From b5c38283d8b6b6a3cc08812720a24819359d6bda Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 12 Dec 2017 21:31:54 -0500 Subject: [PATCH 001/126] Add scripts to build Linux and MacOS wheels --- bin/build-wheels-macos.sh | 59 ++++++++++++++++++++++++++++++++++ bin/build-wheels-manylinux1.sh | 38 ++++++++++++++++++++++ bin/build-wheels.sh | 3 ++ 3 files changed, 100 insertions(+) create mode 100644 bin/build-wheels-macos.sh create mode 100755 bin/build-wheels-manylinux1.sh create mode 100755 bin/build-wheels.sh diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh new file mode 100644 index 000000000..a56df9f8d --- /dev/null +++ b/bin/build-wheels-macos.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Cause the script to exit if a single command fails. +set -e + +# Show explicitly which commands are currently running. +set -x + +MACPYTHON_URL=https://www.python.org/ftp/python +MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions +DOWNLOAD_DIR=python_downloads + +PY_VERSIONS=("3.4.4" + "3.5.3" + "3.6.1") +PY_INSTS=("python-3.4.4-macosx10.6.pkg" + "python-3.5.3-macosx10.6.pkg" + "python-3.6.1-macosx10.6.pkg") +PY_MMS=("3.4" + "3.5" + "3.6") + +mkdir -p $DOWNLOAD_DIR +mkdir -p .whl + +git clone https://bitbucket.org/mpi4py/mpi4py + +for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do + PY_VERSION=${PY_VERSIONS[i]} + PY_INST=${PY_INSTS[i]} + PY_MM=${PY_MMS[i]} + + git clean -f -f -x -d -e .whl -e $DOWNLOAD_DIR + + # Install Python. + INST_PATH=python_downloads/$PY_INST + curl $MACPYTHON_URL/$PY_VERSION/$PY_INST > $INST_PATH + sudo installer -pkg $INST_PATH -target / + + PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM + PIP_CMD="$(dirname $PYTHON_EXE)/pip$PY_MM" + + $PIP_CMD install -q wheel delocate + + pushd mpi4py + git clean -f -f -x -d + $PYTHON_EXE setup.py bdist_wheel + $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-path dist/*.whl + $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl + mv dist/*.whl ../.whl/ + popd + + $PIP_CMD install . + $PYTHON_EXE setup.py bdist_wheel + $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-path dist/*.whl + $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl + mv dist/*.whl .whl/ + +done diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh new file mode 100755 index 000000000..ea7b0e366 --- /dev/null +++ b/bin/build-wheels-manylinux1.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# TODO: We use mpich2, but could download / compile mpich3 if necessary +# TODO: Test against multiple implementations of MPI +# TODO: Upload wheels somewhere appropriate + +# Cause the script to exit if a single command fails. +set -e + +# Show explicitly which commands are currently running. +set -x + +# Install dependencies +yum install -y \ + mpich2-devel \ + libgomp + +# - Use the brainiak source stored in /brainiak via docker run command +# - Use head of mpi4py (we can pick some more suitable tag) +git clone https://bitbucket.org/mpi4py/mpi4py /mpi4py +mkdir /.whl + +for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do + pushd /mpi4py + git clean -f -f -x -d + /opt/python/${PYTHON}/bin/python setup.py bdist_wheel + /opt/python/${PYTHON}/bin/auditwheel repair dist/*.whl + mv wheelhouse/*.whl /.whl/ + popd + + pushd /brainiak + git clean -f -f -x -d + /opt/python/${PYTHON}/bin/python -m pip install . + /opt/python/${PYTHON}/bin/python setup.py bdist_wheel + /opt/python/${PYTHON}/bin/auditwheel repair dist/*.whl + mv wheelhouse/*.whl /.whl/ + popd +done diff --git a/bin/build-wheels.sh b/bin/build-wheels.sh new file mode 100755 index 000000000..cd336c4ca --- /dev/null +++ b/bin/build-wheels.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker run --rm -v `pwd`:/brainiak -ti quay.io/xhochy/arrow_manylinux1_x86_64_base ./brainiak/bin/build-wheels-manylinux1.sh From 07f5a6bf7e6af957ca6f924e4ccb67b3452e3dc2 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 11:32:19 -0500 Subject: [PATCH 002/126] First test for macos wheels --- .travis.yml | 92 ++++++++++++++++++++++----------------- bin/build-wheels-macos.sh | 2 + 2 files changed, 54 insertions(+), 40 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1dda25b66..0266b8de3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,47 +1,59 @@ -branches: - only: - - master +# branches: + # only: + # - master + +# matrix: + # include: + # - os: linux + # dist: trusty + # sudo: required + # language: python + # python: 3.4 + # addons: &linux_addons + # apt: &apt + # packages: + # - build-essential libgomp1 libmpich-dev mpich + + # - os: linux + # dist: trusty + # sudo: required + # language: python + # python: 3.5 + # addons: *linux_addons + + # - os: osx + # osx_image: xcode7.3 + # language: generic + # env: &macos_env + # - CC=/usr/local/opt/llvm/bin/clang + # - CXX=/usr/local/opt/llvm/bin/clang++ + # - LDFLAGS="-L/usr/local/opt/llvm/lib + # -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + # - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + # - HOMEBREW_NO_AUTO_UPDATE=1 + # before_install: &macos_before_install + # - brew install llvm mpich python3 + + # - os: osx + # osx_image: xcode8 + # language: generic + # env: *macos_env + # before_install: *macos_before_install + +# install: + # - python3 -m pip install -U pip + +# script: + # - ./pr-check.sh matrix: include: - - os: linux - dist: trusty - sudo: required - language: python - python: 3.4 - addons: &linux_addons - apt: &apt - packages: - - build-essential libgomp1 libmpich-dev mpich - - - os: linux - dist: trusty - sudo: required - language: python - python: 3.5 - addons: *linux_addons - - os: osx - osx_image: xcode7.3 + osx_image: xcode7 + sudo: required language: generic - env: &macos_env - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib - -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - before_install: &macos_before_install + before_install: - brew install llvm mpich python3 + install: + - ./bin/build-wheels-macos.sh - - os: osx - osx_image: xcode8 - language: generic - env: *macos_env - before_install: *macos_before_install - -install: - - python3 -m pip install -U pip - -script: - - ./pr-check.sh diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index a56df9f8d..b1dce5c4d 100644 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -57,3 +57,5 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do mv dist/*.whl .whl/ done + +ls .whl From dd278b6bc4d615e6461a4fce7b190e6e655b3381 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 11:55:30 -0500 Subject: [PATCH 003/126] Add S3 uploads --- .travis.yml | 79 ++++++++++++++++------------------------------------- 1 file changed, 23 insertions(+), 56 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0266b8de3..8d935fb70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,59 +1,26 @@ -# branches: - # only: - # - master - -# matrix: - # include: - # - os: linux - # dist: trusty - # sudo: required - # language: python - # python: 3.4 - # addons: &linux_addons - # apt: &apt - # packages: - # - build-essential libgomp1 libmpich-dev mpich - - # - os: linux - # dist: trusty - # sudo: required - # language: python - # python: 3.5 - # addons: *linux_addons - - # - os: osx - # osx_image: xcode7.3 - # language: generic - # env: &macos_env - # - CC=/usr/local/opt/llvm/bin/clang - # - CXX=/usr/local/opt/llvm/bin/clang++ - # - LDFLAGS="-L/usr/local/opt/llvm/lib - # -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - # - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - # - HOMEBREW_NO_AUTO_UPDATE=1 - # before_install: &macos_before_install - # - brew install llvm mpich python3 - - # - os: osx - # osx_image: xcode8 - # language: generic - # env: *macos_env - # before_install: *macos_before_install - -# install: - # - python3 -m pip install -U pip - -# script: - # - ./pr-check.sh - matrix: include: - - os: osx - osx_image: xcode7 - sudo: required - language: generic - before_install: - - brew install llvm mpich python3 - install: - - ./bin/build-wheels-macos.sh + - os: osx + osx_image: xcode7 + sudo: required + language: generic + env: MACOS_WHEELS=1 + before_install: + - brew install llvm mpich python3 + install: + - "./bin/build-wheels-macos.sh" + +deploy: + provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + bucket: brainiak + acl: public_read + region: us-east-1 + local_dir: .whl + upload-dir: $TRAVIS_COMMIT + skip_cleanup: true + on: + condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 From 71309800d64f096052c2a994a2ca78787a7b7f6e Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 12:50:52 -0500 Subject: [PATCH 004/126] Add to-do list --- .travis.yml | 53 ++++++++++++++++++++-------------- bin/README.md | 15 ++++++++++ bin/build-wheels-manylinux1.sh | 26 +++++++---------- bin/build-wheels.sh | 2 +- 4 files changed, 58 insertions(+), 38 deletions(-) create mode 100644 bin/README.md diff --git a/.travis.yml b/.travis.yml index 8d935fb70..99dab152b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,35 @@ matrix: include: - - os: osx - osx_image: xcode7 - sudo: required - language: generic - env: MACOS_WHEELS=1 - before_install: - - brew install llvm mpich python3 - install: - - "./bin/build-wheels-macos.sh" + - os: osx + osx_image: xcode7 + sudo: required + language: generic + env: + - MACOS_WHEELS=1 + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib + -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + before_install: + - brew install llvm mpich python3 + install: + - "./bin/build-wheels-macos.sh" + before_script: + - brew uninstall llvm mpich python3 -deploy: - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - acl: public_read - region: us-east-1 - local_dir: .whl - upload-dir: $TRAVIS_COMMIT - skip_cleanup: true - on: - condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 + deploy: + provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + bucket: brainiak + acl: public_read + region: us-east-1 + local_dir: .whl + upload-dir: $TRAVIS_COMMIT + skip_cleanup: true + on: + condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 diff --git a/bin/README.md b/bin/README.md new file mode 100644 index 000000000..7268b76e8 --- /dev/null +++ b/bin/README.md @@ -0,0 +1,15 @@ +# Wheel to-dos + +# TODO +- Verify using other MPI (e.g., Intel MPI, mpich3) +- Test on 32-bit for Linux? +- Test on other python distros (e.g., conda)? + +# To verify +- Upload wheels to somewhere appropriate +- MacOS build script +- MacOS test script +- Linux test script + +# Completed +- Linux build script diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh index ea7b0e366..657462f92 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-wheels-manylinux1.sh @@ -1,10 +1,6 @@ #!/usr/bin/env bash -# TODO: We use mpich2, but could download / compile mpich3 if necessary -# TODO: Test against multiple implementations of MPI -# TODO: Upload wheels somewhere appropriate - -# Cause the script to exit if a single command fails. +# Caust the script to exit if a single command fails. set -e # Show explicitly which commands are currently running. @@ -15,24 +11,24 @@ yum install -y \ mpich2-devel \ libgomp +export WHEEL_DIR=/brainiak/.whl +mkdir -p $WHEEL_DIR + # - Use the brainiak source stored in /brainiak via docker run command # - Use head of mpi4py (we can pick some more suitable tag) git clone https://bitbucket.org/mpi4py/mpi4py /mpi4py -mkdir /.whl for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do pushd /mpi4py git clean -f -f -x -d /opt/python/${PYTHON}/bin/python setup.py bdist_wheel - /opt/python/${PYTHON}/bin/auditwheel repair dist/*.whl - mv wheelhouse/*.whl /.whl/ + auditwheel repair dist/*.whl + mv wheelhouse/*.whl $WHEEL_DIR popd - pushd /brainiak - git clean -f -f -x -d - /opt/python/${PYTHON}/bin/python -m pip install . - /opt/python/${PYTHON}/bin/python setup.py bdist_wheel - /opt/python/${PYTHON}/bin/auditwheel repair dist/*.whl - mv wheelhouse/*.whl /.whl/ - popd + git clean -f -f -x -d + /opt/python/${PYTHON}/bin/python -m pip install . + /opt/python/${PYTHON}/bin/python setup.py bdist_wheel + auditwheel repair dist/*.whl + mv wheelhouse/*.whl $WHEEL_DIR done diff --git a/bin/build-wheels.sh b/bin/build-wheels.sh index cd336c4ca..899af6be6 100755 --- a/bin/build-wheels.sh +++ b/bin/build-wheels.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -docker run --rm -v `pwd`:/brainiak -ti quay.io/xhochy/arrow_manylinux1_x86_64_base ./brainiak/bin/build-wheels-manylinux1.sh +docker run --rm -w /brainiak -v `pwd`:/brainiak -ti quay.io/pypa/manylinux1_x86_64 ./bin/build-wheels-manylinux1.sh From ec6b32de1b6e8bc73211f9696c4388f9752467cf Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 13:05:18 -0500 Subject: [PATCH 005/126] Add test scripts --- bin/README.md | 1 + bin/build-wheels-macos.sh | 2 +- bin/build-wheels-manylinux1.sh | 2 +- bin/test-wheels-macos.sh | 36 ++++++++++++++++++++++++++++++++++ bin/test-wheels-manylinux1.sh | 15 ++++++++++++++ bin/test-wheels.sh | 3 +++ 6 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 bin/test-wheels-macos.sh create mode 100644 bin/test-wheels-manylinux1.sh create mode 100644 bin/test-wheels.sh diff --git a/bin/README.md b/bin/README.md index 7268b76e8..11b4b6947 100644 --- a/bin/README.md +++ b/bin/README.md @@ -1,6 +1,7 @@ # Wheel to-dos # TODO +- Modify ```setup.py``` to handle conditional MPI install - Verify using other MPI (e.g., Intel MPI, mpich3) - Test on 32-bit for Linux? - Test on other python distros (e.g., conda)? diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index b1dce5c4d..b24d15119 100644 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Cause the script to exit if a single command fails. -set -e +# set -e # Show explicitly which commands are currently running. set -x diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh index 657462f92..b196cd6ac 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-wheels-manylinux1.sh @@ -26,7 +26,7 @@ for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do mv wheelhouse/*.whl $WHEEL_DIR popd - git clean -f -f -x -d + git clean -f -f -x -d -e .whl /opt/python/${PYTHON}/bin/python -m pip install . /opt/python/${PYTHON}/bin/python setup.py bdist_wheel auditwheel repair dist/*.whl diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh new file mode 100644 index 000000000..252d0dea8 --- /dev/null +++ b/bin/test-wheels-macos.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Caust the script to exit if a single command fails. +set -e + +# Show explicitly which commands are currently running. +set -x + + +MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions +PY_MMS=("3.4" + "3.5" + "3.6") +# This array is just used to find the right wheel. +PY_WHEEL_VERSIONS=("34" + "35" + "36") + +for ((i=0; i<${#PY_MMS[@]}; ++i)); do + PY_MM=${PY_MMS[i]} + PY_WHEEL_VERSION=${PY_WHEEL_VERSIONS[i]} + + PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM + PIP_CMD="$(dirname $PYTHON_EXE)/pip$PY_MM" + + # Find the appropriate wheel by grepping for the Python version. + PYTHON_WHEEL=$(find $ROOT_DIR/../.whl -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION") + + # TODO: Install MPI wheel + + # Install the wheel. + $PIP_CMD install $PYTHON_WHEEL + + # Run a simple test script to make sure that the wheel works. + # $PYTHON_EXE $TEST_SCRIPT +done diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh new file mode 100644 index 000000000..ee69e4c10 --- /dev/null +++ b/bin/test-wheels-manylinux1.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Caust the script to exit if a single command fails. +set -e + +# Show explicitly which commands are currently running. +set -x + +export WHEEL_DIR=/brainiak/.whl + +for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do + # - Find mpi wheel + # - Find brainiak wheel + # - Install both wheels +done diff --git a/bin/test-wheels.sh b/bin/test-wheels.sh new file mode 100644 index 000000000..1a90eee1e --- /dev/null +++ b/bin/test-wheels.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker run --rm -w /brainiak -v `pwd`:/brainiak -ti quay.io/pypa/manylinux1_x86_64 ./bin/test-wheels-manylinux1.sh From 28c882da2b82152453b2de1aa21a637a4117efd4 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 13:08:19 -0500 Subject: [PATCH 006/126] Add run script --- bin/README.md | 0 bin/build-wheels-macos.sh | 0 bin/build-wheels-manylinux1.sh | 4 ++-- bin/run-manylinux1.sh | 3 +++ bin/test-wheels-macos.sh | 0 bin/test-wheels-manylinux1.sh | 0 bin/test-wheels.sh | 0 7 files changed, 5 insertions(+), 2 deletions(-) mode change 100644 => 100755 bin/README.md mode change 100644 => 100755 bin/build-wheels-macos.sh create mode 100755 bin/run-manylinux1.sh mode change 100644 => 100755 bin/test-wheels-macos.sh mode change 100644 => 100755 bin/test-wheels-manylinux1.sh mode change 100644 => 100755 bin/test-wheels.sh diff --git a/bin/README.md b/bin/README.md old mode 100644 new mode 100755 diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh old mode 100644 new mode 100755 diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh index b196cd6ac..1830dfe41 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-wheels-manylinux1.sh @@ -23,12 +23,12 @@ for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do git clean -f -f -x -d /opt/python/${PYTHON}/bin/python setup.py bdist_wheel auditwheel repair dist/*.whl - mv wheelhouse/*.whl $WHEEL_DIR + mv wheelhouse/*.whl $WHEEL_DIR/ popd git clean -f -f -x -d -e .whl /opt/python/${PYTHON}/bin/python -m pip install . /opt/python/${PYTHON}/bin/python setup.py bdist_wheel auditwheel repair dist/*.whl - mv wheelhouse/*.whl $WHEEL_DIR + mv wheelhouse/*.whl $WHEEL_DIR/ done diff --git a/bin/run-manylinux1.sh b/bin/run-manylinux1.sh new file mode 100755 index 000000000..2dc69eff9 --- /dev/null +++ b/bin/run-manylinux1.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker run --rm -w /brainiak -v `pwd`:/brainiak -ti quay.io/pypa/manylinux1_x86_64 /bin/bash diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh old mode 100644 new mode 100755 diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh old mode 100644 new mode 100755 diff --git a/bin/test-wheels.sh b/bin/test-wheels.sh old mode 100644 new mode 100755 From f93e8d5b6cc1e67f5c39ecd7feeea4f3b1781984 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 13:30:55 -0500 Subject: [PATCH 007/126] Test many linux wheels --- bin/README.md | 1 + bin/build-wheels-manylinux1.sh | 2 +- bin/test-wheels-manylinux1.sh | 13 +++++++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/bin/README.md b/bin/README.md index 11b4b6947..2337eb6ab 100755 --- a/bin/README.md +++ b/bin/README.md @@ -5,6 +5,7 @@ - Verify using other MPI (e.g., Intel MPI, mpich3) - Test on 32-bit for Linux? - Test on other python distros (e.g., conda)? +- Upload wheels to PyPI # To verify - Upload wheels to somewhere appropriate diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh index 1830dfe41..0b62b98f7 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-wheels-manylinux1.sh @@ -11,7 +11,7 @@ yum install -y \ mpich2-devel \ libgomp -export WHEEL_DIR=/brainiak/.whl +WHEEL_DIR=/brainiak/.whl mkdir -p $WHEEL_DIR # - Use the brainiak source stored in /brainiak via docker run command diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index ee69e4c10..a69a52be0 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -6,10 +6,15 @@ set -e # Show explicitly which commands are currently running. set -x -export WHEEL_DIR=/brainiak/.whl +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +WHEEL_DIR=/brainiak/.whl for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do - # - Find mpi wheel - # - Find brainiak wheel - # - Install both wheels + MPI4PY_WHEEL=$(find $SCRIPT_DIR/../.whl -type f | grep $PYTHON | grep mpi4py) + BRAINIAK_WHEEL=$(find $SCRIPT_DIR/../.whl -type f | grep $PYTHON | grep brainiak) + + PIP=/opt/python/${PYTHON}/bin/pip + + $PIP install $MPI4PY_WHEEL + $PIP install $BRAINIAK_WHEEL done From cec0d1be095474cd5cd0d9855e9647fb28942925 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 13:43:08 -0500 Subject: [PATCH 008/126] Add linux tests --- .travis.yml | 78 ++++++++++++++++++++++++++++++--------------------- bin/README.md | 1 + 2 files changed, 47 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index 99dab152b..a95cfb32f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,35 +1,49 @@ matrix: - include: - - os: osx - osx_image: xcode7 - sudo: required - language: generic - env: - - MACOS_WHEELS=1 - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib - -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - before_install: - - brew install llvm mpich python3 - install: - - "./bin/build-wheels-macos.sh" - before_script: - - brew uninstall llvm mpich python3 + include: - deploy: - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - acl: public_read - region: us-east-1 - local_dir: .whl - upload-dir: $TRAVIS_COMMIT - skip_cleanup: true - on: - condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 + # Build Linux wheels + - os: linux + dist: trusty + sudo: required + language: generic + env: + - LINUX_WHEELS=1 + install: + - ./bin/build-wheels.sh + script: + - ./bin/test-wheels.sh + + # Build MacOS wheels + - os: osx + osx_image: xcode7 + sudo: required + language: generic + env: + - MACOS_WHEELS=1 + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib + -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + before_install: + - brew install llvm mpich python3 + install: + - "./bin/build-wheels-macos.sh" + before_script: + - brew uninstall llvm mpich python3 + + deploy: + provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + bucket: brainiak + acl: public_read + region: us-east-1 + local_dir: .whl + upload-dir: $TRAVIS_COMMIT + skip_cleanup: true + on: + condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 diff --git a/bin/README.md b/bin/README.md index 2337eb6ab..39223ee0a 100755 --- a/bin/README.md +++ b/bin/README.md @@ -2,6 +2,7 @@ # TODO - Modify ```setup.py``` to handle conditional MPI install +- Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) - Test on 32-bit for Linux? - Test on other python distros (e.g., conda)? From 6334ac5224846e95b7f68026f379dd338a22779c Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 14:10:28 -0500 Subject: [PATCH 009/126] Make operations quiet so travis logs don't get filled up --- bin/README.md | 4 ++++ bin/build-wheels-macos.sh | 12 ++++++------ bin/build-wheels-manylinux1.sh | 14 +++++++------- bin/test-wheels-macos.sh | 23 ++++++++--------------- bin/test-wheels-manylinux1.sh | 4 ++-- 5 files changed, 27 insertions(+), 30 deletions(-) diff --git a/bin/README.md b/bin/README.md index 39223ee0a..340e2c9a7 100755 --- a/bin/README.md +++ b/bin/README.md @@ -2,11 +2,15 @@ # TODO - Modify ```setup.py``` to handle conditional MPI install +- Build documentation +- Deploy only on master +- Figure out ```pr-check``` and other testing scripts - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) - Test on 32-bit for Linux? - Test on other python distros (e.g., conda)? - Upload wheels to PyPI +- Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8)) # To verify - Upload wheels to somewhere appropriate diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index b24d15119..8f7031105 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -23,14 +23,14 @@ PY_MMS=("3.4" mkdir -p $DOWNLOAD_DIR mkdir -p .whl -git clone https://bitbucket.org/mpi4py/mpi4py +git clone -q https://bitbucket.org/mpi4py/mpi4py for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do PY_VERSION=${PY_VERSIONS[i]} PY_INST=${PY_INSTS[i]} PY_MM=${PY_MMS[i]} - git clean -f -f -x -d -e .whl -e $DOWNLOAD_DIR + git clean -f -f -x -d -q -e .whl -e $DOWNLOAD_DIR # Install Python. INST_PATH=python_downloads/$PY_INST @@ -43,15 +43,15 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do $PIP_CMD install -q wheel delocate pushd mpi4py - git clean -f -f -x -d - $PYTHON_EXE setup.py bdist_wheel + git clean -f -f -x -d -q + $PYTHON_EXE setup.py -q bdist_wheel $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-path dist/*.whl $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl mv dist/*.whl ../.whl/ popd - $PIP_CMD install . - $PYTHON_EXE setup.py bdist_wheel + $PIP_CMD install -q . + $PYTHON_EXE setup.py -q bdist_wheel $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-path dist/*.whl $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl mv dist/*.whl .whl/ diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh index 0b62b98f7..01e320582 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-wheels-manylinux1.sh @@ -7,7 +7,7 @@ set -e set -x # Install dependencies -yum install -y \ +yum install -y -q \ mpich2-devel \ libgomp @@ -16,19 +16,19 @@ mkdir -p $WHEEL_DIR # - Use the brainiak source stored in /brainiak via docker run command # - Use head of mpi4py (we can pick some more suitable tag) -git clone https://bitbucket.org/mpi4py/mpi4py /mpi4py +git clone -q https://bitbucket.org/mpi4py/mpi4py /mpi4py for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do pushd /mpi4py - git clean -f -f -x -d - /opt/python/${PYTHON}/bin/python setup.py bdist_wheel + git clean -f -f -x -d -q + /opt/python/${PYTHON}/bin/python setup.py -q bdist_wheel auditwheel repair dist/*.whl mv wheelhouse/*.whl $WHEEL_DIR/ popd - git clean -f -f -x -d -e .whl - /opt/python/${PYTHON}/bin/python -m pip install . - /opt/python/${PYTHON}/bin/python setup.py bdist_wheel + git clean -f -f -x -d -q -e .whl + /opt/python/${PYTHON}/bin/python -m pip install -q . + /opt/python/${PYTHON}/bin/python setup.py -q bdist_wheel auditwheel repair dist/*.whl mv wheelhouse/*.whl $WHEEL_DIR/ done diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 252d0dea8..d0fc9812d 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -8,29 +8,22 @@ set -x MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions -PY_MMS=("3.4" - "3.5" - "3.6") +PY_MMS=("3.4" "3.5" "3.6") + # This array is just used to find the right wheel. -PY_WHEEL_VERSIONS=("34" - "35" - "36") +PY_WHEEL_VERSIONS=("34" "35" "36") for ((i=0; i<${#PY_MMS[@]}; ++i)); do PY_MM=${PY_MMS[i]} PY_WHEEL_VERSION=${PY_WHEEL_VERSIONS[i]} PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM - PIP_CMD="$(dirname $PYTHON_EXE)/pip$PY_MM" + PIP="$(dirname $PYTHON_EXE)/pip$PY_MM" # Find the appropriate wheel by grepping for the Python version. - PYTHON_WHEEL=$(find $ROOT_DIR/../.whl -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION") - - # TODO: Install MPI wheel - - # Install the wheel. - $PIP_CMD install $PYTHON_WHEEL + MPI4PY_WHEEL=$(find $ROOT_DIR/../.whl -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep mpi4py) + BRAINIAK_WHEEL=$(find $ROOT_DIR/../.whl -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep brainiak) - # Run a simple test script to make sure that the wheel works. - # $PYTHON_EXE $TEST_SCRIPT + $PIP install -q $MPI4PY_WHEEL + $PIP install -q $BRAINIAK_WHEEL done diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index a69a52be0..a39b6060e 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -15,6 +15,6 @@ for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do PIP=/opt/python/${PYTHON}/bin/pip - $PIP install $MPI4PY_WHEEL - $PIP install $BRAINIAK_WHEEL + $PIP install -q $MPI4PY_WHEEL + $PIP install -q $BRAINIAK_WHEEL done From f326f7c71c160586e92c26cd546ccb968740effd Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 14:45:09 -0500 Subject: [PATCH 010/126] Use src instead of brainiak as manylinux1 build directory --- bin/README.md | 1 + bin/build-wheels-manylinux1.sh | 4 ++-- bin/build-wheels.sh | 2 +- bin/test-wheels-macos.sh | 2 ++ bin/test-wheels-manylinux1.sh | 2 +- bin/test-wheels.sh | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/bin/README.md b/bin/README.md index 340e2c9a7..5fc4d584c 100755 --- a/bin/README.md +++ b/bin/README.md @@ -3,6 +3,7 @@ # TODO - Modify ```setup.py``` to handle conditional MPI install - Build documentation +- Add LICENSE into whl file - Deploy only on master - Figure out ```pr-check``` and other testing scripts - Get things working on Jenkins diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh index 01e320582..cb9c272ef 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-wheels-manylinux1.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Caust the script to exit if a single command fails. -set -e +# set -e # Show explicitly which commands are currently running. set -x @@ -11,7 +11,7 @@ yum install -y -q \ mpich2-devel \ libgomp -WHEEL_DIR=/brainiak/.whl +WHEEL_DIR=$PWD/.whl mkdir -p $WHEEL_DIR # - Use the brainiak source stored in /brainiak via docker run command diff --git a/bin/build-wheels.sh b/bin/build-wheels.sh index 899af6be6..0214a98c2 100755 --- a/bin/build-wheels.sh +++ b/bin/build-wheels.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -docker run --rm -w /brainiak -v `pwd`:/brainiak -ti quay.io/pypa/manylinux1_x86_64 ./bin/build-wheels-manylinux1.sh +docker run --rm -w /src -v `pwd`:/src -ti quay.io/pypa/manylinux1_x86_64 ./bin/build-wheels-manylinux1.sh diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index d0fc9812d..9d130f8fa 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -22,6 +22,8 @@ for ((i=0; i<${#PY_MMS[@]}; ++i)); do # Find the appropriate wheel by grepping for the Python version. MPI4PY_WHEEL=$(find $ROOT_DIR/../.whl -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep mpi4py) + + # TODO: this will actually pick up both wheels since brainiak is in the path BRAINIAK_WHEEL=$(find $ROOT_DIR/../.whl -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep brainiak) $PIP install -q $MPI4PY_WHEEL diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index a39b6060e..be90c65bf 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -7,7 +7,7 @@ set -e set -x SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -WHEEL_DIR=/brainiak/.whl +WHEEL_DIR=$PWD/.whl for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do MPI4PY_WHEEL=$(find $SCRIPT_DIR/../.whl -type f | grep $PYTHON | grep mpi4py) diff --git a/bin/test-wheels.sh b/bin/test-wheels.sh index 1a90eee1e..e7a21f5fe 100755 --- a/bin/test-wheels.sh +++ b/bin/test-wheels.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -docker run --rm -w /brainiak -v `pwd`:/brainiak -ti quay.io/pypa/manylinux1_x86_64 ./bin/test-wheels-manylinux1.sh +docker run --rm -w /src -v `pwd`:/src -ti quay.io/pypa/manylinux1_x86_64 ./bin/test-wheels-manylinux1.sh From 91291bd8954d147eb5c55a8132857db87bbc2d70 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 19:00:59 -0500 Subject: [PATCH 011/126] Remove symlink that causes brew to break in macos --- .travis.yml | 28 +++++++++++++++------------- bin/README.md | 1 + 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index a95cfb32f..bcfc98805 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,23 +27,25 @@ matrix: - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 before_install: + - brew install llvm + - rm /usr/local/include/c++ - brew install llvm mpich python3 install: - "./bin/build-wheels-macos.sh" before_script: - brew uninstall llvm mpich python3 - deploy: - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - acl: public_read - region: us-east-1 - local_dir: .whl - upload-dir: $TRAVIS_COMMIT - skip_cleanup: true - on: - condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 +deploy: + provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + bucket: brainiak + acl: public_read + region: us-east-1 + local_dir: .whl + upload-dir: $TRAVIS_COMMIT + skip_cleanup: true + on: + condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 diff --git a/bin/README.md b/bin/README.md index 5fc4d584c..00c604b09 100755 --- a/bin/README.md +++ b/bin/README.md @@ -2,6 +2,7 @@ # TODO - Modify ```setup.py``` to handle conditional MPI install +- Restore ```.travis.yml``` - Build documentation - Add LICENSE into whl file - Deploy only on master From c3f16052721195687a752c7d00b28cdd26e9b12d Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 19:10:52 -0500 Subject: [PATCH 012/126] Comment out linux testing for now --- .travis.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index bcfc98805..3a29913c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,16 +2,16 @@ matrix: include: # Build Linux wheels - - os: linux - dist: trusty - sudo: required - language: generic - env: - - LINUX_WHEELS=1 - install: - - ./bin/build-wheels.sh - script: - - ./bin/test-wheels.sh + # - os: linux + # dist: trusty + # sudo: required + # language: generic + # env: + # - LINUX_WHEELS=1 + # install: + # - ./bin/build-wheels.sh + # script: + # - ./bin/test-wheels.sh # Build MacOS wheels - os: osx From 5e807ee1170abe869d4f659b4cbaf9aae77fa281 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 21:43:14 -0500 Subject: [PATCH 013/126] Fix up wheel dir paths --- bin/build-wheels-macos.sh | 11 +++++++---- bin/build-wheels-manylinux1.sh | 3 ++- bin/test-wheels-macos.sh | 7 +++++-- bin/test-wheels-manylinux1.sh | 6 +++--- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index 8f7031105..230190405 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -21,7 +21,10 @@ PY_MMS=("3.4" "3.6") mkdir -p $DOWNLOAD_DIR -mkdir -p .whl + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +WHEEL_DIR=$SCRIPT_DIR/../.whl +mkdir -p $WHEEL_DIR git clone -q https://bitbucket.org/mpi4py/mpi4py @@ -47,15 +50,15 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do $PYTHON_EXE setup.py -q bdist_wheel $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-path dist/*.whl $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl - mv dist/*.whl ../.whl/ + mv dist/*.whl $WHEEL_DIR/ popd $PIP_CMD install -q . $PYTHON_EXE setup.py -q bdist_wheel $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-path dist/*.whl $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl - mv dist/*.whl .whl/ + mv dist/*.whl $WHEEL_DIR/ done -ls .whl +ls $WHEEL_DIR diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh index cb9c272ef..6005873ac 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-wheels-manylinux1.sh @@ -11,7 +11,8 @@ yum install -y -q \ mpich2-devel \ libgomp -WHEEL_DIR=$PWD/.whl +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +WHEEL_DIR=$SCRIPT_DIR/../.whl mkdir -p $WHEEL_DIR # - Use the brainiak source stored in /brainiak via docker run command diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 9d130f8fa..c708d41fe 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -13,6 +13,9 @@ PY_MMS=("3.4" "3.5" "3.6") # This array is just used to find the right wheel. PY_WHEEL_VERSIONS=("34" "35" "36") +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +WHEEL_DIR=$SCRIPT_DIR/../.whl + for ((i=0; i<${#PY_MMS[@]}; ++i)); do PY_MM=${PY_MMS[i]} PY_WHEEL_VERSION=${PY_WHEEL_VERSIONS[i]} @@ -21,10 +24,10 @@ for ((i=0; i<${#PY_MMS[@]}; ++i)); do PIP="$(dirname $PYTHON_EXE)/pip$PY_MM" # Find the appropriate wheel by grepping for the Python version. - MPI4PY_WHEEL=$(find $ROOT_DIR/../.whl -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep mpi4py) + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep mpi4py) # TODO: this will actually pick up both wheels since brainiak is in the path - BRAINIAK_WHEEL=$(find $ROOT_DIR/../.whl -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep brainiak) + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep brainiak) $PIP install -q $MPI4PY_WHEEL $PIP install -q $BRAINIAK_WHEEL diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index be90c65bf..01a27fbb2 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -7,11 +7,11 @@ set -e set -x SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -WHEEL_DIR=$PWD/.whl +WHEEL_DIR=$SCRIPT_DIR/../.whl for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do - MPI4PY_WHEEL=$(find $SCRIPT_DIR/../.whl -type f | grep $PYTHON | grep mpi4py) - BRAINIAK_WHEEL=$(find $SCRIPT_DIR/../.whl -type f | grep $PYTHON | grep brainiak) + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep mpi4py) + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep brainiak) PIP=/opt/python/${PYTHON}/bin/pip From 2c54322d5dc8170298be5939aef01dfbca3dc854 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 13 Dec 2017 23:01:22 -0500 Subject: [PATCH 014/126] Don't delete mpi4py --- bin/build-wheels-macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index 230190405..dbda40ba4 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -33,7 +33,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do PY_INST=${PY_INSTS[i]} PY_MM=${PY_MMS[i]} - git clean -f -f -x -d -q -e .whl -e $DOWNLOAD_DIR + git clean -f -f -x -d -q -e .whl -e $DOWNLOAD_DIR -e mpi4py # Install Python. INST_PATH=python_downloads/$PY_INST From beb3246052f0beb89fc05269f9b071c8fc77a19c Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 00:16:34 -0500 Subject: [PATCH 015/126] Update pip install --- bin/build-wheels-macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index dbda40ba4..d681d8f3d 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -43,7 +43,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM PIP_CMD="$(dirname $PYTHON_EXE)/pip$PY_MM" - $PIP_CMD install -q wheel delocate + $PIP_CMD install -q wheel delocate numpy cython pybind11 scipy pushd mpi4py git clean -f -f -x -d -q From 0d5505ebecb3d6b973e9fcd5909cd6839ae3c30a Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 00:47:27 -0500 Subject: [PATCH 016/126] Remove quiet flag for debugging --- .travis.yml | 2 +- bin/build-wheels-macos.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3a29913c6..d317c0029 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ matrix: before_install: - brew install llvm - rm /usr/local/include/c++ - - brew install llvm mpich python3 + - brew install mpich python3 install: - "./bin/build-wheels-macos.sh" before_script: diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index d681d8f3d..01029dbfc 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -47,14 +47,14 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do pushd mpi4py git clean -f -f -x -d -q - $PYTHON_EXE setup.py -q bdist_wheel + $PYTHON_EXE setup.py bdist_wheel $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-path dist/*.whl $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl mv dist/*.whl $WHEEL_DIR/ popd $PIP_CMD install -q . - $PYTHON_EXE setup.py -q bdist_wheel + $PYTHON_EXE setup.py bdist_wheel $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-path dist/*.whl $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl mv dist/*.whl $WHEEL_DIR/ From 884738cbf2a8be78a98d85273f376566da38e4a6 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 01:04:02 -0500 Subject: [PATCH 017/126] Specify x86_64 architecture only --- bin/build-wheels-macos.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index 01029dbfc..0fca00fc3 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -47,14 +47,14 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do pushd mpi4py git clean -f -f -x -d -q - $PYTHON_EXE setup.py bdist_wheel + env ARCHFLAGS="-arch x86_64" $PYTHON_EXE setup.py bdist_wheel $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-path dist/*.whl $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl mv dist/*.whl $WHEEL_DIR/ popd - $PIP_CMD install -q . - $PYTHON_EXE setup.py bdist_wheel + env ARCHFLAGS="-arch x86_64" $PIP_CMD install -q . + env ARCHFLAGS="-arch x86_64" $PYTHON_EXE setup.py bdist_wheel $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-path dist/*.whl $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl mv dist/*.whl $WHEEL_DIR/ From d53062c4439dff7615fed565ce260867312e9789 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 01:33:16 -0500 Subject: [PATCH 018/126] Remove delocate-path --- bin/README.md | 1 + bin/build-wheels-macos.sh | 2 -- setup.py | 4 ++++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/README.md b/bin/README.md index 00c604b09..8b4e566a8 100755 --- a/bin/README.md +++ b/bin/README.md @@ -10,6 +10,7 @@ - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) - Test on 32-bit for Linux? +- Consider other OS versions (e.g., xcode8) - Test on other python distros (e.g., conda)? - Upload wheels to PyPI - Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8)) diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index 0fca00fc3..0ebe06ddd 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -48,14 +48,12 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do pushd mpi4py git clean -f -f -x -d -q env ARCHFLAGS="-arch x86_64" $PYTHON_EXE setup.py bdist_wheel - $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-path dist/*.whl $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl mv dist/*.whl $WHEEL_DIR/ popd env ARCHFLAGS="-arch x86_64" $PIP_CMD install -q . env ARCHFLAGS="-arch x86_64" $PYTHON_EXE setup.py bdist_wheel - $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-path dist/*.whl $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl mv dist/*.whl $WHEEL_DIR/ diff --git a/setup.py b/setup.py index c207b7258..14dd7308d 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,10 @@ import sys import setuptools from copy import deepcopy +import pip + +# Attempt to install mpi4py +pip.main(['install', 'mpi4py']) assert sys.version_info >= (3, 4), ( "Please use Python version 3.4 or higher, " From e06e1a4b58424731d673f2d9b93d04176d563c59 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 02:13:01 -0500 Subject: [PATCH 019/126] List wheel dir for debugging --- .travis.yml | 2 ++ bin/README.md | 3 ++- bin/build-wheels-macos.sh | 2 +- setup.py | 10 +++++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d317c0029..d31fc9a4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,8 @@ matrix: - "./bin/build-wheels-macos.sh" before_script: - brew uninstall llvm mpich python3 + script: + - ./bin/test-wheels-macos.sh deploy: provider: s3 diff --git a/bin/README.md b/bin/README.md index 8b4e566a8..ba271bba1 100755 --- a/bin/README.md +++ b/bin/README.md @@ -10,7 +10,8 @@ - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) - Test on 32-bit for Linux? -- Consider other OS versions (e.g., xcode8) +- Consider other OS versions (e.g., xcode8) (or have multi-OS distro) +- Build Windows wheels? - Test on other python distros (e.g., conda)? - Upload wheels to PyPI - Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8)) diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index 0ebe06ddd..6b30819dc 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -59,4 +59,4 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do done -ls $WHEEL_DIR +echo $WHEEL_DIR diff --git a/setup.py b/setup.py index 14dd7308d..f224b19a5 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,15 @@ import pip # Attempt to install mpi4py -pip.main(['install', 'mpi4py']) +result = pip.main(['install', 'mpi4py']) +if result != 0: + # TODO: download mpi4py wheel + # TODO: install mpi4py wheel + version = sys.version_info + + # We could use the platform module, but this may be sufficient + platform = sys.platform + pass assert sys.version_info >= (3, 4), ( "Please use Python version 3.4 or higher, " From 6ae8bee0473f3a027f43128af5fbbc74a721133f Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 02:50:43 -0500 Subject: [PATCH 020/126] Try uploading to s3 --- .travis.yml | 38 +++++++++++++++++++++++++------------- bin/README.md | 10 +++++----- bin/build-wheels-macos.sh | 3 +++ 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index d31fc9a4c..dc79c4d77 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,16 +38,28 @@ matrix: - ./bin/test-wheels-macos.sh deploy: - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - acl: public_read - region: us-east-1 - local_dir: .whl - upload-dir: $TRAVIS_COMMIT - skip_cleanup: true - on: - condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 - + - provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + bucket: brainiak + acl: public_read + region: us-east-1 + local_dir: brainiak/.whl + upload-dir: $TRAVIS_COMMIT + skip_cleanup: true + on: + condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 + # branch: master + # tags: true + # - provider: pypi + # user: brainiak + # password: + # secure: travis encrypt and add to file manually + # on: + # condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 + # tags: true + # branch: master + # TODO: need to verify built wheels are uploaded not ones rebuilt by travis + # distributions: "sdist bdist_wheel" + # skip_cleanup: true diff --git a/bin/README.md b/bin/README.md index ba271bba1..c3cbc0e65 100755 --- a/bin/README.md +++ b/bin/README.md @@ -5,7 +5,6 @@ - Restore ```.travis.yml``` - Build documentation - Add LICENSE into whl file -- Deploy only on master - Figure out ```pr-check``` and other testing scripts - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) @@ -13,14 +12,15 @@ - Consider other OS versions (e.g., xcode8) (or have multi-OS distro) - Build Windows wheels? - Test on other python distros (e.g., conda)? -- Upload wheels to PyPI - Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8)) # To verify - Upload wheels to somewhere appropriate -- MacOS build script -- MacOS test script -- Linux test script +- Upload wheels to PyPI [link](https://pypi.python.org/pypi/twine) (uncomment in ```.travis.yml```) +- Deploy only on master (uncomment in ```.travis.yml```) # Completed - Linux build script +- Linux test script +- MacOS build script +- MacOS test script diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index 6b30819dc..81f15afb6 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -57,6 +57,9 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl mv dist/*.whl $WHEEL_DIR/ + $PIP_CMD uninstall . + $PIP_CMD uninstall numpy cython pybind11 scipy mpi4py + done echo $WHEEL_DIR From 39ff30d69e68bb3248fbfe0564e47a08d8b12e06 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 03:23:42 -0500 Subject: [PATCH 021/126] Uninstall with yes flag --- bin/README.md | 11 ++++++----- bin/build-wheels-macos.sh | 4 ++-- setup.cfg | 3 +++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/bin/README.md b/bin/README.md index c3cbc0e65..6cea51b07 100755 --- a/bin/README.md +++ b/bin/README.md @@ -3,9 +3,9 @@ # TODO - Modify ```setup.py``` to handle conditional MPI install - Restore ```.travis.yml``` -- Build documentation -- Add LICENSE into whl file -- Figure out ```pr-check``` and other testing scripts +- Figure out ```pr-check.sh```, other testing scripts, and documentation + - Change ```pr-check.sh``` to optionally take a ```PYTHON``` argument, using ```python3``` if unspecified + - Change ```pr-check.sh``` to optionally install a wheel (or automatically find the wheel based on the ```PYTHON```argument) insead of from source - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) - Test on 32-bit for Linux? @@ -15,9 +15,10 @@ - Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8)) # To verify -- Upload wheels to somewhere appropriate -- Upload wheels to PyPI [link](https://pypi.python.org/pypi/twine) (uncomment in ```.travis.yml```) +- Upload wheels to S3 +- Upload wheels to PyPI - Deploy only on master (uncomment in ```.travis.yml```) +- Add LICENSE into whl file # Completed - Linux build script diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index 81f15afb6..4f2e7bb71 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -57,8 +57,8 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl mv dist/*.whl $WHEEL_DIR/ - $PIP_CMD uninstall . - $PIP_CMD uninstall numpy cython pybind11 scipy mpi4py + $PIP_CMD uninstall -y . + $PIP_CMD uninstall -y numpy cython pybind11 scipy mpi4py done diff --git a/setup.cfg b/setup.cfg index 80bfe24c9..67b78f40f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,3 +19,6 @@ parallel = True [coverage:report] fail_under = 90 + +[metadata] +license_file = LICENSE From 0fa8411f0e687d69a44ab617584ebbb96f24964f Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 04:11:54 -0500 Subject: [PATCH 022/126] Tag mpi4py and implement conditional install --- .travis.yml | 2 +- bin/README.md | 3 ++- bin/build-wheels-macos.sh | 4 ++++ bin/build-wheels-manylinux1.sh | 5 +++++ setup.py | 39 +++++++++++++++++++++++++--------- 5 files changed, 41 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc79c4d77..6d3bd4206 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ deploy: acl: public_read region: us-east-1 local_dir: brainiak/.whl - upload-dir: $TRAVIS_COMMIT + upload-dir: .whl skip_cleanup: true on: condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 diff --git a/bin/README.md b/bin/README.md index 6cea51b07..0e5ae83c0 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,7 +1,6 @@ # Wheel to-dos # TODO -- Modify ```setup.py``` to handle conditional MPI install - Restore ```.travis.yml``` - Figure out ```pr-check.sh```, other testing scripts, and documentation - Change ```pr-check.sh``` to optionally take a ```PYTHON``` argument, using ```python3``` if unspecified @@ -15,6 +14,8 @@ - Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8)) # To verify +- Modify ```setup.py``` to handle conditional MPI install +- Checkout mpi4py tag so we always create the same wheel - Upload wheels to S3 - Upload wheels to PyPI - Deploy only on master (uncomment in ```.travis.yml```) diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index 4f2e7bb71..06952c697 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -27,6 +27,9 @@ WHEEL_DIR=$SCRIPT_DIR/../.whl mkdir -p $WHEEL_DIR git clone -q https://bitbucket.org/mpi4py/mpi4py +pushd mpi4py +git checkout 3.0.0 +popd for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do PY_VERSION=${PY_VERSIONS[i]} @@ -43,6 +46,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM PIP_CMD="$(dirname $PYTHON_EXE)/pip$PY_MM" + $PIP_CMD install -U pip $PIP_CMD install -q wheel delocate numpy cython pybind11 scipy pushd mpi4py diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh index 6005873ac..bb734282d 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-wheels-manylinux1.sh @@ -18,8 +18,13 @@ mkdir -p $WHEEL_DIR # - Use the brainiak source stored in /brainiak via docker run command # - Use head of mpi4py (we can pick some more suitable tag) git clone -q https://bitbucket.org/mpi4py/mpi4py /mpi4py +pushd /mpi4py +git checkout 3.0.0 +popd for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do + /opt/python/${PYTHON}/bin/python -m pip install -U pip + pushd /mpi4py git clean -f -f -x -d -q /opt/python/${PYTHON}/bin/python setup.py -q bdist_wheel diff --git a/setup.py b/setup.py index f224b19a5..27e91c9e7 100644 --- a/setup.py +++ b/setup.py @@ -7,22 +7,41 @@ import setuptools from copy import deepcopy import pip +import urllib.request + +assert sys.version_info >= (3, 4), ( + "Please use Python version 3.4 or higher, " + "lower versions are not supported" +) # Attempt to install mpi4py result = pip.main(['install', 'mpi4py']) if result != 0: - # TODO: download mpi4py wheel - # TODO: install mpi4py wheel - version = sys.version_info + # MacOS: mpi4py-3.0.0-cp34-cp34m-macosx_10_6_intel.whl + # Linux: mpi4py-3.0.1a0-cp34-cp34m-manylinux1_x86_64.whl - # We could use the platform module, but this may be sufficient - platform = sys.platform - pass + base_url = 'https://s3.amazonaws.com/brainiak/.whl/%s' -assert sys.version_info >= (3, 4), ( - "Please use Python version 3.4 or higher, " - "lower versions are not supported" -) + # Determine wheel file name + # TODO: These templates may change / may need to become more advanced + darwin = 'mpi4py-3.0.0-cp%(version)d-cp%(version)dm-macosx_10_6_intel.whl' + linux = 'mpi4py-3.0.0-cp%(version)d-cp%(version)dm-manylinux1_x86_64.whl' + + wheel_file = darwin if sys.platform == 'darwin' else linux + wheel_file = wheel_file % { + 'version': sys.version_info[0] * 10 + sys.version_info[1] + } + + wheel_url = base_url % wheel_file + + # Download mpi4py wheel + local_filename, headers = urllib.request.urlretrieve(wheel_url) + + # Install mpi4py wheel + result = pip.main(['install', local_filename]) + + if result != 0: + sys.exit('ERROR: failed to install mpi4py') here = os.path.abspath(os.path.dirname(__file__)) From a23a4197307b7916d9b8d27f2efab733a38d3ddf Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 04:12:39 -0500 Subject: [PATCH 023/126] Update README --- bin/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/README.md b/bin/README.md index 0e5ae83c0..e6c7da07a 100755 --- a/bin/README.md +++ b/bin/README.md @@ -17,7 +17,7 @@ - Modify ```setup.py``` to handle conditional MPI install - Checkout mpi4py tag so we always create the same wheel - Upload wheels to S3 -- Upload wheels to PyPI +- Upload wheels to PyPI (uncomment) - Deploy only on master (uncomment in ```.travis.yml```) - Add LICENSE into whl file From c499ccbd281416517789902a50c1c4c8ad7715b4 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 05:10:10 -0500 Subject: [PATCH 024/126] Restore .travis.yml, convert pr-check.sh --- .travis.yml | 130 +++++++++++++++++++++------------- bin/README.md | 12 ++-- bin/test-wheels-macos.sh | 2 + bin/test-wheels-manylinux1.sh | 2 + pr-check.sh | 61 ++++++++++------ run-tests.sh | 6 +- 6 files changed, 136 insertions(+), 77 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d3bd4206..fd12d7480 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,57 +1,89 @@ matrix: - include: + include: + - os: linux + dist: trusty + sudo: required + language: python + python: 3.4 + addons: &linux_addons + apt: &apt + packages: + - build-essential libgomp1 libmpich-dev mpich - # Build Linux wheels - # - os: linux - # dist: trusty - # sudo: required - # language: generic - # env: - # - LINUX_WHEELS=1 - # install: - # - ./bin/build-wheels.sh - # script: - # - ./bin/test-wheels.sh + - os: linux + dist: trusty + sudo: required + language: python + python: 3.5 + addons: *linux_addons - # Build MacOS wheels - - os: osx - osx_image: xcode7 - sudo: required - language: generic - env: - - MACOS_WHEELS=1 - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib - -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - before_install: - - brew install llvm - - rm /usr/local/include/c++ - - brew install mpich python3 - install: - - "./bin/build-wheels-macos.sh" - before_script: - - brew uninstall llvm mpich python3 - script: - - ./bin/test-wheels-macos.sh + - os: osx + osx_image: xcode7.3 + language: generic + env: &macos_env + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib + -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + before_install: &macos_before_install + - brew install llvm mpich python3 + + - os: osx + osx_image: xcode8 + language: generic + env: *macos_env + before_install: *macos_before_install + + # Build Linux wheels + - os: linux + dist: trusty + sudo: required + language: generic + env: + - LINUX_WHEELS=1 + install: + - ./bin/build-wheels.sh + script: + - ./bin/test-wheels.sh + + # Build MacOS wheels + - os: osx + osx_image: xcode7.3 + sudo: required + language: generic + env: *macos_env + - MACOS_WHEELS=1 + before_install: *macos_before_install + install: + - "./bin/build-wheels-macos.sh" + before_script: + - brew uninstall llvm mpich python3 + script: + - ./bin/test-wheels-macos.sh + +install: + - python3 -m pip install -U pip + +script: + - ./pr-check.sh deploy: - - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - acl: public_read - region: us-east-1 - local_dir: brainiak/.whl - upload-dir: .whl - skip_cleanup: true - on: - condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 - # branch: master - # tags: true + - provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + bucket: brainiak + acl: public_read + region: us-east-1 + local_dir: brainiak/.whl + upload-dir: .whl + skip_cleanup: true + on: + condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 + # branch: master + # tags: true # - provider: pypi # user: brainiak # password: diff --git a/bin/README.md b/bin/README.md index e6c7da07a..50b9fcc3e 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,25 +1,23 @@ # Wheel to-dos # TODO -- Restore ```.travis.yml``` -- Figure out ```pr-check.sh```, other testing scripts, and documentation - - Change ```pr-check.sh``` to optionally take a ```PYTHON``` argument, using ```python3``` if unspecified - - Change ```pr-check.sh``` to optionally install a wheel (or automatically find the wheel based on the ```PYTHON```argument) insead of from source - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) -- Test on 32-bit for Linux? -- Consider other OS versions (e.g., xcode8) (or have multi-OS distro) -- Build Windows wheels? +- Build for other OSs (e.g., xcode[VERSION], 32-bit Linux, Windows) - Test on other python distros (e.g., conda)? - Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8)) # To verify +- Restore ```.travis.yml``` - Modify ```setup.py``` to handle conditional MPI install - Checkout mpi4py tag so we always create the same wheel - Upload wheels to S3 - Upload wheels to PyPI (uncomment) - Deploy only on master (uncomment in ```.travis.yml```) - Add LICENSE into whl file +- Figure out ```pr-check.sh```, other testing scripts, and documentation + - Change ```pr-check.sh``` to optionally take a ```PYTHON``` argument, using ```python3``` if unspecified + - Change ```pr-check.sh``` to optionally install a wheel (or automatically find the wheel based on the ```PYTHON```argument) insead of from source # Completed - Linux build script diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index c708d41fe..431e524e1 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -31,4 +31,6 @@ for ((i=0; i<${#PY_MMS[@]}; ++i)); do $PIP install -q $MPI4PY_WHEEL $PIP install -q $BRAINIAK_WHEEL + + PYTHON_EXE=$PYTHON_EXE $SCRIPT_DIR/../pr-check.sh done diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 01a27fbb2..11ebed991 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -17,4 +17,6 @@ for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do $PIP install -q $MPI4PY_WHEEL $PIP install -q $BRAINIAK_WHEEL + + PYTHON_EXE=$PYTHON_EXE $SCRIPT_DIR/../pr-check.sh done diff --git a/pr-check.sh b/pr-check.sh index 045d73e4e..ef405f6d4 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -18,6 +18,17 @@ set -e +# If we pass PYTHON_EXE, then we have a wheel and we don't want environment +if [ -z $PYTHON_EXE ]; then + PYTHON_EXE=python3 + PYTHON_WHL=0 + USE_VENV=1 +else + PYTHON_WHL=1 + USE_VENV=0 +fi +echo $PYTHON_EXE + if [ ! -f brainiak/__init__.py ] then echo "Run "$(basename "$0")" from the root of the BrainIAK hierarchy." @@ -27,7 +38,7 @@ fi basedir=$(pwd) function create_venv_venv { - python3 -m venv ../$1 + $PYTHON_EXE -m venv ../$1 } function activate_venv_venv { @@ -67,10 +78,12 @@ function exit_with_error { } function exit_with_error_and_venv { - $deactivate_venv - cd $basedir - rm -f .coverage.* - $remove_venv $venv + if [ $USE_VENV -eq 1 ]; then + $deactivate_venv + cd $basedir + rm -f .coverage.* + $remove_venv $venv + fi exit_with_error "$1" } @@ -97,25 +110,31 @@ else sdist_mode="--sdist-mode" fi -venv=$(mktemp -u brainiak_pr_venv_XXXXX) || \ - exit_with_error "mktemp -u error" -$create_venv $venv || { - exit_with_error "Virtual environment creation failed." -} -$activate_venv $venv || { - $remove_venv $venv - exit_with_error "Virtual environment activation failed." -} +if [ $USE_VENV -eq 1 ]; then + venv=$(mktemp -u brainiak_pr_venv_XXXXX) || \ + exit_with_error "mktemp -u error" + $create_venv $venv || { + exit_with_error "Virtual environment creation failed." + } + $activate_venv $venv || { + $remove_venv $venv + exit_with_error "Virtual environment activation failed." + } +fi # install brainiak in editable mode (required for testing) # brainiak will also be installed together with the developer dependencies, but # we install it first here to check that installation succeeds without the # developer dependencies. -python3 -m pip install $ignore_installed -U -e . || \ - exit_with_error_and_venv "Failed to install BrainIAK." +$PYTHON_EXE -m pip install -q numpy scipy cython pybind11 + +if [ $PYTHON_WHL -eq 0 ]; then + $PYTHON_EXE -m pip install -q $ignore_installed -U -e $PYTHON_WHL || \ + exit_with_error_and_venv "Failed to install BrainIAK." +fi # install developer dependencies -python3 -m pip install $ignore_installed -U -r requirements-dev.txt || \ +$PYTHON_EXE -m pip install -q $ignore_installed -U -r requirements-dev.txt || \ exit_with_error_and_venv "Failed to install development requirements." # static analysis @@ -123,7 +142,7 @@ python3 -m pip install $ignore_installed -U -r requirements-dev.txt || \ exit_with_error_and_venv "run-checks failed" # run tests -./run-tests.sh $sdist_mode || \ +PYTHON_EXE=$PYTHON_EXE ./run-tests.sh $sdist_mode || \ exit_with_error_and_venv "run-tests failed" # build documentation @@ -140,7 +159,9 @@ $make_wrapper make || { } cd - -$deactivate_venv -$remove_venv $venv +if [ $USE_VENV -eq 1 ]; then + $deactivate_venv + $remove_venv $venv +fi echo "pr-check finished successfully." diff --git a/run-tests.sh b/run-tests.sh index fc1b18da5..ac6a1f43d 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -20,7 +20,11 @@ set -e # detect editable mode. Use the "--sdist-mode" flag. sdist_mode=$1 -python3 -m pip freeze | grep -qi /brainiak \ +if [ -z $PYTHON_EXE ]; then + PYTHON_EXE=python3 +fi + +$PYTHON_EXE -m pip freeze | grep -qi /brainiak \ || [ ${sdist_mode:-default} = "--sdist-mode" ] \ || { echo "You must install brainiak in editable mode"` From 4fc0264a2b79b84d31a0e8040461a2cfa103d132 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 05:26:41 -0500 Subject: [PATCH 025/126] Fix travis.yml --- .travis.yml | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd12d7480..1463bcefb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,38 +36,43 @@ matrix: env: *macos_env before_install: *macos_before_install - # Build Linux wheels + # Build Linux wheels - os: linux dist: trusty sudo: required language: generic - env: - - LINUX_WHEELS=1 + env: LINUX_WHEELS=1 install: - ./bin/build-wheels.sh script: - ./bin/test-wheels.sh - # Build MacOS wheels + # Build MacOS wheels - os: osx osx_image: xcode7.3 sudo: required language: generic - env: *macos_env - - MACOS_WHEELS=1 + env: + - MACOS_WHEELS=1 + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib + -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 before_install: *macos_before_install install: - - "./bin/build-wheels-macos.sh" + - ./bin/build-wheels-macos.sh before_script: - - brew uninstall llvm mpich python3 + - brew uninstall llvm mpich python3 script: - - ./bin/test-wheels-macos.sh + - ./bin/test-wheels-macos.sh install: - - python3 -m pip install -U pip + - python3 -m pip install -U pip script: - - ./pr-check.sh + - ./pr-check.sh deploy: - provider: s3 From fe6d2fd2b30b034923e4eb409adcc3119d69784a Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 05:31:21 -0500 Subject: [PATCH 026/126] Fix wheel typo --- bin/README.md | 2 ++ pr-check.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/README.md b/bin/README.md index 50b9fcc3e..b16e53f14 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,6 +1,8 @@ # Wheel to-dos # TODO +- End-to-end test +- Package documentation [link](http://python-packaging.readthedocs.io/en/latest/non-code-files.html) - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) - Build for other OSs (e.g., xcode[VERSION], 32-bit Linux, Windows) diff --git a/pr-check.sh b/pr-check.sh index ef405f6d4..3c85e3f2d 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -129,7 +129,7 @@ fi $PYTHON_EXE -m pip install -q numpy scipy cython pybind11 if [ $PYTHON_WHL -eq 0 ]; then - $PYTHON_EXE -m pip install -q $ignore_installed -U -e $PYTHON_WHL || \ + $PYTHON_EXE -m pip install -q $ignore_installed -U -e . || \ exit_with_error_and_venv "Failed to install BrainIAK." fi From 9b7b8fdccadfd93c74eaf7ea810ddd0f5dd32cfd Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 05:52:04 -0500 Subject: [PATCH 027/126] Fix manylinux1 pr-check --- .travis.yml | 6 ++++-- bin/README.md | 10 ++++++---- bin/test-wheels-manylinux1.sh | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1463bcefb..e765bf723 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,7 @@ +# branches: + # only: + # - master + matrix: include: - os: linux @@ -87,7 +91,6 @@ deploy: skip_cleanup: true on: condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 - # branch: master # tags: true # - provider: pypi # user: brainiak @@ -96,7 +99,6 @@ deploy: # on: # condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 # tags: true - # branch: master # TODO: need to verify built wheels are uploaded not ones rebuilt by travis # distributions: "sdist bdist_wheel" # skip_cleanup: true diff --git a/bin/README.md b/bin/README.md index b16e53f14..ecb1c92b6 100755 --- a/bin/README.md +++ b/bin/README.md @@ -10,18 +10,20 @@ - Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8)) # To verify -- Restore ```.travis.yml``` - Modify ```setup.py``` to handle conditional MPI install -- Checkout mpi4py tag so we always create the same wheel - Upload wheels to S3 - Upload wheels to PyPI (uncomment) - Deploy only on master (uncomment in ```.travis.yml```) +- Only run on master branch (uncomment) +- Only deploy on tag (uncomment) + +# Completed - Add LICENSE into whl file +- Checkout mpi4py tag so we always create the same wheel - Figure out ```pr-check.sh```, other testing scripts, and documentation - Change ```pr-check.sh``` to optionally take a ```PYTHON``` argument, using ```python3``` if unspecified - Change ```pr-check.sh``` to optionally install a wheel (or automatically find the wheel based on the ```PYTHON```argument) insead of from source - -# Completed +- Restore ```.travis.yml``` and make sure previous tests working - Linux build script - Linux test script - MacOS build script diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 11ebed991..3dd40f264 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -18,5 +18,5 @@ for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do $PIP install -q $MPI4PY_WHEEL $PIP install -q $BRAINIAK_WHEEL - PYTHON_EXE=$PYTHON_EXE $SCRIPT_DIR/../pr-check.sh + PYTHON_EXE=/opt/python/$PYTHON/bin/python $SCRIPT_DIR/../pr-check.sh done From 22dad1a3db7e9747e63fe4e9298eb05476f999a6 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 09:08:36 -0500 Subject: [PATCH 028/126] Fix run-checks errors --- pr-check.sh | 6 ++---- run-checks.sh | 20 ++++++++++++++------ run-tests.sh | 2 +- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/pr-check.sh b/pr-check.sh index 3c85e3f2d..15b5509a3 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -16,7 +16,7 @@ # Check readiness for pull request -set -e +set -e -x # If we pass PYTHON_EXE, then we have a wheel and we don't want environment if [ -z $PYTHON_EXE ]; then @@ -126,8 +126,6 @@ fi # brainiak will also be installed together with the developer dependencies, but # we install it first here to check that installation succeeds without the # developer dependencies. -$PYTHON_EXE -m pip install -q numpy scipy cython pybind11 - if [ $PYTHON_WHL -eq 0 ]; then $PYTHON_EXE -m pip install -q $ignore_installed -U -e . || \ exit_with_error_and_venv "Failed to install BrainIAK." @@ -138,7 +136,7 @@ $PYTHON_EXE -m pip install -q $ignore_installed -U -r requirements-dev.txt || \ exit_with_error_and_venv "Failed to install development requirements." # static analysis -./run-checks.sh || \ +PYTHON_EXE=$PYTHON_EXE ./run-checks.sh || \ exit_with_error_and_venv "run-checks failed" # run tests diff --git a/run-checks.sh b/run-checks.sh index 6df6cfe9f..fda3747a5 100755 --- a/run-checks.sh +++ b/run-checks.sh @@ -14,14 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e +set -ex set -o pipefail -flake8 --config setup.cfg brainiak -flake8 --config tests/.flake8 tests -mypy --ignore-missing-imports brainiak tests/[!_]* -rst-lint ./*.rst | { grep -v "is clean.$" || true; } -towncrier --version=100 --draft > /dev/null 2>&1 \ +if [ $PYTHON_EXE == 'python3' ]; then + RST_LINT=rst-lint + TOWNCRIER=towncrier +else + RST_LINT=$(dirname $PYTHON_EXE)/rst-lint + TOWNCRIER=$(dirname $PYTHON_EXE)/towncrier +fi + +$PYTHON_EXE -m flake8 --config setup.cfg brainiak +$PYTHON_EXE -m flake8 --config tests/.flake8 tests +$PYTHON_EXE -m mypy --ignore-missing-imports brainiak tests/[!_]* +$RST_LINT ./*.rst | { grep -v "is clean.$" || true; } +$TOWNCRIER --version=100 --draft > /dev/null 2>&1 \ || echo "Error assembling news fragments using towncrier." echo "run-checks finished successfully." diff --git a/run-tests.sh b/run-tests.sh index ac6a1f43d..7ad5eeae0 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e +set -e -x # When installing from sdist, the pip freeze output cannot be used to # detect editable mode. Use the "--sdist-mode" flag. From 0fe9ae40a8fd26f2ca6cd3bc9a824ff2a4287c3b Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 10:01:18 -0500 Subject: [PATCH 029/126] Add Python 3.6 and fix linux wheel testing --- .travis.yml | 7 +++++++ bin/test-wheels-manylinux1.sh | 13 ++++++++++++- pr-check.sh | 4 +++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e765bf723..a62f385d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,13 @@ matrix: python: 3.5 addons: *linux_addons + - os: linux + dist: trusty + sudo: required + language: python + python: 3.6 + addons: *linux_addons + - os: osx osx_image: xcode7.3 language: generic diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 3dd40f264..bea777926 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -9,6 +9,7 @@ set -x SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) WHEEL_DIR=$SCRIPT_DIR/../.whl +# Test install for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep mpi4py) BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep brainiak) @@ -17,6 +18,16 @@ for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do $PIP install -q $MPI4PY_WHEEL $PIP install -q $BRAINIAK_WHEEL +done + +# Separate tests into a separate loop because we want to make sure +# we can install brainiak without installing mpich, but require mpiexec during tests - PYTHON_EXE=/opt/python/$PYTHON/bin/python $SCRIPT_DIR/../pr-check.sh +# Install dependencies +yum install -y -q \ + mpich2-devel \ + libgomp + +for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do + PYTHON_EXE=/opt/python/$PYTHON/bin/python $SCRIPT_DIR/../pr-check.sh done diff --git a/pr-check.sh b/pr-check.sh index 15b5509a3..e04f75b2a 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -16,7 +16,7 @@ # Check readiness for pull request -set -e -x +set -ex # If we pass PYTHON_EXE, then we have a wheel and we don't want environment if [ -z $PYTHON_EXE ]; then @@ -126,6 +126,8 @@ fi # brainiak will also be installed together with the developer dependencies, but # we install it first here to check that installation succeeds without the # developer dependencies. +$PYTHON_EXE -m pip install -q numpy scipy cython pybind11 + if [ $PYTHON_WHL -eq 0 ]; then $PYTHON_EXE -m pip install -q $ignore_installed -U -e . || \ exit_with_error_and_venv "Failed to install BrainIAK." From 4b34a3ebe90bd1e92e8ac96bd249de9f93726ab0 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 14:48:29 -0500 Subject: [PATCH 030/126] Fix mpich related bugs in wheel testing --- .travis.yml | 1 - bin/README.md | 5 ++++- bin/test-wheels-manylinux1.sh | 11 +++++++++++ pr-check.sh | 4 +--- run-tests.sh | 24 ++++++++++++++---------- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index a62f385d6..b0113a7c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -98,7 +98,6 @@ deploy: skip_cleanup: true on: condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 - # tags: true # - provider: pypi # user: brainiak # password: diff --git a/bin/README.md b/bin/README.md index ecb1c92b6..ff313e9b1 100755 --- a/bin/README.md +++ b/bin/README.md @@ -2,10 +2,11 @@ # TODO - End-to-end test -- Package documentation [link](http://python-packaging.readthedocs.io/en/latest/non-code-files.html) +- Refactor build and test scripts - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) - Build for other OSs (e.g., xcode[VERSION], 32-bit Linux, Windows) + - Rename MacOS wheels for multiple versions? - Test on other python distros (e.g., conda)? - Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8)) @@ -13,6 +14,7 @@ - Modify ```setup.py``` to handle conditional MPI install - Upload wheels to S3 - Upload wheels to PyPI (uncomment) +- Always upload to S3, only upload to PyPI on tag (uncomment) - Deploy only on master (uncomment in ```.travis.yml```) - Only run on master branch (uncomment) - Only deploy on tag (uncomment) @@ -28,3 +30,4 @@ - Linux test script - MacOS build script - MacOS test script +- ~~Package documentation [link](http://python-packaging.readthedocs.io/en/latest/non-code-files.html)~~ diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index bea777926..6d30855a3 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -20,6 +20,17 @@ for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do $PIP install -q $BRAINIAK_WHEEL done +# Install and configure ssh so we can ssh locally for MPI tests +yum install -y -q openssh-server +service sshd start + +ssh-keygen -f ~/.ssh/id_rsa -t rsa -N '' -b 4096 +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys + +echo "Host *" >> ~/.ssh/config +echo " StrictHostKeyChecking no" >> ~/.ssh/config + # Separate tests into a separate loop because we want to make sure # we can install brainiak without installing mpich, but require mpiexec during tests diff --git a/pr-check.sh b/pr-check.sh index e04f75b2a..55fae4a39 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -126,15 +126,13 @@ fi # brainiak will also be installed together with the developer dependencies, but # we install it first here to check that installation succeeds without the # developer dependencies. -$PYTHON_EXE -m pip install -q numpy scipy cython pybind11 - if [ $PYTHON_WHL -eq 0 ]; then $PYTHON_EXE -m pip install -q $ignore_installed -U -e . || \ exit_with_error_and_venv "Failed to install BrainIAK." fi # install developer dependencies -$PYTHON_EXE -m pip install -q $ignore_installed -U -r requirements-dev.txt || \ +$PYTHON_EXE -m pip install $ignore_installed -U -r requirements-dev.txt || \ exit_with_error_and_venv "Failed to install development requirements." # static analysis diff --git a/run-tests.sh b/run-tests.sh index 7ad5eeae0..d51feedba 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -20,8 +20,17 @@ set -e -x # detect editable mode. Use the "--sdist-mode" flag. sdist_mode=$1 +mpi_command=mpiexec.hydra +COVERAGE="$PYTHON_EXE -m coverage" if [ -z $PYTHON_EXE ]; then PYTHON_EXE=python3 + mpi_command=mpiexec + COVERAGE=coverage +fi + +if [ ! -z $SLURM_NODELIST ] +then + mpi_command=srun fi $PYTHON_EXE -m pip freeze | grep -qi /brainiak \ @@ -35,14 +44,9 @@ $PYTHON_EXE -m pip freeze | grep -qi /brainiak \ # Define MKL env variable that is required by Theano to run with MKL 2018 export MKL_THREADING_LAYER=GNU -mpi_command=mpiexec -if [ ! -z $SLURM_NODELIST ] -then - mpi_command=srun -fi -$mpi_command -n 2 coverage run -m pytest +$mpi_command -n 2 $COVERAGE run -m pytest -coverage combine +$COVERAGE combine # Travis error workaround coverage_report=$(mktemp -u coverage_report_XXXXX) || { @@ -51,11 +55,11 @@ coverage_report=$(mktemp -u coverage_report_XXXXX) || { } set +e -coverage report > $coverage_report +$COVERAGE report > $coverage_report report_exit_code=$? -coverage html -coverage xml +$COVERAGE html +$COVERAGE xml cat $coverage_report rm $coverage_report From 4ca3b394279f31ebca013aa7d62df7aaeefa8f20 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 15:00:31 -0500 Subject: [PATCH 031/126] Update readme --- bin/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/README.md b/bin/README.md index ff313e9b1..6c996d6c1 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,8 +1,10 @@ # Wheel to-dos # TODO +- Split wheel build and test into two travis stages - End-to-end test - Refactor build and test scripts + - Export PATH and clean up logic - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) - Build for other OSs (e.g., xcode[VERSION], 32-bit Linux, Windows) From 79ab35c756a059353d1af7b483815018539500d8 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 15:29:55 -0500 Subject: [PATCH 032/126] Refactor prcheck --- .travis.yml | 34 +++++++++---------- bin/test-wheels-macos.sh | 8 ++++- bin/test-wheels-manylinux1.sh | 4 +-- pr-check.sh | 61 ++++++++++++----------------------- run-checks.sh | 18 +++-------- run-tests.sh | 33 ++++++++----------- 6 files changed, 65 insertions(+), 93 deletions(-) diff --git a/.travis.yml b/.travis.yml index b0113a7c1..a4e38a098 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,24 +28,24 @@ matrix: python: 3.6 addons: *linux_addons - - os: osx - osx_image: xcode7.3 - language: generic - env: &macos_env - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib - -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - before_install: &macos_before_install - - brew install llvm mpich python3 + # - os: osx + # osx_image: xcode7.3 + # language: generic + # env: &macos_env + # - CC=/usr/local/opt/llvm/bin/clang + # - CXX=/usr/local/opt/llvm/bin/clang++ + # - LDFLAGS="-L/usr/local/opt/llvm/lib + # -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + # - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + # - HOMEBREW_NO_AUTO_UPDATE=1 + # before_install: &macos_before_install + # - brew install llvm mpich python3 - - os: osx - osx_image: xcode8 - language: generic - env: *macos_env - before_install: *macos_before_install + # - os: osx + # osx_image: xcode8 + # language: generic + # env: *macos_env + # before_install: *macos_before_install # Build Linux wheels - os: linux diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 431e524e1..b0f1ce2a1 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -16,7 +16,9 @@ PY_WHEEL_VERSIONS=("34" "35" "36") SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) WHEEL_DIR=$SCRIPT_DIR/../.whl +# Test whether we can install without any dependencies for ((i=0; i<${#PY_MMS[@]}; ++i)); do + # TODO: delete this section once we implement travis stages PY_MM=${PY_MMS[i]} PY_WHEEL_VERSION=${PY_WHEEL_VERSIONS[i]} @@ -31,6 +33,10 @@ for ((i=0; i<${#PY_MMS[@]}; ++i)); do $PIP install -q $MPI4PY_WHEEL $PIP install -q $BRAINIAK_WHEEL +done - PYTHON_EXE=$PYTHON_EXE $SCRIPT_DIR/../pr-check.sh +brew install mpich +# Test packages +for ((i=0; i<${#PY_MMS[@]}; ++i)); do + PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $SCRIPT_DIR/../pr-check.sh done diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 6d30855a3..d09d02e04 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -9,7 +9,7 @@ set -x SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) WHEEL_DIR=$SCRIPT_DIR/../.whl -# Test install +# Test whether we can install without any dependencies for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep mpi4py) BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep brainiak) @@ -40,5 +40,5 @@ yum install -y -q \ libgomp for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do - PYTHON_EXE=/opt/python/$PYTHON/bin/python $SCRIPT_DIR/../pr-check.sh + mpi_command=mpiexec.hydra PATH=/opt/python/$PYTHON/bin:$PATH $SCRIPT_DIR/../pr-check.sh done diff --git a/pr-check.sh b/pr-check.sh index 55fae4a39..50a4c51e5 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -18,17 +18,6 @@ set -ex -# If we pass PYTHON_EXE, then we have a wheel and we don't want environment -if [ -z $PYTHON_EXE ]; then - PYTHON_EXE=python3 - PYTHON_WHL=0 - USE_VENV=1 -else - PYTHON_WHL=1 - USE_VENV=0 -fi -echo $PYTHON_EXE - if [ ! -f brainiak/__init__.py ] then echo "Run "$(basename "$0")" from the root of the BrainIAK hierarchy." @@ -38,7 +27,7 @@ fi basedir=$(pwd) function create_venv_venv { - $PYTHON_EXE -m venv ../$1 + python3 -m venv ../$1 } function activate_venv_venv { @@ -78,12 +67,10 @@ function exit_with_error { } function exit_with_error_and_venv { - if [ $USE_VENV -eq 1 ]; then - $deactivate_venv - cd $basedir - rm -f .coverage.* - $remove_venv $venv - fi + $deactivate_venv + cd $basedir + rm -f .coverage.* + $remove_venv $venv exit_with_error "$1" } @@ -110,37 +97,33 @@ else sdist_mode="--sdist-mode" fi -if [ $USE_VENV -eq 1 ]; then - venv=$(mktemp -u brainiak_pr_venv_XXXXX) || \ - exit_with_error "mktemp -u error" - $create_venv $venv || { - exit_with_error "Virtual environment creation failed." - } - $activate_venv $venv || { - $remove_venv $venv - exit_with_error "Virtual environment activation failed." - } -fi +venv=$(mktemp -u brainiak_pr_venv_XXXXX) || \ + exit_with_error "mktemp -u error" +$create_venv $venv || { + exit_with_error "Virtual environment creation failed." +} +$activate_venv $venv || { + $remove_venv $venv + exit_with_error "Virtual environment activation failed." +} # install brainiak in editable mode (required for testing) # brainiak will also be installed together with the developer dependencies, but # we install it first here to check that installation succeeds without the # developer dependencies. -if [ $PYTHON_WHL -eq 0 ]; then - $PYTHON_EXE -m pip install -q $ignore_installed -U -e . || \ - exit_with_error_and_venv "Failed to install BrainIAK." -fi +python3 -m pip install $ignore_installed -U -e . || \ + exit_with_error_and_venv "Failed to install BrainIAK." # install developer dependencies -$PYTHON_EXE -m pip install $ignore_installed -U -r requirements-dev.txt || \ +python3 -m pip install $ignore_installed -U -r requirements-dev.txt || \ exit_with_error_and_venv "Failed to install development requirements." # static analysis -PYTHON_EXE=$PYTHON_EXE ./run-checks.sh || \ +./run-checks.sh || \ exit_with_error_and_venv "run-checks failed" # run tests -PYTHON_EXE=$PYTHON_EXE ./run-tests.sh $sdist_mode || \ +./run-tests.sh $sdist_mode || \ exit_with_error_and_venv "run-tests failed" # build documentation @@ -157,9 +140,7 @@ $make_wrapper make || { } cd - -if [ $USE_VENV -eq 1 ]; then - $deactivate_venv - $remove_venv $venv -fi +$deactivate_venv +$remove_venv $venv echo "pr-check finished successfully." diff --git a/run-checks.sh b/run-checks.sh index fda3747a5..92612f7a1 100755 --- a/run-checks.sh +++ b/run-checks.sh @@ -17,19 +17,11 @@ set -ex set -o pipefail -if [ $PYTHON_EXE == 'python3' ]; then - RST_LINT=rst-lint - TOWNCRIER=towncrier -else - RST_LINT=$(dirname $PYTHON_EXE)/rst-lint - TOWNCRIER=$(dirname $PYTHON_EXE)/towncrier -fi - -$PYTHON_EXE -m flake8 --config setup.cfg brainiak -$PYTHON_EXE -m flake8 --config tests/.flake8 tests -$PYTHON_EXE -m mypy --ignore-missing-imports brainiak tests/[!_]* -$RST_LINT ./*.rst | { grep -v "is clean.$" || true; } -$TOWNCRIER --version=100 --draft > /dev/null 2>&1 \ +flake8 --config setup.cfg brainiak +flake8 --config tests/.flake8 tests +mypy --ignore-missing-imports brainiak tests/[!_]* +rst-lint ./*.rst | { grep -v "is clean.$" || true; } +towncrier --version=100 --draft > /dev/null 2>&1 \ || echo "Error assembling news fragments using towncrier." echo "run-checks finished successfully." diff --git a/run-tests.sh b/run-tests.sh index d51feedba..3f863fda8 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -14,26 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e -x +set -ex # When installing from sdist, the pip freeze output cannot be used to # detect editable mode. Use the "--sdist-mode" flag. sdist_mode=$1 -mpi_command=mpiexec.hydra -COVERAGE="$PYTHON_EXE -m coverage" -if [ -z $PYTHON_EXE ]; then - PYTHON_EXE=python3 - mpi_command=mpiexec - COVERAGE=coverage -fi - -if [ ! -z $SLURM_NODELIST ] -then - mpi_command=srun -fi - -$PYTHON_EXE -m pip freeze | grep -qi /brainiak \ +python3 -m pip freeze | grep -qi /brainiak \ || [ ${sdist_mode:-default} = "--sdist-mode" ] \ || { echo "You must install brainiak in editable mode"` @@ -44,9 +31,15 @@ $PYTHON_EXE -m pip freeze | grep -qi /brainiak \ # Define MKL env variable that is required by Theano to run with MKL 2018 export MKL_THREADING_LAYER=GNU -$mpi_command -n 2 $COVERAGE run -m pytest +if [ -z $mpi_command ]; then + mpi_command=mpiexec +elif [ ! -z $SLURM_NODELIST ]; then + mpi_command=srun +fi +echo $PATH +$mpi_command -n 2 coverage run -m pytest -$COVERAGE combine +coverage combine # Travis error workaround coverage_report=$(mktemp -u coverage_report_XXXXX) || { @@ -55,11 +48,11 @@ coverage_report=$(mktemp -u coverage_report_XXXXX) || { } set +e -$COVERAGE report > $coverage_report +coverage report > $coverage_report report_exit_code=$? -$COVERAGE html -$COVERAGE xml +coverage html +coverage xml cat $coverage_report rm $coverage_report From f7def62779b5980aaa42a49317195a2c96dd0fba Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 17:29:49 -0500 Subject: [PATCH 033/126] Trash commit --- .travis.yml | 46 +++++++++++------------ bin/README.md | 3 ++ bin/build-wheels-manylinux1.sh | 3 +- bin/run-manylinux1.sh | 2 +- bin/test-wheels-macos.sh | 1 + bin/test-wheels-manylinux1.sh | 69 +++++++++++++++++++++++----------- pr-check.sh | 4 +- run-tests.sh | 13 ++++--- 8 files changed, 86 insertions(+), 55 deletions(-) diff --git a/.travis.yml b/.travis.yml index a4e38a098..fce25f514 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,31 +2,31 @@ # only: # - master -matrix: - include: - - os: linux - dist: trusty - sudo: required - language: python - python: 3.4 - addons: &linux_addons - apt: &apt - packages: - - build-essential libgomp1 libmpich-dev mpich +# matrix: + # include: + # - os: linux + # dist: trusty + # sudo: required + # language: python + # python: 3.4 + # addons: &linux_addons + # apt: &apt + # packages: + # - build-essential libgomp1 libmpich-dev mpich - - os: linux - dist: trusty - sudo: required - language: python - python: 3.5 - addons: *linux_addons + # - os: linux + # dist: trusty + # sudo: required + # language: python + # python: 3.5 + # addons: *linux_addons - - os: linux - dist: trusty - sudo: required - language: python - python: 3.6 - addons: *linux_addons + # - os: linux + # dist: trusty + # sudo: required + # language: python + # python: 3.6 + # addons: *linux_addons # - os: osx # osx_image: xcode7.3 diff --git a/bin/README.md b/bin/README.md index 6c996d6c1..8dcb75c6e 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,10 +1,13 @@ # Wheel to-dos # TODO +- Create brainiak/manylinux1 Docker image - Split wheel build and test into two travis stages - End-to-end test - Refactor build and test scripts - Export PATH and clean up logic + - Consolidate python version numbers across various scripts + - Rationalize variables; different scripts call the same things different things - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) - Build for other OSs (e.g., xcode[VERSION], 32-bit Linux, Windows) diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh index bb734282d..69f99001e 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-wheels-manylinux1.sh @@ -8,8 +8,7 @@ set -x # Install dependencies yum install -y -q \ - mpich2-devel \ - libgomp + mpich2-devel SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) WHEEL_DIR=$SCRIPT_DIR/../.whl diff --git a/bin/run-manylinux1.sh b/bin/run-manylinux1.sh index 2dc69eff9..8fe0daacb 100755 --- a/bin/run-manylinux1.sh +++ b/bin/run-manylinux1.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -docker run --rm -w /brainiak -v `pwd`:/brainiak -ti quay.io/pypa/manylinux1_x86_64 /bin/bash +docker run --rm -w /src -v `pwd`:/src -ti quay.io/pypa/manylinux1_x86_64 /bin/bash diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index b0f1ce2a1..ac1df2e05 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -17,6 +17,7 @@ SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) WHEEL_DIR=$SCRIPT_DIR/../.whl # Test whether we can install without any dependencies +# TODO: delete once we have setup.py setup correctly for ((i=0; i<${#PY_MMS[@]}; ++i)); do # TODO: delete this section once we implement travis stages PY_MM=${PY_MMS[i]} diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index d09d02e04..29f8f8bc7 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -6,39 +6,64 @@ set -e # Show explicitly which commands are currently running. set -x -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -WHEEL_DIR=$SCRIPT_DIR/../.whl +# SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +# WHEEL_DIR=$SCRIPT_DIR/../.whl # Test whether we can install without any dependencies -for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep mpi4py) - BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep brainiak) +# TODO: delete once we have setup.py setup correctly +# for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do + # MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep mpi4py) + # BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep brainiak) - PIP=/opt/python/${PYTHON}/bin/pip + # PIP=/opt/python/${PYTHON}/bin/pip - $PIP install -q $MPI4PY_WHEEL - $PIP install -q $BRAINIAK_WHEEL -done + # $PIP install -q $MPI4PY_WHEEL + # $PIP install -q $BRAINIAK_WHEEL +# done # Install and configure ssh so we can ssh locally for MPI tests -yum install -y -q openssh-server -service sshd start +# yum install -y -q openssh-server +# service sshd start -ssh-keygen -f ~/.ssh/id_rsa -t rsa -N '' -b 4096 -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys +# ssh-keygen -f ~/.ssh/id_rsa -t rsa -N '' -b 4096 +# cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +# chmod 600 ~/.ssh/authorized_keys -echo "Host *" >> ~/.ssh/config -echo " StrictHostKeyChecking no" >> ~/.ssh/config +# echo "Host *" >> ~/.ssh/config +# echo " StrictHostKeyChecking no" >> ~/.ssh/config # Separate tests into a separate loop because we want to make sure # we can install brainiak without installing mpich, but require mpiexec during tests -# Install dependencies -yum install -y -q \ - mpich2-devel \ - libgomp +# Install openssl +yum install -y -q openssl-devel + +# Install libpython from source because manylinux1 doesn't pacakge it +PY_MMS=("3.4" "3.5" "3.6") +for ((i=0; i<${#PY_MMS[@]}; ++i)); do + # Get Python major version + PY_MM=${PY_MMS[i]} + + # Get cython minor version + CYTHON=$(find /opt/_internal -maxdepth 1 | grep $PY_MM | xargs basename ) + + # Get minor version + MINOR=$(echo $CYTHON | cut -d- -f2-) -for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do - mpi_command=mpiexec.hydra PATH=/opt/python/$PYTHON/bin:$PATH $SCRIPT_DIR/../pr-check.sh + # Download source + wget https://github.com/python/cpython/archive/v$MINOR.tar.gz + tar zxf v$MINOR + + pushd $CYTHON + ./configure --enable-shared > /dev/null 2>&1 + make -s -j + make install > /dev/null 2>&1 + popd done + +# Install dependencies +# yum install -y -q mpich2-devel + +# for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do + # mpi_command=mpiexec.hydra PATH=/opt/python/$PYTHON/bin:$PATH $SCRIPT_DIR/../pr-check.sh +# done diff --git a/pr-check.sh b/pr-check.sh index 50a4c51e5..cc4643d53 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -111,11 +111,13 @@ $activate_venv $venv || { # brainiak will also be installed together with the developer dependencies, but # we install it first here to check that installation succeeds without the # developer dependencies. + +python3 -m pip install .whl/mpi4py-3.0.0-cp34-cp34m-manylinux1_x86_64.whl python3 -m pip install $ignore_installed -U -e . || \ exit_with_error_and_venv "Failed to install BrainIAK." # install developer dependencies -python3 -m pip install $ignore_installed -U -r requirements-dev.txt || \ +python3 -m pip install -q $ignore_installed -U -r requirements-dev.txt || \ exit_with_error_and_venv "Failed to install development requirements." # static analysis diff --git a/run-tests.sh b/run-tests.sh index 3f863fda8..3c09a616d 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -36,10 +36,11 @@ if [ -z $mpi_command ]; then elif [ ! -z $SLURM_NODELIST ]; then mpi_command=srun fi -echo $PATH -$mpi_command -n 2 coverage run -m pytest +coverage=$(which coverage) -coverage combine +$mpi_command -n 2 $coverage run -m pytest + +$coverage combine # Travis error workaround coverage_report=$(mktemp -u coverage_report_XXXXX) || { @@ -48,11 +49,11 @@ coverage_report=$(mktemp -u coverage_report_XXXXX) || { } set +e -coverage report > $coverage_report +$coverage report > $coverage_report report_exit_code=$? -coverage html -coverage xml +$coverage html +$coverage xml cat $coverage_report rm $coverage_report From de5d38d99272b78558586ea304a92427c3f5084e Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 17:31:05 -0500 Subject: [PATCH 034/126] Slightly less trash commit. Must go home --- bin/test-wheels-manylinux1.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 29f8f8bc7..f24b28136 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -22,15 +22,15 @@ set -x # done # Install and configure ssh so we can ssh locally for MPI tests -# yum install -y -q openssh-server -# service sshd start +yum install -y -q openssh-server +service sshd start -# ssh-keygen -f ~/.ssh/id_rsa -t rsa -N '' -b 4096 -# cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -# chmod 600 ~/.ssh/authorized_keys +ssh-keygen -f ~/.ssh/id_rsa -t rsa -N '' -b 4096 +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys -# echo "Host *" >> ~/.ssh/config -# echo " StrictHostKeyChecking no" >> ~/.ssh/config +echo "Host *" >> ~/.ssh/config +echo " StrictHostKeyChecking no" >> ~/.ssh/config # Separate tests into a separate loop because we want to make sure # we can install brainiak without installing mpich, but require mpiexec during tests @@ -62,7 +62,9 @@ for ((i=0; i<${#PY_MMS[@]}; ++i)); do done # Install dependencies -# yum install -y -q mpich2-devel +yum install -y -q mpich2-devel + +mpi_command=mpiexec.hydra $SCRIPT_DIR/../pr-check.sh # for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do # mpi_command=mpiexec.hydra PATH=/opt/python/$PYTHON/bin:$PATH $SCRIPT_DIR/../pr-check.sh From b46829ab23b99e28a12a32af5f231fb0a1da9e89 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 17:32:54 -0500 Subject: [PATCH 035/126] Maybe the last one for now? --- bin/test-wheels-manylinux1.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index f24b28136..9d7e092eb 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -6,8 +6,8 @@ set -e # Show explicitly which commands are currently running. set -x -# SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -# WHEEL_DIR=$SCRIPT_DIR/../.whl +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +WHEEL_DIR=$SCRIPT_DIR/../.whl # Test whether we can install without any dependencies # TODO: delete once we have setup.py setup correctly From 4340c5764aa7cbece462ceb2bc497ddb9ef878b5 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 19:09:31 -0500 Subject: [PATCH 036/126] manylinux now passing tests --- .travis.yml | 8 +++++--- bin/README.md | 2 +- bin/test-wheels-manylinux1.sh | 15 +++++++++------ pr-check.sh | 14 +++++++++++--- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index fce25f514..38d212047 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,8 @@ # only: # - master -# matrix: - # include: +matrix: + include: # - os: linux # dist: trusty # sudo: required @@ -71,7 +71,9 @@ -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - before_install: *macos_before_install + # before_install: *macos_before_install + before_install: + - brew install llvm mpich python3 install: - ./bin/build-wheels-macos.sh before_script: diff --git a/bin/README.md b/bin/README.md index 8dcb75c6e..9e6cb5f1e 100755 --- a/bin/README.md +++ b/bin/README.md @@ -2,7 +2,7 @@ # TODO - Create brainiak/manylinux1 Docker image -- Split wheel build and test into two travis stages +- Split wheel build, test install, test run into three travis stages - End-to-end test - Refactor build and test scripts - Export PATH and clean up logic diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 9d7e092eb..20c03cbe0 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -36,7 +36,7 @@ echo " StrictHostKeyChecking no" >> ~/.ssh/config # we can install brainiak without installing mpich, but require mpiexec during tests # Install openssl -yum install -y -q openssl-devel +yum install -y -q openssl-devel bzip2-devel # Install libpython from source because manylinux1 doesn't pacakge it PY_MMS=("3.4" "3.5" "3.6") @@ -59,13 +59,16 @@ for ((i=0; i<${#PY_MMS[@]}; ++i)); do make -s -j make install > /dev/null 2>&1 popd + + rm -rf $CYTHON v$MINOR done # Install dependencies yum install -y -q mpich2-devel -mpi_command=mpiexec.hydra $SCRIPT_DIR/../pr-check.sh - -# for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do - # mpi_command=mpiexec.hydra PATH=/opt/python/$PYTHON/bin:$PATH $SCRIPT_DIR/../pr-check.sh -# done +for ((i=0; i<${#PY_MMS[@]}; ++i)); do + PYTHON3=$(which python3) + rm -rf $PYTHON3 + ln -s $(which python${PY_MMS[i]}) $PYTHON3 + mpi_command=mpiexec.hydra $SCRIPT_DIR/../pr-check.sh +done diff --git a/pr-check.sh b/pr-check.sh index cc4643d53..7fa79e799 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -112,9 +112,17 @@ $activate_venv $venv || { # we install it first here to check that installation succeeds without the # developer dependencies. -python3 -m pip install .whl/mpi4py-3.0.0-cp34-cp34m-manylinux1_x86_64.whl -python3 -m pip install $ignore_installed -U -e . || \ - exit_with_error_and_venv "Failed to install BrainIAK." +if [ ! -z $WHEEL_DIR ]; then + PYTHON_MAJOR=$(python3 -c "import sys; v = sys.version_info; print(v[0] * 10 + v[1])") + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON_MAJOR | grep mpi4py) + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON_MAJOR | grep brainiak) + + python3 -m pip install -q $MPI4PY_WHEEL + python3 -m pip install -q $BRAINIAK_WHEEL +else + python3 -m pip install $ignore_installed -U -e . || \ + exit_with_error_and_venv "Failed to install BrainIAK." +fi # install developer dependencies python3 -m pip install -q $ignore_installed -U -r requirements-dev.txt || \ From 102236b4cfda69aa8d4bfd490d10e17ceaf8bc13 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 19:52:42 -0500 Subject: [PATCH 037/126] Fix python3.6 bug --- bin/test-wheels-manylinux1.sh | 4 ++-- pr-check.sh | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 20c03cbe0..353bc6195 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -45,7 +45,7 @@ for ((i=0; i<${#PY_MMS[@]}; ++i)); do PY_MM=${PY_MMS[i]} # Get cython minor version - CYTHON=$(find /opt/_internal -maxdepth 1 | grep $PY_MM | xargs basename ) + CYTHON=$(find /opt/_internal -maxdepth 1 | grep -- -$PY_MM | xargs basename ) # Get minor version MINOR=$(echo $CYTHON | cut -d- -f2-) @@ -70,5 +70,5 @@ for ((i=0; i<${#PY_MMS[@]}; ++i)); do PYTHON3=$(which python3) rm -rf $PYTHON3 ln -s $(which python${PY_MMS[i]}) $PYTHON3 - mpi_command=mpiexec.hydra $SCRIPT_DIR/../pr-check.sh + mpi_command=mpiexec.hydra PYTHON_MAJOR=${PY_MMS[i]} $SCRIPT_DIR/../pr-check.sh done diff --git a/pr-check.sh b/pr-check.sh index 7fa79e799..fa21e055e 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -27,7 +27,11 @@ fi basedir=$(pwd) function create_venv_venv { + if [ ! -z $PYTHON_MAJOR ]; then + python$PYTHON_MAJOR -m venv ../$1 + else python3 -m venv ../$1 + fi } function activate_venv_venv { From ab1327229f6e0f2546eb4488eba90461d6b23a83 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 20:45:50 -0500 Subject: [PATCH 038/126] Use brainiak/manylinux docker for linux wheels --- bin/README.md | 4 ++-- bin/build-wheels-manylinux1.sh | 35 ++++++++++++++++--------------- bin/build-wheels.sh | 2 +- bin/test-wheels-manylinux1.sh | 38 +++++----------------------------- bin/test-wheels.sh | 2 +- pr-check.sh | 8 +++---- 6 files changed, 31 insertions(+), 58 deletions(-) diff --git a/bin/README.md b/bin/README.md index 9e6cb5f1e..5aa7c9f54 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,11 +1,9 @@ # Wheel to-dos # TODO -- Create brainiak/manylinux1 Docker image - Split wheel build, test install, test run into three travis stages - End-to-end test - Refactor build and test scripts - - Export PATH and clean up logic - Consolidate python version numbers across various scripts - Rationalize variables; different scripts call the same things different things - Get things working on Jenkins @@ -25,6 +23,7 @@ - Only deploy on tag (uncomment) # Completed +- Create brainiak/manylinux Docker image - Add LICENSE into whl file - Checkout mpi4py tag so we always create the same wheel - Figure out ```pr-check.sh```, other testing scripts, and documentation @@ -35,4 +34,5 @@ - Linux test script - MacOS build script - MacOS test script +- ~~Export PATH and clean up logic~~ - ~~Package documentation [link](http://python-packaging.readthedocs.io/en/latest/non-code-files.html)~~ diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh index 69f99001e..e61976201 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-wheels-manylinux1.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # Caust the script to exit if a single command fails. -# set -e +set -e # Show explicitly which commands are currently running. set -x # Install dependencies yum install -y -q \ - mpich2-devel + mpich2-devel SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) WHEEL_DIR=$SCRIPT_DIR/../.whl @@ -21,19 +21,20 @@ pushd /mpi4py git checkout 3.0.0 popd -for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do - /opt/python/${PYTHON}/bin/python -m pip install -U pip - - pushd /mpi4py - git clean -f -f -x -d -q - /opt/python/${PYTHON}/bin/python setup.py -q bdist_wheel - auditwheel repair dist/*.whl - mv wheelhouse/*.whl $WHEEL_DIR/ - popd - - git clean -f -f -x -d -q -e .whl - /opt/python/${PYTHON}/bin/python -m pip install -q . - /opt/python/${PYTHON}/bin/python setup.py -q bdist_wheel - auditwheel repair dist/*.whl - mv wheelhouse/*.whl $WHEEL_DIR/ +for VERSION in 3.4 3.5 3.6; do + PYTHON=python$VERSION + $PYTHON -m pip install -U pip wheel + + pushd /mpi4py + git clean -f -f -x -d -q + $PYTHON setup.py -q bdist_wheel + auditwheel repair dist/*.whl + mv wheelhouse/*.whl $WHEEL_DIR/ + popd + + git clean -f -f -x -d -q -e .whl + $PYTHON -m pip install -q . + $PYTHON setup.py -q bdist_wheel + auditwheel repair dist/*.whl + mv wheelhouse/*.whl $WHEEL_DIR/ done diff --git a/bin/build-wheels.sh b/bin/build-wheels.sh index 0214a98c2..abf6cea2d 100755 --- a/bin/build-wheels.sh +++ b/bin/build-wheels.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -docker run --rm -w /src -v `pwd`:/src -ti quay.io/pypa/manylinux1_x86_64 ./bin/build-wheels-manylinux1.sh +docker run --rm -w /src -v `pwd`:/src -ti brainiak/manylinux ./bin/build-wheels-manylinux1.sh diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 353bc6195..00936f0bc 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -35,40 +35,12 @@ echo " StrictHostKeyChecking no" >> ~/.ssh/config # Separate tests into a separate loop because we want to make sure # we can install brainiak without installing mpich, but require mpiexec during tests -# Install openssl -yum install -y -q openssl-devel bzip2-devel - -# Install libpython from source because manylinux1 doesn't pacakge it -PY_MMS=("3.4" "3.5" "3.6") -for ((i=0; i<${#PY_MMS[@]}; ++i)); do - # Get Python major version - PY_MM=${PY_MMS[i]} - - # Get cython minor version - CYTHON=$(find /opt/_internal -maxdepth 1 | grep -- -$PY_MM | xargs basename ) - - # Get minor version - MINOR=$(echo $CYTHON | cut -d- -f2-) - - # Download source - wget https://github.com/python/cpython/archive/v$MINOR.tar.gz - tar zxf v$MINOR - - pushd $CYTHON - ./configure --enable-shared > /dev/null 2>&1 - make -s -j - make install > /dev/null 2>&1 - popd - - rm -rf $CYTHON v$MINOR -done - # Install dependencies yum install -y -q mpich2-devel -for ((i=0; i<${#PY_MMS[@]}; ++i)); do - PYTHON3=$(which python3) - rm -rf $PYTHON3 - ln -s $(which python${PY_MMS[i]}) $PYTHON3 - mpi_command=mpiexec.hydra PYTHON_MAJOR=${PY_MMS[i]} $SCRIPT_DIR/../pr-check.sh +for VERSION in 3.4 3.5 3.6; do + mpi_command=mpiexec.hydra \ + WHEEL_DIR=$WHEEL_DIR \ + VERSION=$VERSION \ + $SCRIPT_DIR/../pr-check.sh done diff --git a/bin/test-wheels.sh b/bin/test-wheels.sh index e7a21f5fe..d149ec536 100755 --- a/bin/test-wheels.sh +++ b/bin/test-wheels.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -docker run --rm -w /src -v `pwd`:/src -ti quay.io/pypa/manylinux1_x86_64 ./bin/test-wheels-manylinux1.sh +docker run --rm -w /src -v `pwd`:/src -ti brainiak/manylinux ./bin/test-wheels-manylinux1.sh diff --git a/pr-check.sh b/pr-check.sh index fa21e055e..a0c4c2f42 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -27,8 +27,8 @@ fi basedir=$(pwd) function create_venv_venv { - if [ ! -z $PYTHON_MAJOR ]; then - python$PYTHON_MAJOR -m venv ../$1 + if [ ! -z $VERSION ]; then + python$VERSION -m venv ../$1 else python3 -m venv ../$1 fi @@ -118,8 +118,8 @@ $activate_venv $venv || { if [ ! -z $WHEEL_DIR ]; then PYTHON_MAJOR=$(python3 -c "import sys; v = sys.version_info; print(v[0] * 10 + v[1])") - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON_MAJOR | grep mpi4py) - BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON_MAJOR | grep brainiak) + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep cp$PYTHON_MAJOR | grep mpi4py) + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep cp$PYTHON_MAJOR | grep brainiak) python3 -m pip install -q $MPI4PY_WHEEL python3 -m pip install -q $BRAINIAK_WHEEL From adab3fb838867f8fd2fafc418234d66ad5b1ba2c Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 21:39:18 -0500 Subject: [PATCH 039/126] Testing stages --- .travis.yml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38d212047..ae9b70c6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,15 @@ +# Testing flow +# - Install and test sdist +# - Build wheel, test locally, and deploy +# - Install wheel +# - Test wheel + # branches: # only: # - master -matrix: - include: +# matrix: + # include: # - os: linux # dist: trusty # sudo: required @@ -47,12 +53,16 @@ matrix: # env: *macos_env # before_install: *macos_before_install +jobs: + include: + - stage: build + # Build Linux wheels - - os: linux + os: linux dist: trusty sudo: required language: generic - env: LINUX_WHEELS=1 + env: DEPLOY_WHEEL=1 install: - ./bin/build-wheels.sh script: @@ -64,7 +74,7 @@ matrix: sudo: required language: generic env: - - MACOS_WHEELS=1 + - DEPLOY_WHEEL=1 - CC=/usr/local/opt/llvm/bin/clang - CXX=/usr/local/opt/llvm/bin/clang++ - LDFLAGS="-L/usr/local/opt/llvm/lib @@ -81,6 +91,7 @@ matrix: script: - ./bin/test-wheels-macos.sh + install: - python3 -m pip install -U pip @@ -99,7 +110,7 @@ deploy: upload-dir: .whl skip_cleanup: true on: - condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 + condition: $DEPLOY_WHEEL=1 # - provider: pypi # user: brainiak # password: From a4c59ed70ded2172a7ed183c20e103a27e9159ec Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 23:33:55 -0500 Subject: [PATCH 040/126] Linux converted --- bin/build-wheels-manylinux1.sh | 12 ++++++------ bin/run-manylinux1.sh | 2 +- bin/test-wheels-manylinux1.sh | 4 ++-- pr-check.sh | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh index e61976201..4a815129a 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-wheels-manylinux1.sh @@ -21,15 +21,15 @@ pushd /mpi4py git checkout 3.0.0 popd -for VERSION in 3.4 3.5 3.6; do - PYTHON=python$VERSION +for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do + PYTHON=/opt/python/$VERSION/bin/python $PYTHON -m pip install -U pip wheel pushd /mpi4py - git clean -f -f -x -d -q - $PYTHON setup.py -q bdist_wheel - auditwheel repair dist/*.whl - mv wheelhouse/*.whl $WHEEL_DIR/ + git clean -f -f -x -d -q + $PYTHON setup.py -q bdist_wheel + auditwheel repair dist/*.whl + mv wheelhouse/*.whl $WHEEL_DIR/ popd git clean -f -f -x -d -q -e .whl diff --git a/bin/run-manylinux1.sh b/bin/run-manylinux1.sh index 8fe0daacb..7b886a5c8 100755 --- a/bin/run-manylinux1.sh +++ b/bin/run-manylinux1.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -docker run --rm -w /src -v `pwd`:/src -ti quay.io/pypa/manylinux1_x86_64 /bin/bash +docker run --rm -w /src -v `pwd`:/src -ti brainiak/manylinux /bin/bash diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 00936f0bc..954922ddf 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -38,9 +38,9 @@ echo " StrictHostKeyChecking no" >> ~/.ssh/config # Install dependencies yum install -y -q mpich2-devel -for VERSION in 3.4 3.5 3.6; do +for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do mpi_command=mpiexec.hydra \ WHEEL_DIR=$WHEEL_DIR \ - VERSION=$VERSION \ + PYTHON=/opt/python/$VERSION/bin/python \ $SCRIPT_DIR/../pr-check.sh done diff --git a/pr-check.sh b/pr-check.sh index a0c4c2f42..9feef9e0f 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -27,8 +27,8 @@ fi basedir=$(pwd) function create_venv_venv { - if [ ! -z $VERSION ]; then - python$VERSION -m venv ../$1 + if [ ! -z $PYTHON ]; then + $PYTHON -m venv ../$1 else python3 -m venv ../$1 fi From caf78c8e6a9ecd94880211fb877ee0208ae0608a Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 23:35:58 -0500 Subject: [PATCH 041/126] FIx wheel directory --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ae9b70c6f..d8539bcdd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -106,7 +106,7 @@ deploy: bucket: brainiak acl: public_read region: us-east-1 - local_dir: brainiak/.whl + local_dir: .whl upload-dir: .whl skip_cleanup: true on: From 338a22b7e0af4c77f5dedd53d6f914e5b3419a84 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 14 Dec 2017 23:40:54 -0500 Subject: [PATCH 042/126] Test s3 upload --- .travis.yml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index d8539bcdd..9e80576ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,27 +69,27 @@ jobs: - ./bin/test-wheels.sh # Build MacOS wheels - - os: osx - osx_image: xcode7.3 - sudo: required - language: generic - env: - - DEPLOY_WHEEL=1 - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib - -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 + # - os: osx + # osx_image: xcode7.3 + # sudo: required + # language: generic + # env: + # - DEPLOY_WHEEL=1 + # - CC=/usr/local/opt/llvm/bin/clang + # - CXX=/usr/local/opt/llvm/bin/clang++ + # - LDFLAGS="-L/usr/local/opt/llvm/lib + # -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + # - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + # - HOMEBREW_NO_AUTO_UPDATE=1 # before_install: *macos_before_install - before_install: - - brew install llvm mpich python3 - install: - - ./bin/build-wheels-macos.sh - before_script: - - brew uninstall llvm mpich python3 - script: - - ./bin/test-wheels-macos.sh + # before_install: + # - brew install llvm mpich python3 + # install: + # - ./bin/build-wheels-macos.sh + # before_script: + # - brew uninstall llvm mpich python3 + # script: + # - ./bin/test-wheels-macos.sh install: From 9a4457e98a32b0818238c6831a1dd1cc28ec57d1 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 00:02:28 -0500 Subject: [PATCH 043/126] Fix MacOS wheel paths --- bin/test-wheels-macos.sh | 23 +++++++++++++---------- bin/test-wheels-manylinux1.sh | 14 +++++++------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index ac1df2e05..70fadfd86 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -19,18 +19,18 @@ WHEEL_DIR=$SCRIPT_DIR/../.whl # Test whether we can install without any dependencies # TODO: delete once we have setup.py setup correctly for ((i=0; i<${#PY_MMS[@]}; ++i)); do - # TODO: delete this section once we implement travis stages - PY_MM=${PY_MMS[i]} - PY_WHEEL_VERSION=${PY_WHEEL_VERSIONS[i]} + # TODO: delete this section once we implement travis stages + PY_MM=${PY_MMS[i]} + PY_WHEEL_VERSION=${PY_WHEEL_VERSIONS[i]} - PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM - PIP="$(dirname $PYTHON_EXE)/pip$PY_MM" + PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM + PIP="$(dirname $PYTHON_EXE)/pip$PY_MM" - # Find the appropriate wheel by grepping for the Python version. - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep mpi4py) + # Find the appropriate wheel by grepping for the Python version. + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep mpi4py) - # TODO: this will actually pick up both wheels since brainiak is in the path - BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep brainiak) + # TODO: this will actually pick up both wheels since brainiak is in the path + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep brainiak) $PIP install -q $MPI4PY_WHEEL $PIP install -q $BRAINIAK_WHEEL @@ -39,5 +39,8 @@ done brew install mpich # Test packages for ((i=0; i<${#PY_MMS[@]}; ++i)); do - PATH=$MACPYTHON_PY_PREFIX/$PY_MM/bin:$PATH $SCRIPT_DIR/../pr-check.sh + PY_MM=${PY_MMS[i]} + WHEEL_DIR=$WHEEL_DIR \ + PYTHON=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM \ + $SCRIPT_DIR/../pr-check.sh done diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 954922ddf..6652890ce 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -11,15 +11,15 @@ WHEEL_DIR=$SCRIPT_DIR/../.whl # Test whether we can install without any dependencies # TODO: delete once we have setup.py setup correctly -# for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do - # MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep mpi4py) - # BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep brainiak) +for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep mpi4py) + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep brainiak) - # PIP=/opt/python/${PYTHON}/bin/pip + PIP=/opt/python/${PYTHON}/bin/pip - # $PIP install -q $MPI4PY_WHEEL - # $PIP install -q $BRAINIAK_WHEEL -# done + $PIP install -q $MPI4PY_WHEEL + $PIP install -q $BRAINIAK_WHEEL +done # Install and configure ssh so we can ssh locally for MPI tests yum install -y -q openssh-server From 09b6905cf14fc9cc1194ca3baf8ad5ab4d1a90e8 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 00:04:08 -0500 Subject: [PATCH 044/126] Enable MacOS --- .travis.yml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e80576ff..d8539bcdd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,27 +69,27 @@ jobs: - ./bin/test-wheels.sh # Build MacOS wheels - # - os: osx - # osx_image: xcode7.3 - # sudo: required - # language: generic - # env: - # - DEPLOY_WHEEL=1 - # - CC=/usr/local/opt/llvm/bin/clang - # - CXX=/usr/local/opt/llvm/bin/clang++ - # - LDFLAGS="-L/usr/local/opt/llvm/lib - # -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - # - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - # - HOMEBREW_NO_AUTO_UPDATE=1 + - os: osx + osx_image: xcode7.3 + sudo: required + language: generic + env: + - DEPLOY_WHEEL=1 + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib + -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 # before_install: *macos_before_install - # before_install: - # - brew install llvm mpich python3 - # install: - # - ./bin/build-wheels-macos.sh - # before_script: - # - brew uninstall llvm mpich python3 - # script: - # - ./bin/test-wheels-macos.sh + before_install: + - brew install llvm mpich python3 + install: + - ./bin/build-wheels-macos.sh + before_script: + - brew uninstall llvm mpich python3 + script: + - ./bin/test-wheels-macos.sh install: From e9bf23dbbbc58a3bdd3762b4cec72bc48107eb05 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 00:12:13 -0500 Subject: [PATCH 045/126] Minimal s3 test --- .travis.yml | 60 ++++++++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index d8539bcdd..07b653de3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,39 +64,39 @@ jobs: language: generic env: DEPLOY_WHEEL=1 install: - - ./bin/build-wheels.sh - script: - - ./bin/test-wheels.sh + # - ./bin/build-wheels.sh + - echo "Hello!" > .whl/tmp + # script: + # - ./bin/test-wheels.sh # Build MacOS wheels - - os: osx - osx_image: xcode7.3 - sudo: required - language: generic - env: - - DEPLOY_WHEEL=1 - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib - -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - # before_install: *macos_before_install - before_install: - - brew install llvm mpich python3 - install: - - ./bin/build-wheels-macos.sh - before_script: - - brew uninstall llvm mpich python3 - script: - - ./bin/test-wheels-macos.sh + # - os: osx + # osx_image: xcode7.3 + # sudo: required + # language: generic + # env: + # - DEPLOY_WHEEL=1 + # - CC=/usr/local/opt/llvm/bin/clang + # - CXX=/usr/local/opt/llvm/bin/clang++ + # - LDFLAGS="-L/usr/local/opt/llvm/lib + # -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + # - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + # - HOMEBREW_NO_AUTO_UPDATE=1 + # before_install: + # - brew install llvm mpich python3 + # install: + # - ./bin/build-wheels-macos.sh + # before_script: + # - brew uninstall llvm mpich python3 + # script: + # - ./bin/test-wheels-macos.sh -install: - - python3 -m pip install -U pip +# install: + # - python3 -m pip install -U pip -script: - - ./pr-check.sh +# script: + # - ./pr-check.sh deploy: - provider: s3 @@ -109,8 +109,8 @@ deploy: local_dir: .whl upload-dir: .whl skip_cleanup: true - on: - condition: $DEPLOY_WHEEL=1 + # on: + # condition: $DEPLOY_WHEEL=1 # - provider: pypi # user: brainiak # password: From 378af962c6b0462729444d46a86fcf9d33e94686 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 00:13:45 -0500 Subject: [PATCH 046/126] Minimal s3 test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 07b653de3..4e72eac28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,7 +65,7 @@ jobs: env: DEPLOY_WHEEL=1 install: # - ./bin/build-wheels.sh - - echo "Hello!" > .whl/tmp + - mkdir .whl && echo "Hello!" >> .whl/tmp # script: # - ./bin/test-wheels.sh From b95d822050a1ff88ab39935796a28699949b962c Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 00:18:03 -0500 Subject: [PATCH 047/126] Minimal s3 test --- .travis.yml | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e72eac28..37175dc1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -98,26 +98,28 @@ jobs: # script: # - ./pr-check.sh -deploy: - - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - acl: public_read - region: us-east-1 - local_dir: .whl - upload-dir: .whl - skip_cleanup: true - # on: - # condition: $DEPLOY_WHEEL=1 - # - provider: pypi - # user: brainiak - # password: - # secure: travis encrypt and add to file manually - # on: - # condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 - # tags: true - # TODO: need to verify built wheels are uploaded not ones rebuilt by travis - # distributions: "sdist bdist_wheel" - # skip_cleanup: true + deploy: + - provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + bucket: brainiak + acl: public_read + region: us-east-1 + local_dir: .whl + upload-dir: .whl + skip_cleanup: true + # on: + # repo: brainiak/brainiak + # condition: $DEPLOY_WHEEL=1 + # - provider: pypi + # user: brainiak + # password: + # secure: travis encrypt and add to file manually + # on: + # condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 + # tags: true + # repo: brainiak/brainiak + # TODO: need to verify built wheels are uploaded not ones rebuilt by travis + # distributions: "sdist bdist_wheel" + # skip_cleanup: true From 91bc32728c6688145fecea606d23177f8bb80d2d Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 10:16:50 -0500 Subject: [PATCH 048/126] Test MacOS build --- .travis.yml | 64 ++++++++++++++++++++++++++------------------------- bin/README.md | 2 ++ 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/.travis.yml b/.travis.yml index 37175dc1c..066a5e12c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,40 +64,34 @@ jobs: language: generic env: DEPLOY_WHEEL=1 install: - # - ./bin/build-wheels.sh - - mkdir .whl && echo "Hello!" >> .whl/tmp - # script: - # - ./bin/test-wheels.sh + - ./bin/build-wheels.sh + script: + - ./bin/test-wheels.sh # Build MacOS wheels - # - os: osx - # osx_image: xcode7.3 - # sudo: required - # language: generic - # env: - # - DEPLOY_WHEEL=1 - # - CC=/usr/local/opt/llvm/bin/clang - # - CXX=/usr/local/opt/llvm/bin/clang++ - # - LDFLAGS="-L/usr/local/opt/llvm/lib - # -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - # - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - # - HOMEBREW_NO_AUTO_UPDATE=1 - # before_install: - # - brew install llvm mpich python3 - # install: - # - ./bin/build-wheels-macos.sh - # before_script: - # - brew uninstall llvm mpich python3 - # script: - # - ./bin/test-wheels-macos.sh - - -# install: - # - python3 -m pip install -U pip - -# script: - # - ./pr-check.sh + - os: osx + osx_image: xcode7.3 + sudo: required + language: generic + env: + - DEPLOY_WHEEL=1 + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib + -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + before_install: + - brew install llvm mpich python3 + install: + - ./bin/build-wheels-macos.sh + before_script: + - brew uninstall llvm mpich python3 + script: + - ./bin/test-wheels-macos.sh + # Deploy wheels to S3 + - stage: deploy deploy: - provider: s3 access_key_id: AKIAJKBW6H2VKKQDHILQ @@ -123,3 +117,11 @@ jobs: # TODO: need to verify built wheels are uploaded not ones rebuilt by travis # distributions: "sdist bdist_wheel" # skip_cleanup: true + + +# install: + # - python3 -m pip install -U pip + +# script: + # - ./pr-check.sh + diff --git a/bin/README.md b/bin/README.md index 5aa7c9f54..f056b0e5c 100755 --- a/bin/README.md +++ b/bin/README.md @@ -2,6 +2,8 @@ # TODO - Split wheel build, test install, test run into three travis stages +- Have dev and prod S3 buckets for testing wheels +- Clean up buckets appropriately - End-to-end test - Refactor build and test scripts - Consolidate python version numbers across various scripts From f2dc79933b3dfc5d6c89d50fedd248816c17cd11 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 11:00:44 -0500 Subject: [PATCH 049/126] Remove quiet flags --- bin/README.md | 1 + pr-check.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/README.md b/bin/README.md index f056b0e5c..cfc176a82 100755 --- a/bin/README.md +++ b/bin/README.md @@ -4,6 +4,7 @@ - Split wheel build, test install, test run into three travis stages - Have dev and prod S3 buckets for testing wheels - Clean up buckets appropriately +- Add back quiet / silent flags - End-to-end test - Refactor build and test scripts - Consolidate python version numbers across various scripts diff --git a/pr-check.sh b/pr-check.sh index 9feef9e0f..c99e21fff 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -121,15 +121,15 @@ if [ ! -z $WHEEL_DIR ]; then MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep cp$PYTHON_MAJOR | grep mpi4py) BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep cp$PYTHON_MAJOR | grep brainiak) - python3 -m pip install -q $MPI4PY_WHEEL - python3 -m pip install -q $BRAINIAK_WHEEL + python3 -m pip install $MPI4PY_WHEEL + python3 -m pip install $BRAINIAK_WHEEL else python3 -m pip install $ignore_installed -U -e . || \ exit_with_error_and_venv "Failed to install BrainIAK." fi # install developer dependencies -python3 -m pip install -q $ignore_installed -U -r requirements-dev.txt || \ +python3 -m pip install $ignore_installed -U -r requirements-dev.txt || \ exit_with_error_and_venv "Failed to install development requirements." # static analysis From 6fde470257bde2fcd885d5f1c19b98f04374ecdd Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 12:04:04 -0500 Subject: [PATCH 050/126] Upgrade pip --- bin/README.md | 3 ++- bin/test-wheels-macos.sh | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/README.md b/bin/README.md index cfc176a82..f285c16fc 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,8 +1,8 @@ # Wheel to-dos # TODO +- Recreate and encrypt keys for brainiak/brainiak - Split wheel build, test install, test run into three travis stages -- Have dev and prod S3 buckets for testing wheels - Clean up buckets appropriately - Add back quiet / silent flags - End-to-end test @@ -27,6 +27,7 @@ # Completed - Create brainiak/manylinux Docker image +- Have dev and prod S3 buckets for testing wheels - Add LICENSE into whl file - Checkout mpi4py tag so we always create the same wheel - Figure out ```pr-check.sh```, other testing scripts, and documentation diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 70fadfd86..b59268d51 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -32,6 +32,8 @@ for ((i=0; i<${#PY_MMS[@]}; ++i)); do # TODO: this will actually pick up both wheels since brainiak is in the path BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep brainiak) + $PIP install --upgrade pip + $PIP install -q $MPI4PY_WHEEL $PIP install -q $BRAINIAK_WHEEL done From c14d369f6de2035b903975920e776c4392ff3f79 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 12:18:02 -0500 Subject: [PATCH 051/126] Check stage publishing --- .travis.yml | 61 +++++++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index 066a5e12c..5cdefdb32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,13 +52,16 @@ # language: generic # env: *macos_env # before_install: *macos_before_install +env: + global: + - BRAINIAK_REPO=danielsuo/brainiak jobs: include: - - stage: build + - stage: test # Build Linux wheels - os: linux + - os: linux dist: trusty sudo: required language: generic @@ -86,42 +89,40 @@ jobs: install: - ./bin/build-wheels-macos.sh before_script: - - brew uninstall llvm mpich python3 + - brew uninstall -f --ignore-dependencies llvm mpich python3 script: - ./bin/test-wheels-macos.sh - # Deploy wheels to S3 - - stage: deploy + # Only publish if we're on master + - stage: publish + if: repo = $BRAINIAK_REPO and branch = asdf + script: skip deploy: - - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - acl: public_read - region: us-east-1 - local_dir: .whl - upload-dir: .whl + - provider: pypi + distributions: "sdist bdist_wheel" skip_cleanup: true - # on: - # repo: brainiak/brainiak - # condition: $DEPLOY_WHEEL=1 - # - provider: pypi - # user: brainiak - # password: - # secure: travis encrypt and add to file manually - # on: - # condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1 - # tags: true - # repo: brainiak/brainiak - # TODO: need to verify built wheels are uploaded not ones rebuilt by travis - # distributions: "sdist bdist_wheel" - # skip_cleanup: true - # install: # - python3 -m pip install -U pip # script: # - ./pr-check.sh - + +# Deploy wheels to S3 only after build wheel jobs +# - Deploy all commits to master with travis-cli S3 user to brainiak +# - This deploy MUST come from brainiak/brainiak +deploy: + - provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + bucket: brainiak + acl: public_read + region: us-east-1 + local_dir: .whl + upload-dir: $TRAVIS_COMMIT/.whl + skip_cleanup: true + on: + repo: $BRAINIAK_REPO + branch: master + condition: $DEPLOY_WHEEL = 1 From 00e6b1495e033df31da940c844d5da5b02ba1cf3 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 12:34:58 -0500 Subject: [PATCH 052/126] Update travis.yml with draft flow --- .travis.yml | 59 +++++++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5cdefdb32..17ee33e58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,21 +4,27 @@ # - Install wheel # - Test wheel -# branches: - # only: - # - master +branches: + only: + - master + +env: + global: + - BRAINIAK_REPO=danielsuo/brainiak + +jobs: + include: + - stage: test -# matrix: - # include: # - os: linux # dist: trusty # sudo: required # language: python # python: 3.4 # addons: &linux_addons - # apt: &apt - # packages: - # - build-essential libgomp1 libmpich-dev mpich + # apt: &apt + # packages: + # - build-essential libgomp1 libmpich-dev mpich # - os: linux # dist: trusty @@ -52,13 +58,6 @@ # language: generic # env: *macos_env # before_install: *macos_before_install -env: - global: - - BRAINIAK_REPO=danielsuo/brainiak - -jobs: - include: - - stage: test # Build Linux wheels - os: linux @@ -93,20 +92,26 @@ jobs: script: - ./bin/test-wheels-macos.sh - # Only publish if we're on master - - stage: publish - if: repo = $BRAINIAK_REPO and branch = asdf - script: skip - deploy: - - provider: pypi - distributions: "sdist bdist_wheel" - skip_cleanup: true + # Make sure we can install our wheels from scratch if we're on mater (not just targeting master) + # - stage: integration + # if: repo = $BRAINIAK_REPO and branch = master + + # Only publish if we're on tagged master commit (not just targeting master) and integration passed. + # - stage: publish + # if: repo = $BRAINIAK_REPO and branch = master + # script: skip + # deploy: + # - provider: pypi + # distributions: "sdist bdist_wheel" + # skip_cleanup: true + # on: + # tags: true -# install: - # - python3 -m pip install -U pip +install: + - python3 -m pip install -U pip -# script: - # - ./pr-check.sh +script: + - ./pr-check.sh # Deploy wheels to S3 only after build wheel jobs # - Deploy all commits to master with travis-cli S3 user to brainiak From 73b785d10f651ba955f90ec4a8ef05e6502d317e Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 13:32:36 -0500 Subject: [PATCH 053/126] Install pip within virtual environment --- .travis.yml | 53 ++++++++++++++++++++++------------------ bin/README.md | 1 + bin/test-wheels-macos.sh | 2 -- pr-check.sh | 3 +++ 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/.travis.yml b/.travis.yml index 17ee33e58..8551114ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,6 +70,25 @@ jobs: script: - ./bin/test-wheels.sh + # Deploy wheels to S3 only after build wheel jobs + # - Deploy all commits to master with travis-cli S3 user to brainiak + # - This deploy MUST come from brainiak/brainiak + deploy: &wheel_deploy + - provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + bucket: brainiak + acl: public_read + region: us-east-1 + local_dir: .whl + upload-dir: $TRAVIS_COMMIT/.whl + skip_cleanup: true + on: + repo: $BRAINIAK_REPO + branch: master + condition: $DEPLOY_WHEEL = 1 + # Build MacOS wheels - os: osx osx_image: xcode7.3 @@ -91,6 +110,7 @@ jobs: - brew uninstall -f --ignore-dependencies llvm mpich python3 script: - ./bin/test-wheels-macos.sh + deploy: *wheel_deploy # Make sure we can install our wheels from scratch if we're on mater (not just targeting master) # - stage: integration @@ -107,27 +127,12 @@ jobs: # on: # tags: true -install: - - python3 -m pip install -U pip - -script: - - ./pr-check.sh - -# Deploy wheels to S3 only after build wheel jobs -# - Deploy all commits to master with travis-cli S3 user to brainiak -# - This deploy MUST come from brainiak/brainiak -deploy: - - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - acl: public_read - region: us-east-1 - local_dir: .whl - upload-dir: $TRAVIS_COMMIT/.whl - skip_cleanup: true - on: - repo: $BRAINIAK_REPO - branch: master - condition: $DEPLOY_WHEEL = 1 + # - stage: pypi-test + +# TODO: Move these into each job; steps outside a job are by default grouped into a stage +# install: + # - python3 -m pip install -U pip + +# script: + # - ./pr-check.sh + diff --git a/bin/README.md b/bin/README.md index f285c16fc..2d5b05e41 100755 --- a/bin/README.md +++ b/bin/README.md @@ -3,6 +3,7 @@ # TODO - Recreate and encrypt keys for brainiak/brainiak - Split wheel build, test install, test run into three travis stages +- Allow setup.py to take argument for mpi4py wheel location - Clean up buckets appropriately - Add back quiet / silent flags - End-to-end test diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index b59268d51..70fadfd86 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -32,8 +32,6 @@ for ((i=0; i<${#PY_MMS[@]}; ++i)); do # TODO: this will actually pick up both wheels since brainiak is in the path BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep brainiak) - $PIP install --upgrade pip - $PIP install -q $MPI4PY_WHEEL $PIP install -q $BRAINIAK_WHEEL done diff --git a/pr-check.sh b/pr-check.sh index c99e21fff..5ce3eb7e7 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -111,6 +111,9 @@ $activate_venv $venv || { exit_with_error "Virtual environment activation failed." } +# Upgrad pip here +python3 -m pip install -U pip + # install brainiak in editable mode (required for testing) # brainiak will also be installed together with the developer dependencies, but # we install it first here to check that installation succeeds without the From 732823d817d64251d9b47da2bd028cc8ad1714ee Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 13:36:37 -0500 Subject: [PATCH 054/126] Stages not well documented --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8551114ad..252ff4254 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ jobs: # before_install: *macos_before_install # Build Linux wheels - - os: linux + os: linux dist: trusty sudo: required language: generic From 232225238f10857ed7c0ea5e2e6dc1fafd7fe587 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 14:52:04 -0500 Subject: [PATCH 055/126] Force ARCH x86_64 --- .travis.yml | 9 +++-- bin/README.md | 1 + bin/build-wheels-macos.sh | 57 ++++++++------------------------ bin/install_python_macos.sh | 18 ++++++++++ bin/install_python_manylinux1.sh | 21 ++++++++++++ 5 files changed, 60 insertions(+), 46 deletions(-) create mode 100644 bin/install_python_macos.sh create mode 100755 bin/install_python_manylinux1.sh diff --git a/.travis.yml b/.travis.yml index 252ff4254..a1a094676 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ branches: env: global: - BRAINIAK_REPO=danielsuo/brainiak + - PYTHON_VERSIONS="3.4.7 3.5.4 3.6.2" jobs: include: @@ -102,12 +103,14 @@ jobs: -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 + - ARCHFLAGS="-arch x86_64" before_install: - - brew install llvm mpich python3 + - brew install llvm mpich + - VERSIONS=$PYTHON_VERSIONS ./bin/install_python_macos.sh install: - - ./bin/build-wheels-macos.sh + - VERSIONS=$PYTHON_VERSIONS ./bin/build-wheels-macos.sh before_script: - - brew uninstall -f --ignore-dependencies llvm mpich python3 + - brew uninstall -f --ignore-dependencies llvm mpich script: - ./bin/test-wheels-macos.sh deploy: *wheel_deploy diff --git a/bin/README.md b/bin/README.md index 2d5b05e41..10b484773 100755 --- a/bin/README.md +++ b/bin/README.md @@ -27,6 +27,7 @@ - Only deploy on tag (uncomment) # Completed +- Separate install python and build wheels in MacOS - Create brainiak/manylinux Docker image - Have dev and prod S3 buckets for testing wheels - Add LICENSE into whl file diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index 06952c697..6b4525ad2 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -1,27 +1,11 @@ #!/usr/bin/env bash # Cause the script to exit if a single command fails. -# set -e +set -e # Show explicitly which commands are currently running. set -x -MACPYTHON_URL=https://www.python.org/ftp/python -MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions -DOWNLOAD_DIR=python_downloads - -PY_VERSIONS=("3.4.4" - "3.5.3" - "3.6.1") -PY_INSTS=("python-3.4.4-macosx10.6.pkg" - "python-3.5.3-macosx10.6.pkg" - "python-3.6.1-macosx10.6.pkg") -PY_MMS=("3.4" - "3.5" - "3.6") - -mkdir -p $DOWNLOAD_DIR - SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) WHEEL_DIR=$SCRIPT_DIR/../.whl mkdir -p $WHEEL_DIR @@ -31,39 +15,26 @@ pushd mpi4py git checkout 3.0.0 popd -for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do - PY_VERSION=${PY_VERSIONS[i]} - PY_INST=${PY_INSTS[i]} - PY_MM=${PY_MMS[i]} - - git clean -f -f -x -d -q -e .whl -e $DOWNLOAD_DIR -e mpi4py +for VERSION in $VERSIONS +do + MAJOR=${VERSION%.*} + PYTHON=python$MAJOR + DELOCATE=$(dirname $PYTHON)/delocate-wheel - # Install Python. - INST_PATH=python_downloads/$PY_INST - curl $MACPYTHON_URL/$PY_VERSION/$PY_INST > $INST_PATH - sudo installer -pkg $INST_PATH -target / - - PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM - PIP_CMD="$(dirname $PYTHON_EXE)/pip$PY_MM" - - $PIP_CMD install -U pip - $PIP_CMD install -q wheel delocate numpy cython pybind11 scipy + git clean -f -f -x -d -q -e .whl -e mpi4py pushd mpi4py git clean -f -f -x -d -q - env ARCHFLAGS="-arch x86_64" $PYTHON_EXE setup.py bdist_wheel - $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl + $PYTHON setup.py bdist_wheel + $DELOCATE dist/*.whl mv dist/*.whl $WHEEL_DIR/ popd - env ARCHFLAGS="-arch x86_64" $PIP_CMD install -q . - env ARCHFLAGS="-arch x86_64" $PYTHON_EXE setup.py bdist_wheel - $MACPYTHON_PY_PREFIX/$PY_MM/bin/delocate-wheel dist/*.whl + $PYTHON -m pip install -q . + $PYTHON setup.py bdist_wheel + $DELOCATE dist/*.whl mv dist/*.whl $WHEEL_DIR/ - $PIP_CMD uninstall -y . - $PIP_CMD uninstall -y numpy cython pybind11 scipy mpi4py - + $PYTHON -m pip uninstall -y . + $PYTHON -m pip uninstall -y numpy cython pybind11 scipy mpi4py done - -echo $WHEEL_DIR diff --git a/bin/install_python_macos.sh b/bin/install_python_macos.sh new file mode 100644 index 000000000..ee5275a89 --- /dev/null +++ b/bin/install_python_macos.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +MACPYTHON_URL=https://www.python.org/ftp/python +DOWNLOAD_DIR=python_downloads +mkdir -p $DOWNLOAD_DIR + +for VERSION in $VERSIONS +do + PKG="python-$VERSION-macosx10.6.pkg" + MAJOR=${VERSION%.*} + + # Install Python. + INST_PATH=$DOWNLOAD_DIR/$PKG + curl $MACPYTHON_URL/$VERSION/$PKG > $INST_PATH + sudo installer -pkg $INST_PATH -target / + + pip$MAJOR install -q -U pip delocate +done diff --git a/bin/install_python_manylinux1.sh b/bin/install_python_manylinux1.sh new file mode 100755 index 000000000..4562fdb31 --- /dev/null +++ b/bin/install_python_manylinux1.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +for VERSION in $VERSIONS +do + # Download source + wget https://github.com/python/cpython/archive/v$VERSION.tar.gz + tar zxf v$VERSION + + # Install + pushd cpython-$VERSION + ./configure --enable-shared + make -j + make install + popd + + PYTHON=python${VERSION%.*} + $PYTHON -m pip install -U pip wheel + + # Clean up + rm -rf cpython-$VERSION v$VERSION +done From a1cce4b7fbe18468f60bf50a73375e9193778131 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 15:25:27 -0500 Subject: [PATCH 056/126] Update macos --- .travis.yml | 2 +- bin/README.md | 2 +- bin/build-wheels-macos.sh | 9 +++++++- bin/test-wheels-macos.sh | 44 +++++++++++++++++---------------------- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index a1a094676..644fdc0c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -112,7 +112,7 @@ jobs: before_script: - brew uninstall -f --ignore-dependencies llvm mpich script: - - ./bin/test-wheels-macos.sh + - VERSIONS=$PYTHON_VERSIONS ./bin/test-wheels-macos.sh deploy: *wheel_deploy # Make sure we can install our wheels from scratch if we're on mater (not just targeting master) diff --git a/bin/README.md b/bin/README.md index 10b484773..ac8ad338f 100755 --- a/bin/README.md +++ b/bin/README.md @@ -2,7 +2,6 @@ # TODO - Recreate and encrypt keys for brainiak/brainiak -- Split wheel build, test install, test run into three travis stages - Allow setup.py to take argument for mpi4py wheel location - Clean up buckets appropriately - Add back quiet / silent flags @@ -27,6 +26,7 @@ - Only deploy on tag (uncomment) # Completed +- Split wheel build, test install, test run into three travis stages - Separate install python and build wheels in MacOS - Create brainiak/manylinux Docker image - Have dev and prod S3 buckets for testing wheels diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index 6b4525ad2..cf35aa9d7 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -18,11 +18,16 @@ popd for VERSION in $VERSIONS do MAJOR=${VERSION%.*} - PYTHON=python$MAJOR + # Replicate readlink -f + PYTHON_DIR=$(cd $(dirname $(readlink $(which python$VERSION))); pwd) + PYTHON=$PYTHON_DIR/python$VERSION DELOCATE=$(dirname $PYTHON)/delocate-wheel git clean -f -f -x -d -q -e .whl -e mpi4py + $PYTHON -m venv venv + source venv/bin/activate + pushd mpi4py git clean -f -f -x -d -q $PYTHON setup.py bdist_wheel @@ -37,4 +42,6 @@ do $PYTHON -m pip uninstall -y . $PYTHON -m pip uninstall -y numpy cython pybind11 scipy mpi4py + + deactivate done diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 70fadfd86..b6f350423 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -6,41 +6,35 @@ set -e # Show explicitly which commands are currently running. set -x - -MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions -PY_MMS=("3.4" "3.5" "3.6") - -# This array is just used to find the right wheel. -PY_WHEEL_VERSIONS=("34" "35" "36") - SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) WHEEL_DIR=$SCRIPT_DIR/../.whl # Test whether we can install without any dependencies # TODO: delete once we have setup.py setup correctly -for ((i=0; i<${#PY_MMS[@]}; ++i)); do - # TODO: delete this section once we implement travis stages - PY_MM=${PY_MMS[i]} - PY_WHEEL_VERSION=${PY_WHEEL_VERSIONS[i]} +for VERSION in $VERSIONS +do + MAJOR=${VERSION%.*} + PYTHON_DIR=$(cd $(dirname $(readlink $(which python$VERSION))); pwd) + PYTHON=$PYTHON_DIR/python$VERSION + WHEEL_VERSION=$(echo $MAJOR | tr -d '.') - PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM - PIP="$(dirname $PYTHON_EXE)/pip$PY_MM" + # Find the appropriate wheel by grepping for the Python version. + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep mpi4py) - # Find the appropriate wheel by grepping for the Python version. - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep mpi4py) + # TODO: this will actually pick up both wheels since brainiak is in the path + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep brainiak) - # TODO: this will actually pick up both wheels since brainiak is in the path - BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$PY_WHEEL_VERSION" | grep brainiak) - - $PIP install -q $MPI4PY_WHEEL - $PIP install -q $BRAINIAK_WHEEL + $PYTHON -m pip install -q $MPI4PY_WHEEL + $PYTHON -m pip install -q $BRAINIAK_WHEEL done brew install mpich + # Test packages -for ((i=0; i<${#PY_MMS[@]}; ++i)); do - PY_MM=${PY_MMS[i]} - WHEEL_DIR=$WHEEL_DIR \ - PYTHON=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM \ - $SCRIPT_DIR/../pr-check.sh +for VERSION in $VERSIONS +do + # TODO: refactor this out + PYTHON_DIR=$(cd $(dirname $(readlink $(which python$VERSION))); pwd) + PYTHON=$PYTHON_DIR/python$VERSION + WHEEL_DIR=$WHEEL_DIR PYTHON=$PYTHON $SCRIPT_DIR/../pr-check.sh done From 17e4a3166d8aee2197d75e67c7b28e912d0ed113 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 15:27:40 -0500 Subject: [PATCH 057/126] Fix --- bin/build-wheels-macos.sh | 4 ++-- bin/test-wheels-macos.sh | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index cf35aa9d7..02ff24cb4 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -19,8 +19,8 @@ for VERSION in $VERSIONS do MAJOR=${VERSION%.*} # Replicate readlink -f - PYTHON_DIR=$(cd $(dirname $(readlink $(which python$VERSION))); pwd) - PYTHON=$PYTHON_DIR/python$VERSION + PYTHON_DIR=$(cd $(dirname $(readlink $(which python$MAJOR))); pwd) + PYTHON=$PYTHON_DIR/python$MAJOR DELOCATE=$(dirname $PYTHON)/delocate-wheel git clean -f -f -x -d -q -e .whl -e mpi4py diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index b6f350423..75dcd4487 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -14,8 +14,8 @@ WHEEL_DIR=$SCRIPT_DIR/../.whl for VERSION in $VERSIONS do MAJOR=${VERSION%.*} - PYTHON_DIR=$(cd $(dirname $(readlink $(which python$VERSION))); pwd) - PYTHON=$PYTHON_DIR/python$VERSION + PYTHON_DIR=$(cd $(dirname $(readlink $(which python$MAJOR))); pwd) + PYTHON=$PYTHON_DIR/python$MAJOR WHEEL_VERSION=$(echo $MAJOR | tr -d '.') # Find the appropriate wheel by grepping for the Python version. @@ -34,7 +34,8 @@ brew install mpich for VERSION in $VERSIONS do # TODO: refactor this out - PYTHON_DIR=$(cd $(dirname $(readlink $(which python$VERSION))); pwd) - PYTHON=$PYTHON_DIR/python$VERSION + MAJOR=${VERSION%.*} + PYTHON_DIR=$(cd $(dirname $(readlink $(which python$MAJOR))); pwd) + PYTHON=$PYTHON_DIR/python$MAJOR WHEEL_DIR=$WHEEL_DIR PYTHON=$PYTHON $SCRIPT_DIR/../pr-check.sh done From f9c6a0cccc8dc59d4424b5c06a4d62bd96607d54 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 13:36:07 -0800 Subject: [PATCH 058/126] Fix MacOS tests --- .travis.yml | 2 +- bin/build-wheels-macos.sh | 4 +--- ...stall_python_macos.sh => install-python-macos.sh} | 0 ...on_manylinux1.sh => install-python-manylinux1.sh} | 0 bin/test-wheels-macos.sh | 6 ++++++ pr-check.sh | 12 ++++++++---- 6 files changed, 16 insertions(+), 8 deletions(-) rename bin/{install_python_macos.sh => install-python-macos.sh} (100%) mode change 100644 => 100755 rename bin/{install_python_manylinux1.sh => install-python-manylinux1.sh} (100%) diff --git a/.travis.yml b/.travis.yml index 644fdc0c2..496d5fae0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -106,7 +106,7 @@ jobs: - ARCHFLAGS="-arch x86_64" before_install: - brew install llvm mpich - - VERSIONS=$PYTHON_VERSIONS ./bin/install_python_macos.sh + - VERSIONS=$PYTHON_VERSIONS ./bin/install-python-macos.sh install: - VERSIONS=$PYTHON_VERSIONS ./bin/build-wheels-macos.sh before_script: diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index 02ff24cb4..53d8b90ad 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -40,8 +40,6 @@ do $DELOCATE dist/*.whl mv dist/*.whl $WHEEL_DIR/ - $PYTHON -m pip uninstall -y . - $PYTHON -m pip uninstall -y numpy cython pybind11 scipy mpi4py - deactivate + rm -rf venv done diff --git a/bin/install_python_macos.sh b/bin/install-python-macos.sh old mode 100644 new mode 100755 similarity index 100% rename from bin/install_python_macos.sh rename to bin/install-python-macos.sh diff --git a/bin/install_python_manylinux1.sh b/bin/install-python-manylinux1.sh similarity index 100% rename from bin/install_python_manylinux1.sh rename to bin/install-python-manylinux1.sh diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 75dcd4487..9fdfa74fc 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -18,6 +18,9 @@ do PYTHON=$PYTHON_DIR/python$MAJOR WHEEL_VERSION=$(echo $MAJOR | tr -d '.') + $PYTHON -m venv venv + source venv/bin/activate + # Find the appropriate wheel by grepping for the Python version. MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep mpi4py) @@ -26,6 +29,9 @@ do $PYTHON -m pip install -q $MPI4PY_WHEEL $PYTHON -m pip install -q $BRAINIAK_WHEEL + + deactivate + rm -rf venv done brew install mpich diff --git a/pr-check.sh b/pr-check.sh index 5ce3eb7e7..92e705e51 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -126,14 +126,18 @@ if [ ! -z $WHEEL_DIR ]; then python3 -m pip install $MPI4PY_WHEEL python3 -m pip install $BRAINIAK_WHEEL + + # We don't want to install brainiak from source + tail -n +2 requirements-dev.txt | python3 -m pip install -r /dev/stdin else python3 -m pip install $ignore_installed -U -e . || \ exit_with_error_and_venv "Failed to install BrainIAK." -fi -# install developer dependencies -python3 -m pip install $ignore_installed -U -r requirements-dev.txt || \ - exit_with_error_and_venv "Failed to install development requirements." + # install developer dependencies + python3 -m pip install $ignore_installed -U -r requirements-dev.txt || \ + exit_with_error_and_venv "Failed to install development requirements." + +fi # static analysis ./run-checks.sh || \ From 3761937c9f820eeeb198a272354291e35636e1c3 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 17:35:53 -0500 Subject: [PATCH 059/126] Skip editable install if wheels --- bin/README.md | 2 ++ bin/build-wheels-macos.sh | 2 ++ bin/build-wheels-manylinux1.sh | 2 ++ pr-check.sh | 32 +++++++++++++++++--------------- run-tests.sh | 5 +++-- 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/bin/README.md b/bin/README.md index ac8ad338f..a34c6bf31 100755 --- a/bin/README.md +++ b/bin/README.md @@ -2,6 +2,7 @@ # TODO - Recreate and encrypt keys for brainiak/brainiak +- Split test and build stages - Allow setup.py to take argument for mpi4py wheel location - Clean up buckets appropriately - Add back quiet / silent flags @@ -18,6 +19,7 @@ # To verify - Modify ```setup.py``` to handle conditional MPI install +- Add bdist exception for editable installs - Upload wheels to S3 - Upload wheels to PyPI (uncomment) - Always upload to S3, only upload to PyPI on tag (uncomment) diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index 53d8b90ad..e98d16c07 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -43,3 +43,5 @@ do deactivate rm -rf venv done + +rm -rf mpi4py diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-wheels-manylinux1.sh index 4a815129a..b8a2c7052 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-wheels-manylinux1.sh @@ -38,3 +38,5 @@ for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do auditwheel repair dist/*.whl mv wheelhouse/*.whl $WHEEL_DIR/ done + +rm -rf /mpi4py diff --git a/pr-check.sh b/pr-check.sh index 92e705e51..939a2a898 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -94,11 +94,14 @@ else fi # Check if running in an sdist -if git ls-files --error-unmatch pr-check.sh 2> /dev/null +if [ ! -z WHEEL_DIR ] +then + dist_mode='--bdist-mode' +elif git ls-files --error-unmatch pr-check.sh 2> /dev/null then git clean -Xf . else - sdist_mode="--sdist-mode" + dist_mode="--sdist-mode" fi venv=$(mktemp -u brainiak_pr_venv_XXXXX) || \ @@ -120,23 +123,22 @@ python3 -m pip install -U pip # developer dependencies. if [ ! -z $WHEEL_DIR ]; then - PYTHON_MAJOR=$(python3 -c "import sys; v = sys.version_info; print(v[0] * 10 + v[1])") - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep cp$PYTHON_MAJOR | grep mpi4py) - BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep cp$PYTHON_MAJOR | grep brainiak) + PYTHON_MAJOR=$(python3 -c "import sys; v = sys.version_info; print(v[0] * 10 + v[1])") + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep cp$PYTHON_MAJOR | grep mpi4py) + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep cp$PYTHON_MAJOR | grep brainiak) - python3 -m pip install $MPI4PY_WHEEL - python3 -m pip install $BRAINIAK_WHEEL + python3 -m pip install $MPI4PY_WHEEL + python3 -m pip install $BRAINIAK_WHEEL - # We don't want to install brainiak from source - tail -n +2 requirements-dev.txt | python3 -m pip install -r /dev/stdin + # We don't want to install brainiak from source + tail -n +2 requirements-dev.txt | python3 -m pip install -r /dev/stdin else - python3 -m pip install $ignore_installed -U -e . || \ - exit_with_error_and_venv "Failed to install BrainIAK." - - # install developer dependencies - python3 -m pip install $ignore_installed -U -r requirements-dev.txt || \ - exit_with_error_and_venv "Failed to install development requirements." + python3 -m pip install $ignore_installed -U -e . || \ + exit_with_error_and_venv "Failed to install BrainIAK." + # install developer dependencies + python3 -m pip install $ignore_installed -U -r requirements-dev.txt || \ + exit_with_error_and_venv "Failed to install development requirements." fi # static analysis diff --git a/run-tests.sh b/run-tests.sh index 3c09a616d..b5aefe3a7 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -18,10 +18,11 @@ set -ex # When installing from sdist, the pip freeze output cannot be used to # detect editable mode. Use the "--sdist-mode" flag. -sdist_mode=$1 +dist_mode=$1 python3 -m pip freeze | grep -qi /brainiak \ - || [ ${sdist_mode:-default} = "--sdist-mode" ] \ + || [ $dist_mode = "--sdist-mode" ] \ + || [ $dist_mode = "--bdist-mode" ] \ || { echo "You must install brainiak in editable mode"` `" before calling "$(basename "$0") From 4a1498393cdf94021e5630bfc550efe58c2c3c09 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 18:18:35 -0500 Subject: [PATCH 060/126] Fix bdist --- pr-check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pr-check.sh b/pr-check.sh index 939a2a898..454c1a0e2 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -146,7 +146,7 @@ fi exit_with_error_and_venv "run-checks failed" # run tests -./run-tests.sh $sdist_mode || \ +./run-tests.sh $dist_mode || \ exit_with_error_and_venv "run-tests failed" # build documentation From 705d919b0cfa69d02f59419289e7c00ae14920ba Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 22:16:33 -0500 Subject: [PATCH 061/126] Fix coverage bugs --- bin/README.md | 3 +++ bin/test-wheels-manylinux1.sh | 2 +- pr-check.sh | 2 +- run-tests.sh | 48 ++++++++++++++++++++--------------- 4 files changed, 32 insertions(+), 23 deletions(-) diff --git a/bin/README.md b/bin/README.md index a34c6bf31..30b40454a 100755 --- a/bin/README.md +++ b/bin/README.md @@ -3,6 +3,7 @@ # TODO - Recreate and encrypt keys for brainiak/brainiak - Split test and build stages +- Update documentation - Allow setup.py to take argument for mpi4py wheel location - Clean up buckets appropriately - Add back quiet / silent flags @@ -28,6 +29,8 @@ - Only deploy on tag (uncomment) # Completed +- Enforce BrainIAK dependencies while installing dev requirements +- Resolve coverage running pytest as module - Split wheel build, test install, test run into three travis stages - Separate install python and build wheels in MacOS - Create brainiak/manylinux Docker image diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 6652890ce..83c5c6814 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -36,7 +36,7 @@ echo " StrictHostKeyChecking no" >> ~/.ssh/config # we can install brainiak without installing mpich, but require mpiexec during tests # Install dependencies -yum install -y -q mpich2-devel +yum install -y -q mpich2 for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do mpi_command=mpiexec.hydra \ diff --git a/pr-check.sh b/pr-check.sh index 454c1a0e2..fb67887bb 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -131,7 +131,7 @@ if [ ! -z $WHEEL_DIR ]; then python3 -m pip install $BRAINIAK_WHEEL # We don't want to install brainiak from source - tail -n +2 requirements-dev.txt | python3 -m pip install -r /dev/stdin + { echo $BRAINIAK_WHEEL; tail -n +2 requirements-dev.txt; } | python3 -m pip install -r /dev/stdin else python3 -m pip install $ignore_installed -U -e . || \ exit_with_error_and_venv "Failed to install BrainIAK." diff --git a/run-tests.sh b/run-tests.sh index b5aefe3a7..2cc8ac0b7 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -21,8 +21,8 @@ set -ex dist_mode=$1 python3 -m pip freeze | grep -qi /brainiak \ - || [ $dist_mode = "--sdist-mode" ] \ - || [ $dist_mode = "--bdist-mode" ] \ + || [ ${dist_mode:-default} = "--sdist-mode" ] \ + || [ ${dist_mode:-default} = "--bdist-mode" ] \ || { echo "You must install brainiak in editable mode"` `" before calling "$(basename "$0") @@ -37,30 +37,36 @@ if [ -z $mpi_command ]; then elif [ ! -z $SLURM_NODELIST ]; then mpi_command=srun fi -coverage=$(which coverage) -$mpi_command -n 2 $coverage run -m pytest +if [ ${dist_mode:-default} = "--bdist-mode" ] +then + $mpi_command -n 2 $(which pytest) -$coverage combine +# Only run coverage for editable or source distribution +else + $mpi_command -n 2 $(which coverage) run -m pytest + coverage combine -# Travis error workaround -coverage_report=$(mktemp -u coverage_report_XXXXX) || { - echo "mktemp -u error" >&2; - exit 1; -} + # Travis error workaround + coverage_report=$(mktemp -u coverage_report_XXXXX) || { + echo "mktemp -u error" >&2; + exit 1; + } -set +e -$coverage report > $coverage_report -report_exit_code=$? + set +e + coverage report > $coverage_report + report_exit_code=$? -$coverage html -$coverage xml + coverage html + coverage xml -cat $coverage_report -rm $coverage_report + cat $coverage_report + rm $coverage_report -if [ $report_exit_code = 2 ] -then - echo "ERROR: Coverage too low." + if [ $report_exit_code = 2 ] + then + echo "ERROR: Coverage too low." + fi + exit $report_exit_code fi -exit $report_exit_code + From 8f1a259fec822132da96e955ddaa714687a87264 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Fri, 15 Dec 2017 23:09:54 -0500 Subject: [PATCH 062/126] Turn on silent --- .travis.yml | 2 +- bin/README.md | 8 ++++---- pr-check.sh | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 496d5fae0..e29eeae5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -83,7 +83,7 @@ jobs: acl: public_read region: us-east-1 local_dir: .whl - upload-dir: $TRAVIS_COMMIT/.whl + upload-dir: .whl skip_cleanup: true on: repo: $BRAINIAK_REPO diff --git a/bin/README.md b/bin/README.md index 30b40454a..11f579102 100755 --- a/bin/README.md +++ b/bin/README.md @@ -6,11 +6,7 @@ - Update documentation - Allow setup.py to take argument for mpi4py wheel location - Clean up buckets appropriately -- Add back quiet / silent flags - End-to-end test -- Refactor build and test scripts - - Consolidate python version numbers across various scripts - - Rationalize variables; different scripts call the same things different things - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) - Build for other OSs (e.g., xcode[VERSION], 32-bit Linux, Windows) @@ -30,6 +26,10 @@ # Completed - Enforce BrainIAK dependencies while installing dev requirements +- Add back quiet / silent flags +- Refactor build and test scripts +- Consolidate python version numbers across various scripts +- Rationalize variables; different scripts call the same things different things - Resolve coverage running pytest as module - Split wheel build, test install, test run into three travis stages - Separate install python and build wheels in MacOS diff --git a/pr-check.sh b/pr-check.sh index fb67887bb..bd9afab14 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -127,11 +127,11 @@ if [ ! -z $WHEEL_DIR ]; then MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep cp$PYTHON_MAJOR | grep mpi4py) BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep cp$PYTHON_MAJOR | grep brainiak) - python3 -m pip install $MPI4PY_WHEEL - python3 -m pip install $BRAINIAK_WHEEL + python3 -m pip install -q $MPI4PY_WHEEL + python3 -m pip install -q $BRAINIAK_WHEEL # We don't want to install brainiak from source - { echo $BRAINIAK_WHEEL; tail -n +2 requirements-dev.txt; } | python3 -m pip install -r /dev/stdin + { echo $BRAINIAK_WHEEL; tail -n +2 requirements-dev.txt; } | python3 -m pip install -q -r /dev/stdin else python3 -m pip install $ignore_installed -U -e . || \ exit_with_error_and_venv "Failed to install BrainIAK." From 11f87261ac31caf941cbde5d0a91f73efa0b3d59 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 08:39:01 -0500 Subject: [PATCH 063/126] Fix MacOS bug --- bin/README.md | 4 ++-- bin/build-wheels-macos.sh | 2 +- bin/test-wheels-macos.sh | 4 ++-- pr-check.sh | 20 ++++++++++++-------- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/bin/README.md b/bin/README.md index 11f579102..2fb902b95 100755 --- a/bin/README.md +++ b/bin/README.md @@ -16,8 +16,6 @@ # To verify - Modify ```setup.py``` to handle conditional MPI install -- Add bdist exception for editable installs -- Upload wheels to S3 - Upload wheels to PyPI (uncomment) - Always upload to S3, only upload to PyPI on tag (uncomment) - Deploy only on master (uncomment in ```.travis.yml```) @@ -25,6 +23,8 @@ - Only deploy on tag (uncomment) # Completed +- Upload wheels to S3 +- Add bdist exception for editable installs - Enforce BrainIAK dependencies while installing dev requirements - Add back quiet / silent flags - Refactor build and test scripts diff --git a/bin/build-wheels-macos.sh b/bin/build-wheels-macos.sh index e98d16c07..cdc4f71ed 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-wheels-macos.sh @@ -19,7 +19,7 @@ for VERSION in $VERSIONS do MAJOR=${VERSION%.*} # Replicate readlink -f - PYTHON_DIR=$(cd $(dirname $(readlink $(which python$MAJOR))); pwd) + PYTHON_DIR=$(dirname $(realpath $(which python$MAJOR))) PYTHON=$PYTHON_DIR/python$MAJOR DELOCATE=$(dirname $PYTHON)/delocate-wheel diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 9fdfa74fc..12ce49d62 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -14,7 +14,7 @@ WHEEL_DIR=$SCRIPT_DIR/../.whl for VERSION in $VERSIONS do MAJOR=${VERSION%.*} - PYTHON_DIR=$(cd $(dirname $(readlink $(which python$MAJOR))); pwd) + PYTHON_DIR=$(dirname $(realpath $(which python$MAJOR))) PYTHON=$PYTHON_DIR/python$MAJOR WHEEL_VERSION=$(echo $MAJOR | tr -d '.') @@ -41,7 +41,7 @@ for VERSION in $VERSIONS do # TODO: refactor this out MAJOR=${VERSION%.*} - PYTHON_DIR=$(cd $(dirname $(readlink $(which python$MAJOR))); pwd) + PYTHON_DIR=$(dirname $(realpath $(which python$MAJOR))) PYTHON=$PYTHON_DIR/python$MAJOR WHEEL_DIR=$WHEEL_DIR PYTHON=$PYTHON $SCRIPT_DIR/../pr-check.sh done diff --git a/pr-check.sh b/pr-check.sh index bd9afab14..07ae6bea8 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -114,7 +114,7 @@ $activate_venv $venv || { exit_with_error "Virtual environment activation failed." } -# Upgrad pip here +# Upgrade pip here (for Linux) python3 -m pip install -U pip # install brainiak in editable mode (required for testing) @@ -123,15 +123,15 @@ python3 -m pip install -U pip # developer dependencies. if [ ! -z $WHEEL_DIR ]; then - PYTHON_MAJOR=$(python3 -c "import sys; v = sys.version_info; print(v[0] * 10 + v[1])") - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep cp$PYTHON_MAJOR | grep mpi4py) - BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep cp$PYTHON_MAJOR | grep brainiak) + MAJOR=$(python3 -c "import sys; v = sys.version_info; print(v[0] * 10 + v[1])") + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep cp$MAJOR | grep mpi4py) + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep cp$MAJOR | grep brainiak) python3 -m pip install -q $MPI4PY_WHEEL python3 -m pip install -q $BRAINIAK_WHEEL # We don't want to install brainiak from source - { echo $BRAINIAK_WHEEL; tail -n +2 requirements-dev.txt; } | python3 -m pip install -q -r /dev/stdin + { echo $BRAINIAK_WHEEL; tail -n +2 requirements-dev.txt; } | python3 -m pip install -r /dev/stdin else python3 -m pip install $ignore_installed -U -e . || \ exit_with_error_and_venv "Failed to install BrainIAK." @@ -141,9 +141,13 @@ else exit_with_error_and_venv "Failed to install development requirements." fi -# static analysis -./run-checks.sh || \ - exit_with_error_and_venv "run-checks failed" +# Only run source checks if we aren't in binary mode +if [ ${dist_mode:-default} != '--bdist-mode' ] +then + # static analysis + ./run-checks.sh || \ + exit_with_error_and_venv "run-checks failed" +fi # run tests ./run-tests.sh $dist_mode || \ From 5c58804eac0d41cc95e53dd026a4b37fc24a2702 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 08:48:55 -0500 Subject: [PATCH 064/126] Show commands in install python --- bin/install-python-macos.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/install-python-macos.sh b/bin/install-python-macos.sh index ee5275a89..e1a826fd9 100755 --- a/bin/install-python-macos.sh +++ b/bin/install-python-macos.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -ex + MACPYTHON_URL=https://www.python.org/ftp/python DOWNLOAD_DIR=python_downloads mkdir -p $DOWNLOAD_DIR From 3421560e21a97a76e63b82511d90828bca6e489d Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 09:01:46 -0500 Subject: [PATCH 065/126] Update MacOS python versions --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e29eeae5d..24b57936b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,8 @@ branches: env: global: - BRAINIAK_REPO=danielsuo/brainiak - - PYTHON_VERSIONS="3.4.7 3.5.4 3.6.2" + # Latest versions with MacOS binaries + - PYTHON_VERSIONS="3.4.4 3.5.4 3.6.4" jobs: include: From e3afa9cf649a54ab6dc02e0d2ff6b0f525f2b350 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 09:04:47 -0500 Subject: [PATCH 066/126] Turn off sudo required for Linux build wheels --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 24b57936b..9d5e01bfb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,6 @@ jobs: # Build Linux wheels os: linux dist: trusty - sudo: required language: generic env: DEPLOY_WHEEL=1 install: From c3739ebb1b8c0defef0d4abb857cadc2e2bc38f8 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 09:06:39 -0500 Subject: [PATCH 067/126] Turn on sudo required for Linux build wheels --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9d5e01bfb..24b57936b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,6 +64,7 @@ jobs: # Build Linux wheels os: linux dist: trusty + sudo: required language: generic env: DEPLOY_WHEEL=1 install: From bb05f85e3afa8c96e2718fe83cb1015abf02c4fb Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 09:07:54 -0500 Subject: [PATCH 068/126] Fix MacOS pkg versions --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 24b57936b..62aaa9b32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ env: global: - BRAINIAK_REPO=danielsuo/brainiak # Latest versions with MacOS binaries - - PYTHON_VERSIONS="3.4.4 3.5.4 3.6.4" + - PYTHON_VERSIONS="3.4.4 3.5.4 3.6.3" jobs: include: From a43f51b967eae9678a7ada277ab2d728827bb428 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 09:33:29 -0500 Subject: [PATCH 069/126] Unset MacOS environment variables before testing --- bin/test-wheels-macos.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 12ce49d62..3527218c3 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -6,6 +6,12 @@ set -e # Show explicitly which commands are currently running. set -x +unset CC +unset CXX +unset LDFLAGS +unset CPPFLAGS +unset ARCHFLAGS + SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) WHEEL_DIR=$SCRIPT_DIR/../.whl From 54e55e3833f0f5a5e9a658b5909b2bc55f463604 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 10:28:21 -0500 Subject: [PATCH 070/126] Fix up linux install --- bin/install-test-deps-manylinux1.sh | 14 ++++++++++++++ bin/test-wheels-manylinux1.sh | 16 +--------------- pr-check.sh | 3 --- run-tests.sh | 2 +- 4 files changed, 16 insertions(+), 19 deletions(-) create mode 100755 bin/install-test-deps-manylinux1.sh diff --git a/bin/install-test-deps-manylinux1.sh b/bin/install-test-deps-manylinux1.sh new file mode 100755 index 000000000..37a020238 --- /dev/null +++ b/bin/install-test-deps-manylinux1.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -ex + +# Install and configure ssh so we can ssh locally for MPI tests +yum install -y -q openssh-server mpich2 +service sshd start + +ssh-keygen -f ~/.ssh/id_rsa -t rsa -N '' -b 4096 +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys + +echo "Host *" >> ~/.ssh/config +echo " StrictHostKeyChecking no" >> ~/.ssh/config diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 83c5c6814..34654e35f 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -21,23 +21,9 @@ for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do $PIP install -q $BRAINIAK_WHEEL done -# Install and configure ssh so we can ssh locally for MPI tests -yum install -y -q openssh-server -service sshd start - -ssh-keygen -f ~/.ssh/id_rsa -t rsa -N '' -b 4096 -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys - -echo "Host *" >> ~/.ssh/config -echo " StrictHostKeyChecking no" >> ~/.ssh/config - # Separate tests into a separate loop because we want to make sure # we can install brainiak without installing mpich, but require mpiexec during tests - -# Install dependencies -yum install -y -q mpich2 - +./bin/install-test-deps-manylinux1.sh for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do mpi_command=mpiexec.hydra \ WHEEL_DIR=$WHEEL_DIR \ diff --git a/pr-check.sh b/pr-check.sh index 07ae6bea8..cebe71cb7 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -114,9 +114,6 @@ $activate_venv $venv || { exit_with_error "Virtual environment activation failed." } -# Upgrade pip here (for Linux) -python3 -m pip install -U pip - # install brainiak in editable mode (required for testing) # brainiak will also be installed together with the developer dependencies, but # we install it first here to check that installation succeeds without the diff --git a/run-tests.sh b/run-tests.sh index 2cc8ac0b7..ad6135654 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -40,7 +40,7 @@ fi if [ ${dist_mode:-default} = "--bdist-mode" ] then - $mpi_command -n 2 $(which pytest) + $mpi_command -n 2 python3 -m pytest # Only run coverage for editable or source distribution else From 2829555defd931d21903641df710be6bee654126 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 11:01:27 -0500 Subject: [PATCH 071/126] Fix linux --- pr-check.sh | 2 +- run-tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pr-check.sh b/pr-check.sh index cebe71cb7..b7a4eb722 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -128,7 +128,7 @@ if [ ! -z $WHEEL_DIR ]; then python3 -m pip install -q $BRAINIAK_WHEEL # We don't want to install brainiak from source - { echo $BRAINIAK_WHEEL; tail -n +2 requirements-dev.txt; } | python3 -m pip install -r /dev/stdin + { echo $BRAINIAK_WHEEL; tail -n +2 requirements-dev.txt; } | python3 -m pip install -q -r /dev/stdin else python3 -m pip install $ignore_installed -U -e . || \ exit_with_error_and_venv "Failed to install BrainIAK." diff --git a/run-tests.sh b/run-tests.sh index ad6135654..07375710d 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -40,7 +40,7 @@ fi if [ ${dist_mode:-default} = "--bdist-mode" ] then - $mpi_command -n 2 python3 -m pytest + $mpi_command -n 2 $(which python3) -m pytest # Only run coverage for editable or source distribution else From 4fdf1a6c2ffc2e6d899dabaf65684650911cba9d Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 13:09:51 -0500 Subject: [PATCH 072/126] MacOS and Linux fixes --- bin/install-python-macos.sh | 6 ++++++ run-tests.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/install-python-macos.sh b/bin/install-python-macos.sh index e1a826fd9..7540bd55f 100755 --- a/bin/install-python-macos.sh +++ b/bin/install-python-macos.sh @@ -17,4 +17,10 @@ do sudo installer -pkg $INST_PATH -target / pip$MAJOR install -q -U pip delocate + + # NOTE: https://stackoverflow.com/questions/41691327/ssl-sslerror-ssl-certificate-verify-failed-certificate-verify-failed-ssl-c + if [[ $MAJOR > "3.5" ]] + then + sudo /Applications/Python\ $MAJOR/Install\ Certificates.command + fi done diff --git a/run-tests.sh b/run-tests.sh index 07375710d..93a7d572e 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -40,7 +40,7 @@ fi if [ ${dist_mode:-default} = "--bdist-mode" ] then - $mpi_command -n 2 $(which python3) -m pytest + $mpi_command -n 2 pytest # Only run coverage for editable or source distribution else From 95d75c11ca9dac197b44469e0eca34988d40249b Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 13:29:46 -0500 Subject: [PATCH 073/126] Fix pytest path --- run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-tests.sh b/run-tests.sh index 93a7d572e..fd5d78139 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -40,7 +40,7 @@ fi if [ ${dist_mode:-default} = "--bdist-mode" ] then - $mpi_command -n 2 pytest + $mpi_command -n 2 $(which pytest) # Only run coverage for editable or source distribution else From 35e703c0ec19bd1b270df42efd98d387b3904ea6 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 13:56:39 -0500 Subject: [PATCH 074/126] MacoS 3.4 only --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 62aaa9b32..7fc00d181 100644 --- a/.travis.yml +++ b/.travis.yml @@ -107,9 +107,11 @@ jobs: - ARCHFLAGS="-arch x86_64" before_install: - brew install llvm mpich - - VERSIONS=$PYTHON_VERSIONS ./bin/install-python-macos.sh + # - VERSIONS=$PYTHON_VERSIONS ./bin/install-python-macos.sh + - VERSIONS="3.4.4" ./bin/install-python-macos.sh install: - - VERSIONS=$PYTHON_VERSIONS ./bin/build-wheels-macos.sh + # - VERSIONS=$PYTHON_VERSIONS ./bin/build-wheels-macos.sh + - VERSIONS="3.4.4" ./bin/build-wheels-macos.sh before_script: - brew uninstall -f --ignore-dependencies llvm mpich script: From 4c06ec78d85ffbb52875aeb0f8d5911dcdedf8d7 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 15:59:04 -0500 Subject: [PATCH 075/126] Mac 3.4.4 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7fc00d181..f2762666f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -115,7 +115,7 @@ jobs: before_script: - brew uninstall -f --ignore-dependencies llvm mpich script: - - VERSIONS=$PYTHON_VERSIONS ./bin/test-wheels-macos.sh + - VERSIONS="3.4.4" ./bin/test-wheels-macos.sh deploy: *wheel_deploy # Make sure we can install our wheels from scratch if we're on mater (not just targeting master) From 85abd8bc139debccbc97c5feba3aaeebf2d0559f Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 16:07:06 -0500 Subject: [PATCH 076/126] Update test wheels --- bin/test-wheels-macos.sh | 3 +++ bin/test-wheels-manylinux1.sh | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 3527218c3..40781f030 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -24,6 +24,8 @@ do PYTHON=$PYTHON_DIR/python$MAJOR WHEEL_VERSION=$(echo $MAJOR | tr -d '.') + git clean -f -f -x -d -q -e .whl + $PYTHON -m venv venv source venv/bin/activate @@ -49,5 +51,6 @@ do MAJOR=${VERSION%.*} PYTHON_DIR=$(dirname $(realpath $(which python$MAJOR))) PYTHON=$PYTHON_DIR/python$MAJOR + git clean -f -f -x -d -q -e .whl WHEEL_DIR=$WHEEL_DIR PYTHON=$PYTHON $SCRIPT_DIR/../pr-check.sh done diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 34654e35f..eb195e379 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -15,16 +15,25 @@ for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep mpi4py) BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep brainiak) - PIP=/opt/python/${PYTHON}/bin/pip + PYTHON=/opt/python/${PYTHON}/bin/python - $PIP install -q $MPI4PY_WHEEL - $PIP install -q $BRAINIAK_WHEEL + git clean -f -f -x -d -q -e .whl + + $PYTHON -m venv venv + source venv/bin/activate + + $PYTHON -m pip install -q $MPI4PY_WHEEL + $PYTHON -m pip install -q $BRAINIAK_WHEEL + + deactivate + rm -rf venv done # Separate tests into a separate loop because we want to make sure # we can install brainiak without installing mpich, but require mpiexec during tests ./bin/install-test-deps-manylinux1.sh for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do + git clean -f -f -x -d -q -e .whl mpi_command=mpiexec.hydra \ WHEEL_DIR=$WHEEL_DIR \ PYTHON=/opt/python/$VERSION/bin/python \ From d425cf0d6660abace98930dde8382429da0fd103 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 16:08:51 -0500 Subject: [PATCH 077/126] asdf --- bin/test-wheels-manylinux1.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index eb195e379..ac3f82b5f 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -11,11 +11,11 @@ WHEEL_DIR=$SCRIPT_DIR/../.whl # Test whether we can install without any dependencies # TODO: delete once we have setup.py setup correctly -for PYTHON in cp34-cp34m cp35-cp35m cp36-cp36m; do - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep mpi4py) - BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $PYTHON | grep brainiak) +for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep mpi4py) + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep brainiak) - PYTHON=/opt/python/${PYTHON}/bin/python + PYTHON=/opt/python/${VERSION}/bin/python3 git clean -f -f -x -d -q -e .whl From be6103245b56ec3d1d3ebba43b03e9945bf28b09 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 17:16:51 -0500 Subject: [PATCH 078/126] Only build docs if not in binary mode --- pr-check.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/pr-check.sh b/pr-check.sh index b7a4eb722..5d7abc7e9 100755 --- a/pr-check.sh +++ b/pr-check.sh @@ -150,19 +150,23 @@ fi ./run-tests.sh $dist_mode || \ exit_with_error_and_venv "run-tests failed" -# build documentation -cd docs -export THEANO_FLAGS='device=cpu,floatX=float64,blas.ldflags=-lblas' - -if [ ! -z $SLURM_NODELIST ] +# Only build documentation if we aren't in binary mode +if [ ${dist_mode:-default} != '--bdist-mode' ] then - make_wrapper="srun -n 1" + # build documentation + cd docs + export THEANO_FLAGS='device=cpu,floatX=float64,blas.ldflags=-lblas' + + if [ ! -z $SLURM_NODELIST ] + then + make_wrapper="srun -n 1" + fi + $make_wrapper make || { + cd - + exit_with_error_and_venv "make docs failed" + } + cd - fi -$make_wrapper make || { - cd - - exit_with_error_and_venv "make docs failed" -} -cd - $deactivate_venv $remove_venv $venv From 9eb2ad74055ae1e031147277651a223ac6eb64ef Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 18:27:00 -0500 Subject: [PATCH 079/126] Test yaml generator --- .travis.yml | 338 +++++++++++++++++++++++---------------- bin/generate-yaml.py | 152 ++++++++++++++++++ bin/test-wheels-macos.sh | 6 - 3 files changed, 352 insertions(+), 144 deletions(-) create mode 100755 bin/generate-yaml.py diff --git a/.travis.yml b/.travis.yml index f2762666f..2a152e568 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,144 +1,206 @@ -# Testing flow -# - Install and test sdist -# - Build wheel, test locally, and deploy -# - Install wheel -# - Test wheel - branches: only: - - master - + - master env: global: - - BRAINIAK_REPO=danielsuo/brainiak - # Latest versions with MacOS binaries - - PYTHON_VERSIONS="3.4.4 3.5.4 3.6.3" - + - BRAINIAK_REPO=danielsuo/brainiak jobs: include: - - stage: test - - # - os: linux - # dist: trusty - # sudo: required - # language: python - # python: 3.4 - # addons: &linux_addons - # apt: &apt - # packages: - # - build-essential libgomp1 libmpich-dev mpich - - # - os: linux - # dist: trusty - # sudo: required - # language: python - # python: 3.5 - # addons: *linux_addons - - # - os: linux - # dist: trusty - # sudo: required - # language: python - # python: 3.6 - # addons: *linux_addons - - # - os: osx - # osx_image: xcode7.3 - # language: generic - # env: &macos_env - # - CC=/usr/local/opt/llvm/bin/clang - # - CXX=/usr/local/opt/llvm/bin/clang++ - # - LDFLAGS="-L/usr/local/opt/llvm/lib - # -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - # - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - # - HOMEBREW_NO_AUTO_UPDATE=1 - # before_install: &macos_before_install - # - brew install llvm mpich python3 - - # - os: osx - # osx_image: xcode8 - # language: generic - # env: *macos_env - # before_install: *macos_before_install - - # Build Linux wheels - os: linux - dist: trusty - sudo: required - language: generic - env: DEPLOY_WHEEL=1 - install: - - ./bin/build-wheels.sh - script: - - ./bin/test-wheels.sh - - # Deploy wheels to S3 only after build wheel jobs - # - Deploy all commits to master with travis-cli S3 user to brainiak - # - This deploy MUST come from brainiak/brainiak - deploy: &wheel_deploy - - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - acl: public_read - region: us-east-1 - local_dir: .whl - upload-dir: .whl - skip_cleanup: true - on: - repo: $BRAINIAK_REPO - branch: master - condition: $DEPLOY_WHEEL = 1 - - # Build MacOS wheels - - os: osx - osx_image: xcode7.3 - sudo: required - language: generic - env: - - DEPLOY_WHEEL=1 - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib - -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - - ARCHFLAGS="-arch x86_64" - before_install: - - brew install llvm mpich - # - VERSIONS=$PYTHON_VERSIONS ./bin/install-python-macos.sh - - VERSIONS="3.4.4" ./bin/install-python-macos.sh - install: - # - VERSIONS=$PYTHON_VERSIONS ./bin/build-wheels-macos.sh - - VERSIONS="3.4.4" ./bin/build-wheels-macos.sh - before_script: - - brew uninstall -f --ignore-dependencies llvm mpich - script: - - VERSIONS="3.4.4" ./bin/test-wheels-macos.sh - deploy: *wheel_deploy - - # Make sure we can install our wheels from scratch if we're on mater (not just targeting master) - # - stage: integration - # if: repo = $BRAINIAK_REPO and branch = master - - # Only publish if we're on tagged master commit (not just targeting master) and integration passed. - # - stage: publish - # if: repo = $BRAINIAK_REPO and branch = master - # script: skip - # deploy: - # - provider: pypi - # distributions: "sdist bdist_wheel" - # skip_cleanup: true - # on: - # tags: true - - # - stage: pypi-test - -# TODO: Move these into each job; steps outside a job are by default grouped into a stage -# install: - # - python3 -m pip install -U pip - -# script: - # - ./pr-check.sh - + - stage: test + - addons: + apt: + packages: + - build-essential libgomp1 libmpich-dev mpich + dist: trusty + language: python + os: linux + python: '3.4' + sudo: required + - addons: + apt: + packages: + - build-essential libgomp1 libmpich-dev mpich + dist: trusty + language: python + os: linux + python: '3.5' + sudo: required + - addons: + apt: + packages: + - build-essential libgomp1 libmpich-dev mpich + dist: trusty + language: python + os: linux + python: '3.6' + sudo: required + - before_install: + - brew install llvm mpich python3 + env: + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + language: generic + os: osx + osx_image: xcode7.3 + - before_install: + - brew install llvm mpich python3 + env: + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + language: generic + os: osx + osx_image: xcode8 + - stage: build + - deploy: + - access_key_id: AKIAJKBW6H2VKKQDHILQ + acl: public_read + bucket: brainiak + local_dir: .whl + 'on': + branch: master + condition: $DEPLOY_WHEEL = 1 + repo: danielsuo/brainiak + provider: s3 + region: us-east-1 + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + skip_cleanup: true + upload-dir: .whl + dist: trusty + env: DEPLOY_WHEEL=1 + install: + - ./bin/build-wheels.sh + language: generic + os: linux + script: + - ./bin/test-wheels.sh + sudo: required + - before_install: + - brew install llvm mpich + - VERSIONS="3.4.4" ./bin/install-python-macos.sh + before_script: + - brew uninstall -f --ignore-dependencies llvm mpich + - unset CC + - unset CXX + - unset LDFLAGS + - unset CPPFLAGS + - unset ARCHFLAGS + deploy: + - access_key_id: AKIAJKBW6H2VKKQDHILQ + acl: public_read + bucket: brainiak + local_dir: .whl + 'on': + branch: master + condition: $DEPLOY_WHEEL = 1 + repo: danielsuo/brainiak + provider: s3 + region: us-east-1 + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + skip_cleanup: true + upload-dir: .whl + env: + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + - DEPLOY_WHEEL=1 + - ARCHFLAGS="-arch x86_64" + install: + - VERSIONS="3.4.4" ./bin/build-wheels-macos.sh + language: generic + os: osx + osx_image: xcode7.3 + script: + - VERSIONS="%s" ./bin/test-wheels-macos.sh + sudo: required + - before_install: + - brew install llvm mpich + - VERSIONS="3.5.3" ./bin/install-python-macos.sh + before_script: + - brew uninstall -f --ignore-dependencies llvm mpich + - unset CC + - unset CXX + - unset LDFLAGS + - unset CPPFLAGS + - unset ARCHFLAGS + deploy: + - access_key_id: AKIAJKBW6H2VKKQDHILQ + acl: public_read + bucket: brainiak + local_dir: .whl + 'on': + branch: master + condition: $DEPLOY_WHEEL = 1 + repo: danielsuo/brainiak + provider: s3 + region: us-east-1 + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + skip_cleanup: true + upload-dir: .whl + env: + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + - DEPLOY_WHEEL=1 + - ARCHFLAGS="-arch x86_64" + install: + - VERSIONS="3.5.3" ./bin/build-wheels-macos.sh + language: generic + os: osx + osx_image: xcode7.3 + script: + - VERSIONS="%s" ./bin/test-wheels-macos.sh + sudo: required + - before_install: + - brew install llvm mpich + - VERSIONS="3.6.0" ./bin/install-python-macos.sh + before_script: + - brew uninstall -f --ignore-dependencies llvm mpich + - unset CC + - unset CXX + - unset LDFLAGS + - unset CPPFLAGS + - unset ARCHFLAGS + deploy: + - access_key_id: AKIAJKBW6H2VKKQDHILQ + acl: public_read + bucket: brainiak + local_dir: .whl + 'on': + branch: master + condition: $DEPLOY_WHEEL = 1 + repo: danielsuo/brainiak + provider: s3 + region: us-east-1 + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + skip_cleanup: true + upload-dir: .whl + env: + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + - DEPLOY_WHEEL=1 + - ARCHFLAGS="-arch x86_64" + install: + - VERSIONS="3.6.0" ./bin/build-wheels-macos.sh + language: generic + os: osx + osx_image: xcode7.3 + script: + - VERSIONS="%s" ./bin/test-wheels-macos.sh + sudo: required diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py new file mode 100755 index 000000000..2f7cd5b56 --- /dev/null +++ b/bin/generate-yaml.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python + +import yaml +import copy + +travis = '.travis.yml' +repo = 'danielsuo/brainiak' +versions = ['3.4.4', '3.5.3', '3.6.0'] +majors = [version[:-2] for version in versions] + +# S3 configuration +bucket = 'brainiak' +region = 'us-east-1' +access_key_id = 'AKIAJKBW6H2VKKQDHILQ' +secret_access_key = "A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg=" + +data = {} + +data['branches'] = { + 'only': ['master'] +} + +data['env'] = { + 'global': [ + 'BRAINIAK_REPO=%s' % repo + ] +} + +data['jobs'] = { + 'include': [] +} + +jobs = data['jobs']['include'] + +# Create test stage +jobs.append({'stage': 'test'}) + +# Linux +test_linux = { + 'os': 'linux', + 'dist': 'trusty', + 'sudo': 'required', + 'language': 'python', + 'addons': { + 'apt': { + 'packages': ['build-essential libgomp1 libmpich-dev mpich'] + } + } +} + +for major in majors: + block = copy.deepcopy(test_linux) + block['python'] = major + jobs.append(block) + +# MacOS +test_macos = { + 'os': 'osx', + 'language': 'generic', + 'env': [ + 'CC=/usr/local/opt/llvm/bin/clang', + 'CXX=/usr/local/opt/llvm/bin/clang++', + 'LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS"', + 'CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS"', + 'HOMEBREW_NO_AUTO_UPDATE=1' + ], + 'before_install': ['brew install llvm mpich python3'] +} + +for osx in ['xcode7.3', 'xcode8']: + block = copy.deepcopy(test_macos) + block['osx_image'] = osx + jobs.append(block) + +# Create build stage +jobs.append({'stage': 'build'}) + +deploy_s3 = [{ + 'provider': 's3', + 'access_key_id': access_key_id, + 'secret_access_key': { + 'secure': secret_access_key + }, + 'bucket': bucket, + 'region': region, + 'acl': 'public_read', + 'local_dir': '.whl', + 'upload-dir': '.whl', + 'skip_cleanup': True, + 'on': { + 'repo': repo, + 'branch': 'master', + 'condition': '$DEPLOY_WHEEL = 1' + } +}] + +jobs.append({ + 'os': 'linux', + 'dist': 'trusty', + 'sudo': 'required', + 'language': 'generic', + 'env': 'DEPLOY_WHEEL=1', + 'install': ['./bin/build-wheels.sh'], + 'script': ['./bin/test-wheels.sh'], + 'deploy': deploy_s3 +}) + + +build_macos_env = copy.deepcopy(test_macos['env']) +build_macos_env.extend([ + 'DEPLOY_WHEEL=1', + 'ARCHFLAGS="-arch x86_64"' +]) +build_macos = { + 'os': 'osx', + 'osx_image': 'xcode7.3', + 'sudo': 'required', + 'language': 'generic', + 'env': build_macos_env, + 'deploy': deploy_s3 +} + +for version in versions: + block = copy.deepcopy(build_macos) + block['before_install'] = [ + 'brew install llvm mpich', + 'VERSIONS="%s" ./bin/install-python-macos.sh' % version + ] + + block['install'] = [ + 'VERSIONS="%s" ./bin/build-wheels-macos.sh' % version + ] + + block['before_script'] = [ + 'brew uninstall -f --ignore-dependencies llvm mpich', + 'unset CC', + 'unset CXX', + 'unset LDFLAGS', + 'unset CPPFLAGS', + 'unset ARCHFLAGS' + ] + + block['script'] = [ + 'VERSIONS="%s" ./bin/test-wheels-macos.sh' + ] + + block['deploy'] = copy.deepcopy(deploy_s3) + + jobs.append(block) + +with open(travis, 'w') as yml: + yaml.dump(data, yml, default_flow_style=False) diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 40781f030..181e56b94 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -6,12 +6,6 @@ set -e # Show explicitly which commands are currently running. set -x -unset CC -unset CXX -unset LDFLAGS -unset CPPFLAGS -unset ARCHFLAGS - SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) WHEEL_DIR=$SCRIPT_DIR/../.whl From 14916fb9b32ef43eeae12185eb04a063eef71da2 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 18:48:47 -0500 Subject: [PATCH 080/126] Update travis --- .travis.yml | 23 ++++++++++++++++++++++- bin/generate-yaml.py | 11 +++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2a152e568..335cf1237 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,33 +6,46 @@ env: - BRAINIAK_REPO=danielsuo/brainiak jobs: include: - - stage: test + - language: generic + stage: test - addons: apt: packages: - build-essential libgomp1 libmpich-dev mpich dist: trusty + install: + - python3 -m pip install -U pip language: python os: linux python: '3.4' + script: + - ./pr-check.sh sudo: required - addons: apt: packages: - build-essential libgomp1 libmpich-dev mpich dist: trusty + install: + - python3 -m pip install -U pip language: python os: linux python: '3.5' + script: + - ./pr-check.sh sudo: required - addons: apt: packages: - build-essential libgomp1 libmpich-dev mpich dist: trusty + install: + - python3 -m pip install -U pip language: python os: linux python: '3.6' + script: + - ./pr-check.sh sudo: required - before_install: - brew install llvm mpich python3 @@ -42,9 +55,13 @@ jobs: - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 + install: + - python3 -m pip install -U pip language: generic os: osx osx_image: xcode7.3 + script: + - ./pr-check.sh - before_install: - brew install llvm mpich python3 env: @@ -53,9 +70,13 @@ jobs: - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 + install: + - python3 -m pip install -U pip language: generic os: osx osx_image: xcode8 + script: + - ./pr-check.sh - stage: build - deploy: - access_key_id: AKIAJKBW6H2VKKQDHILQ diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 2f7cd5b56..d08845260 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -1,5 +1,8 @@ #!/usr/bin/env python +# TODO: the first job in a stage must be part of the job dictinoary, not an +# element in the job array. This results in a dummy job being created + import yaml import copy @@ -33,7 +36,7 @@ jobs = data['jobs']['include'] # Create test stage -jobs.append({'stage': 'test'}) +jobs.append({'stage': 'test', 'language': 'generic'}) # Linux test_linux = { @@ -41,6 +44,8 @@ 'dist': 'trusty', 'sudo': 'required', 'language': 'python', + 'install': ['python3 -m pip install -U pip'], + 'script': ['./pr-check.sh'], 'addons': { 'apt': { 'packages': ['build-essential libgomp1 libmpich-dev mpich'] @@ -64,7 +69,9 @@ 'CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS"', 'HOMEBREW_NO_AUTO_UPDATE=1' ], - 'before_install': ['brew install llvm mpich python3'] + 'before_install': ['brew install llvm mpich python3'], + 'install': ['python3 -m pip install -U pip'], + 'script': ['./pr-check.sh'] } for osx in ['xcode7.3', 'xcode8']: From bc5f86676f8bc4aef26ebbc85183450f3ad39e60 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 18:49:31 -0500 Subject: [PATCH 081/126] Update travis --- bin/generate-yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index d08845260..546d59a30 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -80,7 +80,7 @@ jobs.append(block) # Create build stage -jobs.append({'stage': 'build'}) +jobs.append({'stage': 'build', 'language': 'generic'}) deploy_s3 = [{ 'provider': 's3', From 5637abe93f19486c0019066ccd8e140f3963661e Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 19:03:11 -0500 Subject: [PATCH 082/126] Output ordered .travis.yml [ci skip] --- .travis.yml | 231 ++++++++++++++++++++++--------------------- bin/generate-yaml.py | 67 +++++++++---- 2 files changed, 163 insertions(+), 135 deletions(-) diff --git a/.travis.yml b/.travis.yml index 335cf1237..b9454431e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,93 +6,94 @@ env: - BRAINIAK_REPO=danielsuo/brainiak jobs: include: - - language: generic - stage: test - - addons: - apt: - packages: - - build-essential libgomp1 libmpich-dev mpich + - stage: test + language: generic + - os: linux dist: trusty - install: - - python3 -m pip install -U pip + sudo: required language: python - os: linux python: '3.4' + install: + - python3 -m pip install -U pip script: - ./pr-check.sh - sudo: required - - addons: + addons: apt: packages: - build-essential libgomp1 libmpich-dev mpich + - os: linux dist: trusty - install: - - python3 -m pip install -U pip + sudo: required language: python - os: linux python: '3.5' + install: + - python3 -m pip install -U pip script: - ./pr-check.sh - sudo: required - - addons: + addons: apt: packages: - build-essential libgomp1 libmpich-dev mpich + - os: linux dist: trusty - install: - - python3 -m pip install -U pip + sudo: required language: python - os: linux python: '3.6' + install: + - python3 -m pip install -U pip script: - ./pr-check.sh - sudo: required - - before_install: - - brew install llvm mpich python3 + addons: + apt: + packages: + - build-essential libgomp1 libmpich-dev mpich + - os: osx + language: generic env: - CC=/usr/local/opt/llvm/bin/clang - CXX=/usr/local/opt/llvm/bin/clang++ - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 + before_install: + - brew install llvm mpich python3 install: - python3 -m pip install -U pip - language: generic - os: osx - osx_image: xcode7.3 script: - ./pr-check.sh - - before_install: - - brew install llvm mpich python3 + osx_image: xcode7.3 + - os: osx + language: generic env: - CC=/usr/local/opt/llvm/bin/clang - CXX=/usr/local/opt/llvm/bin/clang++ - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 + before_install: + - brew install llvm mpich python3 install: - python3 -m pip install -U pip - language: generic - os: osx - osx_image: xcode8 script: - ./pr-check.sh + osx_image: xcode8 - stage: build + language: generic - deploy: - - access_key_id: AKIAJKBW6H2VKKQDHILQ - acl: public_read + - provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= bucket: brainiak + region: us-east-1 + acl: public_read local_dir: .whl + upload-dir: .whl + skip_cleanup: true 'on': branch: master condition: $DEPLOY_WHEEL = 1 repo: danielsuo/brainiak - provider: s3 - region: us-east-1 - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - skip_cleanup: true - upload-dir: .whl dist: trusty env: DEPLOY_WHEEL=1 install: @@ -102,31 +103,10 @@ jobs: script: - ./bin/test-wheels.sh sudo: required - - before_install: - - brew install llvm mpich - - VERSIONS="3.4.4" ./bin/install-python-macos.sh - before_script: - - brew uninstall -f --ignore-dependencies llvm mpich - - unset CC - - unset CXX - - unset LDFLAGS - - unset CPPFLAGS - - unset ARCHFLAGS - deploy: - - access_key_id: AKIAJKBW6H2VKKQDHILQ - acl: public_read - bucket: brainiak - local_dir: .whl - 'on': - branch: master - condition: $DEPLOY_WHEEL = 1 - repo: danielsuo/brainiak - provider: s3 - region: us-east-1 - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - skip_cleanup: true - upload-dir: .whl + - os: osx + osx_image: xcode7.3 + sudo: required + language: generic env: - CC=/usr/local/opt/llvm/bin/clang - CXX=/usr/local/opt/llvm/bin/clang++ @@ -135,17 +115,26 @@ jobs: - HOMEBREW_NO_AUTO_UPDATE=1 - DEPLOY_WHEEL=1 - ARCHFLAGS="-arch x86_64" + deploy: + - provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + bucket: brainiak + region: us-east-1 + acl: public_read + local_dir: .whl + upload-dir: .whl + skip_cleanup: true + 'on': + branch: master + condition: $DEPLOY_WHEEL = 1 + repo: danielsuo/brainiak + before_install: + - brew install llvm mpich + - VERSIONS="3.4.4" ./bin/install-python-macos.sh install: - VERSIONS="3.4.4" ./bin/build-wheels-macos.sh - language: generic - os: osx - osx_image: xcode7.3 - script: - - VERSIONS="%s" ./bin/test-wheels-macos.sh - sudo: required - - before_install: - - brew install llvm mpich - - VERSIONS="3.5.3" ./bin/install-python-macos.sh before_script: - brew uninstall -f --ignore-dependencies llvm mpich - unset CC @@ -153,21 +142,12 @@ jobs: - unset LDFLAGS - unset CPPFLAGS - unset ARCHFLAGS - deploy: - - access_key_id: AKIAJKBW6H2VKKQDHILQ - acl: public_read - bucket: brainiak - local_dir: .whl - 'on': - branch: master - condition: $DEPLOY_WHEEL = 1 - repo: danielsuo/brainiak - provider: s3 - region: us-east-1 - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - skip_cleanup: true - upload-dir: .whl + script: + - VERSIONS="%s" ./bin/test-wheels-macos.sh + - os: osx + osx_image: xcode7.3 + sudo: required + language: generic env: - CC=/usr/local/opt/llvm/bin/clang - CXX=/usr/local/opt/llvm/bin/clang++ @@ -176,17 +156,26 @@ jobs: - HOMEBREW_NO_AUTO_UPDATE=1 - DEPLOY_WHEEL=1 - ARCHFLAGS="-arch x86_64" + deploy: + - provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + bucket: brainiak + region: us-east-1 + acl: public_read + local_dir: .whl + upload-dir: .whl + skip_cleanup: true + 'on': + branch: master + condition: $DEPLOY_WHEEL = 1 + repo: danielsuo/brainiak + before_install: + - brew install llvm mpich + - VERSIONS="3.5.3" ./bin/install-python-macos.sh install: - VERSIONS="3.5.3" ./bin/build-wheels-macos.sh - language: generic - os: osx - osx_image: xcode7.3 - script: - - VERSIONS="%s" ./bin/test-wheels-macos.sh - sudo: required - - before_install: - - brew install llvm mpich - - VERSIONS="3.6.0" ./bin/install-python-macos.sh before_script: - brew uninstall -f --ignore-dependencies llvm mpich - unset CC @@ -194,21 +183,12 @@ jobs: - unset LDFLAGS - unset CPPFLAGS - unset ARCHFLAGS - deploy: - - access_key_id: AKIAJKBW6H2VKKQDHILQ - acl: public_read - bucket: brainiak - local_dir: .whl - 'on': - branch: master - condition: $DEPLOY_WHEEL = 1 - repo: danielsuo/brainiak - provider: s3 - region: us-east-1 - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - skip_cleanup: true - upload-dir: .whl + script: + - VERSIONS="%s" ./bin/test-wheels-macos.sh + - os: osx + osx_image: xcode7.3 + sudo: required + language: generic env: - CC=/usr/local/opt/llvm/bin/clang - CXX=/usr/local/opt/llvm/bin/clang++ @@ -217,11 +197,32 @@ jobs: - HOMEBREW_NO_AUTO_UPDATE=1 - DEPLOY_WHEEL=1 - ARCHFLAGS="-arch x86_64" + deploy: + - provider: s3 + access_key_id: AKIAJKBW6H2VKKQDHILQ + secret_access_key: + secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= + bucket: brainiak + region: us-east-1 + acl: public_read + local_dir: .whl + upload-dir: .whl + skip_cleanup: true + 'on': + branch: master + condition: $DEPLOY_WHEEL = 1 + repo: danielsuo/brainiak + before_install: + - brew install llvm mpich + - VERSIONS="3.6.0" ./bin/install-python-macos.sh install: - VERSIONS="3.6.0" ./bin/build-wheels-macos.sh - language: generic - os: osx - osx_image: xcode7.3 + before_script: + - brew uninstall -f --ignore-dependencies llvm mpich + - unset CC + - unset CXX + - unset LDFLAGS + - unset CPPFLAGS + - unset ARCHFLAGS script: - VERSIONS="%s" ./bin/test-wheels-macos.sh - sudo: required diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 546d59a30..b0b4a37ba 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # TODO: the first job in a stage must be part of the job dictinoary, not an # element in the job array. This results in a dummy job being created @@ -6,6 +6,32 @@ import yaml import copy +from collections import OrderedDict + +# Ensure yaml works with OrderedDict +from yaml import Loader, Dumper +from yaml.representer import SafeRepresenter + +_mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG + + +def dict_representer(dumper, data): + return dumper.represent_dict(data.items()) + + +def dict_constructor(loader, node): + return OrderedDict(loader.construct_pairs(node)) + + +Dumper.add_representer(OrderedDict, dict_representer) +Loader.add_constructor(_mapping_tag, dict_constructor) + +Dumper.add_representer(str, + SafeRepresenter.represent_str) + +# Dumper.add_representer(unicode, + # SafeRepresenter.represent_unicode) + travis = '.travis.yml' repo = 'danielsuo/brainiak' versions = ['3.4.4', '3.5.3', '3.6.0'] @@ -17,33 +43,34 @@ access_key_id = 'AKIAJKBW6H2VKKQDHILQ' secret_access_key = "A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg=" -data = {} +data = OrderedDict() -data['branches'] = { +data['branches'] = OrderedDict({ 'only': ['master'] -} +}) -data['env'] = { +data['env'] = OrderedDict({ 'global': [ 'BRAINIAK_REPO=%s' % repo ] -} +}) -data['jobs'] = { +data['jobs'] = OrderedDict({ 'include': [] -} +}) jobs = data['jobs']['include'] # Create test stage -jobs.append({'stage': 'test', 'language': 'generic'}) +jobs.append(OrderedDict({'stage': 'test', 'language': 'generic'})) # Linux -test_linux = { +test_linux = OrderedDict({ 'os': 'linux', 'dist': 'trusty', 'sudo': 'required', 'language': 'python', + 'python': None, 'install': ['python3 -m pip install -U pip'], 'script': ['./pr-check.sh'], 'addons': { @@ -51,7 +78,7 @@ 'packages': ['build-essential libgomp1 libmpich-dev mpich'] } } -} +}) for major in majors: block = copy.deepcopy(test_linux) @@ -59,7 +86,7 @@ jobs.append(block) # MacOS -test_macos = { +test_macos = OrderedDict({ 'os': 'osx', 'language': 'generic', 'env': [ @@ -72,7 +99,7 @@ 'before_install': ['brew install llvm mpich python3'], 'install': ['python3 -m pip install -U pip'], 'script': ['./pr-check.sh'] -} +}) for osx in ['xcode7.3', 'xcode8']: block = copy.deepcopy(test_macos) @@ -80,14 +107,14 @@ jobs.append(block) # Create build stage -jobs.append({'stage': 'build', 'language': 'generic'}) +jobs.append(OrderedDict({'stage': 'build', 'language': 'generic'})) -deploy_s3 = [{ +deploy_s3 = [OrderedDict({ 'provider': 's3', 'access_key_id': access_key_id, 'secret_access_key': { 'secure': secret_access_key - }, + }, 'bucket': bucket, 'region': region, 'acl': 'public_read', @@ -98,8 +125,8 @@ 'repo': repo, 'branch': 'master', 'condition': '$DEPLOY_WHEEL = 1' - } -}] + } + })] jobs.append({ 'os': 'linux', @@ -118,14 +145,14 @@ 'DEPLOY_WHEEL=1', 'ARCHFLAGS="-arch x86_64"' ]) -build_macos = { +build_macos = OrderedDict({ 'os': 'osx', 'osx_image': 'xcode7.3', 'sudo': 'required', 'language': 'generic', 'env': build_macos_env, 'deploy': deploy_s3 -} + }) for version in versions: block = copy.deepcopy(build_macos) From 2132feb1149b226fc210f42f3f5a295f1b78e0b3 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 19:24:12 -0500 Subject: [PATCH 083/126] Upload to TRAVIS_COMMIT --- bin/README.md | 3 ++- bin/generate-yaml.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/README.md b/bin/README.md index 2fb902b95..a60851187 100755 --- a/bin/README.md +++ b/bin/README.md @@ -2,7 +2,7 @@ # TODO - Recreate and encrypt keys for brainiak/brainiak -- Split test and build stages +- Default implementation should grab wheels matching current brainiak version - Update documentation - Allow setup.py to take argument for mpi4py wheel location - Clean up buckets appropriately @@ -24,6 +24,7 @@ # Completed - Upload wheels to S3 +- Split test and build stages - Add bdist exception for editable installs - Enforce BrainIAK dependencies while installing dev requirements - Add back quiet / silent flags diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index b0b4a37ba..43ccd6a92 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -119,7 +119,7 @@ def dict_constructor(loader, node): 'region': region, 'acl': 'public_read', 'local_dir': '.whl', - 'upload-dir': '.whl', + 'upload-dir': '$TRAVIS_COMMIT/.whl', 'skip_cleanup': True, 'on': { 'repo': repo, From dc70206be879ba8ab3a1d57c394f97de1c0b928b Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 19:39:30 -0500 Subject: [PATCH 084/126] Turn on output for pip install in test-wheels-macos --- bin/README.md | 1 + bin/generate-yaml.py | 4 ++-- pr-check.sh => bin/pr-check.sh | 4 ++-- run-checks.sh => bin/run-checks.sh | 0 run-tests.sh => bin/run-tests.sh | 0 bin/test-wheels-macos.sh | 6 +++--- bin/test-wheels-manylinux1.sh | 2 +- 7 files changed, 9 insertions(+), 8 deletions(-) rename pr-check.sh => bin/pr-check.sh (98%) rename run-checks.sh => bin/run-checks.sh (100%) rename run-tests.sh => bin/run-tests.sh (100%) diff --git a/bin/README.md b/bin/README.md index a60851187..43925ea27 100755 --- a/bin/README.md +++ b/bin/README.md @@ -23,6 +23,7 @@ - Only deploy on tag (uncomment) # Completed +- Move all scripts to bin - Upload wheels to S3 - Split test and build stages - Add bdist exception for editable installs diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 43ccd6a92..a1010189d 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -72,7 +72,7 @@ def dict_constructor(loader, node): 'language': 'python', 'python': None, 'install': ['python3 -m pip install -U pip'], - 'script': ['./pr-check.sh'], + 'script': ['./bin/pr-check.sh'], 'addons': { 'apt': { 'packages': ['build-essential libgomp1 libmpich-dev mpich'] @@ -98,7 +98,7 @@ def dict_constructor(loader, node): ], 'before_install': ['brew install llvm mpich python3'], 'install': ['python3 -m pip install -U pip'], - 'script': ['./pr-check.sh'] + 'script': ['./bin/pr-check.sh'] }) for osx in ['xcode7.3', 'xcode8']: diff --git a/pr-check.sh b/bin/pr-check.sh similarity index 98% rename from pr-check.sh rename to bin/pr-check.sh index 5d7abc7e9..6044329e0 100755 --- a/pr-check.sh +++ b/bin/pr-check.sh @@ -142,12 +142,12 @@ fi if [ ${dist_mode:-default} != '--bdist-mode' ] then # static analysis - ./run-checks.sh || \ + ./bin/run-checks.sh || \ exit_with_error_and_venv "run-checks failed" fi # run tests -./run-tests.sh $dist_mode || \ +./bin/run-tests.sh $dist_mode || \ exit_with_error_and_venv "run-tests failed" # Only build documentation if we aren't in binary mode diff --git a/run-checks.sh b/bin/run-checks.sh similarity index 100% rename from run-checks.sh rename to bin/run-checks.sh diff --git a/run-tests.sh b/bin/run-tests.sh similarity index 100% rename from run-tests.sh rename to bin/run-tests.sh diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 181e56b94..44ec047e3 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -29,8 +29,8 @@ do # TODO: this will actually pick up both wheels since brainiak is in the path BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep brainiak) - $PYTHON -m pip install -q $MPI4PY_WHEEL - $PYTHON -m pip install -q $BRAINIAK_WHEEL + $PYTHON -m pip install $MPI4PY_WHEEL + $PYTHON -m pip install $BRAINIAK_WHEEL deactivate rm -rf venv @@ -46,5 +46,5 @@ do PYTHON_DIR=$(dirname $(realpath $(which python$MAJOR))) PYTHON=$PYTHON_DIR/python$MAJOR git clean -f -f -x -d -q -e .whl - WHEEL_DIR=$WHEEL_DIR PYTHON=$PYTHON $SCRIPT_DIR/../pr-check.sh + WHEEL_DIR=$WHEEL_DIR PYTHON=$PYTHON $SCRIPT_DIR/pr-check.sh done diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index ac3f82b5f..59937506e 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -37,5 +37,5 @@ for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do mpi_command=mpiexec.hydra \ WHEEL_DIR=$WHEEL_DIR \ PYTHON=/opt/python/$VERSION/bin/python \ - $SCRIPT_DIR/../pr-check.sh + $SCRIPT_DIR/pr-check.sh done From bd184909ad6a08daa2c71b001b5df24dc9eb1335 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 19:40:28 -0500 Subject: [PATCH 085/126] Fix travis --- bin/generate-yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index a1010189d..65abdeb04 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -175,7 +175,7 @@ def dict_constructor(loader, node): ] block['script'] = [ - 'VERSIONS="%s" ./bin/test-wheels-macos.sh' + 'VERSIONS="%s" ./bin/test-wheels-macos.sh' % version ] block['deploy'] = copy.deepcopy(deploy_s3) From 8b01441eace60cbd39ae455a4549e83ee213050b Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 19:45:15 -0500 Subject: [PATCH 086/126] update travis --- .travis.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index b9454431e..90a298d4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ jobs: install: - python3 -m pip install -U pip script: - - ./pr-check.sh + - ./bin/pr-check.sh addons: apt: packages: @@ -29,7 +29,7 @@ jobs: install: - python3 -m pip install -U pip script: - - ./pr-check.sh + - ./bin/pr-check.sh addons: apt: packages: @@ -42,7 +42,7 @@ jobs: install: - python3 -m pip install -U pip script: - - ./pr-check.sh + - ./bin/pr-check.sh addons: apt: packages: @@ -60,7 +60,7 @@ jobs: install: - python3 -m pip install -U pip script: - - ./pr-check.sh + - ./bin/pr-check.sh osx_image: xcode7.3 - os: osx language: generic @@ -75,7 +75,7 @@ jobs: install: - python3 -m pip install -U pip script: - - ./pr-check.sh + - ./bin/pr-check.sh osx_image: xcode8 - stage: build language: generic @@ -88,7 +88,7 @@ jobs: region: us-east-1 acl: public_read local_dir: .whl - upload-dir: .whl + upload-dir: $TRAVIS_COMMIT/.whl skip_cleanup: true 'on': branch: master @@ -124,7 +124,7 @@ jobs: region: us-east-1 acl: public_read local_dir: .whl - upload-dir: .whl + upload-dir: $TRAVIS_COMMIT/.whl skip_cleanup: true 'on': branch: master @@ -143,7 +143,7 @@ jobs: - unset CPPFLAGS - unset ARCHFLAGS script: - - VERSIONS="%s" ./bin/test-wheels-macos.sh + - VERSIONS="3.4.4" ./bin/test-wheels-macos.sh - os: osx osx_image: xcode7.3 sudo: required @@ -165,7 +165,7 @@ jobs: region: us-east-1 acl: public_read local_dir: .whl - upload-dir: .whl + upload-dir: $TRAVIS_COMMIT/.whl skip_cleanup: true 'on': branch: master @@ -184,7 +184,7 @@ jobs: - unset CPPFLAGS - unset ARCHFLAGS script: - - VERSIONS="%s" ./bin/test-wheels-macos.sh + - VERSIONS="3.5.3" ./bin/test-wheels-macos.sh - os: osx osx_image: xcode7.3 sudo: required @@ -206,7 +206,7 @@ jobs: region: us-east-1 acl: public_read local_dir: .whl - upload-dir: .whl + upload-dir: $TRAVIS_COMMIT/.whl skip_cleanup: true 'on': branch: master @@ -225,4 +225,4 @@ jobs: - unset CPPFLAGS - unset ARCHFLAGS script: - - VERSIONS="%s" ./bin/test-wheels-macos.sh + - VERSIONS="3.6.0" ./bin/test-wheels-macos.sh From c5e68f0313141a05b7b24f9b8a70ac747d0719eb Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sat, 16 Dec 2017 20:17:44 -0500 Subject: [PATCH 087/126] Update brew in hopes that we can download llvm resource --- .travis.yml | 2 ++ bin/generate-yaml.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 90a298d4a..7cff6db38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,6 +56,7 @@ jobs: - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 before_install: + - brew update - brew install llvm mpich python3 install: - python3 -m pip install -U pip @@ -71,6 +72,7 @@ jobs: - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 before_install: + - brew update - brew install llvm mpich python3 install: - python3 -m pip install -U pip diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 65abdeb04..e6094dd55 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -96,7 +96,7 @@ def dict_constructor(loader, node): 'CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS"', 'HOMEBREW_NO_AUTO_UPDATE=1' ], - 'before_install': ['brew install llvm mpich python3'], + 'before_install': ['brew update', 'brew install llvm mpich python3'], 'install': ['python3 -m pip install -U pip'], 'script': ['./bin/pr-check.sh'] }) From 02f50624764a80236409f1d145f1ee14437515b9 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sun, 17 Dec 2017 01:00:03 -0500 Subject: [PATCH 088/126] Add third stage --- .travis.yml | 45 ++++++++++++++++++++++++-------- bin/README.md | 26 +++++++++++------- bin/generate-yaml.py | 55 ++++++++++++++++++++++++++++++++------- bin/test-local-install.sh | 27 +++++++++++++++++++ bin/test-s3-macos.sh | 11 ++++++++ bin/test-s3-manylinux1.sh | 10 +++++++ bin/test-s3.sh | 3 +++ setup.py | 23 +++++++++------- 8 files changed, 160 insertions(+), 40 deletions(-) create mode 100755 bin/test-local-install.sh create mode 100644 bin/test-s3-macos.sh create mode 100644 bin/test-s3-manylinux1.sh create mode 100644 bin/test-s3.sh diff --git a/.travis.yml b/.travis.yml index 7cff6db38..6768c7e3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ env: - BRAINIAK_REPO=danielsuo/brainiak jobs: include: - - stage: test + - stage: test-with-deps language: generic - os: linux dist: trusty @@ -61,6 +61,7 @@ jobs: install: - python3 -m pip install -U pip script: + - ./bin/test-local-install.sh - ./bin/pr-check.sh osx_image: xcode7.3 - os: osx @@ -77,11 +78,21 @@ jobs: install: - python3 -m pip install -U pip script: + - ./bin/test-local-install.sh - ./bin/pr-check.sh osx_image: xcode8 - stage: build language: generic - - deploy: + - os: linux + dist: trusty + sudo: required + language: generic + env: DEPLOY_WHEEL=1 + install: + - ./bin/build-wheels.sh + script: + - ./bin/test-wheels.sh + deploy: - provider: s3 access_key_id: AKIAJKBW6H2VKKQDHILQ secret_access_key: @@ -96,15 +107,6 @@ jobs: branch: master condition: $DEPLOY_WHEEL = 1 repo: danielsuo/brainiak - dist: trusty - env: DEPLOY_WHEEL=1 - install: - - ./bin/build-wheels.sh - language: generic - os: linux - script: - - ./bin/test-wheels.sh - sudo: required - os: osx osx_image: xcode7.3 sudo: required @@ -133,6 +135,7 @@ jobs: condition: $DEPLOY_WHEEL = 1 repo: danielsuo/brainiak before_install: + - brew update - brew install llvm mpich - VERSIONS="3.4.4" ./bin/install-python-macos.sh install: @@ -174,6 +177,7 @@ jobs: condition: $DEPLOY_WHEEL = 1 repo: danielsuo/brainiak before_install: + - brew update - brew install llvm mpich - VERSIONS="3.5.3" ./bin/install-python-macos.sh install: @@ -215,6 +219,7 @@ jobs: condition: $DEPLOY_WHEEL = 1 repo: danielsuo/brainiak before_install: + - brew update - brew install llvm mpich - VERSIONS="3.6.0" ./bin/install-python-macos.sh install: @@ -228,3 +233,21 @@ jobs: - unset ARCHFLAGS script: - VERSIONS="3.6.0" ./bin/test-wheels-macos.sh + - stage: test-wout-deps + language: generic + if: branch = master and repo = danielsuo/brainiak + - os: linux + dist: trusty + sudo: required + language: generic + install: true + script: + - ./bin/test-s3.sh + - os: osx + osx_image: xcode7.3 + sudo: required + language: generic + install: + - ./bin/install-python-macos.sh + script: + - ./bin/test-s3-macos.sh diff --git a/bin/README.md b/bin/README.md index 43925ea27..ea1c7ac14 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,11 +1,16 @@ # Wheel to-dos # TODO +- Test-with-deps: test source with dependencies + can be installed (doesn't check install now) +- Test-wout-deps: test source without dependencies + can be installed (doesn't check tests now) +- Upload wheels to PyPI (uncomment) +- Upload to TestPyPI +- Only upload to PyPI on tagged master - Recreate and encrypt keys for brainiak/brainiak -- Default implementation should grab wheels matching current brainiak version +- Only need to build mpi4py occasionally; not with every commit +- mpi4py should always commit to s3://brainiak/.whl - Update documentation -- Allow setup.py to take argument for mpi4py wheel location -- Clean up buckets appropriately +- Figure out why ```setup.py``` fails - End-to-end test - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) @@ -15,14 +20,17 @@ - Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8)) # To verify -- Modify ```setup.py``` to handle conditional MPI install -- Upload wheels to PyPI (uncomment) -- Always upload to S3, only upload to PyPI on tag (uncomment) -- Deploy only on master (uncomment in ```.travis.yml```) -- Only run on master branch (uncomment) -- Only deploy on tag (uncomment) # Completed +- Clean up buckets appropriately +- Change upload / download to $TRAVIS_COMMIT/.whl +- Test local install (```pip``` vs. ```python setup.py```, editable vs. not) +- Only deploy on tag (uncomment) +- Only run on master branch (uncomment) +- Deploy only on master (uncomment in ```.travis.yml```) +- Default implementation should grab wheels matching current brainiak version +- Allow setup.py to take argument for mpi4py wheel location +- Modify ```setup.py``` to handle conditional MPI install - Move all scripts to bin - Upload wheels to S3 - Split test and build stages diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index e6094dd55..689c5c4b1 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -3,6 +3,15 @@ # TODO: the first job in a stage must be part of the job dictinoary, not an # element in the job array. This results in a dummy job being created +# Stages: (Travis capitalizes first letter and lowers the rest) +# - Test-with-deps: test source and source distribution, dependencies +# - Build: build wheels and test +# - Test-wout-deps: download wheels from S3, install w/out dependencies +# - Testpypi: upload to test PyPI +# - Testpypi-download: download wheels from test PyPI and test +# - Pypi: Upload to PyPI +# - Pypi-download: download wheels from PyPI and test + import yaml import copy @@ -30,7 +39,7 @@ def dict_constructor(loader, node): SafeRepresenter.represent_str) # Dumper.add_representer(unicode, - # SafeRepresenter.represent_unicode) +# SafeRepresenter.represent_unicode) travis = '.travis.yml' repo = 'danielsuo/brainiak' @@ -41,7 +50,7 @@ def dict_constructor(loader, node): bucket = 'brainiak' region = 'us-east-1' access_key_id = 'AKIAJKBW6H2VKKQDHILQ' -secret_access_key = "A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg=" +secret_access_key = 'A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg=' data = OrderedDict() @@ -62,7 +71,7 @@ def dict_constructor(loader, node): jobs = data['jobs']['include'] # Create test stage -jobs.append(OrderedDict({'stage': 'test', 'language': 'generic'})) +jobs.append(OrderedDict({'stage': 'test-with-deps', 'language': 'generic'})) # Linux test_linux = OrderedDict({ @@ -98,7 +107,7 @@ def dict_constructor(loader, node): ], 'before_install': ['brew update', 'brew install llvm mpich python3'], 'install': ['python3 -m pip install -U pip'], - 'script': ['./bin/pr-check.sh'] + 'script': ['./bin/test-local-install.sh', './bin/pr-check.sh'] }) for osx in ['xcode7.3', 'xcode8']: @@ -114,7 +123,7 @@ def dict_constructor(loader, node): 'access_key_id': access_key_id, 'secret_access_key': { 'secure': secret_access_key - }, + }, 'bucket': bucket, 'region': region, 'acl': 'public_read', @@ -125,10 +134,10 @@ def dict_constructor(loader, node): 'repo': repo, 'branch': 'master', 'condition': '$DEPLOY_WHEEL = 1' - } - })] + } +})] -jobs.append({ +jobs.append(OrderedDict({ 'os': 'linux', 'dist': 'trusty', 'sudo': 'required', @@ -137,7 +146,7 @@ def dict_constructor(loader, node): 'install': ['./bin/build-wheels.sh'], 'script': ['./bin/test-wheels.sh'], 'deploy': deploy_s3 -}) +})) build_macos_env = copy.deepcopy(test_macos['env']) @@ -152,11 +161,12 @@ def dict_constructor(loader, node): 'language': 'generic', 'env': build_macos_env, 'deploy': deploy_s3 - }) +}) for version in versions: block = copy.deepcopy(build_macos) block['before_install'] = [ + 'brew update', 'brew install llvm mpich', 'VERSIONS="%s" ./bin/install-python-macos.sh' % version ] @@ -182,5 +192,30 @@ def dict_constructor(loader, node): jobs.append(block) +# S3-Download +jobs.append(OrderedDict({ + 'stage': 'test-wout-deps', + 'language': 'generic', + 'if': 'branch = master and repo = %s' % repo +})) + +jobs.append(OrderedDict({ + 'os': 'linux', + 'dist': 'trusty', + 'sudo': 'required', + 'language': 'generic', + 'install': True, + 'script': ['./bin/test-s3.sh'] +})) + +jobs.append(OrderedDict({ + 'os': 'osx', + 'osx_image': 'xcode7.3', + 'sudo': 'required', + 'language': 'generic', + 'install': ['./bin/install-python-macos.sh'], + 'script': ['./bin/test-s3-macos.sh'] +})) + with open(travis, 'w') as yml: yaml.dump(data, yml, default_flow_style=False) diff --git a/bin/test-local-install.sh b/bin/test-local-install.sh new file mode 100755 index 000000000..51f830bc0 --- /dev/null +++ b/bin/test-local-install.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -ex + +# Install package from source in four ways +installs=( + "pip install ." + "pip install -e ." + + # TODO: Figure out why these don't work + # "python3 setup.py install" + # "python3 setup.py develop" +) + +if [ -z $PYTHON ] +then + PYTHON=python3 +fi + +for install in "${installs[@]}" +do + $PYTHON -m venv venv + source venv/bin/activate + $install + deactivate + rm -rf venv +done diff --git a/bin/test-s3-macos.sh b/bin/test-s3-macos.sh new file mode 100644 index 000000000..a0bf5e043 --- /dev/null +++ b/bin/test-s3-macos.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -ex + +# Test mpi4py wheels from s3 +for VERSION in $VERSIONS +do + MAJOR=${VERSION%.*} + PYTHON=$(realpath $(which python$MAJOR)) + PYTHON=$PYTHON ./bin/test-local-install.sh +done diff --git a/bin/test-s3-manylinux1.sh b/bin/test-s3-manylinux1.sh new file mode 100644 index 000000000..f6350f340 --- /dev/null +++ b/bin/test-s3-manylinux1.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -ex + +# Test mpi4py wheels from s3 +for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m +do + PYTHON=/opt/python/${VERSION}/bin/python3 + PYTHON=$PYTHON ./bin/test-local-install.sh +done diff --git a/bin/test-s3.sh b/bin/test-s3.sh new file mode 100644 index 000000000..bba77cd68 --- /dev/null +++ b/bin/test-s3.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker run --rm -w /src -v `pwd`:/src -ti brainiak/manylinux ./bin/test-s3-manylinux1.sh diff --git a/setup.py b/setup.py index 27e91c9e7..90357e23e 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import sys import setuptools from copy import deepcopy -import pip +from subprocess import check_call, CalledProcessError import urllib.request assert sys.version_info >= (3, 4), ( @@ -15,30 +15,33 @@ ) # Attempt to install mpi4py -result = pip.main(['install', 'mpi4py']) -if result != 0: +try: + result = check_call([sys.executable, '-m', 'pip', 'install', 'mpi4py']) +except CalledProcessError as e: # MacOS: mpi4py-3.0.0-cp34-cp34m-macosx_10_6_intel.whl # Linux: mpi4py-3.0.1a0-cp34-cp34m-manylinux1_x86_64.whl base_url = 'https://s3.amazonaws.com/brainiak/.whl/%s' + mpi4py_version = '3.0.0' # Determine wheel file name # TODO: These templates may change / may need to become more advanced - darwin = 'mpi4py-3.0.0-cp%(version)d-cp%(version)dm-macosx_10_6_intel.whl' - linux = 'mpi4py-3.0.0-cp%(version)d-cp%(version)dm-manylinux1_x86_64.whl' - - wheel_file = darwin if sys.platform == 'darwin' else linux + wheel_file = 'mpi4py-%(mpi4py)s-cp%(py)d-cp%(py)dm-%(dist)s.whl' wheel_file = wheel_file % { - 'version': sys.version_info[0] * 10 + sys.version_info[1] + 'mpi4py': mpi4py_version, + 'py': sys.version_info[0] * 10 + sys.version_info[1], + 'dist': 'macosx_10_6_intel' if sys.platform == 'darwin' + else 'manylinux1_x86_64' } wheel_url = base_url % wheel_file # Download mpi4py wheel - local_filename, headers = urllib.request.urlretrieve(wheel_url) + local_filename, headers = urllib.request.urlretrieve(wheel_url, wheel_file) # Install mpi4py wheel - result = pip.main(['install', local_filename]) + result = check_call([sys.executable, '-m', 'pip', + 'install', local_filename]) if result != 0: sys.exit('ERROR: failed to install mpi4py') From 4fb64af5eb0d4aa84d6d615ea870f0974600bb03 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sun, 17 Dec 2017 08:20:54 -0500 Subject: [PATCH 089/126] Make test s3 scripts executable --- bin/test-s3-macos.sh | 0 bin/test-s3-manylinux1.sh | 0 bin/test-s3.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bin/test-s3-macos.sh mode change 100644 => 100755 bin/test-s3-manylinux1.sh mode change 100644 => 100755 bin/test-s3.sh diff --git a/bin/test-s3-macos.sh b/bin/test-s3-macos.sh old mode 100644 new mode 100755 diff --git a/bin/test-s3-manylinux1.sh b/bin/test-s3-manylinux1.sh old mode 100644 new mode 100755 diff --git a/bin/test-s3.sh b/bin/test-s3.sh old mode 100644 new mode 100755 From a2cd84353065a66ca6de056f5bfe65624f84b636 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sun, 17 Dec 2017 10:05:24 -0500 Subject: [PATCH 090/126] Update MacOX to test properly without dependencies --- .travis.yml | 5 +++-- bin/README.md | 4 ++-- bin/generate-yaml.py | 23 ++++++++++++++++------- requirements-dev.txt | 1 + 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6768c7e3d..0892b619b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -83,6 +83,7 @@ jobs: osx_image: xcode8 - stage: build language: generic + if: branch = master and repo = danielsuo/brainiak - os: linux dist: trusty sudo: required @@ -248,6 +249,6 @@ jobs: sudo: required language: generic install: - - ./bin/install-python-macos.sh + - VERSIONS="3.4.4 3.5.3 3.6.0" ./bin/install-python-macos.sh script: - - ./bin/test-s3-macos.sh + - VERSIONS="3.4.4 3.5.3 3.6.0" ./bin/test-s3-macos.sh diff --git a/bin/README.md b/bin/README.md index ea1c7ac14..32de4a345 100755 --- a/bin/README.md +++ b/bin/README.md @@ -3,8 +3,8 @@ # TODO - Test-with-deps: test source with dependencies + can be installed (doesn't check install now) - Test-wout-deps: test source without dependencies + can be installed (doesn't check tests now) -- Upload wheels to PyPI (uncomment) - Upload to TestPyPI +- Upload wheels to PyPI (uncomment) - Only upload to PyPI on tagged master - Recreate and encrypt keys for brainiak/brainiak - Only need to build mpi4py occasionally; not with every commit @@ -22,11 +22,11 @@ # To verify # Completed +- Confirm stages that run only on master commit to main don't run for PRs other branchs - Clean up buckets appropriately - Change upload / download to $TRAVIS_COMMIT/.whl - Test local install (```pip``` vs. ```python setup.py```, editable vs. not) - Only deploy on tag (uncomment) -- Only run on master branch (uncomment) - Deploy only on master (uncomment in ```.travis.yml```) - Default implementation should grab wheels matching current brainiak version - Allow setup.py to take argument for mpi4py wheel location diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 689c5c4b1..7d0ca3f4b 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -6,11 +6,12 @@ # Stages: (Travis capitalizes first letter and lowers the rest) # - Test-with-deps: test source and source distribution, dependencies # - Build: build wheels and test +# - deploy to s3 # - Test-wout-deps: download wheels from S3, install w/out dependencies -# - Testpypi: upload to test PyPI -# - Testpypi-download: download wheels from test PyPI and test -# - Pypi: Upload to PyPI -# - Pypi-download: download wheels from PyPI and test +# - deploy to testpypi +# - Testpypi: download wheels from test PyPI and test +# - deploy to pypi +# - Pypi: download wheels from PyPI and test import yaml import copy @@ -116,7 +117,11 @@ def dict_constructor(loader, node): jobs.append(block) # Create build stage -jobs.append(OrderedDict({'stage': 'build', 'language': 'generic'})) +jobs.append(OrderedDict({ + 'stage': 'build', + 'language': 'generic', + 'if': 'branch = master and repo = %s' % repo +})) deploy_s3 = [OrderedDict({ 'provider': 's3', @@ -213,8 +218,12 @@ def dict_constructor(loader, node): 'osx_image': 'xcode7.3', 'sudo': 'required', 'language': 'generic', - 'install': ['./bin/install-python-macos.sh'], - 'script': ['./bin/test-s3-macos.sh'] + 'install': [ + 'VERSIONS="%s" ./bin/install-python-macos.sh' % ' '.join(versions) + ], + 'script': [ + 'VERSIONS="%s" ./bin/test-s3-macos.sh' % ' '.join(versions) + ] })) with open(travis, 'w') as yml: diff --git a/requirements-dev.txt b/requirements-dev.txt index 2a887129e..55c4e0005 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,6 +5,7 @@ flake8-print mypy numdifftools pytest +pyyaml restructuredtext-lint setuptools_scm sphinx From 705dae4ecc74111d72f5785371cb5b529c58194b Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Sun, 17 Dec 2017 12:53:42 -0500 Subject: [PATCH 091/126] Update readme [skip ci] --- bin/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/README.md b/bin/README.md index ea1c7ac14..5cc963648 100755 --- a/bin/README.md +++ b/bin/README.md @@ -2,7 +2,8 @@ # TODO - Test-with-deps: test source with dependencies + can be installed (doesn't check install now) -- Test-wout-deps: test source without dependencies + can be installed (doesn't check tests now) +- Test-wout-deps: test binary without dependencies + can be installed (doesn't check tests now) +- Fix stages; it appears that everything just gets lumped into a single stage - Upload wheels to PyPI (uncomment) - Upload to TestPyPI - Only upload to PyPI on tagged master From d25993b69acc6757dbde1951a805cebf25166545 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Mon, 18 Dec 2017 23:30:07 -0500 Subject: [PATCH 092/126] [skip ci] Refactor test stages --- .travis.yml | 85 ++++++++----------- bin/README.md | 6 +- ...ld-wheels-macos.sh => build-dist-macos.sh} | 19 +++-- ...manylinux1.sh => build-dist-manylinux1.sh} | 22 +++-- bin/{test-s3.sh => build-dist.sh} | 2 +- bin/build-wheels.sh | 3 - bin/generate-yaml.py | 72 ++++++---------- bin/test-local-install.sh | 27 ------ bin/test-s3-macos.sh | 11 --- bin/test-s3-manylinux1.sh | 10 --- bin/test-wheels-macos.sh | 6 +- bin/test-wheels-manylinux1.sh | 6 +- 12 files changed, 95 insertions(+), 174 deletions(-) rename bin/{build-wheels-macos.sh => build-dist-macos.sh} (71%) rename bin/{build-wheels-manylinux1.sh => build-dist-manylinux1.sh} (66%) rename bin/{test-s3.sh => build-dist.sh} (75%) delete mode 100755 bin/build-wheels.sh delete mode 100755 bin/test-local-install.sh delete mode 100755 bin/test-s3-macos.sh delete mode 100755 bin/test-s3-manylinux1.sh diff --git a/.travis.yml b/.travis.yml index 0892b619b..afe8d2322 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ env: - BRAINIAK_REPO=danielsuo/brainiak jobs: include: - - stage: test-with-deps + - stage: test language: generic - os: linux dist: trusty @@ -61,7 +61,6 @@ jobs: install: - python3 -m pip install -U pip script: - - ./bin/test-local-install.sh - ./bin/pr-check.sh osx_image: xcode7.3 - os: osx @@ -78,7 +77,6 @@ jobs: install: - python3 -m pip install -U pip script: - - ./bin/test-local-install.sh - ./bin/pr-check.sh osx_image: xcode8 - stage: build @@ -87,12 +85,16 @@ jobs: - os: linux dist: trusty sudo: required - language: generic - env: DEPLOY_WHEEL=1 + language: python + python: '3.4' + env: TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi install: - - ./bin/build-wheels.sh + - python3 -m pip install -U pip twine script: - ./bin/test-wheels.sh + - ./bin/build-dist.sh + after_script: + - twine upload dist/* deploy: - provider: s3 access_key_id: AKIAJKBW6H2VKKQDHILQ @@ -101,12 +103,11 @@ jobs: bucket: brainiak region: us-east-1 acl: public_read - local_dir: .whl - upload-dir: $TRAVIS_COMMIT/.whl + local_dir: dist + upload-dir: $TRAVIS_COMMIT/dist skip_cleanup: true 'on': branch: master - condition: $DEPLOY_WHEEL = 1 repo: danielsuo/brainiak - os: osx osx_image: xcode7.3 @@ -118,8 +119,7 @@ jobs: - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - - DEPLOY_WHEEL=1 - - ARCHFLAGS="-arch x86_64" + - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypiARCHFLAGS="-arch x86_64" deploy: - provider: s3 access_key_id: AKIAJKBW6H2VKKQDHILQ @@ -128,19 +128,18 @@ jobs: bucket: brainiak region: us-east-1 acl: public_read - local_dir: .whl - upload-dir: $TRAVIS_COMMIT/.whl + local_dir: dist + upload-dir: $TRAVIS_COMMIT/dist skip_cleanup: true 'on': branch: master - condition: $DEPLOY_WHEEL = 1 repo: danielsuo/brainiak before_install: - brew update - - brew install llvm mpich - - VERSIONS="3.4.4" ./bin/install-python-macos.sh + - brew install llvm mpich python3 + - python3 -m pip install --user twineVERSIONS="3.4.4" ./bin/install-python-macos.sh install: - - VERSIONS="3.4.4" ./bin/build-wheels-macos.sh + - VERSIONS="3.4.4" ./bin/build-dist-macos.sh before_script: - brew uninstall -f --ignore-dependencies llvm mpich - unset CC @@ -150,6 +149,8 @@ jobs: - unset ARCHFLAGS script: - VERSIONS="3.4.4" ./bin/test-wheels-macos.sh + after_script: + - twine upload dist/* - os: osx osx_image: xcode7.3 sudo: required @@ -160,8 +161,7 @@ jobs: - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - - DEPLOY_WHEEL=1 - - ARCHFLAGS="-arch x86_64" + - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypiARCHFLAGS="-arch x86_64" deploy: - provider: s3 access_key_id: AKIAJKBW6H2VKKQDHILQ @@ -170,19 +170,18 @@ jobs: bucket: brainiak region: us-east-1 acl: public_read - local_dir: .whl - upload-dir: $TRAVIS_COMMIT/.whl + local_dir: dist + upload-dir: $TRAVIS_COMMIT/dist skip_cleanup: true 'on': branch: master - condition: $DEPLOY_WHEEL = 1 repo: danielsuo/brainiak before_install: - brew update - - brew install llvm mpich - - VERSIONS="3.5.3" ./bin/install-python-macos.sh + - brew install llvm mpich python3 + - python3 -m pip install --user twineVERSIONS="3.5.3" ./bin/install-python-macos.sh install: - - VERSIONS="3.5.3" ./bin/build-wheels-macos.sh + - VERSIONS="3.5.3" ./bin/build-dist-macos.sh before_script: - brew uninstall -f --ignore-dependencies llvm mpich - unset CC @@ -192,6 +191,8 @@ jobs: - unset ARCHFLAGS script: - VERSIONS="3.5.3" ./bin/test-wheels-macos.sh + after_script: + - twine upload dist/* - os: osx osx_image: xcode7.3 sudo: required @@ -202,8 +203,7 @@ jobs: - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - - DEPLOY_WHEEL=1 - - ARCHFLAGS="-arch x86_64" + - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypiARCHFLAGS="-arch x86_64" deploy: - provider: s3 access_key_id: AKIAJKBW6H2VKKQDHILQ @@ -212,19 +212,18 @@ jobs: bucket: brainiak region: us-east-1 acl: public_read - local_dir: .whl - upload-dir: $TRAVIS_COMMIT/.whl + local_dir: dist + upload-dir: $TRAVIS_COMMIT/dist skip_cleanup: true 'on': branch: master - condition: $DEPLOY_WHEEL = 1 repo: danielsuo/brainiak before_install: - brew update - - brew install llvm mpich - - VERSIONS="3.6.0" ./bin/install-python-macos.sh + - brew install llvm mpich python3 + - python3 -m pip install --user twineVERSIONS="3.6.0" ./bin/install-python-macos.sh install: - - VERSIONS="3.6.0" ./bin/build-wheels-macos.sh + - VERSIONS="3.6.0" ./bin/build-dist-macos.sh before_script: - brew uninstall -f --ignore-dependencies llvm mpich - unset CC @@ -234,21 +233,5 @@ jobs: - unset ARCHFLAGS script: - VERSIONS="3.6.0" ./bin/test-wheels-macos.sh - - stage: test-wout-deps - language: generic - if: branch = master and repo = danielsuo/brainiak - - os: linux - dist: trusty - sudo: required - language: generic - install: true - script: - - ./bin/test-s3.sh - - os: osx - osx_image: xcode7.3 - sudo: required - language: generic - install: - - VERSIONS="3.4.4 3.5.3 3.6.0" ./bin/install-python-macos.sh - script: - - VERSIONS="3.4.4 3.5.3 3.6.0" ./bin/test-s3-macos.sh + after_script: + - twine upload dist/* diff --git a/bin/README.md b/bin/README.md index 415760d54..93892330d 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,15 +1,13 @@ # Wheel to-dos # TODO -- Test-with-deps: test source with dependencies + can be installed (doesn't check install now) -- Test-wout-deps: test binary without dependencies + can be installed (doesn't check tests now) - Fix stages; it appears that everything just gets lumped into a single stage - Upload wheels to PyPI (uncomment) - Upload to TestPyPI - Upload wheels to PyPI (uncomment) - Only upload to PyPI on tagged master - Recreate and encrypt keys for brainiak/brainiak -- Only need to build mpi4py occasionally; not with every commit +- Only need to build mpi4py occasionally; not with every commit, and not with every job - mpi4py should always commit to s3://brainiak/.whl - Update documentation - Figure out why ```setup.py``` fails @@ -24,6 +22,8 @@ # To verify # Completed +- Test-with-deps: test source with dependencies + can be installed (doesn't check install now) +- Test-wout-deps: test binary without dependencies + can be installed (doesn't check tests now) - Confirm stages that run only on master commit to main don't run for PRs other branchs - Clean up buckets appropriately - Change upload / download to $TRAVIS_COMMIT/.whl diff --git a/bin/build-wheels-macos.sh b/bin/build-dist-macos.sh similarity index 71% rename from bin/build-wheels-macos.sh rename to bin/build-dist-macos.sh index cdc4f71ed..1f2f96647 100755 --- a/bin/build-wheels-macos.sh +++ b/bin/build-dist-macos.sh @@ -6,10 +6,6 @@ set -e # Show explicitly which commands are currently running. set -x -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -WHEEL_DIR=$SCRIPT_DIR/../.whl -mkdir -p $WHEEL_DIR - git clone -q https://bitbucket.org/mpi4py/mpi4py pushd mpi4py git checkout 3.0.0 @@ -23,22 +19,27 @@ do PYTHON=$PYTHON_DIR/python$MAJOR DELOCATE=$(dirname $PYTHON)/delocate-wheel - git clean -f -f -x -d -q -e .whl -e mpi4py + git clean -f -f -x -d -q -e dist -e mpi4py + # Use virtual environments because we want to install later $PYTHON -m venv venv source venv/bin/activate pushd mpi4py git clean -f -f -x -d -q - $PYTHON setup.py bdist_wheel + $PYTHON setup.py -q bdist_wheel $DELOCATE dist/*.whl - mv dist/*.whl $WHEEL_DIR/ popd $PYTHON -m pip install -q . - $PYTHON setup.py bdist_wheel + $PYTHON setup.py -q bdist_wheel $DELOCATE dist/*.whl - mv dist/*.whl $WHEEL_DIR/ + + # Build source distribution + if [ $MAJOR == '3.4' ] + then + $PYTHON setup.py -q sdist + fi deactivate rm -rf venv diff --git a/bin/build-wheels-manylinux1.sh b/bin/build-dist-manylinux1.sh similarity index 66% rename from bin/build-wheels-manylinux1.sh rename to bin/build-dist-manylinux1.sh index b8a2c7052..609f07ba0 100755 --- a/bin/build-wheels-manylinux1.sh +++ b/bin/build-dist-manylinux1.sh @@ -11,8 +11,9 @@ yum install -y -q \ mpich2-devel SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -WHEEL_DIR=$SCRIPT_DIR/../.whl +WHEEL_DIR=$SCRIPT_DIR/../dist mkdir -p $WHEEL_DIR +mkdir -p tmp # - Use the brainiak source stored in /brainiak via docker run command # - Use head of mpi4py (we can pick some more suitable tag) @@ -27,16 +28,21 @@ for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do pushd /mpi4py git clean -f -f -x -d -q - $PYTHON setup.py -q bdist_wheel - auditwheel repair dist/*.whl - mv wheelhouse/*.whl $WHEEL_DIR/ + $PYTHON setup.py -q bdist_wheel -d tmp + auditwheel repair tmp/*.whl -w $WHEEL_DIR popd - git clean -f -f -x -d -q -e .whl + git clean -f -f -x -d -q -e dist $PYTHON -m pip install -q . - $PYTHON setup.py -q bdist_wheel - auditwheel repair dist/*.whl - mv wheelhouse/*.whl $WHEEL_DIR/ + $PYTHON setup.py -q bdist_wheel -d tmp + auditwheel repair tmp/*.whl -w $WHEEL_DIR + + # Build source distribution + if [ $VERSION = "cp34-cp34m" ] + then + $PYTHON setup.py -q sdist + fi done rm -rf /mpi4py +rm -rf tmp diff --git a/bin/test-s3.sh b/bin/build-dist.sh similarity index 75% rename from bin/test-s3.sh rename to bin/build-dist.sh index bba77cd68..7d661298f 100755 --- a/bin/test-s3.sh +++ b/bin/build-dist.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -docker run --rm -w /src -v `pwd`:/src -ti brainiak/manylinux ./bin/test-s3-manylinux1.sh +docker run --rm -w /src -v `pwd`:/src -ti brainiak/manylinux ./bin/build-dist-manylinux1.sh diff --git a/bin/build-wheels.sh b/bin/build-wheels.sh deleted file mode 100755 index abf6cea2d..000000000 --- a/bin/build-wheels.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -docker run --rm -w /src -v `pwd`:/src -ti brainiak/manylinux ./bin/build-wheels-manylinux1.sh diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 7d0ca3f4b..b3df636f3 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -4,11 +4,9 @@ # element in the job array. This results in a dummy job being created # Stages: (Travis capitalizes first letter and lowers the rest) -# - Test-with-deps: test source and source distribution, dependencies +# - Test: test source and source distribution, dependencies # - Build: build wheels and test -# - deploy to s3 -# - Test-wout-deps: download wheels from S3, install w/out dependencies -# - deploy to testpypi +# - deploy to s3 and testpypi # - Testpypi: download wheels from test PyPI and test # - deploy to pypi # - Pypi: download wheels from PyPI and test @@ -72,7 +70,7 @@ def dict_constructor(loader, node): jobs = data['jobs']['include'] # Create test stage -jobs.append(OrderedDict({'stage': 'test-with-deps', 'language': 'generic'})) +jobs.append(OrderedDict({'stage': 'test', 'language': 'generic'})) # Linux test_linux = OrderedDict({ @@ -108,7 +106,7 @@ def dict_constructor(loader, node): ], 'before_install': ['brew update', 'brew install llvm mpich python3'], 'install': ['python3 -m pip install -U pip'], - 'script': ['./bin/test-local-install.sh', './bin/pr-check.sh'] + 'script': ['./bin/pr-check.sh'] }) for osx in ['xcode7.3', 'xcode8']: @@ -132,13 +130,12 @@ def dict_constructor(loader, node): 'bucket': bucket, 'region': region, 'acl': 'public_read', - 'local_dir': '.whl', - 'upload-dir': '$TRAVIS_COMMIT/.whl', + 'local_dir': 'dist', + 'upload-dir': '$TRAVIS_COMMIT/dist', 'skip_cleanup': True, 'on': { 'repo': repo, 'branch': 'master', - 'condition': '$DEPLOY_WHEEL = 1' } })] @@ -146,17 +143,26 @@ def dict_constructor(loader, node): 'os': 'linux', 'dist': 'trusty', 'sudo': 'required', - 'language': 'generic', - 'env': 'DEPLOY_WHEEL=1', - 'install': ['./bin/build-wheels.sh'], - 'script': ['./bin/test-wheels.sh'], + 'language': 'python', + 'python': '3.4', + 'env': 'TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi', + 'install': [ + 'python3 -m pip install -U pip twine' + ], + 'script': [ + './bin/test-wheels.sh', + './bin/build-dist.sh' + ], + 'after_script': [ + 'twine upload dist/*' + ], 'deploy': deploy_s3 })) build_macos_env = copy.deepcopy(test_macos['env']) build_macos_env.extend([ - 'DEPLOY_WHEEL=1', + 'TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi' 'ARCHFLAGS="-arch x86_64"' ]) build_macos = OrderedDict({ @@ -172,12 +178,13 @@ def dict_constructor(loader, node): block = copy.deepcopy(build_macos) block['before_install'] = [ 'brew update', - 'brew install llvm mpich', + 'brew install llvm mpich python3', + 'python3 -m pip install --user twine' 'VERSIONS="%s" ./bin/install-python-macos.sh' % version ] block['install'] = [ - 'VERSIONS="%s" ./bin/build-wheels-macos.sh' % version + 'VERSIONS="%s" ./bin/build-dist-macos.sh' % version ] block['before_script'] = [ @@ -193,38 +200,13 @@ def dict_constructor(loader, node): 'VERSIONS="%s" ./bin/test-wheels-macos.sh' % version ] + block['after_script'] = [ + 'twine upload dist/*' + ] + block['deploy'] = copy.deepcopy(deploy_s3) jobs.append(block) -# S3-Download -jobs.append(OrderedDict({ - 'stage': 'test-wout-deps', - 'language': 'generic', - 'if': 'branch = master and repo = %s' % repo -})) - -jobs.append(OrderedDict({ - 'os': 'linux', - 'dist': 'trusty', - 'sudo': 'required', - 'language': 'generic', - 'install': True, - 'script': ['./bin/test-s3.sh'] -})) - -jobs.append(OrderedDict({ - 'os': 'osx', - 'osx_image': 'xcode7.3', - 'sudo': 'required', - 'language': 'generic', - 'install': [ - 'VERSIONS="%s" ./bin/install-python-macos.sh' % ' '.join(versions) - ], - 'script': [ - 'VERSIONS="%s" ./bin/test-s3-macos.sh' % ' '.join(versions) - ] -})) - with open(travis, 'w') as yml: yaml.dump(data, yml, default_flow_style=False) diff --git a/bin/test-local-install.sh b/bin/test-local-install.sh deleted file mode 100755 index 51f830bc0..000000000 --- a/bin/test-local-install.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -# Install package from source in four ways -installs=( - "pip install ." - "pip install -e ." - - # TODO: Figure out why these don't work - # "python3 setup.py install" - # "python3 setup.py develop" -) - -if [ -z $PYTHON ] -then - PYTHON=python3 -fi - -for install in "${installs[@]}" -do - $PYTHON -m venv venv - source venv/bin/activate - $install - deactivate - rm -rf venv -done diff --git a/bin/test-s3-macos.sh b/bin/test-s3-macos.sh deleted file mode 100755 index a0bf5e043..000000000 --- a/bin/test-s3-macos.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -# Test mpi4py wheels from s3 -for VERSION in $VERSIONS -do - MAJOR=${VERSION%.*} - PYTHON=$(realpath $(which python$MAJOR)) - PYTHON=$PYTHON ./bin/test-local-install.sh -done diff --git a/bin/test-s3-manylinux1.sh b/bin/test-s3-manylinux1.sh deleted file mode 100755 index f6350f340..000000000 --- a/bin/test-s3-manylinux1.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -# Test mpi4py wheels from s3 -for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m -do - PYTHON=/opt/python/${VERSION}/bin/python3 - PYTHON=$PYTHON ./bin/test-local-install.sh -done diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 44ec047e3..1777a2b85 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -7,7 +7,7 @@ set -e set -x SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -WHEEL_DIR=$SCRIPT_DIR/../.whl +WHEEL_DIR=$SCRIPT_DIR/../dist # Test whether we can install without any dependencies # TODO: delete once we have setup.py setup correctly @@ -18,7 +18,7 @@ do PYTHON=$PYTHON_DIR/python$MAJOR WHEEL_VERSION=$(echo $MAJOR | tr -d '.') - git clean -f -f -x -d -q -e .whl + git clean -f -f -x -d -q -e dist $PYTHON -m venv venv source venv/bin/activate @@ -45,6 +45,6 @@ do MAJOR=${VERSION%.*} PYTHON_DIR=$(dirname $(realpath $(which python$MAJOR))) PYTHON=$PYTHON_DIR/python$MAJOR - git clean -f -f -x -d -q -e .whl + git clean -f -f -x -d -q -e dist WHEEL_DIR=$WHEEL_DIR PYTHON=$PYTHON $SCRIPT_DIR/pr-check.sh done diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 59937506e..6ab7cd896 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -7,7 +7,7 @@ set -e set -x SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -WHEEL_DIR=$SCRIPT_DIR/../.whl +WHEEL_DIR=$SCRIPT_DIR/../dist # Test whether we can install without any dependencies # TODO: delete once we have setup.py setup correctly @@ -17,7 +17,7 @@ for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do PYTHON=/opt/python/${VERSION}/bin/python3 - git clean -f -f -x -d -q -e .whl + git clean -f -f -x -d -q -e dist $PYTHON -m venv venv source venv/bin/activate @@ -33,7 +33,7 @@ done # we can install brainiak without installing mpich, but require mpiexec during tests ./bin/install-test-deps-manylinux1.sh for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do - git clean -f -f -x -d -q -e .whl + git clean -f -f -x -d -q -e dist mpi_command=mpiexec.hydra \ WHEEL_DIR=$WHEEL_DIR \ PYTHON=/opt/python/$VERSION/bin/python \ From b22640dc00609542c5f25f7611c3a37b88c7c4e2 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 07:15:07 -0500 Subject: [PATCH 093/126] Comment out pypi upload; update conditional stages --- .travis.yml | 20 ++++++++------------ bin/README.md | 9 ++++----- bin/generate-yaml.py | 14 ++++++++------ 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index afe8d2322..f2a2ced07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,7 +82,8 @@ jobs: - stage: build language: generic if: branch = master and repo = danielsuo/brainiak - - os: linux + - if: branch = master and repo = danielsuo/brainiak + os: linux dist: trusty sudo: required language: python @@ -93,8 +94,6 @@ jobs: script: - ./bin/test-wheels.sh - ./bin/build-dist.sh - after_script: - - twine upload dist/* deploy: - provider: s3 access_key_id: AKIAJKBW6H2VKKQDHILQ @@ -109,7 +108,8 @@ jobs: 'on': branch: master repo: danielsuo/brainiak - - os: osx + - if: branch = master and repo = danielsuo/brainiak + os: osx osx_image: xcode7.3 sudo: required language: generic @@ -149,9 +149,8 @@ jobs: - unset ARCHFLAGS script: - VERSIONS="3.4.4" ./bin/test-wheels-macos.sh - after_script: - - twine upload dist/* - - os: osx + - if: branch = master and repo = danielsuo/brainiak + os: osx osx_image: xcode7.3 sudo: required language: generic @@ -191,9 +190,8 @@ jobs: - unset ARCHFLAGS script: - VERSIONS="3.5.3" ./bin/test-wheels-macos.sh - after_script: - - twine upload dist/* - - os: osx + - if: branch = master and repo = danielsuo/brainiak + os: osx osx_image: xcode7.3 sudo: required language: generic @@ -233,5 +231,3 @@ jobs: - unset ARCHFLAGS script: - VERSIONS="3.6.0" ./bin/test-wheels-macos.sh - after_script: - - twine upload dist/* diff --git a/bin/README.md b/bin/README.md index 93892330d..4dedb4e58 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,11 +1,6 @@ # Wheel to-dos # TODO -- Fix stages; it appears that everything just gets lumped into a single stage -- Upload wheels to PyPI (uncomment) -- Upload to TestPyPI -- Upload wheels to PyPI (uncomment) -- Only upload to PyPI on tagged master - Recreate and encrypt keys for brainiak/brainiak - Only need to build mpi4py occasionally; not with every commit, and not with every job - mpi4py should always commit to s3://brainiak/.whl @@ -20,6 +15,10 @@ - Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8)) # To verify +- Fix stages; it appears that everything just gets lumped into a single stage +- Upload to TestPyPI +- Upload wheels to PyPI (uncomment) +- Only upload to PyPI on tagged master # Completed - Test-with-deps: test source with dependencies + can be installed (doesn't check install now) diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index b3df636f3..0b2e1b2d1 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -140,6 +140,7 @@ def dict_constructor(loader, node): })] jobs.append(OrderedDict({ + 'if': 'branch = master and repo = %s' % repo, 'os': 'linux', 'dist': 'trusty', 'sudo': 'required', @@ -153,9 +154,9 @@ def dict_constructor(loader, node): './bin/test-wheels.sh', './bin/build-dist.sh' ], - 'after_script': [ - 'twine upload dist/*' - ], + # 'after_script': [ + # 'twine upload dist/*' + # ], 'deploy': deploy_s3 })) @@ -166,6 +167,7 @@ def dict_constructor(loader, node): 'ARCHFLAGS="-arch x86_64"' ]) build_macos = OrderedDict({ + 'if': 'branch = master and repo = %s' % repo, 'os': 'osx', 'osx_image': 'xcode7.3', 'sudo': 'required', @@ -200,9 +202,9 @@ def dict_constructor(loader, node): 'VERSIONS="%s" ./bin/test-wheels-macos.sh' % version ] - block['after_script'] = [ - 'twine upload dist/*' - ] + # block['after_script'] = [ + # 'twine upload dist/*' + # ] block['deploy'] = copy.deepcopy(deploy_s3) From eec8321d0baabbdc74cee1bf08f5f10700f22d93 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 08:27:54 -0500 Subject: [PATCH 094/126] Fix change in repo --- .travis.yml | 35 ++++++++----- bin/generate-yaml.py | 94 ++++++++++++++++++++++++++++++----- bin/pr-check.sh | 11 +++- bin/test-wheels-macos.sh | 55 +++++++++++--------- bin/test-wheels-manylinux1.sh | 52 +++++++++++-------- setup.py | 3 ++ 6 files changed, 180 insertions(+), 70 deletions(-) diff --git a/.travis.yml b/.travis.yml index f2a2ced07..11e7e5491 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,8 +92,8 @@ jobs: install: - python3 -m pip install -U pip twine script: - - ./bin/test-wheels.sh - ./bin/build-dist.sh + - ./bin/test-wheels.sh deploy: - provider: s3 access_key_id: AKIAJKBW6H2VKKQDHILQ @@ -120,6 +120,10 @@ jobs: - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypiARCHFLAGS="-arch x86_64" + before_install: + - brew update + - brew install llvm mpich python3 + - python3 -m pip install --user twine deploy: - provider: s3 access_key_id: AKIAJKBW6H2VKKQDHILQ @@ -134,11 +138,8 @@ jobs: 'on': branch: master repo: danielsuo/brainiak - before_install: - - brew update - - brew install llvm mpich python3 - - python3 -m pip install --user twineVERSIONS="3.4.4" ./bin/install-python-macos.sh install: + - VERSIONS="3.4.4" ./bin/install-python-macos.sh - VERSIONS="3.4.4" ./bin/build-dist-macos.sh before_script: - brew uninstall -f --ignore-dependencies llvm mpich @@ -161,6 +162,10 @@ jobs: - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypiARCHFLAGS="-arch x86_64" + before_install: + - brew update + - brew install llvm mpich python3 + - python3 -m pip install --user twine deploy: - provider: s3 access_key_id: AKIAJKBW6H2VKKQDHILQ @@ -175,11 +180,8 @@ jobs: 'on': branch: master repo: danielsuo/brainiak - before_install: - - brew update - - brew install llvm mpich python3 - - python3 -m pip install --user twineVERSIONS="3.5.3" ./bin/install-python-macos.sh install: + - VERSIONS="3.5.3" ./bin/install-python-macos.sh - VERSIONS="3.5.3" ./bin/build-dist-macos.sh before_script: - brew uninstall -f --ignore-dependencies llvm mpich @@ -202,6 +204,10 @@ jobs: - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypiARCHFLAGS="-arch x86_64" + before_install: + - brew update + - brew install llvm mpich python3 + - python3 -m pip install --user twine deploy: - provider: s3 access_key_id: AKIAJKBW6H2VKKQDHILQ @@ -216,11 +222,8 @@ jobs: 'on': branch: master repo: danielsuo/brainiak - before_install: - - brew update - - brew install llvm mpich python3 - - python3 -m pip install --user twineVERSIONS="3.6.0" ./bin/install-python-macos.sh install: + - VERSIONS="3.6.0" ./bin/install-python-macos.sh - VERSIONS="3.6.0" ./bin/build-dist-macos.sh before_script: - brew uninstall -f --ignore-dependencies llvm mpich @@ -231,3 +234,9 @@ jobs: - unset ARCHFLAGS script: - VERSIONS="3.6.0" ./bin/test-wheels-macos.sh + - stage: testpypi + language: generic + if: branch = master and repo = danielsuo/brainiak + - stage: pypi + language: generic + if: branch = master and repo = danielsuo/brainiak diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 0b2e1b2d1..d8d4f0f19 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -139,7 +139,7 @@ def dict_constructor(loader, node): } })] -jobs.append(OrderedDict({ +build_linux = OrderedDict({ 'if': 'branch = master and repo = %s' % repo, 'os': 'linux', 'dist': 'trusty', @@ -151,15 +151,16 @@ def dict_constructor(loader, node): 'python3 -m pip install -U pip twine' ], 'script': [ - './bin/test-wheels.sh', - './bin/build-dist.sh' + './bin/build-dist.sh', + './bin/test-wheels.sh' ], # 'after_script': [ - # 'twine upload dist/*' + # 'twine upload dist/*' # ], 'deploy': deploy_s3 -})) +}) +jobs.append(build_linux) build_macos_env = copy.deepcopy(test_macos['env']) build_macos_env.extend([ @@ -173,19 +174,19 @@ def dict_constructor(loader, node): 'sudo': 'required', 'language': 'generic', 'env': build_macos_env, + 'before_install': [ + 'brew update', + 'brew install llvm mpich python3', + 'python3 -m pip install --user twine' + ], 'deploy': deploy_s3 }) for version in versions: block = copy.deepcopy(build_macos) - block['before_install'] = [ - 'brew update', - 'brew install llvm mpich python3', - 'python3 -m pip install --user twine' - 'VERSIONS="%s" ./bin/install-python-macos.sh' % version - ] block['install'] = [ + 'VERSIONS="%s" ./bin/install-python-macos.sh' % version, 'VERSIONS="%s" ./bin/build-dist-macos.sh' % version ] @@ -203,12 +204,81 @@ def dict_constructor(loader, node): ] # block['after_script'] = [ - # 'twine upload dist/*' + # 'twine upload dist/*' # ] block['deploy'] = copy.deepcopy(deploy_s3) jobs.append(block) +# Test PyPI install +jobs.append(OrderedDict({ + 'stage': 'testpypi', + 'language': 'generic', + 'if': 'branch = master and repo = %s' % repo +})) + +testpypi_linux = copy.deepcopy(build_linux) +testpypi_linux.pop('deploy', None) +testpypi_linux['env'] = [ + 'TWINE_REPOSITORY_URL=https://pypi.python.org/pypi', + 'PYPI_REPOSITORY_URL=https://testpypi.python.org' +] +testpypi_linux['script'] = './bin/test-wheels.sh' + +# jobs.append(testpypi_linux) + +testpypi_macos = copy.deepcopy(build_macos) +testpypi_macos.pop('deploy', None) +testpypi_macos['env'] = [ + 'TWINE_REPOSITORY_URL=https://pypi.python.org/pypi', + 'PYPI_REPOSITORY_URL=https://testpypi.python.org' +] + +for version in versions: + block = copy.deepcopy(testpypi_macos) + block['install'] = [ + 'VERSIONS="%s" ./bin/install-python-macos.sh' % version + ] + + block['script'] = [ + 'VERIONS="%s" ./bin/test-wheels-macos.sh' % version + ] + + # jobs.append(block) + +# Test PyPI install +jobs.append(OrderedDict({ + 'stage': 'pypi', + 'language': 'generic', + 'if': 'branch = master and repo = %s' % repo +})) + +pypi_linux = copy.deepcopy(testpypi_linux) +pypi_linux.pop('after_script', None) +pypi_linux['env'] = [ + 'PYPI_REPOSITORY_URL=https://testpypi.python.org' +] + +# jobs.append(pypi_linux) + +pypi_macos = copy.deepcopy(testpypi_macos) +pypi_macos.pop('after_script', None) +pypi_macos['env'] = [ + 'PYPI_REPOSITORY_URL=https://testpypi.python.org' +] + +for version in versions: + block = copy.deepcopy(pypi_macos) + block['install'] = [ + 'VERSIONS="%s" ./bin/install-python-macos.sh' % version + ] + + block['script'] = [ + 'VERIONS="%s" ./bin/test-wheels-macos.sh' % version + ] + + # jobs.append(block) + with open(travis, 'w') as yml: yaml.dump(data, yml, default_flow_style=False) diff --git a/bin/pr-check.sh b/bin/pr-check.sh index 6044329e0..a08dd161f 100755 --- a/bin/pr-check.sh +++ b/bin/pr-check.sh @@ -94,7 +94,7 @@ else fi # Check if running in an sdist -if [ ! -z WHEEL_DIR ] +if [ ! -z $WHEEL_DIR ] || [ ! -z $PYPI_REPOSITORY_URL ] then dist_mode='--bdist-mode' elif git ls-files --error-unmatch pr-check.sh 2> /dev/null @@ -119,7 +119,8 @@ $activate_venv $venv || { # we install it first here to check that installation succeeds without the # developer dependencies. -if [ ! -z $WHEEL_DIR ]; then +if [ ! -z $WHEEL_DIR ] +then MAJOR=$(python3 -c "import sys; v = sys.version_info; print(v[0] * 10 + v[1])") MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep cp$MAJOR | grep mpi4py) BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep cp$MAJOR | grep brainiak) @@ -129,6 +130,12 @@ if [ ! -z $WHEEL_DIR ]; then # We don't want to install brainiak from source { echo $BRAINIAK_WHEEL; tail -n +2 requirements-dev.txt; } | python3 -m pip install -q -r /dev/stdin +elif [ ! -z $PYPI_REPOSITORY_URL ] +then + python3 -m pip install -q -i $PYPI_REPOSITORY_URL brainiak + + tail -n +2 requirements-dev.txt | python3 -m pip install --index-url $PYPI_REPOSITORY_URL \ + --extra-index-url="https://pypi.python.org/pypi" brainiak -q -r /dev/stdin else python3 -m pip install $ignore_installed -U -e . || \ exit_with_error_and_venv "Failed to install BrainIAK." diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 1777a2b85..4c1c97ac5 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -6,35 +6,38 @@ set -e # Show explicitly which commands are currently running. set -x -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -WHEEL_DIR=$SCRIPT_DIR/../dist +if [ -z $PYPI_REPOSITORY_URL ] +then + SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + WHEEL_DIR=$SCRIPT_DIR/../dist -# Test whether we can install without any dependencies -# TODO: delete once we have setup.py setup correctly -for VERSION in $VERSIONS -do - MAJOR=${VERSION%.*} - PYTHON_DIR=$(dirname $(realpath $(which python$MAJOR))) - PYTHON=$PYTHON_DIR/python$MAJOR - WHEEL_VERSION=$(echo $MAJOR | tr -d '.') + # Test whether we can install without any dependencies + # TODO: delete once we have setup.py setup correctly + for VERSION in $VERSIONS + do + MAJOR=${VERSION%.*} + PYTHON_DIR=$(dirname $(realpath $(which python$MAJOR))) + PYTHON=$PYTHON_DIR/python$MAJOR + WHEEL_VERSION=$(echo $MAJOR | tr -d '.') - git clean -f -f -x -d -q -e dist + git clean -f -f -x -d -q -e dist - $PYTHON -m venv venv - source venv/bin/activate + $PYTHON -m venv venv + source venv/bin/activate - # Find the appropriate wheel by grepping for the Python version. - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep mpi4py) + # Find the appropriate wheel by grepping for the Python version. + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep mpi4py) - # TODO: this will actually pick up both wheels since brainiak is in the path - BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep brainiak) + # TODO: this will actually pick up both wheels since brainiak is in the path + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep brainiak) - $PYTHON -m pip install $MPI4PY_WHEEL - $PYTHON -m pip install $BRAINIAK_WHEEL + $PYTHON -m pip install $MPI4PY_WHEEL + $PYTHON -m pip install $BRAINIAK_WHEEL - deactivate - rm -rf venv -done + deactivate + rm -rf venv + done +fi brew install mpich @@ -46,5 +49,11 @@ do PYTHON_DIR=$(dirname $(realpath $(which python$MAJOR))) PYTHON=$PYTHON_DIR/python$MAJOR git clean -f -f -x -d -q -e dist - WHEEL_DIR=$WHEEL_DIR PYTHON=$PYTHON $SCRIPT_DIR/pr-check.sh + + if [ -z $PYPI_REPOSITORY_URL ] + then + WHEEL_DIR=$WHEEL_DIR PYTHON=$PYTHON $SCRIPT_DIR/pr-check.sh + else + PYTHON=$PYTHON $SCRIPT_DIR/pr-check.sh + fi done diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index 6ab7cd896..dea913ec3 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -6,36 +6,48 @@ set -e # Show explicitly which commands are currently running. set -x -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -WHEEL_DIR=$SCRIPT_DIR/../dist +# If we're downloading from repository, skip +if [ -z $PYPI_REPOSITORY_URL ] +then + SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + WHEEL_DIR=$SCRIPT_DIR/../dist -# Test whether we can install without any dependencies -# TODO: delete once we have setup.py setup correctly -for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep mpi4py) - BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep brainiak) + # Test whether we can install without any dependencies + # TODO: delete once we have setup.py setup correctly + for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep mpi4py) + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep brainiak) - PYTHON=/opt/python/${VERSION}/bin/python3 + PYTHON=/opt/python/${VERSION}/bin/python3 - git clean -f -f -x -d -q -e dist + git clean -f -f -x -d -q -e dist - $PYTHON -m venv venv - source venv/bin/activate + $PYTHON -m venv venv + source venv/bin/activate - $PYTHON -m pip install -q $MPI4PY_WHEEL - $PYTHON -m pip install -q $BRAINIAK_WHEEL + $PYTHON -m pip install -q $MPI4PY_WHEEL + $PYTHON -m pip install -q $BRAINIAK_WHEEL - deactivate - rm -rf venv -done + deactivate + rm -rf venv + done +fi # Separate tests into a separate loop because we want to make sure # we can install brainiak without installing mpich, but require mpiexec during tests ./bin/install-test-deps-manylinux1.sh for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do git clean -f -f -x -d -q -e dist - mpi_command=mpiexec.hydra \ - WHEEL_DIR=$WHEEL_DIR \ - PYTHON=/opt/python/$VERSION/bin/python \ - $SCRIPT_DIR/pr-check.sh + + if [ -z $PYPI_REPOSITORY_URL ] + then + mpi_command=mpiexec.hydra \ + WHEEL_DIR=$WHEEL_DIR \ + PYTHON=/opt/python/$VERSION/bin/python \ + $SCRIPT_DIR/pr-check.sh + else + mpi_command=mpiexec.hydra \ + PYTHON=/opt/python/$VERSION/bin/python \ + $SCRIPT_DIR/pr-check.sh + fi done diff --git a/setup.py b/setup.py index 90357e23e..4427e0fe4 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,9 @@ result = check_call([sys.executable, '-m', 'pip', 'install', local_filename]) + # TODO: Need better clean-up mechanism + os.system('rm -f %s' % local_filename) + if result != 0: sys.exit('ERROR: failed to install mpi4py') From 29cd76d6d8f56d734717a75eaf741f904a84124b Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 10:17:44 -0500 Subject: [PATCH 095/126] Update yaml --- .travis.yml | 9 ++++++--- bin/generate-yaml.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 11e7e5491..1d2aeb2f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -119,7 +119,8 @@ jobs: - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypiARCHFLAGS="-arch x86_64" + - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi + - ARCHFLAGS="-arch x86_64" before_install: - brew update - brew install llvm mpich python3 @@ -161,7 +162,8 @@ jobs: - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypiARCHFLAGS="-arch x86_64" + - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi + - ARCHFLAGS="-arch x86_64" before_install: - brew update - brew install llvm mpich python3 @@ -203,7 +205,8 @@ jobs: - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypiARCHFLAGS="-arch x86_64" + - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi + - ARCHFLAGS="-arch x86_64" before_install: - brew update - brew install llvm mpich python3 diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index d8d4f0f19..e06b63ab1 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -164,7 +164,7 @@ def dict_constructor(loader, node): build_macos_env = copy.deepcopy(test_macos['env']) build_macos_env.extend([ - 'TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi' + 'TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi', 'ARCHFLAGS="-arch x86_64"' ]) build_macos = OrderedDict({ From 2d6c867f071c0446f31e86d3792a0e4d941e4f7a Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 11:24:14 -0500 Subject: [PATCH 096/126] Test pypi credentials --- .travis.yml | 260 +++---------------------------------------- bin/generate-yaml.py | 7 +- 2 files changed, 22 insertions(+), 245 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1d2aeb2f5..13e1cfa09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,245 +1,17 @@ -branches: - only: - - master +os: linux +dist: trusty +python: "3.5" env: - global: - - BRAINIAK_REPO=danielsuo/brainiak -jobs: - include: - - stage: test - language: generic - - os: linux - dist: trusty - sudo: required - language: python - python: '3.4' - install: - - python3 -m pip install -U pip - script: - - ./bin/pr-check.sh - addons: - apt: - packages: - - build-essential libgomp1 libmpich-dev mpich - - os: linux - dist: trusty - sudo: required - language: python - python: '3.5' - install: - - python3 -m pip install -U pip - script: - - ./bin/pr-check.sh - addons: - apt: - packages: - - build-essential libgomp1 libmpich-dev mpich - - os: linux - dist: trusty - sudo: required - language: python - python: '3.6' - install: - - python3 -m pip install -U pip - script: - - ./bin/pr-check.sh - addons: - apt: - packages: - - build-essential libgomp1 libmpich-dev mpich - - os: osx - language: generic - env: - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - before_install: - - brew update - - brew install llvm mpich python3 - install: - - python3 -m pip install -U pip - script: - - ./bin/pr-check.sh - osx_image: xcode7.3 - - os: osx - language: generic - env: - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - before_install: - - brew update - - brew install llvm mpich python3 - install: - - python3 -m pip install -U pip - script: - - ./bin/pr-check.sh - osx_image: xcode8 - - stage: build - language: generic - if: branch = master and repo = danielsuo/brainiak - - if: branch = master and repo = danielsuo/brainiak - os: linux - dist: trusty - sudo: required - language: python - python: '3.4' - env: TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi - install: - - python3 -m pip install -U pip twine - script: - - ./bin/build-dist.sh - - ./bin/test-wheels.sh - deploy: - - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - region: us-east-1 - acl: public_read - local_dir: dist - upload-dir: $TRAVIS_COMMIT/dist - skip_cleanup: true - 'on': - branch: master - repo: danielsuo/brainiak - - if: branch = master and repo = danielsuo/brainiak - os: osx - osx_image: xcode7.3 - sudo: required - language: generic - env: - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi - - ARCHFLAGS="-arch x86_64" - before_install: - - brew update - - brew install llvm mpich python3 - - python3 -m pip install --user twine - deploy: - - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - region: us-east-1 - acl: public_read - local_dir: dist - upload-dir: $TRAVIS_COMMIT/dist - skip_cleanup: true - 'on': - branch: master - repo: danielsuo/brainiak - install: - - VERSIONS="3.4.4" ./bin/install-python-macos.sh - - VERSIONS="3.4.4" ./bin/build-dist-macos.sh - before_script: - - brew uninstall -f --ignore-dependencies llvm mpich - - unset CC - - unset CXX - - unset LDFLAGS - - unset CPPFLAGS - - unset ARCHFLAGS - script: - - VERSIONS="3.4.4" ./bin/test-wheels-macos.sh - - if: branch = master and repo = danielsuo/brainiak - os: osx - osx_image: xcode7.3 - sudo: required - language: generic - env: - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi - - ARCHFLAGS="-arch x86_64" - before_install: - - brew update - - brew install llvm mpich python3 - - python3 -m pip install --user twine - deploy: - - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - region: us-east-1 - acl: public_read - local_dir: dist - upload-dir: $TRAVIS_COMMIT/dist - skip_cleanup: true - 'on': - branch: master - repo: danielsuo/brainiak - install: - - VERSIONS="3.5.3" ./bin/install-python-macos.sh - - VERSIONS="3.5.3" ./bin/build-dist-macos.sh - before_script: - - brew uninstall -f --ignore-dependencies llvm mpich - - unset CC - - unset CXX - - unset LDFLAGS - - unset CPPFLAGS - - unset ARCHFLAGS - script: - - VERSIONS="3.5.3" ./bin/test-wheels-macos.sh - - if: branch = master and repo = danielsuo/brainiak - os: osx - osx_image: xcode7.3 - sudo: required - language: generic - env: - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - - TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi - - ARCHFLAGS="-arch x86_64" - before_install: - - brew update - - brew install llvm mpich python3 - - python3 -m pip install --user twine - deploy: - - provider: s3 - access_key_id: AKIAJKBW6H2VKKQDHILQ - secret_access_key: - secure: A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg= - bucket: brainiak - region: us-east-1 - acl: public_read - local_dir: dist - upload-dir: $TRAVIS_COMMIT/dist - skip_cleanup: true - 'on': - branch: master - repo: danielsuo/brainiak - install: - - VERSIONS="3.6.0" ./bin/install-python-macos.sh - - VERSIONS="3.6.0" ./bin/build-dist-macos.sh - before_script: - - brew uninstall -f --ignore-dependencies llvm mpich - - unset CC - - unset CXX - - unset LDFLAGS - - unset CPPFLAGS - - unset ARCHFLAGS - script: - - VERSIONS="3.6.0" ./bin/test-wheels-macos.sh - - stage: testpypi - language: generic - if: branch = master and repo = danielsuo/brainiak - - stage: pypi - language: generic - if: branch = master and repo = danielsuo/brainiak + - TWINE_USERNAME="brainiak-bot" + - secure: "BHzjkB2HoIIN8njur7R/2+fBqubStLh+KH4vS4WdNMPdshLBYuFXFPAo3kg0CR72jAQ5+pPwPwQDrVbuA4jz8rXSLjpOxwsu8l4TCE6pcISbAGARgWBxOSDsLazIWyKL83aMqwtViEU9ChCdYHOiQZ6YfvVlgGOK4o8hY9eZrLcKy1H66yuMYTJOSu9ct3lUzxexiffGz1JmdJ6lg6jVB1Ga2pYoS1CPk8gK4bf3hRKCBaIhNEV93KGGNtbXbKBhEtHCtJBrCA3aZ9spJySNMt+6ggKCllDdWq26TfMKIl6Q7koV9idgIXkY+hW+3WlPWx5aVH8lR6cw7f0GOiu4E2c5yFKElnGYCqEPkuL2fcg0y9t64T9wMwpMsOgQHD8RX5DPMpK1Rm83mXU2HZLmf4F/FjLWgZmMXQgjYv/dXIMBkO2Ba6cbfCq3a8G2jUxugZw7UcuQpjGKs6XV9/EprrQnuZjSv4SCp0hCNmb9JVv+hd7R+QDtw7sL8VBMQJxZwtbpGDobTGxFS1hC9rmlVnp0jXUhF99ERahhrERj4IdnNuuVvEWpiN085qT3UPp2WU+9o4Cw5NgVdBDTO5wmaEptGdXqfP+EfxdEfQxJNs2WJOHBOA+EeyNsP3LFQVeYLxYOeDBWFaJLh0P5uhb3c+L5UNuO31NpUBBQ+//3b64=" + - TWINE_REPOSITORY_URL="https://testpypi.python.org/pypi" +install: + - python3 -m pip install -U pip twine +script: + - python3 -m pip install -e . + - python3 -m python setup.py sdist + - twine upload dist/* +addons: + apt: + packages: + - build-essential libgomp1 libmpich-dev mpich diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index e06b63ab1..2760ba063 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -42,6 +42,7 @@ def dict_constructor(loader, node): travis = '.travis.yml' repo = 'danielsuo/brainiak' +twine_username = 'brainiak-bot' versions = ['3.4.4', '3.5.3', '3.6.0'] majors = [version[:-2] for version in versions] @@ -51,6 +52,8 @@ def dict_constructor(loader, node): access_key_id = 'AKIAJKBW6H2VKKQDHILQ' secret_access_key = 'A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg=' +twine_password = 'BHzjkB2HoIIN8njur7R/2+fBqubStLh+KH4vS4WdNMPdshLBYuFXFPAo3kg0CR72jAQ5+pPwPwQDrVbuA4jz8rXSLjpOxwsu8l4TCE6pcISbAGARgWBxOSDsLazIWyKL83aMqwtViEU9ChCdYHOiQZ6YfvVlgGOK4o8hY9eZrLcKy1H66yuMYTJOSu9ct3lUzxexiffGz1JmdJ6lg6jVB1Ga2pYoS1CPk8gK4bf3hRKCBaIhNEV93KGGNtbXbKBhEtHCtJBrCA3aZ9spJySNMt+6ggKCllDdWq26TfMKIl6Q7koV9idgIXkY+hW+3WlPWx5aVH8lR6cw7f0GOiu4E2c5yFKElnGYCqEPkuL2fcg0y9t64T9wMwpMsOgQHD8RX5DPMpK1Rm83mXU2HZLmf4F/FjLWgZmMXQgjYv/dXIMBkO2Ba6cbfCq3a8G2jUxugZw7UcuQpjGKs6XV9/EprrQnuZjSv4SCp0hCNmb9JVv+hd7R+QDtw7sL8VBMQJxZwtbpGDobTGxFS1hC9rmlVnp0jXUhF99ERahhrERj4IdnNuuVvEWpiN085qT3UPp2WU+9o4Cw5NgVdBDTO5wmaEptGdXqfP+EfxdEfQxJNs2WJOHBOA+EeyNsP3LFQVeYLxYOeDBWFaJLh0P5uhb3c+L5UNuO31NpUBBQ+//3b64=' + data = OrderedDict() data['branches'] = OrderedDict({ @@ -59,7 +62,9 @@ def dict_constructor(loader, node): data['env'] = OrderedDict({ 'global': [ - 'BRAINIAK_REPO=%s' % repo + 'BRAINIAK_REPO=%s' % repo, + 'TWINE_USERNAME=%s' % twine_username, + 'secure: %s' % twine_password ] }) From f2b5d9081bded428dc5caf532fdae785bf4858f2 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 11:25:56 -0500 Subject: [PATCH 097/126] Fix matrix --- .travis.yml | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 13e1cfa09..7675a7713 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,19 @@ -os: linux -dist: trusty -python: "3.5" -env: - - TWINE_USERNAME="brainiak-bot" - - secure: "BHzjkB2HoIIN8njur7R/2+fBqubStLh+KH4vS4WdNMPdshLBYuFXFPAo3kg0CR72jAQ5+pPwPwQDrVbuA4jz8rXSLjpOxwsu8l4TCE6pcISbAGARgWBxOSDsLazIWyKL83aMqwtViEU9ChCdYHOiQZ6YfvVlgGOK4o8hY9eZrLcKy1H66yuMYTJOSu9ct3lUzxexiffGz1JmdJ6lg6jVB1Ga2pYoS1CPk8gK4bf3hRKCBaIhNEV93KGGNtbXbKBhEtHCtJBrCA3aZ9spJySNMt+6ggKCllDdWq26TfMKIl6Q7koV9idgIXkY+hW+3WlPWx5aVH8lR6cw7f0GOiu4E2c5yFKElnGYCqEPkuL2fcg0y9t64T9wMwpMsOgQHD8RX5DPMpK1Rm83mXU2HZLmf4F/FjLWgZmMXQgjYv/dXIMBkO2Ba6cbfCq3a8G2jUxugZw7UcuQpjGKs6XV9/EprrQnuZjSv4SCp0hCNmb9JVv+hd7R+QDtw7sL8VBMQJxZwtbpGDobTGxFS1hC9rmlVnp0jXUhF99ERahhrERj4IdnNuuVvEWpiN085qT3UPp2WU+9o4Cw5NgVdBDTO5wmaEptGdXqfP+EfxdEfQxJNs2WJOHBOA+EeyNsP3LFQVeYLxYOeDBWFaJLh0P5uhb3c+L5UNuO31NpUBBQ+//3b64=" - - TWINE_REPOSITORY_URL="https://testpypi.python.org/pypi" -install: - - python3 -m pip install -U pip twine -script: - - python3 -m pip install -e . - - python3 -m python setup.py sdist - - twine upload dist/* -addons: - apt: - packages: - - build-essential libgomp1 libmpich-dev mpich +matrix: + include: + - os: linux + dist: trusty + python: "3.5" + env: + - TWINE_USERNAME="brainiak-bot" + - secure: "BHzjkB2HoIIN8njur7R/2+fBqubStLh+KH4vS4WdNMPdshLBYuFXFPAo3kg0CR72jAQ5+pPwPwQDrVbuA4jz8rXSLjpOxwsu8l4TCE6pcISbAGARgWBxOSDsLazIWyKL83aMqwtViEU9ChCdYHOiQZ6YfvVlgGOK4o8hY9eZrLcKy1H66yuMYTJOSu9ct3lUzxexiffGz1JmdJ6lg6jVB1Ga2pYoS1CPk8gK4bf3hRKCBaIhNEV93KGGNtbXbKBhEtHCtJBrCA3aZ9spJySNMt+6ggKCllDdWq26TfMKIl6Q7koV9idgIXkY+hW+3WlPWx5aVH8lR6cw7f0GOiu4E2c5yFKElnGYCqEPkuL2fcg0y9t64T9wMwpMsOgQHD8RX5DPMpK1Rm83mXU2HZLmf4F/FjLWgZmMXQgjYv/dXIMBkO2Ba6cbfCq3a8G2jUxugZw7UcuQpjGKs6XV9/EprrQnuZjSv4SCp0hCNmb9JVv+hd7R+QDtw7sL8VBMQJxZwtbpGDobTGxFS1hC9rmlVnp0jXUhF99ERahhrERj4IdnNuuVvEWpiN085qT3UPp2WU+9o4Cw5NgVdBDTO5wmaEptGdXqfP+EfxdEfQxJNs2WJOHBOA+EeyNsP3LFQVeYLxYOeDBWFaJLh0P5uhb3c+L5UNuO31NpUBBQ+//3b64=" + - TWINE_REPOSITORY_URL="https://testpypi.python.org/pypi" + install: + - python3 -m pip install -U pip twine + script: + - python3 -m pip install -e . + - python3 -m python setup.py sdist + - twine upload dist/* + addons: + apt: + packages: + - build-essential libgomp1 libmpich-dev mpich From af4e215572553649d99d33e2a8b8b19a47237753 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 11:29:23 -0500 Subject: [PATCH 098/126] Add language key --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 7675a7713..733991938 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ matrix: include: - os: linux dist: trusty + language: python python: "3.5" env: - TWINE_USERNAME="brainiak-bot" From ad1ed69c41caabed724448fa08d6ca5326a53135 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 11:34:17 -0500 Subject: [PATCH 099/126] Silly bug --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 733991938..81c2af618 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ matrix: - python3 -m pip install -U pip twine script: - python3 -m pip install -e . - - python3 -m python setup.py sdist + - python3 setup.py sdist - twine upload dist/* addons: apt: From 740bc47a7debd3f8390ffee28491168ecaadbce3 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 11:41:33 -0500 Subject: [PATCH 100/126] Move virtual environments to directory above current --- .travis.yml | 2 +- bin/build-dist-macos.sh | 4 +++- bin/test-wheels-macos.sh | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 81c2af618..a0160d119 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ matrix: env: - TWINE_USERNAME="brainiak-bot" - secure: "BHzjkB2HoIIN8njur7R/2+fBqubStLh+KH4vS4WdNMPdshLBYuFXFPAo3kg0CR72jAQ5+pPwPwQDrVbuA4jz8rXSLjpOxwsu8l4TCE6pcISbAGARgWBxOSDsLazIWyKL83aMqwtViEU9ChCdYHOiQZ6YfvVlgGOK4o8hY9eZrLcKy1H66yuMYTJOSu9ct3lUzxexiffGz1JmdJ6lg6jVB1Ga2pYoS1CPk8gK4bf3hRKCBaIhNEV93KGGNtbXbKBhEtHCtJBrCA3aZ9spJySNMt+6ggKCllDdWq26TfMKIl6Q7koV9idgIXkY+hW+3WlPWx5aVH8lR6cw7f0GOiu4E2c5yFKElnGYCqEPkuL2fcg0y9t64T9wMwpMsOgQHD8RX5DPMpK1Rm83mXU2HZLmf4F/FjLWgZmMXQgjYv/dXIMBkO2Ba6cbfCq3a8G2jUxugZw7UcuQpjGKs6XV9/EprrQnuZjSv4SCp0hCNmb9JVv+hd7R+QDtw7sL8VBMQJxZwtbpGDobTGxFS1hC9rmlVnp0jXUhF99ERahhrERj4IdnNuuVvEWpiN085qT3UPp2WU+9o4Cw5NgVdBDTO5wmaEptGdXqfP+EfxdEfQxJNs2WJOHBOA+EeyNsP3LFQVeYLxYOeDBWFaJLh0P5uhb3c+L5UNuO31NpUBBQ+//3b64=" - - TWINE_REPOSITORY_URL="https://testpypi.python.org/pypi" + - TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/" install: - python3 -m pip install -U pip twine script: diff --git a/bin/build-dist-macos.sh b/bin/build-dist-macos.sh index 1f2f96647..c8aa053cc 100755 --- a/bin/build-dist-macos.sh +++ b/bin/build-dist-macos.sh @@ -22,8 +22,10 @@ do git clean -f -f -x -d -q -e dist -e mpi4py # Use virtual environments because we want to install later + pushd .. $PYTHON -m venv venv source venv/bin/activate + popd pushd mpi4py git clean -f -f -x -d -q @@ -42,7 +44,7 @@ do fi deactivate - rm -rf venv + rm -rf ../venv done rm -rf mpi4py diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 4c1c97ac5..22f7d469f 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -22,8 +22,10 @@ then git clean -f -f -x -d -q -e dist + pushd .. $PYTHON -m venv venv source venv/bin/activate + popd # Find the appropriate wheel by grepping for the Python version. MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep mpi4py) @@ -35,7 +37,7 @@ then $PYTHON -m pip install $BRAINIAK_WHEEL deactivate - rm -rf venv + rm -rf ../venv done fi From 428a2427b6739d36dd099680b068701fb8768bd1 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 14:15:14 -0500 Subject: [PATCH 101/126] Update environment var --- .travis.yml | 2 +- bin/generate-yaml.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0160d119..a7cd15454 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ matrix: python: "3.5" env: - TWINE_USERNAME="brainiak-bot" - - secure: "BHzjkB2HoIIN8njur7R/2+fBqubStLh+KH4vS4WdNMPdshLBYuFXFPAo3kg0CR72jAQ5+pPwPwQDrVbuA4jz8rXSLjpOxwsu8l4TCE6pcISbAGARgWBxOSDsLazIWyKL83aMqwtViEU9ChCdYHOiQZ6YfvVlgGOK4o8hY9eZrLcKy1H66yuMYTJOSu9ct3lUzxexiffGz1JmdJ6lg6jVB1Ga2pYoS1CPk8gK4bf3hRKCBaIhNEV93KGGNtbXbKBhEtHCtJBrCA3aZ9spJySNMt+6ggKCllDdWq26TfMKIl6Q7koV9idgIXkY+hW+3WlPWx5aVH8lR6cw7f0GOiu4E2c5yFKElnGYCqEPkuL2fcg0y9t64T9wMwpMsOgQHD8RX5DPMpK1Rm83mXU2HZLmf4F/FjLWgZmMXQgjYv/dXIMBkO2Ba6cbfCq3a8G2jUxugZw7UcuQpjGKs6XV9/EprrQnuZjSv4SCp0hCNmb9JVv+hd7R+QDtw7sL8VBMQJxZwtbpGDobTGxFS1hC9rmlVnp0jXUhF99ERahhrERj4IdnNuuVvEWpiN085qT3UPp2WU+9o4Cw5NgVdBDTO5wmaEptGdXqfP+EfxdEfQxJNs2WJOHBOA+EeyNsP3LFQVeYLxYOeDBWFaJLh0P5uhb3c+L5UNuO31NpUBBQ+//3b64=" + - secure: "Ptttm+7EYhN6FiusRQV4lWeAh/48lghaotSvmDw/PLCh69Vqq7YnM6qLS+r+K99qvJLctHj+jPX3TT5xYZhcLD8S5WWq04N2fQyxd23Yr8cL0QeFHE1KeDQqOpcoYW1eMEHx2Xv6YHOrGlQIvi4FCL6T/jEXORfQGwjn4UxCtSU1eP6FLsSdBq3hFnV7hwK9rywzCiWacCahIgiWx6n1qt/Qj3FFbBZmsbBvH1lJdagj1wwCMEwSS57fKq1j1LcJRFzA97jG+mYJHEEAhj6Eyuc7m29hDhFCnbYZ8N+LHBmuSS1zweVgKignBEz2cpEvEtB0eTA/n+SAqbrRPzxYIcHA3wTTz9Cr1HDMNdYox2ZBoWSECPWv+CDrts/s2WFbC8UpClQ67BPctKwQNLuQhf4ztWu2QclDjtND1RBMeoZlwZeCcGhJaDLPtIuiaZRqvh0O8a2Kyq+eJs62zNu/B8VXd6e8WcRdodh4uNc7UOJBwnYk9bmWuP28rATaLIjdFuVtWMlTHO8TyBjqFgeB61SbONBqD7pvv9Oxjrf/wEjifZSoRivoVtfqS231UB+1m2VdI6Ow28G9SOkjaFHkrL/WaoUhWL4oedbpoKwrpWK/qnnFO19J1G5M6p/Zsw7iAiawdkoEU+w01nG/7PdX9DpbLtWghxhd+e3t9WS+9+I=" - TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/" install: - python3 -m pip install -U pip twine diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 2760ba063..fbca83c07 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -151,7 +151,7 @@ def dict_constructor(loader, node): 'sudo': 'required', 'language': 'python', 'python': '3.4', - 'env': 'TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi', + 'env': 'TWINE_REPOSITORY_URL=https://test.pypi.org/legacy', 'install': [ 'python3 -m pip install -U pip twine' ], @@ -169,7 +169,7 @@ def dict_constructor(loader, node): build_macos_env = copy.deepcopy(test_macos['env']) build_macos_env.extend([ - 'TWINE_REPOSITORY_URL=https://testpypi.python.org/pypi', + 'TWINE_REPOSITORY_URL=https://test.pypi.org/legacy', 'ARCHFLAGS="-arch x86_64"' ]) build_macos = OrderedDict({ From 6485784c8d0d9d4fea11dda9ef7988cd60f7bd0d Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 14:25:01 -0500 Subject: [PATCH 102/126] Update environment var --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a7cd15454..cf86fc58f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ matrix: python: "3.5" env: - TWINE_USERNAME="brainiak-bot" - - secure: "Ptttm+7EYhN6FiusRQV4lWeAh/48lghaotSvmDw/PLCh69Vqq7YnM6qLS+r+K99qvJLctHj+jPX3TT5xYZhcLD8S5WWq04N2fQyxd23Yr8cL0QeFHE1KeDQqOpcoYW1eMEHx2Xv6YHOrGlQIvi4FCL6T/jEXORfQGwjn4UxCtSU1eP6FLsSdBq3hFnV7hwK9rywzCiWacCahIgiWx6n1qt/Qj3FFbBZmsbBvH1lJdagj1wwCMEwSS57fKq1j1LcJRFzA97jG+mYJHEEAhj6Eyuc7m29hDhFCnbYZ8N+LHBmuSS1zweVgKignBEz2cpEvEtB0eTA/n+SAqbrRPzxYIcHA3wTTz9Cr1HDMNdYox2ZBoWSECPWv+CDrts/s2WFbC8UpClQ67BPctKwQNLuQhf4ztWu2QclDjtND1RBMeoZlwZeCcGhJaDLPtIuiaZRqvh0O8a2Kyq+eJs62zNu/B8VXd6e8WcRdodh4uNc7UOJBwnYk9bmWuP28rATaLIjdFuVtWMlTHO8TyBjqFgeB61SbONBqD7pvv9Oxjrf/wEjifZSoRivoVtfqS231UB+1m2VdI6Ow28G9SOkjaFHkrL/WaoUhWL4oedbpoKwrpWK/qnnFO19J1G5M6p/Zsw7iAiawdkoEU+w01nG/7PdX9DpbLtWghxhd+e3t9WS+9+I=" + - secure: "YDzVN9IjAu7kEAPaIDSQi48cGYV8HoQQJpb4ZzGeTA6lfs5OmpQ4QF6Bte7FUpMU5/ibNVaEG+uihk70/3rVCl+hWxSYo7MlvS4viYbZCUE5suQnT0ncasGYYoqdZCroMuTmt4reSn3vThWKPEtURUVac4nb+F0VpNSGhFH9PFwu/7b4Ph37N5ge2KXrjQBEKLc8Q71LlnHASCROjkMcG+6tAOA3kVERInZ/U3gRcjdzekzpLm+YDduKQqSkqNTAus4mNXa5baLGVcq5gvCXZxTntx6GCSBCLbzG8tj8VniCgNkDi6BsQzqdI1rJQI1I7DUxzyIf6OEBoBA/VCl5e+ZxCRGS7mRoMKioQcORLpUGEYF0nY8RrG3C6cxWF9LXqr44HuyYyS8EByn0tjZYgljBEsAxI15FvBibsVpWza9lHksj0sXolBenf6DQCDJlt4V26mbafvRkkJyV2Xcaks7/YfKngxawanRpl5pvzkjl21k7GKWLr3i4gs1Ammbm6Cr3n505Plae/fp+wZCkiHv3GA2cznvPumKaB7VNVWMvr/CClbY5Qe7vY8ZSoChV6V0RSzEW0+h7F+80kCGvA2QhJ4UdiSkYB3+dmGvC2nFLD/TIpjQn8mLbYlRfTwdnbOp1J0eVd+xqPlMY8HmfndYMzmQc8FteNcI7oluZlbU=" - TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/" install: - python3 -m pip install -U pip twine From 3acd919e31408fcf37743d86b3c8fd4a6795ed02 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 14:32:03 -0500 Subject: [PATCH 103/126] Update environment var --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cf86fc58f..3d1fee6c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ matrix: python: "3.5" env: - TWINE_USERNAME="brainiak-bot" - - secure: "YDzVN9IjAu7kEAPaIDSQi48cGYV8HoQQJpb4ZzGeTA6lfs5OmpQ4QF6Bte7FUpMU5/ibNVaEG+uihk70/3rVCl+hWxSYo7MlvS4viYbZCUE5suQnT0ncasGYYoqdZCroMuTmt4reSn3vThWKPEtURUVac4nb+F0VpNSGhFH9PFwu/7b4Ph37N5ge2KXrjQBEKLc8Q71LlnHASCROjkMcG+6tAOA3kVERInZ/U3gRcjdzekzpLm+YDduKQqSkqNTAus4mNXa5baLGVcq5gvCXZxTntx6GCSBCLbzG8tj8VniCgNkDi6BsQzqdI1rJQI1I7DUxzyIf6OEBoBA/VCl5e+ZxCRGS7mRoMKioQcORLpUGEYF0nY8RrG3C6cxWF9LXqr44HuyYyS8EByn0tjZYgljBEsAxI15FvBibsVpWza9lHksj0sXolBenf6DQCDJlt4V26mbafvRkkJyV2Xcaks7/YfKngxawanRpl5pvzkjl21k7GKWLr3i4gs1Ammbm6Cr3n505Plae/fp+wZCkiHv3GA2cznvPumKaB7VNVWMvr/CClbY5Qe7vY8ZSoChV6V0RSzEW0+h7F+80kCGvA2QhJ4UdiSkYB3+dmGvC2nFLD/TIpjQn8mLbYlRfTwdnbOp1J0eVd+xqPlMY8HmfndYMzmQc8FteNcI7oluZlbU=" + - secure: "HEqZQD3+9qAaFXrwczgTeCyaqiMpzJW2fyyEQh09x2bRsBrkfV5I5nnQLiL9bb8CbFGrYHLqcHAgUElQbneHk3dHuvw0eHy5bpJVMJFqQKKi0ycZscMG5Y5RdE0BIxiQJ6HIGy4crEs6T/TcpScIbtxZA43zUoeYBpp/C8hrAhkgSqUsDvvQVRYK4o/dt4h6EecuqcCUDe5ERF0K78GJx/2x4Yg7uYO6DcFemqo72zxNs24fjYpwU0G0mIznFCc13ZgZu0eJBLq1/LK9QhALj8cCgkVytB/ZzJe6Z7qxQxoVHaSHOs3fPmamGQ5VOLwKlwm4IGjKQ8781nPd3UTHiRtkO+QvXq5nCE1i5Mk9OJSEjFcw98SHeqLJPkOLW9+a2fagsOwM2Y6OPnWrz/Wk7P8tn2WefnHQImmrX1atryCaFOzblIip8PNWA25anxBMs5iSBW/8uSMKA08Hi7HrSVuvqIOC/NC2G6KUt8Yka1yDNkNzkL+i3rOzNeqBbcRkXezsMxEU6Dr4w/9VYpfyscU+pVBuicOsq8n99bDigPwHRPBRMuRwdQczlQXLYhXbQYWY3mtlukyhvIaDJXKaJiJAWKZcsWDx12HnixvbhETzp22cmMadvw5TKLaafBTwoUdTJXCNBD3fxdvZpw3nOmbvH2Phc95ubRVXw77pqpI=" - TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/" install: - python3 -m pip install -U pip twine From 46132256ec8ec6b5827fd72fee79735c9970021d Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 14:37:47 -0500 Subject: [PATCH 104/126] Update environment var --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3d1fee6c0..a057a8875 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ matrix: python: "3.5" env: - TWINE_USERNAME="brainiak-bot" - - secure: "HEqZQD3+9qAaFXrwczgTeCyaqiMpzJW2fyyEQh09x2bRsBrkfV5I5nnQLiL9bb8CbFGrYHLqcHAgUElQbneHk3dHuvw0eHy5bpJVMJFqQKKi0ycZscMG5Y5RdE0BIxiQJ6HIGy4crEs6T/TcpScIbtxZA43zUoeYBpp/C8hrAhkgSqUsDvvQVRYK4o/dt4h6EecuqcCUDe5ERF0K78GJx/2x4Yg7uYO6DcFemqo72zxNs24fjYpwU0G0mIznFCc13ZgZu0eJBLq1/LK9QhALj8cCgkVytB/ZzJe6Z7qxQxoVHaSHOs3fPmamGQ5VOLwKlwm4IGjKQ8781nPd3UTHiRtkO+QvXq5nCE1i5Mk9OJSEjFcw98SHeqLJPkOLW9+a2fagsOwM2Y6OPnWrz/Wk7P8tn2WefnHQImmrX1atryCaFOzblIip8PNWA25anxBMs5iSBW/8uSMKA08Hi7HrSVuvqIOC/NC2G6KUt8Yka1yDNkNzkL+i3rOzNeqBbcRkXezsMxEU6Dr4w/9VYpfyscU+pVBuicOsq8n99bDigPwHRPBRMuRwdQczlQXLYhXbQYWY3mtlukyhvIaDJXKaJiJAWKZcsWDx12HnixvbhETzp22cmMadvw5TKLaafBTwoUdTJXCNBD3fxdvZpw3nOmbvH2Phc95ubRVXw77pqpI=" + - secure: "JWCpO0caWN4M0xaGPqKMPPHg8WB6sybPQPcCFtXc+dwsD3mu+mDv7lGV3l/5sRoDqIi87uhXBtfnPVKao7AdPaLZoZc8ktBP3b7QwG49U479odN1OuxZcdLBmQBPZlAnqyH7wbM0xhkskr6Yl8gkKWynDfyod1GWbr/plcGRuX82unvuq5tkX8gNSdJ4vHbHGGcvgUbDkyEJdQz7y96qJn7VUpD4FhujO7bGyGVB2ZVcAcEyHuQZiyPxknrXSE0dnk0IeEH77DY40SLpPf5WG4fl/gGYLlccw2ZmBOAOYYE/nq9HF+/b5gnYp+0zLUCgA1tK9uC1UdT3IE24hr8XJMdMIog1BSVjv5k31RjFi0mwCVaasiCN/IOYvwrg+yod5UjXBag7f/Wjo3AEoUSByFERTr+Y7Gs+uCHQKT8Jw4mCTwmI+3JnH+4ZcuMbIQUr10uaocmpNuWhA1IVMTewIKjaAgY9qWy2X5e7X5/faYXlUhV68QiA5TB4MCvA+jeq84+bIk00HBnOadL/iAKFVhBlbkqe8OZWNbsBKFERWEVJFIoOLMg43NORWRs54Mi3eie4bNxWUwwPsxHR4FoBSeWjm2yorDbUIFDK57xZH3jUFu11xSXW3x9BqGWdgQg71j/P1nVKGMoHvPUBX0wO2CLQnSl6katSwgM6aLeJl1U=" - TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/" install: - python3 -m pip install -U pip twine From 72c7d7ffb7e885343532cdbec79f0f1e0526b20a Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 14:48:02 -0500 Subject: [PATCH 105/126] Add version string --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4427e0fe4..4074ee4ec 100644 --- a/setup.py +++ b/setup.py @@ -150,7 +150,8 @@ def finalize_options(self): setup( name='brainiak', - use_scm_version=True, + # use_scm_version=True, + version='0.2.1', setup_requires=[ 'cython', 'numpy', From ac47db28834b298efc0295d5e97a5a8e9a4dfb75 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 15:27:20 -0500 Subject: [PATCH 106/126] Full test --- .travis.yml | 325 +++++++++++++++++++++++++++++++++++++++++-- bin/README.md | 4 + bin/generate-yaml.py | 127 ++++++++++++----- setup.py | 4 +- 4 files changed, 412 insertions(+), 48 deletions(-) diff --git a/.travis.yml b/.travis.yml index a057a8875..6c7d57c69 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,323 @@ -matrix: +branches: + only: + - master +env: + global: + - BRAINIAK_REPO=danielsuo/brainiak + - TWINE_USERNAME=brainiak-bot + - 'secure: "JWCpO0caWN4M0xaGPqKMPPHg8WB6sybPQPcCFtXc+dwsD3mu+mDv7lGV3l/5sRoDqIi87uhXBtfnPVKao7AdPaLZoZc8ktBP3b7QwG49U479odN1OuxZcdLBmQBPZlAnqyH7wbM0xhkskr6Yl8gkKWynDfyod1GWbr/plcGRuX82unvuq5tkX8gNSdJ4vHbHGGcvgUbDkyEJdQz7y96qJn7VUpD4FhujO7bGyGVB2ZVcAcEyHuQZiyPxknrXSE0dnk0IeEH77DY40SLpPf5WG4fl/gGYLlccw2ZmBOAOYYE/nq9HF+/b5gnYp+0zLUCgA1tK9uC1UdT3IE24hr8XJMdMIog1BSVjv5k31RjFi0mwCVaasiCN/IOYvwrg+yod5UjXBag7f/Wjo3AEoUSByFERTr+Y7Gs+uCHQKT8Jw4mCTwmI+3JnH+4ZcuMbIQUr10uaocmpNuWhA1IVMTewIKjaAgY9qWy2X5e7X5/faYXlUhV68QiA5TB4MCvA+jeq84+bIk00HBnOadL/iAKFVhBlbkqe8OZWNbsBKFERWEVJFIoOLMg43NORWRs54Mi3eie4bNxWUwwPsxHR4FoBSeWjm2yorDbUIFDK57xZH3jUFu11xSXW3x9BqGWdgQg71j/P1nVKGMoHvPUBX0wO2CLQnSl6katSwgM6aLeJl1U="' + - AWS_ACCESS_KEY_ID="AKIAJKBW6H2VKKQDHILQ" + - 'secure: "gCGZuvBQVZ09rmH5tPUn3GT17NzyBX9JRO3SJwq6WcLbmKDRo50xAhqib/VAKfMXmcteaIPQdxjWUcohPXAoFQ67Z3Jx8abjxEIJ6t7IwIH3aRYWh7PX28Iu5Cr3b/pLbSBw0r+tj/X0mHsbS1dF/uVOBo4fSGN5f4li3EMv+iWQNimwC64uT+ywbzjdUPwJBZ+rVoyh/uxBR/14CpYMi2b1Bf79iVYZr1INShWH+4JNqw6RkJQVhQbAXq9CdSH3bNgyrnASno542cDa9l++xJhJ9okFTdWchiDj1FH4BhBLdEQHVEm790dAZGcE41APg8gW/4hDA8G5wXrCN/T+QtEEgzIDKJ9o/VImabjWcuB1oMERM4Rcvr0wpRNd+DCcPNyEpLPHlOV8WGiIu/5cgLv7O8n38+LmyjCNeJKKR9TxcCoZ53LhrwHgfcokElz1aXQvHcNh+IAG0WF0+OxclBiqEwvasyq8PT4r9MMNYwklINcSDO5RdK2xVJJbehFT81LodAJ9ydgrfagAB7haLq6JfdIucvlllmeVyLwznJm/Gtx9WNOFeSVEN0pufJWsiQJelu4S+4OesfdCCzypAXkSWQ2M4vQBcWNYjJ6Ns6svhNX/9wC9+QLPusAieLan2WDFjxrQwUM7n+J3/GxEEPI6heUGNh3n0wimmVpGxZo="' + - AWS_DEFAULT_REGION="us-east-1" +jobs: include: + - stage: test + language: generic - os: linux dist: trusty + sudo: required language: python - python: "3.5" - env: - - TWINE_USERNAME="brainiak-bot" - - secure: "JWCpO0caWN4M0xaGPqKMPPHg8WB6sybPQPcCFtXc+dwsD3mu+mDv7lGV3l/5sRoDqIi87uhXBtfnPVKao7AdPaLZoZc8ktBP3b7QwG49U479odN1OuxZcdLBmQBPZlAnqyH7wbM0xhkskr6Yl8gkKWynDfyod1GWbr/plcGRuX82unvuq5tkX8gNSdJ4vHbHGGcvgUbDkyEJdQz7y96qJn7VUpD4FhujO7bGyGVB2ZVcAcEyHuQZiyPxknrXSE0dnk0IeEH77DY40SLpPf5WG4fl/gGYLlccw2ZmBOAOYYE/nq9HF+/b5gnYp+0zLUCgA1tK9uC1UdT3IE24hr8XJMdMIog1BSVjv5k31RjFi0mwCVaasiCN/IOYvwrg+yod5UjXBag7f/Wjo3AEoUSByFERTr+Y7Gs+uCHQKT8Jw4mCTwmI+3JnH+4ZcuMbIQUr10uaocmpNuWhA1IVMTewIKjaAgY9qWy2X5e7X5/faYXlUhV68QiA5TB4MCvA+jeq84+bIk00HBnOadL/iAKFVhBlbkqe8OZWNbsBKFERWEVJFIoOLMg43NORWRs54Mi3eie4bNxWUwwPsxHR4FoBSeWjm2yorDbUIFDK57xZH3jUFu11xSXW3x9BqGWdgQg71j/P1nVKGMoHvPUBX0wO2CLQnSl6katSwgM6aLeJl1U=" - - TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/" + python: '3.4' + install: + - python3 -m pip install -U pip + script: + - ./bin/pr-check.sh + addons: + apt: + packages: + - build-essential libgomp1 libmpich-dev mpich + - os: linux + dist: trusty + sudo: required + language: python + python: '3.5' + install: + - python3 -m pip install -U pip + script: + - ./bin/pr-check.sh + addons: + apt: + packages: + - build-essential libgomp1 libmpich-dev mpich + - os: linux + dist: trusty + sudo: required + language: python + python: '3.6' install: - - python3 -m pip install -U pip twine + - python3 -m pip install -U pip script: - - python3 -m pip install -e . - - python3 setup.py sdist - - twine upload dist/* + - ./bin/pr-check.sh addons: apt: packages: - - build-essential libgomp1 libmpich-dev mpich + - build-essential libgomp1 libmpich-dev mpich + - os: osx + language: generic + env: + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + before_install: + - brew update + - brew install llvm mpich python3 + install: + - python3 -m pip install -U pip + script: + - ./bin/pr-check.sh + osx_image: xcode7.3 + - os: osx + language: generic + env: + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + before_install: + - brew update + - brew install llvm mpich python3 + install: + - python3 -m pip install -U pip + script: + - ./bin/pr-check.sh + osx_image: xcode8 + - stage: build + language: generic + if: branch = master and repo = danielsuo/brainiak + - if: branch = master and repo = danielsuo/brainiak + os: linux + dist: trusty + sudo: required + language: python + python: '3.4' + env: TWINE_REPOSITORY_URL=https://test.pypi.org/legacy + install: + - python3 -m pip install -U pip awscli + script: + - ./bin/build-dist.sh + - ./bin/test-wheels.sh + after_script: + - aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read + - if: branch = master and repo = danielsuo/brainiak + os: osx + osx_image: xcode7.3 + sudo: required + language: generic + env: + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + - TWINE_REPOSITORY_URL=https://test.pypi.org/legacy + - ARCHFLAGS="-arch x86_64" + before_install: + - brew update + - brew install llvm mpich python3 + - python3 -m pip install --user pip awscli + install: + - VERSIONS="3.4.4" ./bin/install-python-macos.sh + - VERSIONS="3.4.4" ./bin/build-dist-macos.sh + before_script: + - brew uninstall -f --ignore-dependencies llvm mpich + - unset CC + - unset CXX + - unset LDFLAGS + - unset CPPFLAGS + - unset ARCHFLAGS + script: + - VERSIONS="3.4.4" ./bin/test-wheels-macos.sh + after_script: + - aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read + - if: branch = master and repo = danielsuo/brainiak + os: osx + osx_image: xcode7.3 + sudo: required + language: generic + env: + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + - TWINE_REPOSITORY_URL=https://test.pypi.org/legacy + - ARCHFLAGS="-arch x86_64" + before_install: + - brew update + - brew install llvm mpich python3 + - python3 -m pip install --user pip awscli + install: + - VERSIONS="3.5.3" ./bin/install-python-macos.sh + - VERSIONS="3.5.3" ./bin/build-dist-macos.sh + before_script: + - brew uninstall -f --ignore-dependencies llvm mpich + - unset CC + - unset CXX + - unset LDFLAGS + - unset CPPFLAGS + - unset ARCHFLAGS + script: + - VERSIONS="3.5.3" ./bin/test-wheels-macos.sh + after_script: + - aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read + - if: branch = master and repo = danielsuo/brainiak + os: osx + osx_image: xcode7.3 + sudo: required + language: generic + env: + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + - TWINE_REPOSITORY_URL=https://test.pypi.org/legacy + - ARCHFLAGS="-arch x86_64" + before_install: + - brew update + - brew install llvm mpich python3 + - python3 -m pip install --user pip awscli + install: + - VERSIONS="3.6.0" ./bin/install-python-macos.sh + - VERSIONS="3.6.0" ./bin/build-dist-macos.sh + before_script: + - brew uninstall -f --ignore-dependencies llvm mpich + - unset CC + - unset CXX + - unset LDFLAGS + - unset CPPFLAGS + - unset ARCHFLAGS + script: + - VERSIONS="3.6.0" ./bin/test-wheels-macos.sh + after_script: + - aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read + - stage: s3 + language: generic + if: branch = mater and repo = danielsuo/brainiak + - if: branch = master and repo = danielsuo/brainiak + os: linux + dist: trusty + sudo: required + env: TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/" + language: python + python: '3.4' + install: + - python3 -m pip install -U pip awscli twine + - ./bin/download-wheels-s3.sh + script: + - ./bin/test-wheels.sh + after_script: + - twine upload dist/* + - if: branch = master and repo = danielsuo/brainiak + os: osx + osx_image: xcode7.3 + sudo: required + language: generic + before_install: + - brew update + - brew install python3 + - python3 -m pip install -U pip awscli twine + - ./bin/download-wheels-s3.sh + install: + - VERSIONS="3.4.4" ./bin/install-python-macos.sh + script: + - VERIONS="3.4.4" ./bin/test-wheels-macos.sh + - if: branch = master and repo = danielsuo/brainiak + os: osx + osx_image: xcode7.3 + sudo: required + language: generic + before_install: + - brew update + - brew install python3 + - python3 -m pip install -U pip awscli twine + - ./bin/download-wheels-s3.sh + install: + - VERSIONS="3.5.3" ./bin/install-python-macos.sh + script: + - VERIONS="3.5.3" ./bin/test-wheels-macos.sh + - if: branch = master and repo = danielsuo/brainiak + os: osx + osx_image: xcode7.3 + sudo: required + language: generic + before_install: + - brew update + - brew install python3 + - python3 -m pip install -U pip awscli twine + - ./bin/download-wheels-s3.sh + install: + - VERSIONS="3.6.0" ./bin/install-python-macos.sh + script: + - VERIONS="3.6.0" ./bin/test-wheels-macos.sh + - stage: testpypi + language: generic + if: branch = master and repo = danielsuo/brainiak + - if: branch = master and repo = danielsuo/brainiak + os: linux + dist: trusty + sudo: required + language: python + python: '3.4' + env: + - TWINE_REPOSITORY_URL=https://pypi.python.org/pypi + - PYPI_REPOSITORY_URL=https://testpypi.python.org + install: + - python3 -m pip install -U pip awscli + script: ./bin/test-wheels.sh + after_script: + - aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read + - if: branch = master and repo = danielsuo/brainiak + os: osx + osx_image: xcode7.3 + sudo: required + language: generic + env: + - TWINE_REPOSITORY_URL=https://pypi.python.org/pypi + - PYPI_REPOSITORY_URL=https://testpypi.python.org + before_install: + - brew update + - brew install llvm mpich python3 + - python3 -m pip install --user pip awscli + install: + - VERSIONS="3.4.4" ./bin/install-python-macos.sh + script: + - VERIONS="3.4.4" ./bin/test-wheels-macos.sh + - if: branch = master and repo = danielsuo/brainiak + os: osx + osx_image: xcode7.3 + sudo: required + language: generic + env: + - TWINE_REPOSITORY_URL=https://pypi.python.org/pypi + - PYPI_REPOSITORY_URL=https://testpypi.python.org + before_install: + - brew update + - brew install llvm mpich python3 + - python3 -m pip install --user pip awscli + install: + - VERSIONS="3.5.3" ./bin/install-python-macos.sh + script: + - VERIONS="3.5.3" ./bin/test-wheels-macos.sh + - if: branch = master and repo = danielsuo/brainiak + os: osx + osx_image: xcode7.3 + sudo: required + language: generic + env: + - TWINE_REPOSITORY_URL=https://pypi.python.org/pypi + - PYPI_REPOSITORY_URL=https://testpypi.python.org + before_install: + - brew update + - brew install llvm mpich python3 + - python3 -m pip install --user pip awscli + install: + - VERSIONS="3.6.0" ./bin/install-python-macos.sh + script: + - VERIONS="3.6.0" ./bin/test-wheels-macos.sh + - stage: pypi + language: generic + if: branch = master and repo = danielsuo/brainiak diff --git a/bin/README.md b/bin/README.md index 4dedb4e58..1229f2ff7 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,6 +1,10 @@ # Wheel to-dos # TODO +- Deal with versions and tagging (pypi only accepts non-dev tags) +- Download from S3 and install + - Pick up from the correct travis commit +- Upload to S3 even for PRs (custom deploy) - Recreate and encrypt keys for brainiak/brainiak - Only need to build mpi4py occasionally; not with every commit, and not with every job - mpi4py should always commit to s3://brainiak/.whl diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index fbca83c07..b8c226367 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -6,7 +6,9 @@ # Stages: (Travis capitalizes first letter and lowers the rest) # - Test: test source and source distribution, dependencies # - Build: build wheels and test -# - deploy to s3 and testpypi +# - deploy to s3 +# - S3: download from s3 and test +# - deploy to testpypi # - Testpypi: download wheels from test PyPI and test # - deploy to pypi # - Pypi: download wheels from PyPI and test @@ -51,8 +53,8 @@ def dict_constructor(loader, node): region = 'us-east-1' access_key_id = 'AKIAJKBW6H2VKKQDHILQ' secret_access_key = 'A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg=' - -twine_password = 'BHzjkB2HoIIN8njur7R/2+fBqubStLh+KH4vS4WdNMPdshLBYuFXFPAo3kg0CR72jAQ5+pPwPwQDrVbuA4jz8rXSLjpOxwsu8l4TCE6pcISbAGARgWBxOSDsLazIWyKL83aMqwtViEU9ChCdYHOiQZ6YfvVlgGOK4o8hY9eZrLcKy1H66yuMYTJOSu9ct3lUzxexiffGz1JmdJ6lg6jVB1Ga2pYoS1CPk8gK4bf3hRKCBaIhNEV93KGGNtbXbKBhEtHCtJBrCA3aZ9spJySNMt+6ggKCllDdWq26TfMKIl6Q7koV9idgIXkY+hW+3WlPWx5aVH8lR6cw7f0GOiu4E2c5yFKElnGYCqEPkuL2fcg0y9t64T9wMwpMsOgQHD8RX5DPMpK1Rm83mXU2HZLmf4F/FjLWgZmMXQgjYv/dXIMBkO2Ba6cbfCq3a8G2jUxugZw7UcuQpjGKs6XV9/EprrQnuZjSv4SCp0hCNmb9JVv+hd7R+QDtw7sL8VBMQJxZwtbpGDobTGxFS1hC9rmlVnp0jXUhF99ERahhrERj4IdnNuuVvEWpiN085qT3UPp2WU+9o4Cw5NgVdBDTO5wmaEptGdXqfP+EfxdEfQxJNs2WJOHBOA+EeyNsP3LFQVeYLxYOeDBWFaJLh0P5uhb3c+L5UNuO31NpUBBQ+//3b64=' +secret_access_key_env = 'gCGZuvBQVZ09rmH5tPUn3GT17NzyBX9JRO3SJwq6WcLbmKDRo50xAhqib/VAKfMXmcteaIPQdxjWUcohPXAoFQ67Z3Jx8abjxEIJ6t7IwIH3aRYWh7PX28Iu5Cr3b/pLbSBw0r+tj/X0mHsbS1dF/uVOBo4fSGN5f4li3EMv+iWQNimwC64uT+ywbzjdUPwJBZ+rVoyh/uxBR/14CpYMi2b1Bf79iVYZr1INShWH+4JNqw6RkJQVhQbAXq9CdSH3bNgyrnASno542cDa9l++xJhJ9okFTdWchiDj1FH4BhBLdEQHVEm790dAZGcE41APg8gW/4hDA8G5wXrCN/T+QtEEgzIDKJ9o/VImabjWcuB1oMERM4Rcvr0wpRNd+DCcPNyEpLPHlOV8WGiIu/5cgLv7O8n38+LmyjCNeJKKR9TxcCoZ53LhrwHgfcokElz1aXQvHcNh+IAG0WF0+OxclBiqEwvasyq8PT4r9MMNYwklINcSDO5RdK2xVJJbehFT81LodAJ9ydgrfagAB7haLq6JfdIucvlllmeVyLwznJm/Gtx9WNOFeSVEN0pufJWsiQJelu4S+4OesfdCCzypAXkSWQ2M4vQBcWNYjJ6Ns6svhNX/9wC9+QLPusAieLan2WDFjxrQwUM7n+J3/GxEEPI6heUGNh3n0wimmVpGxZo=' +twine_password = 'JWCpO0caWN4M0xaGPqKMPPHg8WB6sybPQPcCFtXc+dwsD3mu+mDv7lGV3l/5sRoDqIi87uhXBtfnPVKao7AdPaLZoZc8ktBP3b7QwG49U479odN1OuxZcdLBmQBPZlAnqyH7wbM0xhkskr6Yl8gkKWynDfyod1GWbr/plcGRuX82unvuq5tkX8gNSdJ4vHbHGGcvgUbDkyEJdQz7y96qJn7VUpD4FhujO7bGyGVB2ZVcAcEyHuQZiyPxknrXSE0dnk0IeEH77DY40SLpPf5WG4fl/gGYLlccw2ZmBOAOYYE/nq9HF+/b5gnYp+0zLUCgA1tK9uC1UdT3IE24hr8XJMdMIog1BSVjv5k31RjFi0mwCVaasiCN/IOYvwrg+yod5UjXBag7f/Wjo3AEoUSByFERTr+Y7Gs+uCHQKT8Jw4mCTwmI+3JnH+4ZcuMbIQUr10uaocmpNuWhA1IVMTewIKjaAgY9qWy2X5e7X5/faYXlUhV68QiA5TB4MCvA+jeq84+bIk00HBnOadL/iAKFVhBlbkqe8OZWNbsBKFERWEVJFIoOLMg43NORWRs54Mi3eie4bNxWUwwPsxHR4FoBSeWjm2yorDbUIFDK57xZH3jUFu11xSXW3x9BqGWdgQg71j/P1nVKGMoHvPUBX0wO2CLQnSl6katSwgM6aLeJl1U=' data = OrderedDict() @@ -64,7 +66,10 @@ def dict_constructor(loader, node): 'global': [ 'BRAINIAK_REPO=%s' % repo, 'TWINE_USERNAME=%s' % twine_username, - 'secure: %s' % twine_password + 'secure: "%s"' % twine_password, + 'AWS_ACCESS_KEY_ID="%s"' % access_key_id, + 'secure: "%s"' % secret_access_key_env, + 'AWS_DEFAULT_REGION="%s"' % region ] }) @@ -126,23 +131,27 @@ def dict_constructor(loader, node): 'if': 'branch = master and repo = %s' % repo })) -deploy_s3 = [OrderedDict({ - 'provider': 's3', - 'access_key_id': access_key_id, - 'secret_access_key': { - 'secure': secret_access_key - }, - 'bucket': bucket, - 'region': region, - 'acl': 'public_read', - 'local_dir': 'dist', - 'upload-dir': '$TRAVIS_COMMIT/dist', - 'skip_cleanup': True, - 'on': { - 'repo': repo, - 'branch': 'master', - } -})] +# deploy_s3 = [OrderedDict({ +# 'provider': 's3', +# 'access_key_id': access_key_id, +# 'secret_access_key': { +# 'secure': secret_access_key +# }, +# 'bucket': bucket, +# 'region': region, +# 'acl': 'public_read', +# 'local_dir': 'dist', +# 'upload-dir': '$TRAVIS_COMMIT/dist', +# 'skip_cleanup': True, +# 'on': { +# 'repo': repo, +# 'branch': 'master', +# } +# })] + +upload_s3 = [ + 'aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read' +] build_linux = OrderedDict({ 'if': 'branch = master and repo = %s' % repo, @@ -153,16 +162,13 @@ def dict_constructor(loader, node): 'python': '3.4', 'env': 'TWINE_REPOSITORY_URL=https://test.pypi.org/legacy', 'install': [ - 'python3 -m pip install -U pip twine' + 'python3 -m pip install -U pip awscli' ], 'script': [ './bin/build-dist.sh', './bin/test-wheels.sh' ], - # 'after_script': [ - # 'twine upload dist/*' - # ], - 'deploy': deploy_s3 + 'after_script': copy.deepcopy(upload_s3) }) jobs.append(build_linux) @@ -182,9 +188,8 @@ def dict_constructor(loader, node): 'before_install': [ 'brew update', 'brew install llvm mpich python3', - 'python3 -m pip install --user twine' - ], - 'deploy': deploy_s3 + 'python3 -m pip install --user pip awscli' + ] }) for version in versions: @@ -208,11 +213,61 @@ def dict_constructor(loader, node): 'VERSIONS="%s" ./bin/test-wheels-macos.sh' % version ] - # block['after_script'] = [ - # 'twine upload dist/*' - # ] + block['after_script'] = copy.deepcopy(upload_s3) + + jobs.append(block) - block['deploy'] = copy.deepcopy(deploy_s3) +jobs.append(OrderedDict({ + 'stage': 's3', + 'language': 'generic', + 'if': 'branch = mater and repo = %s' % repo +})) + +s3_linux = OrderedDict({ + 'if': 'branch = master and repo = %s' % repo, + 'os': 'linux', + 'dist': 'trusty', + 'sudo': 'required', + 'env': 'TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/"', + 'language': 'python', + 'python': '3.4', + 'install': [ + 'python3 -m pip install -U pip awscli twine', + './bin/download-wheels-s3.sh' + ], + 'script': ['./bin/test-wheels.sh'], + 'after_script': ['twine upload dist/*'] +}) + +jobs.append(s3_linux) + +s3_macos = OrderedDict({ + 'if': 'branch = master and repo = %s' % repo, + 'os': 'osx', + 'osx_image': 'xcode7.3', + 'sudo': 'required', + 'language': 'generic', + 'before_install': [ + 'brew update', + 'brew install python3', + 'python3 -m pip install -U pip awscli twine', + './bin/download-wheels-s3.sh' + ], + 'install': [ + './bin/install-python-macos.sh' + ], + 'script': ['./bin/test-wheels-macos.sh'] +}) + +for version in versions: + block = copy.deepcopy(s3_macos) + block['install'] = [ + 'VERSIONS="%s" ./bin/install-python-macos.sh' % version + ] + + block['script'] = [ + 'VERIONS="%s" ./bin/test-wheels-macos.sh' % version + ] jobs.append(block) @@ -231,7 +286,7 @@ def dict_constructor(loader, node): ] testpypi_linux['script'] = './bin/test-wheels.sh' -# jobs.append(testpypi_linux) +jobs.append(testpypi_linux) testpypi_macos = copy.deepcopy(build_macos) testpypi_macos.pop('deploy', None) @@ -250,9 +305,9 @@ def dict_constructor(loader, node): 'VERIONS="%s" ./bin/test-wheels-macos.sh' % version ] - # jobs.append(block) + jobs.append(block) -# Test PyPI install +# Pypi install jobs.append(OrderedDict({ 'stage': 'pypi', 'language': 'generic', diff --git a/setup.py b/setup.py index 4074ee4ec..e77f31f5b 100644 --- a/setup.py +++ b/setup.py @@ -150,8 +150,10 @@ def finalize_options(self): setup( name='brainiak', + + # TODO: Change this back # use_scm_version=True, - version='0.2.1', + version='0.2.2', setup_requires=[ 'cython', 'numpy', From 5ffdfd300d64f55cccdef3a05449e02ded60da3c Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 15:28:50 -0500 Subject: [PATCH 107/126] Typo --- bin/generate-yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index b8c226367..7f4ef817b 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -220,7 +220,7 @@ def dict_constructor(loader, node): jobs.append(OrderedDict({ 'stage': 's3', 'language': 'generic', - 'if': 'branch = mater and repo = %s' % repo + 'if': 'branch = master and repo = %s' % repo })) s3_linux = OrderedDict({ From c89c3458e023f7380a653ab4d3d0580e5dd65b5c Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 15:29:11 -0500 Subject: [PATCH 108/126] Typo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6c7d57c69..046407acc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -196,7 +196,7 @@ jobs: - aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read - stage: s3 language: generic - if: branch = mater and repo = danielsuo/brainiak + if: branch = master and repo = danielsuo/brainiak - if: branch = master and repo = danielsuo/brainiak os: linux dist: trusty From 0e3b88998377f87104cb5f9cc52d1f259561947f Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 16:34:55 -0500 Subject: [PATCH 109/126] Refactor generate yaml [skip ci] --- bin/generate-yaml.py | 260 ++++++++++++++++++------------------------- 1 file changed, 111 insertions(+), 149 deletions(-) diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 7f4ef817b..9e314a0f2 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -3,6 +3,8 @@ # TODO: the first job in a stage must be part of the job dictinoary, not an # element in the job array. This results in a dummy job being created +# NOTE: we use deep copies even though we could rely on YAML references + # Stages: (Travis capitalizes first letter and lowers the rest) # - Test: test source and source distribution, dependencies # - Build: build wheels and test @@ -73,132 +75,116 @@ def dict_constructor(loader, node): ] }) -data['jobs'] = OrderedDict({ - 'include': [] -}) - +data['jobs'] = OrderedDict({'include': []}) jobs = data['jobs']['include'] -# Create test stage -jobs.append(OrderedDict({'stage': 'test', 'language': 'generic'})) +conditions = { + 'pr': 'branch = master', + 'master': 'branch = master and repo = %s' % repo, + 'tag': 'branch = master and repo = %s and tag IS present' % repo +} -# Linux -test_linux = OrderedDict({ +actions = { + 'download-s3': 'aws s3 cp s3://brainiak/$TRAVIS_COMMIT/dist dist --recursive', + 'upload-s3': 'aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read', + 'upload-testpypi': 'if [ ! -z $TRAVIS_TAG ]; then twine upload dist/*; fi', + + # TODO: populate this command + 'upload-pypi': ';' +} + +linux = OrderedDict({ 'os': 'linux', 'dist': 'trusty', 'sudo': 'required', 'language': 'python', - 'python': None, - 'install': ['python3 -m pip install -U pip'], - 'script': ['./bin/pr-check.sh'], - 'addons': { - 'apt': { - 'packages': ['build-essential libgomp1 libmpich-dev mpich'] - } - } + 'python': '3.4', + 'before_install': ['python3 -m pip install -U pip awscli twine'] +}) + +macos = OrderedDict({ + 'os': 'osx', + 'osx_image': 'xcode7.3', + 'language': 'generic', + 'env': ['HOMEBREW_NO_AUTO_UPDATE=1'], + 'before_install': [ + 'brew update', + 'brew install python3', + 'python3 -m pip install -U pip awscli twine' + ] }) +############################################################################### +# Stage: Test +############################################################################### +jobs.append(OrderedDict({'stage': 'test', 'language': 'generic'})) + +# Linux +test_linux = copy.deepcopy(linux) +test_linux['script'] = ['./bin/pr-check.sh'] +test_linux['addons'] = { + 'apt': { + 'packages': ['build-essential libgomp1 libmpich-dev mpich'] + } +} + for major in majors: block = copy.deepcopy(test_linux) block['python'] = major jobs.append(block) # MacOS -test_macos = OrderedDict({ - 'os': 'osx', - 'language': 'generic', - 'env': [ - 'CC=/usr/local/opt/llvm/bin/clang', - 'CXX=/usr/local/opt/llvm/bin/clang++', - 'LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS"', - 'CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS"', - 'HOMEBREW_NO_AUTO_UPDATE=1' - ], - 'before_install': ['brew update', 'brew install llvm mpich python3'], - 'install': ['python3 -m pip install -U pip'], - 'script': ['./bin/pr-check.sh'] -}) +macos_build_env = [ + 'CC=/usr/local/opt/llvm/bin/clang', + 'CXX=/usr/local/opt/llvm/bin/clang++', + 'LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS"', + 'CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS"', + 'HOMEBREW_NO_AUTO_UPDATE=1' +] + +test_macos = copy.deepcopy(macos) +test_macos['env'] = macos_build_env +test_macos['install'] = ['brew install llvm mpich'] +test_macos['script'] = ['./bin/pr-check.sh'] for osx in ['xcode7.3', 'xcode8']: block = copy.deepcopy(test_macos) block['osx_image'] = osx jobs.append(block) -# Create build stage +############################################################################### +# Stage: Build +############################################################################### jobs.append(OrderedDict({ 'stage': 'build', 'language': 'generic', - 'if': 'branch = master and repo = %s' % repo + 'if': conditions['pr'] })) -# deploy_s3 = [OrderedDict({ -# 'provider': 's3', -# 'access_key_id': access_key_id, -# 'secret_access_key': { -# 'secure': secret_access_key -# }, -# 'bucket': bucket, -# 'region': region, -# 'acl': 'public_read', -# 'local_dir': 'dist', -# 'upload-dir': '$TRAVIS_COMMIT/dist', -# 'skip_cleanup': True, -# 'on': { -# 'repo': repo, -# 'branch': 'master', -# } -# })] - -upload_s3 = [ - 'aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read' -] - -build_linux = OrderedDict({ - 'if': 'branch = master and repo = %s' % repo, - 'os': 'linux', - 'dist': 'trusty', - 'sudo': 'required', - 'language': 'python', - 'python': '3.4', - 'env': 'TWINE_REPOSITORY_URL=https://test.pypi.org/legacy', - 'install': [ - 'python3 -m pip install -U pip awscli' - ], - 'script': [ - './bin/build-dist.sh', - './bin/test-wheels.sh' - ], - 'after_script': copy.deepcopy(upload_s3) -}) - +# Linux +build_linux = copy.deepcopy(linux) +build_linux['if'] = conditions['pr'] +build_linux['install'] = ['./bin/build-dist.sh'] +build_linux['script'] = ['./bin/test-wheels.sh'] +build_linux['after_script'] = [actions['upload-s3']] jobs.append(build_linux) +# MacOS build_macos_env = copy.deepcopy(test_macos['env']) -build_macos_env.extend([ - 'TWINE_REPOSITORY_URL=https://test.pypi.org/legacy', - 'ARCHFLAGS="-arch x86_64"' -]) -build_macos = OrderedDict({ - 'if': 'branch = master and repo = %s' % repo, - 'os': 'osx', - 'osx_image': 'xcode7.3', - 'sudo': 'required', - 'language': 'generic', - 'env': build_macos_env, - 'before_install': [ - 'brew update', - 'brew install llvm mpich python3', - 'python3 -m pip install --user pip awscli' - ] -}) +build_macos_env.extend(['ARCHFLAGS="-arch x86_64"']) + +build_macos = copy.deepcopy(macos) +build_macos['if'] = conditions['pr'] +build_macos['env'] = build_macos_env +build_macos['install'] = ['brew install llvm mpich'] for version in versions: block = copy.deepcopy(build_macos) - block['install'] = [ + block['install'].extend([ 'VERSIONS="%s" ./bin/install-python-macos.sh' % version, 'VERSIONS="%s" ./bin/build-dist-macos.sh' % version - ] + ]) block['before_script'] = [ 'brew uninstall -f --ignore-dependencies llvm mpich', @@ -213,51 +199,32 @@ def dict_constructor(loader, node): 'VERSIONS="%s" ./bin/test-wheels-macos.sh' % version ] - block['after_script'] = copy.deepcopy(upload_s3) + block['after_script'] = [actions['upload-s3']] jobs.append(block) +############################################################################### +# Stage: S3 +############################################################################### jobs.append(OrderedDict({ 'stage': 's3', 'language': 'generic', - 'if': 'branch = master and repo = %s' % repo + 'if': conditions['pr'] })) -s3_linux = OrderedDict({ - 'if': 'branch = master and repo = %s' % repo, - 'os': 'linux', - 'dist': 'trusty', - 'sudo': 'required', - 'env': 'TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/"', - 'language': 'python', - 'python': '3.4', - 'install': [ - 'python3 -m pip install -U pip awscli twine', - './bin/download-wheels-s3.sh' - ], - 'script': ['./bin/test-wheels.sh'], - 'after_script': ['twine upload dist/*'] -}) - +# Linux +s3_linux = copy.deepcopy(linux) +s3_linux['if'] = conditions['pr'] +s3_linux['env'] = 'TWINE_REPOSITORY_URL=https://test.pypi.org/legacy' +s3_linux['install'] = actions['download-s3'] +s3_linux['script'] = ['./bin/test-wheels.sh'] +s3_linux['after_script'] = [actions['upload-testpypi']] jobs.append(s3_linux) -s3_macos = OrderedDict({ - 'if': 'branch = master and repo = %s' % repo, - 'os': 'osx', - 'osx_image': 'xcode7.3', - 'sudo': 'required', - 'language': 'generic', - 'before_install': [ - 'brew update', - 'brew install python3', - 'python3 -m pip install -U pip awscli twine', - './bin/download-wheels-s3.sh' - ], - 'install': [ - './bin/install-python-macos.sh' - ], - 'script': ['./bin/test-wheels-macos.sh'] -}) +# MacOS +s3_macos = copy.deepcopy(macos) +s3_macos['if'] = conditions['pr'] +s3_macos['before_install'].append(actions['download-s3']) for version in versions: block = copy.deepcopy(s3_macos) @@ -271,29 +238,26 @@ def dict_constructor(loader, node): jobs.append(block) -# Test PyPI install +############################################################################### +# Stage: Testpypi +############################################################################### jobs.append(OrderedDict({ 'stage': 'testpypi', 'language': 'generic', - 'if': 'branch = master and repo = %s' % repo + 'if': conditions['master'] })) -testpypi_linux = copy.deepcopy(build_linux) -testpypi_linux.pop('deploy', None) +testpypi_linux = copy.deepcopy(linux) testpypi_linux['env'] = [ 'TWINE_REPOSITORY_URL=https://pypi.python.org/pypi', 'PYPI_REPOSITORY_URL=https://testpypi.python.org' ] testpypi_linux['script'] = './bin/test-wheels.sh' - +testpypi_linux['after_script'] = [actions['upload-pypi']] jobs.append(testpypi_linux) testpypi_macos = copy.deepcopy(build_macos) -testpypi_macos.pop('deploy', None) -testpypi_macos['env'] = [ - 'TWINE_REPOSITORY_URL=https://pypi.python.org/pypi', - 'PYPI_REPOSITORY_URL=https://testpypi.python.org' -] +testpypi_macos['env'] = copy.deepcopy(testpypi_linux['env']) for version in versions: block = copy.deepcopy(testpypi_macos) @@ -305,28 +269,26 @@ def dict_constructor(loader, node): 'VERIONS="%s" ./bin/test-wheels-macos.sh' % version ] + block['after_script'] = [actions['upload-pypi']] + jobs.append(block) -# Pypi install +############################################################################### +# Stage: Pypi +############################################################################### jobs.append(OrderedDict({ 'stage': 'pypi', 'language': 'generic', - 'if': 'branch = master and repo = %s' % repo + 'if': conditions['tag'] })) -pypi_linux = copy.deepcopy(testpypi_linux) -pypi_linux.pop('after_script', None) -pypi_linux['env'] = [ - 'PYPI_REPOSITORY_URL=https://testpypi.python.org' -] - +pypi_linux = copy.deepcopy(linux) +pypi_linux['env'] = ['PYPI_REPOSITORY_URL=https://pypi.python.org'] +pypi_linux['install'] = ['./bin/test-wheels.sh'] # jobs.append(pypi_linux) -pypi_macos = copy.deepcopy(testpypi_macos) -pypi_macos.pop('after_script', None) -pypi_macos['env'] = [ - 'PYPI_REPOSITORY_URL=https://testpypi.python.org' -] +pypi_macos = copy.deepcopy(macos) +pypi_macos['env'] = copy.deepcopy(pypi_linux['env']) for version in versions: block = copy.deepcopy(pypi_macos) From c99b77fb3c5292eecf17cbcaa54de79919e839ea Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 16:36:11 -0500 Subject: [PATCH 110/126] Small edit [skip ci] --- bin/generate-yaml.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 9e314a0f2..6f773cf81 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -41,9 +41,6 @@ def dict_constructor(loader, node): Dumper.add_representer(str, SafeRepresenter.represent_str) -# Dumper.add_representer(unicode, -# SafeRepresenter.represent_unicode) - travis = '.travis.yml' repo = 'danielsuo/brainiak' twine_username = 'brainiak-bot' From c7f341a6368bcc1bc753f2a51768fc4c248122b1 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 17:20:26 -0500 Subject: [PATCH 111/126] [skip ci] Regenerate yaml post refactor --- .travis.yml | 156 +++++++++++++++++++++++++++------------------------- 1 file changed, 80 insertions(+), 76 deletions(-) diff --git a/.travis.yml b/.travis.yml index 046407acc..163d5e30b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,8 +18,8 @@ jobs: sudo: required language: python python: '3.4' - install: - - python3 -m pip install -U pip + before_install: + - python3 -m pip install -U pip awscli twine script: - ./bin/pr-check.sh addons: @@ -31,8 +31,8 @@ jobs: sudo: required language: python python: '3.5' - install: - - python3 -m pip install -U pip + before_install: + - python3 -m pip install -U pip awscli twine script: - ./bin/pr-check.sh addons: @@ -44,8 +44,8 @@ jobs: sudo: required language: python python: '3.6' - install: - - python3 -m pip install -U pip + before_install: + - python3 -m pip install -U pip awscli twine script: - ./bin/pr-check.sh addons: @@ -53,6 +53,7 @@ jobs: packages: - build-essential libgomp1 libmpich-dev mpich - os: osx + osx_image: xcode7.3 language: generic env: - CC=/usr/local/opt/llvm/bin/clang @@ -62,13 +63,14 @@ jobs: - HOMEBREW_NO_AUTO_UPDATE=1 before_install: - brew update - - brew install llvm mpich python3 + - brew install python3 + - python3 -m pip install -U pip awscli twine install: - - python3 -m pip install -U pip + - brew install llvm mpich script: - ./bin/pr-check.sh - osx_image: xcode7.3 - os: osx + osx_image: xcode8 language: generic env: - CC=/usr/local/opt/llvm/bin/clang @@ -78,33 +80,31 @@ jobs: - HOMEBREW_NO_AUTO_UPDATE=1 before_install: - brew update - - brew install llvm mpich python3 + - brew install python3 + - python3 -m pip install -U pip awscli twine install: - - python3 -m pip install -U pip + - brew install llvm mpich script: - ./bin/pr-check.sh - osx_image: xcode8 - stage: build language: generic - if: branch = master and repo = danielsuo/brainiak - - if: branch = master and repo = danielsuo/brainiak - os: linux + if: branch = master + - os: linux dist: trusty sudo: required language: python python: '3.4' - env: TWINE_REPOSITORY_URL=https://test.pypi.org/legacy + before_install: + - python3 -m pip install -U pip awscli twine + if: branch = master install: - - python3 -m pip install -U pip awscli - script: - ./bin/build-dist.sh + script: - ./bin/test-wheels.sh after_script: - aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read - - if: branch = master and repo = danielsuo/brainiak - os: osx + - os: osx osx_image: xcode7.3 - sudo: required language: generic env: - CC=/usr/local/opt/llvm/bin/clang @@ -112,13 +112,14 @@ jobs: - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - - TWINE_REPOSITORY_URL=https://test.pypi.org/legacy - ARCHFLAGS="-arch x86_64" before_install: - brew update - - brew install llvm mpich python3 - - python3 -m pip install --user pip awscli + - brew install python3 + - python3 -m pip install -U pip awscli twine + if: branch = master install: + - brew install llvm mpich - VERSIONS="3.4.4" ./bin/install-python-macos.sh - VERSIONS="3.4.4" ./bin/build-dist-macos.sh before_script: @@ -132,10 +133,8 @@ jobs: - VERSIONS="3.4.4" ./bin/test-wheels-macos.sh after_script: - aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read - - if: branch = master and repo = danielsuo/brainiak - os: osx + - os: osx osx_image: xcode7.3 - sudo: required language: generic env: - CC=/usr/local/opt/llvm/bin/clang @@ -143,13 +142,14 @@ jobs: - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - - TWINE_REPOSITORY_URL=https://test.pypi.org/legacy - ARCHFLAGS="-arch x86_64" before_install: - brew update - - brew install llvm mpich python3 - - python3 -m pip install --user pip awscli + - brew install python3 + - python3 -m pip install -U pip awscli twine + if: branch = master install: + - brew install llvm mpich - VERSIONS="3.5.3" ./bin/install-python-macos.sh - VERSIONS="3.5.3" ./bin/build-dist-macos.sh before_script: @@ -163,10 +163,8 @@ jobs: - VERSIONS="3.5.3" ./bin/test-wheels-macos.sh after_script: - aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read - - if: branch = master and repo = danielsuo/brainiak - os: osx + - os: osx osx_image: xcode7.3 - sudo: required language: generic env: - CC=/usr/local/opt/llvm/bin/clang @@ -174,13 +172,14 @@ jobs: - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - HOMEBREW_NO_AUTO_UPDATE=1 - - TWINE_REPOSITORY_URL=https://test.pypi.org/legacy - ARCHFLAGS="-arch x86_64" before_install: - brew update - - brew install llvm mpich python3 - - python3 -m pip install --user pip awscli + - brew install python3 + - python3 -m pip install -U pip awscli twine + if: branch = master install: + - brew install llvm mpich - VERSIONS="3.6.0" ./bin/install-python-macos.sh - VERSIONS="3.6.0" ./bin/build-dist-macos.sh before_script: @@ -196,59 +195,62 @@ jobs: - aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read - stage: s3 language: generic - if: branch = master and repo = danielsuo/brainiak - - if: branch = master and repo = danielsuo/brainiak - os: linux + if: branch = master + - os: linux dist: trusty sudo: required - env: TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/" language: python python: '3.4' - install: + before_install: - python3 -m pip install -U pip awscli twine - - ./bin/download-wheels-s3.sh + if: branch = master + env: TWINE_REPOSITORY_URL=https://test.pypi.org/legacy + install: aws s3 cp s3://brainiak/$TRAVIS_COMMIT/dist dist --recursive script: - ./bin/test-wheels.sh after_script: - - twine upload dist/* - - if: branch = master and repo = danielsuo/brainiak - os: osx + - if [ ! -z $TRAVIS_TAG ]; then twine upload dist/*; fi + - os: osx osx_image: xcode7.3 - sudo: required language: generic + env: + - HOMEBREW_NO_AUTO_UPDATE=1 before_install: - brew update - brew install python3 - python3 -m pip install -U pip awscli twine - - ./bin/download-wheels-s3.sh + - aws s3 cp s3://brainiak/$TRAVIS_COMMIT/dist dist --recursive + if: branch = master install: - VERSIONS="3.4.4" ./bin/install-python-macos.sh script: - VERIONS="3.4.4" ./bin/test-wheels-macos.sh - - if: branch = master and repo = danielsuo/brainiak - os: osx + - os: osx osx_image: xcode7.3 - sudo: required language: generic + env: + - HOMEBREW_NO_AUTO_UPDATE=1 before_install: - brew update - brew install python3 - python3 -m pip install -U pip awscli twine - - ./bin/download-wheels-s3.sh + - aws s3 cp s3://brainiak/$TRAVIS_COMMIT/dist dist --recursive + if: branch = master install: - VERSIONS="3.5.3" ./bin/install-python-macos.sh script: - VERIONS="3.5.3" ./bin/test-wheels-macos.sh - - if: branch = master and repo = danielsuo/brainiak - os: osx + - os: osx osx_image: xcode7.3 - sudo: required language: generic + env: + - HOMEBREW_NO_AUTO_UPDATE=1 before_install: - brew update - brew install python3 - python3 -m pip install -U pip awscli twine - - ./bin/download-wheels-s3.sh + - aws s3 cp s3://brainiak/$TRAVIS_COMMIT/dist dist --recursive + if: branch = master install: - VERSIONS="3.6.0" ./bin/install-python-macos.sh script: @@ -256,68 +258,70 @@ jobs: - stage: testpypi language: generic if: branch = master and repo = danielsuo/brainiak - - if: branch = master and repo = danielsuo/brainiak - os: linux + - os: linux dist: trusty sudo: required language: python python: '3.4' + before_install: + - python3 -m pip install -U pip awscli twine env: - TWINE_REPOSITORY_URL=https://pypi.python.org/pypi - PYPI_REPOSITORY_URL=https://testpypi.python.org - install: - - python3 -m pip install -U pip awscli script: ./bin/test-wheels.sh after_script: - - aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read - - if: branch = master and repo = danielsuo/brainiak - os: osx + - ; + - os: osx osx_image: xcode7.3 - sudo: required language: generic env: - TWINE_REPOSITORY_URL=https://pypi.python.org/pypi - PYPI_REPOSITORY_URL=https://testpypi.python.org before_install: - brew update - - brew install llvm mpich python3 - - python3 -m pip install --user pip awscli + - brew install python3 + - python3 -m pip install -U pip awscli twine + if: branch = master install: - VERSIONS="3.4.4" ./bin/install-python-macos.sh script: - VERIONS="3.4.4" ./bin/test-wheels-macos.sh - - if: branch = master and repo = danielsuo/brainiak - os: osx + after_script: + - ; + - os: osx osx_image: xcode7.3 - sudo: required language: generic env: - TWINE_REPOSITORY_URL=https://pypi.python.org/pypi - PYPI_REPOSITORY_URL=https://testpypi.python.org before_install: - brew update - - brew install llvm mpich python3 - - python3 -m pip install --user pip awscli + - brew install python3 + - python3 -m pip install -U pip awscli twine + if: branch = master install: - VERSIONS="3.5.3" ./bin/install-python-macos.sh script: - VERIONS="3.5.3" ./bin/test-wheels-macos.sh - - if: branch = master and repo = danielsuo/brainiak - os: osx + after_script: + - ; + - os: osx osx_image: xcode7.3 - sudo: required language: generic env: - TWINE_REPOSITORY_URL=https://pypi.python.org/pypi - PYPI_REPOSITORY_URL=https://testpypi.python.org before_install: - brew update - - brew install llvm mpich python3 - - python3 -m pip install --user pip awscli + - brew install python3 + - python3 -m pip install -U pip awscli twine + if: branch = master install: - VERSIONS="3.6.0" ./bin/install-python-macos.sh script: - VERIONS="3.6.0" ./bin/test-wheels-macos.sh + after_script: + - ; - stage: pypi language: generic - if: branch = master and repo = danielsuo/brainiak + if: branch = master and repo = danielsuo/brainiak and tag IS present From 7b0c2062689fa95f5806687bb481bd874fa1651c Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 21:20:35 -0500 Subject: [PATCH 112/126] Remove Test phase for faster testing --- .travis.yml | 75 ----------------------------------------------------- 1 file changed, 75 deletions(-) diff --git a/.travis.yml b/.travis.yml index 163d5e30b..9bc369a6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,81 +11,6 @@ env: - AWS_DEFAULT_REGION="us-east-1" jobs: include: - - stage: test - language: generic - - os: linux - dist: trusty - sudo: required - language: python - python: '3.4' - before_install: - - python3 -m pip install -U pip awscli twine - script: - - ./bin/pr-check.sh - addons: - apt: - packages: - - build-essential libgomp1 libmpich-dev mpich - - os: linux - dist: trusty - sudo: required - language: python - python: '3.5' - before_install: - - python3 -m pip install -U pip awscli twine - script: - - ./bin/pr-check.sh - addons: - apt: - packages: - - build-essential libgomp1 libmpich-dev mpich - - os: linux - dist: trusty - sudo: required - language: python - python: '3.6' - before_install: - - python3 -m pip install -U pip awscli twine - script: - - ./bin/pr-check.sh - addons: - apt: - packages: - - build-essential libgomp1 libmpich-dev mpich - - os: osx - osx_image: xcode7.3 - language: generic - env: - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - before_install: - - brew update - - brew install python3 - - python3 -m pip install -U pip awscli twine - install: - - brew install llvm mpich - script: - - ./bin/pr-check.sh - - os: osx - osx_image: xcode8 - language: generic - env: - - CC=/usr/local/opt/llvm/bin/clang - - CXX=/usr/local/opt/llvm/bin/clang++ - - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" - - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" - - HOMEBREW_NO_AUTO_UPDATE=1 - before_install: - - brew update - - brew install python3 - - python3 -m pip install -U pip awscli twine - install: - - brew install llvm mpich - script: - - ./bin/pr-check.sh - stage: build language: generic if: branch = master From bca9d56d043bd86928cc359a4e53ecff5b951b39 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 19 Dec 2017 21:23:51 -0500 Subject: [PATCH 113/126] Remove quiet flags so mac jobs don't time out --- bin/build-dist-macos.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/build-dist-macos.sh b/bin/build-dist-macos.sh index c8aa053cc..41a01a86f 100755 --- a/bin/build-dist-macos.sh +++ b/bin/build-dist-macos.sh @@ -33,14 +33,14 @@ do $DELOCATE dist/*.whl popd - $PYTHON -m pip install -q . - $PYTHON setup.py -q bdist_wheel + $PYTHON -m pip install . + $PYTHON setup.py bdist_wheel $DELOCATE dist/*.whl # Build source distribution if [ $MAJOR == '3.4' ] then - $PYTHON setup.py -q sdist + $PYTHON setup.py sdist fi deactivate From 7eb122b3853e22649c207b9a53d859d97fc43242 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 20 Dec 2017 10:49:48 -0500 Subject: [PATCH 114/126] Update keys --- .travis.yml | 2 +- bin/generate-yaml.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9bc369a6f..3b67ee4ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ env: - TWINE_USERNAME=brainiak-bot - 'secure: "JWCpO0caWN4M0xaGPqKMPPHg8WB6sybPQPcCFtXc+dwsD3mu+mDv7lGV3l/5sRoDqIi87uhXBtfnPVKao7AdPaLZoZc8ktBP3b7QwG49U479odN1OuxZcdLBmQBPZlAnqyH7wbM0xhkskr6Yl8gkKWynDfyod1GWbr/plcGRuX82unvuq5tkX8gNSdJ4vHbHGGcvgUbDkyEJdQz7y96qJn7VUpD4FhujO7bGyGVB2ZVcAcEyHuQZiyPxknrXSE0dnk0IeEH77DY40SLpPf5WG4fl/gGYLlccw2ZmBOAOYYE/nq9HF+/b5gnYp+0zLUCgA1tK9uC1UdT3IE24hr8XJMdMIog1BSVjv5k31RjFi0mwCVaasiCN/IOYvwrg+yod5UjXBag7f/Wjo3AEoUSByFERTr+Y7Gs+uCHQKT8Jw4mCTwmI+3JnH+4ZcuMbIQUr10uaocmpNuWhA1IVMTewIKjaAgY9qWy2X5e7X5/faYXlUhV68QiA5TB4MCvA+jeq84+bIk00HBnOadL/iAKFVhBlbkqe8OZWNbsBKFERWEVJFIoOLMg43NORWRs54Mi3eie4bNxWUwwPsxHR4FoBSeWjm2yorDbUIFDK57xZH3jUFu11xSXW3x9BqGWdgQg71j/P1nVKGMoHvPUBX0wO2CLQnSl6katSwgM6aLeJl1U="' - AWS_ACCESS_KEY_ID="AKIAJKBW6H2VKKQDHILQ" - - 'secure: "gCGZuvBQVZ09rmH5tPUn3GT17NzyBX9JRO3SJwq6WcLbmKDRo50xAhqib/VAKfMXmcteaIPQdxjWUcohPXAoFQ67Z3Jx8abjxEIJ6t7IwIH3aRYWh7PX28Iu5Cr3b/pLbSBw0r+tj/X0mHsbS1dF/uVOBo4fSGN5f4li3EMv+iWQNimwC64uT+ywbzjdUPwJBZ+rVoyh/uxBR/14CpYMi2b1Bf79iVYZr1INShWH+4JNqw6RkJQVhQbAXq9CdSH3bNgyrnASno542cDa9l++xJhJ9okFTdWchiDj1FH4BhBLdEQHVEm790dAZGcE41APg8gW/4hDA8G5wXrCN/T+QtEEgzIDKJ9o/VImabjWcuB1oMERM4Rcvr0wpRNd+DCcPNyEpLPHlOV8WGiIu/5cgLv7O8n38+LmyjCNeJKKR9TxcCoZ53LhrwHgfcokElz1aXQvHcNh+IAG0WF0+OxclBiqEwvasyq8PT4r9MMNYwklINcSDO5RdK2xVJJbehFT81LodAJ9ydgrfagAB7haLq6JfdIucvlllmeVyLwznJm/Gtx9WNOFeSVEN0pufJWsiQJelu4S+4OesfdCCzypAXkSWQ2M4vQBcWNYjJ6Ns6svhNX/9wC9+QLPusAieLan2WDFjxrQwUM7n+J3/GxEEPI6heUGNh3n0wimmVpGxZo="' + - 'secure: "ZkEtQyYsMAiomNX0UCgF8ExvCIselUY9z0cSdbatmiSSHRlKxjBk9wS1CQM+Y9+Eeri7aUkS1PbCrz3vNVbTtj9VEZJjxpFNDzmKziIDBJm6DCE5k0JV7dQbKS1jPrX1/+Az1mMCy+EN6ZtA2WUXCLZsqkmG9rdEUJMBM5SYFBCfTzrTWxwqJZU46ErKrdn5d+ywi9O4Cu7/i2JjRpEejC2iYWSDNEgqHvxDo6lbG5sVSn2pns4Dzqwh3n9zLxqWeQalvgisjcGuBoRLTyCEeJNgDL43PkdgqflyanleLzlDCvTowfrqWZF7Xn8TmkZslunlRe3zMBaiDHxg1h60rju4inXzIahHL7keFDB32QWBZb33jf2BZRStjjJDgO8jvPMBmhxwV5LLzSuW0cHxEm+f5O6eWPQjT4dX04dGMabYU7iMgR5t0+uZDqeW17J4py75UXC3u4KWkzWKvRIe78nEXSE4F1SSiV9eQWdBtkJcF0i3UQzQW5quKUWcYp2sw4SEyz/zCc+CLCtjheIg39rrWVSRKL3t4iCebxyVU/YLgOpBW2BVYtY4vUsyLma7Mr+rcVLZggcU4qnoz3Q6jgDwPP1jeNC9RZNw/Tida9hQgykmqNM9OPkgrRVgL3g0NfYMdysupqezplywjvIDODh11rg8+Cjk6nRb28S4wVQ="' - AWS_DEFAULT_REGION="us-east-1" jobs: include: diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 6f773cf81..c6edff1d3 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -52,7 +52,7 @@ def dict_constructor(loader, node): region = 'us-east-1' access_key_id = 'AKIAJKBW6H2VKKQDHILQ' secret_access_key = 'A1wrWjyfpCAPCYfu/Y4JpKOgjaAbZTubDfNur1K4rXqLWsi5JHWW9UUcmVXHHZGxy5wM56dTa5Y5smarjNno+KU21ioZ9u4LKthbMq/aDtLc9bMXbWJ+k1fu+jJT5yZ174NwrYFtyOrkwRcJR7ttfBIapY31IgCCkNQ6NtzFLFsf0rNEaW1K0lZIj8k0MvD5aJ77Pi06zRRZdwTibAu27w+FHQzDYTRfPGcutlS/3zfdvBEWC7FpZK772bJFfUsSZ3tUy8BBLhQztnssC3jCvIv4zFkeG7PnZULPjq4f/0EfvvNt7aF2cxsdbwG16L2Ia++/aS98qgA9+f5u+LB83rt7fWmxSyc47kRmyrXKipv9o/mDjZXW7OmlqHVgwRUBkZ6suPwrrv1ZBAbYCk8uNk5wGt69OyJDsyegEoKSSGkYDhQZ8I5JgbiB1myJf2wUVyyd8g71U0/W0CtboqCXiHYZWPIhyIYzN6n044IoNpWleusIAABqp2TU/zSAM+sOjqJqZ59mNVYU5hpPUGVJPuoZ9TW63oOX1q/eO5XSnl2asoNFjjooTr3A38YQ5PdWz+IbIlBJL35pZgnzxOkyskNIYuwOTexLqx7G4nZj9wgDxUUL8UA48wmiu8MnfNxBeZcnmxPqRPVQM3qf9nhpaM3OaX3Cs/OO5leHK/BSPPg=' -secret_access_key_env = 'gCGZuvBQVZ09rmH5tPUn3GT17NzyBX9JRO3SJwq6WcLbmKDRo50xAhqib/VAKfMXmcteaIPQdxjWUcohPXAoFQ67Z3Jx8abjxEIJ6t7IwIH3aRYWh7PX28Iu5Cr3b/pLbSBw0r+tj/X0mHsbS1dF/uVOBo4fSGN5f4li3EMv+iWQNimwC64uT+ywbzjdUPwJBZ+rVoyh/uxBR/14CpYMi2b1Bf79iVYZr1INShWH+4JNqw6RkJQVhQbAXq9CdSH3bNgyrnASno542cDa9l++xJhJ9okFTdWchiDj1FH4BhBLdEQHVEm790dAZGcE41APg8gW/4hDA8G5wXrCN/T+QtEEgzIDKJ9o/VImabjWcuB1oMERM4Rcvr0wpRNd+DCcPNyEpLPHlOV8WGiIu/5cgLv7O8n38+LmyjCNeJKKR9TxcCoZ53LhrwHgfcokElz1aXQvHcNh+IAG0WF0+OxclBiqEwvasyq8PT4r9MMNYwklINcSDO5RdK2xVJJbehFT81LodAJ9ydgrfagAB7haLq6JfdIucvlllmeVyLwznJm/Gtx9WNOFeSVEN0pufJWsiQJelu4S+4OesfdCCzypAXkSWQ2M4vQBcWNYjJ6Ns6svhNX/9wC9+QLPusAieLan2WDFjxrQwUM7n+J3/GxEEPI6heUGNh3n0wimmVpGxZo=' +secret_access_key_env = 'ZkEtQyYsMAiomNX0UCgF8ExvCIselUY9z0cSdbatmiSSHRlKxjBk9wS1CQM+Y9+Eeri7aUkS1PbCrz3vNVbTtj9VEZJjxpFNDzmKziIDBJm6DCE5k0JV7dQbKS1jPrX1/+Az1mMCy+EN6ZtA2WUXCLZsqkmG9rdEUJMBM5SYFBCfTzrTWxwqJZU46ErKrdn5d+ywi9O4Cu7/i2JjRpEejC2iYWSDNEgqHvxDo6lbG5sVSn2pns4Dzqwh3n9zLxqWeQalvgisjcGuBoRLTyCEeJNgDL43PkdgqflyanleLzlDCvTowfrqWZF7Xn8TmkZslunlRe3zMBaiDHxg1h60rju4inXzIahHL7keFDB32QWBZb33jf2BZRStjjJDgO8jvPMBmhxwV5LLzSuW0cHxEm+f5O6eWPQjT4dX04dGMabYU7iMgR5t0+uZDqeW17J4py75UXC3u4KWkzWKvRIe78nEXSE4F1SSiV9eQWdBtkJcF0i3UQzQW5quKUWcYp2sw4SEyz/zCc+CLCtjheIg39rrWVSRKL3t4iCebxyVU/YLgOpBW2BVYtY4vUsyLma7Mr+rcVLZggcU4qnoz3Q6jgDwPP1jeNC9RZNw/Tida9hQgykmqNM9OPkgrRVgL3g0NfYMdysupqezplywjvIDODh11rg8+Cjk6nRb28S4wVQ=' twine_password = 'JWCpO0caWN4M0xaGPqKMPPHg8WB6sybPQPcCFtXc+dwsD3mu+mDv7lGV3l/5sRoDqIi87uhXBtfnPVKao7AdPaLZoZc8ktBP3b7QwG49U479odN1OuxZcdLBmQBPZlAnqyH7wbM0xhkskr6Yl8gkKWynDfyod1GWbr/plcGRuX82unvuq5tkX8gNSdJ4vHbHGGcvgUbDkyEJdQz7y96qJn7VUpD4FhujO7bGyGVB2ZVcAcEyHuQZiyPxknrXSE0dnk0IeEH77DY40SLpPf5WG4fl/gGYLlccw2ZmBOAOYYE/nq9HF+/b5gnYp+0zLUCgA1tK9uC1UdT3IE24hr8XJMdMIog1BSVjv5k31RjFi0mwCVaasiCN/IOYvwrg+yod5UjXBag7f/Wjo3AEoUSByFERTr+Y7Gs+uCHQKT8Jw4mCTwmI+3JnH+4ZcuMbIQUr10uaocmpNuWhA1IVMTewIKjaAgY9qWy2X5e7X5/faYXlUhV68QiA5TB4MCvA+jeq84+bIk00HBnOadL/iAKFVhBlbkqe8OZWNbsBKFERWEVJFIoOLMg43NORWRs54Mi3eie4bNxWUwwPsxHR4FoBSeWjm2yorDbUIFDK57xZH3jUFu11xSXW3x9BqGWdgQg71j/P1nVKGMoHvPUBX0wO2CLQnSl6katSwgM6aLeJl1U=' data = OrderedDict() From f6382ba39e1af14de857bf0434632fa86fa3423e Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 20 Dec 2017 11:33:48 -0500 Subject: [PATCH 115/126] Update README [skip ci] --- bin/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bin/README.md b/bin/README.md index 1229f2ff7..4b7827da6 100755 --- a/bin/README.md +++ b/bin/README.md @@ -2,14 +2,10 @@ # TODO - Deal with versions and tagging (pypi only accepts non-dev tags) -- Download from S3 and install - - Pick up from the correct travis commit -- Upload to S3 even for PRs (custom deploy) - Recreate and encrypt keys for brainiak/brainiak - Only need to build mpi4py occasionally; not with every commit, and not with every job - mpi4py should always commit to s3://brainiak/.whl - Update documentation -- Figure out why ```setup.py``` fails - End-to-end test - Get things working on Jenkins - Verify using other MPI (e.g., Intel MPI, mpich3) @@ -19,12 +15,15 @@ - Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8)) # To verify -- Fix stages; it appears that everything just gets lumped into a single stage +- Upload to S3 even for PRs (custom deploy) +- Download from S3 and install +- Pick up from the correct travis commit - Upload to TestPyPI - Upload wheels to PyPI (uncomment) - Only upload to PyPI on tagged master # Completed +- Fix stages; it appears that everything just gets lumped into a single stage - Test-with-deps: test source with dependencies + can be installed (doesn't check install now) - Test-wout-deps: test binary without dependencies + can be installed (doesn't check tests now) - Confirm stages that run only on master commit to main don't run for PRs other branchs From e4c5e2c38ede674fd95dbf73c440f2a4aad7c15b Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 20 Dec 2017 17:32:59 -0500 Subject: [PATCH 116/126] Store mpi4py wheels --- bin/build-dist-macos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/build-dist-macos.sh b/bin/build-dist-macos.sh index 41a01a86f..eacb5133b 100755 --- a/bin/build-dist-macos.sh +++ b/bin/build-dist-macos.sh @@ -32,6 +32,7 @@ do $PYTHON setup.py -q bdist_wheel $DELOCATE dist/*.whl popd + mv mpi4py/dist/*.whl dist/ $PYTHON -m pip install . $PYTHON setup.py bdist_wheel From c91d5423dfa9b213bc14047574eb8bbef885c154 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 20 Dec 2017 18:55:44 -0500 Subject: [PATCH 117/126] Update wheel dir --- bin/build-dist-macos.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bin/build-dist-macos.sh b/bin/build-dist-macos.sh index eacb5133b..5a71c1845 100755 --- a/bin/build-dist-macos.sh +++ b/bin/build-dist-macos.sh @@ -6,6 +6,10 @@ set -e # Show explicitly which commands are currently running. set -x +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +WHEEL_DIR=$SCRIPT_DIR/../dist +mkdir -p $WHEEL_DIR + git clone -q https://bitbucket.org/mpi4py/mpi4py pushd mpi4py git checkout 3.0.0 @@ -29,14 +33,13 @@ do pushd mpi4py git clean -f -f -x -d -q - $PYTHON setup.py -q bdist_wheel - $DELOCATE dist/*.whl + $PYTHON setup.py -q bdist_wheel -d $WHEEL_DIR + $DELOCATE $WHEEL_DIR/*.whl popd - mv mpi4py/dist/*.whl dist/ $PYTHON -m pip install . - $PYTHON setup.py bdist_wheel - $DELOCATE dist/*.whl + $PYTHON setup.py bdist_wheel -d $WHEEL_DIR + $DELOCATE $WHEEL_DIR/*.whl # Build source distribution if [ $MAJOR == '3.4' ] From b6e7d34f3fe2b5e7bea242de1d9a8a7f71d4e0c1 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Wed, 20 Dec 2017 21:47:09 -0500 Subject: [PATCH 118/126] Don't pick up wrong platform --- bin/pr-check.sh | 17 +++++++++++++++-- bin/test-wheels-macos.sh | 4 ++-- bin/test-wheels-manylinux1.sh | 4 ++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/bin/pr-check.sh b/bin/pr-check.sh index a08dd161f..05e3b29b9 100755 --- a/bin/pr-check.sh +++ b/bin/pr-check.sh @@ -121,9 +121,22 @@ $activate_venv $venv || { if [ ! -z $WHEEL_DIR ] then + platform="unknown" + unamestr="$(uname)" + if [[ "$unamestr" == "Linux" ]] + then + platform="manylinux" + elif [[ "$unamestr" == "Darwin" ]] + then + platform="macosx" + else + echo "Unrecognized platform." + exit 1 + fi + MAJOR=$(python3 -c "import sys; v = sys.version_info; print(v[0] * 10 + v[1])") - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep cp$MAJOR | grep mpi4py) - BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep cp$MAJOR | grep brainiak) + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep cp$MAJOR | grep mpi4py | grep $platform) + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep cp$MAJOR | grep brainiak | grep $platform) python3 -m pip install -q $MPI4PY_WHEEL python3 -m pip install -q $BRAINIAK_WHEEL diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 22f7d469f..383bd0ea0 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -28,10 +28,10 @@ then popd # Find the appropriate wheel by grepping for the Python version. - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep mpi4py) + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep mpi4py | grep macosx) # TODO: this will actually pick up both wheels since brainiak is in the path - BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep brainiak) + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep brainiak | grep macosx) $PYTHON -m pip install $MPI4PY_WHEEL $PYTHON -m pip install $BRAINIAK_WHEEL diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index dea913ec3..aa8599846 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -15,8 +15,8 @@ then # Test whether we can install without any dependencies # TODO: delete once we have setup.py setup correctly for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep mpi4py) - BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep brainiak) + MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep mpi4py | grep manylinux) + BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep brainiak | grep manylinux) PYTHON=/opt/python/${VERSION}/bin/python3 From 19eecabdc35ae566b058e7d281e57a3009f4ae50 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 21 Dec 2017 01:03:55 -0500 Subject: [PATCH 119/126] Pipeline test --- .travis.yml | 146 ++++++++++++++++++++++++++++++++++++++++--- bin/generate-yaml.py | 13 ++-- 2 files changed, 147 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3b67ee4ac..080fe916e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,81 @@ env: - AWS_DEFAULT_REGION="us-east-1" jobs: include: + - stage: test + language: generic + - os: linux + dist: trusty + sudo: required + language: python + python: '3.4' + before_install: + - python3 -m pip install -U pip awscli twine + script: + - ./bin/pr-check.sh + addons: + apt: + packages: + - build-essential libgomp1 libmpich-dev mpich + - os: linux + dist: trusty + sudo: required + language: python + python: '3.5' + before_install: + - python3 -m pip install -U pip awscli twine + script: + - ./bin/pr-check.sh + addons: + apt: + packages: + - build-essential libgomp1 libmpich-dev mpich + - os: linux + dist: trusty + sudo: required + language: python + python: '3.6' + before_install: + - python3 -m pip install -U pip awscli twine + script: + - ./bin/pr-check.sh + addons: + apt: + packages: + - build-essential libgomp1 libmpich-dev mpich + - os: osx + osx_image: xcode7.3 + language: generic + env: + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + before_install: + - brew update + - brew install python3 + - python3 -m pip install -U pip awscli twine + install: + - brew install llvm mpich + script: + - ./bin/pr-check.sh + - os: osx + osx_image: xcode8 + language: generic + env: + - CC=/usr/local/opt/llvm/bin/clang + - CXX=/usr/local/opt/llvm/bin/clang++ + - LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib $LDFLAGS" + - CPPFLAGS="-I/usr/local/opt/llvm/include $CPPFLAGS" + - HOMEBREW_NO_AUTO_UPDATE=1 + before_install: + - brew update + - brew install python3 + - python3 -m pip install -U pip awscli twine + install: + - brew install llvm mpich + script: + - ./bin/pr-check.sh - stage: build language: generic if: branch = master @@ -134,7 +209,7 @@ jobs: script: - ./bin/test-wheels.sh after_script: - - if [ ! -z $TRAVIS_TAG ]; then twine upload dist/*; fi + - twine upload dist/* - os: osx osx_image: xcode7.3 language: generic @@ -190,12 +265,13 @@ jobs: python: '3.4' before_install: - python3 -m pip install -U pip awscli twine + if: branch = master and repo = danielsuo/brainiak env: - TWINE_REPOSITORY_URL=https://pypi.python.org/pypi - PYPI_REPOSITORY_URL=https://testpypi.python.org script: ./bin/test-wheels.sh after_script: - - ; + - 'true' - os: osx osx_image: xcode7.3 language: generic @@ -206,13 +282,13 @@ jobs: - brew update - brew install python3 - python3 -m pip install -U pip awscli twine - if: branch = master + if: branch = master and repo = danielsuo/brainiak install: - VERSIONS="3.4.4" ./bin/install-python-macos.sh script: - VERIONS="3.4.4" ./bin/test-wheels-macos.sh after_script: - - ; + - 'true' - os: osx osx_image: xcode7.3 language: generic @@ -223,13 +299,13 @@ jobs: - brew update - brew install python3 - python3 -m pip install -U pip awscli twine - if: branch = master + if: branch = master and repo = danielsuo/brainiak install: - VERSIONS="3.5.3" ./bin/install-python-macos.sh script: - VERIONS="3.5.3" ./bin/test-wheels-macos.sh after_script: - - ; + - 'true' - os: osx osx_image: xcode7.3 language: generic @@ -240,13 +316,67 @@ jobs: - brew update - brew install python3 - python3 -m pip install -U pip awscli twine - if: branch = master + if: branch = master and repo = danielsuo/brainiak install: - VERSIONS="3.6.0" ./bin/install-python-macos.sh script: - VERIONS="3.6.0" ./bin/test-wheels-macos.sh after_script: - - ; + - 'true' - stage: pypi language: generic if: branch = master and repo = danielsuo/brainiak and tag IS present + - os: linux + dist: trusty + sudo: required + language: python + python: '3.4' + before_install: + - python3 -m pip install -U pip awscli twine + if: branch = master and repo = danielsuo/brainiak and tag IS present + env: + - PYPI_REPOSITORY_URL=https://pypi.python.org + install: + - ./bin/test-wheels.sh + - os: osx + osx_image: xcode7.3 + language: generic + env: + - PYPI_REPOSITORY_URL=https://pypi.python.org + before_install: + - brew update + - brew install python3 + - python3 -m pip install -U pip awscli twine + if: branch = master and repo = danielsuo/brainiak and tag IS present + install: + - VERSIONS="3.4.4" ./bin/install-python-macos.sh + script: + - VERIONS="3.4.4" ./bin/test-wheels-macos.sh + - os: osx + osx_image: xcode7.3 + language: generic + env: + - PYPI_REPOSITORY_URL=https://pypi.python.org + before_install: + - brew update + - brew install python3 + - python3 -m pip install -U pip awscli twine + if: branch = master and repo = danielsuo/brainiak and tag IS present + install: + - VERSIONS="3.5.3" ./bin/install-python-macos.sh + script: + - VERIONS="3.5.3" ./bin/test-wheels-macos.sh + - os: osx + osx_image: xcode7.3 + language: generic + env: + - PYPI_REPOSITORY_URL=https://pypi.python.org + before_install: + - brew update + - brew install python3 + - python3 -m pip install -U pip awscli twine + if: branch = master and repo = danielsuo/brainiak and tag IS present + install: + - VERSIONS="3.6.0" ./bin/install-python-macos.sh + script: + - VERIONS="3.6.0" ./bin/test-wheels-macos.sh diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index c6edff1d3..8e133b1b5 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -84,10 +84,11 @@ def dict_constructor(loader, node): actions = { 'download-s3': 'aws s3 cp s3://brainiak/$TRAVIS_COMMIT/dist dist --recursive', 'upload-s3': 'aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read', - 'upload-testpypi': 'if [ ! -z $TRAVIS_TAG ]; then twine upload dist/*; fi', + # 'upload-testpypi': 'if [ ! -z $TRAVIS_TAG ]; then twine upload dist/*; fi', + 'upload-testpypi': 'twine upload dist/*', # TODO: populate this command - 'upload-pypi': ';' + 'upload-pypi': 'true' } linux = OrderedDict({ @@ -245,6 +246,7 @@ def dict_constructor(loader, node): })) testpypi_linux = copy.deepcopy(linux) +testpypi_linux['if'] = conditions['master'] testpypi_linux['env'] = [ 'TWINE_REPOSITORY_URL=https://pypi.python.org/pypi', 'PYPI_REPOSITORY_URL=https://testpypi.python.org' @@ -254,6 +256,7 @@ def dict_constructor(loader, node): jobs.append(testpypi_linux) testpypi_macos = copy.deepcopy(build_macos) +testpypi_macos['if'] = conditions['master'] testpypi_macos['env'] = copy.deepcopy(testpypi_linux['env']) for version in versions: @@ -280,11 +283,13 @@ def dict_constructor(loader, node): })) pypi_linux = copy.deepcopy(linux) +pypi_linux['if'] = conditions['tag'] pypi_linux['env'] = ['PYPI_REPOSITORY_URL=https://pypi.python.org'] pypi_linux['install'] = ['./bin/test-wheels.sh'] -# jobs.append(pypi_linux) +jobs.append(pypi_linux) pypi_macos = copy.deepcopy(macos) +pypi_macos['if'] = conditions['tag'] pypi_macos['env'] = copy.deepcopy(pypi_linux['env']) for version in versions: @@ -297,7 +302,7 @@ def dict_constructor(loader, node): 'VERIONS="%s" ./bin/test-wheels-macos.sh' % version ] - # jobs.append(block) + jobs.append(block) with open(travis, 'w') as yml: yaml.dump(data, yml, default_flow_style=False) From c03a7797e4c8ad6813544114b62ac5c98fb7b7c1 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 21 Dec 2017 11:52:20 -0500 Subject: [PATCH 120/126] Update generate yaml [skip ci] --- bin/README.md | 1 + bin/generate-yaml.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/bin/README.md b/bin/README.md index 4b7827da6..f51abc451 100755 --- a/bin/README.md +++ b/bin/README.md @@ -16,6 +16,7 @@ # To verify - Upload to S3 even for PRs (custom deploy) + - NOTE: untrusted builds don't have access to encrypted environment variables or deploy - Download from S3 and install - Pick up from the correct travis commit - Upload to TestPyPI diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 8e133b1b5..37464f418 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -83,7 +83,11 @@ def dict_constructor(loader, node): actions = { 'download-s3': 'aws s3 cp s3://brainiak/$TRAVIS_COMMIT/dist dist --recursive', + + # TODO: make this for master only 'upload-s3': 'aws s3 cp dist s3://brainiak/$TRAVIS_COMMIT/dist --recursive --acl=public-read', + + # TODO: make this for master only / possibly only for tag # 'upload-testpypi': 'if [ ! -z $TRAVIS_TAG ]; then twine upload dist/*; fi', 'upload-testpypi': 'twine upload dist/*', @@ -234,6 +238,8 @@ def dict_constructor(loader, node): 'VERIONS="%s" ./bin/test-wheels-macos.sh' % version ] + block['after_script'] = [actions['upload-testpypi']] + jobs.append(block) ############################################################################### From 2a5078032899d0aa03fcb28e48e6299eae006dcb Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 21 Dec 2017 11:52:33 -0500 Subject: [PATCH 121/126] Generate yaml --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 080fe916e..4b6bcbfa2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -225,6 +225,8 @@ jobs: - VERSIONS="3.4.4" ./bin/install-python-macos.sh script: - VERIONS="3.4.4" ./bin/test-wheels-macos.sh + after_script: + - twine upload dist/* - os: osx osx_image: xcode7.3 language: generic @@ -240,6 +242,8 @@ jobs: - VERSIONS="3.5.3" ./bin/install-python-macos.sh script: - VERIONS="3.5.3" ./bin/test-wheels-macos.sh + after_script: + - twine upload dist/* - os: osx osx_image: xcode7.3 language: generic @@ -255,6 +259,8 @@ jobs: - VERSIONS="3.6.0" ./bin/install-python-macos.sh script: - VERIONS="3.6.0" ./bin/test-wheels-macos.sh + after_script: + - twine upload dist/* - stage: testpypi language: generic if: branch = master and repo = danielsuo/brainiak From 231b9e9b6724d4cf51dcb2b28970c602174eea63 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 21 Dec 2017 11:55:37 -0500 Subject: [PATCH 122/126] Update README [skip ci] --- bin/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/README.md b/bin/README.md index f51abc451..c00071585 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,6 +1,11 @@ # Wheel to-dos # TODO +- Problematic issues with installs + - `pip install` hangs + - `brew` hangs when updating, or unable to download bottle -> forces compilation + - `apt` unreachable + - Python packages incorrectly installed / files not found (e.g., six) - Deal with versions and tagging (pypi only accepts non-dev tags) - Recreate and encrypt keys for brainiak/brainiak - Only need to build mpi4py occasionally; not with every commit, and not with every job From 9e6cc169c6a163ea01aaa1417bebcb6c431ff6d5 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 21 Dec 2017 12:01:23 -0500 Subject: [PATCH 123/126] Update generate yaml --- .travis.yml | 4 ++-- bin/generate-yaml.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4b6bcbfa2..8540ede79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,10 +5,10 @@ env: global: - BRAINIAK_REPO=danielsuo/brainiak - TWINE_USERNAME=brainiak-bot - - 'secure: "JWCpO0caWN4M0xaGPqKMPPHg8WB6sybPQPcCFtXc+dwsD3mu+mDv7lGV3l/5sRoDqIi87uhXBtfnPVKao7AdPaLZoZc8ktBP3b7QwG49U479odN1OuxZcdLBmQBPZlAnqyH7wbM0xhkskr6Yl8gkKWynDfyod1GWbr/plcGRuX82unvuq5tkX8gNSdJ4vHbHGGcvgUbDkyEJdQz7y96qJn7VUpD4FhujO7bGyGVB2ZVcAcEyHuQZiyPxknrXSE0dnk0IeEH77DY40SLpPf5WG4fl/gGYLlccw2ZmBOAOYYE/nq9HF+/b5gnYp+0zLUCgA1tK9uC1UdT3IE24hr8XJMdMIog1BSVjv5k31RjFi0mwCVaasiCN/IOYvwrg+yod5UjXBag7f/Wjo3AEoUSByFERTr+Y7Gs+uCHQKT8Jw4mCTwmI+3JnH+4ZcuMbIQUr10uaocmpNuWhA1IVMTewIKjaAgY9qWy2X5e7X5/faYXlUhV68QiA5TB4MCvA+jeq84+bIk00HBnOadL/iAKFVhBlbkqe8OZWNbsBKFERWEVJFIoOLMg43NORWRs54Mi3eie4bNxWUwwPsxHR4FoBSeWjm2yorDbUIFDK57xZH3jUFu11xSXW3x9BqGWdgQg71j/P1nVKGMoHvPUBX0wO2CLQnSl6katSwgM6aLeJl1U="' - AWS_ACCESS_KEY_ID="AKIAJKBW6H2VKKQDHILQ" - - 'secure: "ZkEtQyYsMAiomNX0UCgF8ExvCIselUY9z0cSdbatmiSSHRlKxjBk9wS1CQM+Y9+Eeri7aUkS1PbCrz3vNVbTtj9VEZJjxpFNDzmKziIDBJm6DCE5k0JV7dQbKS1jPrX1/+Az1mMCy+EN6ZtA2WUXCLZsqkmG9rdEUJMBM5SYFBCfTzrTWxwqJZU46ErKrdn5d+ywi9O4Cu7/i2JjRpEejC2iYWSDNEgqHvxDo6lbG5sVSn2pns4Dzqwh3n9zLxqWeQalvgisjcGuBoRLTyCEeJNgDL43PkdgqflyanleLzlDCvTowfrqWZF7Xn8TmkZslunlRe3zMBaiDHxg1h60rju4inXzIahHL7keFDB32QWBZb33jf2BZRStjjJDgO8jvPMBmhxwV5LLzSuW0cHxEm+f5O6eWPQjT4dX04dGMabYU7iMgR5t0+uZDqeW17J4py75UXC3u4KWkzWKvRIe78nEXSE4F1SSiV9eQWdBtkJcF0i3UQzQW5quKUWcYp2sw4SEyz/zCc+CLCtjheIg39rrWVSRKL3t4iCebxyVU/YLgOpBW2BVYtY4vUsyLma7Mr+rcVLZggcU4qnoz3Q6jgDwPP1jeNC9RZNw/Tida9hQgykmqNM9OPkgrRVgL3g0NfYMdysupqezplywjvIDODh11rg8+Cjk6nRb28S4wVQ="' - AWS_DEFAULT_REGION="us-east-1" + - secure: JWCpO0caWN4M0xaGPqKMPPHg8WB6sybPQPcCFtXc+dwsD3mu+mDv7lGV3l/5sRoDqIi87uhXBtfnPVKao7AdPaLZoZc8ktBP3b7QwG49U479odN1OuxZcdLBmQBPZlAnqyH7wbM0xhkskr6Yl8gkKWynDfyod1GWbr/plcGRuX82unvuq5tkX8gNSdJ4vHbHGGcvgUbDkyEJdQz7y96qJn7VUpD4FhujO7bGyGVB2ZVcAcEyHuQZiyPxknrXSE0dnk0IeEH77DY40SLpPf5WG4fl/gGYLlccw2ZmBOAOYYE/nq9HF+/b5gnYp+0zLUCgA1tK9uC1UdT3IE24hr8XJMdMIog1BSVjv5k31RjFi0mwCVaasiCN/IOYvwrg+yod5UjXBag7f/Wjo3AEoUSByFERTr+Y7Gs+uCHQKT8Jw4mCTwmI+3JnH+4ZcuMbIQUr10uaocmpNuWhA1IVMTewIKjaAgY9qWy2X5e7X5/faYXlUhV68QiA5TB4MCvA+jeq84+bIk00HBnOadL/iAKFVhBlbkqe8OZWNbsBKFERWEVJFIoOLMg43NORWRs54Mi3eie4bNxWUwwPsxHR4FoBSeWjm2yorDbUIFDK57xZH3jUFu11xSXW3x9BqGWdgQg71j/P1nVKGMoHvPUBX0wO2CLQnSl6katSwgM6aLeJl1U= + - secure: ZkEtQyYsMAiomNX0UCgF8ExvCIselUY9z0cSdbatmiSSHRlKxjBk9wS1CQM+Y9+Eeri7aUkS1PbCrz3vNVbTtj9VEZJjxpFNDzmKziIDBJm6DCE5k0JV7dQbKS1jPrX1/+Az1mMCy+EN6ZtA2WUXCLZsqkmG9rdEUJMBM5SYFBCfTzrTWxwqJZU46ErKrdn5d+ywi9O4Cu7/i2JjRpEejC2iYWSDNEgqHvxDo6lbG5sVSn2pns4Dzqwh3n9zLxqWeQalvgisjcGuBoRLTyCEeJNgDL43PkdgqflyanleLzlDCvTowfrqWZF7Xn8TmkZslunlRe3zMBaiDHxg1h60rju4inXzIahHL7keFDB32QWBZb33jf2BZRStjjJDgO8jvPMBmhxwV5LLzSuW0cHxEm+f5O6eWPQjT4dX04dGMabYU7iMgR5t0+uZDqeW17J4py75UXC3u4KWkzWKvRIe78nEXSE4F1SSiV9eQWdBtkJcF0i3UQzQW5quKUWcYp2sw4SEyz/zCc+CLCtjheIg39rrWVSRKL3t4iCebxyVU/YLgOpBW2BVYtY4vUsyLma7Mr+rcVLZggcU4qnoz3Q6jgDwPP1jeNC9RZNw/Tida9hQgykmqNM9OPkgrRVgL3g0NfYMdysupqezplywjvIDODh11rg8+Cjk6nRb28S4wVQ= jobs: include: - stage: test diff --git a/bin/generate-yaml.py b/bin/generate-yaml.py index 37464f418..dcff7a06e 100755 --- a/bin/generate-yaml.py +++ b/bin/generate-yaml.py @@ -65,10 +65,14 @@ def dict_constructor(loader, node): 'global': [ 'BRAINIAK_REPO=%s' % repo, 'TWINE_USERNAME=%s' % twine_username, - 'secure: "%s"' % twine_password, 'AWS_ACCESS_KEY_ID="%s"' % access_key_id, - 'secure: "%s"' % secret_access_key_env, - 'AWS_DEFAULT_REGION="%s"' % region + 'AWS_DEFAULT_REGION="%s"' % region, + { + 'secure': twine_password + }, + { + 'secure': secret_access_key_env + } ] }) From 5bd6daefdf1cfdf841315f21c6c912c7f94e0e74 Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 21 Dec 2017 12:53:25 -0500 Subject: [PATCH 124/126] Update readme [skip ci] --- bin/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/README.md b/bin/README.md index c00071585..dfb1ac92e 100755 --- a/bin/README.md +++ b/bin/README.md @@ -1,6 +1,7 @@ # Wheel to-dos # TODO +- Investigate [caching](https://docs.travis-ci.com/user/caching) for pip / brew and for getting wheels from job to job - Problematic issues with installs - `pip install` hangs - `brew` hangs when updating, or unable to download bottle -> forces compilation From 6d01809736d4e222901378987836ef353556573e Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Thu, 21 Dec 2017 13:05:43 -0500 Subject: [PATCH 125/126] Check if we can pass build without building or installing mpi4py wheels every build --- bin/README.md | 4 ++-- bin/build-dist-macos.sh | 20 ++++++++++---------- bin/build-dist-manylinux1.sh | 20 ++++++++++---------- bin/build-wheel.sh | 19 +++++++++++++++++++ bin/install-wheel.sh | 3 +++ bin/pr-check.sh | 4 ++-- bin/test-wheels-macos.sh | 4 ++-- bin/test-wheels-manylinux1.sh | 4 ++-- 8 files changed, 50 insertions(+), 28 deletions(-) create mode 100644 bin/build-wheel.sh create mode 100644 bin/install-wheel.sh diff --git a/bin/README.md b/bin/README.md index dfb1ac92e..82d85fba8 100755 --- a/bin/README.md +++ b/bin/README.md @@ -9,8 +9,6 @@ - Python packages incorrectly installed / files not found (e.g., six) - Deal with versions and tagging (pypi only accepts non-dev tags) - Recreate and encrypt keys for brainiak/brainiak -- Only need to build mpi4py occasionally; not with every commit, and not with every job -- mpi4py should always commit to s3://brainiak/.whl - Update documentation - End-to-end test - Get things working on Jenkins @@ -28,6 +26,8 @@ - Upload to TestPyPI - Upload wheels to PyPI (uncomment) - Only upload to PyPI on tagged master +- Only need to build mpi4py occasionally; not with every commit, and not with every job +- mpi4py should always commit to s3://brainiak/.whl # Completed - Fix stages; it appears that everything just gets lumped into a single stage diff --git a/bin/build-dist-macos.sh b/bin/build-dist-macos.sh index 5a71c1845..2f3c1f2f1 100755 --- a/bin/build-dist-macos.sh +++ b/bin/build-dist-macos.sh @@ -10,10 +10,10 @@ SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) WHEEL_DIR=$SCRIPT_DIR/../dist mkdir -p $WHEEL_DIR -git clone -q https://bitbucket.org/mpi4py/mpi4py -pushd mpi4py -git checkout 3.0.0 -popd +# git clone -q https://bitbucket.org/mpi4py/mpi4py +# pushd mpi4py +# git checkout 3.0.0 +# popd for VERSION in $VERSIONS do @@ -31,11 +31,11 @@ do source venv/bin/activate popd - pushd mpi4py - git clean -f -f -x -d -q - $PYTHON setup.py -q bdist_wheel -d $WHEEL_DIR - $DELOCATE $WHEEL_DIR/*.whl - popd + # pushd mpi4py + # git clean -f -f -x -d -q + # $PYTHON setup.py -q bdist_wheel -d $WHEEL_DIR + # $DELOCATE $WHEEL_DIR/*.whl + # popd $PYTHON -m pip install . $PYTHON setup.py bdist_wheel -d $WHEEL_DIR @@ -51,4 +51,4 @@ do rm -rf ../venv done -rm -rf mpi4py +# rm -rf mpi4py diff --git a/bin/build-dist-manylinux1.sh b/bin/build-dist-manylinux1.sh index 609f07ba0..c17e3d3d8 100755 --- a/bin/build-dist-manylinux1.sh +++ b/bin/build-dist-manylinux1.sh @@ -17,20 +17,20 @@ mkdir -p tmp # - Use the brainiak source stored in /brainiak via docker run command # - Use head of mpi4py (we can pick some more suitable tag) -git clone -q https://bitbucket.org/mpi4py/mpi4py /mpi4py -pushd /mpi4py -git checkout 3.0.0 -popd +# git clone -q https://bitbucket.org/mpi4py/mpi4py /mpi4py +# pushd /mpi4py +# git checkout 3.0.0 +# popd for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do PYTHON=/opt/python/$VERSION/bin/python $PYTHON -m pip install -U pip wheel - pushd /mpi4py - git clean -f -f -x -d -q - $PYTHON setup.py -q bdist_wheel -d tmp - auditwheel repair tmp/*.whl -w $WHEEL_DIR - popd + # pushd /mpi4py + # git clean -f -f -x -d -q + # $PYTHON setup.py -q bdist_wheel -d tmp + # auditwheel repair tmp/*.whl -w $WHEEL_DIR + # popd git clean -f -f -x -d -q -e dist $PYTHON -m pip install -q . @@ -44,5 +44,5 @@ for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do fi done -rm -rf /mpi4py +# rm -rf /mpi4py rm -rf tmp diff --git a/bin/build-wheel.sh b/bin/build-wheel.sh new file mode 100644 index 000000000..86bb02b1e --- /dev/null +++ b/bin/build-wheel.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + + +$PYTHON -m pip install -q . +$PYTHON setup.py -q bdist_wheel -d tmp + + platform="unknown" + unamestr="$(uname)" + if [[ "$unamestr" == "Linux" ]] + then + auditwheel repair tmp/*.whl -w $WHEEL_DIR + elif [[ "$unamestr" == "Darwin" ]] + then + $DELOCATE tmp/*.whl + mv tmp/*.whl $WHEEL_DIR + else + echo "Unrecognized platform." + exit 1 + fi diff --git a/bin/install-wheel.sh b/bin/install-wheel.sh new file mode 100644 index 000000000..1329677cd --- /dev/null +++ b/bin/install-wheel.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + + diff --git a/bin/pr-check.sh b/bin/pr-check.sh index 05e3b29b9..ca39e351b 100755 --- a/bin/pr-check.sh +++ b/bin/pr-check.sh @@ -135,10 +135,10 @@ then fi MAJOR=$(python3 -c "import sys; v = sys.version_info; print(v[0] * 10 + v[1])") - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep cp$MAJOR | grep mpi4py | grep $platform) + # MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep cp$MAJOR | grep mpi4py | grep $platform) BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep cp$MAJOR | grep brainiak | grep $platform) - python3 -m pip install -q $MPI4PY_WHEEL + # python3 -m pip install -q $MPI4PY_WHEEL python3 -m pip install -q $BRAINIAK_WHEEL # We don't want to install brainiak from source diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index 383bd0ea0..a6551fa3a 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -28,12 +28,12 @@ then popd # Find the appropriate wheel by grepping for the Python version. - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep mpi4py | grep macosx) + # MPI4PY_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep mpi4py | grep macosx) # TODO: this will actually pick up both wheels since brainiak is in the path BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep brainiak | grep macosx) - $PYTHON -m pip install $MPI4PY_WHEEL + # $PYTHON -m pip install $MPI4PY_WHEEL $PYTHON -m pip install $BRAINIAK_WHEEL deactivate diff --git a/bin/test-wheels-manylinux1.sh b/bin/test-wheels-manylinux1.sh index aa8599846..c0a722a9f 100755 --- a/bin/test-wheels-manylinux1.sh +++ b/bin/test-wheels-manylinux1.sh @@ -15,7 +15,7 @@ then # Test whether we can install without any dependencies # TODO: delete once we have setup.py setup correctly for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do - MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep mpi4py | grep manylinux) + # MPI4PY_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep mpi4py | grep manylinux) BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f | grep $VERSION | grep brainiak | grep manylinux) PYTHON=/opt/python/${VERSION}/bin/python3 @@ -25,7 +25,7 @@ then $PYTHON -m venv venv source venv/bin/activate - $PYTHON -m pip install -q $MPI4PY_WHEEL + # $PYTHON -m pip install -q $MPI4PY_WHEEL $PYTHON -m pip install -q $BRAINIAK_WHEEL deactivate From e2afb078739b55e89c57e7ac6aaff7cf43e8cd5c Mon Sep 17 00:00:00 2001 From: Daniel Suo Date: Tue, 9 Jan 2018 15:44:54 -0800 Subject: [PATCH 126/126] [skip ci] Fix bug where test-wheels-macos grabs wrong python --- bin/build-dist-macos.sh | 2 +- bin/test-wheels-macos.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/build-dist-macos.sh b/bin/build-dist-macos.sh index 2f3c1f2f1..8159a6f47 100755 --- a/bin/build-dist-macos.sh +++ b/bin/build-dist-macos.sh @@ -23,7 +23,7 @@ do PYTHON=$PYTHON_DIR/python$MAJOR DELOCATE=$(dirname $PYTHON)/delocate-wheel - git clean -f -f -x -d -q -e dist -e mpi4py + git clean -f -f -x -d -q -e dist # Use virtual environments because we want to install later pushd .. diff --git a/bin/test-wheels-macos.sh b/bin/test-wheels-macos.sh index a6551fa3a..4f96e7d7b 100755 --- a/bin/test-wheels-macos.sh +++ b/bin/test-wheels-macos.sh @@ -34,7 +34,7 @@ then BRAINIAK_WHEEL=$(find $WHEEL_DIR -type f -maxdepth 1 -print | grep "$WHEEL_VERSION" | grep brainiak | grep macosx) # $PYTHON -m pip install $MPI4PY_WHEEL - $PYTHON -m pip install $BRAINIAK_WHEEL + python3 -m pip install $BRAINIAK_WHEEL deactivate rm -rf ../venv