danielsuo · Dec 13, 2017 · Dec 13, 2017 · Dec 13, 2017 · Dec 13, 2017 · Dec 13, 2017
diff --git a/.travis.yml b/.travis.yml
diff --git a/bin/README.md b/bin/README.md
@@ -0,0 +1,70 @@
+# Wheel to-dos
+
+# TODO
+- Investigate [caching](https://docs.travis-ci.com/user/caching) for pip / brew and for getting wheels from job to job
+- Problematic issues with installs
+   - `pip install` hangs
+   - `brew` hangs when updating, or unable to download bottle -> forces compilation
+   - `apt` unreachable
+   - Python packages incorrectly installed / files not found (e.g., six)
+- Deal with versions and tagging (pypi only accepts non-dev tags)
+- Recreate and encrypt keys for brainiak/brainiak
+- Update documentation
+- End-to-end test
+- Get things working on Jenkins
+- Verify using other MPI (e.g., Intel MPI, mpich3)
+- Build for other OSs (e.g., xcode[VERSION], 32-bit Linux, Windows)
+   - Rename MacOS wheels for multiple versions?
+- Test on other python distros (e.g., conda)?
+- Change quiet to no progress bar (merged, but not released in pip. See [here](https://github.com/pypa/pip/pull/4194/commits/0124945031e93236c2300eb45c2f962768be62d8))
+
+# To verify
+- Upload to S3 even for PRs (custom deploy)
+   - NOTE: untrusted builds don't have access to encrypted environment variables or deploy
+- Download from S3 and install
+- Pick up from the correct travis commit
+- Upload to TestPyPI
+- Upload wheels to PyPI (uncomment)
+- Only upload to PyPI on tagged master
+- Only need to build mpi4py occasionally; not with every commit, and not with every job
+- mpi4py should always commit to s3://brainiak/.whl
+
+# Completed
+- Fix stages; it appears that everything just gets lumped into a single stage
+- Test-with-deps: test source with dependencies + can be installed (doesn't check install now)
+- Test-wout-deps: test binary without dependencies + can be installed (doesn't check tests now)
+- Confirm stages that run only on master commit to main don't run for PRs other branchs
+- Clean up buckets appropriately
+- Change upload / download to $TRAVIS_COMMIT/.whl
+- Test local install (```pip``` vs. ```python setup.py```, editable vs. not)
+- Only deploy on tag (uncomment)
+- Deploy only on master (uncomment in ```.travis.yml```)
+- Default implementation should grab wheels matching current brainiak version
+- Allow setup.py to take argument for mpi4py wheel location
+- Modify ```setup.py``` to handle conditional MPI install
+- Move all scripts to bin
+- Upload wheels to S3
+- Split test and build stages
+- Add bdist exception for editable installs
+- Enforce BrainIAK dependencies while installing dev requirements
+- Add back quiet / silent flags
+- Refactor build and test scripts
+- Consolidate python version numbers across various scripts
+- Rationalize variables; different scripts call the same things different things
+- Resolve coverage running pytest as module
+- Split wheel build, test install, test run into three travis stages
+- Separate install python and build wheels in MacOS
+- Create brainiak/manylinux Docker image
+- Have dev and prod S3 buckets for testing wheels
+- Add LICENSE into whl file
+- Checkout mpi4py tag so we always create the same wheel
+- Figure out ```pr-check.sh```, other testing scripts, and documentation
+   - Change ```pr-check.sh``` to optionally take a ```PYTHON``` argument, using ```python3``` if unspecified
+   - Change ```pr-check.sh``` to optionally install a wheel (or automatically find the wheel based on the ```PYTHON```argument) insead of from source
+- Restore ```.travis.yml``` and make sure previous tests working
+- Linux build script
+- Linux test script
+- MacOS build script
+- MacOS test script
+- ~~Export PATH and clean up logic~~
+- ~~Package documentation [link](http://python-packaging.readthedocs.io/en/latest/non-code-files.html)~~
diff --git a/bin/build-dist-macos.sh b/bin/build-dist-macos.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Cause the script to exit if a single command fails.
+set -e
+
+# Show explicitly which commands are currently running.
+set -x
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+WHEEL_DIR=$SCRIPT_DIR/../dist
+mkdir -p $WHEEL_DIR
+
+# git clone -q https://bitbucket.org/mpi4py/mpi4py
+# pushd mpi4py
+# git checkout 3.0.0
+# popd
+
+for VERSION in $VERSIONS
+do
+  MAJOR=${VERSION%.*}
+  # Replicate readlink -f
+  PYTHON_DIR=$(dirname $(realpath $(which python$MAJOR)))
+  PYTHON=$PYTHON_DIR/python$MAJOR
+  DELOCATE=$(dirname $PYTHON)/delocate-wheel
+
+  git clean -f -f -x -d -q -e dist
+
+  # Use virtual environments because we want to install later
+  pushd ..
+  $PYTHON -m venv venv
+  source venv/bin/activate
+  popd
+
+  # pushd mpi4py
+    # git clean -f -f -x -d -q
+    # $PYTHON setup.py -q bdist_wheel -d $WHEEL_DIR
+    # $DELOCATE $WHEEL_DIR/*.whl
+  # popd
+
+  $PYTHON -m pip install .
+  $PYTHON setup.py bdist_wheel -d $WHEEL_DIR
+  $DELOCATE $WHEEL_DIR/*.whl
+
+  # Build source distribution
+  if [ $MAJOR == '3.4' ]
+  then
+     $PYTHON setup.py sdist
+  fi
+
+  deactivate
+  rm -rf ../venv
+done
+
+# rm -rf mpi4py
diff --git a/bin/build-dist-manylinux1.sh b/bin/build-dist-manylinux1.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# Caust the script to exit if a single command fails.
+set -e
+
+# Show explicitly which commands are currently running.
+set -x
+
+# Install dependencies
+yum install -y -q \
+   mpich2-devel
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+WHEEL_DIR=$SCRIPT_DIR/../dist
+mkdir -p $WHEEL_DIR
+mkdir -p tmp
+
+# - Use the brainiak source stored in /brainiak via docker run command
+# - Use head of mpi4py (we can pick some more suitable tag)
+# git clone -q https://bitbucket.org/mpi4py/mpi4py /mpi4py
+# pushd /mpi4py
+# git checkout 3.0.0
+# popd
+
+for VERSION in cp34-cp34m cp35-cp35m cp36-cp36m; do
+   PYTHON=/opt/python/$VERSION/bin/python
+   $PYTHON -m pip install -U pip wheel
+
+   # pushd /mpi4py
+      # git clean -f -f -x -d -q
+      # $PYTHON setup.py -q bdist_wheel -d tmp
+      # auditwheel repair tmp/*.whl -w $WHEEL_DIR
+   # popd
+
+   git clean -f -f -x -d -q -e dist
+   $PYTHON -m pip install -q .
+   $PYTHON setup.py -q bdist_wheel -d tmp
+   auditwheel repair tmp/*.whl -w $WHEEL_DIR
+
+   # Build source distribution
+   if [ $VERSION = "cp34-cp34m" ]
+   then
+      $PYTHON setup.py -q sdist
+   fi
+done
+
+# rm -rf /mpi4py
+rm -rf tmp
diff --git a/bin/build-dist.sh b/bin/build-dist.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker run --rm -w /src -v `pwd`:/src -ti brainiak/manylinux ./bin/build-dist-manylinux1.sh
diff --git a/bin/build-wheel.sh b/bin/build-wheel.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+
+$PYTHON -m pip install -q .
+$PYTHON setup.py -q bdist_wheel -d tmp
+
+ platform="unknown"
+ unamestr="$(uname)"
+ if [[ "$unamestr" == "Linux" ]]
+ then
+    auditwheel repair tmp/*.whl -w $WHEEL_DIR
+ elif [[ "$unamestr" == "Darwin" ]]
+ then
+    $DELOCATE tmp/*.whl
+    mv tmp/*.whl $WHEEL_DIR
+ else
+    echo "Unrecognized platform."
+    exit 1
+ fi
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		#!/usr/bin/env bash

		docker run --rm -w /src -v `pwd`:/src -ti brainiak/manylinux ./bin/build-dist-manylinux1.sh