diff --git a/.circleci/config.yml b/.circleci/config.yml index b7dd30fa7..5125eb943 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,10 +64,14 @@ jobs: TAHOE_LAFS_TOX_ENVIRONMENT: "coverage" # Additional arguments to pass to tox. TAHOE_LAFS_TOX_ARGS: "" + # Convince all of our pip invocations to look at the cached wheelhouse + # we maintain. + WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse" + PIP_FIND_LINKS: "file:///tmp/wheelhouse" steps: - run: &INSTALL_GIT - node: "Install Git" + name: "Install Git" command: | apt-get --quiet update apt-get --quiet --yes install git @@ -79,6 +83,36 @@ jobs: command: | ~/project/.circleci/bootstrap-test-environment.sh ~/project "${EXTRA_PACKAGES}" + - restore_cache: &RESTORE_HTTP_CACHE + name: "Restoring pip HTTP cache" + keys: + # An exact match on the http cache key is great. It should have + # exactly the packages (tgz, whl, whatever) we need. + - v5-pip-http-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }} + # A prefix match is okay too. It might have a + # partially-overlapping set of packages. That's a head-start, at + # least. We might have to download a few more things but at least + # we saved a little time. After we download some more stuff we'll + # create a new cache entry with the full key above and the next + # build will get a better cache hit. + - v5-pip-http- + + - restore_cache: &RESTORE_WHEELHOUSE + name: "Restoring wheelhouse" + keys: + # As above, an exact match is great. Here, we also need to + # include the job name to make sure the platform ABI matches. + # There are binary wheels in this wheelhouse and we're not taking + # care to make manylinux1 wheels. The binary wheels in this cache + # will only work on some Linux distros. + - v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }} + # A partial match is okay too. It'll get us at least some of the + # wheels. We do need to keep the job name as part of the key or + # we might get binary wheels build against an incompatible ABI and + # we won't be able to use them (and they'll break the build rather + # than being ignored). + - v4-wheelhouse-{{ .Environment.CIRCLE_JOB }} + - run: &SETUP_VIRTUALENV name: "Setup virtualenv" # pip cannot install packages if the working directory is not @@ -89,6 +123,21 @@ jobs: "${TAHOE_LAFS_TOX_ENVIRONMENT}" \ "${TAHOE_LAFS_TOX_ARGS}" + - save_cache: &SAVE_HTTP_CACHE + name: "Saving pip HTTP cache" + key: v5-pip-http-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }} + paths: + # Perfectly valid for Linux. Note we exclude the wheel cache + # because we want this cache to be valid across all platforms and + # the wheels in the pip wheel cache are not necessarily so. + - "/tmp/nobody/.cache/pip/http" + + - save_cache: &SAVE_WHEELHOUSE + name: "Caching wheelhouse" + key: v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }} + paths: + - *WHEELHOUSE_PATH + - run: &RUN_TESTS name: "Run test suite" # Something about when it re-uses an existing environment blows up @@ -183,7 +232,7 @@ jobs: steps: - run: - node: "Install Git" + name: "Install Git" command: | yum install --assumeyes git @@ -219,7 +268,14 @@ jobs: yum install --assumeyes \ net-tools + - restore_cache: *RESTORE_HTTP_CACHE + - restore_cache: *RESTORE_WHEELHOUSE + - run: *SETUP_VIRTUALENV + + - save_cache: *SAVE_HTTP_CACHE + - save_cache: *SAVE_WHEELHOUSE + - run: *RUN_TESTS - store_test_results: *STORE_TEST_RESULTS @@ -248,7 +304,7 @@ jobs: steps: - run: - node: "Install Git" + name: "Install Git" command: | slackpkg update slackpkg install openssh-7.4p1 git-2.14.4 = 1.2) are incompatible with the old TLS clients # available to those systems. Installing it ahead of time (with pip) avoids -# this problem. -sudo --set-home -u nobody /tmp/tests/bin/pip install certifi +# this problem. Make sure this step comes before any other attempts to +# install things using pip! +sudo --set-home -u nobody \ + PIP_FIND_LINKS="${PIP_FIND_LINKS}" \ + /tmp/tests/bin/pip install certifi -# Python packages we need to support the test infrastructure. *Not* packages -# Tahoe-LAFS itself (implementation or test suite) need. -TEST_DEPS="tox codecov" +# Get a new, awesome version of pip and setuptools. For example, the +# distro-packaged virtualenv's pip may not know about wheels. +sudo --set-home -u nobody \ + PIP_FIND_LINKS="${PIP_FIND_LINKS}" \ + /tmp/tests/bin/pip install --upgrade pip setuptools wheel -# Python packages we need to generate test reports for CI infrastructure. -# *Not* packages Tahoe-LAFS itself (implement or test suite) need. -REPORTING_DEPS="python-subunit junitxml subunitreporter" +# Populate the wheelhouse, if necessary. +sudo --set-home -u nobody \ + PIP_FIND_LINKS="${PIP_FIND_LINKS}" \ + /tmp/tests/bin/pip \ + wheel \ + --wheel-dir "${WHEELHOUSE_PATH}" \ + /tmp/project ${TEST_DEPS} ${REPORTING_DEPS} -sudo --set-home -u nobody /tmp/tests/bin/pip install ${TEST_DEPS} ${REPORTING_DEPS} +sudo --set-home -u nobody \ + PIP_FIND_LINKS="${PIP_FIND_LINKS}" \ + /tmp/tests/bin/pip install ${TEST_DEPS} ${REPORTING_DEPS} # Get everything else installed in it, too. -sudo --set-home -u nobody /tmp/tests/bin/tox \ +sudo --set-home -u nobody \ + PIP_FIND_LINKS="${PIP_FIND_LINKS}" \ + /tmp/tests/bin/tox \ -c /tmp/project/tox.ini \ --workdir /tmp/tahoe-lafs.tox \ --notest \ diff --git a/tox.ini b/tox.ini index f7a7f96b3..5a08728f8 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ skipsdist = True [testenv] basepython=python2.7 -passenv = TAHOE_LAFS_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH +passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH # Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes # a package to be installed (with setuptools) then it'll fail on certain # platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS