.circleci/config.yml

@@ -64,10 +64,14 @@ jobs:
       TAHOE_LAFS_TOX_ENVIRONMENT: "coverage"
       # Additional arguments to pass to tox.
       TAHOE_LAFS_TOX_ARGS: ""
+      # Convince all of our pip invocations to look at the cached wheelhouse
+      # we maintain.
+      WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
+      PIP_FIND_LINKS: "file:///tmp/wheelhouse"
 
     steps:
       - run: &INSTALL_GIT
-          node: "Install Git"
+          name: "Install Git"
           command: |
             apt-get --quiet update
             apt-get --quiet --yes install git
@@ -79,6 +83,36 @@ jobs:
           command: |
             ~/project/.circleci/bootstrap-test-environment.sh ~/project "${EXTRA_PACKAGES}"
 
+      - restore_cache: &RESTORE_HTTP_CACHE
+          name: "Restoring pip HTTP cache"
+          keys:
+            # An exact match on the http cache key is great.  It should have
+            # exactly the packages (tgz, whl, whatever) we need.
+            - v5-pip-http-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
+            # A prefix match is okay too.  It might have a
+            # partially-overlapping set of packages.  That's a head-start, at
+            # least.  We might have to download a few more things but at least
+            # we saved a little time.  After we download some more stuff we'll
+            # create a new cache entry with the full key above and the next
+            # build will get a better cache hit.
+            - v5-pip-http-
+
+      - restore_cache: &RESTORE_WHEELHOUSE
+          name: "Restoring wheelhouse"
+          keys:
+            # As above, an exact match is great.  Here, we also need to
+            # include the job name to make sure the platform ABI matches.
+            # There are binary wheels in this wheelhouse and we're not taking
+            # care to make manylinux1 wheels.  The binary wheels in this cache
+            # will only work on some Linux distros.
+            - v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
+            # A partial match is okay too.  It'll get us at least some of the
+            # wheels.  We do need to keep the job name as part of the key or
+            # we might get binary wheels build against an incompatible ABI and
+            # we won't be able to use them (and they'll break the build rather
+            # than being ignored).
+            - v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}
+
       - run: &SETUP_VIRTUALENV
           name: "Setup virtualenv"
           # pip cannot install packages if the working directory is not
@@ -89,6 +123,21 @@ jobs:
                 "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
                 "${TAHOE_LAFS_TOX_ARGS}"
 
+      - save_cache: &SAVE_HTTP_CACHE
+          name: "Saving pip HTTP cache"
+          key: v5-pip-http-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
+          paths:
+            # Perfectly valid for Linux.  Note we exclude the wheel cache
+            # because we want this cache to be valid across all platforms and
+            # the wheels in the pip wheel cache are not necessarily so.
+            - "/tmp/nobody/.cache/pip/http"
+
+      - save_cache: &SAVE_WHEELHOUSE
+          name: "Caching wheelhouse"
+          key: v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
+          paths:
+            - *WHEELHOUSE_PATH
+
       - run: &RUN_TESTS
           name: "Run test suite"
           # Something about when it re-uses an existing environment blows up
@@ -183,7 +232,7 @@ jobs:
 
     steps:
       - run:
-          node: "Install Git"
+          name: "Install Git"
           command: |
             yum install --assumeyes git
 
@@ -219,7 +268,14 @@ jobs:
             yum install --assumeyes \
                 net-tools
 
+      - restore_cache: *RESTORE_HTTP_CACHE
+      - restore_cache: *RESTORE_WHEELHOUSE
+
       - run: *SETUP_VIRTUALENV
+
+      - save_cache: *SAVE_HTTP_CACHE
+      - save_cache: *SAVE_WHEELHOUSE
+
       - run: *RUN_TESTS
 
       - store_test_results: *STORE_TEST_RESULTS
@@ -248,7 +304,7 @@ jobs:
 
     steps:
       - run:
-          node: "Install Git"
+          name: "Install Git"
           command: |
             slackpkg update
             slackpkg install openssh-7.4p1 git-2.14.4 </dev/null
@@ -301,7 +357,14 @@ jobs:
             python get-pip.py
             pip install virtualenv
 
+      - restore_cache: *RESTORE_HTTP_CACHE
+      - restore_cache: *RESTORE_WHEELHOUSE
+
       - run: *SETUP_VIRTUALENV
+
+      - save_cache: *SAVE_HTTP_CACHE
+      - save_cache: *SAVE_WHEELHOUSE
+
       - run: *RUN_TESTS
 
       - store_test_results: *STORE_TEST_RESULTS

.circleci/run-tests.sh

@@ -31,6 +31,7 @@ JUNITXML="${ARTIFACTS}"/junit/unittests/results.xml
 sudo \
     SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}" \
     TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file" \
+    PIP_NO_INDEX="1" \
     --set-home \
     --user nobody \
     /tmp/tests/bin/tox \

.circleci/setup-virtualenv.sh

@@ -6,6 +6,19 @@ shift
 TAHOE_LAFS_TOX_ARGS=$1
 shift || :
 
+# Python packages we need to support the test infrastructure.  *Not* packages
+# Tahoe-LAFS itself (implementation or test suite) need.
+TEST_DEPS="tox codecov"
+
+# Python packages we need to generate test reports for CI infrastructure.
+# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
+REPORTING_DEPS="python-subunit junitxml subunitreporter"
+
+# Make sure the ownership of the pip cache directory is correct.  The CircleCI
+# cache management operations seem to mess it up.  The cache directory might
+# not exist if there was no matching cache to restore.
+[ -e /tmp/nobody/.cache ] && chown --recursive nobody /tmp/nobody/.cache
+
 # Set up the virtualenv as a non-root user so we can run the test suite as a
 # non-root user.  See below.
 sudo --set-home -u nobody virtualenv --python python2.7 /tmp/tests
@@ -15,21 +28,34 @@ sudo --set-home -u nobody virtualenv --python python2.7 /tmp/tests
 # platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
 # requirements (TLS >= 1.2) are incompatible with the old TLS clients
 # available to those systems.  Installing it ahead of time (with pip) avoids
-# this problem.
-sudo --set-home -u nobody /tmp/tests/bin/pip install certifi
+# this problem.  Make sure this step comes before any other attempts to
+# install things using pip!
+sudo --set-home -u nobody \
+     PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
+     /tmp/tests/bin/pip install certifi
 
-# Python packages we need to support the test infrastructure.  *Not* packages
-# Tahoe-LAFS itself (implementation or test suite) need.
-TEST_DEPS="tox codecov"
+# Get a new, awesome version of pip and setuptools.  For example, the
+# distro-packaged virtualenv's pip may not know about wheels.
+sudo --set-home -u nobody \
+     PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
+     /tmp/tests/bin/pip install --upgrade pip setuptools wheel
 
-# Python packages we need to generate test reports for CI infrastructure.
-# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
-REPORTING_DEPS="python-subunit junitxml subunitreporter"
+# Populate the wheelhouse, if necessary.
+sudo --set-home -u nobody \
+     PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
+     /tmp/tests/bin/pip \
+     wheel \
+     --wheel-dir "${WHEELHOUSE_PATH}" \
+     /tmp/project ${TEST_DEPS} ${REPORTING_DEPS}
 
-sudo --set-home -u nobody /tmp/tests/bin/pip install ${TEST_DEPS} ${REPORTING_DEPS}
+sudo --set-home -u nobody \
+     PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
+     /tmp/tests/bin/pip install ${TEST_DEPS} ${REPORTING_DEPS}
 
 # Get everything else installed in it, too.
-sudo --set-home -u nobody /tmp/tests/bin/tox \
+sudo --set-home -u nobody \
+     PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
+     /tmp/tests/bin/tox \
      -c /tmp/project/tox.ini \
      --workdir /tmp/tahoe-lafs.tox \
      --notest \

tox.ini

@@ -13,7 +13,7 @@ skipsdist = True
 
 [testenv]
 basepython=python2.7
-passenv = TAHOE_LAFS_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
+passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
 # Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
 # a package to be installed (with setuptools) then it'll fail on certain
 # platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS