Merge pull request #509 from LeastAuthority/circleci-store-test-results

Provide detailed test results to CircleCI.

This enables improved tracking and reporting in the CircleCI UI.

.circleci/config.yml

@@ -85,8 +85,9 @@ jobs:
           # readable.
           working_directory: "/tmp"
           command: |
-            env
+            /tmp/project/.circleci/setup-virtualenv.sh \
-            /tmp/project/.circleci/setup-virtualenv.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}"
+                "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
+                "${TAHOE_LAFS_TOX_ARGS}"
 
       - run: &RUN_TESTS
           name: "Run test suite"
@@ -94,8 +95,16 @@ jobs:
           # if the working directory is not readable.
           working_directory: "/tmp"
           command: |
-            env
+            /tmp/project/.circleci/run-tests.sh \
-            /tmp/project/.circleci/run-tests.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}"
+                /tmp/artifacts \
+                "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
+                "${TAHOE_LAFS_TOX_ARGS}"
+          # trial output gets directed straight to a log.  avoid the circleci
+          # timeout while the test suite runs.
+          no_output_timeout: "20m"
+
+      - store_test_results: &STORE_TEST_RESULTS
+          path: "/tmp/artifacts/junit"
 
       - store_artifacts: &STORE_TEST_LOG
           # Despite passing --workdir /tmp to tox above, it still runs trial
@@ -213,6 +222,7 @@ jobs:
       - run: *SETUP_VIRTUALENV
       - run: *RUN_TESTS
 
+      - store_test_results: *STORE_TEST_RESULTS
       - store_artifacts: *STORE_TEST_LOG
       - store_artifacts: *STORE_OTHER_ARTIFACTS
       - run: *SUBMIT_COVERAGE
@@ -262,6 +272,7 @@ jobs:
             chown --recursive nobody:nobody /tmp/project
 
             slackpkg install \
+                ca-certificates \
                 sudo-1.8.20p2 \
                 make-4.1 \
                 automake-1.15 \
@@ -293,6 +304,7 @@ jobs:
       - run: *SETUP_VIRTUALENV
       - run: *RUN_TESTS
 
+      - store_test_results: *STORE_TEST_RESULTS
       - store_artifacts: *STORE_TEST_LOG
       - store_artifacts: *STORE_OTHER_ARTIFACTS
       - run: *SUBMIT_COVERAGE

.circleci/run-tests.sh

@@ -1,11 +1,23 @@
 #!/bin/bash -e
 
+ARTIFACTS=$1
+shift
+
 TAHOE_LAFS_TOX_ENVIRONMENT=$1
 shift
 
 TAHOE_LAFS_TOX_ARGS=$1
 shift || :
 
+# Make sure we can actually write things to this directory.
+sudo --user nobody mkdir -p "${ARTIFACTS}"
+
+SUBUNIT2="${ARTIFACTS}"/results.subunit2
+
+# Use an intermediate directory here because CircleCI extracts some label
+# information from its name.
+JUNITXML="${ARTIFACTS}"/junit/unittests/results.xml
+
 # Run the test suite as a non-root user.  This is the expected usage some
 # small areas of the test suite assume non-root privileges (such as unreadable
 # files being unreadable).
@@ -13,4 +25,20 @@ shift || :
 # Also run with /tmp as a workdir because the non-root user won't be able to
 # create the tox working filesystem state in the source checkout because it is
 # owned by root.
-sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS}
+#
+# Send the output directly to a file because transporting the binary subunit2
+# via tox and then scraping it out is hideous and failure prone.
+sudo \
+    SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}" \
+    TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file" \
+     --set-home \
+     --user nobody \
+     /tmp/tests/bin/tox \
+     -c /tmp/project/tox.ini \
+     --workdir /tmp/tahoe-lafs.tox \
+     -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
+     ${TAHOE_LAFS_TOX_ARGS}
+
+# Create a junitxml results area.
+mkdir -p "$(dirname "${JUNITXML}")"
+/tmp/tests/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}"

.circleci/setup-virtualenv.sh

@@ -10,11 +10,28 @@ shift || :
 # non-root user.  See below.
 sudo --set-home -u nobody virtualenv --python python2.7 /tmp/tests
 
-# Slackware has non-working SSL support in setuptools until certifi is
+# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
-# installed.  SSL support in setuptools is needed in case packages use
+# a package to be installed (with setuptools) then it'll fail on certain
-# `setup_requires` which gets satisfied by setuptools instead of by pip.
+# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
-# txi2p (vcversioner) is one such package.  Twisted (incremental) is another.
+# requirements (TLS >= 1.2) are incompatible with the old TLS clients
-sudo --set-home -u nobody /tmp/tests/bin/pip install tox codecov
+# available to those systems.  Installing it ahead of time (with pip) avoids
+# this problem.
+sudo --set-home -u nobody /tmp/tests/bin/pip install certifi
+
+# Python packages we need to support the test infrastructure.  *Not* packages
+# Tahoe-LAFS itself (implementation or test suite) need.
+TEST_DEPS="tox codecov"
+
+# Python packages we need to generate test reports for CI infrastructure.
+# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
+REPORTING_DEPS="python-subunit junitxml subunitreporter"
+
+sudo --set-home -u nobody /tmp/tests/bin/pip install ${TEST_DEPS} ${REPORTING_DEPS}
 
 # Get everything else installed in it, too.
-sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp --notest -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS}
+sudo --set-home -u nobody /tmp/tests/bin/tox \
+     -c /tmp/project/tox.ini \
+     --workdir /tmp/tahoe-lafs.tox \
+     --notest \
+     -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
+     ${TAHOE_LAFS_TOX_ARGS}

tox.ini

@@ -13,14 +13,16 @@ skipsdist = True
 
 [testenv]
 basepython=python2.7
-passenv = TAHOE_LAFS_* USERPROFILE HOMEDRIVE HOMEPATH
+passenv = TAHOE_LAFS_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
 # Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
 # a package to be installed (with setuptools) then it'll fail on certain
 # platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
 # requirements (TLS >= 1.2) are incompatible with the old TLS clients
 # available to those systems.  Installing it ahead of time (with pip) avoids
 # this problem.
-deps = certifi
+deps =
+     certifi
+     subunitreporter
 # We add usedevelop=True for speed, and extras=test to get things like "mock"
 # that are required for our unit tests.
 usedevelop = True
@@ -59,6 +61,8 @@ commands =
 setenv =
          PYTHONWARNINGS=default::DeprecationWarning
 deps =
+     # Take the base deps as well!
+     {[testenv]deps}
      git+https://github.com/twisted/twisted
      git+https://github.com/warner/foolscap
 commands =