Merge branch 'master' into 3603.scripts
This commit is contained in:
commit
df137cca0a
|
|
@ -29,7 +29,7 @@ workflows:
|
||||||
- "debian-9": &DOCKERHUB_CONTEXT
|
- "debian-9": &DOCKERHUB_CONTEXT
|
||||||
context: "dockerhub-auth"
|
context: "dockerhub-auth"
|
||||||
|
|
||||||
- "debian-8":
|
- "debian-10":
|
||||||
<<: *DOCKERHUB_CONTEXT
|
<<: *DOCKERHUB_CONTEXT
|
||||||
requires:
|
requires:
|
||||||
- "debian-9"
|
- "debian-9"
|
||||||
|
|
@ -86,11 +86,6 @@ workflows:
|
||||||
# integration tests.
|
# integration tests.
|
||||||
- "debian-9"
|
- "debian-9"
|
||||||
|
|
||||||
# Generate the underlying data for a visualization to aid with Python 3
|
|
||||||
# porting.
|
|
||||||
- "build-porting-depgraph":
|
|
||||||
<<: *DOCKERHUB_CONTEXT
|
|
||||||
|
|
||||||
- "typechecks":
|
- "typechecks":
|
||||||
<<: *DOCKERHUB_CONTEXT
|
<<: *DOCKERHUB_CONTEXT
|
||||||
|
|
||||||
|
|
@ -107,7 +102,7 @@ workflows:
|
||||||
- "master"
|
- "master"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
- "build-image-debian-8":
|
- "build-image-debian-10":
|
||||||
<<: *DOCKERHUB_CONTEXT
|
<<: *DOCKERHUB_CONTEXT
|
||||||
- "build-image-debian-9":
|
- "build-image-debian-9":
|
||||||
<<: *DOCKERHUB_CONTEXT
|
<<: *DOCKERHUB_CONTEXT
|
||||||
|
|
@ -213,7 +208,7 @@ jobs:
|
||||||
# filenames and argv).
|
# filenames and argv).
|
||||||
LANG: "en_US.UTF-8"
|
LANG: "en_US.UTF-8"
|
||||||
# Select a tox environment to run for this job.
|
# Select a tox environment to run for this job.
|
||||||
TAHOE_LAFS_TOX_ENVIRONMENT: "py27-coverage"
|
TAHOE_LAFS_TOX_ENVIRONMENT: "py27"
|
||||||
# Additional arguments to pass to tox.
|
# Additional arguments to pass to tox.
|
||||||
TAHOE_LAFS_TOX_ARGS: ""
|
TAHOE_LAFS_TOX_ARGS: ""
|
||||||
# The path in which test artifacts will be placed.
|
# The path in which test artifacts will be placed.
|
||||||
|
|
@ -223,7 +218,7 @@ jobs:
|
||||||
WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
|
WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
|
||||||
PIP_FIND_LINKS: "file:///tmp/wheelhouse"
|
PIP_FIND_LINKS: "file:///tmp/wheelhouse"
|
||||||
# Upload the coverage report.
|
# Upload the coverage report.
|
||||||
UPLOAD_COVERAGE: "yes"
|
UPLOAD_COVERAGE: ""
|
||||||
|
|
||||||
# pip cannot install packages if the working directory is not readable.
|
# pip cannot install packages if the working directory is not readable.
|
||||||
# We want to run a lot of steps as nobody instead of as root.
|
# We want to run a lot of steps as nobody instead of as root.
|
||||||
|
|
@ -277,11 +272,11 @@ jobs:
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
debian-8:
|
debian-10:
|
||||||
<<: *DEBIAN
|
<<: *DEBIAN
|
||||||
docker:
|
docker:
|
||||||
- <<: *DOCKERHUB_AUTH
|
- <<: *DOCKERHUB_AUTH
|
||||||
image: "tahoelafsci/debian:8-py2.7"
|
image: "tahoelafsci/debian:10-py2.7"
|
||||||
user: "nobody"
|
user: "nobody"
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -376,7 +371,7 @@ jobs:
|
||||||
# this reporter on Python 3. So drop that and just specify the
|
# this reporter on Python 3. So drop that and just specify the
|
||||||
# reporter.
|
# reporter.
|
||||||
TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
|
TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
|
||||||
TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage"
|
TAHOE_LAFS_TOX_ENVIRONMENT: "py36"
|
||||||
|
|
||||||
|
|
||||||
ubuntu-20-04:
|
ubuntu-20-04:
|
||||||
|
|
@ -451,33 +446,6 @@ jobs:
|
||||||
# them in parallel.
|
# them in parallel.
|
||||||
nix-build --cores 3 --max-jobs 2 nix/
|
nix-build --cores 3 --max-jobs 2 nix/
|
||||||
|
|
||||||
# Generate up-to-date data for the dependency graph visualizer.
|
|
||||||
build-porting-depgraph:
|
|
||||||
# Get a system in which we can easily install Tahoe-LAFS and all its
|
|
||||||
# dependencies. The dependency graph analyzer works by executing the code.
|
|
||||||
# It's Python, what do you expect?
|
|
||||||
<<: *DEBIAN
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- "checkout"
|
|
||||||
|
|
||||||
- add_ssh_keys:
|
|
||||||
fingerprints:
|
|
||||||
# Jean-Paul Calderone <exarkun@twistedmatrix.com> (CircleCI depgraph key)
|
|
||||||
# This lets us push to tahoe-lafs/tahoe-depgraph in the next step.
|
|
||||||
- "86:38:18:a7:c0:97:42:43:18:46:55:d6:21:b0:5f:d4"
|
|
||||||
|
|
||||||
- run:
|
|
||||||
name: "Setup Python Environment"
|
|
||||||
command: |
|
|
||||||
/tmp/venv/bin/pip install -e /tmp/project
|
|
||||||
|
|
||||||
- run:
|
|
||||||
name: "Generate dependency graph data"
|
|
||||||
command: |
|
|
||||||
. /tmp/venv/bin/activate
|
|
||||||
./misc/python3/depgraph.sh
|
|
||||||
|
|
||||||
typechecks:
|
typechecks:
|
||||||
docker:
|
docker:
|
||||||
- <<: *DOCKERHUB_AUTH
|
- <<: *DOCKERHUB_AUTH
|
||||||
|
|
@ -529,12 +497,12 @@ jobs:
|
||||||
docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}
|
docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}
|
||||||
|
|
||||||
|
|
||||||
build-image-debian-8:
|
build-image-debian-10:
|
||||||
<<: *BUILD_IMAGE
|
<<: *BUILD_IMAGE
|
||||||
|
|
||||||
environment:
|
environment:
|
||||||
DISTRO: "debian"
|
DISTRO: "debian"
|
||||||
TAG: "8"
|
TAG: "10"
|
||||||
PYTHON_VERSION: "2.7"
|
PYTHON_VERSION: "2.7"
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
48
.codecov.yml
48
.codecov.yml
|
|
@ -1,48 +0,0 @@
|
||||||
# Override defaults for codecov.io checks.
|
|
||||||
#
|
|
||||||
# Documentation is at https://docs.codecov.io/docs/codecov-yaml;
|
|
||||||
# reference is at https://docs.codecov.io/docs/codecovyml-reference.
|
|
||||||
#
|
|
||||||
# To validate this file, use:
|
|
||||||
#
|
|
||||||
# curl --data-binary @.codecov.yml https://codecov.io/validate
|
|
||||||
#
|
|
||||||
# Codecov's defaults seem to leave red marks in GitHub CI checks in a
|
|
||||||
# rather arbitrary manner, probably because of non-determinism in
|
|
||||||
# coverage (see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2891)
|
|
||||||
# and maybe because computers are bad with floating point numbers.
|
|
||||||
|
|
||||||
# Allow coverage percentage a precision of zero decimals, and round to
|
|
||||||
# the nearest number (for example, 89.957 to to 90; 89.497 to 89%).
|
|
||||||
# Coverage above 90% is good, below 80% is bad.
|
|
||||||
coverage:
|
|
||||||
round: nearest
|
|
||||||
range: 80..90
|
|
||||||
precision: 0
|
|
||||||
|
|
||||||
# Aim for a target test coverage of 90% in codecov/project check (do
|
|
||||||
# not allow project coverage to drop below that), and allow
|
|
||||||
# codecov/patch a threshold of 1% (allow coverage in changes to drop
|
|
||||||
# by that much, and no less). That should be good enough for us.
|
|
||||||
status:
|
|
||||||
project:
|
|
||||||
default:
|
|
||||||
target: 90%
|
|
||||||
threshold: 1%
|
|
||||||
patch:
|
|
||||||
default:
|
|
||||||
threshold: 1%
|
|
||||||
|
|
||||||
|
|
||||||
codecov:
|
|
||||||
# This is a public repository so supposedly we don't "need" to use an upload
|
|
||||||
# token. However, using one makes sure that CI jobs running against forked
|
|
||||||
# repositories have coverage uploaded to the right place in codecov so
|
|
||||||
# their reports aren't incomplete.
|
|
||||||
token: "abf679b6-e2e6-4b33-b7b5-6cfbd41ee691"
|
|
||||||
|
|
||||||
notify:
|
|
||||||
# The reference documentation suggests that this is the default setting:
|
|
||||||
# https://docs.codecov.io/docs/codecovyml-reference#codecovnotifywait_for_ci
|
|
||||||
# However observation suggests otherwise.
|
|
||||||
wait_for_ci: true
|
|
||||||
|
|
@ -79,11 +79,110 @@ jobs:
|
||||||
name: eliot.log
|
name: eliot.log
|
||||||
path: eliot.log
|
path: eliot.log
|
||||||
|
|
||||||
- name: Upload coverage report
|
# Upload this job's coverage data to Coveralls. While there is a GitHub
|
||||||
uses: codecov/codecov-action@v1
|
# Action for this, as of Jan 2021 it does not support Python coverage
|
||||||
with:
|
# files - only lcov files. Therefore, we use coveralls-python, the
|
||||||
token: abf679b6-e2e6-4b33-b7b5-6cfbd41ee691
|
# coveralls.io-supplied Python reporter, for this.
|
||||||
file: coverage.xml
|
- name: "Report Coverage to Coveralls"
|
||||||
|
run: |
|
||||||
|
pip install coveralls
|
||||||
|
python -m coveralls
|
||||||
|
env:
|
||||||
|
# Some magic value required for some magic reason.
|
||||||
|
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
||||||
|
# Help coveralls identify our project.
|
||||||
|
COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o"
|
||||||
|
# Every source of coverage reports needs a unique "flag name".
|
||||||
|
# Construct one by smashing a few variables from the matrix together
|
||||||
|
# here.
|
||||||
|
COVERALLS_FLAG_NAME: "run-${{ matrix.os }}-${{ matrix.python-version }}"
|
||||||
|
# Mark the data as just one piece of many because we have more than
|
||||||
|
# one instance of this job (Windows, macOS) which collects and
|
||||||
|
# reports coverage. This is necessary to cause Coveralls to merge
|
||||||
|
# multiple coverage results into a single report. Note the merge
|
||||||
|
# only happens when we "finish" a particular build, as identified by
|
||||||
|
# its "build_num" (aka "service_number").
|
||||||
|
COVERALLS_PARALLEL: true
|
||||||
|
|
||||||
|
# Tell Coveralls that we're done reporting coverage data. Since we're using
|
||||||
|
# the "parallel" mode where more than one coverage data file is merged into
|
||||||
|
# a single report, we have to tell Coveralls when we've uploaded all of the
|
||||||
|
# data files. This does it. We make sure it runs last by making it depend
|
||||||
|
# on *all* of the coverage-collecting jobs.
|
||||||
|
finish-coverage-report:
|
||||||
|
# There happens to just be one coverage-collecting job at the moment. If
|
||||||
|
# the coverage reports are broken and someone added more
|
||||||
|
# coverage-collecting jobs to this workflow but didn't update this, that's
|
||||||
|
# why.
|
||||||
|
needs:
|
||||||
|
- "coverage"
|
||||||
|
runs-on: "ubuntu-latest"
|
||||||
|
steps:
|
||||||
|
- name: "Check out Tahoe-LAFS sources"
|
||||||
|
uses: "actions/checkout@v2"
|
||||||
|
|
||||||
|
- name: "Finish Coveralls Reporting"
|
||||||
|
run: |
|
||||||
|
# coveralls-python does have a `--finish` option but it doesn't seem
|
||||||
|
# to work, at least for us.
|
||||||
|
# https://github.com/coveralls-clients/coveralls-python/issues/248
|
||||||
|
#
|
||||||
|
# But all it does is this simple POST so we can just send it
|
||||||
|
# ourselves. The only hard part is guessing what the POST
|
||||||
|
# parameters mean. And I've done that for you already.
|
||||||
|
#
|
||||||
|
# Since the build is done I'm going to guess that "done" is a fine
|
||||||
|
# value for status.
|
||||||
|
#
|
||||||
|
# That leaves "build_num". The coveralls documentation gives some
|
||||||
|
# hints about it. It suggests using $CIRCLE_WORKFLOW_ID if your job
|
||||||
|
# is on CircleCI. CircleCI documentation says this about
|
||||||
|
# CIRCLE_WORKFLOW_ID:
|
||||||
|
#
|
||||||
|
# Observation of the coveralls.io web interface, logs from the
|
||||||
|
# coveralls command in action, and experimentation suggests the
|
||||||
|
# value for PRs is something more like:
|
||||||
|
#
|
||||||
|
# <GIT MERGE COMMIT HASH>-PR-<PR NUM>
|
||||||
|
#
|
||||||
|
# For branches, it's just the git branch tip hash.
|
||||||
|
|
||||||
|
# For pull requests, refs/pull/<PR NUM>/merge was just checked out
|
||||||
|
# by so HEAD will refer to the right revision. For branches, HEAD
|
||||||
|
# is also the tip of the branch.
|
||||||
|
REV=$(git rev-parse HEAD)
|
||||||
|
|
||||||
|
# We can get the PR number from the "context".
|
||||||
|
#
|
||||||
|
# https://docs.github.com/en/free-pro-team@latest/developers/webhooks-and-events/webhook-events-and-payloads#pull_request
|
||||||
|
#
|
||||||
|
# (via <https://github.community/t/github-ref-is-inconsistent/17728/3>).
|
||||||
|
#
|
||||||
|
# If this is a pull request, `github.event` is a `pull_request`
|
||||||
|
# structure which has `number` right in it.
|
||||||
|
#
|
||||||
|
# If this is a push, `github.event` is a `push` instead but we only
|
||||||
|
# need the revision to construct the build_num.
|
||||||
|
|
||||||
|
PR=${{ github.event.number }}
|
||||||
|
|
||||||
|
if [ "${PR}" = "" ]; then
|
||||||
|
BUILD_NUM=$REV
|
||||||
|
else
|
||||||
|
BUILD_NUM=$REV-PR-$PR
|
||||||
|
fi
|
||||||
|
REPO_NAME=$GITHUB_REPOSITORY
|
||||||
|
|
||||||
|
curl \
|
||||||
|
-k \
|
||||||
|
https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN \
|
||||||
|
-d \
|
||||||
|
"payload[build_num]=$BUILD_NUM&payload[status]=done&payload[repo_name]=$REPO_NAME"
|
||||||
|
env:
|
||||||
|
# Some magic value required for some magic reason.
|
||||||
|
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
||||||
|
# Help coveralls identify our project.
|
||||||
|
COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o"
|
||||||
|
|
||||||
integration:
|
integration:
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ Free and Open decentralized data store
|
||||||
|
|
||||||
`Tahoe-LAFS <https://www.tahoe-lafs.org>`__ (Tahoe Least-Authority File Store) is the first free software / open-source storage technology that distributes your data across multiple servers. Even if some servers fail or are taken over by an attacker, the entire file store continues to function correctly, preserving your privacy and security.
|
`Tahoe-LAFS <https://www.tahoe-lafs.org>`__ (Tahoe Least-Authority File Store) is the first free software / open-source storage technology that distributes your data across multiple servers. Even if some servers fail or are taken over by an attacker, the entire file store continues to function correctly, preserving your privacy and security.
|
||||||
|
|
||||||
|Contributor Covenant| |readthedocs| |travis| |circleci| |codecov|
|
|Contributor Covenant| |readthedocs| |travis| |circleci| |coveralls|
|
||||||
|
|
||||||
|
|
||||||
Table of contents
|
Table of contents
|
||||||
|
|
@ -125,9 +125,9 @@ See `TGPPL.PDF <https://tahoe-lafs.org/~zooko/tgppl.pdf>`__ for why the TGPPL ex
|
||||||
.. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg
|
.. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg
|
||||||
:target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs
|
:target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs
|
||||||
|
|
||||||
.. |codecov| image:: https://codecov.io/github/tahoe-lafs/tahoe-lafs/coverage.svg?branch=master
|
.. |coveralls| image:: https://coveralls.io/repos/github/tahoe-lafs/tahoe-lafs/badge.svg
|
||||||
:alt: test coverage percentage
|
:alt: code coverage
|
||||||
:target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master
|
:target: https://coveralls.io/github/tahoe-lafs/tahoe-lafs
|
||||||
|
|
||||||
.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg
|
.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg
|
||||||
:alt: code of conduct
|
:alt: code of conduct
|
||||||
|
|
|
||||||
|
|
@ -173,7 +173,9 @@ from PyPI with ``venv/bin/pip install tahoe-lafs``. After installation, run
|
||||||
Install From a Source Tarball
|
Install From a Source Tarball
|
||||||
-----------------------------
|
-----------------------------
|
||||||
|
|
||||||
You can also install directly from the source tarball URL::
|
You can also install directly from the source tarball URL. To verify
|
||||||
|
signatures, first see verifying_signatures_ and replace the URL in the
|
||||||
|
following instructions with the local filename.
|
||||||
|
|
||||||
% virtualenv venv
|
% virtualenv venv
|
||||||
New python executable in ~/venv/bin/python2.7
|
New python executable in ~/venv/bin/python2.7
|
||||||
|
|
@ -189,6 +191,40 @@ You can also install directly from the source tarball URL::
|
||||||
tahoe-lafs: 1.14.0
|
tahoe-lafs: 1.14.0
|
||||||
...
|
...
|
||||||
|
|
||||||
|
.. _verifying_signatures:
|
||||||
|
|
||||||
|
Verifying Signatures
|
||||||
|
--------------------
|
||||||
|
|
||||||
|
First download the source tarball and then any signatures. There are several
|
||||||
|
developers who are able to produce signatures for a release. A release may
|
||||||
|
have multiple signatures. All should be valid and you should confirm at least
|
||||||
|
one of them (ideally, confirm all).
|
||||||
|
|
||||||
|
This statement, signed by the existing Tahoe release-signing key, attests to
|
||||||
|
those developers authorized to sign a Tahoe release:
|
||||||
|
|
||||||
|
.. include:: developer-release-signatures
|
||||||
|
:code:
|
||||||
|
|
||||||
|
Signatures are made available beside the release. So for example, a release
|
||||||
|
like ``https://tahoe-lafs.org/downloads/tahoe-lafs-1.16.0.tar.bz2`` might
|
||||||
|
have signatures ``tahoe-lafs-1.16.0.tar.bz2.meejah.asc`` and
|
||||||
|
``tahoe-lafs-1.16.0.tar.bz2.warner.asc``.
|
||||||
|
|
||||||
|
To verify the signatures using GnuPG::
|
||||||
|
|
||||||
|
% gpg --verify tahoe-lafs-1.16.0.tar.bz2.meejah.asc tahoe-lafs-1.16.0.tar.bz2
|
||||||
|
gpg: Signature made XXX
|
||||||
|
gpg: using RSA key 9D5A2BD5688ECB889DEBCD3FC2602803128069A7
|
||||||
|
gpg: Good signature from "meejah <meejah@meejah.ca>" [full]
|
||||||
|
% gpg --verify tahoe-lafs-1.16.0.tar.bz2.warner.asc tahoe-lafs-1.16.0.tar.bz2
|
||||||
|
gpg: Signature made XXX
|
||||||
|
gpg: using RSA key 967EFE06699872411A77DF36D43B4C9C73225AAF
|
||||||
|
gpg: Good signature from "Brian Warner <warner@lothar.com>" [full]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Extras
|
Extras
|
||||||
------
|
------
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
-----BEGIN PGP SIGNED MESSAGE-----
|
||||||
|
Hash: SHA512
|
||||||
|
|
||||||
|
|
||||||
|
January 20, 2021
|
||||||
|
|
||||||
|
Any of the following core Tahoe contributers may sign a release. Each
|
||||||
|
release MUST be signed by at least one developer but MAY have
|
||||||
|
additional signatures. Each developer independently produces a
|
||||||
|
signature which is made available beside Tahoe releases after 1.15.0
|
||||||
|
|
||||||
|
This statement is signed by the existing Tahoe release key. Any future
|
||||||
|
such statements may be signed by it OR by any two developers (for
|
||||||
|
example, to add or remove developers from the list).
|
||||||
|
|
||||||
|
meejah
|
||||||
|
0xC2602803128069A7
|
||||||
|
9D5A 2BD5 688E CB88 9DEB CD3F C260 2803 1280 69A7
|
||||||
|
https://meejah.ca/meejah.asc
|
||||||
|
|
||||||
|
jean-paul calderone (exarkun)
|
||||||
|
0xE27B085EDEAA4B1B
|
||||||
|
96B9 C5DA B2EA 9EB6 7941 9DB7 E27B 085E DEAA 4B1B
|
||||||
|
https://twistedmatrix.com/~exarkun/E27B085EDEAA4B1B.asc
|
||||||
|
|
||||||
|
brian warner (lothar)
|
||||||
|
0x863333C265497810
|
||||||
|
5810 F125 7F8C F753 7753 895A 8633 33C2 6549 7810
|
||||||
|
https://www.lothar.com/warner-gpg.html
|
||||||
|
|
||||||
|
|
||||||
|
-----BEGIN PGP SIGNATURE-----
|
||||||
|
|
||||||
|
iQEzBAEBCgAdFiEE405i0G0Oac/KQXn/veDTHWhmanoFAmAHIyIACgkQveDTHWhm
|
||||||
|
anqhqQf/YSbMXL+gwFhAZsjX39EVlbr/Ik7WPPkJW7v1oHybTnwFpFIc52COU1x/
|
||||||
|
sqRfk4OyYtz9IBgOPXoWgXu9R4qdK6vYKxEsekcGT9C5l0OyDz8YWXEWgbGK5mvI
|
||||||
|
aEub9WucD8r2uOQnnW6DtznFuEpvOjtf/+2BU767+bvLsbViW88ocbuLfCqLdOgD
|
||||||
|
WZT9j3M+Y2Dc56DAJzP/4fkrUSVIofZStYp5u9HBjburgcYIp0g/cyc4xXRoi6Mp
|
||||||
|
lFTRFv3MIjmoamzSQseoIgP6fi8QRqPrffPrsyqAp+06mJnPhxxFqxtO/ZErmpSa
|
||||||
|
+BGrLBxdWa8IF9U1A4Fs5nuAzAKMEg==
|
||||||
|
=E9J+
|
||||||
|
-----END PGP SIGNATURE-----
|
||||||
|
|
@ -137,6 +137,12 @@ Did anyone contribute a hack since the last release? If so, then
|
||||||
https://tahoe-lafs.org/hacktahoelafs/ needs to be updated.
|
https://tahoe-lafs.org/hacktahoelafs/ needs to be updated.
|
||||||
|
|
||||||
|
|
||||||
|
Sign Git Tag
|
||||||
|
````````````
|
||||||
|
|
||||||
|
- git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-X.Y.Z" tahoe-lafs-X.Y.Z
|
||||||
|
|
||||||
|
|
||||||
Upload Artifacts
|
Upload Artifacts
|
||||||
````````````````
|
````````````````
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,7 @@ class ProvisioningTool(rend.Page):
|
||||||
req = inevow.IRequest(ctx)
|
req = inevow.IRequest(ctx)
|
||||||
|
|
||||||
def getarg(name, astype=int):
|
def getarg(name, astype=int):
|
||||||
if req.method != "POST":
|
if req.method != b"POST":
|
||||||
return None
|
return None
|
||||||
if name in req.fields:
|
if name in req.fields:
|
||||||
return astype(req.fields[name].value)
|
return astype(req.fields[name].value)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
Debian 8 support has been replaced with Debian 10 support.
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
The Tahoe command line now always uses UTF-8 to decode its arguments, regardless of locale.
|
||||||
|
|
@ -501,7 +501,7 @@ def list_aliases(options):
|
||||||
rc = tahoe_add_alias.list_aliases(options)
|
rc = tahoe_add_alias.list_aliases(options)
|
||||||
return rc
|
return rc
|
||||||
|
|
||||||
def list(options):
|
def list_(options):
|
||||||
from allmydata.scripts import tahoe_ls
|
from allmydata.scripts import tahoe_ls
|
||||||
rc = tahoe_ls.list(options)
|
rc = tahoe_ls.list(options)
|
||||||
return rc
|
return rc
|
||||||
|
|
@ -587,7 +587,7 @@ dispatch = {
|
||||||
"add-alias": add_alias,
|
"add-alias": add_alias,
|
||||||
"create-alias": create_alias,
|
"create-alias": create_alias,
|
||||||
"list-aliases": list_aliases,
|
"list-aliases": list_aliases,
|
||||||
"ls": list,
|
"ls": list_,
|
||||||
"get": get,
|
"get": get,
|
||||||
"put": put,
|
"put": put,
|
||||||
"cp": cp,
|
"cp": cp,
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,197 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
## Copyright (C) 2021 Valentin Lab
|
||||||
|
##
|
||||||
|
## Redistribution and use in source and binary forms, with or without
|
||||||
|
## modification, are permitted provided that the following conditions
|
||||||
|
## are met:
|
||||||
|
##
|
||||||
|
## 1. Redistributions of source code must retain the above copyright
|
||||||
|
## notice, this list of conditions and the following disclaimer.
|
||||||
|
##
|
||||||
|
## 2. Redistributions in binary form must reproduce the above
|
||||||
|
## copyright notice, this list of conditions and the following
|
||||||
|
## disclaimer in the documentation and/or other materials provided
|
||||||
|
## with the distribution.
|
||||||
|
##
|
||||||
|
## 3. Neither the name of the copyright holder nor the names of its
|
||||||
|
## contributors may be used to endorse or promote products derived
|
||||||
|
## from this software without specific prior written permission.
|
||||||
|
##
|
||||||
|
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
## "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
## LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
|
||||||
|
## FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||||
|
## COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
|
||||||
|
## INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||||
|
## (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||||
|
## SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||||
|
## HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||||
|
## STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||||
|
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
|
||||||
|
## OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
##
|
||||||
|
|
||||||
|
## issue: https://bugs.python.org/issue19264
|
||||||
|
|
||||||
|
# See allmydata/windows/fixups.py
|
||||||
|
import sys
|
||||||
|
assert sys.platform == "win32"
|
||||||
|
|
||||||
|
import os
|
||||||
|
import ctypes
|
||||||
|
import subprocess
|
||||||
|
import _subprocess
|
||||||
|
from ctypes import byref, windll, c_char_p, c_wchar_p, c_void_p, \
|
||||||
|
Structure, sizeof, c_wchar, WinError
|
||||||
|
from ctypes.wintypes import BYTE, WORD, LPWSTR, BOOL, DWORD, LPVOID, \
|
||||||
|
HANDLE
|
||||||
|
|
||||||
|
|
||||||
|
##
|
||||||
|
## Types
|
||||||
|
##
|
||||||
|
|
||||||
|
CREATE_UNICODE_ENVIRONMENT = 0x00000400
|
||||||
|
LPCTSTR = c_char_p
|
||||||
|
LPTSTR = c_wchar_p
|
||||||
|
LPSECURITY_ATTRIBUTES = c_void_p
|
||||||
|
LPBYTE = ctypes.POINTER(BYTE)
|
||||||
|
|
||||||
|
class STARTUPINFOW(Structure):
|
||||||
|
_fields_ = [
|
||||||
|
("cb", DWORD), ("lpReserved", LPWSTR),
|
||||||
|
("lpDesktop", LPWSTR), ("lpTitle", LPWSTR),
|
||||||
|
("dwX", DWORD), ("dwY", DWORD),
|
||||||
|
("dwXSize", DWORD), ("dwYSize", DWORD),
|
||||||
|
("dwXCountChars", DWORD), ("dwYCountChars", DWORD),
|
||||||
|
("dwFillAtrribute", DWORD), ("dwFlags", DWORD),
|
||||||
|
("wShowWindow", WORD), ("cbReserved2", WORD),
|
||||||
|
("lpReserved2", LPBYTE), ("hStdInput", HANDLE),
|
||||||
|
("hStdOutput", HANDLE), ("hStdError", HANDLE),
|
||||||
|
]
|
||||||
|
|
||||||
|
LPSTARTUPINFOW = ctypes.POINTER(STARTUPINFOW)
|
||||||
|
|
||||||
|
|
||||||
|
class PROCESS_INFORMATION(Structure):
|
||||||
|
_fields_ = [
|
||||||
|
("hProcess", HANDLE), ("hThread", HANDLE),
|
||||||
|
("dwProcessId", DWORD), ("dwThreadId", DWORD),
|
||||||
|
]
|
||||||
|
|
||||||
|
LPPROCESS_INFORMATION = ctypes.POINTER(PROCESS_INFORMATION)
|
||||||
|
|
||||||
|
|
||||||
|
class DUMMY_HANDLE(ctypes.c_void_p):
|
||||||
|
|
||||||
|
def __init__(self, *a, **kw):
|
||||||
|
super(DUMMY_HANDLE, self).__init__(*a, **kw)
|
||||||
|
self.closed = False
|
||||||
|
|
||||||
|
def Close(self):
|
||||||
|
if not self.closed:
|
||||||
|
windll.kernel32.CloseHandle(self)
|
||||||
|
self.closed = True
|
||||||
|
|
||||||
|
def __int__(self):
|
||||||
|
return self.value
|
||||||
|
|
||||||
|
|
||||||
|
CreateProcessW = windll.kernel32.CreateProcessW
|
||||||
|
CreateProcessW.argtypes = [
|
||||||
|
LPCTSTR, LPTSTR, LPSECURITY_ATTRIBUTES,
|
||||||
|
LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCTSTR,
|
||||||
|
LPSTARTUPINFOW, LPPROCESS_INFORMATION,
|
||||||
|
]
|
||||||
|
CreateProcessW.restype = BOOL
|
||||||
|
|
||||||
|
|
||||||
|
##
|
||||||
|
## Patched functions/classes
|
||||||
|
##
|
||||||
|
|
||||||
|
def CreateProcess(executable, args, _p_attr, _t_attr,
|
||||||
|
inherit_handles, creation_flags, env, cwd,
|
||||||
|
startup_info):
|
||||||
|
"""Create a process supporting unicode executable and args for win32
|
||||||
|
|
||||||
|
Python implementation of CreateProcess using CreateProcessW for Win32
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
si = STARTUPINFOW(
|
||||||
|
dwFlags=startup_info.dwFlags,
|
||||||
|
wShowWindow=startup_info.wShowWindow,
|
||||||
|
cb=sizeof(STARTUPINFOW),
|
||||||
|
## XXXvlab: not sure of the casting here to ints.
|
||||||
|
hStdInput=int(startup_info.hStdInput),
|
||||||
|
hStdOutput=int(startup_info.hStdOutput),
|
||||||
|
hStdError=int(startup_info.hStdError),
|
||||||
|
)
|
||||||
|
|
||||||
|
wenv = None
|
||||||
|
if env is not None:
|
||||||
|
## LPCWSTR seems to be c_wchar_p, so let's say CWSTR is c_wchar
|
||||||
|
env = (unicode("").join([
|
||||||
|
unicode("%s=%s\0") % (k, v)
|
||||||
|
for k, v in env.items()])) + unicode("\0")
|
||||||
|
wenv = (c_wchar * len(env))()
|
||||||
|
wenv.value = env
|
||||||
|
|
||||||
|
pi = PROCESS_INFORMATION()
|
||||||
|
creation_flags |= CREATE_UNICODE_ENVIRONMENT
|
||||||
|
|
||||||
|
if CreateProcessW(executable, args, None, None,
|
||||||
|
inherit_handles, creation_flags,
|
||||||
|
wenv, cwd, byref(si), byref(pi)):
|
||||||
|
return (DUMMY_HANDLE(pi.hProcess), DUMMY_HANDLE(pi.hThread),
|
||||||
|
pi.dwProcessId, pi.dwThreadId)
|
||||||
|
raise WinError()
|
||||||
|
|
||||||
|
|
||||||
|
class Popen(subprocess.Popen):
|
||||||
|
"""This superseeds Popen and corrects a bug in cPython 2.7 implem"""
|
||||||
|
|
||||||
|
def _execute_child(self, args, executable, preexec_fn, close_fds,
|
||||||
|
cwd, env, universal_newlines,
|
||||||
|
startupinfo, creationflags, shell, to_close,
|
||||||
|
p2cread, p2cwrite,
|
||||||
|
c2pread, c2pwrite,
|
||||||
|
errread, errwrite):
|
||||||
|
"""Code from part of _execute_child from Python 2.7 (9fbb65e)
|
||||||
|
|
||||||
|
There are only 2 little changes concerning the construction of
|
||||||
|
the the final string in shell mode: we preempt the creation of
|
||||||
|
the command string when shell is True, because original function
|
||||||
|
will try to encode unicode args which we want to avoid to be able to
|
||||||
|
sending it as-is to ``CreateProcess``.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if not isinstance(args, subprocess.types.StringTypes):
|
||||||
|
args = subprocess.list2cmdline(args)
|
||||||
|
|
||||||
|
if startupinfo is None:
|
||||||
|
startupinfo = subprocess.STARTUPINFO()
|
||||||
|
if shell:
|
||||||
|
startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW
|
||||||
|
startupinfo.wShowWindow = _subprocess.SW_HIDE
|
||||||
|
comspec = os.environ.get("COMSPEC", unicode("cmd.exe"))
|
||||||
|
args = unicode('{} /c "{}"').format(comspec, args)
|
||||||
|
if (_subprocess.GetVersion() >= 0x80000000 or
|
||||||
|
os.path.basename(comspec).lower() == "command.com"):
|
||||||
|
w9xpopen = self._find_w9xpopen()
|
||||||
|
args = unicode('"%s" %s') % (w9xpopen, args)
|
||||||
|
creationflags |= _subprocess.CREATE_NEW_CONSOLE
|
||||||
|
|
||||||
|
cp = _subprocess.CreateProcess
|
||||||
|
_subprocess.CreateProcess = CreateProcess
|
||||||
|
try:
|
||||||
|
super(Popen, self)._execute_child(
|
||||||
|
args, executable,
|
||||||
|
preexec_fn, close_fds, cwd, env, universal_newlines,
|
||||||
|
startupinfo, creationflags, False, to_close, p2cread,
|
||||||
|
p2cwrite, c2pread, c2pwrite, errread, errwrite,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
_subprocess.CreateProcess = cp
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
from ...util.encodingutil import unicode_to_argv
|
from six import ensure_str
|
||||||
|
|
||||||
from ...scripts import runner
|
from ...scripts import runner
|
||||||
from ..common_util import ReallyEqualMixin, run_cli, run_cli_unicode
|
from ..common_util import ReallyEqualMixin, run_cli, run_cli_unicode
|
||||||
|
|
||||||
|
|
@ -45,6 +46,12 @@ class CLITestMixin(ReallyEqualMixin):
|
||||||
# client_num is used to execute client CLI commands on a specific
|
# client_num is used to execute client CLI commands on a specific
|
||||||
# client.
|
# client.
|
||||||
client_num = kwargs.pop("client_num", 0)
|
client_num = kwargs.pop("client_num", 0)
|
||||||
client_dir = unicode_to_argv(self.get_clientdir(i=client_num))
|
# If we were really going to launch a child process then
|
||||||
|
# `unicode_to_argv` would be the right thing to do here. However,
|
||||||
|
# we're just going to call some Python functions directly and those
|
||||||
|
# Python functions want native strings. So ignore the requirements
|
||||||
|
# for passing arguments to another process and make sure this argument
|
||||||
|
# is a native string.
|
||||||
|
client_dir = ensure_str(self.get_clientdir(i=client_num))
|
||||||
nodeargs = [ b"--node-directory", client_dir ]
|
nodeargs = [ b"--node-directory", client_dir ]
|
||||||
return run_cli(verb, *args, nodeargs=nodeargs, **kwargs)
|
return run_cli(verb, *args, nodeargs=nodeargs, **kwargs)
|
||||||
|
|
|
||||||
|
|
@ -99,22 +99,6 @@ class ListAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_list_latin_1(self):
|
|
||||||
"""
|
|
||||||
An alias composed of all Latin-1-encodeable code points can be created
|
|
||||||
when the active encoding is Latin-1.
|
|
||||||
|
|
||||||
This is very similar to ``test_list_utf_8`` but the assumption of
|
|
||||||
UTF-8 is nearly ubiquitous and explicitly exercising the codepaths
|
|
||||||
with a UTF-8-incompatible encoding helps flush out unintentional UTF-8
|
|
||||||
assumptions.
|
|
||||||
"""
|
|
||||||
return self._check_create_alias(
|
|
||||||
u"taho\N{LATIN SMALL LETTER E WITH ACUTE}",
|
|
||||||
encoding="latin-1",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_list_utf_8(self):
|
def test_list_utf_8(self):
|
||||||
"""
|
"""
|
||||||
An alias composed of all UTF-8-encodeable code points can be created when
|
An alias composed of all UTF-8-encodeable code points can be created when
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ from allmydata.scripts.common import get_aliases
|
||||||
from allmydata.scripts import cli
|
from allmydata.scripts import cli
|
||||||
from ..no_network import GridTestMixin
|
from ..no_network import GridTestMixin
|
||||||
from ..common_util import skip_if_cannot_represent_filename
|
from ..common_util import skip_if_cannot_represent_filename
|
||||||
from allmydata.util.encodingutil import get_io_encoding, unicode_to_argv
|
from allmydata.util.encodingutil import get_io_encoding
|
||||||
from allmydata.util.fileutil import abspath_expanduser_unicode
|
from allmydata.util.fileutil import abspath_expanduser_unicode
|
||||||
from .common import CLITestMixin
|
from .common import CLITestMixin
|
||||||
|
|
||||||
|
|
@ -46,21 +46,21 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
|
||||||
self.basedir = "cli/Put/unlinked_immutable_from_file"
|
self.basedir = "cli/Put/unlinked_immutable_from_file"
|
||||||
self.set_up_grid(oneshare=True)
|
self.set_up_grid(oneshare=True)
|
||||||
|
|
||||||
rel_fn = os.path.join(self.basedir, "DATAFILE")
|
rel_fn = unicode(os.path.join(self.basedir, "DATAFILE"))
|
||||||
abs_fn = unicode_to_argv(abspath_expanduser_unicode(unicode(rel_fn)))
|
abs_fn = abspath_expanduser_unicode(rel_fn)
|
||||||
# we make the file small enough to fit in a LIT file, for speed
|
# we make the file small enough to fit in a LIT file, for speed
|
||||||
fileutil.write(rel_fn, "short file")
|
fileutil.write(rel_fn, "short file")
|
||||||
d = self.do_cli("put", rel_fn)
|
d = self.do_cli_unicode(u"put", [rel_fn])
|
||||||
def _uploaded(args):
|
def _uploaded(args):
|
||||||
(rc, out, err) = args
|
(rc, out, err) = args
|
||||||
readcap = out
|
readcap = out
|
||||||
self.failUnless(readcap.startswith("URI:LIT:"), readcap)
|
self.failUnless(readcap.startswith("URI:LIT:"), readcap)
|
||||||
self.readcap = readcap
|
self.readcap = readcap
|
||||||
d.addCallback(_uploaded)
|
d.addCallback(_uploaded)
|
||||||
d.addCallback(lambda res: self.do_cli("put", "./" + rel_fn))
|
d.addCallback(lambda res: self.do_cli_unicode(u"put", [u"./" + rel_fn]))
|
||||||
d.addCallback(lambda rc_stdout_stderr:
|
d.addCallback(lambda rc_stdout_stderr:
|
||||||
self.failUnlessReallyEqual(rc_stdout_stderr[1], self.readcap))
|
self.failUnlessReallyEqual(rc_stdout_stderr[1], self.readcap))
|
||||||
d.addCallback(lambda res: self.do_cli("put", abs_fn))
|
d.addCallback(lambda res: self.do_cli_unicode(u"put", [abs_fn]))
|
||||||
d.addCallback(lambda rc_stdout_stderr:
|
d.addCallback(lambda rc_stdout_stderr:
|
||||||
self.failUnlessReallyEqual(rc_stdout_stderr[1], self.readcap))
|
self.failUnlessReallyEqual(rc_stdout_stderr[1], self.readcap))
|
||||||
# we just have to assume that ~ is handled properly
|
# we just have to assume that ~ is handled properly
|
||||||
|
|
|
||||||
|
|
@ -9,10 +9,15 @@ __all__ = [
|
||||||
"flush_logged_errors",
|
"flush_logged_errors",
|
||||||
"skip",
|
"skip",
|
||||||
"skipIf",
|
"skipIf",
|
||||||
|
|
||||||
|
# Selected based on platform and re-exported for convenience.
|
||||||
|
"Popen",
|
||||||
|
"PIPE",
|
||||||
]
|
]
|
||||||
|
|
||||||
from past.builtins import chr as byteschr, unicode
|
from past.builtins import chr as byteschr, unicode
|
||||||
|
|
||||||
|
import sys
|
||||||
import os, random, struct
|
import os, random, struct
|
||||||
import six
|
import six
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
@ -101,6 +106,21 @@ from .eliotutil import (
|
||||||
)
|
)
|
||||||
from .common_util import ShouldFailMixin # noqa: F401
|
from .common_util import ShouldFailMixin # noqa: F401
|
||||||
|
|
||||||
|
if sys.platform == "win32":
|
||||||
|
# Python 2.7 doesn't have good options for launching a process with
|
||||||
|
# non-ASCII in its command line. So use this alternative that does a
|
||||||
|
# better job. However, only use it on Windows because it doesn't work
|
||||||
|
# anywhere else.
|
||||||
|
from ._win_subprocess import (
|
||||||
|
Popen,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
from subprocess import (
|
||||||
|
Popen,
|
||||||
|
)
|
||||||
|
from subprocess import (
|
||||||
|
PIPE,
|
||||||
|
)
|
||||||
|
|
||||||
TEST_RSA_KEY_SIZE = 522
|
TEST_RSA_KEY_SIZE = 522
|
||||||
|
|
||||||
|
|
@ -432,7 +452,7 @@ class FakeCHKFileNode(object): # type: ignore # incomplete implementation
|
||||||
return self.storage_index
|
return self.storage_index
|
||||||
|
|
||||||
def check(self, monitor, verify=False, add_lease=False):
|
def check(self, monitor, verify=False, add_lease=False):
|
||||||
s = StubServer("\x00"*20)
|
s = StubServer(b"\x00"*20)
|
||||||
r = CheckResults(self.my_uri, self.storage_index,
|
r = CheckResults(self.my_uri, self.storage_index,
|
||||||
healthy=True, recoverable=True,
|
healthy=True, recoverable=True,
|
||||||
count_happiness=10,
|
count_happiness=10,
|
||||||
|
|
@ -566,12 +586,12 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
|
||||||
self.file_types[self.storage_index] = version
|
self.file_types[self.storage_index] = version
|
||||||
initial_contents = self._get_initial_contents(contents)
|
initial_contents = self._get_initial_contents(contents)
|
||||||
data = initial_contents.read(initial_contents.get_size())
|
data = initial_contents.read(initial_contents.get_size())
|
||||||
data = "".join(data)
|
data = b"".join(data)
|
||||||
self.all_contents[self.storage_index] = data
|
self.all_contents[self.storage_index] = data
|
||||||
return defer.succeed(self)
|
return defer.succeed(self)
|
||||||
def _get_initial_contents(self, contents):
|
def _get_initial_contents(self, contents):
|
||||||
if contents is None:
|
if contents is None:
|
||||||
return MutableData("")
|
return MutableData(b"")
|
||||||
|
|
||||||
if IMutableUploadable.providedBy(contents):
|
if IMutableUploadable.providedBy(contents):
|
||||||
return contents
|
return contents
|
||||||
|
|
@ -625,7 +645,7 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
|
||||||
def raise_error(self):
|
def raise_error(self):
|
||||||
pass
|
pass
|
||||||
def get_writekey(self):
|
def get_writekey(self):
|
||||||
return "\x00"*16
|
return b"\x00"*16
|
||||||
def get_size(self):
|
def get_size(self):
|
||||||
return len(self.all_contents[self.storage_index])
|
return len(self.all_contents[self.storage_index])
|
||||||
def get_current_size(self):
|
def get_current_size(self):
|
||||||
|
|
@ -644,7 +664,7 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
|
||||||
return self.file_types[self.storage_index]
|
return self.file_types[self.storage_index]
|
||||||
|
|
||||||
def check(self, monitor, verify=False, add_lease=False):
|
def check(self, monitor, verify=False, add_lease=False):
|
||||||
s = StubServer("\x00"*20)
|
s = StubServer(b"\x00"*20)
|
||||||
r = CheckResults(self.my_uri, self.storage_index,
|
r = CheckResults(self.my_uri, self.storage_index,
|
||||||
healthy=True, recoverable=True,
|
healthy=True, recoverable=True,
|
||||||
count_happiness=10,
|
count_happiness=10,
|
||||||
|
|
@ -655,7 +675,7 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
|
||||||
count_recoverable_versions=1,
|
count_recoverable_versions=1,
|
||||||
count_unrecoverable_versions=0,
|
count_unrecoverable_versions=0,
|
||||||
servers_responding=[s],
|
servers_responding=[s],
|
||||||
sharemap={"seq1-abcd-sh0": [s]},
|
sharemap={b"seq1-abcd-sh0": [s]},
|
||||||
count_wrong_shares=0,
|
count_wrong_shares=0,
|
||||||
list_corrupt_shares=[],
|
list_corrupt_shares=[],
|
||||||
count_corrupt_shares=0,
|
count_corrupt_shares=0,
|
||||||
|
|
@ -709,7 +729,7 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
|
||||||
def overwrite(self, new_contents):
|
def overwrite(self, new_contents):
|
||||||
assert not self.is_readonly()
|
assert not self.is_readonly()
|
||||||
new_data = new_contents.read(new_contents.get_size())
|
new_data = new_contents.read(new_contents.get_size())
|
||||||
new_data = "".join(new_data)
|
new_data = b"".join(new_data)
|
||||||
self.all_contents[self.storage_index] = new_data
|
self.all_contents[self.storage_index] = new_data
|
||||||
return defer.succeed(None)
|
return defer.succeed(None)
|
||||||
def modify(self, modifier):
|
def modify(self, modifier):
|
||||||
|
|
@ -740,7 +760,7 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
|
||||||
def update(self, data, offset):
|
def update(self, data, offset):
|
||||||
assert not self.is_readonly()
|
assert not self.is_readonly()
|
||||||
def modifier(old, servermap, first_time):
|
def modifier(old, servermap, first_time):
|
||||||
new = old[:offset] + "".join(data.read(data.get_size()))
|
new = old[:offset] + b"".join(data.read(data.get_size()))
|
||||||
new += old[len(new):]
|
new += old[len(new):]
|
||||||
return new
|
return new
|
||||||
return self.modify(modifier)
|
return self.modify(modifier)
|
||||||
|
|
@ -859,6 +879,8 @@ class WebErrorMixin(object):
|
||||||
body = yield response.content()
|
body = yield response.content()
|
||||||
self.assertEquals(response.code, code)
|
self.assertEquals(response.code, code)
|
||||||
if response_substring is not None:
|
if response_substring is not None:
|
||||||
|
if isinstance(response_substring, unicode):
|
||||||
|
response_substring = response_substring.encode("utf-8")
|
||||||
self.assertIn(response_substring, body)
|
self.assertIn(response_substring, body)
|
||||||
returnValue(body)
|
returnValue(body)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -203,6 +203,14 @@ def flip_one_bit(s, offset=0, size=None):
|
||||||
class ReallyEqualMixin(object):
|
class ReallyEqualMixin(object):
|
||||||
def failUnlessReallyEqual(self, a, b, msg=None):
|
def failUnlessReallyEqual(self, a, b, msg=None):
|
||||||
self.assertEqual(a, b, msg)
|
self.assertEqual(a, b, msg)
|
||||||
|
# Make sure unicode strings are a consistent type. Specifically there's
|
||||||
|
# Future newstr (backported Unicode type) vs. Python 2 native unicode
|
||||||
|
# type. They're equal, and _logically_ the same type, but have
|
||||||
|
# different types in practice.
|
||||||
|
if a.__class__ == future_str:
|
||||||
|
a = unicode(a)
|
||||||
|
if b.__class__ == future_str:
|
||||||
|
b = unicode(b)
|
||||||
self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg))
|
self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg))
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,29 +6,43 @@ Tools aimed at the interaction between tests and Eliot.
|
||||||
# Can't use `builtins.str` because it's not JSON encodable:
|
# Can't use `builtins.str` because it's not JSON encodable:
|
||||||
# `exceptions.TypeError: <class 'future.types.newstr.newstr'> is not JSON-encodeable`
|
# `exceptions.TypeError: <class 'future.types.newstr.newstr'> is not JSON-encodeable`
|
||||||
from past.builtins import unicode as str
|
from past.builtins import unicode as str
|
||||||
from future.utils import PY3
|
from future.utils import PY2
|
||||||
|
from six import ensure_text
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"RUN_TEST",
|
"RUN_TEST",
|
||||||
"EliotLoggedRunTest",
|
"EliotLoggedRunTest",
|
||||||
"eliot_logged_test",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
|
try:
|
||||||
|
from typing import Callable
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
|
||||||
from functools import (
|
from functools import (
|
||||||
wraps,
|
|
||||||
partial,
|
partial,
|
||||||
|
wraps,
|
||||||
)
|
)
|
||||||
|
|
||||||
import attr
|
import attr
|
||||||
|
|
||||||
|
from zope.interface import (
|
||||||
|
implementer,
|
||||||
|
)
|
||||||
|
|
||||||
from eliot import (
|
from eliot import (
|
||||||
ActionType,
|
ActionType,
|
||||||
Field,
|
Field,
|
||||||
|
MemoryLogger,
|
||||||
|
ILogger,
|
||||||
|
)
|
||||||
|
from eliot.testing import (
|
||||||
|
swap_logger,
|
||||||
|
check_for_errors,
|
||||||
)
|
)
|
||||||
from eliot.testing import capture_logging
|
|
||||||
|
|
||||||
from twisted.internet.defer import (
|
from twisted.python.monkey import (
|
||||||
maybeDeferred,
|
MonkeyPatcher,
|
||||||
)
|
)
|
||||||
|
|
||||||
from ..util.jsonbytes import BytesJSONEncoder
|
from ..util.jsonbytes import BytesJSONEncoder
|
||||||
|
|
@ -48,92 +62,12 @@ RUN_TEST = ActionType(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def eliot_logged_test(f):
|
# On Python 3, we want to use our custom JSON encoder when validating messages
|
||||||
"""
|
# can be encoded to JSON:
|
||||||
Decorate a test method to run in a dedicated Eliot action context.
|
if PY2:
|
||||||
|
_memory_logger = MemoryLogger
|
||||||
The action will finish after the test is done (after the returned Deferred
|
else:
|
||||||
fires, if a Deferred is returned). It will note the name of the test
|
_memory_logger = lambda: MemoryLogger(encoder=BytesJSONEncoder)
|
||||||
being run.
|
|
||||||
|
|
||||||
All messages emitted by the test will be validated. They will still be
|
|
||||||
delivered to the global logger.
|
|
||||||
"""
|
|
||||||
# A convenient, mutable container into which nested functions can write
|
|
||||||
# state to be shared among them.
|
|
||||||
class storage(object):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
# On Python 3, we want to use our custom JSON encoder when validating
|
|
||||||
# messages can be encoded to JSON:
|
|
||||||
if PY3:
|
|
||||||
capture = lambda f : capture_logging(None, encoder_=BytesJSONEncoder)(f)
|
|
||||||
else:
|
|
||||||
capture = lambda f : capture_logging(None)(f)
|
|
||||||
|
|
||||||
@wraps(f)
|
|
||||||
def run_and_republish(self, *a, **kw):
|
|
||||||
# Unfortunately the only way to get at the global/default logger...
|
|
||||||
# This import is delayed here so that we get the *current* default
|
|
||||||
# logger at the time the decorated function is run.
|
|
||||||
from eliot._output import _DEFAULT_LOGGER as default_logger
|
|
||||||
|
|
||||||
def republish():
|
|
||||||
# This is called as a cleanup function after capture_logging has
|
|
||||||
# restored the global/default logger to its original state. We
|
|
||||||
# can now emit messages that go to whatever global destinations
|
|
||||||
# are installed.
|
|
||||||
|
|
||||||
# storage.logger.serialize() seems like it would make more sense
|
|
||||||
# than storage.logger.messages here. However, serialize()
|
|
||||||
# explodes, seemingly as a result of double-serializing the logged
|
|
||||||
# messages. I don't understand this.
|
|
||||||
for msg in storage.logger.messages:
|
|
||||||
default_logger.write(msg)
|
|
||||||
|
|
||||||
# And now that we've re-published all of the test's messages, we
|
|
||||||
# can finish the test's action.
|
|
||||||
storage.action.finish()
|
|
||||||
|
|
||||||
@capture
|
|
||||||
def run(self, logger):
|
|
||||||
# Record the MemoryLogger for later message extraction.
|
|
||||||
storage.logger = logger
|
|
||||||
# Give the test access to the logger as well. It would be just
|
|
||||||
# fine to pass this as a keyword argument to `f` but implementing
|
|
||||||
# that now will give me conflict headaches so I'm not doing it.
|
|
||||||
self.eliot_logger = logger
|
|
||||||
return f(self, *a, **kw)
|
|
||||||
|
|
||||||
# Arrange for all messages written to the memory logger that
|
|
||||||
# `capture_logging` installs to be re-written to the global/default
|
|
||||||
# logger so they might end up in a log file somewhere, if someone
|
|
||||||
# wants. This has to be done in a cleanup function (or later) because
|
|
||||||
# capture_logging restores the original logger in a cleanup function.
|
|
||||||
# We install our cleanup function here, before we call run, so that it
|
|
||||||
# runs *after* the cleanup function capture_logging installs (cleanup
|
|
||||||
# functions are a stack).
|
|
||||||
self.addCleanup(republish)
|
|
||||||
|
|
||||||
# Begin an action that should comprise all messages from the decorated
|
|
||||||
# test method.
|
|
||||||
with RUN_TEST(name=self.id()).context() as action:
|
|
||||||
# When the test method Deferred fires, the RUN_TEST action is
|
|
||||||
# done. However, we won't have re-published the MemoryLogger
|
|
||||||
# messages into the global/default logger when this Deferred
|
|
||||||
# fires. So we need to delay finishing the action until that has
|
|
||||||
# happened. Record the action so we can do that.
|
|
||||||
storage.action = action
|
|
||||||
|
|
||||||
# Support both Deferred-returning and non-Deferred-returning
|
|
||||||
# tests.
|
|
||||||
d = maybeDeferred(run, self)
|
|
||||||
|
|
||||||
# Let the test runner do its thing.
|
|
||||||
return d
|
|
||||||
|
|
||||||
return run_and_republish
|
|
||||||
|
|
||||||
|
|
||||||
@attr.s
|
@attr.s
|
||||||
|
|
@ -174,10 +108,91 @@ class EliotLoggedRunTest(object):
|
||||||
def id(self):
|
def id(self):
|
||||||
return self.case.id()
|
return self.case.id()
|
||||||
|
|
||||||
@eliot_logged_test
|
def run(self, result):
|
||||||
def run(self, result=None):
|
"""
|
||||||
return self._run_tests_with_factory(
|
Run the test case in the context of a distinct Eliot action.
|
||||||
self.case,
|
|
||||||
self.handlers,
|
The action will finish after the test is done. It will note the name of
|
||||||
self.last_resort,
|
the test being run.
|
||||||
).run(result)
|
|
||||||
|
All messages emitted by the test will be validated. They will still be
|
||||||
|
delivered to the global logger.
|
||||||
|
"""
|
||||||
|
# The idea here is to decorate the test method itself so that all of
|
||||||
|
# the extra logic happens at the point where test/application logic is
|
||||||
|
# expected to be. This `run` method is more like test infrastructure
|
||||||
|
# and things do not go well when we add too much extra behavior here.
|
||||||
|
# For example, exceptions raised here often just kill the whole
|
||||||
|
# runner.
|
||||||
|
patcher = MonkeyPatcher()
|
||||||
|
|
||||||
|
# So, grab the test method.
|
||||||
|
name = self.case._testMethodName
|
||||||
|
original = getattr(self.case, name)
|
||||||
|
decorated = with_logging(ensure_text(self.case.id()), original)
|
||||||
|
patcher.addPatch(self.case, name, decorated)
|
||||||
|
try:
|
||||||
|
# Patch it in
|
||||||
|
patcher.patch()
|
||||||
|
# Then use the rest of the machinery to run it.
|
||||||
|
return self._run_tests_with_factory(
|
||||||
|
self.case,
|
||||||
|
self.handlers,
|
||||||
|
self.last_resort,
|
||||||
|
).run(result)
|
||||||
|
finally:
|
||||||
|
# Clean up the patching for idempotency or something.
|
||||||
|
patcher.restore()
|
||||||
|
|
||||||
|
|
||||||
|
def with_logging(
|
||||||
|
test_id, # type: str
|
||||||
|
test_method, # type: Callable
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Decorate a test method with additional log-related behaviors.
|
||||||
|
|
||||||
|
1. The test method will run in a distinct Eliot action.
|
||||||
|
2. Typed log messages will be validated.
|
||||||
|
3. Logged tracebacks will be added as errors.
|
||||||
|
|
||||||
|
:param test_id: The full identifier of the test being decorated.
|
||||||
|
:param test_method: The method itself.
|
||||||
|
"""
|
||||||
|
@wraps(test_method)
|
||||||
|
def run_with_logging(*args, **kwargs):
|
||||||
|
validating_logger = _memory_logger()
|
||||||
|
original = swap_logger(None)
|
||||||
|
try:
|
||||||
|
swap_logger(_TwoLoggers(original, validating_logger))
|
||||||
|
with RUN_TEST(name=test_id):
|
||||||
|
try:
|
||||||
|
return test_method(*args, **kwargs)
|
||||||
|
finally:
|
||||||
|
check_for_errors(validating_logger)
|
||||||
|
finally:
|
||||||
|
swap_logger(original)
|
||||||
|
return run_with_logging
|
||||||
|
|
||||||
|
|
||||||
|
@implementer(ILogger)
|
||||||
|
class _TwoLoggers(object):
|
||||||
|
"""
|
||||||
|
Log to two loggers.
|
||||||
|
|
||||||
|
A single logger can have multiple destinations so this isn't typically a
|
||||||
|
useful thing to do. However, MemoryLogger has inline validation instead
|
||||||
|
of destinations. That means this *is* useful to simultaneously write to
|
||||||
|
the normal places and validate all written log messages.
|
||||||
|
"""
|
||||||
|
def __init__(self, a, b):
|
||||||
|
"""
|
||||||
|
:param ILogger a: One logger
|
||||||
|
:param ILogger b: Another logger
|
||||||
|
"""
|
||||||
|
self._a = a # type: ILogger
|
||||||
|
self._b = b # type: ILogger
|
||||||
|
|
||||||
|
def write(self, dictionary, serializer=None):
|
||||||
|
self._a.write(dictionary, serializer)
|
||||||
|
self._b.write(dictionary, serializer)
|
||||||
|
|
|
||||||
|
|
@ -18,17 +18,25 @@ if PY2:
|
||||||
from sys import stdout
|
from sys import stdout
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
|
from unittest import (
|
||||||
|
skip,
|
||||||
|
)
|
||||||
|
|
||||||
from fixtures import (
|
from fixtures import (
|
||||||
TempDir,
|
TempDir,
|
||||||
)
|
)
|
||||||
from testtools import (
|
from testtools import (
|
||||||
TestCase,
|
TestCase,
|
||||||
)
|
)
|
||||||
|
from testtools import (
|
||||||
|
TestResult,
|
||||||
|
)
|
||||||
from testtools.matchers import (
|
from testtools.matchers import (
|
||||||
Is,
|
Is,
|
||||||
IsInstance,
|
IsInstance,
|
||||||
MatchesStructure,
|
MatchesStructure,
|
||||||
Equals,
|
Equals,
|
||||||
|
HasLength,
|
||||||
AfterPreprocessing,
|
AfterPreprocessing,
|
||||||
)
|
)
|
||||||
from testtools.twistedsupport import (
|
from testtools.twistedsupport import (
|
||||||
|
|
@ -38,12 +46,16 @@ from testtools.twistedsupport import (
|
||||||
|
|
||||||
from eliot import (
|
from eliot import (
|
||||||
Message,
|
Message,
|
||||||
|
MessageType,
|
||||||
|
fields,
|
||||||
FileDestination,
|
FileDestination,
|
||||||
|
MemoryLogger,
|
||||||
)
|
)
|
||||||
from eliot.twisted import DeferredContext
|
from eliot.twisted import DeferredContext
|
||||||
from eliot.testing import (
|
from eliot.testing import (
|
||||||
capture_logging,
|
capture_logging,
|
||||||
assertHasAction,
|
assertHasAction,
|
||||||
|
swap_logger,
|
||||||
)
|
)
|
||||||
|
|
||||||
from twisted.internet.defer import (
|
from twisted.internet.defer import (
|
||||||
|
|
@ -173,6 +185,62 @@ class EliotLoggingTests(TestCase):
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_validation_failure(self):
|
||||||
|
"""
|
||||||
|
If a test emits a log message that fails validation then an error is added
|
||||||
|
to the result.
|
||||||
|
"""
|
||||||
|
# Make sure we preserve the original global Eliot state.
|
||||||
|
original = swap_logger(MemoryLogger())
|
||||||
|
self.addCleanup(lambda: swap_logger(original))
|
||||||
|
|
||||||
|
class ValidationFailureProbe(SyncTestCase):
|
||||||
|
def test_bad_message(self):
|
||||||
|
# This message does not validate because "Hello" is not an
|
||||||
|
# int.
|
||||||
|
MSG = MessageType("test:eliotutil", fields(foo=int))
|
||||||
|
MSG(foo="Hello").write()
|
||||||
|
|
||||||
|
result = TestResult()
|
||||||
|
case = ValidationFailureProbe("test_bad_message")
|
||||||
|
case.run(result)
|
||||||
|
|
||||||
|
self.assertThat(
|
||||||
|
result.errors,
|
||||||
|
HasLength(1),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_skip_cleans_up(self):
|
||||||
|
"""
|
||||||
|
After a skipped test the global Eliot logging state is restored.
|
||||||
|
"""
|
||||||
|
# Save the logger that's active before we do anything so that we can
|
||||||
|
# restore it later. Also install another logger so we can compare it
|
||||||
|
# to the active logger later.
|
||||||
|
expected = MemoryLogger()
|
||||||
|
original = swap_logger(expected)
|
||||||
|
|
||||||
|
# Restore it, whatever else happens.
|
||||||
|
self.addCleanup(lambda: swap_logger(original))
|
||||||
|
|
||||||
|
class SkipProbe(SyncTestCase):
|
||||||
|
@skip("It's a skip test.")
|
||||||
|
def test_skipped(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
case = SkipProbe("test_skipped")
|
||||||
|
case.run()
|
||||||
|
|
||||||
|
# Retrieve the logger that's active now that the skipped test is done
|
||||||
|
# so we can check it against the expected value.
|
||||||
|
actual = swap_logger(MemoryLogger())
|
||||||
|
self.assertThat(
|
||||||
|
actual,
|
||||||
|
Is(expected),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class LogCallDeferredTests(TestCase):
|
class LogCallDeferredTests(TestCase):
|
||||||
"""
|
"""
|
||||||
Tests for ``log_call_deferred``.
|
Tests for ``log_call_deferred``.
|
||||||
|
|
|
||||||
|
|
@ -70,7 +70,7 @@ if __name__ == "__main__":
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
import os, sys, locale
|
import os, sys
|
||||||
from unittest import skipIf
|
from unittest import skipIf
|
||||||
|
|
||||||
from twisted.trial import unittest
|
from twisted.trial import unittest
|
||||||
|
|
@ -81,99 +81,28 @@ from allmydata.test.common_util import (
|
||||||
ReallyEqualMixin, skip_if_cannot_represent_filename,
|
ReallyEqualMixin, skip_if_cannot_represent_filename,
|
||||||
)
|
)
|
||||||
from allmydata.util import encodingutil, fileutil
|
from allmydata.util import encodingutil, fileutil
|
||||||
from allmydata.util.encodingutil import argv_to_unicode, unicode_to_url, \
|
from allmydata.util.encodingutil import unicode_to_url, \
|
||||||
unicode_to_output, quote_output, quote_path, quote_local_unicode_path, \
|
unicode_to_output, quote_output, quote_path, quote_local_unicode_path, \
|
||||||
quote_filepath, unicode_platform, listdir_unicode, FilenameEncodingError, \
|
quote_filepath, unicode_platform, listdir_unicode, FilenameEncodingError, \
|
||||||
get_io_encoding, get_filesystem_encoding, to_bytes, from_utf8_or_none, _reload, \
|
get_filesystem_encoding, to_bytes, from_utf8_or_none, _reload, \
|
||||||
to_filepath, extend_filepath, unicode_from_filepath, unicode_segments_from, \
|
to_filepath, extend_filepath, unicode_from_filepath, unicode_segments_from, \
|
||||||
unicode_to_argv
|
unicode_to_argv
|
||||||
from twisted.python import usage
|
|
||||||
|
|
||||||
|
|
||||||
class MockStdout(object):
|
class MockStdout(object):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
class EncodingUtilErrors(ReallyEqualMixin, unittest.TestCase):
|
|
||||||
def test_get_io_encoding(self):
|
|
||||||
mock_stdout = MockStdout()
|
|
||||||
self.patch(sys, 'stdout', mock_stdout)
|
|
||||||
|
|
||||||
mock_stdout.encoding = 'UTF-8'
|
|
||||||
_reload()
|
|
||||||
self.failUnlessReallyEqual(get_io_encoding(), 'utf-8')
|
|
||||||
|
|
||||||
mock_stdout.encoding = 'cp65001'
|
|
||||||
_reload()
|
|
||||||
self.assertEqual(get_io_encoding(), 'utf-8')
|
|
||||||
|
|
||||||
mock_stdout.encoding = 'koi8-r'
|
|
||||||
expected = sys.platform == "win32" and 'utf-8' or 'koi8-r'
|
|
||||||
_reload()
|
|
||||||
self.failUnlessReallyEqual(get_io_encoding(), expected)
|
|
||||||
|
|
||||||
mock_stdout.encoding = 'nonexistent_encoding'
|
|
||||||
if sys.platform == "win32":
|
|
||||||
_reload()
|
|
||||||
self.failUnlessReallyEqual(get_io_encoding(), 'utf-8')
|
|
||||||
else:
|
|
||||||
self.failUnlessRaises(AssertionError, _reload)
|
|
||||||
|
|
||||||
def test_get_io_encoding_not_from_stdout(self):
|
|
||||||
preferredencoding = 'koi8-r'
|
|
||||||
def call_locale_getpreferredencoding():
|
|
||||||
return preferredencoding
|
|
||||||
self.patch(locale, 'getpreferredencoding', call_locale_getpreferredencoding)
|
|
||||||
mock_stdout = MockStdout()
|
|
||||||
self.patch(sys, 'stdout', mock_stdout)
|
|
||||||
|
|
||||||
expected = sys.platform == "win32" and 'utf-8' or 'koi8-r'
|
|
||||||
_reload()
|
|
||||||
self.failUnlessReallyEqual(get_io_encoding(), expected)
|
|
||||||
|
|
||||||
mock_stdout.encoding = None
|
|
||||||
_reload()
|
|
||||||
self.failUnlessReallyEqual(get_io_encoding(), expected)
|
|
||||||
|
|
||||||
preferredencoding = None
|
|
||||||
_reload()
|
|
||||||
self.assertEqual(get_io_encoding(), 'utf-8')
|
|
||||||
|
|
||||||
def test_argv_to_unicode(self):
|
|
||||||
encodingutil.io_encoding = 'utf-8'
|
|
||||||
self.failUnlessRaises(usage.UsageError,
|
|
||||||
argv_to_unicode,
|
|
||||||
lumiere_nfc.encode('latin1'))
|
|
||||||
|
|
||||||
@skipIf(PY3, "Python 2 only.")
|
|
||||||
def test_unicode_to_output(self):
|
|
||||||
encodingutil.io_encoding = 'koi8-r'
|
|
||||||
self.failUnlessRaises(UnicodeEncodeError, unicode_to_output, lumiere_nfc)
|
|
||||||
|
|
||||||
def test_no_unicode_normalization(self):
|
|
||||||
# Pretend to run on a Unicode platform.
|
|
||||||
# listdir_unicode normalized to NFC in 1.7beta, but now doesn't.
|
|
||||||
|
|
||||||
def call_os_listdir(path):
|
|
||||||
return [Artonwall_nfd]
|
|
||||||
self.patch(os, 'listdir', call_os_listdir)
|
|
||||||
self.patch(sys, 'platform', 'darwin')
|
|
||||||
|
|
||||||
_reload()
|
|
||||||
self.failUnlessReallyEqual(listdir_unicode(u'/dummy'), [Artonwall_nfd])
|
|
||||||
|
|
||||||
|
|
||||||
# The following tests apply only to platforms that don't store filenames as
|
# The following tests apply only to platforms that don't store filenames as
|
||||||
# Unicode entities on the filesystem.
|
# Unicode entities on the filesystem.
|
||||||
class EncodingUtilNonUnicodePlatform(unittest.TestCase):
|
class EncodingUtilNonUnicodePlatform(unittest.TestCase):
|
||||||
@skipIf(PY3, "Python 3 is always Unicode, regardless of OS.")
|
@skipIf(PY3, "Python 3 is always Unicode, regardless of OS.")
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
# Mock sys.platform because unicode_platform() uses it
|
# Make sure everything goes back to the way it was at the end of the
|
||||||
self.original_platform = sys.platform
|
# test.
|
||||||
sys.platform = 'linux'
|
self.addCleanup(_reload)
|
||||||
|
|
||||||
def tearDown(self):
|
# Mock sys.platform because unicode_platform() uses it. Cleanups run
|
||||||
sys.platform = self.original_platform
|
# in reverse order so we do this second so it gets undone first.
|
||||||
_reload()
|
self.patch(sys, "platform", "linux")
|
||||||
|
|
||||||
def test_listdir_unicode(self):
|
def test_listdir_unicode(self):
|
||||||
# What happens if latin1-encoded filenames are encountered on an UTF-8
|
# What happens if latin1-encoded filenames are encountered on an UTF-8
|
||||||
|
|
@ -206,25 +135,8 @@ class EncodingUtilNonUnicodePlatform(unittest.TestCase):
|
||||||
|
|
||||||
class EncodingUtil(ReallyEqualMixin):
|
class EncodingUtil(ReallyEqualMixin):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
self.original_platform = sys.platform
|
self.addCleanup(_reload)
|
||||||
sys.platform = self.platform
|
self.patch(sys, "platform", self.platform)
|
||||||
|
|
||||||
def tearDown(self):
|
|
||||||
sys.platform = self.original_platform
|
|
||||||
_reload()
|
|
||||||
|
|
||||||
def test_argv_to_unicode(self):
|
|
||||||
if 'argv' not in dir(self):
|
|
||||||
return
|
|
||||||
|
|
||||||
mock_stdout = MockStdout()
|
|
||||||
mock_stdout.encoding = self.io_encoding
|
|
||||||
self.patch(sys, 'stdout', mock_stdout)
|
|
||||||
|
|
||||||
argu = lumiere_nfc
|
|
||||||
argv = self.argv
|
|
||||||
_reload()
|
|
||||||
self.failUnlessReallyEqual(argv_to_unicode(argv), argu)
|
|
||||||
|
|
||||||
def test_unicode_to_url(self):
|
def test_unicode_to_url(self):
|
||||||
self.failUnless(unicode_to_url(lumiere_nfc), b"lumi\xc3\xa8re")
|
self.failUnless(unicode_to_url(lumiere_nfc), b"lumi\xc3\xa8re")
|
||||||
|
|
@ -245,15 +157,19 @@ class EncodingUtil(ReallyEqualMixin):
|
||||||
def test_unicode_to_output_py3(self):
|
def test_unicode_to_output_py3(self):
|
||||||
self.failUnlessReallyEqual(unicode_to_output(lumiere_nfc), lumiere_nfc)
|
self.failUnlessReallyEqual(unicode_to_output(lumiere_nfc), lumiere_nfc)
|
||||||
|
|
||||||
@skipIf(PY3, "Python 2 only.")
|
def test_unicode_to_argv(self):
|
||||||
def test_unicode_to_argv_py2(self):
|
"""
|
||||||
"""unicode_to_argv() converts to bytes on Python 2."""
|
unicode_to_argv() returns its unicode argument on Windows and Python 2 and
|
||||||
self.assertEqual(unicode_to_argv("abc"), u"abc".encode(self.io_encoding))
|
converts to bytes using UTF-8 elsewhere.
|
||||||
|
"""
|
||||||
|
result = unicode_to_argv(lumiere_nfc)
|
||||||
|
if PY3 or self.platform == "win32":
|
||||||
|
expected_value = lumiere_nfc
|
||||||
|
else:
|
||||||
|
expected_value = lumiere_nfc.encode(self.io_encoding)
|
||||||
|
|
||||||
@skipIf(PY2, "Python 3 only.")
|
self.assertIsInstance(result, type(expected_value))
|
||||||
def test_unicode_to_argv_py3(self):
|
self.assertEqual(result, expected_value)
|
||||||
"""unicode_to_argv() is noop on Python 3."""
|
|
||||||
self.assertEqual(unicode_to_argv("abc"), "abc")
|
|
||||||
|
|
||||||
@skipIf(PY3, "Python 3 only.")
|
@skipIf(PY3, "Python 3 only.")
|
||||||
def test_unicode_platform_py2(self):
|
def test_unicode_platform_py2(self):
|
||||||
|
|
@ -463,13 +379,6 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase):
|
||||||
check(u"\n", u"\"\\x0a\"", quote_newlines=True)
|
check(u"\n", u"\"\\x0a\"", quote_newlines=True)
|
||||||
|
|
||||||
def test_quote_output_default(self):
|
def test_quote_output_default(self):
|
||||||
self.patch(encodingutil, 'io_encoding', 'ascii')
|
|
||||||
self.test_quote_output_ascii(None)
|
|
||||||
|
|
||||||
self.patch(encodingutil, 'io_encoding', 'latin1')
|
|
||||||
self.test_quote_output_latin1(None)
|
|
||||||
|
|
||||||
self.patch(encodingutil, 'io_encoding', 'utf-8')
|
|
||||||
self.test_quote_output_utf8(None)
|
self.test_quote_output_utf8(None)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -581,14 +490,6 @@ class UbuntuKarmicUTF8(EncodingUtil, unittest.TestCase):
|
||||||
io_encoding = 'UTF-8'
|
io_encoding = 'UTF-8'
|
||||||
dirlist = [b'test_file', b'\xc3\x84rtonwall.mp3', b'Blah blah.txt']
|
dirlist = [b'test_file', b'\xc3\x84rtonwall.mp3', b'Blah blah.txt']
|
||||||
|
|
||||||
class UbuntuKarmicLatin1(EncodingUtil, unittest.TestCase):
|
|
||||||
uname = 'Linux korn 2.6.31-14-generic #48-Ubuntu SMP Fri Oct 16 14:05:01 UTC 2009 x86_64'
|
|
||||||
argv = b'lumi\xe8re'
|
|
||||||
platform = 'linux2'
|
|
||||||
filesystem_encoding = 'ISO-8859-1'
|
|
||||||
io_encoding = 'ISO-8859-1'
|
|
||||||
dirlist = [b'test_file', b'Blah blah.txt', b'\xc4rtonwall.mp3']
|
|
||||||
|
|
||||||
class Windows(EncodingUtil, unittest.TestCase):
|
class Windows(EncodingUtil, unittest.TestCase):
|
||||||
uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD'
|
uname = 'Windows XP 5.1.2600 x86 x86 Family 15 Model 75 Step ping 2, AuthenticAMD'
|
||||||
argv = b'lumi\xc3\xa8re'
|
argv = b'lumi\xc3\xa8re'
|
||||||
|
|
@ -605,20 +506,6 @@ class MacOSXLeopard(EncodingUtil, unittest.TestCase):
|
||||||
io_encoding = 'UTF-8'
|
io_encoding = 'UTF-8'
|
||||||
dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file']
|
dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file']
|
||||||
|
|
||||||
class MacOSXLeopard7bit(EncodingUtil, unittest.TestCase):
|
|
||||||
uname = 'Darwin g5.local 9.8.0 Darwin Kernel Version 9.8.0: Wed Jul 15 16:57:01 PDT 2009; root:xnu-1228.15.4~1/RELEASE_PPC Power Macintosh powerpc'
|
|
||||||
platform = 'darwin'
|
|
||||||
filesystem_encoding = 'utf-8'
|
|
||||||
io_encoding = 'US-ASCII'
|
|
||||||
dirlist = [u'A\u0308rtonwall.mp3', u'Blah blah.txt', u'test_file']
|
|
||||||
|
|
||||||
class OpenBSD(EncodingUtil, unittest.TestCase):
|
|
||||||
uname = 'OpenBSD 4.1 GENERIC#187 i386 Intel(R) Celeron(R) CPU 2.80GHz ("GenuineIntel" 686-class)'
|
|
||||||
platform = 'openbsd4'
|
|
||||||
filesystem_encoding = '646'
|
|
||||||
io_encoding = '646'
|
|
||||||
# Oops, I cannot write filenames containing non-ascii characters
|
|
||||||
|
|
||||||
|
|
||||||
class TestToFromStr(ReallyEqualMixin, unittest.TestCase):
|
class TestToFromStr(ReallyEqualMixin, unittest.TestCase):
|
||||||
def test_to_bytes(self):
|
def test_to_bytes(self):
|
||||||
|
|
|
||||||
|
|
@ -126,6 +126,42 @@ class HashUtilTests(unittest.TestCase):
|
||||||
base32.a2b(b"2ckv3dfzh6rgjis6ogfqhyxnzy"),
|
base32.a2b(b"2ckv3dfzh6rgjis6ogfqhyxnzy"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_convergence_hasher_tag(self):
|
||||||
|
"""
|
||||||
|
``_convergence_hasher_tag`` constructs the convergence hasher tag from a
|
||||||
|
unique prefix, the required, total, and segment size parameters, and a
|
||||||
|
convergence secret.
|
||||||
|
"""
|
||||||
|
self.assertEqual(
|
||||||
|
b"allmydata_immutable_content_to_key_with_added_secret_v1+"
|
||||||
|
b"16:\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42,"
|
||||||
|
b"9:3,10,1024,",
|
||||||
|
hashutil._convergence_hasher_tag(
|
||||||
|
k=3,
|
||||||
|
n=10,
|
||||||
|
segsize=1024,
|
||||||
|
convergence=b"\x42" * 16,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_convergence_hasher_out_of_bounds(self):
|
||||||
|
"""
|
||||||
|
``_convergence_hasher_tag`` raises ``ValueError`` if k or n is not between
|
||||||
|
1 and 256 inclusive or if k is greater than n.
|
||||||
|
"""
|
||||||
|
segsize = 1024
|
||||||
|
secret = b"\x42" * 16
|
||||||
|
for bad_k in (0, 2, 257):
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
hashutil._convergence_hasher_tag(
|
||||||
|
k=bad_k, n=1, segsize=segsize, convergence=secret,
|
||||||
|
)
|
||||||
|
for bad_n in (0, 1, 257):
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
hashutil._convergence_hasher_tag(
|
||||||
|
k=2, n=bad_n, segsize=segsize, convergence=secret,
|
||||||
|
)
|
||||||
|
|
||||||
def test_known_answers(self):
|
def test_known_answers(self):
|
||||||
"""
|
"""
|
||||||
Verify backwards compatibility by comparing hash outputs for some
|
Verify backwards compatibility by comparing hash outputs for some
|
||||||
|
|
|
||||||
|
|
@ -14,9 +14,11 @@ from testtools.matchers import (
|
||||||
)
|
)
|
||||||
|
|
||||||
BLACKLIST = {
|
BLACKLIST = {
|
||||||
"allmydata.test.check_load",
|
|
||||||
"allmydata.windows.registry",
|
|
||||||
"allmydata.scripts.types_",
|
"allmydata.scripts.types_",
|
||||||
|
"allmydata.test.check_load",
|
||||||
|
"allmydata.test._win_subprocess",
|
||||||
|
"allmydata.windows.registry",
|
||||||
|
"allmydata.windows.fixups",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,10 @@ from __future__ import (
|
||||||
import os.path, re, sys
|
import os.path, re, sys
|
||||||
from os import linesep
|
from os import linesep
|
||||||
|
|
||||||
|
from eliot import (
|
||||||
|
log_call,
|
||||||
|
)
|
||||||
|
|
||||||
from twisted.trial import unittest
|
from twisted.trial import unittest
|
||||||
|
|
||||||
from twisted.internet import reactor
|
from twisted.internet import reactor
|
||||||
|
|
@ -19,22 +23,25 @@ from twisted.python.runtime import (
|
||||||
platform,
|
platform,
|
||||||
)
|
)
|
||||||
from allmydata.util import fileutil, pollmixin
|
from allmydata.util import fileutil, pollmixin
|
||||||
from allmydata.util.encodingutil import unicode_to_argv, unicode_to_output
|
from allmydata.util.encodingutil import unicode_to_argv, get_filesystem_encoding
|
||||||
from allmydata.test import common_util
|
from allmydata.test import common_util
|
||||||
import allmydata
|
import allmydata
|
||||||
from .common_util import parse_cli, run_cli
|
from .common import (
|
||||||
|
PIPE,
|
||||||
|
Popen,
|
||||||
|
)
|
||||||
|
from .common_util import (
|
||||||
|
parse_cli,
|
||||||
|
run_cli,
|
||||||
|
)
|
||||||
from .cli_node_api import (
|
from .cli_node_api import (
|
||||||
CLINodeAPI,
|
CLINodeAPI,
|
||||||
Expect,
|
Expect,
|
||||||
on_stdout,
|
on_stdout,
|
||||||
on_stdout_and_stderr,
|
on_stdout_and_stderr,
|
||||||
)
|
)
|
||||||
from ._twisted_9607 import (
|
|
||||||
getProcessOutputAndValue,
|
|
||||||
)
|
|
||||||
from ..util.eliotutil import (
|
from ..util.eliotutil import (
|
||||||
inline_callbacks,
|
inline_callbacks,
|
||||||
log_call_deferred,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_root_from_file(src):
|
def get_root_from_file(src):
|
||||||
|
|
@ -54,93 +61,92 @@ srcfile = allmydata.__file__
|
||||||
rootdir = get_root_from_file(srcfile)
|
rootdir = get_root_from_file(srcfile)
|
||||||
|
|
||||||
|
|
||||||
class RunBinTahoeMixin(object):
|
@log_call(action_type="run-bin-tahoe")
|
||||||
@log_call_deferred(action_type="run-bin-tahoe")
|
def run_bintahoe(extra_argv, python_options=None):
|
||||||
def run_bintahoe(self, args, stdin=None, python_options=[], env=None):
|
"""
|
||||||
command = sys.executable
|
Run the main Tahoe entrypoint in a child process with the given additional
|
||||||
argv = python_options + ["-m", "allmydata.scripts.runner"] + args
|
arguments.
|
||||||
|
|
||||||
if env is None:
|
:param [unicode] extra_argv: More arguments for the child process argv.
|
||||||
env = os.environ
|
|
||||||
|
|
||||||
d = getProcessOutputAndValue(command, argv, env, stdinBytes=stdin)
|
:return: A three-tuple of stdout (unicode), stderr (unicode), and the
|
||||||
def fix_signal(result):
|
child process "returncode" (int).
|
||||||
# Mirror subprocess.Popen.returncode structure
|
"""
|
||||||
(out, err, signal) = result
|
argv = [sys.executable.decode(get_filesystem_encoding())]
|
||||||
return (out, err, -signal)
|
if python_options is not None:
|
||||||
d.addErrback(fix_signal)
|
argv.extend(python_options)
|
||||||
return d
|
argv.extend([u"-m", u"allmydata.scripts.runner"])
|
||||||
|
argv.extend(extra_argv)
|
||||||
|
argv = list(unicode_to_argv(arg) for arg in argv)
|
||||||
|
p = Popen(argv, stdout=PIPE, stderr=PIPE)
|
||||||
|
out = p.stdout.read().decode("utf-8")
|
||||||
|
err = p.stderr.read().decode("utf-8")
|
||||||
|
returncode = p.wait()
|
||||||
|
return (out, err, returncode)
|
||||||
|
|
||||||
|
|
||||||
class BinTahoe(common_util.SignalMixin, unittest.TestCase, RunBinTahoeMixin):
|
class BinTahoe(common_util.SignalMixin, unittest.TestCase):
|
||||||
def test_unicode_arguments_and_output(self):
|
def test_unicode_arguments_and_output(self):
|
||||||
|
"""
|
||||||
|
The runner script receives unmangled non-ASCII values in argv.
|
||||||
|
"""
|
||||||
tricky = u"\u2621"
|
tricky = u"\u2621"
|
||||||
try:
|
out, err, returncode = run_bintahoe([tricky])
|
||||||
tricky_arg = unicode_to_argv(tricky, mangle=True)
|
self.assertEqual(returncode, 1)
|
||||||
tricky_out = unicode_to_output(tricky)
|
self.assertIn(u"Unknown command: " + tricky, out)
|
||||||
except UnicodeEncodeError:
|
|
||||||
raise unittest.SkipTest("A non-ASCII argument/output could not be encoded on this platform.")
|
|
||||||
|
|
||||||
d = self.run_bintahoe([tricky_arg])
|
def test_with_python_options(self):
|
||||||
def _cb(res):
|
"""
|
||||||
out, err, rc_or_sig = res
|
Additional options for the Python interpreter don't prevent the runner
|
||||||
self.failUnlessEqual(rc_or_sig, 1, str(res))
|
script from receiving the arguments meant for it.
|
||||||
self.failUnlessIn("Unknown command: "+tricky_out, out)
|
"""
|
||||||
d.addCallback(_cb)
|
# This seems like a redundant test for someone else's functionality
|
||||||
return d
|
# but on Windows we parse the whole command line string ourselves so
|
||||||
|
# we have to have our own implementation of skipping these options.
|
||||||
|
|
||||||
def test_run_with_python_options(self):
|
# -t is a harmless option that warns about tabs so we can add it
|
||||||
# -t is a harmless option that warns about tabs.
|
# without impacting other behavior noticably.
|
||||||
d = self.run_bintahoe(["--version"], python_options=["-t"])
|
out, err, returncode = run_bintahoe([u"--version"], python_options=[u"-t"])
|
||||||
def _cb(res):
|
self.assertEqual(returncode, 0)
|
||||||
out, err, rc_or_sig = res
|
self.assertTrue(out.startswith(allmydata.__appname__ + '/'))
|
||||||
self.assertEqual(rc_or_sig, 0, str(res))
|
|
||||||
self.assertTrue(out.startswith(allmydata.__appname__ + '/'), str(res))
|
|
||||||
d.addCallback(_cb)
|
|
||||||
return d
|
|
||||||
|
|
||||||
@inlineCallbacks
|
|
||||||
def test_help_eliot_destinations(self):
|
def test_help_eliot_destinations(self):
|
||||||
out, err, rc_or_sig = yield self.run_bintahoe(["--help-eliot-destinations"])
|
out, err, returncode = run_bintahoe([u"--help-eliot-destinations"])
|
||||||
self.assertIn("\tfile:<path>", out)
|
self.assertIn(u"\tfile:<path>", out)
|
||||||
self.assertEqual(rc_or_sig, 0)
|
self.assertEqual(returncode, 0)
|
||||||
|
|
||||||
@inlineCallbacks
|
|
||||||
def test_eliot_destination(self):
|
def test_eliot_destination(self):
|
||||||
out, err, rc_or_sig = yield self.run_bintahoe([
|
out, err, returncode = run_bintahoe([
|
||||||
# Proves little but maybe more than nothing.
|
# Proves little but maybe more than nothing.
|
||||||
"--eliot-destination=file:-",
|
u"--eliot-destination=file:-",
|
||||||
# Throw in *some* command or the process exits with error, making
|
# Throw in *some* command or the process exits with error, making
|
||||||
# it difficult for us to see if the previous arg was accepted or
|
# it difficult for us to see if the previous arg was accepted or
|
||||||
# not.
|
# not.
|
||||||
"--help",
|
u"--help",
|
||||||
])
|
])
|
||||||
self.assertEqual(rc_or_sig, 0)
|
self.assertEqual(returncode, 0)
|
||||||
|
|
||||||
@inlineCallbacks
|
|
||||||
def test_unknown_eliot_destination(self):
|
def test_unknown_eliot_destination(self):
|
||||||
out, err, rc_or_sig = yield self.run_bintahoe([
|
out, err, returncode = run_bintahoe([
|
||||||
"--eliot-destination=invalid:more",
|
u"--eliot-destination=invalid:more",
|
||||||
])
|
])
|
||||||
self.assertEqual(1, rc_or_sig)
|
self.assertEqual(1, returncode)
|
||||||
self.assertIn("Unknown destination description", out)
|
self.assertIn(u"Unknown destination description", out)
|
||||||
self.assertIn("invalid:more", out)
|
self.assertIn(u"invalid:more", out)
|
||||||
|
|
||||||
@inlineCallbacks
|
|
||||||
def test_malformed_eliot_destination(self):
|
def test_malformed_eliot_destination(self):
|
||||||
out, err, rc_or_sig = yield self.run_bintahoe([
|
out, err, returncode = run_bintahoe([
|
||||||
"--eliot-destination=invalid",
|
u"--eliot-destination=invalid",
|
||||||
])
|
])
|
||||||
self.assertEqual(1, rc_or_sig)
|
self.assertEqual(1, returncode)
|
||||||
self.assertIn("must be formatted like", out)
|
self.assertIn(u"must be formatted like", out)
|
||||||
|
|
||||||
@inlineCallbacks
|
|
||||||
def test_escape_in_eliot_destination(self):
|
def test_escape_in_eliot_destination(self):
|
||||||
out, err, rc_or_sig = yield self.run_bintahoe([
|
out, err, returncode = run_bintahoe([
|
||||||
"--eliot-destination=file:@foo",
|
u"--eliot-destination=file:@foo",
|
||||||
])
|
])
|
||||||
self.assertEqual(1, rc_or_sig)
|
self.assertEqual(1, returncode)
|
||||||
self.assertIn("Unsupported escape character", out)
|
self.assertIn(u"Unsupported escape character", out)
|
||||||
|
|
||||||
|
|
||||||
class CreateNode(unittest.TestCase):
|
class CreateNode(unittest.TestCase):
|
||||||
|
|
@ -250,8 +256,7 @@ class CreateNode(unittest.TestCase):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin,
|
class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
|
||||||
RunBinTahoeMixin):
|
|
||||||
"""
|
"""
|
||||||
exercise "tahoe run" for both introducer and client node, by spawning
|
exercise "tahoe run" for both introducer and client node, by spawning
|
||||||
"tahoe run" as a subprocess. This doesn't get us line-level coverage, but
|
"tahoe run" as a subprocess. This doesn't get us line-level coverage, but
|
||||||
|
|
@ -271,18 +276,18 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin,
|
||||||
The introducer furl is stable across restarts.
|
The introducer furl is stable across restarts.
|
||||||
"""
|
"""
|
||||||
basedir = self.workdir("test_introducer")
|
basedir = self.workdir("test_introducer")
|
||||||
c1 = os.path.join(basedir, "c1")
|
c1 = os.path.join(basedir, u"c1")
|
||||||
tahoe = CLINodeAPI(reactor, FilePath(c1))
|
tahoe = CLINodeAPI(reactor, FilePath(c1))
|
||||||
self.addCleanup(tahoe.stop_and_wait)
|
self.addCleanup(tahoe.stop_and_wait)
|
||||||
|
|
||||||
out, err, rc_or_sig = yield self.run_bintahoe([
|
out, err, returncode = run_bintahoe([
|
||||||
"--quiet",
|
u"--quiet",
|
||||||
"create-introducer",
|
u"create-introducer",
|
||||||
"--basedir", c1,
|
u"--basedir", c1,
|
||||||
"--hostname", "127.0.0.1",
|
u"--hostname", u"127.0.0.1",
|
||||||
])
|
])
|
||||||
|
|
||||||
self.assertEqual(rc_or_sig, 0)
|
self.assertEqual(returncode, 0)
|
||||||
|
|
||||||
# This makes sure that node.url is written, which allows us to
|
# This makes sure that node.url is written, which allows us to
|
||||||
# detect when the introducer restarts in _node_has_restarted below.
|
# detect when the introducer restarts in _node_has_restarted below.
|
||||||
|
|
@ -350,18 +355,18 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin,
|
||||||
3) Verify that the pid file is removed after SIGTERM (on POSIX).
|
3) Verify that the pid file is removed after SIGTERM (on POSIX).
|
||||||
"""
|
"""
|
||||||
basedir = self.workdir("test_client")
|
basedir = self.workdir("test_client")
|
||||||
c1 = os.path.join(basedir, "c1")
|
c1 = os.path.join(basedir, u"c1")
|
||||||
|
|
||||||
tahoe = CLINodeAPI(reactor, FilePath(c1))
|
tahoe = CLINodeAPI(reactor, FilePath(c1))
|
||||||
# Set this up right now so we don't forget later.
|
# Set this up right now so we don't forget later.
|
||||||
self.addCleanup(tahoe.cleanup)
|
self.addCleanup(tahoe.cleanup)
|
||||||
|
|
||||||
out, err, rc_or_sig = yield self.run_bintahoe([
|
out, err, returncode = run_bintahoe([
|
||||||
"--quiet", "create-node", "--basedir", c1,
|
u"--quiet", u"create-node", u"--basedir", c1,
|
||||||
"--webport", "0",
|
u"--webport", u"0",
|
||||||
"--hostname", "localhost",
|
u"--hostname", u"localhost",
|
||||||
])
|
])
|
||||||
self.failUnlessEqual(rc_or_sig, 0)
|
self.failUnlessEqual(returncode, 0)
|
||||||
|
|
||||||
# Check that the --webport option worked.
|
# Check that the --webport option worked.
|
||||||
config = fileutil.read(tahoe.config_file.path)
|
config = fileutil.read(tahoe.config_file.path)
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,10 @@ from twisted.python.filepath import (
|
||||||
FilePath,
|
FilePath,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
from ._twisted_9607 import (
|
||||||
|
getProcessOutputAndValue,
|
||||||
|
)
|
||||||
|
|
||||||
from .common import (
|
from .common import (
|
||||||
TEST_RSA_KEY_SIZE,
|
TEST_RSA_KEY_SIZE,
|
||||||
SameProcessStreamEndpointAssigner,
|
SameProcessStreamEndpointAssigner,
|
||||||
|
|
@ -61,13 +65,32 @@ from .web.common import (
|
||||||
)
|
)
|
||||||
|
|
||||||
# TODO: move this to common or common_util
|
# TODO: move this to common or common_util
|
||||||
from allmydata.test.test_runner import RunBinTahoeMixin
|
|
||||||
from . import common_util as testutil
|
from . import common_util as testutil
|
||||||
from .common_util import run_cli_unicode
|
from .common_util import run_cli_unicode
|
||||||
from ..scripts.common import (
|
from ..scripts.common import (
|
||||||
write_introducer,
|
write_introducer,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
class RunBinTahoeMixin(object):
|
||||||
|
def run_bintahoe(self, args, stdin=None, python_options=[], env=None):
|
||||||
|
# test_runner.run_bintahoe has better unicode support but doesn't
|
||||||
|
# support env yet and is also synchronous. If we could get rid of
|
||||||
|
# this in favor of that, though, it would probably be an improvement.
|
||||||
|
command = sys.executable
|
||||||
|
argv = python_options + ["-m", "allmydata.scripts.runner"] + args
|
||||||
|
|
||||||
|
if env is None:
|
||||||
|
env = os.environ
|
||||||
|
|
||||||
|
d = getProcessOutputAndValue(command, argv, env, stdinBytes=stdin)
|
||||||
|
def fix_signal(result):
|
||||||
|
# Mirror subprocess.Popen.returncode structure
|
||||||
|
(out, err, signal) = result
|
||||||
|
return (out, err, -signal)
|
||||||
|
d.addErrback(fix_signal)
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
def run_cli(*args, **kwargs):
|
def run_cli(*args, **kwargs):
|
||||||
"""
|
"""
|
||||||
Run a Tahoe-LAFS CLI utility, but inline.
|
Run a Tahoe-LAFS CLI utility, but inline.
|
||||||
|
|
|
||||||
|
|
@ -888,6 +888,34 @@ def is_happy_enough(servertoshnums, h, k):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class FileHandleTests(unittest.TestCase):
|
||||||
|
"""
|
||||||
|
Tests for ``FileHandle``.
|
||||||
|
"""
|
||||||
|
def test_get_encryption_key_convergent(self):
|
||||||
|
"""
|
||||||
|
When ``FileHandle`` is initialized with a convergence secret,
|
||||||
|
``FileHandle.get_encryption_key`` returns a deterministic result that
|
||||||
|
is a function of that secret.
|
||||||
|
"""
|
||||||
|
secret = b"\x42" * 16
|
||||||
|
handle = upload.FileHandle(BytesIO(b"hello world"), secret)
|
||||||
|
handle.set_default_encoding_parameters({
|
||||||
|
"k": 3,
|
||||||
|
"happy": 5,
|
||||||
|
"n": 10,
|
||||||
|
# Remember this is the *max* segment size. In reality, the data
|
||||||
|
# size is much smaller so the actual segment size incorporated
|
||||||
|
# into the encryption key is also smaller.
|
||||||
|
"max_segment_size": 128 * 1024,
|
||||||
|
})
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
b64encode(self.successResultOf(handle.get_encryption_key())),
|
||||||
|
b"oBcuR/wKdCgCV2GKKXqiNg==",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin,
|
class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin,
|
||||||
ShouldFailMixin):
|
ShouldFailMixin):
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -491,12 +491,16 @@ class JSONBytes(unittest.TestCase):
|
||||||
"""Tests for BytesJSONEncoder."""
|
"""Tests for BytesJSONEncoder."""
|
||||||
|
|
||||||
def test_encode_bytes(self):
|
def test_encode_bytes(self):
|
||||||
"""BytesJSONEncoder can encode bytes."""
|
"""BytesJSONEncoder can encode bytes.
|
||||||
|
|
||||||
|
Bytes are presumed to be UTF-8 encoded.
|
||||||
|
"""
|
||||||
|
snowman = u"def\N{SNOWMAN}\uFF00"
|
||||||
data = {
|
data = {
|
||||||
b"hello": [1, b"cd"],
|
b"hello": [1, b"cd", {b"abc": [123, snowman.encode("utf-8")]}],
|
||||||
}
|
}
|
||||||
expected = {
|
expected = {
|
||||||
u"hello": [1, u"cd"],
|
u"hello": [1, u"cd", {u"abc": [123, snowman]}],
|
||||||
}
|
}
|
||||||
# Bytes get passed through as if they were UTF-8 Unicode:
|
# Bytes get passed through as if they were UTF-8 Unicode:
|
||||||
encoded = jsonbytes.dumps(data)
|
encoded = jsonbytes.dumps(data)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,225 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Tahoe-LAFS -- secure, distributed storage grid
|
||||||
|
#
|
||||||
|
# Copyright © 2020 The Tahoe-LAFS Software Foundation
|
||||||
|
#
|
||||||
|
# This file is part of Tahoe-LAFS.
|
||||||
|
#
|
||||||
|
# See the docs/about.rst file for licensing information.
|
||||||
|
|
||||||
|
"""
|
||||||
|
Tests for the ``allmydata.windows``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import division
|
||||||
|
from __future__ import absolute_import
|
||||||
|
from __future__ import print_function
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from future.utils import PY2
|
||||||
|
if PY2:
|
||||||
|
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||||
|
|
||||||
|
from sys import (
|
||||||
|
executable,
|
||||||
|
)
|
||||||
|
from json import (
|
||||||
|
load,
|
||||||
|
)
|
||||||
|
from textwrap import (
|
||||||
|
dedent,
|
||||||
|
)
|
||||||
|
from twisted.python.filepath import (
|
||||||
|
FilePath,
|
||||||
|
)
|
||||||
|
from twisted.python.runtime import (
|
||||||
|
platform,
|
||||||
|
)
|
||||||
|
|
||||||
|
from testtools import (
|
||||||
|
skipUnless,
|
||||||
|
)
|
||||||
|
|
||||||
|
from testtools.matchers import (
|
||||||
|
MatchesAll,
|
||||||
|
AllMatch,
|
||||||
|
IsInstance,
|
||||||
|
Equals,
|
||||||
|
)
|
||||||
|
|
||||||
|
from hypothesis import (
|
||||||
|
HealthCheck,
|
||||||
|
settings,
|
||||||
|
given,
|
||||||
|
note,
|
||||||
|
)
|
||||||
|
|
||||||
|
from hypothesis.strategies import (
|
||||||
|
lists,
|
||||||
|
text,
|
||||||
|
characters,
|
||||||
|
)
|
||||||
|
|
||||||
|
from .common import (
|
||||||
|
PIPE,
|
||||||
|
Popen,
|
||||||
|
SyncTestCase,
|
||||||
|
)
|
||||||
|
|
||||||
|
slow_settings = settings(
|
||||||
|
suppress_health_check=[HealthCheck.too_slow],
|
||||||
|
deadline=None,
|
||||||
|
|
||||||
|
# Reduce the number of examples required to consider the test a success.
|
||||||
|
# The default is 100. Launching a process is expensive so we'll try to do
|
||||||
|
# it as few times as we can get away with. To maintain good coverage,
|
||||||
|
# we'll try to pass as much data to each process as we can so we're still
|
||||||
|
# covering a good portion of the space.
|
||||||
|
max_examples=10,
|
||||||
|
)
|
||||||
|
|
||||||
|
@skipUnless(platform.isWindows(), "get_argv is Windows-only")
|
||||||
|
class GetArgvTests(SyncTestCase):
|
||||||
|
"""
|
||||||
|
Tests for ``get_argv``.
|
||||||
|
"""
|
||||||
|
def test_get_argv_return_type(self):
|
||||||
|
"""
|
||||||
|
``get_argv`` returns a list of unicode strings
|
||||||
|
"""
|
||||||
|
# Hide the ``allmydata.windows.fixups.get_argv`` import here so it
|
||||||
|
# doesn't cause failures on non-Windows platforms.
|
||||||
|
from ..windows.fixups import (
|
||||||
|
get_argv,
|
||||||
|
)
|
||||||
|
argv = get_argv()
|
||||||
|
|
||||||
|
# We don't know what this process's command line was so we just make
|
||||||
|
# structural assertions here.
|
||||||
|
self.assertThat(
|
||||||
|
argv,
|
||||||
|
MatchesAll(
|
||||||
|
IsInstance(list),
|
||||||
|
AllMatch(IsInstance(str)),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# This test runs a child process. This is unavoidably slow and variable.
|
||||||
|
# Disable the two time-based Hypothesis health checks.
|
||||||
|
@slow_settings
|
||||||
|
@given(
|
||||||
|
lists(
|
||||||
|
text(
|
||||||
|
alphabet=characters(
|
||||||
|
blacklist_categories=('Cs',),
|
||||||
|
# Windows CommandLine is a null-terminated string,
|
||||||
|
# analogous to POSIX exec* arguments. So exclude nul from
|
||||||
|
# our generated arguments.
|
||||||
|
blacklist_characters=('\x00',),
|
||||||
|
),
|
||||||
|
min_size=10,
|
||||||
|
max_size=20,
|
||||||
|
),
|
||||||
|
min_size=10,
|
||||||
|
max_size=20,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
def test_argv_values(self, argv):
|
||||||
|
"""
|
||||||
|
``get_argv`` returns a list representing the result of tokenizing the
|
||||||
|
"command line" argument string provided to Windows processes.
|
||||||
|
"""
|
||||||
|
working_path = FilePath(self.mktemp())
|
||||||
|
working_path.makedirs()
|
||||||
|
save_argv_path = working_path.child("script.py")
|
||||||
|
saved_argv_path = working_path.child("data.json")
|
||||||
|
with open(save_argv_path.path, "wt") as f:
|
||||||
|
# A simple program to save argv to a file. Using the file saves
|
||||||
|
# us having to figure out how to reliably get non-ASCII back over
|
||||||
|
# stdio which may pose an independent set of challenges. At least
|
||||||
|
# file I/O is relatively simple and well-understood.
|
||||||
|
f.write(dedent(
|
||||||
|
"""
|
||||||
|
from allmydata.windows.fixups import (
|
||||||
|
get_argv,
|
||||||
|
)
|
||||||
|
import json
|
||||||
|
with open({!r}, "wt") as f:
|
||||||
|
f.write(json.dumps(get_argv()))
|
||||||
|
""".format(saved_argv_path.path)),
|
||||||
|
)
|
||||||
|
argv = [executable.decode("utf-8"), save_argv_path.path] + argv
|
||||||
|
p = Popen(argv, stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
||||||
|
p.stdin.close()
|
||||||
|
stdout = p.stdout.read()
|
||||||
|
stderr = p.stderr.read()
|
||||||
|
returncode = p.wait()
|
||||||
|
|
||||||
|
note("stdout: {!r}".format(stdout))
|
||||||
|
note("stderr: {!r}".format(stderr))
|
||||||
|
|
||||||
|
self.assertThat(
|
||||||
|
returncode,
|
||||||
|
Equals(0),
|
||||||
|
)
|
||||||
|
with open(saved_argv_path.path, "rt") as f:
|
||||||
|
saved_argv = load(f)
|
||||||
|
|
||||||
|
self.assertThat(
|
||||||
|
saved_argv,
|
||||||
|
Equals(argv),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@skipUnless(platform.isWindows(), "intended for Windows-only codepaths")
|
||||||
|
class UnicodeOutputTests(SyncTestCase):
|
||||||
|
"""
|
||||||
|
Tests for writing unicode to stdout and stderr.
|
||||||
|
"""
|
||||||
|
@slow_settings
|
||||||
|
@given(characters(), characters())
|
||||||
|
def test_write_non_ascii(self, stdout_char, stderr_char):
|
||||||
|
"""
|
||||||
|
Non-ASCII unicode characters can be written to stdout and stderr with
|
||||||
|
automatic UTF-8 encoding.
|
||||||
|
"""
|
||||||
|
working_path = FilePath(self.mktemp())
|
||||||
|
working_path.makedirs()
|
||||||
|
script = working_path.child("script.py")
|
||||||
|
script.setContent(dedent(
|
||||||
|
"""
|
||||||
|
from future.utils import PY2
|
||||||
|
if PY2:
|
||||||
|
from future.builtins import chr
|
||||||
|
|
||||||
|
from allmydata.windows.fixups import initialize
|
||||||
|
initialize()
|
||||||
|
|
||||||
|
# XXX A shortcoming of the monkey-patch approach is that you'd
|
||||||
|
# better not import stdout or stderr before you call initialize.
|
||||||
|
from sys import argv, stdout, stderr
|
||||||
|
|
||||||
|
stdout.write(chr(int(argv[1])))
|
||||||
|
stdout.close()
|
||||||
|
stderr.write(chr(int(argv[2])))
|
||||||
|
stderr.close()
|
||||||
|
"""
|
||||||
|
))
|
||||||
|
p = Popen([
|
||||||
|
executable,
|
||||||
|
script.path,
|
||||||
|
str(ord(stdout_char)),
|
||||||
|
str(ord(stderr_char)),
|
||||||
|
], stdout=PIPE, stderr=PIPE)
|
||||||
|
stdout = p.stdout.read().decode("utf-8").replace("\r\n", "\n")
|
||||||
|
stderr = p.stderr.read().decode("utf-8").replace("\r\n", "\n")
|
||||||
|
returncode = p.wait()
|
||||||
|
|
||||||
|
self.assertThat(
|
||||||
|
(stdout, stderr, returncode),
|
||||||
|
Equals((
|
||||||
|
stdout_char,
|
||||||
|
stderr_char,
|
||||||
|
0,
|
||||||
|
)),
|
||||||
|
)
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -199,5 +199,6 @@ PORTED_TEST_MODULES = [
|
||||||
"allmydata.test.web.test_root",
|
"allmydata.test.web.test_root",
|
||||||
"allmydata.test.web.test_status",
|
"allmydata.test.web.test_status",
|
||||||
"allmydata.test.web.test_util",
|
"allmydata.test.web.test_util",
|
||||||
|
"allmydata.test.web.test_web",
|
||||||
"allmydata.test.web.test_webish",
|
"allmydata.test.web.test_webish",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -18,8 +18,9 @@ if PY2:
|
||||||
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
|
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
|
||||||
|
|
||||||
from past.builtins import unicode
|
from past.builtins import unicode
|
||||||
|
from six import ensure_str
|
||||||
|
|
||||||
import sys, os, re, locale
|
import sys, os, re
|
||||||
import unicodedata
|
import unicodedata
|
||||||
import warnings
|
import warnings
|
||||||
|
|
||||||
|
|
@ -50,36 +51,25 @@ def check_encoding(encoding):
|
||||||
try:
|
try:
|
||||||
u"test".encode(encoding)
|
u"test".encode(encoding)
|
||||||
except (LookupError, AttributeError):
|
except (LookupError, AttributeError):
|
||||||
raise AssertionError("The character encoding '%s' is not supported for conversion." % (encoding,))
|
raise AssertionError(
|
||||||
|
"The character encoding '%s' is not supported for conversion." % (encoding,),
|
||||||
|
)
|
||||||
|
|
||||||
|
# On Windows we install UTF-8 stream wrappers for sys.stdout and
|
||||||
|
# sys.stderr, and reencode the arguments as UTF-8 (see scripts/runner.py).
|
||||||
|
#
|
||||||
|
# On POSIX, we are moving towards a UTF-8-everything and ignore the locale.
|
||||||
|
io_encoding = "utf-8"
|
||||||
|
|
||||||
filesystem_encoding = None
|
filesystem_encoding = None
|
||||||
io_encoding = None
|
|
||||||
is_unicode_platform = False
|
is_unicode_platform = False
|
||||||
use_unicode_filepath = False
|
use_unicode_filepath = False
|
||||||
|
|
||||||
def _reload():
|
def _reload():
|
||||||
global filesystem_encoding, io_encoding, is_unicode_platform, use_unicode_filepath
|
global filesystem_encoding, is_unicode_platform, use_unicode_filepath
|
||||||
|
|
||||||
filesystem_encoding = canonical_encoding(sys.getfilesystemencoding())
|
filesystem_encoding = canonical_encoding(sys.getfilesystemencoding())
|
||||||
check_encoding(filesystem_encoding)
|
check_encoding(filesystem_encoding)
|
||||||
|
|
||||||
if sys.platform == 'win32':
|
|
||||||
# On Windows we install UTF-8 stream wrappers for sys.stdout and
|
|
||||||
# sys.stderr, and reencode the arguments as UTF-8 (see scripts/runner.py).
|
|
||||||
io_encoding = 'utf-8'
|
|
||||||
else:
|
|
||||||
ioenc = None
|
|
||||||
if hasattr(sys.stdout, 'encoding'):
|
|
||||||
ioenc = sys.stdout.encoding
|
|
||||||
if ioenc is None:
|
|
||||||
try:
|
|
||||||
ioenc = locale.getpreferredencoding()
|
|
||||||
except Exception:
|
|
||||||
pass # work around <http://bugs.python.org/issue1443504>
|
|
||||||
io_encoding = canonical_encoding(ioenc)
|
|
||||||
|
|
||||||
check_encoding(io_encoding)
|
|
||||||
|
|
||||||
is_unicode_platform = PY3 or sys.platform in ["win32", "darwin"]
|
is_unicode_platform = PY3 or sys.platform in ["win32", "darwin"]
|
||||||
|
|
||||||
# Despite the Unicode-mode FilePath support added to Twisted in
|
# Despite the Unicode-mode FilePath support added to Twisted in
|
||||||
|
|
@ -110,6 +100,8 @@ def get_io_encoding():
|
||||||
def argv_to_unicode(s):
|
def argv_to_unicode(s):
|
||||||
"""
|
"""
|
||||||
Decode given argv element to unicode. If this fails, raise a UsageError.
|
Decode given argv element to unicode. If this fails, raise a UsageError.
|
||||||
|
|
||||||
|
This is the inverse of ``unicode_to_argv``.
|
||||||
"""
|
"""
|
||||||
if isinstance(s, unicode):
|
if isinstance(s, unicode):
|
||||||
return s
|
return s
|
||||||
|
|
@ -133,26 +125,22 @@ def argv_to_abspath(s, **kwargs):
|
||||||
% (quote_output(s), quote_output(os.path.join('.', s))))
|
% (quote_output(s), quote_output(os.path.join('.', s))))
|
||||||
return abspath_expanduser_unicode(decoded, **kwargs)
|
return abspath_expanduser_unicode(decoded, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
def unicode_to_argv(s, mangle=False):
|
def unicode_to_argv(s, mangle=False):
|
||||||
"""
|
"""
|
||||||
Encode the given Unicode argument as a bytestring.
|
Make the given unicode string suitable for use in an argv list.
|
||||||
If the argument is to be passed to a different process, then the 'mangle' argument
|
|
||||||
should be true; on Windows, this uses a mangled encoding that will be reversed by
|
|
||||||
code in runner.py.
|
|
||||||
|
|
||||||
On Python 3, just return the string unchanged, since argv is unicode.
|
On Python 2 on POSIX, this encodes using UTF-8. On Python 3 and on
|
||||||
|
Windows, this returns the input unmodified.
|
||||||
"""
|
"""
|
||||||
precondition(isinstance(s, unicode), s)
|
precondition(isinstance(s, unicode), s)
|
||||||
if PY3:
|
if PY3:
|
||||||
warnings.warn("This will be unnecessary once Python 2 is dropped.",
|
warnings.warn("This will be unnecessary once Python 2 is dropped.",
|
||||||
DeprecationWarning)
|
DeprecationWarning)
|
||||||
|
if sys.platform == "win32":
|
||||||
return s
|
return s
|
||||||
|
return ensure_str(s)
|
||||||
|
|
||||||
if mangle and sys.platform == "win32":
|
|
||||||
# This must be the same as 'mangle' in bin/tahoe-script.template.
|
|
||||||
return bytes(re.sub(u'[^\\x20-\\x7F]', lambda m: u'\x7F%x;' % (ord(m.group(0)),), s), io_encoding)
|
|
||||||
else:
|
|
||||||
return s.encode(io_encoding)
|
|
||||||
|
|
||||||
def unicode_to_url(s):
|
def unicode_to_url(s):
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -176,10 +176,44 @@ def convergence_hash(k, n, segsize, data, convergence):
|
||||||
return h.digest()
|
return h.digest()
|
||||||
|
|
||||||
|
|
||||||
def convergence_hasher(k, n, segsize, convergence):
|
def _convergence_hasher_tag(k, n, segsize, convergence):
|
||||||
|
"""
|
||||||
|
Create the convergence hashing tag.
|
||||||
|
|
||||||
|
:param int k: Required shares (in [1..256]).
|
||||||
|
:param int n: Total shares (in [1..256]).
|
||||||
|
:param int segsize: Maximum segment size.
|
||||||
|
:param bytes convergence: The convergence secret.
|
||||||
|
|
||||||
|
:return bytes: The bytestring to use as a tag in the convergence hash.
|
||||||
|
"""
|
||||||
assert isinstance(convergence, bytes)
|
assert isinstance(convergence, bytes)
|
||||||
|
if k > n:
|
||||||
|
raise ValueError(
|
||||||
|
"k > n not allowed; k = {}, n = {}".format(k, n),
|
||||||
|
)
|
||||||
|
if k < 1 or n < 1:
|
||||||
|
# It doesn't make sense to have zero shares. Zero shares carry no
|
||||||
|
# information, cannot encode any part of the application data.
|
||||||
|
raise ValueError(
|
||||||
|
"k, n < 1 not allowed; k = {}, n = {}".format(k, n),
|
||||||
|
)
|
||||||
|
if k > 256 or n > 256:
|
||||||
|
# ZFEC supports encoding application data into a maximum of 256
|
||||||
|
# shares. If we ignore the limitations of ZFEC, it may be fine to use
|
||||||
|
# a configuration with more shares than that and it may be fine to
|
||||||
|
# construct a convergence tag from such a configuration. Since ZFEC
|
||||||
|
# is the only supported encoder, though, this is moot for now.
|
||||||
|
raise ValueError(
|
||||||
|
"k, n > 256 not allowed; k = {}, n = {}".format(k, n),
|
||||||
|
)
|
||||||
param_tag = netstring(b"%d,%d,%d" % (k, n, segsize))
|
param_tag = netstring(b"%d,%d,%d" % (k, n, segsize))
|
||||||
tag = CONVERGENT_ENCRYPTION_TAG + netstring(convergence) + param_tag
|
tag = CONVERGENT_ENCRYPTION_TAG + netstring(convergence) + param_tag
|
||||||
|
return tag
|
||||||
|
|
||||||
|
|
||||||
|
def convergence_hasher(k, n, segsize, convergence):
|
||||||
|
tag = _convergence_hasher_tag(k, n, segsize, convergence)
|
||||||
return tagged_hasher(tag, KEYLEN)
|
return tagged_hasher(tag, KEYLEN)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -13,20 +13,34 @@ from future.utils import PY2
|
||||||
if PY2:
|
if PY2:
|
||||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||||
|
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
|
||||||
|
|
||||||
|
def _bytes_to_unicode(obj):
|
||||||
|
"""Convert any bytes objects to unicode, recursively."""
|
||||||
|
if isinstance(obj, bytes):
|
||||||
|
return obj.decode("utf-8")
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
new_obj = {}
|
||||||
|
for k, v in obj.items():
|
||||||
|
if isinstance(k, bytes):
|
||||||
|
k = k.decode("utf-8")
|
||||||
|
v = _bytes_to_unicode(v)
|
||||||
|
new_obj[k] = v
|
||||||
|
return new_obj
|
||||||
|
if isinstance(obj, (list, set, tuple)):
|
||||||
|
return [_bytes_to_unicode(i) for i in obj]
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
class BytesJSONEncoder(json.JSONEncoder):
|
class BytesJSONEncoder(json.JSONEncoder):
|
||||||
"""
|
"""
|
||||||
A JSON encoder than can also encode bytes.
|
A JSON encoder than can also encode bytes.
|
||||||
|
|
||||||
The bytes are assumed to be UTF-8 encoded Unicode strings.
|
The bytes are assumed to be UTF-8 encoded Unicode strings.
|
||||||
"""
|
"""
|
||||||
def default(self, o):
|
def iterencode(self, o, **kwargs):
|
||||||
if isinstance(o, bytes):
|
return json.JSONEncoder.iterencode(self, _bytes_to_unicode(o), **kwargs)
|
||||||
return o.decode("utf-8")
|
|
||||||
return json.JSONEncoder.default(self, o)
|
|
||||||
|
|
||||||
|
|
||||||
def dumps(obj, *args, **kwargs):
|
def dumps(obj, *args, **kwargs):
|
||||||
|
|
@ -34,13 +48,6 @@ def dumps(obj, *args, **kwargs):
|
||||||
|
|
||||||
The bytes are assumed to be UTF-8 encoded Unicode strings.
|
The bytes are assumed to be UTF-8 encoded Unicode strings.
|
||||||
"""
|
"""
|
||||||
if isinstance(obj, dict):
|
|
||||||
new_obj = {}
|
|
||||||
for k, v in obj.items():
|
|
||||||
if isinstance(k, bytes):
|
|
||||||
k = k.decode("utf-8")
|
|
||||||
new_obj[k] = v
|
|
||||||
obj = new_obj
|
|
||||||
return json.dumps(obj, cls=BytesJSONEncoder, *args, **kwargs)
|
return json.dumps(obj, cls=BytesJSONEncoder, *args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -432,7 +432,7 @@ class DeepCheckResultsRenderer(MultiFormatResource):
|
||||||
return CheckResultsRenderer(self._client,
|
return CheckResultsRenderer(self._client,
|
||||||
r.get_results_for_storage_index(si))
|
r.get_results_for_storage_index(si))
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise WebError("No detailed results for SI %s" % html.escape(name),
|
raise WebError("No detailed results for SI %s" % html.escape(str(name, "utf-8")),
|
||||||
http.NOT_FOUND)
|
http.NOT_FOUND)
|
||||||
|
|
||||||
@render_exception
|
@render_exception
|
||||||
|
|
|
||||||
|
|
@ -186,7 +186,7 @@ def convert_children_json(nodemaker, children_json):
|
||||||
children = {}
|
children = {}
|
||||||
if children_json:
|
if children_json:
|
||||||
data = json.loads(children_json)
|
data = json.loads(children_json)
|
||||||
for (namex, (ctype, propdict)) in data.iteritems():
|
for (namex, (ctype, propdict)) in data.items():
|
||||||
namex = unicode(namex)
|
namex = unicode(namex)
|
||||||
writecap = to_bytes(propdict.get("rw_uri"))
|
writecap = to_bytes(propdict.get("rw_uri"))
|
||||||
readcap = to_bytes(propdict.get("ro_uri"))
|
readcap = to_bytes(propdict.get("ro_uri"))
|
||||||
|
|
@ -283,8 +283,8 @@ def render_time_attr(t):
|
||||||
# actual exception). The latter is growing increasingly annoying.
|
# actual exception). The latter is growing increasingly annoying.
|
||||||
|
|
||||||
def should_create_intermediate_directories(req):
|
def should_create_intermediate_directories(req):
|
||||||
t = get_arg(req, "t", "").strip()
|
t = unicode(get_arg(req, "t", "").strip(), "ascii")
|
||||||
return bool(req.method in ("PUT", "POST") and
|
return bool(req.method in (b"PUT", b"POST") and
|
||||||
t not in ("delete", "rename", "rename-form", "check"))
|
t not in ("delete", "rename", "rename-form", "check"))
|
||||||
|
|
||||||
def humanize_exception(exc):
|
def humanize_exception(exc):
|
||||||
|
|
@ -674,7 +674,7 @@ def url_for_string(req, url_string):
|
||||||
and the given URL string.
|
and the given URL string.
|
||||||
"""
|
"""
|
||||||
url = DecodedURL.from_text(url_string.decode("utf-8"))
|
url = DecodedURL.from_text(url_string.decode("utf-8"))
|
||||||
if url.host == b"":
|
if not url.host:
|
||||||
root = req.URLPath()
|
root = req.URLPath()
|
||||||
netloc = root.netloc.split(b":", 1)
|
netloc = root.netloc.split(b":", 1)
|
||||||
if len(netloc) == 1:
|
if len(netloc) == 1:
|
||||||
|
|
|
||||||
|
|
@ -40,8 +40,12 @@ def get_arg(req, argname, default=None, multiple=False):
|
||||||
results = []
|
results = []
|
||||||
if argname in req.args:
|
if argname in req.args:
|
||||||
results.extend(req.args[argname])
|
results.extend(req.args[argname])
|
||||||
if req.fields and argname in req.fields:
|
argname_unicode = unicode(argname, "utf-8")
|
||||||
results.append(req.fields[argname].value)
|
if req.fields and argname_unicode in req.fields:
|
||||||
|
value = req.fields[argname_unicode].value
|
||||||
|
if isinstance(value, unicode):
|
||||||
|
value = value.encode("utf-8")
|
||||||
|
results.append(value)
|
||||||
if multiple:
|
if multiple:
|
||||||
return tuple(results)
|
return tuple(results)
|
||||||
if results:
|
if results:
|
||||||
|
|
@ -79,7 +83,13 @@ class MultiFormatResource(resource.Resource, object):
|
||||||
if isinstance(t, bytes):
|
if isinstance(t, bytes):
|
||||||
t = unicode(t, "ascii")
|
t = unicode(t, "ascii")
|
||||||
renderer = self._get_renderer(t)
|
renderer = self._get_renderer(t)
|
||||||
return renderer(req)
|
result = renderer(req)
|
||||||
|
# On Python 3, json.dumps() returns Unicode for example, but
|
||||||
|
# twisted.web expects bytes. Instead of updating every single render
|
||||||
|
# method, just handle Unicode one time here.
|
||||||
|
if isinstance(result, unicode):
|
||||||
|
result = result.encode("utf-8")
|
||||||
|
return result
|
||||||
|
|
||||||
def _get_renderer(self, fmt):
|
def _get_renderer(self, fmt):
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,11 @@
|
||||||
|
"""
|
||||||
|
TODO: When porting to Python 3, the filename handling logic seems wrong. On
|
||||||
|
Python 3 filename will _already_ be correctly decoded. So only decode if it's
|
||||||
|
bytes.
|
||||||
|
|
||||||
|
Also there's a lot of code duplication I think.
|
||||||
|
"""
|
||||||
|
|
||||||
from past.builtins import unicode
|
from past.builtins import unicode
|
||||||
|
|
||||||
from urllib.parse import quote as url_quote
|
from urllib.parse import quote as url_quote
|
||||||
|
|
@ -135,7 +143,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
|
||||||
terminal = (req.prepath + req.postpath)[-1].decode('utf8') == name
|
terminal = (req.prepath + req.postpath)[-1].decode('utf8') == name
|
||||||
nonterminal = not terminal #len(req.postpath) > 0
|
nonterminal = not terminal #len(req.postpath) > 0
|
||||||
|
|
||||||
t = get_arg(req, b"t", b"").strip()
|
t = unicode(get_arg(req, b"t", b"").strip(), "ascii")
|
||||||
if isinstance(node_or_failure, Failure):
|
if isinstance(node_or_failure, Failure):
|
||||||
f = node_or_failure
|
f = node_or_failure
|
||||||
f.trap(NoSuchChildError)
|
f.trap(NoSuchChildError)
|
||||||
|
|
@ -150,10 +158,10 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
|
||||||
else:
|
else:
|
||||||
# terminal node
|
# terminal node
|
||||||
terminal_requests = (
|
terminal_requests = (
|
||||||
("POST", "mkdir"),
|
(b"POST", "mkdir"),
|
||||||
("PUT", "mkdir"),
|
(b"PUT", "mkdir"),
|
||||||
("POST", "mkdir-with-children"),
|
(b"POST", "mkdir-with-children"),
|
||||||
("POST", "mkdir-immutable")
|
(b"POST", "mkdir-immutable")
|
||||||
)
|
)
|
||||||
if (req.method, t) in terminal_requests:
|
if (req.method, t) in terminal_requests:
|
||||||
# final directory
|
# final directory
|
||||||
|
|
@ -182,8 +190,8 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
|
||||||
)
|
)
|
||||||
return d
|
return d
|
||||||
leaf_requests = (
|
leaf_requests = (
|
||||||
("PUT",""),
|
(b"PUT",""),
|
||||||
("PUT","uri"),
|
(b"PUT","uri"),
|
||||||
)
|
)
|
||||||
if (req.method, t) in leaf_requests:
|
if (req.method, t) in leaf_requests:
|
||||||
# we were trying to find the leaf filenode (to put a new
|
# we were trying to find the leaf filenode (to put a new
|
||||||
|
|
@ -224,7 +232,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
|
||||||
FIXED_OUTPUT_TYPES = ["", "json", "uri", "readonly-uri"]
|
FIXED_OUTPUT_TYPES = ["", "json", "uri", "readonly-uri"]
|
||||||
if not self.node.is_mutable() and t in FIXED_OUTPUT_TYPES:
|
if not self.node.is_mutable() and t in FIXED_OUTPUT_TYPES:
|
||||||
si = self.node.get_storage_index()
|
si = self.node.get_storage_index()
|
||||||
if si and req.setETag('DIR:%s-%s' % (base32.b2a(si), t or "")):
|
if si and req.setETag(b'DIR:%s-%s' % (base32.b2a(si), t.encode("ascii") or b"")):
|
||||||
return b""
|
return b""
|
||||||
|
|
||||||
if not t:
|
if not t:
|
||||||
|
|
@ -255,7 +263,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
|
||||||
|
|
||||||
@render_exception
|
@render_exception
|
||||||
def render_PUT(self, req):
|
def render_PUT(self, req):
|
||||||
t = get_arg(req, b"t", b"").strip()
|
t = unicode(get_arg(req, b"t", b"").strip(), "ascii")
|
||||||
replace = parse_replace_arg(get_arg(req, "replace", "true"))
|
replace = parse_replace_arg(get_arg(req, "replace", "true"))
|
||||||
|
|
||||||
if t == "mkdir":
|
if t == "mkdir":
|
||||||
|
|
@ -364,7 +372,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
|
||||||
return d
|
return d
|
||||||
|
|
||||||
def _POST_upload(self, req):
|
def _POST_upload(self, req):
|
||||||
charset = get_arg(req, "_charset", "utf-8")
|
charset = unicode(get_arg(req, "_charset", b"utf-8"), "utf-8")
|
||||||
contents = req.fields["file"]
|
contents = req.fields["file"]
|
||||||
assert contents.filename is None or isinstance(contents.filename, str)
|
assert contents.filename is None or isinstance(contents.filename, str)
|
||||||
name = get_arg(req, "name")
|
name = get_arg(req, "name")
|
||||||
|
|
@ -374,8 +382,8 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
|
||||||
if not name:
|
if not name:
|
||||||
# this prohibts empty, missing, and all-whitespace filenames
|
# this prohibts empty, missing, and all-whitespace filenames
|
||||||
raise WebError("upload requires a name")
|
raise WebError("upload requires a name")
|
||||||
assert isinstance(name, str)
|
if isinstance(name, bytes):
|
||||||
name = name.decode(charset)
|
name = name.decode(charset)
|
||||||
if "/" in name:
|
if "/" in name:
|
||||||
raise WebError("name= may not contain a slash", http.BAD_REQUEST)
|
raise WebError("name= may not contain a slash", http.BAD_REQUEST)
|
||||||
assert isinstance(name, unicode)
|
assert isinstance(name, unicode)
|
||||||
|
|
@ -413,7 +421,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
|
||||||
name = get_arg(req, "name")
|
name = get_arg(req, "name")
|
||||||
if not name:
|
if not name:
|
||||||
raise WebError("set-uri requires a name")
|
raise WebError("set-uri requires a name")
|
||||||
charset = get_arg(req, "_charset", "utf-8")
|
charset = unicode(get_arg(req, "_charset", b"utf-8"), "ascii")
|
||||||
name = name.decode(charset)
|
name = name.decode(charset)
|
||||||
replace = parse_replace_arg(get_arg(req, "replace", "true"))
|
replace = parse_replace_arg(get_arg(req, "replace", "true"))
|
||||||
|
|
||||||
|
|
@ -436,8 +444,8 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
|
||||||
# a slightly confusing error message if someone does a POST
|
# a slightly confusing error message if someone does a POST
|
||||||
# without a name= field. For our own HTML this isn't a big
|
# without a name= field. For our own HTML this isn't a big
|
||||||
# deal, because we create the 'unlink' POST buttons ourselves.
|
# deal, because we create the 'unlink' POST buttons ourselves.
|
||||||
name = ''
|
name = b''
|
||||||
charset = get_arg(req, "_charset", "utf-8")
|
charset = unicode(get_arg(req, "_charset", b"utf-8"), "ascii")
|
||||||
name = name.decode(charset)
|
name = name.decode(charset)
|
||||||
d = self.node.delete(name)
|
d = self.node.delete(name)
|
||||||
d.addCallback(lambda res: "thing unlinked")
|
d.addCallback(lambda res: "thing unlinked")
|
||||||
|
|
@ -453,7 +461,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
|
||||||
return self._POST_relink(req)
|
return self._POST_relink(req)
|
||||||
|
|
||||||
def _POST_relink(self, req):
|
def _POST_relink(self, req):
|
||||||
charset = get_arg(req, "_charset", "utf-8")
|
charset = unicode(get_arg(req, "_charset", b"utf-8"), "ascii")
|
||||||
replace = parse_replace_arg(get_arg(req, "replace", "true"))
|
replace = parse_replace_arg(get_arg(req, "replace", "true"))
|
||||||
|
|
||||||
from_name = get_arg(req, "from_name")
|
from_name = get_arg(req, "from_name")
|
||||||
|
|
@ -624,14 +632,14 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
|
||||||
# TODO test handling of bad JSON
|
# TODO test handling of bad JSON
|
||||||
raise
|
raise
|
||||||
cs = {}
|
cs = {}
|
||||||
for name, (file_or_dir, mddict) in children.iteritems():
|
for name, (file_or_dir, mddict) in children.items():
|
||||||
name = unicode(name) # json returns str *or* unicode
|
name = unicode(name) # json returns str *or* unicode
|
||||||
writecap = mddict.get('rw_uri')
|
writecap = mddict.get('rw_uri')
|
||||||
if writecap is not None:
|
if writecap is not None:
|
||||||
writecap = str(writecap)
|
writecap = writecap.encode("utf-8")
|
||||||
readcap = mddict.get('ro_uri')
|
readcap = mddict.get('ro_uri')
|
||||||
if readcap is not None:
|
if readcap is not None:
|
||||||
readcap = str(readcap)
|
readcap = readcap.encode("utf-8")
|
||||||
cs[name] = (writecap, readcap, mddict.get('metadata'))
|
cs[name] = (writecap, readcap, mddict.get('metadata'))
|
||||||
d = self.node.set_children(cs, replace)
|
d = self.node.set_children(cs, replace)
|
||||||
d.addCallback(lambda res: "Okay so I did it.")
|
d.addCallback(lambda res: "Okay so I did it.")
|
||||||
|
|
@ -1144,8 +1152,8 @@ def _slashify_path(path):
|
||||||
in it
|
in it
|
||||||
"""
|
"""
|
||||||
if not path:
|
if not path:
|
||||||
return ""
|
return b""
|
||||||
return "/".join([p.encode("utf-8") for p in path])
|
return b"/".join([p.encode("utf-8") for p in path])
|
||||||
|
|
||||||
|
|
||||||
def _cap_to_link(root, path, cap):
|
def _cap_to_link(root, path, cap):
|
||||||
|
|
@ -1234,10 +1242,10 @@ class ManifestResults(MultiFormatResource, ReloadMixin):
|
||||||
req.setHeader("content-type", "text/plain")
|
req.setHeader("content-type", "text/plain")
|
||||||
lines = []
|
lines = []
|
||||||
is_finished = self.monitor.is_finished()
|
is_finished = self.monitor.is_finished()
|
||||||
lines.append("finished: " + {True: "yes", False: "no"}[is_finished])
|
lines.append(b"finished: " + {True: b"yes", False: b"no"}[is_finished])
|
||||||
for path, cap in self.monitor.get_status()["manifest"]:
|
for path, cap in self.monitor.get_status()["manifest"]:
|
||||||
lines.append(_slashify_path(path) + " " + cap)
|
lines.append(_slashify_path(path) + b" " + cap)
|
||||||
return "\n".join(lines) + "\n"
|
return b"\n".join(lines) + b"\n"
|
||||||
|
|
||||||
def render_JSON(self, req):
|
def render_JSON(self, req):
|
||||||
req.setHeader("content-type", "text/plain")
|
req.setHeader("content-type", "text/plain")
|
||||||
|
|
@ -1290,7 +1298,7 @@ class DeepSizeResults(MultiFormatResource):
|
||||||
+ stats.get("size-mutable-files", 0)
|
+ stats.get("size-mutable-files", 0)
|
||||||
+ stats.get("size-directories", 0))
|
+ stats.get("size-directories", 0))
|
||||||
output += "size: %d\n" % total
|
output += "size: %d\n" % total
|
||||||
return output
|
return output.encode("utf-8")
|
||||||
render_TEXT = render_HTML
|
render_TEXT = render_HTML
|
||||||
|
|
||||||
def render_JSON(self, req):
|
def render_JSON(self, req):
|
||||||
|
|
@ -1315,7 +1323,7 @@ class DeepStatsResults(Resource, object):
|
||||||
req.setHeader("content-type", "text/plain")
|
req.setHeader("content-type", "text/plain")
|
||||||
s = self.monitor.get_status().copy()
|
s = self.monitor.get_status().copy()
|
||||||
s["finished"] = self.monitor.is_finished()
|
s["finished"] = self.monitor.is_finished()
|
||||||
return json.dumps(s, indent=1)
|
return json.dumps(s, indent=1).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
@implementer(IPushProducer)
|
@implementer(IPushProducer)
|
||||||
|
|
|
||||||
|
|
@ -127,7 +127,7 @@ class PlaceHolderNodeHandler(Resource, ReplaceMeMixin):
|
||||||
http.NOT_IMPLEMENTED)
|
http.NOT_IMPLEMENTED)
|
||||||
if not t:
|
if not t:
|
||||||
return self.replace_me_with_a_child(req, self.client, replace)
|
return self.replace_me_with_a_child(req, self.client, replace)
|
||||||
if t == "uri":
|
if t == b"uri":
|
||||||
return self.replace_me_with_a_childcap(req, self.client, replace)
|
return self.replace_me_with_a_childcap(req, self.client, replace)
|
||||||
|
|
||||||
raise WebError("PUT to a file: bad t=%s" % t)
|
raise WebError("PUT to a file: bad t=%s" % t)
|
||||||
|
|
@ -188,8 +188,8 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object):
|
||||||
# if the client already has the ETag then we can
|
# if the client already has the ETag then we can
|
||||||
# short-circuit the whole process.
|
# short-circuit the whole process.
|
||||||
si = self.node.get_storage_index()
|
si = self.node.get_storage_index()
|
||||||
if si and req.setETag('%s-%s' % (base32.b2a(si), t or "")):
|
if si and req.setETag(b'%s-%s' % (base32.b2a(si), t.encode("ascii") or b"")):
|
||||||
return ""
|
return b""
|
||||||
|
|
||||||
if not t:
|
if not t:
|
||||||
# just get the contents
|
# just get the contents
|
||||||
|
|
@ -281,7 +281,7 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object):
|
||||||
assert self.parentnode and self.name
|
assert self.parentnode and self.name
|
||||||
return self.replace_me_with_a_child(req, self.client, replace)
|
return self.replace_me_with_a_child(req, self.client, replace)
|
||||||
|
|
||||||
if t == "uri":
|
if t == b"uri":
|
||||||
if not replace:
|
if not replace:
|
||||||
raise ExistingChildError()
|
raise ExistingChildError()
|
||||||
assert self.parentnode and self.name
|
assert self.parentnode and self.name
|
||||||
|
|
@ -309,7 +309,7 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object):
|
||||||
assert self.parentnode and self.name
|
assert self.parentnode and self.name
|
||||||
d = self.replace_me_with_a_formpost(req, self.client, replace)
|
d = self.replace_me_with_a_formpost(req, self.client, replace)
|
||||||
else:
|
else:
|
||||||
raise WebError("POST to file: bad t=%s" % t)
|
raise WebError("POST to file: bad t=%s" % unicode(t, "ascii"))
|
||||||
|
|
||||||
return handle_when_done(req, d)
|
return handle_when_done(req, d)
|
||||||
|
|
||||||
|
|
@ -439,7 +439,7 @@ class FileDownloader(Resource, object):
|
||||||
# bytes we were given in the URL. See the comment in
|
# bytes we were given in the URL. See the comment in
|
||||||
# FileNodeHandler.render_GET for the sad details.
|
# FileNodeHandler.render_GET for the sad details.
|
||||||
req.setHeader("content-disposition",
|
req.setHeader("content-disposition",
|
||||||
'attachment; filename="%s"' % self.filename)
|
b'attachment; filename="%s"' % self.filename)
|
||||||
|
|
||||||
filesize = self.filenode.get_size()
|
filesize = self.filenode.get_size()
|
||||||
assert isinstance(filesize, (int,long)), filesize
|
assert isinstance(filesize, (int,long)), filesize
|
||||||
|
|
@ -475,8 +475,8 @@ class FileDownloader(Resource, object):
|
||||||
size = contentsize
|
size = contentsize
|
||||||
|
|
||||||
req.setHeader("content-length", b"%d" % contentsize)
|
req.setHeader("content-length", b"%d" % contentsize)
|
||||||
if req.method == "HEAD":
|
if req.method == b"HEAD":
|
||||||
return ""
|
return b""
|
||||||
|
|
||||||
d = self.filenode.read(req, first, size)
|
d = self.filenode.read(req, first, size)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
|
|
||||||
import os, urllib
|
import os
|
||||||
|
from urllib.parse import quote as urlquote
|
||||||
|
|
||||||
from twisted.python.filepath import FilePath
|
from twisted.python.filepath import FilePath
|
||||||
from twisted.web.template import tags as T, Element, renderElement, XMLFile, renderer
|
from twisted.web.template import tags as T, Element, renderElement, XMLFile, renderer
|
||||||
|
|
@ -180,7 +181,7 @@ class MoreInfoElement(Element):
|
||||||
else:
|
else:
|
||||||
return ""
|
return ""
|
||||||
root = self.get_root(req)
|
root = self.get_root(req)
|
||||||
quoted_uri = urllib.quote(node.get_uri())
|
quoted_uri = urlquote(node.get_uri())
|
||||||
text_plain_url = "%s/file/%s/@@named=/raw.txt" % (root, quoted_uri)
|
text_plain_url = "%s/file/%s/@@named=/raw.txt" % (root, quoted_uri)
|
||||||
return T.li("Raw data as ", T.a("text/plain", href=text_plain_url))
|
return T.li("Raw data as ", T.a("text/plain", href=text_plain_url))
|
||||||
|
|
||||||
|
|
@ -196,7 +197,7 @@ class MoreInfoElement(Element):
|
||||||
@renderer
|
@renderer
|
||||||
def check_form(self, req, tag):
|
def check_form(self, req, tag):
|
||||||
node = self.original
|
node = self.original
|
||||||
quoted_uri = urllib.quote(node.get_uri())
|
quoted_uri = urlquote(node.get_uri())
|
||||||
target = self.get_root(req) + "/uri/" + quoted_uri
|
target = self.get_root(req) + "/uri/" + quoted_uri
|
||||||
if IDirectoryNode.providedBy(node):
|
if IDirectoryNode.providedBy(node):
|
||||||
target += "/"
|
target += "/"
|
||||||
|
|
@ -236,8 +237,8 @@ class MoreInfoElement(Element):
|
||||||
def overwrite_form(self, req, tag):
|
def overwrite_form(self, req, tag):
|
||||||
node = self.original
|
node = self.original
|
||||||
root = self.get_root(req)
|
root = self.get_root(req)
|
||||||
action = "%s/uri/%s" % (root, urllib.quote(node.get_uri()))
|
action = "%s/uri/%s" % (root, urlquote(node.get_uri()))
|
||||||
done_url = "%s/uri/%s?t=info" % (root, urllib.quote(node.get_uri()))
|
done_url = "%s/uri/%s?t=info" % (root, urlquote(node.get_uri()))
|
||||||
overwrite = T.form(action=action, method="post",
|
overwrite = T.form(action=action, method="post",
|
||||||
enctype="multipart/form-data")(
|
enctype="multipart/form-data")(
|
||||||
T.fieldset(
|
T.fieldset(
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
from past.builtins import unicode
|
||||||
|
|
||||||
import time
|
import time
|
||||||
from hyperlink import (
|
from hyperlink import (
|
||||||
|
|
@ -101,12 +102,12 @@ class OphandleTable(resource.Resource, service.Service):
|
||||||
def getChild(self, name, req):
|
def getChild(self, name, req):
|
||||||
ophandle = name
|
ophandle = name
|
||||||
if ophandle not in self.handles:
|
if ophandle not in self.handles:
|
||||||
raise WebError("unknown/expired handle '%s'" % escape(ophandle),
|
raise WebError("unknown/expired handle '%s'" % escape(unicode(ophandle, "utf-8")),
|
||||||
NOT_FOUND)
|
NOT_FOUND)
|
||||||
(monitor, renderer, when_added) = self.handles[ophandle]
|
(monitor, renderer, when_added) = self.handles[ophandle]
|
||||||
|
|
||||||
t = get_arg(req, "t", "status")
|
t = get_arg(req, "t", "status")
|
||||||
if t == "cancel" and req.method == "POST":
|
if t == b"cancel" and req.method == b"POST":
|
||||||
monitor.cancel()
|
monitor.cancel()
|
||||||
# return the status anyways, but release the handle
|
# return the status anyways, but release the handle
|
||||||
self._release_ophandle(ophandle)
|
self._release_ophandle(ophandle)
|
||||||
|
|
@ -151,7 +152,7 @@ class ReloadMixin(object):
|
||||||
@renderer
|
@renderer
|
||||||
def refresh(self, req, tag):
|
def refresh(self, req, tag):
|
||||||
if self.monitor.is_finished():
|
if self.monitor.is_finished():
|
||||||
return ""
|
return b""
|
||||||
tag.attributes["http-equiv"] = "refresh"
|
tag.attributes["http-equiv"] = "refresh"
|
||||||
tag.attributes["content"] = str(self.REFRESH_TIME)
|
tag.attributes["content"] = str(self.REFRESH_TIME)
|
||||||
return tag
|
return tag
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
from future.utils import PY3
|
from future.utils import PY3
|
||||||
|
from past.builtins import unicode
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
|
|
@ -97,7 +98,7 @@ class URIHandler(resource.Resource, object):
|
||||||
either "PUT /uri" to create an unlinked file, or
|
either "PUT /uri" to create an unlinked file, or
|
||||||
"PUT /uri?t=mkdir" to create an unlinked directory
|
"PUT /uri?t=mkdir" to create an unlinked directory
|
||||||
"""
|
"""
|
||||||
t = get_arg(req, "t", "").strip()
|
t = unicode(get_arg(req, "t", "").strip(), "utf-8")
|
||||||
if t == "":
|
if t == "":
|
||||||
file_format = get_format(req, "CHK")
|
file_format = get_format(req, "CHK")
|
||||||
mutable_type = get_mutable_type(file_format)
|
mutable_type = get_mutable_type(file_format)
|
||||||
|
|
@ -120,7 +121,7 @@ class URIHandler(resource.Resource, object):
|
||||||
unlinked file or "POST /uri?t=mkdir" to create a
|
unlinked file or "POST /uri?t=mkdir" to create a
|
||||||
new directory
|
new directory
|
||||||
"""
|
"""
|
||||||
t = get_arg(req, "t", "").strip()
|
t = unicode(get_arg(req, "t", "").strip(), "ascii")
|
||||||
if t in ("", "upload"):
|
if t in ("", "upload"):
|
||||||
file_format = get_format(req)
|
file_format = get_format(req)
|
||||||
mutable_type = get_mutable_type(file_format)
|
mutable_type = get_mutable_type(file_format)
|
||||||
|
|
@ -177,7 +178,7 @@ class FileHandler(resource.Resource, object):
|
||||||
|
|
||||||
@exception_to_child
|
@exception_to_child
|
||||||
def getChild(self, name, req):
|
def getChild(self, name, req):
|
||||||
if req.method not in ("GET", "HEAD"):
|
if req.method not in (b"GET", b"HEAD"):
|
||||||
raise WebError("/file can only be used with GET or HEAD")
|
raise WebError("/file can only be used with GET or HEAD")
|
||||||
# 'name' must be a file URI
|
# 'name' must be a file URI
|
||||||
try:
|
try:
|
||||||
|
|
@ -200,7 +201,7 @@ class IncidentReporter(MultiFormatResource):
|
||||||
|
|
||||||
@render_exception
|
@render_exception
|
||||||
def render(self, req):
|
def render(self, req):
|
||||||
if req.method != "POST":
|
if req.method != b"POST":
|
||||||
raise WebError("/report_incident can only be used with POST")
|
raise WebError("/report_incident can only be used with POST")
|
||||||
|
|
||||||
log.msg(format="User reports incident through web page: %(details)s",
|
log.msg(format="User reports incident through web page: %(details)s",
|
||||||
|
|
@ -255,11 +256,11 @@ class Root(MultiFormatResource):
|
||||||
if not path:
|
if not path:
|
||||||
# Render "/" path.
|
# Render "/" path.
|
||||||
return self
|
return self
|
||||||
if path == "helper_status":
|
if path == b"helper_status":
|
||||||
# the Helper isn't attached until after the Tub starts, so this child
|
# the Helper isn't attached until after the Tub starts, so this child
|
||||||
# needs to created on each request
|
# needs to created on each request
|
||||||
return status.HelperStatus(self._client.helper)
|
return status.HelperStatus(self._client.helper)
|
||||||
if path == "storage":
|
if path == b"storage":
|
||||||
# Storage isn't initialized until after the web hierarchy is
|
# Storage isn't initialized until after the web hierarchy is
|
||||||
# constructed so this child needs to be created later than
|
# constructed so this child needs to be created later than
|
||||||
# `__init__`.
|
# `__init__`.
|
||||||
|
|
@ -293,7 +294,7 @@ class Root(MultiFormatResource):
|
||||||
self._describe_server(server)
|
self._describe_server(server)
|
||||||
for server
|
for server
|
||||||
in broker.get_known_servers()
|
in broker.get_known_servers()
|
||||||
))
|
), key=lambda o: sorted(o.items()))
|
||||||
|
|
||||||
|
|
||||||
def _describe_server(self, server):
|
def _describe_server(self, server):
|
||||||
|
|
|
||||||
|
|
@ -284,7 +284,7 @@ def _find_overlap(events, start_key, end_key):
|
||||||
rows = []
|
rows = []
|
||||||
for ev in events:
|
for ev in events:
|
||||||
ev = ev.copy()
|
ev = ev.copy()
|
||||||
if ev.has_key('server'):
|
if 'server' in ev:
|
||||||
ev["serverid"] = ev["server"].get_longname()
|
ev["serverid"] = ev["server"].get_longname()
|
||||||
del ev["server"]
|
del ev["server"]
|
||||||
# find an empty slot in the rows
|
# find an empty slot in the rows
|
||||||
|
|
@ -362,8 +362,8 @@ def _find_overlap_requests(events):
|
||||||
def _color(server):
|
def _color(server):
|
||||||
h = hashlib.sha256(server.get_serverid()).digest()
|
h = hashlib.sha256(server.get_serverid()).digest()
|
||||||
def m(c):
|
def m(c):
|
||||||
return min(ord(c) / 2 + 0x80, 0xff)
|
return min(ord(c) // 2 + 0x80, 0xff)
|
||||||
return "#%02x%02x%02x" % (m(h[0]), m(h[1]), m(h[2]))
|
return "#%02x%02x%02x" % (m(h[0:1]), m(h[1:2]), m(h[2:3]))
|
||||||
|
|
||||||
class _EventJson(Resource, object):
|
class _EventJson(Resource, object):
|
||||||
|
|
||||||
|
|
@ -426,7 +426,7 @@ class DownloadStatusPage(Resource, object):
|
||||||
"""
|
"""
|
||||||
super(DownloadStatusPage, self).__init__()
|
super(DownloadStatusPage, self).__init__()
|
||||||
self._download_status = download_status
|
self._download_status = download_status
|
||||||
self.putChild("event_json", _EventJson(self._download_status))
|
self.putChild(b"event_json", _EventJson(self._download_status))
|
||||||
|
|
||||||
@render_exception
|
@render_exception
|
||||||
def render_GET(self, req):
|
def render_GET(self, req):
|
||||||
|
|
@ -1288,14 +1288,14 @@ class Status(MultiFormatResource):
|
||||||
# final URL segment will be an empty string. Resources can
|
# final URL segment will be an empty string. Resources can
|
||||||
# thus know if they were requested with or without a final
|
# thus know if they were requested with or without a final
|
||||||
# slash."
|
# slash."
|
||||||
if not path and request.postpath != ['']:
|
if not path and request.postpath != [b'']:
|
||||||
return self
|
return self
|
||||||
|
|
||||||
h = self.history
|
h = self.history
|
||||||
try:
|
try:
|
||||||
stype, count_s = path.split("-")
|
stype, count_s = path.split(b"-")
|
||||||
except ValueError:
|
except ValueError:
|
||||||
raise WebError("no '-' in '{}'".format(path))
|
raise WebError("no '-' in '{}'".format(unicode(path, "utf-8")))
|
||||||
count = int(count_s)
|
count = int(count_s)
|
||||||
stype = unicode(stype, "ascii")
|
stype = unicode(stype, "ascii")
|
||||||
if stype == "up":
|
if stype == "up":
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
|
from past.builtins import unicode
|
||||||
|
|
||||||
import urllib
|
from urllib.parse import quote as urlquote
|
||||||
|
|
||||||
from twisted.web import http
|
from twisted.web import http
|
||||||
from twisted.internet import defer
|
from twisted.internet import defer
|
||||||
|
|
@ -65,8 +66,8 @@ def POSTUnlinkedCHK(req, client):
|
||||||
# if when_done= is provided, return a redirect instead of our
|
# if when_done= is provided, return a redirect instead of our
|
||||||
# usual upload-results page
|
# usual upload-results page
|
||||||
def _done(upload_results, redir_to):
|
def _done(upload_results, redir_to):
|
||||||
if "%(uri)s" in redir_to:
|
if b"%(uri)s" in redir_to:
|
||||||
redir_to = redir_to.replace("%(uri)s", urllib.quote(upload_results.get_uri()))
|
redir_to = redir_to.replace(b"%(uri)s", urlquote(upload_results.get_uri()).encode("utf-8"))
|
||||||
return url_for_string(req, redir_to)
|
return url_for_string(req, redir_to)
|
||||||
d.addCallback(_done, when_done)
|
d.addCallback(_done, when_done)
|
||||||
else:
|
else:
|
||||||
|
|
@ -118,8 +119,8 @@ class UploadResultsElement(status.UploadResultsRendererMixin):
|
||||||
def download_link(self, req, tag):
|
def download_link(self, req, tag):
|
||||||
d = self.upload_results()
|
d = self.upload_results()
|
||||||
d.addCallback(lambda res:
|
d.addCallback(lambda res:
|
||||||
tags.a("/uri/" + res.get_uri(),
|
tags.a("/uri/" + unicode(res.get_uri(), "utf-8"),
|
||||||
href="/uri/" + urllib.quote(res.get_uri())))
|
href="/uri/" + urlquote(unicode(res.get_uri(), "utf-8"))))
|
||||||
return d
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -158,7 +159,7 @@ def POSTUnlinkedCreateDirectory(req, client):
|
||||||
redirect = get_arg(req, "redirect_to_result", "false")
|
redirect = get_arg(req, "redirect_to_result", "false")
|
||||||
if boolean_of_arg(redirect):
|
if boolean_of_arg(redirect):
|
||||||
def _then_redir(res):
|
def _then_redir(res):
|
||||||
new_url = "uri/" + urllib.quote(res.get_uri())
|
new_url = "uri/" + urlquote(res.get_uri())
|
||||||
req.setResponseCode(http.SEE_OTHER) # 303
|
req.setResponseCode(http.SEE_OTHER) # 303
|
||||||
req.setHeader('location', new_url)
|
req.setHeader('location', new_url)
|
||||||
return ''
|
return ''
|
||||||
|
|
@ -176,7 +177,7 @@ def POSTUnlinkedCreateDirectoryWithChildren(req, client):
|
||||||
redirect = get_arg(req, "redirect_to_result", "false")
|
redirect = get_arg(req, "redirect_to_result", "false")
|
||||||
if boolean_of_arg(redirect):
|
if boolean_of_arg(redirect):
|
||||||
def _then_redir(res):
|
def _then_redir(res):
|
||||||
new_url = "uri/" + urllib.quote(res.get_uri())
|
new_url = "uri/" + urlquote(res.get_uri())
|
||||||
req.setResponseCode(http.SEE_OTHER) # 303
|
req.setResponseCode(http.SEE_OTHER) # 303
|
||||||
req.setHeader('location', new_url)
|
req.setHeader('location', new_url)
|
||||||
return ''
|
return ''
|
||||||
|
|
@ -194,7 +195,7 @@ def POSTUnlinkedCreateImmutableDirectory(req, client):
|
||||||
redirect = get_arg(req, "redirect_to_result", "false")
|
redirect = get_arg(req, "redirect_to_result", "false")
|
||||||
if boolean_of_arg(redirect):
|
if boolean_of_arg(redirect):
|
||||||
def _then_redir(res):
|
def _then_redir(res):
|
||||||
new_url = "uri/" + urllib.quote(res.get_uri())
|
new_url = "uri/" + urlquote(res.get_uri())
|
||||||
req.setResponseCode(http.SEE_OTHER) # 303
|
req.setResponseCode(http.SEE_OTHER) # 303
|
||||||
req.setHeader('location', new_url)
|
req.setHeader('location', new_url)
|
||||||
return ''
|
return ''
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,43 @@ from .web.storage_plugins import (
|
||||||
StoragePlugins,
|
StoragePlugins,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if PY2:
|
||||||
|
FileUploadFieldStorage = FieldStorage
|
||||||
|
else:
|
||||||
|
class FileUploadFieldStorage(FieldStorage):
|
||||||
|
"""
|
||||||
|
Do terrible things to ensure files are still bytes.
|
||||||
|
|
||||||
|
On Python 2, uploaded files were always bytes. On Python 3, there's a
|
||||||
|
heuristic: if the filename is set on a field, it's assumed to be a file
|
||||||
|
upload and therefore bytes. If no filename is set, it's Unicode.
|
||||||
|
|
||||||
|
Unfortunately, we always want it to be bytes, and Tahoe-LAFS also
|
||||||
|
enables setting the filename not via the MIME filename, but via a
|
||||||
|
separate field called "name".
|
||||||
|
|
||||||
|
Thus we need to do this ridiculous workaround. Mypy doesn't like it
|
||||||
|
either, thus the ``# type: ignore`` below.
|
||||||
|
|
||||||
|
Source for idea:
|
||||||
|
https://mail.python.org/pipermail/python-dev/2017-February/147402.html
|
||||||
|
"""
|
||||||
|
@property # type: ignore
|
||||||
|
def filename(self):
|
||||||
|
if self.name == "file" and not self._mime_filename:
|
||||||
|
# We use the file field to upload files, see directory.py's
|
||||||
|
# _POST_upload. Lack of _mime_filename means we need to trick
|
||||||
|
# FieldStorage into thinking there is a filename so it'll
|
||||||
|
# return bytes.
|
||||||
|
return "unknown-filename"
|
||||||
|
return self._mime_filename
|
||||||
|
|
||||||
|
@filename.setter
|
||||||
|
def filename(self, value):
|
||||||
|
self._mime_filename = value
|
||||||
|
|
||||||
|
|
||||||
class TahoeLAFSRequest(Request, object):
|
class TahoeLAFSRequest(Request, object):
|
||||||
"""
|
"""
|
||||||
``TahoeLAFSRequest`` adds several features to a Twisted Web ``Request``
|
``TahoeLAFSRequest`` adds several features to a Twisted Web ``Request``
|
||||||
|
|
@ -94,7 +131,8 @@ class TahoeLAFSRequest(Request, object):
|
||||||
headers['content-length'] = str(self.content.tell())
|
headers['content-length'] = str(self.content.tell())
|
||||||
self.content.seek(0)
|
self.content.seek(0)
|
||||||
|
|
||||||
self.fields = FieldStorage(self.content, headers, environ={'REQUEST_METHOD': 'POST'})
|
self.fields = FileUploadFieldStorage(
|
||||||
|
self.content, headers, environ={'REQUEST_METHOD': 'POST'})
|
||||||
self.content.seek(0)
|
self.content.seek(0)
|
||||||
|
|
||||||
self._tahoeLAFSSecurityPolicy()
|
self._tahoeLAFSSecurityPolicy()
|
||||||
|
|
@ -211,7 +249,7 @@ class WebishServer(service.MultiService):
|
||||||
# use to test ophandle expiration.
|
# use to test ophandle expiration.
|
||||||
self._operations = OphandleTable(clock)
|
self._operations = OphandleTable(clock)
|
||||||
self._operations.setServiceParent(self)
|
self._operations.setServiceParent(self)
|
||||||
self.root.putChild("operations", self._operations)
|
self.root.putChild(b"operations", self._operations)
|
||||||
|
|
||||||
self.root.putChild(b"storage-plugins", StoragePlugins(client))
|
self.root.putChild(b"storage-plugins", StoragePlugins(client))
|
||||||
|
|
||||||
|
|
@ -220,7 +258,7 @@ class WebishServer(service.MultiService):
|
||||||
self.site = TahoeLAFSSite(tempdir, self.root)
|
self.site = TahoeLAFSSite(tempdir, self.root)
|
||||||
self.staticdir = staticdir # so tests can check
|
self.staticdir = staticdir # so tests can check
|
||||||
if staticdir:
|
if staticdir:
|
||||||
self.root.putChild("static", static.File(staticdir))
|
self.root.putChild(b"static", static.File(staticdir))
|
||||||
if re.search(r'^\d', webport):
|
if re.search(r'^\d', webport):
|
||||||
webport = "tcp:"+webport # twisted warns about bare "0" or "3456"
|
webport = "tcp:"+webport # twisted warns about bare "0" or "3456"
|
||||||
# strports must be native strings.
|
# strports must be native strings.
|
||||||
|
|
|
||||||
|
|
@ -1,29 +1,123 @@
|
||||||
from __future__ import print_function
|
from __future__ import print_function
|
||||||
|
|
||||||
done = False
|
# This code isn't loadable or sensible except on Windows. Importers all know
|
||||||
|
# this and are careful. Normally I would just let an import error from ctypes
|
||||||
|
# explain any mistakes but Mypy also needs some help here. This assert
|
||||||
|
# explains to it that this module is Windows-only. This prevents errors about
|
||||||
|
# ctypes.windll and such which only exist when running on Windows.
|
||||||
|
#
|
||||||
|
# Beware of the limitations of the Mypy AST analyzer. The check needs to take
|
||||||
|
# exactly this form or it may not be recognized.
|
||||||
|
#
|
||||||
|
# https://mypy.readthedocs.io/en/stable/common_issues.html?highlight=platform#python-version-and-system-platform-checks
|
||||||
|
import sys
|
||||||
|
assert sys.platform == "win32"
|
||||||
|
|
||||||
|
import codecs
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
|
from ctypes import WINFUNCTYPE, windll, POINTER, c_int, WinError, byref, get_last_error
|
||||||
|
from ctypes.wintypes import BOOL, HANDLE, DWORD, LPWSTR, LPCWSTR, LPVOID
|
||||||
|
|
||||||
|
# <https://msdn.microsoft.com/en-us/library/ms680621%28VS.85%29.aspx>
|
||||||
|
from win32api import (
|
||||||
|
STD_OUTPUT_HANDLE,
|
||||||
|
STD_ERROR_HANDLE,
|
||||||
|
SetErrorMode,
|
||||||
|
|
||||||
|
# <https://msdn.microsoft.com/en-us/library/ms683231(VS.85).aspx>
|
||||||
|
# HANDLE WINAPI GetStdHandle(DWORD nStdHandle);
|
||||||
|
# returns INVALID_HANDLE_VALUE, NULL, or a valid handle
|
||||||
|
GetStdHandle,
|
||||||
|
)
|
||||||
|
from win32con import (
|
||||||
|
SEM_FAILCRITICALERRORS,
|
||||||
|
SEM_NOOPENFILEERRORBOX,
|
||||||
|
)
|
||||||
|
|
||||||
|
from win32file import (
|
||||||
|
INVALID_HANDLE_VALUE,
|
||||||
|
FILE_TYPE_CHAR,
|
||||||
|
|
||||||
|
# <https://msdn.microsoft.com/en-us/library/aa364960(VS.85).aspx>
|
||||||
|
# DWORD WINAPI GetFileType(DWORD hFile);
|
||||||
|
GetFileType,
|
||||||
|
)
|
||||||
|
|
||||||
|
from allmydata.util import (
|
||||||
|
log,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Keep track of whether `initialize` has run so we don't do any of the
|
||||||
|
# initialization more than once.
|
||||||
|
_done = False
|
||||||
|
|
||||||
|
#
|
||||||
|
# pywin32 for Python 2.7 does not bind any of these *W variants so we do it
|
||||||
|
# ourselves.
|
||||||
|
#
|
||||||
|
|
||||||
|
# <https://msdn.microsoft.com/en-us/library/windows/desktop/ms687401%28v=vs.85%29.aspx>
|
||||||
|
# BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars,
|
||||||
|
# LPDWORD lpCharsWritten, LPVOID lpReserved);
|
||||||
|
WriteConsoleW = WINFUNCTYPE(
|
||||||
|
BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID,
|
||||||
|
use_last_error=True
|
||||||
|
)(("WriteConsoleW", windll.kernel32))
|
||||||
|
|
||||||
|
# <https://msdn.microsoft.com/en-us/library/windows/desktop/ms683156%28v=vs.85%29.aspx>
|
||||||
|
GetCommandLineW = WINFUNCTYPE(
|
||||||
|
LPWSTR,
|
||||||
|
use_last_error=True
|
||||||
|
)(("GetCommandLineW", windll.kernel32))
|
||||||
|
|
||||||
|
# <https://msdn.microsoft.com/en-us/library/windows/desktop/bb776391%28v=vs.85%29.aspx>
|
||||||
|
CommandLineToArgvW = WINFUNCTYPE(
|
||||||
|
POINTER(LPWSTR), LPCWSTR, POINTER(c_int),
|
||||||
|
use_last_error=True
|
||||||
|
)(("CommandLineToArgvW", windll.shell32))
|
||||||
|
|
||||||
|
# <https://msdn.microsoft.com/en-us/library/ms683167(VS.85).aspx>
|
||||||
|
# BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode);
|
||||||
|
GetConsoleMode = WINFUNCTYPE(
|
||||||
|
BOOL, HANDLE, POINTER(DWORD),
|
||||||
|
use_last_error=True
|
||||||
|
)(("GetConsoleMode", windll.kernel32))
|
||||||
|
|
||||||
|
|
||||||
|
STDOUT_FILENO = 1
|
||||||
|
STDERR_FILENO = 2
|
||||||
|
|
||||||
|
def get_argv():
|
||||||
|
"""
|
||||||
|
:return [unicode]: The argument list this process was invoked with, as
|
||||||
|
unicode.
|
||||||
|
|
||||||
|
Python 2 does not do a good job exposing this information in
|
||||||
|
``sys.argv`` on Windows so this code re-retrieves the underlying
|
||||||
|
information using Windows API calls and massages it into the right
|
||||||
|
shape.
|
||||||
|
"""
|
||||||
|
command_line = GetCommandLineW()
|
||||||
|
argc = c_int(0)
|
||||||
|
argv_unicode = CommandLineToArgvW(command_line, byref(argc))
|
||||||
|
if argv_unicode is None:
|
||||||
|
raise WinError(get_last_error())
|
||||||
|
|
||||||
|
# Convert it to a normal Python list
|
||||||
|
return list(
|
||||||
|
argv_unicode[i]
|
||||||
|
for i
|
||||||
|
in range(argc.value)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def initialize():
|
def initialize():
|
||||||
global done
|
global _done
|
||||||
import sys
|
import sys
|
||||||
if sys.platform != "win32" or done:
|
if sys.platform != "win32" or _done:
|
||||||
return True
|
return True
|
||||||
done = True
|
_done = True
|
||||||
|
|
||||||
import codecs, re
|
|
||||||
from ctypes import WINFUNCTYPE, WinError, windll, POINTER, byref, c_int, get_last_error
|
|
||||||
from ctypes.wintypes import BOOL, HANDLE, DWORD, UINT, LPWSTR, LPCWSTR, LPVOID
|
|
||||||
|
|
||||||
from allmydata.util import log
|
|
||||||
from allmydata.util.encodingutil import canonical_encoding
|
|
||||||
|
|
||||||
# <https://msdn.microsoft.com/en-us/library/ms680621%28VS.85%29.aspx>
|
|
||||||
SetErrorMode = WINFUNCTYPE(
|
|
||||||
UINT, UINT,
|
|
||||||
use_last_error=True
|
|
||||||
)(("SetErrorMode", windll.kernel32))
|
|
||||||
|
|
||||||
SEM_FAILCRITICALERRORS = 0x0001
|
|
||||||
SEM_NOOPENFILEERRORBOX = 0x8000
|
|
||||||
|
|
||||||
SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX)
|
SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX)
|
||||||
|
|
||||||
|
|
@ -33,10 +127,12 @@ def initialize():
|
||||||
# which makes for frustrating debugging if stderr is directed to our wrapper.
|
# which makes for frustrating debugging if stderr is directed to our wrapper.
|
||||||
# So be paranoid about catching errors and reporting them to original_stderr,
|
# So be paranoid about catching errors and reporting them to original_stderr,
|
||||||
# so that we can at least see them.
|
# so that we can at least see them.
|
||||||
def _complain(message):
|
def _complain(output_file, message):
|
||||||
print(isinstance(message, str) and message or repr(message), file=original_stderr)
|
print(isinstance(message, str) and message or repr(message), file=output_file)
|
||||||
log.msg(message, level=log.WEIRD)
|
log.msg(message, level=log.WEIRD)
|
||||||
|
|
||||||
|
_complain = partial(_complain, original_stderr)
|
||||||
|
|
||||||
# Work around <http://bugs.python.org/issue6058>.
|
# Work around <http://bugs.python.org/issue6058>.
|
||||||
codecs.register(lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None)
|
codecs.register(lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None)
|
||||||
|
|
||||||
|
|
@ -46,45 +142,6 @@ def initialize():
|
||||||
# and TZOmegaTZIOY
|
# and TZOmegaTZIOY
|
||||||
# <http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-causes-python-crash/1432462#1432462>.
|
# <http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-causes-python-crash/1432462#1432462>.
|
||||||
try:
|
try:
|
||||||
# <https://msdn.microsoft.com/en-us/library/ms683231(VS.85).aspx>
|
|
||||||
# HANDLE WINAPI GetStdHandle(DWORD nStdHandle);
|
|
||||||
# returns INVALID_HANDLE_VALUE, NULL, or a valid handle
|
|
||||||
#
|
|
||||||
# <https://msdn.microsoft.com/en-us/library/aa364960(VS.85).aspx>
|
|
||||||
# DWORD WINAPI GetFileType(DWORD hFile);
|
|
||||||
#
|
|
||||||
# <https://msdn.microsoft.com/en-us/library/ms683167(VS.85).aspx>
|
|
||||||
# BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode);
|
|
||||||
|
|
||||||
GetStdHandle = WINFUNCTYPE(
|
|
||||||
HANDLE, DWORD,
|
|
||||||
use_last_error=True
|
|
||||||
)(("GetStdHandle", windll.kernel32))
|
|
||||||
|
|
||||||
STD_OUTPUT_HANDLE = DWORD(-11)
|
|
||||||
STD_ERROR_HANDLE = DWORD(-12)
|
|
||||||
|
|
||||||
GetFileType = WINFUNCTYPE(
|
|
||||||
DWORD, DWORD,
|
|
||||||
use_last_error=True
|
|
||||||
)(("GetFileType", windll.kernel32))
|
|
||||||
|
|
||||||
FILE_TYPE_CHAR = 0x0002
|
|
||||||
FILE_TYPE_REMOTE = 0x8000
|
|
||||||
|
|
||||||
GetConsoleMode = WINFUNCTYPE(
|
|
||||||
BOOL, HANDLE, POINTER(DWORD),
|
|
||||||
use_last_error=True
|
|
||||||
)(("GetConsoleMode", windll.kernel32))
|
|
||||||
|
|
||||||
INVALID_HANDLE_VALUE = DWORD(-1).value
|
|
||||||
|
|
||||||
def not_a_console(handle):
|
|
||||||
if handle == INVALID_HANDLE_VALUE or handle is None:
|
|
||||||
return True
|
|
||||||
return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
|
|
||||||
or GetConsoleMode(handle, byref(DWORD())) == 0)
|
|
||||||
|
|
||||||
old_stdout_fileno = None
|
old_stdout_fileno = None
|
||||||
old_stderr_fileno = None
|
old_stderr_fileno = None
|
||||||
if hasattr(sys.stdout, 'fileno'):
|
if hasattr(sys.stdout, 'fileno'):
|
||||||
|
|
@ -92,151 +149,145 @@ def initialize():
|
||||||
if hasattr(sys.stderr, 'fileno'):
|
if hasattr(sys.stderr, 'fileno'):
|
||||||
old_stderr_fileno = sys.stderr.fileno()
|
old_stderr_fileno = sys.stderr.fileno()
|
||||||
|
|
||||||
STDOUT_FILENO = 1
|
|
||||||
STDERR_FILENO = 2
|
|
||||||
real_stdout = (old_stdout_fileno == STDOUT_FILENO)
|
real_stdout = (old_stdout_fileno == STDOUT_FILENO)
|
||||||
real_stderr = (old_stderr_fileno == STDERR_FILENO)
|
real_stderr = (old_stderr_fileno == STDERR_FILENO)
|
||||||
|
|
||||||
if real_stdout:
|
if real_stdout:
|
||||||
hStdout = GetStdHandle(STD_OUTPUT_HANDLE)
|
hStdout = GetStdHandle(STD_OUTPUT_HANDLE)
|
||||||
if not_a_console(hStdout):
|
if not a_console(hStdout):
|
||||||
real_stdout = False
|
real_stdout = False
|
||||||
|
|
||||||
if real_stderr:
|
if real_stderr:
|
||||||
hStderr = GetStdHandle(STD_ERROR_HANDLE)
|
hStderr = GetStdHandle(STD_ERROR_HANDLE)
|
||||||
if not_a_console(hStderr):
|
if not a_console(hStderr):
|
||||||
real_stderr = False
|
real_stderr = False
|
||||||
|
|
||||||
if real_stdout or real_stderr:
|
if real_stdout:
|
||||||
# <https://msdn.microsoft.com/en-us/library/windows/desktop/ms687401%28v=vs.85%29.aspx>
|
sys.stdout = UnicodeOutput(hStdout, None, STDOUT_FILENO, '<Unicode console stdout>', _complain)
|
||||||
# BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars,
|
else:
|
||||||
# LPDWORD lpCharsWritten, LPVOID lpReserved);
|
sys.stdout = UnicodeOutput(None, sys.stdout, old_stdout_fileno, '<Unicode redirected stdout>', _complain)
|
||||||
|
|
||||||
WriteConsoleW = WINFUNCTYPE(
|
if real_stderr:
|
||||||
BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID,
|
sys.stderr = UnicodeOutput(hStderr, None, STDERR_FILENO, '<Unicode console stderr>', _complain)
|
||||||
use_last_error=True
|
else:
|
||||||
)(("WriteConsoleW", windll.kernel32))
|
sys.stderr = UnicodeOutput(None, sys.stderr, old_stderr_fileno, '<Unicode redirected stderr>', _complain)
|
||||||
|
|
||||||
class UnicodeOutput(object):
|
|
||||||
def __init__(self, hConsole, stream, fileno, name):
|
|
||||||
self._hConsole = hConsole
|
|
||||||
self._stream = stream
|
|
||||||
self._fileno = fileno
|
|
||||||
self.closed = False
|
|
||||||
self.softspace = False
|
|
||||||
self.mode = 'w'
|
|
||||||
self.encoding = 'utf-8'
|
|
||||||
self.name = name
|
|
||||||
if hasattr(stream, 'encoding') and canonical_encoding(stream.encoding) != 'utf-8':
|
|
||||||
log.msg("%s: %r had encoding %r, but we're going to write UTF-8 to it" %
|
|
||||||
(name, stream, stream.encoding), level=log.CURIOUS)
|
|
||||||
self.flush()
|
|
||||||
|
|
||||||
def isatty(self):
|
|
||||||
return False
|
|
||||||
def close(self):
|
|
||||||
# don't really close the handle, that would only cause problems
|
|
||||||
self.closed = True
|
|
||||||
def fileno(self):
|
|
||||||
return self._fileno
|
|
||||||
def flush(self):
|
|
||||||
if self._hConsole is None:
|
|
||||||
try:
|
|
||||||
self._stream.flush()
|
|
||||||
except Exception as e:
|
|
||||||
_complain("%s.flush: %r from %r" % (self.name, e, self._stream))
|
|
||||||
raise
|
|
||||||
|
|
||||||
def write(self, text):
|
|
||||||
try:
|
|
||||||
if self._hConsole is None:
|
|
||||||
if isinstance(text, unicode):
|
|
||||||
text = text.encode('utf-8')
|
|
||||||
self._stream.write(text)
|
|
||||||
else:
|
|
||||||
if not isinstance(text, unicode):
|
|
||||||
text = str(text).decode('utf-8')
|
|
||||||
remaining = len(text)
|
|
||||||
while remaining > 0:
|
|
||||||
n = DWORD(0)
|
|
||||||
# There is a shorter-than-documented limitation on the length of the string
|
|
||||||
# passed to WriteConsoleW (see #1232).
|
|
||||||
retval = WriteConsoleW(self._hConsole, text, min(remaining, 10000), byref(n), None)
|
|
||||||
if retval == 0:
|
|
||||||
raise IOError("WriteConsoleW failed with WinError: %s" % (WinError(get_last_error()),))
|
|
||||||
if n.value == 0:
|
|
||||||
raise IOError("WriteConsoleW returned %r, n.value = 0" % (retval,))
|
|
||||||
remaining -= n.value
|
|
||||||
if remaining == 0: break
|
|
||||||
text = text[n.value:]
|
|
||||||
except Exception as e:
|
|
||||||
_complain("%s.write: %r" % (self.name, e))
|
|
||||||
raise
|
|
||||||
|
|
||||||
def writelines(self, lines):
|
|
||||||
try:
|
|
||||||
for line in lines:
|
|
||||||
self.write(line)
|
|
||||||
except Exception as e:
|
|
||||||
_complain("%s.writelines: %r" % (self.name, e))
|
|
||||||
raise
|
|
||||||
|
|
||||||
if real_stdout:
|
|
||||||
sys.stdout = UnicodeOutput(hStdout, None, STDOUT_FILENO, '<Unicode console stdout>')
|
|
||||||
else:
|
|
||||||
sys.stdout = UnicodeOutput(None, sys.stdout, old_stdout_fileno, '<Unicode redirected stdout>')
|
|
||||||
|
|
||||||
if real_stderr:
|
|
||||||
sys.stderr = UnicodeOutput(hStderr, None, STDERR_FILENO, '<Unicode console stderr>')
|
|
||||||
else:
|
|
||||||
sys.stderr = UnicodeOutput(None, sys.stderr, old_stderr_fileno, '<Unicode redirected stderr>')
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
_complain("exception %r while fixing up sys.stdout and sys.stderr" % (e,))
|
_complain("exception %r while fixing up sys.stdout and sys.stderr" % (e,))
|
||||||
|
|
||||||
# This works around <http://bugs.python.org/issue2128>.
|
argv = list(arg.encode("utf-8") for arg in get_argv())
|
||||||
|
|
||||||
# <https://msdn.microsoft.com/en-us/library/windows/desktop/ms683156%28v=vs.85%29.aspx>
|
|
||||||
GetCommandLineW = WINFUNCTYPE(
|
|
||||||
LPWSTR,
|
|
||||||
use_last_error=True
|
|
||||||
)(("GetCommandLineW", windll.kernel32))
|
|
||||||
|
|
||||||
# <https://msdn.microsoft.com/en-us/library/windows/desktop/bb776391%28v=vs.85%29.aspx>
|
|
||||||
CommandLineToArgvW = WINFUNCTYPE(
|
|
||||||
POINTER(LPWSTR), LPCWSTR, POINTER(c_int),
|
|
||||||
use_last_error=True
|
|
||||||
)(("CommandLineToArgvW", windll.shell32))
|
|
||||||
|
|
||||||
argc = c_int(0)
|
|
||||||
argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))
|
|
||||||
if argv_unicode is None:
|
|
||||||
raise WinError(get_last_error())
|
|
||||||
|
|
||||||
# Because of <http://bugs.python.org/issue8775> (and similar limitations in
|
|
||||||
# twisted), the 'bin/tahoe' script cannot invoke us with the actual Unicode arguments.
|
|
||||||
# Instead it "mangles" or escapes them using \x7F as an escape character, which we
|
|
||||||
# unescape here.
|
|
||||||
def unmangle(s):
|
|
||||||
return re.sub(
|
|
||||||
u'\\x7F[0-9a-fA-F]*\\;',
|
|
||||||
# type ignored for 'unichr' (Python 2 only)
|
|
||||||
lambda m: unichr(int(m.group(0)[1:-1], 16)), # type: ignore
|
|
||||||
s,
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
argv = [unmangle(argv_unicode[i]).encode('utf-8') for i in xrange(0, argc.value)]
|
|
||||||
except Exception as e:
|
|
||||||
_complain("%s: could not unmangle Unicode arguments.\n%r"
|
|
||||||
% (sys.argv[0], [argv_unicode[i] for i in xrange(0, argc.value)]))
|
|
||||||
raise
|
|
||||||
|
|
||||||
# Take only the suffix with the same number of arguments as sys.argv.
|
# Take only the suffix with the same number of arguments as sys.argv.
|
||||||
# This accounts for anything that can cause initial arguments to be stripped,
|
# This accounts for anything that can cause initial arguments to be stripped,
|
||||||
# for example, the Python interpreter or any options passed to it, or runner
|
# for example, the Python interpreter or any options passed to it, or runner
|
||||||
# scripts such as 'coverage run'. It works even if there are no such arguments,
|
# scripts such as 'coverage run'. It works even if there are no such arguments,
|
||||||
# as in the case of a frozen executable created by bb-freeze or similar.
|
# as in the case of a frozen executable created by bb-freeze or similar.
|
||||||
|
|
||||||
sys.argv = argv[-len(sys.argv):]
|
sys.argv = argv[-len(sys.argv):]
|
||||||
if sys.argv[0].endswith('.pyscript'):
|
|
||||||
sys.argv[0] = sys.argv[0][:-9]
|
|
||||||
|
def a_console(handle):
|
||||||
|
"""
|
||||||
|
:return: ``True`` if ``handle`` refers to a console, ``False`` otherwise.
|
||||||
|
"""
|
||||||
|
if handle == INVALID_HANDLE_VALUE:
|
||||||
|
return False
|
||||||
|
return (
|
||||||
|
# It's a character file (eg a printer or a console)
|
||||||
|
GetFileType(handle) == FILE_TYPE_CHAR and
|
||||||
|
# Checking the console mode doesn't fail (thus it's a console)
|
||||||
|
GetConsoleMode(handle, byref(DWORD())) != 0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class UnicodeOutput(object):
|
||||||
|
"""
|
||||||
|
``UnicodeOutput`` is a file-like object that encodes unicode to UTF-8 and
|
||||||
|
writes it to another file or writes unicode natively to the Windows
|
||||||
|
console.
|
||||||
|
"""
|
||||||
|
def __init__(self, hConsole, stream, fileno, name, _complain):
|
||||||
|
"""
|
||||||
|
:param hConsole: ``None`` or a handle on the console to which to write
|
||||||
|
unicode. Mutually exclusive with ``stream``.
|
||||||
|
|
||||||
|
:param stream: ``None`` or a file-like object to which to write bytes.
|
||||||
|
|
||||||
|
:param fileno: A result to hand back from method of the same name.
|
||||||
|
|
||||||
|
:param name: A human-friendly identifier for this output object.
|
||||||
|
|
||||||
|
:param _complain: A one-argument callable which accepts bytes to be
|
||||||
|
written when there's a problem. Care should be taken to not make
|
||||||
|
this do a write on this object.
|
||||||
|
"""
|
||||||
|
self._hConsole = hConsole
|
||||||
|
self._stream = stream
|
||||||
|
self._fileno = fileno
|
||||||
|
self.closed = False
|
||||||
|
self.softspace = False
|
||||||
|
self.mode = 'w'
|
||||||
|
self.encoding = 'utf-8'
|
||||||
|
self.name = name
|
||||||
|
|
||||||
|
self._complain = _complain
|
||||||
|
|
||||||
|
from allmydata.util.encodingutil import canonical_encoding
|
||||||
|
from allmydata.util import log
|
||||||
|
if hasattr(stream, 'encoding') and canonical_encoding(stream.encoding) != 'utf-8':
|
||||||
|
log.msg("%s: %r had encoding %r, but we're going to write UTF-8 to it" %
|
||||||
|
(name, stream, stream.encoding), level=log.CURIOUS)
|
||||||
|
self.flush()
|
||||||
|
|
||||||
|
def isatty(self):
|
||||||
|
return False
|
||||||
|
def close(self):
|
||||||
|
# don't really close the handle, that would only cause problems
|
||||||
|
self.closed = True
|
||||||
|
def fileno(self):
|
||||||
|
return self._fileno
|
||||||
|
def flush(self):
|
||||||
|
if self._hConsole is None:
|
||||||
|
try:
|
||||||
|
self._stream.flush()
|
||||||
|
except Exception as e:
|
||||||
|
self._complain("%s.flush: %r from %r" % (self.name, e, self._stream))
|
||||||
|
raise
|
||||||
|
|
||||||
|
def write(self, text):
|
||||||
|
try:
|
||||||
|
if self._hConsole is None:
|
||||||
|
# There is no Windows console available. That means we are
|
||||||
|
# responsible for encoding the unicode to a byte string to
|
||||||
|
# write it to a Python file object.
|
||||||
|
if isinstance(text, unicode):
|
||||||
|
text = text.encode('utf-8')
|
||||||
|
self._stream.write(text)
|
||||||
|
else:
|
||||||
|
# There is a Windows console available. That means Windows is
|
||||||
|
# responsible for dealing with the unicode itself.
|
||||||
|
if not isinstance(text, unicode):
|
||||||
|
text = str(text).decode('utf-8')
|
||||||
|
remaining = len(text)
|
||||||
|
while remaining > 0:
|
||||||
|
n = DWORD(0)
|
||||||
|
# There is a shorter-than-documented limitation on the
|
||||||
|
# length of the string passed to WriteConsoleW (see
|
||||||
|
# #1232).
|
||||||
|
retval = WriteConsoleW(self._hConsole, text, min(remaining, 10000), byref(n), None)
|
||||||
|
if retval == 0:
|
||||||
|
raise IOError("WriteConsoleW failed with WinError: %s" % (WinError(get_last_error()),))
|
||||||
|
if n.value == 0:
|
||||||
|
raise IOError("WriteConsoleW returned %r, n.value = 0" % (retval,))
|
||||||
|
remaining -= n.value
|
||||||
|
if remaining == 0: break
|
||||||
|
text = text[n.value:]
|
||||||
|
except Exception as e:
|
||||||
|
self._complain("%s.write: %r" % (self.name, e))
|
||||||
|
raise
|
||||||
|
|
||||||
|
def writelines(self, lines):
|
||||||
|
try:
|
||||||
|
for line in lines:
|
||||||
|
self.write(line)
|
||||||
|
except Exception as e:
|
||||||
|
self._complain("%s.writelines: %r" % (self.name, e))
|
||||||
|
raise
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue