tahoe-lafs/tahoe-lafs
tahoe-lafs
/
tahoe-lafs
2
0
Fork 0
Code Issues Pull Requests 23 Packages Projects Releases 1 Wiki Activity

Merge remote-tracking branch 'origin/master' into 3376.encodingutil-python-3

This commit is contained in:
Itamar Turner-Trauring 2020-08-14 13:09:08 -04:00
parent 97dd2fab72 bc787975da
commit 2fc8f896b3
20 changed files with 1025 additions and 1133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.circleci/config.yml
View File

@ -281,6 +281,10 @@ jobs:
environment: environment:
<<: *UTF_8_ENVIRONMENT <<: *UTF_8_ENVIRONMENT
# The default trial args include --rterrors which is incompatible with
# this reporter on Python 3. So drop that and just specify the
# reporter.
TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
TAHOE_LAFS_TOX_ENVIRONMENT: "py36" TAHOE_LAFS_TOX_ENVIRONMENT: "py36"

2
.circleci/run-tests.sh
View File

@ -65,7 +65,7 @@ TIMEOUT="timeout --kill-after 1m 15m"
# Send the output directly to a file because transporting the binary subunit2 # Send the output directly to a file because transporting the binary subunit2
# via tox and then scraping it out is hideous and failure prone. # via tox and then scraping it out is hideous and failure prone.
export SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}" export SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}"
export TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file --rterrors" export TAHOE_LAFS_TRIAL_ARGS="${TAHOE_LAFS_TRIAL_ARGS:---reporter=subunitv2-file --rterrors}"
export PIP_NO_INDEX="1" export PIP_NO_INDEX="1"
if [ "${ALLOWED_FAILURE}" = "yes" ]; then if [ "${ALLOWED_FAILURE}" = "yes" ]; then

236
misc/python3/ratchet-passing
View File

@ -1,236 +0,0 @@
allmydata.test.mutable.test_exceptions.Exceptions.test_repr
allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_1s
allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_25s
allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_day
allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_future_5_minutes
allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_hours
allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_month
allmydata.test.test_abbreviate.Abbreviate.test_abbrev_time_year
allmydata.test.test_abbreviate.Abbreviate.test_parse_space
allmydata.test.test_abbreviate.Abbreviate.test_space
allmydata.test.test_abbreviate.Abbreviate.test_time
allmydata.test.test_backupdb.BackupDB.test_basic
allmydata.test.test_backupdb.BackupDB.test_upgrade_v1_v2
allmydata.test.test_backupdb.BackupDB.test_wrong_version
allmydata.test.test_base32.Base32.test_a2b
allmydata.test.test_base32.Base32.test_a2b_b2a_match_Pythons
allmydata.test.test_base32.Base32.test_b2a
allmydata.test.test_base32.Base32.test_b2a_or_none
allmydata.test.test_base62.Base62.test_ende_0x00
allmydata.test.test_base62.Base62.test_ende_0x000000
allmydata.test.test_base62.Base62.test_ende_0x01
allmydata.test.test_base62.Base62.test_ende_0x0100
allmydata.test.test_base62.Base62.test_ende_0x010000
allmydata.test.test_base62.Base62.test_ende_longrandstr
allmydata.test.test_base62.Base62.test_ende_randstr
allmydata.test.test_base62.Base62.test_known_values
allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars
allmydata.test.test_base62.Base62.test_odd_sizes
allmydata.test.test_base62.Base62.test_roundtrip
allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes
allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes
allmydata.test.test_crypto.TestEd25519.test_key_serialization
allmydata.test.test_crypto.TestEd25519.test_sign_invalid_pubkey
allmydata.test.test_crypto.TestEd25519.test_signature_data_not_bytes
allmydata.test.test_crypto.TestEd25519.test_signature_not_bytes
allmydata.test.test_crypto.TestEd25519.test_signed_data_not_bytes
allmydata.test.test_crypto.TestEd25519.test_verify_invalid_pubkey
allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_long_input
allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_short_input
allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_long_input
allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_short_input
allmydata.test.test_crypto.TestRegression.test_decode_ed15519_keypair
allmydata.test.test_crypto.TestRegression.test_decode_rsa_keypair
allmydata.test.test_crypto.TestRegression.test_encrypt_data_not_bytes
allmydata.test.test_crypto.TestRegression.test_incorrect_iv_size
allmydata.test.test_crypto.TestRegression.test_iv_not_bytes
allmydata.test.test_crypto.TestRegression.test_key_incorrect_size
allmydata.test.test_crypto.TestRegression.test_old_start_up_test
allmydata.test.test_crypto.TestRsa.test_keys
allmydata.test.test_crypto.TestRsa.test_sign_invalid_pubkey
allmydata.test.test_crypto.TestRsa.test_verify_invalid_pubkey
allmydata.test.test_crypto.TestUtil.test_remove_prefix_bad
allmydata.test.test_crypto.TestUtil.test_remove_prefix_entire_string
allmydata.test.test_crypto.TestUtil.test_remove_prefix_good
allmydata.test.test_crypto.TestUtil.test_remove_prefix_partial
allmydata.test.test_crypto.TestUtil.test_remove_prefix_zero
allmydata.test.test_deferredutil.DeferredUtilTests.test_failure
allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results
allmydata.test.test_deferredutil.DeferredUtilTests.test_success
allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls
allmydata.test.test_dictutil.DictUtil.test_auxdict
allmydata.test.test_dictutil.DictUtil.test_dict_of_sets
allmydata.test.test_encodingutil.EncodingUtilErrors.test_argv_to_unicode
allmydata.test.test_encodingutil.EncodingUtilErrors.test_get_io_encoding
allmydata.test.test_encodingutil.EncodingUtilErrors.test_get_io_encoding_not_from_stdout
allmydata.test.test_encodingutil.EncodingUtilErrors.test_no_unicode_normalization
allmydata.test.test_encodingutil.EncodingUtilErrors.test_unicode_to_output
allmydata.test.test_encodingutil.FilePaths.test_extend_filepath
allmydata.test.test_encodingutil.FilePaths.test_to_filepath
allmydata.test.test_encodingutil.FilePaths.test_unicode_from_filepath
allmydata.test.test_encodingutil.FilePaths.test_unicode_segments_from
allmydata.test.test_encodingutil.MacOSXLeopard.test_argv_to_unicode
allmydata.test.test_encodingutil.MacOSXLeopard.test_listdir_unicode
allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_platform_py3
allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_to_argv_py3
allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_to_output
allmydata.test.test_encodingutil.MacOSXLeopard.test_unicode_to_url
allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_argv_to_unicode
allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_listdir_unicode
allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_platform_py3
allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_to_argv_py3
allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_to_output
allmydata.test.test_encodingutil.MacOSXLeopard7bit.test_unicode_to_url
allmydata.test.test_encodingutil.OpenBSD.test_argv_to_unicode
allmydata.test.test_encodingutil.OpenBSD.test_listdir_unicode
allmydata.test.test_encodingutil.OpenBSD.test_unicode_platform_py3
allmydata.test.test_encodingutil.OpenBSD.test_unicode_to_argv_py3
allmydata.test.test_encodingutil.OpenBSD.test_unicode_to_output
allmydata.test.test_encodingutil.OpenBSD.test_unicode_to_url
allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_ascii
allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_default
allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_latin1
allmydata.test.test_encodingutil.QuoteOutput.test_quote_output_utf8
allmydata.test.test_encodingutil.QuotePaths.test_quote_filepath
allmydata.test.test_encodingutil.QuotePaths.test_quote_path
allmydata.test.test_encodingutil.StdlibUnicode.test_mkdir_open_exists_abspath_listdir_expanduser
allmydata.test.test_encodingutil.TestToFromStr.test_from_utf8_or_none
allmydata.test.test_encodingutil.TestToFromStr.test_to_str
allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_argv_to_unicode
allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_listdir_unicode
allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_platform_py3
allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_to_argv_py3
allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_to_output
allmydata.test.test_encodingutil.UbuntuKarmicLatin1.test_unicode_to_url
allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_argv_to_unicode
allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_listdir_unicode
allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_platform_py3
allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_to_argv_py3
allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_to_output
allmydata.test.test_encodingutil.UbuntuKarmicUTF8.test_unicode_to_url
allmydata.test.test_encodingutil.Windows.test_argv_to_unicode
allmydata.test.test_encodingutil.Windows.test_unicode_platform_py3
allmydata.test.test_encodingutil.Windows.test_unicode_to_argv_py3
allmydata.test.test_encodingutil.Windows.test_unicode_to_output
allmydata.test.test_encodingutil.Windows.test_unicode_to_url
allmydata.test.test_happiness.Happiness.test_100
allmydata.test.test_happiness.Happiness.test_calc_happy
allmydata.test.test_happiness.Happiness.test_everything_broken
allmydata.test.test_happiness.Happiness.test_hypothesis0
allmydata.test.test_happiness.Happiness.test_hypothesis_0
allmydata.test.test_happiness.Happiness.test_hypothesis_1
allmydata.test.test_happiness.Happiness.test_placement_1
allmydata.test.test_happiness.Happiness.test_placement_simple
allmydata.test.test_happiness.Happiness.test_redistribute
allmydata.test.test_happiness.Happiness.test_unhappy
allmydata.test.test_happiness.HappinessUtils.test_residual_0
allmydata.test.test_happiness.HappinessUtils.test_trivial_flow_graph
allmydata.test.test_happiness.HappinessUtils.test_trivial_maximum_graph
allmydata.test.test_happiness.PlacementTests.test_hypothesis_unhappy
allmydata.test.test_happiness.PlacementTests.test_more_hypothesis
allmydata.test.test_hashtree.Complete.test_create
allmydata.test.test_hashtree.Complete.test_dump
allmydata.test.test_hashtree.Complete.test_needed_hashes
allmydata.test.test_hashtree.Incomplete.test_check
allmydata.test.test_hashtree.Incomplete.test_create
allmydata.test.test_hashtree.Incomplete.test_depth_of
allmydata.test.test_hashtree.Incomplete.test_large
allmydata.test.test_hashtree.Incomplete.test_needed_hashes
allmydata.test.test_hashutil.HashUtilTests.test_chk
allmydata.test.test_hashutil.HashUtilTests.test_hashers
allmydata.test.test_hashutil.HashUtilTests.test_known_answers
allmydata.test.test_hashutil.HashUtilTests.test_random_key
allmydata.test.test_hashutil.HashUtilTests.test_sha256d
allmydata.test.test_hashutil.HashUtilTests.test_sha256d_truncated
allmydata.test.test_hashutil.HashUtilTests.test_timing_safe_compare
allmydata.test.test_humanreadable.HumanReadable.test_repr
allmydata.test.test_iputil.GcUtil.test_gc_after_allocations
allmydata.test.test_iputil.GcUtil.test_release_delays_gc
allmydata.test.test_iputil.ListAddresses.test_get_local_ip_for
allmydata.test.test_iputil.ListAddresses.test_list_async
allmydata.test.test_iputil.ListAddresses.test_list_async_mock_cygwin
allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ifconfig
allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr
allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route
allmydata.test.test_iputil.ListenOnUsed.test_random_port
allmydata.test.test_iputil.ListenOnUsed.test_specific_port
allmydata.test.test_log.Log.test_default_facility
allmydata.test.test_log.Log.test_err
allmydata.test.test_log.Log.test_grandparent_id
allmydata.test.test_log.Log.test_no_prefix
allmydata.test.test_log.Log.test_numming
allmydata.test.test_log.Log.test_parent_id
allmydata.test.test_log.Log.test_with_bytes_prefix
allmydata.test.test_log.Log.test_with_prefix
allmydata.test.test_netstring.Netstring.test_encode
allmydata.test.test_netstring.Netstring.test_extra
allmydata.test.test_netstring.Netstring.test_nested
allmydata.test.test_netstring.Netstring.test_split
allmydata.test.test_observer.Observer.test_lazy_oneshot
allmydata.test.test_observer.Observer.test_observerlist
allmydata.test.test_observer.Observer.test_oneshot
allmydata.test.test_observer.Observer.test_oneshot_fireagain
allmydata.test.test_pipeline.Pipeline.test_basic
allmydata.test.test_pipeline.Pipeline.test_errors
allmydata.test.test_pipeline.Pipeline.test_errors2
allmydata.test.test_python3.Python3PortingEffortTests.test_finished_porting
allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_distinct
allmydata.test.test_python3.Python3PortingEffortTests.test_ported_modules_exist
allmydata.test.test_spans.ByteSpans.test_basic
allmydata.test.test_spans.ByteSpans.test_large
allmydata.test.test_spans.ByteSpans.test_math
allmydata.test.test_spans.ByteSpans.test_overlap
allmydata.test.test_spans.ByteSpans.test_random
allmydata.test.test_spans.StringSpans.test_basic
allmydata.test.test_spans.StringSpans.test_random
allmydata.test.test_spans.StringSpans.test_test
allmydata.test.test_statistics.Statistics.test_binomial_coeff
allmydata.test.test_statistics.Statistics.test_binomial_distribution_pmf
allmydata.test.test_statistics.Statistics.test_convolve
allmydata.test.test_statistics.Statistics.test_find_k
allmydata.test.test_statistics.Statistics.test_pr_backup_file_loss
allmydata.test.test_statistics.Statistics.test_pr_file_loss
allmydata.test.test_statistics.Statistics.test_repair_cost
allmydata.test.test_statistics.Statistics.test_repair_count_pmf
allmydata.test.test_statistics.Statistics.test_survival_pmf
allmydata.test.test_time_format.TimeFormat.test_epoch
allmydata.test.test_time_format.TimeFormat.test_epoch_in_London
allmydata.test.test_time_format.TimeFormat.test_format_delta
allmydata.test.test_time_format.TimeFormat.test_format_time
allmydata.test.test_time_format.TimeFormat.test_format_time_y2038
allmydata.test.test_time_format.TimeFormat.test_iso_utc
allmydata.test.test_time_format.TimeFormat.test_parse_date
allmydata.test.test_time_format.TimeFormat.test_parse_duration
allmydata.test.test_util.FileUtil.test_abspath_expanduser_unicode
allmydata.test.test_util.FileUtil.test_create_long_path
allmydata.test.test_util.FileUtil.test_disk_stats
allmydata.test.test_util.FileUtil.test_disk_stats_avail_nonnegative
allmydata.test.test_util.FileUtil.test_du
allmydata.test.test_util.FileUtil.test_encrypted_tempfile
allmydata.test.test_util.FileUtil.test_get_pathinfo
allmydata.test.test_util.FileUtil.test_get_pathinfo_symlink
allmydata.test.test_util.FileUtil.test_make_dirs_with_absolute_mode
allmydata.test.test_util.FileUtil.test_remove_if_possible
allmydata.test.test_util.FileUtil.test_rename
allmydata.test.test_util.FileUtil.test_rename_no_overwrite
allmydata.test.test_util.FileUtil.test_replace_file
allmydata.test.test_util.FileUtil.test_rm_dir
allmydata.test.test_util.FileUtil.test_windows_expanduser_win7
allmydata.test.test_util.FileUtil.test_windows_expanduser_xp
allmydata.test.test_util.FileUtil.test_write_atomically
allmydata.test.test_util.IDLib.test_nodeid_b2a
allmydata.test.test_util.Math.test_round_sigfigs
allmydata.test.test_util.PollMixinTests.test_PollMixin_False_then_True
allmydata.test.test_util.PollMixinTests.test_PollMixin_True
allmydata.test.test_util.PollMixinTests.test_timeout
allmydata.test.test_util.YAML.test_convert
allmydata.test.test_version.CheckRequirement.test_cross_check
allmydata.test.test_version.CheckRequirement.test_cross_check_unparseable_versions
allmydata.test.test_version.CheckRequirement.test_extract_openssl_version
allmydata.test.test_version.CheckRequirement.test_packages_from_pkg_resources
allmydata.test.test_version.T.test_report_import_error
allmydata.test.test_version.VersionTestCase.test_basic_versions
allmydata.test.test_version.VersionTestCase.test_comparison
allmydata.test.test_version.VersionTestCase.test_from_parts
allmydata.test.test_version.VersionTestCase.test_irrational_versions
allmydata.test.test_version.VersionTestCase.test_suggest_normalized_version

409
misc/python3/ratchet.py
View File

@ -1,409 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''Ratchet up passing tests, or ratchet down failing tests.
Usage:
ratchet.py <"up" or "down"> <junitxml file path> <tracking file path>
This script helps when you expect a large test suite to fail spectactularly in
some environment, and you want to gradually improve the situation with minimal
impact to forward development of the same codebase for other environments. The
initial and primary usecase is porting from Python 2 to Python 3.
The idea is to emit JUnit XML from your test runner, and then invoke ratchet.py
to consume this XML output and operate on a so-called "tracking" file. When
ratcheting up passing tests, the tracking file will contain a list of tests,
one per line, that passed. When ratching down, the tracking file contains a
list of failing tests. On each subsequent run, ratchet.py will compare the
prior results in the tracking file with the new results in the XML, and will
report on both welcome and unwelcome changes. It will modify the tracking file
in the case of welcome changes, and therein lies the ratcheting.
The exit codes are:
0 - no changes observed
1 - changes observed, whether welcome or unwelcome
2 - invocation error
If <junitxml file path> does not exist, you'll get a FileNotFoundError:
>>> _test('up', None, None) # doctest: +ELLIPSIS
Traceback (most recent call last):
...
FileNotFoundError: ...
If <tracking file path> does not exist, that's fine:
>>> _test('up', '1', None)
Some tests not required to pass did:
c0.t
Conveniently, they have been added to `<tracking_path>` for you. Perhaps commit that?
Eep! 0 test(s) were required to pass, but instead 1 did. 🐭
Same if you're ratcheting down:
>>> _test('down', '1', None)
All and only tests expected to fail did. 💃
If the test run has the same output as last time, it's all good:
>>> _test('up', '01001110', '01001110')
All and only tests required to pass did. 💃
>>> _test('down', '01001110', '10110001')
All and only tests expected to fail did. 💃
If there's a welcome change, that's noted:
>>> _test('up', '0101', '0100')
Some tests not required to pass did:
c3.t
Conveniently, they have been added to `<tracking_path>` for you. Perhaps commit that?
Eep! 1 test(s) were required to pass, but instead 2 did. 🐭
>>> _test('down', '0011', '1110')
Some tests expected to fail didn't:
c2.t
Conveniently, they have been removed from `<tracking_path>` for you. Perhaps commit that?
Eep! 3 test(s) were expected to fail, but instead 2 did. 🐭
And if there is an unwelcome change, that is noted as well:
>>> _test('up', '1101', '1111')
Some tests required to pass didn't:
c2.t
Eep! 4 test(s) were required to pass, but instead 3 did. 🐭
>>> _test('down', '0000', '1101')
Some tests not expected to fail did:
c2.t
Eep! 3 test(s) were expected to fail, but instead 4 did. 🐭
And if there are both welcome and unwelcome changes, they are both noted:
>>> _test('up', '1101', '1011')
Some tests not required to pass did:
c1.t
Conveniently, they have been added to `<tracking_path>` for you. Perhaps commit that?
Some tests required to pass didn't:
c2.t
Eep! 3 test(s) were required to pass, but instead 3 did. 🐭
>>> _test('down', '0100', '1100')
Some tests not expected to fail did:
c2.t
c3.t
Some tests expected to fail didn't:
c1.t
Conveniently, they have been removed from `<tracking_path>` for you. Perhaps commit that?
Eep! 2 test(s) were expected to fail, but instead 3 did. 🐭
To test ratchet.py itself:
python3 -m doctest ratchet.py
'''
from __future__ import absolute_import, division, print_function, unicode_literals
import io
import os
import re
import sys
import tempfile
import xml.etree.ElementTree as Etree
class JUnitXMLFile(object):
'''Represent a file containing test results in JUnit XML format.
>>> eg = _mktemp_junitxml('0100111')
>>> results = JUnitXMLFile(eg.name).parse()
>>> results.failed
['c0.t', 'c2.t', 'c3.t']
>>> results.passed
['c1.t', 'c4.t', 'c5.t', 'c6.t']
'''
def __init__(self, filepath):
self.filepath = filepath
self.failed = []
self.failed_aggregates = {}
self.stderr_output = []
self.passed = []
self._tree = None
def parse(self):
if self._tree:
raise RuntimeError('already parsed')
self._tree = Etree.parse(self.filepath)
for testcase in self._tree.findall('testcase'):
self.process_testcase(testcase)
return self
def process_testcase(self, case):
key = self.case_key(case)
# look at children but throw away stderr output
nonpassing = [c for c in case if not c.tag == 'system-err']
n = len(nonpassing)
if n > 1:
raise RuntimeError(f'multiple results for {key}: {nonpassing}')
elif n == 1:
result = nonpassing.pop()
self.failed.append(key)
message = result.get('message')
self.failed_aggregates.setdefault(message, []).append(key)
else:
self.passed.append(key)
@staticmethod
def case_key(case):
return f'{case.get("classname")}.{case.get("name")}'
def report(self, details=False):
for k, v in sorted(
self.failed_aggregates.items(),
key = lambda i: len(i[1]),
reverse=True):
print(f'# {k}')
for t in v:
print(f' - {t}')
def load_previous_results(txt):
try:
previous_results = open(txt).read()
except FileNotFoundError:
previous_results = ''
parsed = set()
for line in previous_results.splitlines():
if not line or line.startswith('#'):
continue
parsed.add(line)
return parsed
def print_tests(tests):
for test in sorted(tests):
print(' ', test)
def ratchet_up_passing(tracking_path, tests):
try:
old = set(open(tracking_path, 'r'))
except FileNotFoundError:
old = set()
new = set(t + '\n' for t in tests)
merged = sorted(old | new)
open(tracking_path, 'w+').writelines(merged)
def ratchet_down_failing(tracking_path, tests):
new = set(t + '\n' for t in tests)
open(tracking_path, 'w+').writelines(sorted(new))
def main(direction, junitxml_path, tracking_path):
'''Takes a string indicating which direction to ratchet, "up" or "down,"
and two paths, one to test-runner output in JUnit XML format, the other to
a file tracking test results (one test case dotted name per line). Walk the
former looking for the latter, and react appropriately.
>>> inp = _mktemp_junitxml('0100111')
>>> out = _mktemp_tracking('0000000')
>>> _test_main('up', inp.name, out.name)
Some tests not required to pass did:
c1.t
c4.t
c5.t
c6.t
Conveniently, they have been added to `<tracking_path>` for you. Perhaps commit that?
Eep! 0 test(s) were required to pass, but instead 4 did. 🐭
'''
results = JUnitXMLFile(junitxml_path).parse()
if tracking_path == '...':
# Shortcut to aid in debugging XML parsing issues.
results.report()
return
previous = load_previous_results(tracking_path)
current = set(results.passed if direction == 'up' else results.failed)
subjunctive = {'up': 'required to pass', 'down': 'expected to fail'}[direction]
ratchet = None
too_many = current - previous
if too_many:
print(f'Some tests not {subjunctive} did:')
print_tests(too_many)
if direction == 'up':
# Too many passing tests is good -- let's do more of those!
ratchet_up_passing(tracking_path, current)
print(f'Conveniently, they have been added to `{tracking_path}` for you. Perhaps commit that?')
not_enough = previous - current
if not_enough:
print(f'Some tests {subjunctive} didn\'t:')
print_tests(not_enough)
if direction == 'down':
# Not enough failing tests is good -- let's do more of those!
ratchet_down_failing(tracking_path, current)
print(f'Conveniently, they have been removed from `{tracking_path}` for you. Perhaps commit that?')
if too_many or not_enough:
print(f'Eep! {len(previous)} test(s) were {subjunctive}, but instead {len(current)} did. 🐭')
return 1
print(f'All and only tests {subjunctive} did. 💃')
return 0
# When called as an executable ...
if __name__ == '__main__':
try:
direction, junitxml_path, tracking_path = sys.argv[1:4]
if direction not in ('up', 'down'):
raise ValueError
except ValueError:
doc = '\n'.join(__doc__.splitlines()[:6])
doc = re.sub(' ratchet.py', f' {sys.argv[0]}', doc)
print(doc, file=sys.stderr)
exit_code = 2
else:
exit_code = main(direction, junitxml_path, tracking_path)
sys.exit(exit_code)
# Helpers for when called under doctest ...
def _test(*a):
return _test_main(*_mk(*a))
def _test_main(direction, junitxml, tracking):
'''Takes a string 'up' or 'down' and paths to (or open file objects for)
the JUnit XML and tracking files to use for this test run. Captures and
emits stdout (slightly modified) for inspection via doctest.'''
junitxml_path = junitxml.name if hasattr(junitxml, 'name') else junitxml
tracking_path = tracking.name if hasattr(tracking, 'name') else tracking
old_stdout = sys.stdout
sys.stdout = io.StringIO()
try:
main(direction, junitxml_path, tracking_path)
finally:
sys.stdout.seek(0)
out = sys.stdout.read()
out = re.sub('`.*?`', '`<tracking_path>`', out).strip()
sys.stdout = old_stdout
print(out)
class _PotentialFile(object):
'''Represent a file that we are able to create but which doesn't exist yet,
and which, if we create it, will be automatically torn down when the test
run is over.'''
def __init__(self, filename):
self.d = tempfile.TemporaryDirectory()
self.name = os.path.join(self.d.name, filename)
def _mk(direction, spec_junitxml, spec_tracking):
'''Takes a string 'up' or 'down' and two bit strings specifying the state
of the JUnit XML results file and the tracking file to set up for this test
case. Returns the direction (unharmed) and two file-ish objects.
If a spec string is None the corresponding return value will be a
_PotentialFile object, which has a .name attribute (like a true file
object) that points to a file that does not exist, but could.
The reason not to simply return the path in all cases is that the file
objects are actually temporary file objects that destroy the underlying
file when they go out of scope, and we want to keep the underlying file
around until the end of the test run.'''
if None not in(spec_junitxml, spec_tracking):
if len(spec_junitxml) != len(spec_tracking):
raise ValueError('if both given, must be the same length: `{spec_junitxml}` and `{spec_tracking}`')
if spec_junitxml is None:
junitxml_fp = _PotentialFile('results.xml')
else:
junitxml_fp = _mktemp_junitxml(spec_junitxml)
if spec_tracking is None:
tracking_fp = _PotentialFile('tracking')
else:
tracking_fp = _mktemp_tracking(spec_tracking)
return direction, junitxml_fp, tracking_fp
def _mktemp_junitxml(spec):
'''Test helper to generate a raw JUnit XML file.
>>> fp = _mktemp_junitxml('00101')
>>> open(fp.name).read()[:11]
'<testsuite>'
'''
fp = tempfile.NamedTemporaryFile()
fp.write(b'<testsuite>')
passed = '''\
<testcase classname="c{i}" name="t"></testcase>
'''
failed = '''\
<testcase classname="c{i}" name="t">
<failure>Traceback (most recent call last):
File "/foo/bar/baz/buz.py", line 1, in &lt;module>
NameError: name 'heck' is not defined
</failure>
</testcase>
'''
i = 0
for c in spec:
if c == '0':
out = failed
elif c == '1':
out = passed
else:
raise ValueError(f'bad c: `{c}`')
fp.write(out.format(i=i).encode('utf8'))
i += 1
fp.write(b'</testsuite>')
fp.flush()
return fp
def _mktemp_tracking(spec):
'''Test helper to prefabricate a tracking file.
>>> fp = _mktemp_tracking('01101')
>>> print(open(fp.name).read()[:-1])
c1.t
c2.t
c4.t
'''
fp = tempfile.NamedTemporaryFile()
i = 0
for c in spec:
if c == '0':
pass
elif c == '1':
fp.write(f'c{i}.t\n'.encode('utf8'))
else:
raise ValueError(f'bad c: `{c}`')
i += 1
fp.flush()
return fp

1
newsfragments/3316.minor Normal file
View File

@ -0,0 +1 @@
Port checker result pages' rendering from nevow to twisted web templates.

1
newsfragments/3372.minor
View File

@ -0,0 +1 @@

37
src/allmydata/test/python3_tests.py Normal file
View File

@ -0,0 +1,37 @@
"""
This module defines the subset of the full test suite which is expected to
pass on Python 3 in a way which makes that suite discoverable by trial.
This module has been ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from twisted.python.reflect import (
namedModule,
)
from twisted.trial.runner import (
TestLoader,
)
from twisted.trial.unittest import (
TestSuite,
)
from allmydata.util._python3 import (
PORTED_TEST_MODULES,
)
def testSuite():
loader = TestLoader()
return TestSuite(list(
loader.loadModule(namedModule(module))
for module
in PORTED_TEST_MODULES
))

428
src/allmydata/test/test_checker.py
View File

@ -1,10 +1,25 @@
import json import json
import os.path, shutil import os.path, shutil
from bs4 import BeautifulSoup
from twisted.trial import unittest from twisted.trial import unittest
from twisted.internet import defer from twisted.internet import defer
from nevow.inevow import IRequest
from zope.interface import implementer
from twisted.web.server import Request
from twisted.web.test.requesthelper import DummyChannel
from twisted.web.template import flattenString
from allmydata import check_results, uri from allmydata import check_results, uri
from allmydata import uri as tahoe_uri from allmydata import uri as tahoe_uri
from allmydata.interfaces import (
IServer,
ICheckResults,
ICheckAndRepairResults,
)
from allmydata.util import base32 from allmydata.util import base32
from allmydata.web import check_results as web_check_results from allmydata.web import check_results as web_check_results
from allmydata.storage_client import StorageFarmBroker, NativeStorageServer from allmydata.storage_client import StorageFarmBroker, NativeStorageServer
@ -12,18 +27,115 @@ from allmydata.storage.server import storage_index_to_dir
from allmydata.monitor import Monitor from allmydata.monitor import Monitor
from allmydata.test.no_network import GridTestMixin from allmydata.test.no_network import GridTestMixin
from allmydata.immutable.upload import Data from allmydata.immutable.upload import Data
from allmydata.test.common_web import WebRenderingMixin
from allmydata.mutable.publish import MutableData from allmydata.mutable.publish import MutableData
from .common import ( from .common import (
EMPTY_CLIENT_CONFIG, EMPTY_CLIENT_CONFIG,
) )
from .web.common import (
assert_soup_has_favicon,
assert_soup_has_tag_with_content,
)
class FakeClient(object): class FakeClient(object):
def get_storage_broker(self): def get_storage_broker(self):
return self.storage_broker return self.storage_broker
class WebResultsRendering(unittest.TestCase, WebRenderingMixin): @implementer(IRequest)
class TestRequest(Request, object):
"""
A minimal Request class to use in tests.
XXX: We have to have this class because `common.get_arg()` expects
a `nevow.inevow.IRequest`, which `twisted.web.server.Request`
isn't. The request needs to have `args`, `fields`, `prepath`, and
`postpath` properties so that `allmydata.web.common.get_arg()`
won't complain.
"""
def __init__(self, args=None, fields=None):
super(TestRequest, self).__init__(DummyChannel())
self.args = args or {}
self.fields = fields or {}
self.prepath = [b""]
self.postpath = [b""]
@implementer(IServer)
class FakeServer(object):
def get_name(self):
return "fake name"
def get_longname(self):
return "fake longname"
def get_nickname(self):
return "fake nickname"
@implementer(ICheckResults)
class FakeCheckResults(object):
def __init__(self, si=None,
healthy=False, recoverable=False,
summary="fake summary"):
self._storage_index = si
self._is_healthy = healthy
self._is_recoverable = recoverable
self._summary = summary
def get_storage_index(self):
return self._storage_index
def get_storage_index_string(self):
return base32.b2a_or_none(self._storage_index)
def is_healthy(self):
return self._is_healthy
def is_recoverable(self):
return self._is_recoverable
def get_summary(self):
return self._summary
def get_corrupt_shares(self):
# returns a list of (IServer, storage_index, sharenum)
return [(FakeServer(), "<fake-si>", 0)]
@implementer(ICheckAndRepairResults)
class FakeCheckAndRepairResults(object):
def __init__(self, si=None,
repair_attempted=False,
repair_success=False):
self._storage_index = si
self._repair_attempted = repair_attempted
self._repair_success = repair_success
def get_storage_index(self):
return self._storage_index
def get_pre_repair_results(self):
return FakeCheckResults()
def get_post_repair_results(self):
return FakeCheckResults()
def get_repair_attempted(self):
return self._repair_attempted
def get_repair_successful(self):
return self._repair_success
class WebResultsRendering(unittest.TestCase):
@staticmethod
def remove_tags(html):
return BeautifulSoup(html).get_text(separator=" ")
def create_fake_client(self): def create_fake_client(self):
sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
@ -51,34 +163,31 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
c.storage_broker = sb c.storage_broker = sb
return c return c
def render_json(self, page): def render_json(self, resource):
d = self.render1(page, args={"output": ["json"]}) return resource.render(TestRequest(args={"output": ["json"]}))
return d
def render_element(self, element, args=None):
d = flattenString(TestRequest(args), element)
return unittest.TestCase().successResultOf(d)
def test_literal(self): def test_literal(self):
lcr = web_check_results.LiteralCheckResultsRendererElement()
html = self.render_element(lcr)
self.failUnlessIn("Literal files are always healthy", html)
html = self.render_element(lcr, args={"return_to": ["FOOURL"]})
self.failUnlessIn("Literal files are always healthy", html)
self.failUnlessIn('<a href="FOOURL">Return to file.</a>', html)
c = self.create_fake_client() c = self.create_fake_client()
lcr = web_check_results.LiteralCheckResultsRenderer(c) lcr = web_check_results.LiteralCheckResultsRenderer(c)
d = self.render1(lcr) js = self.render_json(lcr)
def _check(html): j = json.loads(js)
s = self.remove_tags(html) self.failUnlessEqual(j["storage-index"], "")
self.failUnlessIn("Literal files are always healthy", s) self.failUnlessEqual(j["results"]["healthy"], True)
d.addCallback(_check)
d.addCallback(lambda ignored:
self.render1(lcr, args={"return_to": ["FOOURL"]}))
def _check_return_to(html):
s = self.remove_tags(html)
self.failUnlessIn("Literal files are always healthy", s)
self.failUnlessIn('<a href="FOOURL">Return to file.</a>',
html)
d.addCallback(_check_return_to)
d.addCallback(lambda ignored: self.render_json(lcr))
def _check_json(js):
j = json.loads(js)
self.failUnlessEqual(j["storage-index"], "")
self.failUnlessEqual(j["results"]["healthy"], True)
d.addCallback(_check_json)
return d
def test_check(self): def test_check(self):
c = self.create_fake_client() c = self.create_fake_client()
@ -108,8 +217,8 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
healthy=True, recoverable=True, healthy=True, recoverable=True,
summary="groovy", summary="groovy",
**data) **data)
w = web_check_results.CheckResultsRenderer(c, cr) w = web_check_results.CheckResultsRendererElement(c, cr)
html = self.render2(w) html = self.render_element(w)
s = self.remove_tags(html) s = self.remove_tags(html)
self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
self.failUnlessIn("Healthy : groovy", s) self.failUnlessIn("Healthy : groovy", s)
@ -120,14 +229,14 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
self.failUnlessIn("Wrong Shares: 0", s) self.failUnlessIn("Wrong Shares: 0", s)
self.failUnlessIn("Recoverable Versions: 1", s) self.failUnlessIn("Recoverable Versions: 1", s)
self.failUnlessIn("Unrecoverable Versions: 0", s) self.failUnlessIn("Unrecoverable Versions: 0", s)
self.failUnlessIn("Good Shares (sorted in share order): Share ID Nickname Node ID shareid1 peer-0 00000000 peer-f ffffffff", s) self.failUnlessIn("Good Shares (sorted in share order): Share ID Nickname Node ID shareid1 peer-0 00000000 peer-f ffffffff", s)
cr = check_results.CheckResults(u, u.get_storage_index(), cr = check_results.CheckResults(u, u.get_storage_index(),
healthy=False, recoverable=True, healthy=False, recoverable=True,
summary="ungroovy", summary="ungroovy",
**data) **data)
w = web_check_results.CheckResultsRenderer(c, cr) w = web_check_results.CheckResultsRendererElement(c, cr)
html = self.render2(w) html = self.render_element(w)
s = self.remove_tags(html) s = self.remove_tags(html)
self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
self.failUnlessIn("Not Healthy! : ungroovy", s) self.failUnlessIn("Not Healthy! : ungroovy", s)
@ -138,22 +247,23 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
healthy=False, recoverable=False, healthy=False, recoverable=False,
summary="rather dead", summary="rather dead",
**data) **data)
w = web_check_results.CheckResultsRenderer(c, cr) w = web_check_results.CheckResultsRendererElement(c, cr)
html = self.render2(w) html = self.render_element(w)
s = self.remove_tags(html) s = self.remove_tags(html)
self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
self.failUnlessIn("Not Recoverable! : rather dead", s) self.failUnlessIn("Not Recoverable! : rather dead", s)
self.failUnlessIn("Corrupt shares: Share ID Nickname Node ID sh#2 peer-0 00000000", s) self.failUnlessIn("Corrupt shares: Share ID Nickname Node ID sh#2 peer-0 00000000", s)
html = self.render2(w) html = self.render_element(w)
s = self.remove_tags(html) s = self.remove_tags(html)
self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated
self.failUnlessIn("Not Recoverable! : rather dead", s) self.failUnlessIn("Not Recoverable! : rather dead", s)
html = self.render2(w, args={"return_to": ["FOOURL"]}) html = self.render_element(w, args={"return_to": ["FOOURL"]})
self.failUnlessIn('<a href="FOOURL">Return to file/directory.</a>', self.failUnlessIn('<a href="FOOURL">Return to file/directory.</a>',
html) html)
w = web_check_results.CheckResultsRenderer(c, cr)
d = self.render_json(w) d = self.render_json(w)
def _check_json(jdata): def _check_json(jdata):
j = json.loads(jdata) j = json.loads(jdata)
@ -178,15 +288,15 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
'recoverable': False, 'recoverable': False,
} }
self.failUnlessEqual(j["results"], expected) self.failUnlessEqual(j["results"], expected)
d.addCallback(_check_json) _check_json(d)
d.addCallback(lambda ignored: self.render1(w))
w = web_check_results.CheckResultsRendererElement(c, cr)
d = self.render_element(w)
def _check(html): def _check(html):
s = self.remove_tags(html) s = self.remove_tags(html)
self.failUnlessIn("File Check Results for SI=2k6avp", s) self.failUnlessIn("File Check Results for SI=2k6avp", s)
self.failUnlessIn("Not Recoverable! : rather dead", s) self.failUnlessIn("Not Recoverable! : rather dead", s)
d.addCallback(_check) _check(html)
return d
def test_check_and_repair(self): def test_check_and_repair(self):
c = self.create_fake_client() c = self.create_fake_client()
@ -244,8 +354,8 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
crr.post_repair_results = post_cr crr.post_repair_results = post_cr
crr.repair_attempted = False crr.repair_attempted = False
w = web_check_results.CheckAndRepairResultsRenderer(c, crr) w = web_check_results.CheckAndRepairResultsRendererElement(c, crr)
html = self.render2(w) html = self.render_element(w)
s = self.remove_tags(html) s = self.remove_tags(html)
self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s) self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s)
@ -256,7 +366,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
crr.repair_attempted = True crr.repair_attempted = True
crr.repair_successful = True crr.repair_successful = True
html = self.render2(w) html = self.render_element(w)
s = self.remove_tags(html) s = self.remove_tags(html)
self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s) self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s)
@ -271,7 +381,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
summary="better", summary="better",
**data) **data)
crr.post_repair_results = post_cr crr.post_repair_results = post_cr
html = self.render2(w) html = self.render_element(w)
s = self.remove_tags(html) s = self.remove_tags(html)
self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s) self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s)
@ -286,7 +396,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
summary="worse", summary="worse",
**data) **data)
crr.post_repair_results = post_cr crr.post_repair_results = post_cr
html = self.render2(w) html = self.render_element(w)
s = self.remove_tags(html) s = self.remove_tags(html)
self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s) self.failUnlessIn("File Check-And-Repair Results for SI=2k6avp", s)
@ -294,24 +404,218 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
self.failUnlessIn("Repair unsuccessful", s) self.failUnlessIn("Repair unsuccessful", s)
self.failUnlessIn("Post-Repair Checker Results:", s) self.failUnlessIn("Post-Repair Checker Results:", s)
d = self.render_json(w) w = web_check_results.CheckAndRepairResultsRenderer(c, crr)
def _got_json(data): j = json.loads(self.render_json(w))
j = json.loads(data) self.failUnlessEqual(j["repair-attempted"], True)
self.failUnlessEqual(j["repair-attempted"], True) self.failUnlessEqual(j["storage-index"],
self.failUnlessEqual(j["storage-index"], "2k6avpjga3dho3zsjo6nnkt7n4")
"2k6avpjga3dho3zsjo6nnkt7n4") self.failUnlessEqual(j["pre-repair-results"]["summary"], "illing")
self.failUnlessEqual(j["pre-repair-results"]["summary"], "illing") self.failUnlessEqual(j["post-repair-results"]["summary"], "worse")
self.failUnlessEqual(j["post-repair-results"]["summary"], "worse")
d.addCallback(_got_json) w = web_check_results.CheckAndRepairResultsRenderer(c, None)
j = json.loads(self.render_json(w))
self.failUnlessEqual(j["repair-attempted"], False)
self.failUnlessEqual(j["storage-index"], "")
def test_deep_check_renderer(self):
status = check_results.DeepCheckResults("fake-root-si")
status.add_check(
FakeCheckResults("<unhealthy/unrecoverable>", False, False),
(u"fake", u"unhealthy", u"unrecoverable")
)
status.add_check(
FakeCheckResults("<healthy/recoverable>", True, True),
(u"fake", u"healthy", u"recoverable")
)
status.add_check(
FakeCheckResults("<healthy/unrecoverable>", True, False),
(u"fake", u"healthy", u"unrecoverable")
)
status.add_check(
FakeCheckResults("<unhealthy/unrecoverable>", False, True),
(u"fake", u"unhealthy", u"recoverable")
)
monitor = Monitor()
monitor.set_status(status)
elem = web_check_results.DeepCheckResultsRendererElement(monitor)
doc = self.render_element(elem)
soup = BeautifulSoup(doc, 'html5lib')
assert_soup_has_favicon(self, soup)
assert_soup_has_tag_with_content(
self, soup, u"title",
u"Tahoe-LAFS - Deep Check Results"
)
assert_soup_has_tag_with_content(
self, soup, u"h1",
"Deep-Check Results for root SI="
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Objects Checked: 4"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Objects Healthy: 2"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Objects Unhealthy: 2"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Objects Unrecoverable: 2"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Corrupt Shares: 4"
)
assert_soup_has_tag_with_content(
self, soup, u"h2",
u"Files/Directories That Had Problems:"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"fake/unhealthy/recoverable: fake summary"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"fake/unhealthy/unrecoverable: fake summary"
)
assert_soup_has_tag_with_content(
self, soup, u"h2",
u"Servers on which corrupt shares were found"
)
assert_soup_has_tag_with_content(
self, soup, u"h2",
u"Corrupt Shares"
)
assert_soup_has_tag_with_content(
self, soup, u"h2",
u"All Results"
)
def test_deep_check_and_repair_renderer(self):
status = check_results.DeepCheckAndRepairResults("")
status.add_check_and_repair(
FakeCheckAndRepairResults("attempted/success", True, True),
(u"attempted", u"success")
)
status.add_check_and_repair(
FakeCheckAndRepairResults("attempted/failure", True, False),
(u"attempted", u"failure")
)
status.add_check_and_repair(
FakeCheckAndRepairResults("unattempted/failure", False, False),
(u"unattempted", u"failure")
)
monitor = Monitor()
monitor.set_status(status)
elem = web_check_results.DeepCheckAndRepairResultsRendererElement(monitor)
doc = self.render_element(elem)
soup = BeautifulSoup(doc, 'html5lib')
assert_soup_has_favicon(self, soup)
assert_soup_has_tag_with_content(
self, soup, u"title",
u"Tahoe-LAFS - Deep Check Results"
)
assert_soup_has_tag_with_content(
self, soup, u"h1",
u"Deep-Check-And-Repair Results for root SI="
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Objects Checked: 3"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Objects Healthy (before repair): 0"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Objects Unhealthy (before repair): 3"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Corrupt Shares (before repair): 3"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Repairs Attempted: 2"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Repairs Successful: 1"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
"Repairs Unsuccessful: 1"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Objects Healthy (after repair): 0"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Objects Unhealthy (after repair): 3"
)
assert_soup_has_tag_with_content(
self, soup, u"li",
u"Corrupt Shares (after repair): 3"
)
assert_soup_has_tag_with_content(
self, soup, u"h2",
u"Files/Directories That Had Problems:"
)
assert_soup_has_tag_with_content(
self, soup, u"h2",
u"Files/Directories That Still Have Problems:"
)
assert_soup_has_tag_with_content(
self, soup, u"h2",
u"Servers on which corrupt shares were found"
)
assert_soup_has_tag_with_content(
self, soup, u"h2",
u"Remaining Corrupt Shares"
)
w2 = web_check_results.CheckAndRepairResultsRenderer(c, None)
d.addCallback(lambda ignored: self.render_json(w2))
def _got_lit_results(data):
j = json.loads(data)
self.failUnlessEqual(j["repair-attempted"], False)
self.failUnlessEqual(j["storage-index"], "")
d.addCallback(_got_lit_results)
return d
class BalancingAct(GridTestMixin, unittest.TestCase): class BalancingAct(GridTestMixin, unittest.TestCase):
# test for #1115 regarding the 'count-good-share-hosts' metric # test for #1115 regarding the 'count-good-share-hosts' metric

2
src/allmydata/test/web/common.py
View File

@ -5,8 +5,6 @@ unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8') unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8') unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
def assert_soup_has_favicon(testcase, soup): def assert_soup_has_favicon(testcase, soup):
""" """

15
src/allmydata/test/web/test_grid.py
View File

@ -21,7 +21,12 @@ from allmydata.mutable import publish
from .. import common_util as testutil from .. import common_util as testutil
from ..common import WebErrorMixin, ShouldFailMixin from ..common import WebErrorMixin, ShouldFailMixin
from ..no_network import GridTestMixin from ..no_network import GridTestMixin
from .common import unknown_rwcap, unknown_rocap, unknown_immcap, FAVICON_MARKUP from .common import (
assert_soup_has_favicon,
unknown_immcap,
unknown_rocap,
unknown_rwcap,
)
DIR_HTML_TAG = '<html lang="en">' DIR_HTML_TAG = '<html lang="en">'
@ -92,7 +97,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
def _got_html_good(res): def _got_html_good(res):
self.failUnlessIn("Healthy", res) self.failUnlessIn("Healthy", res)
self.failIfIn("Not Healthy", res) self.failIfIn("Not Healthy", res)
self.failUnlessIn(FAVICON_MARKUP, res) soup = BeautifulSoup(res, 'html5lib')
assert_soup_has_favicon(self, soup)
d.addCallback(_got_html_good) d.addCallback(_got_html_good)
d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere") d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere")
def _got_html_good_return_to(res): def _got_html_good_return_to(res):
@ -235,7 +242,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
self.failUnlessIn("Healthy", res) self.failUnlessIn("Healthy", res)
self.failIfIn("Not Healthy", res) self.failIfIn("Not Healthy", res)
self.failUnlessIn("No repair necessary", res) self.failUnlessIn("No repair necessary", res)
self.failUnlessIn(FAVICON_MARKUP, res) soup = BeautifulSoup(res, 'html5lib')
assert_soup_has_favicon(self, soup)
d.addCallback(_got_html_good) d.addCallback(_got_html_good)
d.addCallback(self.CHECK, "sick", "t=check&repair=true") d.addCallback(self.CHECK, "sick", "t=check&repair=true")

13
src/allmydata/test/web/test_web.py
View File

@ -54,6 +54,9 @@ from .common import (
assert_soup_has_tag_with_attributes, assert_soup_has_tag_with_attributes,
assert_soup_has_tag_with_content, assert_soup_has_tag_with_content,
assert_soup_has_tag_with_attributes_and_content, assert_soup_has_tag_with_attributes_and_content,
unknown_rwcap,
unknown_rocap,
unknown_immcap,
) )
from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
@ -65,7 +68,6 @@ from ..common_web import (
Error, Error,
) )
from allmydata.client import _Client, SecretHolder from allmydata.client import _Client, SecretHolder
from .common import unknown_rwcap, unknown_rocap, unknown_immcap, FAVICON_MARKUP
# create a fake uploader/downloader, and a couple of fake dirnodes, then # create a fake uploader/downloader, and a couple of fake dirnodes, then
# create a webserver that works against them # create a webserver that works against them
@ -3262,13 +3264,15 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
res = yield self.get_operation_results(None, "123", "html") res = yield self.get_operation_results(None, "123", "html")
self.failUnlessIn("Objects Checked: <span>11</span>", res) self.failUnlessIn("Objects Checked: <span>11</span>", res)
self.failUnlessIn("Objects Healthy: <span>11</span>", res) self.failUnlessIn("Objects Healthy: <span>11</span>", res)
self.failUnlessIn(FAVICON_MARKUP, res) soup = BeautifulSoup(res, 'html5lib')
assert_soup_has_favicon(self, soup)
res = yield self.GET("/operations/123/") res = yield self.GET("/operations/123/")
# should be the same as without the slash # should be the same as without the slash
self.failUnlessIn("Objects Checked: <span>11</span>", res) self.failUnlessIn("Objects Checked: <span>11</span>", res)
self.failUnlessIn("Objects Healthy: <span>11</span>", res) self.failUnlessIn("Objects Healthy: <span>11</span>", res)
self.failUnlessIn(FAVICON_MARKUP, res) soup = BeautifulSoup(res, 'html5lib')
assert_soup_has_favicon(self, soup)
yield self.shouldFail2(error.Error, "one", "404 Not Found", yield self.shouldFail2(error.Error, "one", "404 Not Found",
"No detailed results for SI bogus", "No detailed results for SI bogus",
@ -3318,7 +3322,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res) self.failUnlessIn("Objects Unhealthy (after repair): <span>0</span>", res)
self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res) self.failUnlessIn("Corrupt Shares (after repair): <span>0</span>", res)
self.failUnlessIn(FAVICON_MARKUP, res) soup = BeautifulSoup(res, 'html5lib')
assert_soup_has_favicon(self, soup)
d.addCallback(_check_html) d.addCallback(_check_html)
return d return d

13
src/allmydata/util/_python3.py
View File

@ -1,6 +1,15 @@
""" """
Track the port to Python 3. Track the port to Python 3.
The two easiest ways to run the part of the test suite which is expected to
pass on Python 3 are::
$ tox -e py36
and::
$ trial allmydata.test.python3_tests
This module has been ported to Python 3. This module has been ported to Python 3.
""" """
@ -73,7 +82,3 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_util", "allmydata.test.test_util",
"allmydata.test.test_version", "allmydata.test.test_version",
] ]
if __name__ == '__main__':
from subprocess import check_call
check_call(["trial"] + PORTED_TEST_MODULES)

14
src/allmydata/web/check-and-repair-results.xhtml
View File

@ -1,4 +1,4 @@
<html xmlns:n="http://nevow.com/ns/nevow/0.1"> <html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
<head> <head>
<title>Tahoe-LAFS - Check Results</title> <title>Tahoe-LAFS - Check Results</title>
<link href="/tahoe.css" rel="stylesheet" type="text/css"/> <link href="/tahoe.css" rel="stylesheet" type="text/css"/>
@ -7,17 +7,17 @@
</head> </head>
<body> <body>
<h1>File Check-And-Repair Results for SI=<span n:render="storage_index" /></h1> <h1>File Check-And-Repair Results for SI=<span t:render="storage_index" /></h1>
<div n:render="summary" /> <div t:render="summary" />
<div n:render="repair_results" /> <div t:render="repair_results" />
<div n:render="post_repair_results" /> <div t:render="post_repair_results" />
<div n:render="maybe_pre_repair_results" /> <div t:render="maybe_pre_repair_results" />
<div n:render="return" /> <div t:render="return_to" />
</body> </body>
</html> </html>

12
src/allmydata/web/check-results.xhtml
View File

@ -1,4 +1,4 @@
<html xmlns:n="http://nevow.com/ns/nevow/0.1"> <html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
<head> <head>
<title>Tahoe-LAFS - Check Results</title> <title>Tahoe-LAFS - Check Results</title>
<link href="/tahoe.css" rel="stylesheet" type="text/css"/> <link href="/tahoe.css" rel="stylesheet" type="text/css"/>
@ -7,17 +7,17 @@
</head> </head>
<body> <body>
<h1>File Check Results for SI=<span n:render="storage_index" /></h1> <h1>File Check Results for SI=<span t:render="storage_index" /></h1>
<div> <div>
<span n:render="summary" /> <span t:render="summary" />
</div> </div>
<div n:render="repair" /> <div t:render="repair" />
<div n:render="results" /> <div t:render="results" />
<div n:render="return" /> <div t:render="return_to" />
</body> </body>
</html> </html>

753
src/allmydata/web/check_results.py
View File

@ -1,12 +1,35 @@
import time import time
import json import json
from nevow import rend, inevow, tags as T
from twisted.web import http, html from twisted.web import (
from allmydata.web.common import getxmlfile, get_arg, get_root, WebError http,
html,
)
from twisted.python.filepath import FilePath
from twisted.web.template import (
Element,
XMLFile,
renderer,
renderElement,
tags,
)
from allmydata.web.common import (
get_arg,
get_root,
WebError,
MultiFormatResource,
SlotsSequenceElement,
)
from allmydata.web.operations import ReloadMixin from allmydata.web.operations import ReloadMixin
from allmydata.interfaces import ICheckAndRepairResults, ICheckResults from allmydata.interfaces import (
from allmydata.util import base32, dictutil ICheckAndRepairResults,
ICheckResults,
)
from allmydata.util import (
base32,
dictutil,
)
def json_check_counts(r): def json_check_counts(r):
@ -64,53 +87,64 @@ def json_check_and_repair_results(r):
return data return data
class ResultsBase(object): class ResultsBase(object):
# self.client must point to the Client, so we can get nicknames and # self._client must point to the Client, so we can get nicknames and
# determine the permuted peer order # determine the permuted peer order
def _join_pathstring(self, path): def _join_pathstring(self, path):
"""
:param tuple path: a path represented by a tuple, such as
``(u'some', u'dir', u'file')``.
:return: a string joined by path separaters, such as
``u'some/dir/file'``.
"""
if path: if path:
pathstring = "/".join(self._html(path)) pathstring = "/".join(self._html(path))
else: else:
pathstring = "<root>" pathstring = "<root>"
return pathstring return pathstring
def _render_results(self, ctx, cr): def _render_results(self, req, cr):
assert ICheckResults(cr) assert ICheckResults(cr)
c = self.client c = self._client
sb = c.get_storage_broker() sb = c.get_storage_broker()
r = [] r = []
def add(name, value): def add(name, value):
r.append(T.li[name + ": ", value]) r.append(tags.li(name + ": ", value))
add("Report", tags.pre("\n".join(self._html(cr.get_report()))))
add("Report", T.pre["\n".join(self._html(cr.get_report()))])
add("Share Counts", add("Share Counts",
"need %d-of-%d, have %d" % (cr.get_encoding_needed(), "need %d-of-%d, have %d" % (cr.get_encoding_needed(),
cr.get_encoding_expected(), cr.get_encoding_expected(),
cr.get_share_counter_good())) cr.get_share_counter_good()))
add("Happiness Level", cr.get_happiness()) add("Happiness Level", str(cr.get_happiness()))
add("Hosts with good shares", cr.get_host_counter_good_shares()) add("Hosts with good shares", str(cr.get_host_counter_good_shares()))
if cr.get_corrupt_shares(): if cr.get_corrupt_shares():
badsharemap = [] badsharemap = []
for (s, si, shnum) in cr.get_corrupt_shares(): for (s, si, shnum) in cr.get_corrupt_shares():
d = T.tr[T.td["sh#%d" % shnum], d = tags.tr(tags.td("sh#%d" % shnum),
T.td[T.div(class_="nickname")[s.get_nickname()], tags.td(tags.div(s.get_nickname(), class_="nickname"),
T.div(class_="nodeid")[T.tt[s.get_name()]]], tags.div(tags.tt(s.get_name()), class_="nodeid")),)
]
badsharemap.append(d) badsharemap.append(d)
add("Corrupt shares", T.table()[ add("Corrupt shares",
T.tr[T.th["Share ID"], tags.table(
T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]], tags.tr(tags.th("Share ID"),
badsharemap]) tags.th((tags.div("Nickname"), tags.div("Node ID", class_="nodeid")), class_="nickname-and-peerid")),
badsharemap))
else: else:
add("Corrupt shares", "none") add("Corrupt shares", "none")
add("Wrong Shares", cr.get_share_counter_wrong()) add("Wrong Shares", str(cr.get_share_counter_wrong()))
sharemap_data = [] sharemap_data = []
shares_on_server = dictutil.DictOfSets() shares_on_server = dictutil.DictOfSets()
# FIXME: The two tables below contain nickname-and-nodeid table column markup which is duplicated with each other, introducer.xhtml, and deep-check-results.xhtml. All of these (and any other presentations of nickname-and-nodeid) should be combined. # FIXME: The two tables below contain nickname-and-nodeid
# table column markup which is duplicated with each other,
# introducer.xhtml, and deep-check-results.xhtml. All of these
# (and any other presentations of nickname-and-nodeid) should be combined.
for shareid in sorted(cr.get_sharemap().keys()): for shareid in sorted(cr.get_sharemap().keys()):
servers = sorted(cr.get_sharemap()[shareid], servers = sorted(cr.get_sharemap()[shareid],
@ -119,19 +153,20 @@ class ResultsBase(object):
shares_on_server.add(s, shareid) shares_on_server.add(s, shareid)
shareid_s = "" shareid_s = ""
if i == 0: if i == 0:
shareid_s = shareid shareid_s = str(shareid)
d = T.tr[T.td[shareid_s], d = tags.tr(tags.td(shareid_s),
T.td[T.div(class_="nickname")[s.get_nickname()], tags.td(tags.div(s.get_nickname(), class_="nickname"),
T.div(class_="nodeid")[T.tt[s.get_name()]]] tags.div(tags.tt(s.get_name()), class_="nodeid")))
]
sharemap_data.append(d) sharemap_data.append(d)
add("Good Shares (sorted in share order)", add("Good Shares (sorted in share order)",
T.table()[T.tr[T.th["Share ID"], T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]]], tags.table(tags.tr(tags.th("Share ID"),
sharemap_data]) tags.th(tags.div("Nickname"),
tags.div("Node ID", class_="nodeid"), class_="nickname-and-peerid")),
sharemap_data))
add("Recoverable Versions", str(cr.get_version_counter_recoverable()))
add("Recoverable Versions", cr.get_version_counter_recoverable()) add("Unrecoverable Versions", str(cr.get_version_counter_unrecoverable()))
add("Unrecoverable Versions", cr.get_version_counter_unrecoverable())
# this table is sorted by permuted order # this table is sorted by permuted order
permuted_servers = [s permuted_servers = [s
@ -144,20 +179,23 @@ class ResultsBase(object):
for s in permuted_servers: for s in permuted_servers:
shareids = list(shares_on_server.get(s, [])) shareids = list(shares_on_server.get(s, []))
shareids.reverse() shareids.reverse()
shareids_s = [ T.tt[shareid, " "] for shareid in sorted(shareids) ] shareids_s = [tags.tt(str(shareid), " ") for shareid in sorted(shareids)]
d = T.tr[T.td[T.div(class_="nickname")[s.get_nickname()],
T.div(class_="nodeid")[T.tt[s.get_name()]]], d = tags.tr(tags.td(tags.div(s.get_nickname(), class_="nickname"),
T.td[shareids_s], tags.div(tags.tt(s.get_name()), class_="nodeid")),
] tags.td(shareids_s), )
servermap.append(d) servermap.append(d)
num_shares_left -= len(shareids) num_shares_left -= len(shareids)
if not num_shares_left: if not num_shares_left:
break break
add("Share Balancing (servers in permuted order)",
T.table()[T.tr[T.th(class_="nickname-and-peerid")[T.div["Nickname"], T.div(class_="nodeid")["Node ID"]], T.th["Share IDs"]],
servermap])
return T.ul[r] add("Share Balancing (servers in permuted order)",
tags.table(tags.tr(tags.th(tags.div("Nickname"),
tags.div("Node ID", class_="nodeid"), class_="nickname-and-peerid"),
tags.th("Share IDs")),
servermap))
return tags.ul(r)
def _html(self, s): def _html(self, s):
if isinstance(s, (str, unicode)): if isinstance(s, (str, unicode)):
@ -165,91 +203,114 @@ class ResultsBase(object):
assert isinstance(s, (list, tuple)) assert isinstance(s, (list, tuple))
return [html.escape(w) for w in s] return [html.escape(w) for w in s]
def want_json(self, ctx): def _render_si_link(self, req, storage_index):
output = get_arg(inevow.IRequest(ctx), "output", "").lower()
if output.lower() == "json":
return True
return False
def _render_si_link(self, ctx, storage_index):
si_s = base32.b2a(storage_index) si_s = base32.b2a(storage_index)
req = inevow.IRequest(ctx)
ophandle = req.prepath[-1] ophandle = req.prepath[-1]
target = "%s/operations/%s/%s" % (get_root(ctx), ophandle, si_s) target = "%s/operations/%s/%s" % (get_root(req), ophandle, si_s)
output = get_arg(ctx, "output") output = get_arg(req, "output")
if output: if output:
target = target + "?output=%s" % output target = target + "?output=%s" % output
return T.a(href=target)[si_s] return tags.a(si_s, href=target)
class LiteralCheckResultsRenderer(rend.Page, ResultsBase):
docFactory = getxmlfile("literal-check-results.xhtml") class LiteralCheckResultsRenderer(MultiFormatResource, ResultsBase):
formatArgument = "output"
def __init__(self, client): def __init__(self, client):
self.client = client """
rend.Page.__init__(self, client) :param allmydata.interfaces.IStatsProducer client: stats provider.
"""
super(LiteralCheckResultsRenderer, self).__init__()
self._client = client
def renderHTTP(self, ctx): def render_HTML(self, req):
if self.want_json(ctx): return renderElement(req, LiteralCheckResultsRendererElement())
return self.json(ctx)
return rend.Page.renderHTTP(self, ctx)
def json(self, ctx): def render_JSON(self, req):
inevow.IRequest(ctx).setHeader("content-type", "text/plain") req.setHeader("content-type", "text/plain")
data = json_check_results(None) data = json_check_results(None)
return json.dumps(data, indent=1) + "\n" return json.dumps(data, indent=1) + "\n"
def render_return(self, ctx, data):
req = inevow.IRequest(ctx) class LiteralCheckResultsRendererElement(Element):
loader = XMLFile(FilePath(__file__).sibling("literal-check-results.xhtml"))
def __init__(self):
super(LiteralCheckResultsRendererElement, self).__init__()
@renderer
def return_to(self, req, tag):
return_to = get_arg(req, "return_to", None) return_to = get_arg(req, "return_to", None)
if return_to: if return_to:
return T.div[T.a(href=return_to)["Return to file."]] return tags.div(tags.a("Return to file.", href=return_to))
return "" return ""
class CheckerBase(object): class CheckerBase(object):
def renderHTTP(self, ctx): @renderer
if self.want_json(ctx): def storage_index(self, req, tag):
return self.json(ctx) return self._results.get_storage_index_string()
return rend.Page.renderHTTP(self, ctx)
def render_storage_index(self, ctx, data): @renderer
return self.r.get_storage_index_string() def return_to(self, req, tag):
def render_return(self, ctx, data):
req = inevow.IRequest(ctx)
return_to = get_arg(req, "return_to", None) return_to = get_arg(req, "return_to", None)
if return_to: if return_to:
return T.div[T.a(href=return_to)["Return to file/directory."]] return tags.div(tags.a("Return to file/directory.", href=return_to))
return "" return ""
class CheckResultsRenderer(CheckerBase, rend.Page, ResultsBase):
docFactory = getxmlfile("check-results.xhtml") class CheckResultsRenderer(MultiFormatResource):
formatArgument = "output"
def __init__(self, client, results): def __init__(self, client, results):
self.client = client """
self.r = ICheckResults(results) :param allmydata.interfaces.IStatsProducer client: stats provider.
rend.Page.__init__(self, results) :param allmydata.interfaces.ICheckResults results: results of check/vefify operation.
"""
super(CheckResultsRenderer, self).__init__()
self._client = client
self._results = ICheckResults(results)
def json(self, ctx): def render_HTML(self, req):
inevow.IRequest(ctx).setHeader("content-type", "text/plain") return renderElement(req, CheckResultsRendererElement(self._client, self._results))
data = json_check_results(self.r)
def render_JSON(self, req):
req.setHeader("content-type", "text/plain")
data = json_check_results(self._results)
return json.dumps(data, indent=1) + "\n" return json.dumps(data, indent=1) + "\n"
def render_summary(self, ctx, data):
class CheckResultsRendererElement(Element, CheckerBase, ResultsBase):
loader = XMLFile(FilePath(__file__).sibling("check-results.xhtml"))
def __init__(self, client, results):
super(CheckResultsRendererElement, self).__init__()
self._client = client
self._results = results
@renderer
def summary(self, req, tag):
results = [] results = []
if data.is_healthy(): if self._results.is_healthy():
results.append("Healthy") results.append("Healthy")
elif data.is_recoverable(): elif self._results.is_recoverable():
results.append("Not Healthy!") results.append("Not Healthy!")
else: else:
results.append("Not Recoverable!") results.append("Not Recoverable!")
results.append(" : ") results.append(" : ")
results.append(self._html(data.get_summary())) results.append(self._html(self._results.get_summary()))
return ctx.tag[results] return tag(results)
def render_repair(self, ctx, data): @renderer
if data.is_healthy(): def repair(self, req, tag):
if self._results.is_healthy():
return "" return ""
#repair = T.form(action=".", method="post", #repair = T.form(action=".", method="post",
# enctype="multipart/form-data")[ # enctype="multipart/form-data")[
# T.fieldset[ # T.fieldset[
@ -258,30 +319,52 @@ class CheckResultsRenderer(CheckerBase, rend.Page, ResultsBase):
# T.input(type="submit", value="Repair"), # T.input(type="submit", value="Repair"),
# ]] # ]]
#return ctx.tag[repair] #return ctx.tag[repair]
return "" # repair button disabled until we make it work correctly, return "" # repair button disabled until we make it work correctly,
# see #622 for details # see #622 for details
def render_results(self, ctx, data): @renderer
cr = self._render_results(ctx, data) def results(self, req, tag):
return ctx.tag[cr] cr = self._render_results(req, self._results)
return tag(cr)
class CheckAndRepairResultsRenderer(CheckerBase, rend.Page, ResultsBase): class CheckAndRepairResultsRenderer(MultiFormatResource):
docFactory = getxmlfile("check-and-repair-results.xhtml")
formatArgument = "output"
def __init__(self, client, results): def __init__(self, client, results):
self.client = client """
self.r = None :param allmydata.interfaces.IStatsProducer client: stats provider.
:param allmydata.interfaces.ICheckResults results: check/verify results.
"""
super(CheckAndRepairResultsRenderer, self).__init__()
self._client = client
self._results = None
if results: if results:
self.r = ICheckAndRepairResults(results) self._results = ICheckAndRepairResults(results)
rend.Page.__init__(self, results)
def json(self, ctx): def render_HTML(self, req):
inevow.IRequest(ctx).setHeader("content-type", "text/plain") elem = CheckAndRepairResultsRendererElement(self._client, self._results)
data = json_check_and_repair_results(self.r) return renderElement(req, elem)
def render_JSON(self, req):
req.setHeader("content-type", "text/plain")
data = json_check_and_repair_results(self._results)
return json.dumps(data, indent=1) + "\n" return json.dumps(data, indent=1) + "\n"
def render_summary(self, ctx, data):
cr = data.get_post_repair_results() class CheckAndRepairResultsRendererElement(Element, CheckerBase, ResultsBase):
loader = XMLFile(FilePath(__file__).sibling("check-and-repair-results.xhtml"))
def __init__(self, client, results):
super(CheckAndRepairResultsRendererElement, self).__init__()
self._client = client
self._results = results
@renderer
def summary(self, req, tag):
cr = self._results.get_post_repair_results()
results = [] results = []
if cr.is_healthy(): if cr.is_healthy():
results.append("Healthy") results.append("Healthy")
@ -291,35 +374,44 @@ class CheckAndRepairResultsRenderer(CheckerBase, rend.Page, ResultsBase):
results.append("Not Recoverable!") results.append("Not Recoverable!")
results.append(" : ") results.append(" : ")
results.append(self._html(cr.get_summary())) results.append(self._html(cr.get_summary()))
return ctx.tag[results] return tag(results)
def render_repair_results(self, ctx, data): @renderer
if data.get_repair_attempted(): def repair_results(self, req, tag):
if data.get_repair_successful(): if self._results.get_repair_attempted():
return ctx.tag["Repair successful"] if self._results.get_repair_successful():
return tag("Repair successful")
else: else:
return ctx.tag["Repair unsuccessful"] return tag("Repair unsuccessful")
return ctx.tag["No repair necessary"] return tag("No repair necessary")
def render_post_repair_results(self, ctx, data): @renderer
cr = self._render_results(ctx, data.get_post_repair_results()) def post_repair_results(self, req, tag):
return ctx.tag[T.div["Post-Repair Checker Results:"], cr] cr = self._render_results(req, self._results.get_post_repair_results())
return tag(tags.div("Post-Repair Checker Results:"), cr)
def render_maybe_pre_repair_results(self, ctx, data): @renderer
if data.get_repair_attempted(): def maybe_pre_repair_results(self, req, tag):
cr = self._render_results(ctx, data.get_pre_repair_results()) if self._results.get_repair_attempted():
return ctx.tag[T.div["Pre-Repair Checker Results:"], cr] cr = self._render_results(req, self._results.get_pre_repair_results())
return tag(tags.div("Pre-Repair Checker Results:"), cr)
return "" return ""
class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin): class DeepCheckResultsRenderer(MultiFormatResource):
docFactory = getxmlfile("deep-check-results.xhtml")
formatArgument = "output"
def __init__(self, client, monitor): def __init__(self, client, monitor):
self.client = client """
:param allmydata.interfaces.IStatsProducer client: stats provider.
:param allmydata.monitor.IMonitor monitor: status, progress, and cancellation provider.
"""
super(DeepCheckResultsRenderer, self).__init__()
self._client = client
self.monitor = monitor self.monitor = monitor
def childFactory(self, ctx, name): def getChild(self, name, req):
if not name: if not name:
return self return self
# /operation/$OPHANDLE/$STORAGEINDEX provides detailed information # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
@ -327,19 +419,18 @@ class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin):
si = base32.a2b(name) si = base32.a2b(name)
r = self.monitor.get_status() r = self.monitor.get_status()
try: try:
return CheckResultsRenderer(self.client, return CheckResultsRenderer(self._client,
r.get_results_for_storage_index(si)) r.get_results_for_storage_index(si))
except KeyError: except KeyError:
raise WebError("No detailed results for SI %s" % html.escape(name), raise WebError("No detailed results for SI %s" % html.escape(name),
http.NOT_FOUND) http.NOT_FOUND)
def renderHTTP(self, ctx): def render_HTML(self, req):
if self.want_json(ctx): elem = DeepCheckResultsRendererElement(self.monitor)
return self.json(ctx) return renderElement(req, elem)
return rend.Page.renderHTTP(self, ctx)
def json(self, ctx): def render_JSON(self, req):
inevow.IRequest(ctx).setHeader("content-type", "text/plain") req.setHeader("content-type", "text/plain")
data = {} data = {}
data["finished"] = self.monitor.is_finished() data["finished"] = self.monitor.is_finished()
res = self.monitor.get_status() res = self.monitor.get_status()
@ -361,116 +452,170 @@ class DeepCheckResultsRenderer(rend.Page, ResultsBase, ReloadMixin):
data["stats"] = res.get_stats() data["stats"] = res.get_stats()
return json.dumps(data, indent=1) + "\n" return json.dumps(data, indent=1) + "\n"
def render_root_storage_index(self, ctx, data):
class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin):
loader = XMLFile(FilePath(__file__).sibling("deep-check-results.xhtml"))
def __init__(self, monitor):
super(DeepCheckResultsRendererElement, self).__init__()
self.monitor = monitor
@renderer
def root_storage_index(self, req, tag):
if not self.monitor.get_status():
return ""
return self.monitor.get_status().get_root_storage_index_string() return self.monitor.get_status().get_root_storage_index_string()
def data_objects_checked(self, ctx, data): def _get_monitor_counter(self, name):
return self.monitor.get_status().get_counters()["count-objects-checked"] if not self.monitor.get_status():
def data_objects_healthy(self, ctx, data): return ""
return self.monitor.get_status().get_counters()["count-objects-healthy"] return str(self.monitor.get_status().get_counters().get(name))
def data_objects_unhealthy(self, ctx, data):
return self.monitor.get_status().get_counters()["count-objects-unhealthy"]
def data_objects_unrecoverable(self, ctx, data):
return self.monitor.get_status().get_counters()["count-objects-unrecoverable"]
def data_count_corrupt_shares(self, ctx, data): @renderer
return self.monitor.get_status().get_counters()["count-corrupt-shares"] def objects_checked(self, req, tag):
return self._get_monitor_counter("count-objects-checked")
def render_problems_p(self, ctx, data): @renderer
c = self.monitor.get_status().get_counters() def objects_healthy(self, req, tag):
if c["count-objects-unhealthy"]: return self._get_monitor_counter("count-objects-healthy")
return ctx.tag
@renderer
def objects_unhealthy(self, req, tag):
return self._get_monitor_counter("count-objects-unhealthy")
@renderer
def objects_unrecoverable(self, req, tag):
return self._get_monitor_counter("count-objects-unrecoverable")
@renderer
def count_corrupt_shares(self, req, tag):
return self._get_monitor_counter("count-corrupt-shares")
@renderer
def problems_p(self, req, tag):
if self._get_monitor_counter("count-objects-unhealthy"):
return tag
return "" return ""
def data_problems(self, ctx, data): @renderer
def problems(self, req, tag):
all_objects = self.monitor.get_status().get_all_results() all_objects = self.monitor.get_status().get_all_results()
problems = []
for path in sorted(all_objects.keys()): for path in sorted(all_objects.keys()):
cr = all_objects[path] cr = all_objects[path]
assert ICheckResults.providedBy(cr) assert ICheckResults.providedBy(cr)
if not cr.is_healthy(): if not cr.is_healthy():
yield path, cr summary_text = ""
summary = cr.get_summary()
if summary:
summary_text = ": " + summary
summary_text += " [SI: %s]" % cr.get_storage_index_string()
problems.append({
# Not sure self._join_pathstring(path) is the
# right thing to use here.
"problem": self._join_pathstring(path) + self._html(summary_text),
})
def render_problem(self, ctx, data): return SlotsSequenceElement(tag, problems)
path, cr = data
summary_text = ""
summary = cr.get_summary()
if summary:
summary_text = ": " + summary
summary_text += " [SI: %s]" % cr.get_storage_index_string()
return ctx.tag[self._join_pathstring(path), self._html(summary_text)]
@renderer
def render_servers_with_corrupt_shares_p(self, ctx, data): def servers_with_corrupt_shares_p(self, req, tag):
if self.monitor.get_status().get_counters()["count-corrupt-shares"]: if self._get_monitor_counter("count-corrupt-shares"):
return ctx.tag return tag
return "" return ""
def data_servers_with_corrupt_shares(self, ctx, data): @renderer
def servers_with_corrupt_shares(self, req, tag):
servers = [s servers = [s
for (s, storage_index, sharenum) for (s, storage_index, sharenum)
in self.monitor.get_status().get_corrupt_shares()] in self.monitor.get_status().get_corrupt_shares()]
servers.sort(key=lambda s: s.get_longname()) servers.sort(key=lambda s: s.get_longname())
return servers
def render_server_problem(self, ctx, server): problems = []
data = [server.get_name()]
nickname = server.get_nickname()
if nickname:
data.append(" (%s)" % self._html(nickname))
return ctx.tag[data]
for server in servers:
name = [server.get_name()]
nickname = server.get_nickname()
if nickname:
name.append(" (%s)" % self._html(nickname))
problems.append({"problem": name})
def render_corrupt_shares_p(self, ctx, data): return SlotsSequenceElement(tag, problems)
if self.monitor.get_status().get_counters()["count-corrupt-shares"]:
return ctx.tag @renderer
def corrupt_shares_p(self, req, tag):
if self._get_monitor_counter("count-corrupt-shares"):
return tag
return "" return ""
def data_corrupt_shares(self, ctx, data):
return self.monitor.get_status().get_corrupt_shares()
def render_share_problem(self, ctx, data):
server, storage_index, sharenum = data
nickname = server.get_nickname()
ctx.fillSlots("serverid", server.get_name())
if nickname:
ctx.fillSlots("nickname", self._html(nickname))
ctx.fillSlots("si", self._render_si_link(ctx, storage_index))
ctx.fillSlots("shnum", str(sharenum))
return ctx.tag
def render_return(self, ctx, data): @renderer
req = inevow.IRequest(ctx) def corrupt_shares(self, req, tag):
shares = self.monitor.get_status().get_corrupt_shares()
problems = []
for share in shares:
server, storage_index, sharenum = share
nickname = server.get_nickname()
problem = {
"serverid": server.get_name(),
"nickname": self._html(nickname),
"si": self._render_si_link(req, storage_index),
"shnum": str(sharenum),
}
problems.append(problem)
return SlotsSequenceElement(tag, problems)
@renderer
def return_to(self, req, tag):
return_to = get_arg(req, "return_to", None) return_to = get_arg(req, "return_to", None)
if return_to: if return_to:
return T.div[T.a(href=return_to)["Return to file/directory."]] return tags.div(tags.a("Return to file/directory.", href=return_to))
return "" return ""
def data_all_objects(self, ctx, data): @renderer
r = self.monitor.get_status().get_all_results() def all_objects(self, req, tag):
for path in sorted(r.keys()): results = self.monitor.get_status().get_all_results()
yield (path, r[path]) objects = []
def render_object(self, ctx, data): for path in sorted(results.keys()):
path, r = data result = results.get(path)
ctx.fillSlots("path", self._join_pathstring(path)) storage_index = result.get_storage_index()
ctx.fillSlots("healthy", str(r.is_healthy())) object = {
ctx.fillSlots("recoverable", str(r.is_recoverable())) "path": self._join_pathstring(path),
storage_index = r.get_storage_index() "healthy": str(result.is_healthy()),
ctx.fillSlots("storage_index", self._render_si_link(ctx, storage_index)) "recoverable": str(result.is_recoverable()),
ctx.fillSlots("summary", self._html(r.get_summary())) "storage_index": self._render_si_link(req, storage_index),
return ctx.tag "summary": self._html(result.get_summary()),
}
objects.append(object)
def render_runtime(self, ctx, data): return SlotsSequenceElement(tag, objects)
req = inevow.IRequest(ctx)
runtime = time.time() - req.processing_started_timestamp
return ctx.tag["runtime: %s seconds" % runtime]
class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin): @renderer
docFactory = getxmlfile("deep-check-and-repair-results.xhtml") def runtime(self, req, tag):
runtime = 'unknown'
if hasattr(req, 'processing_started_timestamp'):
runtime = time.time() - req.processing_started_timestamp
return tag("runtime: %s seconds" % runtime)
class DeepCheckAndRepairResultsRenderer(MultiFormatResource):
formatArgument = "output"
def __init__(self, client, monitor): def __init__(self, client, monitor):
self.client = client """
:param allmydata.interfaces.IStatsProducer client: stats provider.
:param allmydata.monitor.IMonitor monitor: status, progress, and cancellation provider.
"""
super(DeepCheckAndRepairResultsRenderer, self).__init__()
self._client = client
self.monitor = monitor self.monitor = monitor
def childFactory(self, ctx, name): def getChild(self, name, req):
if not name: if not name:
return self return self
# /operation/$OPHANDLE/$STORAGEINDEX provides detailed information # /operation/$OPHANDLE/$STORAGEINDEX provides detailed information
@ -479,18 +624,17 @@ class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin):
s = self.monitor.get_status() s = self.monitor.get_status()
try: try:
results = s.get_results_for_storage_index(si) results = s.get_results_for_storage_index(si)
return CheckAndRepairResultsRenderer(self.client, results) return CheckAndRepairResultsRenderer(self._client, results)
except KeyError: except KeyError:
raise WebError("No detailed results for SI %s" % html.escape(name), raise WebError("No detailed results for SI %s" % html.escape(name),
http.NOT_FOUND) http.NOT_FOUND)
def renderHTTP(self, ctx): def render_HTML(self, req):
if self.want_json(ctx): elem = DeepCheckAndRepairResultsRendererElement(self.monitor)
return self.json(ctx) return renderElement(req, elem)
return rend.Page.renderHTTP(self, ctx)
def json(self, ctx): def render_JSON(self, req):
inevow.IRequest(ctx).setHeader("content-type", "text/plain") req.setHeader("content-type", "text/plain")
res = self.monitor.get_status() res = self.monitor.get_status()
data = {} data = {}
data["finished"] = self.monitor.is_finished() data["finished"] = self.monitor.is_finished()
@ -531,119 +675,132 @@ class DeepCheckAndRepairResultsRenderer(rend.Page, ResultsBase, ReloadMixin):
data["stats"] = res.get_stats() data["stats"] = res.get_stats()
return json.dumps(data, indent=1) + "\n" return json.dumps(data, indent=1) + "\n"
def render_root_storage_index(self, ctx, data):
return self.monitor.get_status().get_root_storage_index_string()
def data_objects_checked(self, ctx, data): class DeepCheckAndRepairResultsRendererElement(DeepCheckResultsRendererElement):
return self.monitor.get_status().get_counters()["count-objects-checked"] """
The page generated here has several elements common to "deep check
results" page; hence the code reuse.
"""
def data_objects_healthy(self, ctx, data): loader = XMLFile(FilePath(__file__).sibling("deep-check-and-repair-results.xhtml"))
return self.monitor.get_status().get_counters()["count-objects-healthy-pre-repair"]
def data_objects_unhealthy(self, ctx, data):
return self.monitor.get_status().get_counters()["count-objects-unhealthy-pre-repair"]
def data_corrupt_shares(self, ctx, data):
return self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]
def data_repairs_attempted(self, ctx, data): def __init__(self, monitor):
return self.monitor.get_status().get_counters()["count-repairs-attempted"] super(DeepCheckAndRepairResultsRendererElement, self).__init__(monitor)
def data_repairs_successful(self, ctx, data): self.monitor = monitor
return self.monitor.get_status().get_counters()["count-repairs-successful"]
def data_repairs_unsuccessful(self, ctx, data):
return self.monitor.get_status().get_counters()["count-repairs-unsuccessful"]
def data_objects_healthy_post(self, ctx, data): @renderer
return self.monitor.get_status().get_counters()["count-objects-healthy-post-repair"] def objects_healthy(self, req, tag):
def data_objects_unhealthy_post(self, ctx, data): return self._get_monitor_counter("count-objects-healthy-pre-repair")
return self.monitor.get_status().get_counters()["count-objects-unhealthy-post-repair"]
def data_corrupt_shares_post(self, ctx, data):
return self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]
def render_pre_repair_problems_p(self, ctx, data): @renderer
c = self.monitor.get_status().get_counters() def objects_unhealthy(self, req, tag):
if c["count-objects-unhealthy-pre-repair"]: return self._get_monitor_counter("count-objects-unhealthy-pre-repair")
return ctx.tag
@renderer
def corrupt_shares(self, req, tag):
return self._get_monitor_counter("count-corrupt-shares-pre-repair")
@renderer
def repairs_attempted(self, req, tag):
return self._get_monitor_counter("count-repairs-attempted")
@renderer
def repairs_successful(self, req, tag):
return self._get_monitor_counter("count-repairs-successful")
@renderer
def repairs_unsuccessful(self, req, tag):
return self._get_monitor_counter("count-repairs-unsuccessful")
@renderer
def objects_healthy_post(self, req, tag):
return self._get_monitor_counter("count-objects-healthy-post-repair")
@renderer
def objects_unhealthy_post(self, req, tag):
return self._get_monitor_counter("count-objects-unhealthy-post-repair")
@renderer
def corrupt_shares_post(self, req, tag):
return self._get_monitor_counter("count-corrupt-shares-post-repair")
@renderer
def pre_repair_problems_p(self, req, tag):
if self._get_monitor_counter("count-objects-unhealthy-pre-repair"):
return tag
return "" return ""
def data_pre_repair_problems(self, ctx, data): @renderer
def pre_repair_problems(self, req, tag):
all_objects = self.monitor.get_status().get_all_results() all_objects = self.monitor.get_status().get_all_results()
problems = []
for path in sorted(all_objects.keys()): for path in sorted(all_objects.keys()):
r = all_objects[path] r = all_objects[path]
assert ICheckAndRepairResults.providedBy(r) assert ICheckAndRepairResults.providedBy(r)
cr = r.get_pre_repair_results() cr = r.get_pre_repair_results()
if not cr.is_healthy(): if not cr.is_healthy():
yield path, cr problem = self._join_pathstring(path), ": ", self._html(cr.get_summary())
problems.append({"problem": problem})
def render_problem(self, ctx, data): return SlotsSequenceElement(tag, problems)
path, cr = data
return ctx.tag[self._join_pathstring(path), ": ",
self._html(cr.get_summary())]
def render_post_repair_problems_p(self, ctx, data): @renderer
c = self.monitor.get_status().get_counters() def post_repair_problems_p(self, req, tag):
if (c["count-objects-unhealthy-post-repair"] if (self._get_monitor_counter("count-objects-unhealthy-post-repair")
or c["count-corrupt-shares-post-repair"]): or self._get_monitor_counter("count-corrupt-shares-post-repair")):
return ctx.tag return tag
return "" return ""
def data_post_repair_problems(self, ctx, data): @renderer
def post_repair_problems(self, req, tag):
all_objects = self.monitor.get_status().get_all_results() all_objects = self.monitor.get_status().get_all_results()
problems = []
for path in sorted(all_objects.keys()): for path in sorted(all_objects.keys()):
r = all_objects[path] r = all_objects[path]
assert ICheckAndRepairResults.providedBy(r) assert ICheckAndRepairResults.providedBy(r)
cr = r.get_post_repair_results() cr = r.get_post_repair_results()
if not cr.is_healthy(): if not cr.is_healthy():
yield path, cr problem = self._join_pathstring(path), ": ", self._html(cr.get_summary())
problems.append({"problem": problem})
def render_servers_with_corrupt_shares_p(self, ctx, data): return SlotsSequenceElement(tag, problems)
if self.monitor.get_status().get_counters()["count-corrupt-shares-pre-repair"]:
return ctx.tag
return ""
def data_servers_with_corrupt_shares(self, ctx, data):
return [] # TODO
def render_server_problem(self, ctx, data):
pass
@renderer
def render_remaining_corrupt_shares_p(self, ctx, data): def remaining_corrupt_shares_p(self, req, tag):
if self.monitor.get_status().get_counters()["count-corrupt-shares-post-repair"]: if self._get_monitor_counter("count-corrupt-shares-post-repair"):
return ctx.tag return tag
return ""
def data_post_repair_corrupt_shares(self, ctx, data):
return [] # TODO
def render_share_problem(self, ctx, data):
pass
def render_return(self, ctx, data):
req = inevow.IRequest(ctx)
return_to = get_arg(req, "return_to", None)
if return_to:
return T.div[T.a(href=return_to)["Return to file/directory."]]
return "" return ""
def data_all_objects(self, ctx, data): @renderer
r = self.monitor.get_status().get_all_results() def post_repair_corrupt_shares(self, req, tag):
for path in sorted(r.keys()): # TODO: this was not implemented before porting to
yield (path, r[path]) # twisted.web.template; leaving it as such.
#
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3371
corrupt = [{"share":"unimplemented"}]
return SlotsSequenceElement(tag, corrupt)
def render_object(self, ctx, data): @renderer
path, r = data def all_objects(self, req, tag):
ctx.fillSlots("path", self._join_pathstring(path)) results = {}
ctx.fillSlots("healthy_pre_repair", if self.monitor.get_status():
str(r.get_pre_repair_results().is_healthy())) results = self.monitor.get_status().get_all_results()
ctx.fillSlots("recoverable_pre_repair", objects = []
str(r.get_pre_repair_results().is_recoverable()))
ctx.fillSlots("healthy_post_repair", for path in sorted(results.keys()):
str(r.get_post_repair_results().is_healthy())) result = results[path]
storage_index = r.get_storage_index() storage_index = result.get_storage_index()
ctx.fillSlots("storage_index", obj = {
self._render_si_link(ctx, storage_index)) "path": self._join_pathstring(path),
ctx.fillSlots("summary", "healthy_pre_repair": str(result.get_pre_repair_results().is_healthy()),
self._html(r.get_pre_repair_results().get_summary())) "recoverable_pre_repair": str(result.get_pre_repair_results().is_recoverable()),
return ctx.tag "healthy_post_repair": str(result.get_post_repair_results().is_healthy()),
"storage_index": self._render_si_link(req, storage_index),
"summary": self._html(result.get_pre_repair_results().get_summary()),
}
objects.append(obj)
return SlotsSequenceElement(tag, objects)
def render_runtime(self, ctx, data):
req = inevow.IRequest(ctx)
runtime = time.time() - req.processing_started_timestamp
return ctx.tag["runtime: %s seconds" % runtime]

105
src/allmydata/web/deep-check-and-repair-results.xhtml
View File

@ -1,95 +1,106 @@
<html xmlns:n="http://nevow.com/ns/nevow/0.1"> <html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
<head> <head>
<title>Tahoe-LAFS - Deep Check Results</title> <title>Tahoe-LAFS - Deep Check Results</title>
<link href="/tahoe.css" rel="stylesheet" type="text/css"/> <link href="/tahoe.css" rel="stylesheet" type="text/css"/>
<link href="/icon.png" rel="shortcut icon" /> <link href="/icon.png" rel="shortcut icon" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta n:render="refresh" /> <meta t:render="refresh" />
</head> </head>
<body> <body>
<h1>Deep-Check-And-Repair Results for root <h1>Deep-Check-And-Repair Results for root
SI=<span n:render="root_storage_index" /></h1> SI=<span t:render="root_storage_index" /></h1>
<h2 n:render="reload" /> <h2 t:render="reload" />
<p>Counters:</p> <p>Counters:</p>
<ul> <ul>
<li>Objects Checked: <span n:render="data" n:data="objects_checked" /></li> <li>Objects Checked: <span><t:transparent t:render="objects_checked" /></span></li>
<li>Objects Healthy (before repair): <span n:render="data" n:data="objects_healthy" /></li> <li>Objects Healthy (before repair): <span><t:transparent t:render="objects_healthy" /></span></li>
<li>Objects Unhealthy (before repair): <span n:render="data" n:data="objects_unhealthy" /></li> <li>Objects Unhealthy (before repair): <span><t:transparent t:render="objects_unhealthy" /></span></li>
<li>Corrupt Shares (before repair): <span n:render="data" n:data="corrupt_shares" /></li> <li>Corrupt Shares (before repair): <span><t:transparent t:render="corrupt_shares" /></span></li>
<li>Repairs Attempted: <span n:render="data" n:data="repairs_attempted" /></li> <li>Repairs Attempted: <span><t:transparent t:render="repairs_attempted" /></span></li>
<li>Repairs Successful: <span n:render="data" n:data="repairs_successful" /></li> <li>Repairs Successful: <span><t:transparent t:render="repairs_successful" /></span></li>
<li>Repairs Unsuccessful: <span n:render="data" n:data="repairs_unsuccessful" /></li> <li>Repairs Unsuccessful: <span><t:transparent t:render="repairs_unsuccessful" /></span></li>
<li>Objects Healthy (after repair): <span n:render="data" n:data="objects_healthy_post" /></li> <li>Objects Healthy (after repair): <span><t:transparent t:render="objects_healthy_post" /></span></li>
<li>Objects Unhealthy (after repair): <span n:render="data" n:data="objects_unhealthy_post" /></li> <li>Objects Unhealthy (after repair): <span><t:transparent t:render="objects_unhealthy_post" /></span></li>
<li>Corrupt Shares (after repair): <span n:render="data" n:data="corrupt_shares_post" /></li> <li>Corrupt Shares (after repair): <span><t:transparent t:render="corrupt_shares_post" /></span></li>
</ul> </ul>
<div n:render="pre_repair_problems_p"> <div t:render="pre_repair_problems_p">
<h2>Files/Directories That Had Problems:</h2> <h2>Files/Directories That Had Problems:</h2>
<ul n:render="sequence" n:data="pre_repair_problems"> <ul t:render="pre_repair_problems">
<li n:pattern="item" n:render="problem"/> <li t:render="item">
<li n:pattern="empty">None</li> <t:slot name="problem" />
</li>
<li t:render="empty">None</li>
</ul> </ul>
</div> </div>
<div n:render="post_repair_problems_p"> <div t:render="post_repair_problems_p">
<h2>Files/Directories That Still Have Problems:</h2> <h2>Files/Directories That Still Have Problems:</h2>
<ul n:render="sequence" n:data="post_repair_problems"> <ul t:render="post_repair_problems">
<li n:pattern="item" n:render="problem"/> <li t:render="item">
<li n:pattern="empty">None</li> <t:slot name="problem" />
</li>
<li t:render="empty">None</li>
</ul> </ul>
</div> </div>
<div n:render="servers_with_corrupt_shares_p"> <div t:render="servers_with_corrupt_shares_p">
<h2>Servers on which corrupt shares were found</h2> <h2>Servers on which corrupt shares were found</h2>
<ul n:render="sequence" n:data="servers_with_corrupt_shares"> <ul t:render="servers_with_corrupt_shares">
<li n:pattern="item" n:render="server_problem"/> <li t:render="item">
<li n:pattern="empty">None</li> <t:slot name="problem" />
</li>
<li t:render="empty">None</li>
</ul> </ul>
</div> </div>
<div n:render="remaining_corrupt_shares_p"> <div t:render="remaining_corrupt_shares_p">
<h2>Remaining Corrupt Shares</h2> <h2>Remaining Corrupt Shares</h2>
<p>These shares need to be manually inspected and removed.</p> <p>These shares need to be manually inspected and removed.</p>
<ul n:render="sequence" n:data="post_repair_corrupt_shares"> <ul t:render="post_repair_corrupt_shares">
<li n:pattern="item" n:render="share_problem"/> <li t:render="item">
<li n:pattern="empty">None</li> <t:slot name="share" />
</li>
<li t:render="empty">None</li>
</ul> </ul>
</div> </div>
<div n:render="return" /> <div t:render="return_to" />
<div> <div>
<table n:render="sequence" n:data="all_objects"> <table t:render="all_objects">
<tr n:pattern="header"> <tr t:render="header">
<td>Relative Path</td> <th>Relative Path</th>
<td>Healthy Pre-Repair</td> <th>Healthy Pre-Repair</th>
<td>Recoverable Pre-Repair</td> <th>Recoverable Pre-Repair</th>
<td>Healthy Post-Repair</td> <th>Healthy Post-Repair</th>
<td>Storage Index</td> <th>Storage Index</th>
<td>Summary</td> <th>Summary</th>
</tr> </tr>
<tr n:pattern="item" n:render="object"> <tr t:render="item">
<td><n:slot name="path"/></td> <td><t:slot name="path"/></td>
<td><n:slot name="healthy_pre_repair"/></td> <td><t:slot name="healthy_pre_repair"/></td>
<td><n:slot name="recoverable_pre_repair"/></td> <td><t:slot name="recoverable_pre_repair"/></td>
<td><n:slot name="healthy_post_repair"/></td> <td><t:slot name="healthy_post_repair"/></td>
<td><n:slot name="storage_index"/></td> <td><t:slot name="storage_index"/></td>
<td><n:slot name="summary"/></td> <td><t:slot name="summary"/></td>
</tr>
<tr t:render="empty">
<td>Nothing to report yet.</td>
</tr> </tr>
</table> </table>
</div> </div>
<div n:render="runtime" /> <div t:render="runtime" />
</body> </body>
</html> </html>

96
src/allmydata/web/deep-check-results.xhtml
View File

@ -1,87 +1,93 @@
<html xmlns:n="http://nevow.com/ns/nevow/0.1"> <html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
<head> <head>
<title>Tahoe-LAFS - Deep Check Results</title> <title>Tahoe-LAFS - Deep Check Results</title>
<link href="/tahoe.css" rel="stylesheet" type="text/css"/> <link href="/tahoe.css" rel="stylesheet" type="text/css"/>
<link href="/icon.png" rel="shortcut icon" /> <link href="/icon.png" rel="shortcut icon" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta n:render="refresh" /> <meta t:render="refresh" />
</head> </head>
<body> <body>
<h1>Deep-Check Results for root SI=<span n:render="root_storage_index" /></h1> <h1>Deep-Check Results for root SI=<span t:render="root_storage_index" /></h1>
<h2 n:render="reload" /> <h2 t:render="reload" />
<p>Counters:</p> <p>Counters:</p>
<ul> <ul>
<li>Objects Checked: <span n:render="data" n:data="objects_checked" /></li> <li>Objects Checked: <span><t:transparent t:render="objects_checked" /></span></li>
<li>Objects Healthy: <span n:render="data" n:data="objects_healthy" /></li> <li>Objects Healthy: <span><t:transparent t:render="objects_healthy" /></span></li>
<li>Objects Unhealthy: <span n:render="data" n:data="objects_unhealthy" /></li> <li>Objects Unhealthy: <span><t:transparent t:render="objects_unhealthy" /></span></li>
<li>Objects Unrecoverable: <span n:render="data" n:data="objects_unrecoverable" /></li> <li>Objects Unrecoverable: <span><t:transparent t:render="objects_unrecoverable" /></span></li>
<li>Corrupt Shares: <span n:render="data" n:data="count_corrupt_shares" /></li> <li>Corrupt Shares: <span><t:transparent t:render="count_corrupt_shares" /></span></li>
</ul> </ul>
<div n:render="problems_p"> <div t:render="problems_p">
<h2>Files/Directories That Had Problems:</h2> <h2>Files/Directories That Had Problems:</h2>
<ul n:render="sequence" n:data="problems"> <ul t:render="problems">
<li n:pattern="item" n:render="problem"/> <li t:render="item">
<li n:pattern="empty">None</li> <t:slot name="problem" />
</li>
<li t:render="empty">None</li>
</ul> </ul>
</div> </div>
<div n:render="servers_with_corrupt_shares_p"> <div t:render="servers_with_corrupt_shares_p">
<h2>Servers on which corrupt shares were found</h2> <h2>Servers on which corrupt shares were found</h2>
<ul n:render="sequence" n:data="servers_with_corrupt_shares"> <ul t:render="servers_with_corrupt_shares">
<li n:pattern="item" n:render="server_problem"/> <li t:render="item">
<li n:pattern="empty">None</li> <t:slot name="problem" />
</li>
<li t:render="empty">None</li>
</ul> </ul>
</div> </div>
<div n:render="corrupt_shares_p"> <div t:render="corrupt_shares_p">
<h2>Corrupt Shares</h2> <h2>Corrupt Shares</h2>
<p>If repair fails, these shares need to be manually inspected and removed.</p> <p>If repair fails, these shares need to be manually inspected and removed.</p>
<table n:render="sequence" n:data="corrupt_shares"> <table t:render="corrupt_shares">
<tr n:pattern="header"> <tr t:render="header">
<td>Server</td> <th>Server</th>
<td>Server Nickname</td> <th>Server Nickname</th>
<td>Storage Index</td> <th>Storage Index</th>
<td>Share Number</td> <th>Share Number</th>
</tr> </tr>
<tr n:pattern="item" n:render="share_problem"> <tr t:render="item">
<td><n:slot name="serverid"/></td> <td><t:slot name="serverid"/></td>
<td><n:slot name="nickname"/></td> <td><t:slot name="nickname"/></td>
<td><n:slot name="si"/></td> <td><t:slot name="si"/></td>
<td><n:slot name="shnum"/></td> <td><t:slot name="shnum"/></td>
</tr> </tr>
</table> </table>
</div> </div>
<div n:render="return" /> <div t:render="return_to" />
<div> <div>
<h2>All Results</h2> <h2>All Results</h2>
<table n:render="sequence" n:data="all_objects"> <table t:render="all_objects">
<tr n:pattern="header"> <tr t:render="header">
<td>Relative Path</td> <th>Relative Path</th>
<td>Healthy</td> <th>Healthy</th>
<td>Recoverable</td> <th>Recoverable</th>
<td>Storage Index</td> <th>Storage Index</th>
<td>Summary</td> <th>Summary</th>
</tr> </tr>
<tr n:pattern="item" n:render="object"> <tr t:render="item">
<td><n:slot name="path"/></td> <td><t:slot name="path"/></td>
<td><n:slot name="healthy"/></td> <td><t:slot name="healthy"/></td>
<td><n:slot name="recoverable"/></td> <td><t:slot name="recoverable"/></td>
<td><tt><n:slot name="storage_index"/></tt></td> <td><tt><t:slot name="storage_index"/></tt></td>
<td><n:slot name="summary"/></td> <td><t:slot name="summary"/></td>
</tr>
<tr t:render="empty">
<td>Nothing to report yet.</td>
</tr> </tr>
</table> </table>
</div> </div>
<div n:render="runtime" /> <div t:render="runtime" />
</body> </body>
</html> </html>

8
src/allmydata/web/info.py
View File

@ -261,7 +261,7 @@ class MoreInfoElement(Element):
@renderer @renderer
def deep_check_form(self, req, tag): def deep_check_form(self, req, tag):
ophandle = base32.b2a(os.urandom(16)) ophandle = base32.b2a(os.urandom(16))
deep_check = T.form(action=".", method="post", deep_check = T.form(action=req.path, method="post",
enctype="multipart/form-data")( enctype="multipart/form-data")(
T.fieldset( T.fieldset(
T.input(type="hidden", name="t", value="start-deep-check"), T.input(type="hidden", name="t", value="start-deep-check"),
@ -287,7 +287,7 @@ class MoreInfoElement(Element):
@renderer @renderer
def deep_size_form(self, req, tag): def deep_size_form(self, req, tag):
ophandle = base32.b2a(os.urandom(16)) ophandle = base32.b2a(os.urandom(16))
deep_size = T.form(action=".", method="post", deep_size = T.form(action=req.path, method="post",
enctype="multipart/form-data")( enctype="multipart/form-data")(
T.fieldset( T.fieldset(
T.input(type="hidden", name="t", value="start-deep-size"), T.input(type="hidden", name="t", value="start-deep-size"),
@ -300,7 +300,7 @@ class MoreInfoElement(Element):
@renderer @renderer
def deep_stats_form(self, req, tag): def deep_stats_form(self, req, tag):
ophandle = base32.b2a(os.urandom(16)) ophandle = base32.b2a(os.urandom(16))
deep_stats = T.form(action=".", method="post", deep_stats = T.form(action=req.path, method="post",
enctype="multipart/form-data")( enctype="multipart/form-data")(
T.fieldset( T.fieldset(
T.input(type="hidden", name="t", value="start-deep-stats"), T.input(type="hidden", name="t", value="start-deep-stats"),
@ -313,7 +313,7 @@ class MoreInfoElement(Element):
@renderer @renderer
def manifest_form(self, req, tag): def manifest_form(self, req, tag):
ophandle = base32.b2a(os.urandom(16)) ophandle = base32.b2a(os.urandom(16))
manifest = T.form(action=".", method="post", manifest = T.form(action=req.path, method="post",
enctype="multipart/form-data")( enctype="multipart/form-data")(
T.fieldset( T.fieldset(
T.input(type="hidden", name="t", value="start-manifest"), T.input(type="hidden", name="t", value="start-manifest"),

4
src/allmydata/web/literal-check-results.xhtml
View File

@ -1,4 +1,4 @@
<html xmlns:n="http://nevow.com/ns/nevow/0.1"> <html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
<head> <head>
<title>Tahoe-LAFS - Check Results</title> <title>Tahoe-LAFS - Check Results</title>
<link href="/tahoe.css" rel="stylesheet" type="text/css"/> <link href="/tahoe.css" rel="stylesheet" type="text/css"/>
@ -11,7 +11,7 @@
<div>Literal files are always healthy: their data is contained in the URI</div> <div>Literal files are always healthy: their data is contained in the URI</div>
<div n:render="return" /> <div t:render="return_to" />
</body> </body>
</html> </html>

5
tox.ini
View File

@ -49,9 +49,8 @@ commands =
tahoe --version tahoe --version
[testenv:py36] [testenv:py36]
# On macOS, git inside of ratchet.sh needs $HOME. commands =
passenv = {[testenv]passenv} HOME trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata.test.python3_tests}
commands = {toxinidir}/misc/python3/ratchet.sh
[testenv:integration] [testenv:integration]
setenv = setenv =