working test of 'in the wild' data, working converters
This commit is contained in:
parent
bf5e682d71
commit
89c2aacadc
|
|
@ -34,7 +34,7 @@ def _convert_pickle_state_to_json(state):
|
|||
:return dict: the state in the JSON form
|
||||
"""
|
||||
# ["cycle-to-date"]["corrupt-shares"] from 2-tuple to list
|
||||
# ["leases-per-share-histogram"] gets str keys instead of int
|
||||
# ["cycle-to-date"]["leases-per-share-histogram"] gets str keys instead of int
|
||||
# ["cycle-start-finish-times"] from 2-tuple to list
|
||||
# ["configured-expiration-mode"] from 4-tuple to list
|
||||
# ["history"] keys are strings
|
||||
|
|
@ -43,12 +43,6 @@ def _convert_pickle_state_to_json(state):
|
|||
"Unknown version {version} in pickle state".format(**state)
|
||||
)
|
||||
|
||||
def convert_lpsh(value):
|
||||
return {
|
||||
str(k): v
|
||||
for k, v in value.items()
|
||||
}
|
||||
|
||||
def convert_cem(value):
|
||||
# original is a 4-tuple, with the last element being a 2-tuple
|
||||
# .. convert both to lists
|
||||
|
|
@ -59,20 +53,28 @@ def _convert_pickle_state_to_json(state):
|
|||
list(value[3]),
|
||||
]
|
||||
|
||||
def convert_history(value):
|
||||
print("convert history")
|
||||
print(value)
|
||||
def convert_ctd(value):
|
||||
ctd_converter = {
|
||||
"lease-age-histogram": lambda value: {
|
||||
"{},{}".format(k[0], k[1]): v
|
||||
for k, v in value.items()
|
||||
},
|
||||
"corrupt-shares": lambda value: [
|
||||
list(x)
|
||||
for x in value
|
||||
],
|
||||
}
|
||||
return {
|
||||
str(k): v
|
||||
for k, v in value
|
||||
k: ctd_converter.get(k, lambda z: z)(v)
|
||||
for k, v in value.items()
|
||||
}
|
||||
|
||||
# we don't convert "history" here because that's in a separate
|
||||
# file; see expirer.py
|
||||
converters = {
|
||||
"cycle-to-date": list,
|
||||
"leases-per-share-histogram": convert_lpsh,
|
||||
"cycle-to-date": convert_ctd,
|
||||
"cycle-starte-finish-times": list,
|
||||
"configured-expiration-mode": convert_cem,
|
||||
"history": convert_history,
|
||||
}
|
||||
|
||||
def convert_value(key, value):
|
||||
|
|
@ -116,10 +118,10 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle):
|
|||
import pickle
|
||||
with state_path.open("r") as f:
|
||||
state = pickle.load(f)
|
||||
state = convert_pickle(state)
|
||||
new_state = convert_pickle(state)
|
||||
json_state_path = state_path.siblingExtension(".json")
|
||||
with json_state_path.open("w") as f:
|
||||
json.dump(state, f)
|
||||
json.dump(new_state, f)
|
||||
# we've written the JSON, delete the pickle
|
||||
state_path.remove()
|
||||
return json_state_path.path
|
||||
|
|
|
|||
|
|
@ -1160,6 +1160,171 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
|
|||
self.assertNot(test_pickle.exists())
|
||||
self.assertTrue(test_pickle.siblingExtension(".json").exists())
|
||||
|
||||
self.assertEqual(
|
||||
serial.load(),
|
||||
{
|
||||
u'last-complete-prefix': None,
|
||||
u'version': 1,
|
||||
u'current-cycle-start-time': 1635003106.611748,
|
||||
u'last-cycle-finished': 312,
|
||||
u'cycle-to-date': {
|
||||
u'leases-per-share-histogram': {
|
||||
u'1': 36793,
|
||||
u'2': 1,
|
||||
},
|
||||
u'space-recovered': {
|
||||
u'examined-buckets-immutable': 17183,
|
||||
u'configured-buckets-mutable': 0,
|
||||
u'examined-shares-mutable': 1796,
|
||||
u'original-shares-mutable': 1563,
|
||||
u'configured-buckets-immutable': 0,
|
||||
u'original-shares-immutable': 27926,
|
||||
u'original-diskbytes-immutable': 431149056,
|
||||
u'examined-shares-immutable': 34998,
|
||||
u'original-buckets': 14661,
|
||||
u'actual-shares-immutable': 0,
|
||||
u'configured-shares': 0,
|
||||
u'original-buckets-mutable': 899,
|
||||
u'actual-diskbytes': 4096,
|
||||
u'actual-shares-mutable': 0,
|
||||
u'configured-buckets': 1,
|
||||
u'examined-buckets-unknown': 14,
|
||||
u'actual-sharebytes': 0,
|
||||
u'original-shares': 29489,
|
||||
u'actual-buckets-immutable': 0,
|
||||
u'original-sharebytes': 312664812,
|
||||
u'examined-sharebytes-immutable': 383801602,
|
||||
u'actual-shares': 0,
|
||||
u'actual-sharebytes-immutable': 0,
|
||||
u'original-diskbytes': 441643008,
|
||||
u'configured-diskbytes-mutable': 0,
|
||||
u'configured-sharebytes-immutable': 0,
|
||||
u'configured-shares-mutable': 0,
|
||||
u'actual-diskbytes-immutable': 0,
|
||||
u'configured-diskbytes-immutable': 0,
|
||||
u'original-diskbytes-mutable': 10489856,
|
||||
u'actual-sharebytes-mutable': 0,
|
||||
u'configured-sharebytes': 0,
|
||||
u'examined-shares': 36794,
|
||||
u'actual-diskbytes-mutable': 0,
|
||||
u'actual-buckets': 1,
|
||||
u'original-buckets-immutable': 13761,
|
||||
u'configured-sharebytes-mutable': 0,
|
||||
u'examined-sharebytes': 390369660,
|
||||
u'original-sharebytes-immutable': 308125753,
|
||||
u'original-sharebytes-mutable': 4539059,
|
||||
u'actual-buckets-mutable': 0,
|
||||
u'examined-buckets-mutable': 1043,
|
||||
u'configured-shares-immutable': 0,
|
||||
u'examined-diskbytes': 476598272,
|
||||
u'examined-diskbytes-mutable': 9154560,
|
||||
u'examined-sharebytes-mutable': 6568058,
|
||||
u'examined-buckets': 18241,
|
||||
u'configured-diskbytes': 4096,
|
||||
u'examined-diskbytes-immutable': 467443712},
|
||||
u'corrupt-shares': [
|
||||
[u'2dn6xnlnsqwtnapwxfdivpm3s4', 4],
|
||||
[u'2dn6xnlnsqwtnapwxfdivpm3s4', 1],
|
||||
[u'2rrzthwsrrxolevmwdvbdy3rqi', 4],
|
||||
[u'2rrzthwsrrxolevmwdvbdy3rqi', 1],
|
||||
[u'2skfngcto6h7eqmn4uo7ntk3ne', 4],
|
||||
[u'2skfngcto6h7eqmn4uo7ntk3ne', 1],
|
||||
[u'32d5swqpqx2mwix7xmqzvhdwje', 4],
|
||||
[u'32d5swqpqx2mwix7xmqzvhdwje', 1],
|
||||
[u'5mmayp66yflmpon3o6unsnbaca', 4],
|
||||
[u'5mmayp66yflmpon3o6unsnbaca', 1],
|
||||
[u'6ixhpvbtre7fnrl6pehlrlflc4', 4],
|
||||
[u'6ixhpvbtre7fnrl6pehlrlflc4', 1],
|
||||
[u'ewzhvswjsz4vp2bqkb6mi3bz2u', 4],
|
||||
[u'ewzhvswjsz4vp2bqkb6mi3bz2u', 1],
|
||||
[u'fu7pazf6ogavkqj6z4q5qqex3u', 4],
|
||||
[u'fu7pazf6ogavkqj6z4q5qqex3u', 1],
|
||||
[u'hbyjtqvpcimwxiyqbcbbdn2i4a', 4],
|
||||
[u'hbyjtqvpcimwxiyqbcbbdn2i4a', 1],
|
||||
[u'pmcjbdkbjdl26k3e6yja77femq', 4],
|
||||
[u'pmcjbdkbjdl26k3e6yja77femq', 1],
|
||||
[u'r6swof4v2uttbiiqwj5pi32cm4', 4],
|
||||
[u'r6swof4v2uttbiiqwj5pi32cm4', 1],
|
||||
[u't45v5akoktf53evc2fi6gwnv6y', 4],
|
||||
[u't45v5akoktf53evc2fi6gwnv6y', 1],
|
||||
[u'y6zb4faar3rdvn3e6pfg4wlotm', 4],
|
||||
[u'y6zb4faar3rdvn3e6pfg4wlotm', 1],
|
||||
[u'z3yghutvqoqbchjao4lndnrh3a', 4],
|
||||
[u'z3yghutvqoqbchjao4lndnrh3a', 1],
|
||||
],
|
||||
u'lease-age-histogram': {
|
||||
"1641600,1728000": 78,
|
||||
"12441600,12528000": 78,
|
||||
"8640000,8726400": 32,
|
||||
"1814400,1900800": 1860,
|
||||
"2764800,2851200": 76,
|
||||
"11491200,11577600": 20,
|
||||
"10713600,10800000": 183,
|
||||
"47865600,47952000": 7,
|
||||
"3110400,3196800": 328,
|
||||
"10627200,10713600": 43,
|
||||
"45619200,45705600": 4,
|
||||
"12873600,12960000": 5,
|
||||
"7430400,7516800": 7228,
|
||||
"1555200,1641600": 492,
|
||||
"38880000,38966400": 3,
|
||||
"12528000,12614400": 193,
|
||||
"7344000,7430400": 12689,
|
||||
"2678400,2764800": 278,
|
||||
"2332800,2419200": 12,
|
||||
"9244800,9331200": 73,
|
||||
"12787200,12873600": 218,
|
||||
"49075200,49161600": 19,
|
||||
"10368000,10454400": 117,
|
||||
"4665600,4752000": 256,
|
||||
"7516800,7603200": 993,
|
||||
"42336000,42422400": 33,
|
||||
"10972800,11059200": 122,
|
||||
"39052800,39139200": 51,
|
||||
"12614400,12700800": 210,
|
||||
"7603200,7689600": 2004,
|
||||
"10540800,10627200": 16,
|
||||
"950400,1036800": 4435,
|
||||
"42076800,42163200": 4,
|
||||
"8812800,8899200": 57,
|
||||
"5788800,5875200": 954,
|
||||
"36374400,36460800": 3,
|
||||
"9331200,9417600": 12,
|
||||
"30499200,30585600": 5,
|
||||
"12700800,12787200": 25,
|
||||
"2073600,2160000": 388,
|
||||
"12960000,13046400": 8,
|
||||
"11923200,12009600": 89,
|
||||
"3369600,3456000": 79,
|
||||
"3196800,3283200": 628,
|
||||
"37497600,37584000": 11,
|
||||
"33436800,33523200": 7,
|
||||
"44928000,45014400": 2,
|
||||
"37929600,38016000": 3,
|
||||
"38966400,39052800": 61,
|
||||
"3283200,3369600": 86,
|
||||
"11750400,11836800": 7,
|
||||
"3801600,3888000": 32,
|
||||
"46310400,46396800": 1,
|
||||
"4838400,4924800": 386,
|
||||
"8208000,8294400": 38,
|
||||
"37411200,37497600": 4,
|
||||
"12009600,12096000": 329,
|
||||
"10454400,10540800": 1239,
|
||||
"40176000,40262400": 1,
|
||||
"3715200,3801600": 104,
|
||||
"44409600,44496000": 13,
|
||||
"38361600,38448000": 5,
|
||||
"12268800,12355200": 2,
|
||||
"28771200,28857600": 6,
|
||||
"41990400,42076800": 10,
|
||||
"2592000,2678400": 40,
|
||||
},
|
||||
},
|
||||
'current-cycle': None,
|
||||
'last-complete-bucket': None,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class WebStatus(unittest.TestCase, pollmixin.PollMixin):
|
||||
|
|
|
|||
Loading…
Reference in New Issue