Fri Sep 10 11:35:20 MDT 2010 zooko@zooko.com
* fileutil: copy in the get_disk_stats() and get_available_space() functions from storage/server.py
Fri Sep 10 11:36:29 MDT 2010 zooko@zooko.com
* storage: use fileutil's version of get_disk_stats() and get_available_space(), use mockery/fakery in tests, enable large share test on platforms with sparse files and if > 4 GiB of disk space is currently available
New patches:
[fileutil: copy in the get_disk_stats() and get_available_space() functions from storage/server.py
zooko@zooko.com**20100910173520
Ignore-this: 8b15569715f710f4fc5092f7ca109253
] {
hunk ./src/allmydata/test/test_util.py 506
finally:
os.chdir(saved_cwd)
+ def test_disk_stats(self):
+ avail = fileutil.get_available_space('.', 2**14)
+ if avail == 0:
+ raise unittest.SkipTest("This test will spuriously fail there is no disk space left.")
+
+ disk = fileutil.get_disk_stats('.', 2**13)
+ self.failUnless(disk['total'] > 0, disk['total'])
+ self.failUnless(disk['used'] > 0, disk['used'])
+ self.failUnless(disk['free_for_root'] > 0, disk['free_for_root'])
+ self.failUnless(disk['free_for_nonroot'] > 0, disk['free_for_nonroot'])
+ self.failUnless(disk['avail'] > 0, disk['avail'])
+
+ def test_disk_stats_avail_nonnegative(self):
+ # This test will spuriously fail if you have more than 2^128
+ # bytes of available space on your filesystem.
+ disk = fileutil.get_disk_stats('.', 2**128)
+ self.failUnlessEqual(disk['avail'], 0)
+
class PollMixinTests(unittest.TestCase):
def setUp(self):
self.pm = pollmixin.PollMixin()
hunk ./src/allmydata/util/fileutil.py 308
# there is always at least one Unicode path component.
return os.path.normpath(path)
+windows = False
+try:
+ import win32api, win32con
+except ImportError:
+ pass
+else:
+ windows = True
+ # <http://msdn.microsoft.com/en-us/library/ms680621%28VS.85%29.aspx>
+ win32api.SetErrorMode(win32con.SEM_FAILCRITICALERRORS |
+ win32con.SEM_NOOPENFILEERRORBOX)
+
+def get_disk_stats(whichdir, reserved_space=0):
+ """Return disk statistics for the storage disk, in the form of a dict
+ with the following fields.
+ total: total bytes on disk
+ free_for_root: bytes actually free on disk
+ free_for_nonroot: bytes free for "a non-privileged user" [Unix] or
+ the current user [Windows]; might take into
+ account quotas depending on platform
+ used: bytes used on disk
+ avail: bytes available excluding reserved space
+ An AttributeError can occur if the OS has no API to get disk information.
+ An EnvironmentError can occur if the OS call fails.
+
+ whichdir is a directory on the filesystem in question -- the
+ answer is about the filesystem, not about the directory, so the
+ directory is used only to specify which filesystem.
+
+ reserved_space is how many bytes to subtract from the answer, so
+ you can pass how many bytes you would like to leave unused on this
+ filesystem as reserved_space.
+ """
+
+ if windows:
+ # For Windows systems, where os.statvfs is not available, use GetDiskFreeSpaceEx.
+ # <http://docs.activestate.com/activepython/2.5/pywin32/win32api__GetDiskFreeSpaceEx_meth.html>
+ #
+ # Although the docs say that the argument should be the root directory
+ # of a disk, GetDiskFreeSpaceEx actually accepts any path on that disk
+ # (like its Win32 equivalent).
+
+ (free_for_nonroot, total, free_for_root) = win32api.GetDiskFreeSpaceEx(whichdir)
+ else:
+ # For Unix-like systems.
+ # <http://docs.python.org/library/os.html#os.statvfs>
+ # <http://opengroup.org/onlinepubs/7990989799/xsh/fstatvfs.html>
+ # <http://opengroup.org/onlinepubs/7990989799/xsh/sysstatvfs.h.html>
+ s = os.statvfs(whichdir)
+
+ # on my mac laptop:
+ # statvfs(2) is a wrapper around statfs(2).
+ # statvfs.f_frsize = statfs.f_bsize :
+ # "minimum unit of allocation" (statvfs)
+ # "fundamental file system block size" (statfs)
+ # statvfs.f_bsize = statfs.f_iosize = stat.st_blocks : preferred IO size
+ # on an encrypted home directory ("FileVault"), it gets f_blocks
+ # wrong, and s.f_blocks*s.f_frsize is twice the size of my disk,
+ # but s.f_bavail*s.f_frsize is correct
+
+ total = s.f_frsize * s.f_blocks
+ free_for_root = s.f_frsize * s.f_bfree
+ free_for_nonroot = s.f_frsize * s.f_bavail
+
+ # valid for all platforms:
+ used = total - free_for_root
+ avail = max(free_for_nonroot - reserved_space, 0)
+
+ return { 'total': total, 'free_for_root': free_for_root,
+ 'free_for_nonroot': free_for_nonroot,
+ 'used': used, 'avail': avail, }
+
+def get_available_space(whichdir, reserved_space):
+ """Returns available space for share storage in bytes, or None if no
+ API to get this information is available.
+
+ whichdir is a directory on the filesystem in question -- the
+ answer is about the filesystem, not about the directory, so the
+ directory is used only to specify which filesystem.
+
+ reserved_space is how many bytes to subtract from the answer, so
+ you can pass how many bytes you would like to leave unused on this
+ filesystem as reserved_space.
+ """
+ try:
+ return get_disk_stats(whichdir, reserved_space)['avail']
+ except AttributeError:
+ return None
+ except EnvironmentError:
+ log.msg("OS call to get disk statistics failed")
+ return 0
+
}
[storage: use fileutil's version of get_disk_stats() and get_available_space(), use mockery/fakery in tests, enable large share test on platforms with sparse files and if > 4 GiB of disk space is currently available
zooko@zooko.com**20100910173629
Ignore-this: 1304f1164c661de6d5304f993eb9b27b
]
<
[fileutil: copy in the get_disk_stats() and get_available_space() functions from storage/server.py
zooko@zooko.com**20100910173520
Ignore-this: 8b15569715f710f4fc5092f7ca109253
]
> {
hunk ./src/allmydata/storage/server.py 39
implements(RIStorageServer, IStatsProducer)
name = 'storage'
LeaseCheckerClass = LeaseCheckingCrawler
- windows = False
-
- try:
- import win32api, win32con
- windows = True
- # <http://msdn.microsoft.com/en-us/library/ms680621%28VS.85%29.aspx>
- win32api.SetErrorMode(win32con.SEM_FAILCRITICALERRORS |
- win32con.SEM_NOOPENFILEERRORBOX)
- except ImportError:
- pass
def __init__(self, storedir, nodeid, reserved_space=0,
discard_storage=False, readonly_storage=False,
hunk ./src/allmydata/storage/server.py 153
def _clean_incomplete(self):
fileutil.rm_dir(self.incomingdir)
- def get_disk_stats(self):
- """Return disk statistics for the storage disk, in the form of a dict
- with the following fields.
- total: total bytes on disk
- free_for_root: bytes actually free on disk
- free_for_nonroot: bytes free for "a non-privileged user" [Unix] or
- the current user [Windows]; might take into
- account quotas depending on platform
- used: bytes used on disk
- avail: bytes available excluding reserved space
- An AttributeError can occur if the OS has no API to get disk information.
- An EnvironmentError can occur if the OS call fails."""
-
- if self.windows:
- # For Windows systems, where os.statvfs is not available, use GetDiskFreeSpaceEx.
- # <http://docs.activestate.com/activepython/2.5/pywin32/win32api__GetDiskFreeSpaceEx_meth.html>
- #
- # Although the docs say that the argument should be the root directory
- # of a disk, GetDiskFreeSpaceEx actually accepts any path on that disk
- # (like its Win32 equivalent).
-
- (free_for_nonroot, total, free_for_root) = self.win32api.GetDiskFreeSpaceEx(self.storedir)
- else:
- # For Unix-like systems.
- # <http://docs.python.org/library/os.html#os.statvfs>
- # <http://opengroup.org/onlinepubs/7990989799/xsh/fstatvfs.html>
- # <http://opengroup.org/onlinepubs/7990989799/xsh/sysstatvfs.h.html>
- s = os.statvfs(self.storedir)
-
- # on my mac laptop:
- # statvfs(2) is a wrapper around statfs(2).
- # statvfs.f_frsize = statfs.f_bsize :
- # "minimum unit of allocation" (statvfs)
- # "fundamental file system block size" (statfs)
- # statvfs.f_bsize = statfs.f_iosize = stat.st_blocks : preferred IO size
- # on an encrypted home directory ("FileVault"), it gets f_blocks
- # wrong, and s.f_blocks*s.f_frsize is twice the size of my disk,
- # but s.f_bavail*s.f_frsize is correct
-
- total = s.f_frsize * s.f_blocks
- free_for_root = s.f_frsize * s.f_bfree
- free_for_nonroot = s.f_frsize * s.f_bavail
-
- # valid for all platforms:
- used = total - free_for_root
- avail = max(free_for_nonroot - self.reserved_space, 0)
-
- return { 'total': total, 'free_for_root': free_for_root,
- 'free_for_nonroot': free_for_nonroot,
- 'used': used, 'avail': avail, }
-
def get_stats(self):
# remember: RIStatsProvider requires that our return dict
# contains numeric values.
hunk ./src/allmydata/storage/server.py 163
stats['storage_server.latencies.%s.%s' % (category, name)] = v
try:
- disk = self.get_disk_stats()
+ disk = fileutil.get_disk_stats(self.storedir, self.reserved_space)
writeable = disk['avail'] > 0
# spacetime predictors should use disk_avail / (d(disk_used)/dt)
hunk ./src/allmydata/storage/server.py 195
if self.readonly_storage:
return 0
- try:
- return self.get_disk_stats()['avail']
- except AttributeError:
- return None
- except EnvironmentError:
- log.msg("OS call to get disk statistics failed", level=log.UNUSUAL)
- return 0
+ return fileutil.get_available_space(self.storedir, self.reserved_space)
def allocated_size(self):
space = 0
hunk ./src/allmydata/test/test_storage.py 1
+import time, os.path, platform, stat, re, simplejson, struct
hunk ./src/allmydata/test/test_storage.py 3
-import time, os.path, stat, re, simplejson, struct
+from allmydata.util import log
+
+import mock
from twisted.trial import unittest
hunk ./src/allmydata/test/test_storage.py 233
return self._do_test_readwrite("test_readwrite_v2",
0x44, WriteBucketProxy_v2, ReadBucketProxy)
-class FakeDiskStorageServer(StorageServer):
- DISKAVAIL = 0
- def get_disk_stats(self):
- return { 'free_for_nonroot': self.DISKAVAIL, 'avail': max(self.DISKAVAIL - self.reserved_space, 0), }
-
class Server(unittest.TestCase):
def setUp(self):
hunk ./src/allmydata/test/test_storage.py 266
sharenums, size, canary)
def test_large_share(self):
+ syslow = platform.system().lower()
+ if 'cygwin' in syslow or 'windows' in syslow or 'darwin' in syslow:
+ raise unittest.SkipTest("If your filesystem doesn't support efficient sparse files then it is very expensive (Mac OS X and Windows don't support efficient sparse files).")
+
+ avail = fileutil.get_available_space('.', 2**14)
+ if avail <= 4*2**30:
+ raise unittest.SkipTest("This test will spuriously fail if you have less than 4 GiB free on your filesystem.")
+
ss = self.create("test_large_share")
already,writers = self.allocate(ss, "allocate", [0], 2**32+2)
hunk ./src/allmydata/test/test_storage.py 288
readers = ss.remote_get_buckets("allocate")
reader = readers[shnum]
self.failUnlessEqual(reader.remote_read(2**32, 2), "ab")
- test_large_share.skip = "This test can spuriously fail if you have less than 4 GiB free on your filesystem, and if your filesystem doesn't support efficient sparse files then it is very expensive (Mac OS X and Windows don't support efficient sparse files)."
def test_dont_overfill_dirs(self):
"""
hunk ./src/allmydata/test/test_storage.py 431
self.failUnlessEqual(already, set())
self.failUnlessEqual(set(writers.keys()), set([0,1,2]))
- def test_reserved_space(self):
- ss = self.create("test_reserved_space", reserved_space=10000,
- klass=FakeDiskStorageServer)
- # the FakeDiskStorageServer doesn't do real calls to get_disk_stats
- ss.DISKAVAIL = 15000
+ @mock.patch('allmydata.util.fileutil.get_disk_stats')
+ def test_reserved_space(self, mock_get_disk_stats):
+ reserved_space=10000
+ mock_get_disk_stats.return_value = {
+ 'free_for_nonroot': 15000,
+ 'avail': max(15000 - reserved_space, 0),
+ }
+
+ ss = self.create("test_reserved_space", reserved_space=reserved_space)
# 15k available, 10k reserved, leaves 5k for shares
# a newly created and filled share incurs this much overhead, beyond
hunk ./src/allmydata/test/test_storage.py 478
allocated = 1001 + OVERHEAD + LEASE_SIZE
- # we have to manually increase DISKAVAIL, since we're not doing real
+ # we have to manually increase available, since we're not doing real
# disk measurements
hunk ./src/allmydata/test/test_storage.py 480
- ss.DISKAVAIL -= allocated
+ mock_get_disk_stats.return_value = {
+ 'free_for_nonroot': 15000 - allocated,
+ 'avail': max(15000 - allocated - reserved_space, 0),
+ }
# now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and
# 5000-1085=3915 free, therefore we can fit 39 100byte shares
hunk ./src/allmydata/test/test_storage.py 497
ss.disownServiceParent()
del ss
- def test_disk_stats(self):
- # This will spuriously fail if there is zero disk space left (but so will other tests).
- ss = self.create("test_disk_stats", reserved_space=0)
-
- disk = ss.get_disk_stats()
- self.failUnless(disk['total'] > 0, disk['total'])
- self.failUnless(disk['used'] > 0, disk['used'])
- self.failUnless(disk['free_for_root'] > 0, disk['free_for_root'])
- self.failUnless(disk['free_for_nonroot'] > 0, disk['free_for_nonroot'])
- self.failUnless(disk['avail'] > 0, disk['avail'])
-
- def test_disk_stats_avail_nonnegative(self):
- ss = self.create("test_disk_stats_avail_nonnegative", reserved_space=2**64)
-
- disk = ss.get_disk_stats()
- self.failUnlessEqual(disk['avail'], 0)
-
def test_seek(self):
basedir = self.workdir("test_seek_behavior")
fileutil.make_dirs(basedir)
hunk ./src/allmydata/test/test_storage.py 2459
d = self.render1(page, args={"t": ["json"]})
return d
-class NoDiskStatsServer(StorageServer):
- def get_disk_stats(self):
- raise AttributeError
-
-class BadDiskStatsServer(StorageServer):
- def get_disk_stats(self):
- raise OSError
-
class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
def setUp(self):
hunk ./src/allmydata/test/test_storage.py 2500
d = self.render1(page, args={"t": ["json"]})
return d
- def test_status_no_disk_stats(self):
+ @mock.patch('allmydata.util.fileutil.get_disk_stats')
+ def test_status_no_disk_stats(self, mock_get_disk_stats):
+ mock_get_disk_stats.side_effect = AttributeError()
+
# Some platforms may have no disk stats API. Make sure the code can handle that
# (test runs on all platforms).
basedir = "storage/WebStatus/status_no_disk_stats"
hunk ./src/allmydata/test/test_storage.py 2508
fileutil.make_dirs(basedir)
- ss = NoDiskStatsServer(basedir, "\x00" * 20)
+ ss = StorageServer(basedir, "\x00" * 20)
ss.setServiceParent(self.s)
w = StorageStatus(ss)
html = w.renderSynchronously()
hunk ./src/allmydata/test/test_storage.py 2519
self.failUnlessIn("Space Available to Tahoe: ?", s)
self.failUnless(ss.get_available_space() is None)
- def test_status_bad_disk_stats(self):
+ @mock.patch('allmydata.util.fileutil.get_disk_stats')
+ def test_status_bad_disk_stats(self, mock_get_disk_stats):
+ mock_get_disk_stats.side_effect = OSError()
+
# If the API to get disk stats exists but a call to it fails, then the status should
# show that no shares will be accepted, and get_available_space() should be 0.
basedir = "storage/WebStatus/status_bad_disk_stats"
hunk ./src/allmydata/test/test_storage.py 2527
fileutil.make_dirs(basedir)
- ss = BadDiskStatsServer(basedir, "\x00" * 20)
+ ss = StorageServer(basedir, "\x00" * 20)
ss.setServiceParent(self.s)
w = StorageStatus(ss)
html = w.renderSynchronously()
hunk ./src/allmydata/test/test_storage.py 2579
self.failUnlessEqual(w.render_abbrev_space(None, 10e6), "10.00 MB")
self.failUnlessEqual(remove_prefix("foo.bar", "foo."), "bar")
self.failUnlessEqual(remove_prefix("foo.bar", "baz."), None)
-
hunk ./src/allmydata/test/test_system.py 1784
d.addCallback(_got_lit_filenode)
return d
-
}
Context:
[test: make tests stop relying on pyutil version class accepting the string 'unknown' for its version, and make them forward-compatible with the future Python Rational Version Numbering standard
zooko@zooko.com**20100910154135
Ignore-this: d051b071f33595493be5df218f5015a6
]
[setup: copy in this fix from zetuptoolz and the accompanying new version number of zetuptoolz: http://tahoe-lafs.org/trac/zetuptoolz/ticket/1
zooko@zooko.com**20100910061411
Ignore-this: cb0ddce66b2a71666df3e22375fa581a
]
[immutable download: have the finder inform its share consumer "no more shares" in a subsequent tick, thus avoiding accidentally telling it "no more shares" now and then telling it "here's another share" in a subsequent tick
Brian Warner <warner@lothar.com>**20100910043038
Ignore-this: 47595fb2b87867d3d75695d51344c484
fixes #1191
Patch by Brian. This patch description was actually written by Zooko, but I forged Brian's name on the "author" field so that he would get credit for this patch in revision control history.
]
[immutable downloader: add a test specifically of whether the finder sometimes announces "no more shares ever" and then announces a newly found share
zooko@zooko.com**20100909041654
Ignore-this: ec0d5febc499f974b167465290770abd
(The current code fails this test, ref #1191.)
]
[docs/frontends/FTP-and-SFTP.txt : ftpd and sftpd doesn't listen on loopback interface only
marc.doudiet@nimag.net**20100813140853
Ignore-this: 5b5dfd0e5991a2669fe41ba13ea21bd4
]
[tests: assign the storage servers to a fixed order which triggers a bug in new downloader every time this test is run (formerly this test would detect the bug in new-downloader only sporadically)
zooko@zooko.com**20100904041515
Ignore-this: 33155dcc03e84217ec5541addd3a16fc
If you are investigating the bug in new-downloader, one way to investigate might be to change this ordering to a different fixed order (e.g. rotate by 4 instead of rotate by 5) and observe how the behavior of new-downloader differs in that case.
]
[TAG allmydata-tahoe-1.8.0c3
zooko@zooko.com**20100902212140
Ignore-this: e4550de37f57e5c1a591e549a104565d
]
Patch bundle hash:
c8607a41f8ddd2d945ae8ed6d309413d178c6c84