Created
March 5, 2012 15:23
-
-
Save ulope/1978730 to your computer and use it in GitHub Desktop.
FS Stressing (my use case: Provoking ZFS on Linux hang bugs)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# Copyright (c) 2012 Ulrich Petri <mail@ulo.pe> | |
# | |
# Permission is hereby granted, free of charge, to any person obtaining a copy | |
# of this software and associated documentation files (the "Software"), to deal | |
# in the Software without restriction, including without limitation the rights | |
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |
# copies of the Software, and to permit persons to whom the Software is | |
# furnished to do so, subject to the following conditions: | |
# | |
# The above copyright notice and this permission notice shall be included in all | |
# copies or substantial portions of the Software. | |
# | |
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
# SOFTWARE. | |
""" | |
fs_stress.py | |
This script uses dd to generate read and write load on the path specified as | |
first command line argument. | |
WARNING: There are no checks for free space left on the device. This script | |
will keep going indefinitely. | |
Abort with Ctrl-C | |
Usage: ./fs_stress.py /path/to/test/directory | |
""" | |
from itertools import cycle | |
import os | |
import random | |
import sys | |
from tempfile import NamedTemporaryFile | |
from threading import Thread | |
from subprocess import call, Popen, PIPE, CalledProcessError | |
import time | |
import threading | |
try: | |
from subprocess import check_output | |
except ImportError: | |
def check_output(*popenargs, **kwargs): | |
r"""Run command with arguments and return its output as a byte string. | |
Backported from Python 2.7 as it's implemented as pure python on stdlib. | |
""" | |
process = Popen(stdout=PIPE, *popenargs, **kwargs) | |
output, unused_err = process.communicate() | |
retcode = process.poll() | |
if retcode: | |
cmd = kwargs.get("args") | |
if cmd is None: | |
cmd = popenargs[0] | |
error = CalledProcessError(retcode, cmd) | |
error.output = output | |
raise error | |
return output | |
DD_PATH = check_output("which dd", shell=True) | |
if not DD_PATH: | |
print "This utility needs the 'dd' command. Something seems to be wrong on your system." | |
sys.exit(1) | |
DD_CMD_TEMPLATE = ("%s if=%%(source)s of=%%(target)s" % (DD_PATH,)).split() | |
DD_COUNT_PART = "count=%(count)d" | |
SPINNER_CHARS = "-\|/" | |
# check if we've got a GNU dd | |
if call("dd --version > /dev/null 2>&1", shell=True) > 0: | |
DD_CMD_TEMPLATE.append("bs=1m") | |
else: | |
DD_CMD_TEMPLATE.append("bs=1M") | |
class SourceGatherer(Thread): | |
limit = 9 | |
def __init__(self, base_path, **kwargs): | |
super(SourceGatherer, self).__init__(**kwargs) | |
self.daemon = True | |
self.base_path = base_path | |
self.sources = [] | |
self.name = self.__class__.__name__ | |
def run(self): | |
while True: | |
if len(self.sources) > self.limit: | |
time.sleep(.125) | |
continue | |
dir2 = None | |
p1 = "" | |
while not dir2: | |
path1 = "" | |
while not os.path.isdir(path1): | |
dir = None | |
while not dir: | |
dir = os.listdir(self.base_path) | |
time.sleep(.0625) | |
p1 = random.choice(dir) | |
path1 = os.path.join(self.base_path, p1) | |
dir2 = os.listdir(path1) | |
p2 = random.choice(dir2) | |
self.sources.append(os.path.join(self.base_path, p1, p2)) | |
def get_source(self): | |
while not len(self.sources): | |
time.sleep(0.125) | |
return self.sources.pop() | |
class StresserBase(Thread): | |
count = None | |
source = None | |
target = None | |
mode = "?" | |
def __init__(self, base_path, **kwargs): | |
super(StresserBase, self).__init__(**kwargs) | |
self.base_path = base_path | |
self.name = self.__class__.__name__ | |
def run(self): | |
cmd = DD_CMD_TEMPLATE[:] | |
if self.count: | |
cmd.append(DD_COUNT_PART) | |
params = dict( | |
source=self.source, | |
target=self.target, | |
count=self.count, | |
mode=self.mode | |
) | |
call([part % params for part in cmd], stderr=PIPE) | |
class WriteStresserBase(StresserBase): | |
_target = None | |
mode = "W" | |
@property | |
def target(self): | |
if not self._target: | |
path = os.path.join(self.base_path, "%02d" % (random.randint(1, 99),)) | |
if not os.path.exists(path): | |
os.mkdir(path) | |
tmpfile = NamedTemporaryFile( | |
dir=path, | |
delete=False | |
) | |
self._target = tmpfile.name | |
tmpfile.close() | |
return self._target | |
class ReadStresser(StresserBase): | |
target = "/dev/null" | |
mode = "R" | |
def __init__(self, base_path, source, **kwargs): | |
super(ReadStresser, self).__init__(base_path, **kwargs) | |
self.source = source | |
class BigWriteStresser(WriteStresserBase): | |
count = 1024 | |
source = "/dev/zero" | |
class SmallWriteStresser(WriteStresserBase): | |
count = 32 | |
source = "/dev/urandom" | |
class Stresser(object): | |
def __init__(self, cls, arg_provider=None): | |
self.cls = cls | |
self.arg_provider = arg_provider | |
self.instance = None | |
self.spinner = cycle(SPINNER_CHARS) | |
self.start_time = 0 | |
self.mb = 0 | |
self.iteration = 0 | |
def update(self, should_exit=False): | |
if not self.instance and not should_exit: | |
if self.arg_provider: | |
self.instance = self.cls(*self.arg_provider()) | |
else: | |
self.instance = self.cls(sys.argv[1]) | |
self.start_time = time.time() | |
self.iteration += 1 | |
self.instance.start() | |
if self.instance: | |
if self.instance.is_alive(): | |
return "[%s: %04d: %s GB] %s (%02d s) %s => %s" % ( | |
self.instance.mode, | |
self.iteration, | |
"%07.2f" % (self.mb / 1024.0) if self.instance.mode == "W" else " - ", | |
self.spinner.next(), | |
time.time() - self.start_time, | |
self.instance.source, | |
self.instance.target | |
) | |
else: | |
if self.instance.mode == "W" and self.instance.count: | |
self.mb += self.instance.count | |
self.instance = None | |
self.start_time = 0 | |
return "[-: -]" | |
def thread_count(): | |
return len([t for t in threading.enumerate() if not t.daemon and t.name != "MainThread"]) | |
def main(): | |
if len(sys.argv) < 2: | |
print "Usage: %s BASE_PATH" % (os.path.basename(sys.argv[0]),) | |
sys.exit(1) | |
sg = SourceGatherer(sys.argv[1]) | |
sg.start() | |
reader_args = lambda: (sys.argv[1], sg.get_source()) | |
stressers = ( | |
Stresser(BigWriteStresser), | |
Stresser(SmallWriteStresser), | |
Stresser(SmallWriteStresser), | |
Stresser(SmallWriteStresser), | |
Stresser(ReadStresser, reader_args), | |
Stresser(ReadStresser, reader_args), | |
Stresser(ReadStresser, reader_args), | |
Stresser(ReadStresser, reader_args), | |
Stresser(ReadStresser, reader_args), | |
) | |
should_exit = False | |
while not should_exit or (should_exit and thread_count() != 0): | |
try: | |
out = [] | |
for stresser in stressers: | |
out.append(stresser.update(should_exit)) | |
os.system("clear") | |
print "\n".join(out) | |
print "Threads running: %02d" % (thread_count(), ) | |
if should_exit: | |
print "Ctrl-C pressed. Waiting for threads to finish...\nPress again to force quit." | |
time.sleep(.0625) | |
except KeyboardInterrupt: | |
if should_exit: | |
# force quit | |
os._exit(1) | |
should_exit = True | |
if __name__ == "__main__": | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment