Last active
August 29, 2015 14:19
-
-
Save junhe/2a7ab13c05fd029a3189 to your computer and use it in GitHub Desktop.
create ext4 on loop dev
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
import itertools | |
import random | |
import argparse | |
import re | |
import subprocess | |
import os | |
import sys | |
import shlex | |
import time | |
import glob | |
from time import localtime, strftime | |
def shcmd(cmd, ignore_error=False): | |
print 'Doing:', cmd | |
ret = subprocess.call(cmd, shell=True) | |
print 'Returned', ret, cmd | |
if ignore_error == False and ret != 0: | |
exit(ret) | |
return ret | |
class cd: | |
"""Context manager for changing the current working directory""" | |
def __init__(self, newPath): | |
self.newPath = newPath | |
def __enter__(self): | |
self.savedPath = os.getcwd() | |
os.chdir(self.newPath) | |
def __exit__(self, etype, value, traceback): | |
os.chdir(self.savedPath) | |
########################################################### | |
########################################################### | |
def compile_linux(): | |
#with cd("/mnt/scratch-sda4/"): | |
#shcmd("git clone git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git") | |
with cd("/mnt/scratch-sda4/linux"): | |
#shcmd("cp -vi /boot/config-`uname -r` .config") | |
#shcmd("yes ''| make oldconfig") | |
#shcmd("sudo apt-get install -y git-core libncurses5 libncurses5-dev libelf-dev asciidoc binutils-dev linux-source qt3-dev-tools libqt3-mt-dev libncurses5 libncurses5-dev fakeroot build-essential crash kexec-tools makedumpfile kernel-wedge kernel-package") | |
#shcmd("make menuconfig") | |
#shcmd("make -j3") | |
shcmd("sudo make INSTALL_MOD_PATH=/mnt/scratch-sda4/ modules_install") | |
shcmd("sudo make install") | |
def mkLoopDevOnFile(devname, filepath): | |
cmd = ['losetup', devname, filepath] | |
cmd = [str(x) for x in cmd] | |
print " ".join(cmd), "......" | |
proc = subprocess.Popen(cmd) | |
proc.wait() | |
return proc.returncode | |
def delLoopDev(devname): | |
cmd = ['losetup', '-d', devname] | |
cmd = [str(x) for x in cmd] | |
print " ".join(cmd), "......" | |
proc = subprocess.Popen(cmd) | |
proc.wait() | |
return proc.returncode | |
def isMounted(name): | |
"only check is a name is in mounted list" | |
name = name.rstrip('/') | |
print "isMounted: name:", name | |
with open('/etc/mtab', 'r') as f: | |
for line in f: | |
#print "line:", line, | |
line = " " + line + " " # a hack | |
if re.search(r'\s'+name+r'\s', line): | |
#print " YES" | |
return True | |
#print " NO" | |
return False | |
def isLoopDevUsed(path): | |
cmd = ['losetup','-f'] | |
proc = subprocess.Popen(cmd, | |
stdout=subprocess.PIPE) | |
proc.wait() | |
outstr = proc.communicate()[0] | |
outstr = outstr.strip() | |
print "isLoopUsed:", outstr+"END" | |
if outstr > path: | |
return True | |
else: | |
return False | |
def umountFS(mountpoint): | |
cmd = ["umount", mountpoint] | |
p = subprocess.Popen(cmd) | |
p.wait() | |
print "umountFS:", p.returncode | |
return p.returncode | |
def makeLoopDevice(devname, tmpfs_mountpoint, sizeMB, img_file=None): | |
"size is in MB. The tmpfs for this device might be bigger than sizeMB" | |
if not devname.startswith('/dev/loop'): | |
print 'you are requesting to create loop device on a non-loop device path' | |
exit(1) | |
if not os.path.exists(tmpfs_mountpoint): | |
os.makedirs(tmpfs_mountpoint) | |
# umount the FS mounted on loop dev | |
if isMounted(devname): | |
if umountFS(devname) != 0: | |
print "unable to umount", devname | |
exit(1) | |
else: | |
print devname, 'umounted' | |
else: | |
print devname, "is not mounted" | |
# delete the loop device | |
if isLoopDevUsed(devname): | |
if delLoopDev(devname) != 0: | |
print "Failed to delete loop device" | |
exit(1) | |
else: | |
print devname, "is not in use" | |
# umount the tmpfs the loop device is on | |
if isMounted(tmpfs_mountpoint): | |
if umountFS(tmpfs_mountpoint) != 0: | |
print "unable to umount tmpfs at", tmpfs_mountpoint | |
exit(1) | |
print tmpfs_mountpoint, "umounted" | |
else: | |
print tmpfs_mountpoint, "is not mounted" | |
mountTmpfs(tmpfs_mountpoint, int(sizeMB*1024*1024*1.1)) | |
imgpath = os.path.join(tmpfs_mountpoint, "disk.img") | |
if img_file == None: | |
mkImageFile(imgpath, sizeMB) | |
else: | |
cmd = ['cp', img_file, imgpath] | |
print 'doing...', cmd | |
subprocess.call(cmd) | |
mkLoopDevOnFile(devname, imgpath) | |
def mkImageFile(filepath, size): | |
"size is in MB" | |
#cmd = ['dd', 'if=/dev/zero', 'of='+filepath, | |
#'bs=1M', 'count='+str(size)] | |
cmd = ['truncate', '-s', str(size*1024*1024), filepath] | |
print " ".join(cmd), "......" | |
proc = subprocess.Popen(cmd) | |
proc.wait() | |
return proc.returncode | |
def mountTmpfs(mountpoint, size): | |
if not os.path.exists(mountpoint): | |
os.makedirs(mountpoint) | |
cmd = ['mount', '-t', 'tmpfs', | |
'-o', 'size='+str(size), 'tmpfs', mountpoint] | |
cmd = [str(x) for x in cmd] | |
print " ".join(cmd), "......" | |
proc = subprocess.Popen(cmd) | |
proc.wait() | |
return proc.returncode | |
def mountExt4(devname, mountpoint): | |
if not os.path.exists(mountpoint): | |
os.makedirs(mountpoint) | |
cmd = ["mount", "-t", "ext4", devname, mountpoint] | |
p = subprocess.Popen(cmd) | |
p.wait() | |
print "mountExt4:", p.returncode | |
return p.returncode | |
def makeExt4(devname, blocksize=4096, makeopts=None): | |
if makeopts == None: | |
cmd = ["mkfs.ext4", | |
"-b", blocksize, | |
"-O", "^has_journal,extent,huge_file,flex_bg,uninit_bg,dir_nlink,extra_isize", | |
devname] | |
else: | |
cmd = ["mkfs.ext4", | |
"-b", blocksize] | |
cmd.extend(makeopts) | |
cmd.extend([devname]) | |
cmd = [str(x) for x in cmd] | |
p = subprocess.Popen(cmd) | |
p.wait() | |
print "makeExt4:", p.returncode | |
return p.returncode | |
def create_ext4_on_loop(): | |
makeLoopDevice("/dev/loop0", "/mnt/tmpfs", 4096, img_file=None) | |
makeExt4("/dev/loop0", blocksize=4096, makeopts=None) | |
mountExt4(devname="/dev/loop0", mountpoint="/mnt/ext4onloop") | |
def create_ext4(): | |
#makeLoopDevice("/dev/loop0", "/mnt/tmpfs", 4096, img_file=None) | |
#makeLoopDevice("/dev/loop1", "/mnt/tmpfs1", 4096, img_file=None) | |
makeExt4("/dev/sdb1", blocksize=4096, makeopts=None) | |
makeExt4("/dev/sdb2", blocksize=4096, makeopts=None) | |
mountExt4(devname="/dev/sdb1", mountpoint="/mnt/ext4a") | |
mountExt4(devname="/dev/sdb2", mountpoint="/mnt/ext4b") | |
shcmd("sudo chown jhe:FSPerfAtScale -R /mnt/ext4a") | |
shcmd("sudo chown jhe:FSPerfAtScale -R /mnt/ext4b") | |
def xfstest_install_packages(): | |
shcmd("sudo apt-get install -y xfslibs-dev uuid-dev libtool e2fsprogs automake gcc libuuid1 quota attr libattr1-dev libacl1-dev libaio-dev xfsprogs libgdbm-dev gawk fio dbench") | |
shcmd("git clone git://oss.sgi.com/xfs/cmds/xfstests") | |
with cd("xfstests"): | |
shcmd("make") | |
shcmd("make install") | |
shcmd("sudo useradd fsgqa") | |
def run_xfstests(): | |
"you have to run it by $sudo ./Makefile.py, so env var can be passed" | |
os.environ["TEST_DEV"]="/dev/sdb1" | |
os.environ["TEST_DIR"]="/mnt/ext4a" | |
os.environ["SCRATCH_DEV"]="/dev/sdb2" | |
os.environ["SCRATCH_MNT"]="/mnt/ext4b" | |
#shcmd("export SCRATCH_DEV=scratch_device-name") | |
#shcmd("export SCRATCH_MNT=scratch_device-mount-point") | |
os.environ["FSTYP"]="ext4" | |
shcmd("echo $FSTYP") | |
#with cd("./xfstests"): | |
#shcmd("./check -g auto") | |
#with cd("./xfstests"): | |
#shcmd("./check -g quick ") | |
#with cd("./xfstests"): | |
#shcmd("./check generic/013 generic/053 generic/062 generic/105 generic/228 generic/237 generic/307 generic/318 generic/319 ") | |
# failed runs with check -g auto | |
with cd("./xfstests"): | |
shcmd("./check generic/062 generic/128 generic/228 generic/256 generic/269") | |
def setup_xfstests_env(): | |
os.environ["TEST_DEV"]="/dev/loop0" | |
os.environ["TEST_DIR"]="/mnt/ext4onloop" | |
os.environ["SCRATCH_DEV"]="/dev/loop1" | |
os.environ["SCRATCH_MNT"]="/mnt/ext4onloop1" | |
#shcmd("export SCRATCH_DEV=scratch_device-name") | |
#shcmd("export SCRATCH_MNT=scratch_device-mount-point") | |
os.environ["FSTYP"]="ext4" | |
shcmd("echo $FSTYP") | |
#with cd("./xfstests"): | |
#shcmd("./check -g auto") | |
with cd("./xfstests"): | |
shcmd("sudo ./check -g quick ") | |
def cloudlab(): | |
def create_ext4(): | |
makeExt4("/dev/sdb1", blocksize=4096, makeopts=None) | |
makeExt4("/dev/sdc1", blocksize=4096, makeopts=None) | |
mountExt4(devname="/dev/sdb1", mountpoint="/mnt/ext4a") | |
mountExt4(devname="/dev/sdc1", mountpoint="/mnt/ext4b") | |
shcmd("sudo chown jhe:fsperfatscale-PG -R /mnt/ext4a") | |
shcmd("sudo chown jhe:fsperfatscale-PG -R /mnt/ext4b") | |
def run_xfstests(): | |
"you have to run it by $sudo ./Makefile.py, so env var can be passed" | |
os.environ["TEST_DEV"]="/dev/sdb1" | |
os.environ["TEST_DIR"]="/mnt/ext4a" | |
os.environ["SCRATCH_DEV"]="/dev/sdc1" | |
os.environ["SCRATCH_MNT"]="/mnt/ext4b" | |
#shcmd("export SCRATCH_DEV=scratch_device-name") | |
#shcmd("export SCRATCH_MNT=scratch_device-mount-point") | |
os.environ["FSTYP"]="ext4" | |
shcmd("echo $FSTYP") | |
with cd("./xfstests"): | |
shcmd("./check -g auto") | |
#with cd("./xfstests"): | |
#shcmd("./check -g quick ") | |
#with cd("./xfstests"): | |
#shcmd("./check generic/013 generic/053 generic/062 generic/105 generic/228 generic/237 generic/307 generic/318 generic/319 ") | |
# failed runs with check -g auto | |
#with cd("./xfstests"): | |
#shcmd("./check generic/062 generic/128 generic/228 generic/256 generic/269") | |
#xfstest_install_packages() | |
create_ext4() | |
run_xfstests() | |
########################################################### | |
########################################################### | |
def main(): | |
#function you want to call | |
#compile_linux() | |
#xfstest_install_packages() | |
create_ext4_on_loop() | |
#create_ext4() | |
#setup_xfstests_env() | |
#run_xfstests() | |
# cloudlab() | |
pass | |
def _main(): | |
parser = argparse.ArgumentParser( | |
description="This file hold command stream." \ | |
'Example: python Makefile.py doexp1 ' | |
) | |
parser.add_argument('-t', '--target', action='store') | |
args = parser.parse_args() | |
if args.target == None: | |
main() | |
else: | |
# WARNING! Using argument will make it less reproducible | |
# because you have to remember what argument you used! | |
targets = args.target.split(';') | |
for target in targets: | |
eval(target) | |
if __name__ == '__main__': | |
_main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment