create ext4 on loop dev

create-ext4-on-loop.py

#!/usr/bin/env python
import itertools
import random
import argparse
import re
import subprocess
import os
import sys
import shlex
import time
import glob
from time import localtime, strftime

def shcmd(cmd, ignore_error=False):
    print 'Doing:', cmd
    ret = subprocess.call(cmd, shell=True)
    print 'Returned', ret, cmd
    if ignore_error == False and ret != 0:
        exit(ret)
    return ret

class cd:
    """Context manager for changing the current working directory"""
    def __init__(self, newPath):
        self.newPath = newPath

    def __enter__(self):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, etype, value, traceback):
        os.chdir(self.savedPath)


###########################################################
###########################################################
def compile_linux():
    #with cd("/mnt/scratch-sda4/"):
        #shcmd("git clone git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git")

    with cd("/mnt/scratch-sda4/linux"):
        #shcmd("cp -vi /boot/config-`uname -r` .config")
        #shcmd("yes ''| make oldconfig")
        #shcmd("sudo apt-get install -y git-core libncurses5 libncurses5-dev libelf-dev asciidoc binutils-dev linux-source qt3-dev-tools libqt3-mt-dev libncurses5 libncurses5-dev fakeroot build-essential crash kexec-tools makedumpfile kernel-wedge kernel-package")
        #shcmd("make menuconfig")
        #shcmd("make -j3")
        shcmd("sudo make INSTALL_MOD_PATH=/mnt/scratch-sda4/ modules_install")
        shcmd("sudo make install")

def mkLoopDevOnFile(devname, filepath):
    cmd = ['losetup', devname, filepath]
    cmd = [str(x) for x in cmd]
    print " ".join(cmd), "......"
    proc = subprocess.Popen(cmd)
    proc.wait()

    return proc.returncode

def delLoopDev(devname):
    cmd = ['losetup', '-d', devname]
    cmd = [str(x) for x in cmd]
    print " ".join(cmd), "......"
    proc = subprocess.Popen(cmd)
    proc.wait()

    return proc.returncode

def isMounted(name):
    "only check is a name is in mounted list"
    name = name.rstrip('/')
    print "isMounted: name:", name
    with open('/etc/mtab', 'r') as f:
        for line in f:
            #print "line:", line,
            line = " " + line + " " # a hack
            if re.search(r'\s'+name+r'\s', line):
                #print " YES"
                return True
            #print " NO"
    return False

def isLoopDevUsed(path):
    cmd = ['losetup','-f']
    proc = subprocess.Popen(cmd,
            stdout=subprocess.PIPE)

    proc.wait()

    outstr = proc.communicate()[0]
    outstr = outstr.strip()
    print "isLoopUsed:", outstr+"END"
    if outstr > path:
        return True
    else:
        return False

def umountFS(mountpoint):
    cmd = ["umount", mountpoint]
    p = subprocess.Popen(cmd)
    p.wait()
    print "umountFS:", p.returncode
    return p.returncode

def makeLoopDevice(devname, tmpfs_mountpoint, sizeMB, img_file=None):
    "size is in MB. The tmpfs for this device might be bigger than sizeMB"
    if not devname.startswith('/dev/loop'):
        print 'you are requesting to create loop device on a non-loop device path'
        exit(1)

    if not os.path.exists(tmpfs_mountpoint):
        os.makedirs(tmpfs_mountpoint)

    # umount the FS mounted on loop dev
    if isMounted(devname):
        if umountFS(devname) != 0:
            print "unable to umount", devname
            exit(1)
        else:
            print devname, 'umounted'
    else:
        print devname, "is not mounted"

    # delete the loop device
    if isLoopDevUsed(devname):
        if delLoopDev(devname) != 0:
            print "Failed to delete loop device"
            exit(1)
    else:
        print devname, "is not in use"


    # umount the tmpfs the loop device is on
    if isMounted(tmpfs_mountpoint):
        if umountFS(tmpfs_mountpoint) != 0:
            print "unable to umount tmpfs at", tmpfs_mountpoint
            exit(1)
        print tmpfs_mountpoint, "umounted"
    else:
        print tmpfs_mountpoint, "is not mounted"


    mountTmpfs(tmpfs_mountpoint, int(sizeMB*1024*1024*1.1))
    imgpath = os.path.join(tmpfs_mountpoint, "disk.img")
    if img_file == None:
        mkImageFile(imgpath, sizeMB)
    else:
        cmd = ['cp', img_file, imgpath]
        print 'doing...', cmd
        subprocess.call(cmd)

    mkLoopDevOnFile(devname, imgpath)

def mkImageFile(filepath, size):
    "size is in MB"
    #cmd = ['dd', 'if=/dev/zero', 'of='+filepath,
           #'bs=1M', 'count='+str(size)]
    cmd = ['truncate', '-s', str(size*1024*1024), filepath]
    print " ".join(cmd), "......"
    proc = subprocess.Popen(cmd)
    proc.wait()
    return proc.returncode

def mountTmpfs(mountpoint, size):
    if not os.path.exists(mountpoint):
        os.makedirs(mountpoint)
    cmd = ['mount', '-t', 'tmpfs',
           '-o', 'size='+str(size), 'tmpfs', mountpoint]
    cmd = [str(x) for x in cmd]
    print " ".join(cmd), "......"
    proc = subprocess.Popen(cmd)
    proc.wait()

    return proc.returncode

def mountExt4(devname, mountpoint):
    if not os.path.exists(mountpoint):
        os.makedirs(mountpoint)

    cmd = ["mount", "-t", "ext4", devname, mountpoint]
    p = subprocess.Popen(cmd)
    p.wait()
    print "mountExt4:", p.returncode
    return p.returncode

def makeExt4(devname, blocksize=4096, makeopts=None):

    if makeopts == None:
        cmd = ["mkfs.ext4",
               "-b", blocksize,
               "-O", "^has_journal,extent,huge_file,flex_bg,uninit_bg,dir_nlink,extra_isize",
               devname]
    else:
        cmd = ["mkfs.ext4",
               "-b", blocksize]
        cmd.extend(makeopts)
        cmd.extend([devname])

    cmd = [str(x) for x in cmd]
    p = subprocess.Popen(cmd)
    p.wait()
    print "makeExt4:", p.returncode
    return p.returncode

def create_ext4_on_loop():
    makeLoopDevice("/dev/loop0", "/mnt/tmpfs", 4096, img_file=None)
    makeExt4("/dev/loop0", blocksize=4096, makeopts=None)
    mountExt4(devname="/dev/loop0", mountpoint="/mnt/ext4onloop")

def create_ext4():
    #makeLoopDevice("/dev/loop0", "/mnt/tmpfs", 4096, img_file=None)
    #makeLoopDevice("/dev/loop1", "/mnt/tmpfs1", 4096, img_file=None)
    makeExt4("/dev/sdb1", blocksize=4096, makeopts=None)
    makeExt4("/dev/sdb2", blocksize=4096, makeopts=None)
    mountExt4(devname="/dev/sdb1", mountpoint="/mnt/ext4a")
    mountExt4(devname="/dev/sdb2", mountpoint="/mnt/ext4b")
    shcmd("sudo chown jhe:FSPerfAtScale -R /mnt/ext4a")
    shcmd("sudo chown jhe:FSPerfAtScale -R /mnt/ext4b")

def xfstest_install_packages():
    shcmd("sudo apt-get install -y xfslibs-dev uuid-dev libtool e2fsprogs automake gcc libuuid1 quota attr libattr1-dev libacl1-dev libaio-dev xfsprogs libgdbm-dev gawk fio dbench")
    shcmd("git clone git://oss.sgi.com/xfs/cmds/xfstests")

    with cd("xfstests"):
        shcmd("make")
        shcmd("make install")

    shcmd("sudo useradd fsgqa")

def run_xfstests():
    "you have to run it by $sudo ./Makefile.py, so env var can be passed"
    os.environ["TEST_DEV"]="/dev/sdb1"
    os.environ["TEST_DIR"]="/mnt/ext4a"
    os.environ["SCRATCH_DEV"]="/dev/sdb2"
    os.environ["SCRATCH_MNT"]="/mnt/ext4b"
    #shcmd("export SCRATCH_DEV=scratch_device-name")
    #shcmd("export SCRATCH_MNT=scratch_device-mount-point")
    os.environ["FSTYP"]="ext4"
    shcmd("echo $FSTYP")

    #with cd("./xfstests"):
        #shcmd("./check -g auto")
    #with cd("./xfstests"):
        #shcmd("./check -g quick ")
    #with cd("./xfstests"):
        #shcmd("./check generic/013 generic/053 generic/062 generic/105 generic/228 generic/237 generic/307 generic/318 generic/319 ")

    # failed runs with check -g auto
    with cd("./xfstests"):
        shcmd("./check generic/062 generic/128 generic/228 generic/256 generic/269")

def setup_xfstests_env():
    os.environ["TEST_DEV"]="/dev/loop0"
    os.environ["TEST_DIR"]="/mnt/ext4onloop"
    os.environ["SCRATCH_DEV"]="/dev/loop1"
    os.environ["SCRATCH_MNT"]="/mnt/ext4onloop1"
    #shcmd("export SCRATCH_DEV=scratch_device-name")
    #shcmd("export SCRATCH_MNT=scratch_device-mount-point")
    os.environ["FSTYP"]="ext4"
    shcmd("echo $FSTYP")

    #with cd("./xfstests"):
        #shcmd("./check -g auto")
    with cd("./xfstests"):
        shcmd("sudo ./check -g quick ")


def cloudlab():
    def create_ext4():
        makeExt4("/dev/sdb1", blocksize=4096, makeopts=None)
        makeExt4("/dev/sdc1", blocksize=4096, makeopts=None)
        mountExt4(devname="/dev/sdb1", mountpoint="/mnt/ext4a")
        mountExt4(devname="/dev/sdc1", mountpoint="/mnt/ext4b")
        shcmd("sudo chown jhe:fsperfatscale-PG -R /mnt/ext4a")
        shcmd("sudo chown jhe:fsperfatscale-PG -R /mnt/ext4b")

    def run_xfstests():
        "you have to run it by $sudo ./Makefile.py, so env var can be passed"
        os.environ["TEST_DEV"]="/dev/sdb1"
        os.environ["TEST_DIR"]="/mnt/ext4a"
        os.environ["SCRATCH_DEV"]="/dev/sdc1"
        os.environ["SCRATCH_MNT"]="/mnt/ext4b"
        #shcmd("export SCRATCH_DEV=scratch_device-name")
        #shcmd("export SCRATCH_MNT=scratch_device-mount-point")
        os.environ["FSTYP"]="ext4"
        shcmd("echo $FSTYP")

        with cd("./xfstests"):
            shcmd("./check -g auto")
        #with cd("./xfstests"):
            #shcmd("./check -g quick ")
        #with cd("./xfstests"):
            #shcmd("./check generic/013 generic/053 generic/062 generic/105 generic/228 generic/237 generic/307 generic/318 generic/319 ")

        # failed runs with check -g auto
        #with cd("./xfstests"):
            #shcmd("./check generic/062 generic/128 generic/228 generic/256 generic/269")
    #xfstest_install_packages()
    create_ext4()
    run_xfstests()


###########################################################
###########################################################

def main():
    #function you want to call
    #compile_linux()
    #xfstest_install_packages()
    create_ext4_on_loop()
    #create_ext4()
    #setup_xfstests_env()
    #run_xfstests()
    # cloudlab()
    pass

def _main():
    parser = argparse.ArgumentParser(
            description="This file hold command stream." \
            'Example: python Makefile.py doexp1 '
            )
    parser.add_argument('-t', '--target', action='store')
    args = parser.parse_args()

    if args.target == None:
        main()
    else:
        # WARNING! Using argument will make it less reproducible
        # because you have to remember what argument you used!
        targets = args.target.split(';')
        for target in targets:
            eval(target)

if __name__ == '__main__':
    _main()