aoeblk.c

[cremes@box2 aoe]$ cat aoeblk.c 
/* Copyright (c) 2011 Coraid, Inc.  See COPYING for GPL terms. */
/*
 * aoeblk.c
 * block device routines
 */

#include <linux/hdreg.h>
#include <linux/blkdev.h>
#include <linux/fs.h>
#include <linux/ioctl.h>
#include <linux/genhd.h>
#include <linux/netdevice.h>
#include <scsi/sg.h>
#include "aoe.h"
#include "disk_attr.h"

static struct kmem_cache *buf_pool_cache;

/* GPFS needs a larger value than the default. */
static int aoe_maxsectors = 0;
module_param(aoe_maxsectors, int, 0644);
MODULE_PARM_DESC(aoe_maxsectors,
	"When nonzero, set the maximum number of sectors "
	"per I/O request in new devices.");

/* workaround for XFS and bios with zero pageref pages in general */
void
aoe_bio_pagedec(struct bio *bio)
{
	struct bio_vec *bv;
	int i;

	bio_for_each_segment(bv, bio, i)
		atomic_dec(&bv->bv_page->_count);
}

static ssize_t aoedisk_show_state(struct device *dev,
				  struct device_attribute *attr, char *page)
{
	struct gendisk *disk = dev_to_disk(dev);
	struct aoedev *d = disk->private_data;

	return snprintf(page, PAGE_SIZE,
			"%s%s\n",
			(d->flags & DEVFL_UP) ? "up" : "down",
			(d->flags & DEVFL_KICKME) ? ",kickme" :
			(d->nopen && !(d->flags & DEVFL_UP)) ?
			",closewait" : "");
	/* I'd rather see nopen exported so we can ditch closewait */
}
static ssize_t aoedisk_show_mac(struct device *dev,
				struct device_attribute *attr, char *page)
{
	struct gendisk *disk = dev_to_disk(dev);
	struct aoedev *d = disk->private_data;
	struct aoetgt *t = d->targets[0];

	if (t == NULL)
		return snprintf(page, PAGE_SIZE, "none\n");
	return snprintf(page, PAGE_SIZE, "%012llx\n", mac_addr(t->addr));
}
static ssize_t aoedisk_show_netif(struct device *dev,
				  struct device_attribute *attr, char *page)
{
	struct gendisk *disk = dev_to_disk(dev);
	struct aoedev *d = disk->private_data;
	struct net_device *nds[8], **nd, **nnd, **ne;
	struct aoetgt **t, **te;
	struct aoeif *ifp, *e;
	char *p;

	memset(nds, 0, sizeof nds);
	nd = nds;
	ne = nd + ARRAY_SIZE(nds);
	t = d->targets;
	te = t + d->ntargets;
	for (; t<te && *t; t++) {
		ifp = (*t)->ifs;
		e = ifp + NAOEIFS;
		for (; ifp<e && ifp->nd; ifp++) {
			for (nnd=nds; nnd<nd; nnd++)
				if (*nnd == ifp->nd)
					break;
			if (nnd == nd && nd != ne)
				*nd++ = ifp->nd;
		}
	}

	ne = nd;
	nd = nds;
	if (*nd == NULL)
		return snprintf(page, PAGE_SIZE, "none\n");
	for (p=page; nd<ne; nd++)
		p += snprintf(p, PAGE_SIZE - (p-page), "%s%s",
			p == page ? "" : ",", (*nd)->name);
	p += snprintf(p, PAGE_SIZE - (p-page), "\n");
	return p-page;
}
/* firmware version */
static ssize_t aoedisk_show_fwver(struct device *dev,
				  struct device_attribute *attr, char *page)
{
	struct gendisk *disk = dev_to_disk(dev);
	struct aoedev *d = disk->private_data;

	return snprintf(page, PAGE_SIZE,
			"0x%04x\n", (unsigned int) d->fw_ver);
}
static ssize_t aoedisk_show_payload(struct device *dev,
				    struct device_attribute *attr, char *page)
{
	struct gendisk *disk = dev_to_disk(dev);
	struct aoedev *d = disk->private_data;

	return snprintf(page, PAGE_SIZE, "%lu\n", d->maxbcnt);
}
static ssize_t aoedisk_show_debug(struct device *dev,
				  struct device_attribute *attr, char *page)
{
	struct gendisk *disk = dev_to_disk(dev);
	struct aoedev *d = disk->private_data;
	struct aoetgt **t, **te;
	char *p;

// senseless obfuscation
#define so(arg...) snprintf(p, PAGE_SIZE - (p-page), ## arg)

	p = page;
	p += so("rttavg: %d rttdev: %d\n",
		d->rttavg >> RTTSCALE,
		d->rttdev >> RTTDSCALE);
	p += so("nskbpool: %d\n", d->nskbpool);
	p += so("kicked: %ld\n", d->kicked);
	p += so("maxbcnt: %ld\n", d->maxbcnt);
	p += so("ref: %ld\n", d->ref);

	t = d->targets;
	te = t + d->ntargets;
	for (; t<te && *t; t++) {
		struct aoeif *ifp, *ife;
		char c = '\t';

		p += so("falloc: %ld\n", (*t)->falloc);
		p += so("ffree: %p\n", list_head_el(&(*t)->ffree));
		p += so("%012llx:%d:%d:%d\n", mac_addr((*t)->addr), (*t)->nout,
			(*t)->maxout, (*t)->nframes);
		p += so("\tssthresh:%d\n", (*t)->ssthresh);
		p += so("\tlost:%lu\n", (*t)->lost);
		p += so("\ttaint:%d\n", (*t)->taint);
		p += so("\tr:%d\n", (*t)->rpkts);
		p += so("\tw:%d\n", (*t)->wpkts);
		ifp = (*t)->ifs;
		ife = ifp + ARRAY_SIZE((*t)->ifs);
		for (; ifp->nd && ifp<ife; ifp++) {
			p += so("%c%s", c, ifp->nd->name);
			c = ',';
		}
		p += so("\n");
	}

	return p-page;
}

static DEVICE_ATTR(state, S_IRUGO, aoedisk_show_state, NULL);
static DEVICE_ATTR(mac, S_IRUGO, aoedisk_show_mac, NULL);
static DEVICE_ATTR(netif, S_IRUGO, aoedisk_show_netif, NULL);
static struct device_attribute dev_attr_firmware_version = {
	.attr = {
		.name = "firmware-version",
		.mode = S_IRUGO,
	},
	.show = aoedisk_show_fwver,
};
static DEVICE_ATTR(payload, S_IRUGO, aoedisk_show_payload, NULL);
static DEVICE_ATTR(debug, S_IRUGO, aoedisk_show_debug, NULL);

static struct attribute *aoe_attrs[] = {
	&dev_attr_state.attr,
	&dev_attr_mac.attr,
	&dev_attr_netif.attr,
	&dev_attr_firmware_version.attr,
	&dev_attr_payload.attr,
	&dev_attr_debug.attr,
	NULL,
};

static const struct attribute_group attr_group = {
	.attrs = aoe_attrs,
};

static int
aoedisk_add_sysfs(struct aoedev *d)
{
	return sysfs_create_group(&disk_to_dev(d->gd)->kobj, &attr_group);
}
void
aoedisk_rm_sysfs(struct aoedev *d)
{
	sysfs_remove_group(&disk_to_dev(d->gd)->kobj, &attr_group);
}

static int
aoeblk_open(struct block_device *bdev, fmode_t mode)
{
	struct aoedev *d = bdev->bd_disk->private_data;
	ulong flags;

	if (!virt_addr_valid(d)) {
		printk(KERN_CRIT
			"aoe: invalid device pointer in %s\n",
			__func__);
		WARN_ON(1);
		return -ENODEV;
	}
	if (!(d->flags & DEVFL_UP) || d->flags & DEVFL_TKILL)
		return -ENODEV;

	spin_lock_irqsave(&d->lock, flags);
	if (d->flags & DEVFL_UP && !(d->flags & DEVFL_TKILL)) {
		d->nopen++;
		spin_unlock_irqrestore(&d->lock, flags);
		return 0;
	}
	spin_unlock_irqrestore(&d->lock, flags);
	return -ENODEV;
}

static int
aoeblk_release(struct gendisk *disk, fmode_t mode)
{
	struct aoedev *d = disk->private_data;
	ulong flags;

	spin_lock_irqsave(&d->lock, flags);

	if (--d->nopen == 0) {
		spin_unlock_irqrestore(&d->lock, flags);
		aoecmd_cfg(d->aoemajor, d->aoeminor);
		return 0;
	}
	spin_unlock_irqrestore(&d->lock, flags);

	return 0;
}

static void
aoeblk_request(struct request_queue *q)
{
	struct aoedev *d;
	struct request *rq;

	d = q->queuedata;
	if ((d->flags & DEVFL_UP) == 0) {
		printk(KERN_INFO "aoe: device %ld.%d is not up\n",
			d->aoemajor, d->aoeminor);
		while ((rq = blk_peek_request(q))) {
			blk_start_request(rq);
			aoe_end_request(d, rq, 1);
		}
		return;
	}
	aoecmd_work(d);
}

static int
aoeblk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
{
	struct aoedev *d = bdev->bd_disk->private_data;

	if ((d->flags & DEVFL_UP) == 0) {
		printk(KERN_ERR "aoe: disk not up\n");
		return -ENODEV;
	}

	geo->cylinders = d->geo.cylinders;
	geo->heads = d->geo.heads;
	geo->sectors = d->geo.sectors;
	return 0;
}

static int
aoeblk_ioctl(struct block_device *bdev, fmode_t mode, uint cmd, ulong arg)
{
	struct aoedev *d;

	if (!arg)
		return -EINVAL;

	d = bdev->bd_disk->private_data;
	if ((d->flags & DEVFL_UP) == 0) {
		printk(KERN_ERR "aoe: disk not up\n");
		return -ENODEV;
	}

	if (cmd == HDIO_GET_IDENTITY) {
		if (!copy_to_user((void __user *) arg, &d->ident,
				sizeof d->ident))
			return 0;
		return -EFAULT;
	}

	/* udev calls scsi_id, which uses SG_IO, resulting in noise */
	if (cmd != SG_IO)
		printk(KERN_INFO "aoe: unknown ioctl 0x%x\n", cmd);

	return -ENOTTY;		/* for older kernels */
}

static struct block_device_operations aoe_bdops = {
	.open = aoeblk_open,
	.release = aoeblk_release,
	.ioctl = aoeblk_ioctl,
	.getgeo = aoeblk_getgeo,
	.owner = THIS_MODULE,
};

/* alloc_disk and add_disk can sleep */
void
aoeblk_gdalloc(void *vp)
{
	struct aoedev *d = vp;
	struct gendisk *gd;
	mempool_t *mp;
	struct request_queue *q;
	enum { KB= 1024, MB= KB*KB, READ_AHEAD= 2*MB, };
	ulong flags;
	int late = 0;

	spin_lock_irqsave(&d->lock, flags);
	if (d->flags & DEVFL_GDALLOC
	&& !(d->flags & DEVFL_TKILL)
	&& !(d->flags & DEVFL_GD_NOW))
		d->flags |= DEVFL_GD_NOW;
	else
		late = 1;
	spin_unlock_irqrestore(&d->lock, flags);
	if (late)
		return;

	gd = alloc_disk(AOE_PARTITIONS);
	if (gd == NULL) {
		printk(KERN_ERR "aoe: cannot allocate disk structure for %ld.%d\n",
			d->aoemajor, d->aoeminor);
		goto gderr;
	}
	mp = mempool_create(MIN_BUFS, mempool_alloc_slab, mempool_free_slab,
		buf_pool_cache);
	if (mp == NULL) {
		printk(KERN_ERR "aoe: cannot allocate bufpool for %ld.%d\n",
			d->aoemajor, d->aoeminor);
		goto mperr;
	}
	q = blk_init_queue(aoeblk_request, &d->lock);
	if (q == NULL) {
		printk(KERN_ERR "aoe: cannot allocate block queue for %ld.%d\n",
			d->aoemajor, d->aoeminor);
		mempool_destroy(mp);
mperr:		put_disk(gd);
gderr:		spin_lock_irqsave(&d->lock, flags);
		d->flags &= ~DEVFL_GD_NOW;
		schedule_work(&d->work);
		spin_unlock_irqrestore(&d->lock, flags);
		return;
	}

	spin_lock_irqsave(&d->lock, flags);
	WARN_ON(!(d->flags & DEVFL_GD_NOW));
	WARN_ON(!(d->flags & DEVFL_GDALLOC));
	WARN_ON(d->flags & DEVFL_TKILL);
	WARN_ON(d->gd);
	WARN_ON(d->flags & DEVFL_UP);

	q->backing_dev_info.ra_pages = READ_AHEAD / PAGE_CACHE_SIZE;

	d->bufpool = mp;
	d->blkq = gd->queue = q;
	q->queuedata = d;
	d->gd = gd;
	if (aoe_maxsectors)
		blk_queue_max_hw_sectors(q, aoe_maxsectors);
	gd->major = AOE_MAJOR;
	gd->first_minor = d->sysminor;
	gd->fops = &aoe_bdops;
	gd->private_data = d;
	set_capacity(gd, d->ssize);
	snprintf(gd->disk_name, sizeof gd->disk_name, "%s/e%ld.%d",
		DEVSUBDIR, d->aoemajor, d->aoeminor);

	d->flags |= DEVFL_UP;
	d->flags &= ~DEVFL_GDALLOC;
	spin_unlock_irqrestore(&d->lock, flags);

	add_disk(gd);
	aoedisk_add_sysfs(d);

	spin_lock_irqsave(&d->lock, flags);
	WARN_ON(!(d->flags & DEVFL_GD_NOW));
	d->flags &= ~DEVFL_GD_NOW;
	spin_unlock_irqrestore(&d->lock, flags);
}

void
aoeblk_exit(void)
{
	kmem_cache_destroy(buf_pool_cache);
}

int __init
aoeblk_init(void)
{
	buf_pool_cache = kmem_cache_create("aoe_bufs",
					   sizeof(struct buf),
					   0, 0, NULL);
	if (buf_pool_cache == NULL)
		return -ENOMEM;

	return 0;
}

[cremes@box2 aoe]$ 

install_readme.txt

Instructions for installing AoE driver under ArchLinux

ArchLinux comes with an outdated version of the aoe kernel module (version 47). ArchLinux distributes its code updates as binary form instead of source form; this includes the kernel. So the usual steps for compiling and installing the aoe driver don't immediately apply due to the missing kernel build directory.

To build the driver, we first need to install and build the kernel sources. In ArchLinux this is done with ABS (Arch Build System).

1. We're going to need to build the *latest* kernel from source. ABS primarily maintains source for the latest official package in the pacman database. If you are unable to update your kernel to the latest ArchLinux release, then you are on your own. It may be possible to get the ABS data for your specific kernel since it maintains that information in SVN. Figuring out how to do so is left as an exercise for the reader.

So, update your pacman package database to the latest revision.

% sudo pacman -Sy

2. Update your system to the latest and greatest kernel.

% sudo pacman -S linux

3. Now install ABS.

% sudo pacman -S abs # install ABS

4. Issue the command to synchronize the /var/abs directory with the latest SVN snapshot.

% sudo abs

5. Copy the kernel sources to a local working directory. The kernel will be built here.

% cp /var/abs/core/linux/* <working_dir>/

% cd <working_dir>

5b. Edit /etc/makepkg.conf and set the MAKEFLAGS to at least -j2 otherwise the kernel compile will take forever.

6. Don't modify any of the files. We essentially want to recreate the stock kernel build environment.

% makepkg

This command will take approximately 1 hour to complete depending on the performance of the computer.


7. At this point the kernel has been built and packaged in the working directory. There is no need to install this specific package using pacman. All of this work was to get the kernel sources primed and ready for the aoe driver build step.

8. Change directories to the src/linux-x.y.z/ kernel directory. Confirm that it contains a .config file.

% cd <working_dir>/src/linux-x.y.z/
% ls -al .config

9. Prepare the kernel sources for building a kernel module. The modern Makefile (as of 20100519) has the command as a PHONY no op, but we should do it anyway just in case the behavior of the Makefile changes again in the future.

% make modules_prepare

10. Move to the aoe driver directory and build the driver using the path leading to the kernel sources you just built.

% cd <aoe driver dir>

% make KDIR=path/to/working_dir/src/linux-x.y.z \
       INSTDIR=/lib/modules/<kernel name>/kernel/drivers/block/aoe

11. And install the driver as root.

% su -

# cd <aoe driver dir>

# make install KDIR=path/to/working_dir/src/linux-x.y.z \
               INSTDIR=/lib/modules/<kernel name>/kernel/drivers/block/aoe
               
12. Refer to the README file packaged with the aoe driver for any additional details.

stdout.txt

[cremes@box2 aoe6-78]$ make KDIR=/home/cremes/dev/kernel/src/linux-3.2 \
> INSTDIR=/lib/modules/   
3.2.4-1-ARCH/          extramodules-3.2-ARCH/ 
> INSTDIR=/lib/modules/3.2.4-1-ARCH/kernel/drivers/block/aoe 
cd aoetools-32 && make 
make[1]: Entering directory `/home/cremes/dev/aoe6-78/aoetools-32'
+ sed -e 's!@devdir@!/dev/etherd!g' -e 's!@npershelf@!16!g' aoe-discover.in
+ sed -e 's!@devdir@!/dev/etherd!g' -e 's!@npershelf@!16!g' aoe-interfaces.in
+ sed -e 's!@devdir@!/dev/etherd!g' -e 's!@npershelf@!16!g' aoe-mkshelf.in
+ sed -e 's!@devdir@!/dev/etherd!g' -e 's!@npershelf@!16!g' aoe-revalidate.in
+ sed -e 's!@devdir@!/dev/etherd!g' -e 's!@npershelf@!16!g' aoe-flush.in
+ sed -e 's!@devdir@!/dev/etherd!g' -e 's!@npershelf@!16!g' aoe-stat.in
make[1]: Leaving directory `/home/cremes/dev/aoe6-78/aoetools-32'
sh -c "cd linux/drivers/block/aoe && rm -f *.o *.ko core"
ensuring compatibility ... 1 
patching file linux/drivers/block/aoe/disk_attr.h
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 
patching file linux/drivers/block/aoe/aoecmd.c
24 25 26 27 28 29 30 31 32 33 34 35 ok
echo > conf/done
make -C /home/cremes/dev/kernel/src/linux-3.2 CONFIG_ATA_OVER_ETH=m KDIR=/home/cremes/dev/kernel/src/linux-3.2 SUBDIRS="/home/cremes/dev/aoe6-78/linux/drivers/block/aoe" EXTRA_CFLAGS="-DAOE_PARTITIONS=16 -DDEVSUBDIR='\"etherd\"' -DAOE_DYNDEVS=1" modules
make[1]: Entering directory `/home/cremes/dev/kernel/src/linux-3.2'
  CC [M]  /home/cremes/dev/aoe6-78/linux/drivers/block/aoe/aoeblk.o
/home/cremes/dev/aoe6-78/linux/drivers/block/aoe/aoeblk.c:21:30: error: expected ‘)’ before ‘int’
/home/cremes/dev/aoe6-78/linux/drivers/block/aoe/aoeblk.c:23:2: error: expected ‘)’ before string constant
/home/cremes/dev/aoe6-78/linux/drivers/block/aoe/aoeblk.c:314:11: error: ‘THIS_MODULE’ undeclared here (not in a function)
make[2]: *** [/home/cremes/dev/aoe6-78/linux/drivers/block/aoe/aoeblk.o] Error 1
make[1]: *** [_module_/home/cremes/dev/aoe6-78/linux/drivers/block/aoe] Error 2
make[1]: Leaving directory `/home/cremes/dev/kernel/src/linux-3.2'
make: *** [linux/drivers/block/aoe/aoe.ko] Error 2