aoecmd.c

   1/* Copyright (c) 2013 Coraid, Inc.  See COPYING for GPL terms. */
   2/*
   3 * aoecmd.c
   4 * Filesystem request handling methods
   5 */
   6
   7#include <linux/ata.h>
   8#include <linux/slab.h>
   9#include <linux/hdreg.h>
  10#include <linux/blk-mq.h>
  11#include <linux/skbuff.h>
  12#include <linux/netdevice.h>
  13#include <linux/moduleparam.h>
  14#include <linux/workqueue.h>
  15#include <linux/kthread.h>
  16#include <net/net_namespace.h>
  17#include <linux/unaligned.h>
  18#include <linux/uio.h>
  19#include "aoe.h"
  20
  21#define MAXIOC (8192)	/* default meant to avoid most soft lockups */
  22
  23static void ktcomplete(struct frame *, struct sk_buff *);
  24static int count_targets(struct aoedev *d, int *untainted);
  25
  26static struct buf *nextbuf(struct aoedev *);
  27
  28static int aoe_deadsecs = 60 * 3;
  29module_param(aoe_deadsecs, int, 0644);
  30MODULE_PARM_DESC(aoe_deadsecs, "After aoe_deadsecs seconds, give up and fail dev.");
  31
  32static int aoe_maxout = 64;
  33module_param(aoe_maxout, int, 0644);
  34MODULE_PARM_DESC(aoe_maxout,
  35	"Only aoe_maxout outstanding packets for every MAC on eX.Y.");
  36
  37/* The number of online cpus during module initialization gives us a
  38 * convenient heuristic cap on the parallelism used for ktio threads
  39 * doing I/O completion.  It is not important that the cap equal the
  40 * actual number of running CPUs at any given time, but because of CPU
  41 * hotplug, we take care to use ncpus instead of using
  42 * num_online_cpus() after module initialization.
  43 */
  44static int ncpus;
  45
  46/* mutex lock used for synchronization while thread spawning */
  47static DEFINE_MUTEX(ktio_spawn_lock);
  48
  49static wait_queue_head_t *ktiowq;
  50static struct ktstate *kts;
  51
  52/* io completion queue */
  53struct iocq_ktio {
  54	struct list_head head;
  55	spinlock_t lock;
  56};
  57static struct iocq_ktio *iocq;
  58
  59static struct page *empty_page;
  60
  61static struct sk_buff *
  62new_skb(ulong len)
  63{
  64	struct sk_buff *skb;
  65
  66	skb = alloc_skb(len + MAX_HEADER, GFP_ATOMIC);
  67	if (skb) {
  68		skb_reserve(skb, MAX_HEADER);
  69		skb_reset_mac_header(skb);
  70		skb_reset_network_header(skb);
  71		skb->protocol = __constant_htons(ETH_P_AOE);
  72		skb_checksum_none_assert(skb);
  73	}
  74	return skb;
  75}
  76
  77static struct frame *
  78getframe_deferred(struct aoedev *d, u32 tag)
  79{
  80	struct list_head *head, *pos, *nx;
  81	struct frame *f;
  82
  83	head = &d->rexmitq;
  84	list_for_each_safe(pos, nx, head) {
  85		f = list_entry(pos, struct frame, head);
  86		if (f->tag == tag) {
  87			list_del(pos);
  88			return f;
  89		}
  90	}
  91	return NULL;
  92}
  93
  94static struct frame *
  95getframe(struct aoedev *d, u32 tag)
  96{
  97	struct frame *f;
  98	struct list_head *head, *pos, *nx;
  99	u32 n;
 100
 101	n = tag % NFACTIVE;
 102	head = &d->factive[n];
 103	list_for_each_safe(pos, nx, head) {
 104		f = list_entry(pos, struct frame, head);
 105		if (f->tag == tag) {
 106			list_del(pos);
 107			return f;
 108		}
 109	}
 110	return NULL;
 111}
 112
 113/*
 114 * Leave the top bit clear so we have tagspace for userland.
 115 * The bottom 16 bits are the xmit tick for rexmit/rttavg processing.
 116 * This driver reserves tag -1 to mean "unused frame."
 117 */
 118static int
 119newtag(struct aoedev *d)
 120{
 121	register ulong n;
 122
 123	n = jiffies & 0xffff;
 124	return n | (++d->lasttag & 0x7fff) << 16;
 125}
 126
 127static u32
 128aoehdr_atainit(struct aoedev *d, struct aoetgt *t, struct aoe_hdr *h)
 129{
 130	u32 host_tag = newtag(d);
 131
 132	memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
 133	memcpy(h->dst, t->addr, sizeof h->dst);
 134	h->type = __constant_cpu_to_be16(ETH_P_AOE);
 135	h->verfl = AOE_HVER;
 136	h->major = cpu_to_be16(d->aoemajor);
 137	h->minor = d->aoeminor;
 138	h->cmd = AOECMD_ATA;
 139	h->tag = cpu_to_be32(host_tag);
 140
 141	return host_tag;
 142}
 143
 144static inline void
 145put_lba(struct aoe_atahdr *ah, sector_t lba)
 146{
 147	ah->lba0 = lba;
 148	ah->lba1 = lba >>= 8;
 149	ah->lba2 = lba >>= 8;
 150	ah->lba3 = lba >>= 8;
 151	ah->lba4 = lba >>= 8;
 152	ah->lba5 = lba >>= 8;
 153}
 154
 155static struct aoeif *
 156ifrotate(struct aoetgt *t)
 157{
 158	struct aoeif *ifp;
 159
 160	ifp = t->ifp;
 161	ifp++;
 162	if (ifp >= &t->ifs[NAOEIFS] || ifp->nd == NULL)
 163		ifp = t->ifs;
 164	if (ifp->nd == NULL)
 165		return NULL;
 166	return t->ifp = ifp;
 167}
 168
 169static void
 170skb_pool_put(struct aoedev *d, struct sk_buff *skb)
 171{
 172	__skb_queue_tail(&d->skbpool, skb);
 173}
 174
 175static struct sk_buff *
 176skb_pool_get(struct aoedev *d)
 177{
 178	struct sk_buff *skb = skb_peek(&d->skbpool);
 179
 180	if (skb && atomic_read(&skb_shinfo(skb)->dataref) == 1) {
 181		__skb_unlink(skb, &d->skbpool);
 182		return skb;
 183	}
 184	if (skb_queue_len(&d->skbpool) < NSKBPOOLMAX &&
 185	    (skb = new_skb(ETH_ZLEN)))
 186		return skb;
 187
 188	return NULL;
 189}
 190
 191void
 192aoe_freetframe(struct frame *f)
 193{
 194	struct aoetgt *t;
 195
 196	t = f->t;
 197	f->buf = NULL;
 198	memset(&f->iter, 0, sizeof(f->iter));
 199	f->r_skb = NULL;
 200	f->flags = 0;
 201	list_add(&f->head, &t->ffree);
 202}
 203
 204static struct frame *
 205newtframe(struct aoedev *d, struct aoetgt *t)
 206{
 207	struct frame *f;
 208	struct sk_buff *skb;
 209	struct list_head *pos;
 210
 211	if (list_empty(&t->ffree)) {
 212		if (t->falloc >= NSKBPOOLMAX*2)
 213			return NULL;
 214		f = kcalloc(1, sizeof(*f), GFP_ATOMIC);
 215		if (f == NULL)
 216			return NULL;
 217		t->falloc++;
 218		f->t = t;
 219	} else {
 220		pos = t->ffree.next;
 221		list_del(pos);
 222		f = list_entry(pos, struct frame, head);
 223	}
 224
 225	skb = f->skb;
 226	if (skb == NULL) {
 227		f->skb = skb = new_skb(ETH_ZLEN);
 228		if (!skb) {
 229bail:			aoe_freetframe(f);
 230			return NULL;
 231		}
 232	}
 233
 234	if (atomic_read(&skb_shinfo(skb)->dataref) != 1) {
 235		skb = skb_pool_get(d);
 236		if (skb == NULL)
 237			goto bail;
 238		skb_pool_put(d, f->skb);
 239		f->skb = skb;
 240	}
 241
 242	skb->truesize -= skb->data_len;
 243	skb_shinfo(skb)->nr_frags = skb->data_len = 0;
 244	skb_trim(skb, 0);
 245	return f;
 246}
 247
 248static struct frame *
 249newframe(struct aoedev *d)
 250{
 251	struct frame *f;
 252	struct aoetgt *t, **tt;
 253	int totout = 0;
 254	int use_tainted;
 255	int has_untainted;
 256
 257	if (!d->targets || !d->targets[0]) {
 258		printk(KERN_ERR "aoe: NULL TARGETS!\n");
 259		return NULL;
 260	}
 261	tt = d->tgt;	/* last used target */
 262	for (use_tainted = 0, has_untainted = 0;;) {
 263		tt++;
 264		if (tt >= &d->targets[d->ntargets] || !*tt)
 265			tt = d->targets;
 266		t = *tt;
 267		if (!t->taint) {
 268			has_untainted = 1;
 269			totout += t->nout;
 270		}
 271		if (t->nout < t->maxout
 272		&& (use_tainted || !t->taint)
 273		&& t->ifp->nd) {
 274			f = newtframe(d, t);
 275			if (f) {
 276				ifrotate(t);
 277				d->tgt = tt;
 278				return f;
 279			}
 280		}
 281		if (tt == d->tgt) {	/* we've looped and found nada */
 282			if (!use_tainted && !has_untainted)
 283				use_tainted = 1;
 284			else
 285				break;
 286		}
 287	}
 288	if (totout == 0) {
 289		d->kicked++;
 290		d->flags |= DEVFL_KICKME;
 291	}
 292	return NULL;
 293}
 294
 295static void
 296skb_fillup(struct sk_buff *skb, struct bio *bio, struct bvec_iter iter)
 297{
 298	int frag = 0;
 299	struct bio_vec bv;
 300
 301	__bio_for_each_segment(bv, bio, iter, iter)
 302		skb_fill_page_desc(skb, frag++, bv.bv_page,
 303				   bv.bv_offset, bv.bv_len);
 304}
 305
 306static void
 307fhash(struct frame *f)
 308{
 309	struct aoedev *d = f->t->d;
 310	u32 n;
 311
 312	n = f->tag % NFACTIVE;
 313	list_add_tail(&f->head, &d->factive[n]);
 314}
 315
 316static void
 317ata_rw_frameinit(struct frame *f)
 318{
 319	struct aoetgt *t;
 320	struct aoe_hdr *h;
 321	struct aoe_atahdr *ah;
 322	struct sk_buff *skb;
 323	char writebit, extbit;
 324
 325	skb = f->skb;
 326	h = (struct aoe_hdr *) skb_mac_header(skb);
 327	ah = (struct aoe_atahdr *) (h + 1);
 328	skb_put(skb, sizeof(*h) + sizeof(*ah));
 329	memset(h, 0, skb->len);
 330
 331	writebit = 0x10;
 332	extbit = 0x4;
 333
 334	t = f->t;
 335	f->tag = aoehdr_atainit(t->d, t, h);
 336	fhash(f);
 337	t->nout++;
 338	f->waited = 0;
 339	f->waited_total = 0;
 340
 341	/* set up ata header */
 342	ah->scnt = f->iter.bi_size >> 9;
 343	put_lba(ah, f->iter.bi_sector);
 344	if (t->d->flags & DEVFL_EXT) {
 345		ah->aflags |= AOEAFL_EXT;
 346	} else {
 347		extbit = 0;
 348		ah->lba3 &= 0x0f;
 349		ah->lba3 |= 0xe0;	/* LBA bit + obsolete 0xa0 */
 350	}
 351	if (f->buf && bio_data_dir(f->buf->bio) == WRITE) {
 352		skb_fillup(skb, f->buf->bio, f->iter);
 353		ah->aflags |= AOEAFL_WRITE;
 354		skb->len += f->iter.bi_size;
 355		skb->data_len = f->iter.bi_size;
 356		skb->truesize += f->iter.bi_size;
 357		t->wpkts++;
 358	} else {
 359		t->rpkts++;
 360		writebit = 0;
 361	}
 362
 363	ah->cmdstat = ATA_CMD_PIO_READ | writebit | extbit;
 364	dev_hold(t->ifp->nd);
 365	skb->dev = t->ifp->nd;
 366}
 367
 368static int
 369aoecmd_ata_rw(struct aoedev *d)
 370{
 371	struct frame *f;
 372	struct buf *buf;
 373	struct sk_buff *skb;
 374	struct sk_buff_head queue;
 375
 376	buf = nextbuf(d);
 377	if (buf == NULL)
 378		return 0;
 379	f = newframe(d);
 380	if (f == NULL)
 381		return 0;
 382
 383	/* initialize the headers & frame */
 384	f->buf = buf;
 385	f->iter = buf->iter;
 386	f->iter.bi_size = min_t(unsigned long,
 387				d->maxbcnt ?: DEFAULTBCNT,
 388				f->iter.bi_size);
 389	bio_advance_iter(buf->bio, &buf->iter, f->iter.bi_size);
 390
 391	if (!buf->iter.bi_size)
 392		d->ip.buf = NULL;
 393
 394	/* mark all tracking fields and load out */
 395	buf->nframesout += 1;
 396
 397	ata_rw_frameinit(f);
 398
 399	skb = skb_clone(f->skb, GFP_ATOMIC);
 400	if (skb) {
 401		f->sent = ktime_get();
 402		__skb_queue_head_init(&queue);
 403		__skb_queue_tail(&queue, skb);
 404		aoenet_xmit(&queue);
 405	} else {
 406		dev_put(f->t->ifp->nd);
 407	}
 408	return 1;
 409}
 410
 411/* some callers cannot sleep, and they can call this function,
 412 * transmitting the packets later, when interrupts are on
 413 */
 414static void
 415aoecmd_cfg_pkts(ushort aoemajor, unsigned char aoeminor, struct sk_buff_head *queue)
 416{
 417	struct aoe_hdr *h;
 418	struct aoe_cfghdr *ch;
 419	struct sk_buff *skb;
 420	struct net_device *ifp;
 421
 422	rcu_read_lock();
 423	for_each_netdev_rcu(&init_net, ifp) {
 424		dev_hold(ifp);
 425		if (!is_aoe_netif(ifp)) {
 426			dev_put(ifp);
 427			continue;
 428		}
 429
 430		skb = new_skb(sizeof *h + sizeof *ch);
 431		if (skb == NULL) {
 432			printk(KERN_INFO "aoe: skb alloc failure\n");
 433			dev_put(ifp);
 434			continue;
 435		}
 436		skb_put(skb, sizeof *h + sizeof *ch);
 437		skb->dev = ifp;
 438		__skb_queue_tail(queue, skb);
 439		h = (struct aoe_hdr *) skb_mac_header(skb);
 440		memset(h, 0, sizeof *h + sizeof *ch);
 441
 442		memset(h->dst, 0xff, sizeof h->dst);
 443		memcpy(h->src, ifp->dev_addr, sizeof h->src);
 444		h->type = __constant_cpu_to_be16(ETH_P_AOE);
 445		h->verfl = AOE_HVER;
 446		h->major = cpu_to_be16(aoemajor);
 447		h->minor = aoeminor;
 448		h->cmd = AOECMD_CFG;
 449	}
 450	rcu_read_unlock();
 451}
 452
 453static void
 454resend(struct aoedev *d, struct frame *f)
 455{
 456	struct sk_buff *skb;
 457	struct sk_buff_head queue;
 458	struct aoe_hdr *h;
 459	struct aoetgt *t;
 460	char buf[128];
 461	u32 n;
 462
 463	t = f->t;
 464	n = newtag(d);
 465	skb = f->skb;
 466	if (ifrotate(t) == NULL) {
 467		/* probably can't happen, but set it up to fail anyway */
 468		pr_info("aoe: resend: no interfaces to rotate to.\n");
 469		ktcomplete(f, NULL);
 470		return;
 471	}
 472	h = (struct aoe_hdr *) skb_mac_header(skb);
 473
 474	if (!(f->flags & FFL_PROBE)) {
 475		snprintf(buf, sizeof(buf),
 476			"%15s e%ld.%d oldtag=%08x@%08lx newtag=%08x s=%pm d=%pm nout=%d\n",
 477			"retransmit", d->aoemajor, d->aoeminor,
 478			f->tag, jiffies, n,
 479			h->src, h->dst, t->nout);
 480		aoechr_error(buf);
 481	}
 482
 483	f->tag = n;
 484	fhash(f);
 485	h->tag = cpu_to_be32(n);
 486	memcpy(h->dst, t->addr, sizeof h->dst);
 487	memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
 488
 489	dev_hold(t->ifp->nd);
 490	skb->dev = t->ifp->nd;
 491	skb = skb_clone(skb, GFP_ATOMIC);
 492	if (skb == NULL) {
 493		dev_put(t->ifp->nd);
 494		return;
 495	}
 496	f->sent = ktime_get();
 497	__skb_queue_head_init(&queue);
 498	__skb_queue_tail(&queue, skb);
 499	aoenet_xmit(&queue);
 500}
 501
 502static int
 503tsince_hr(struct frame *f)
 504{
 505	u64 delta = ktime_to_ns(ktime_sub(ktime_get(), f->sent));
 506
 507	/* delta is normally under 4.2 seconds, avoid 64-bit division */
 508	if (likely(delta <= UINT_MAX))
 509		return (u32)delta / NSEC_PER_USEC;
 510
 511	/* avoid overflow after 71 minutes */
 512	if (delta > ((u64)INT_MAX * NSEC_PER_USEC))
 513		return INT_MAX;
 514
 515	return div_u64(delta, NSEC_PER_USEC);
 516}
 517
 518static int
 519tsince(u32 tag)
 520{
 521	int n;
 522
 523	n = jiffies & 0xffff;
 524	n -= tag & 0xffff;
 525	if (n < 0)
 526		n += 1<<16;
 527	return jiffies_to_usecs(n + 1);
 528}
 529
 530static struct aoeif *
 531getif(struct aoetgt *t, struct net_device *nd)
 532{
 533	struct aoeif *p, *e;
 534
 535	p = t->ifs;
 536	e = p + NAOEIFS;
 537	for (; p < e; p++)
 538		if (p->nd == nd)
 539			return p;
 540	return NULL;
 541}
 542
 543static void
 544ejectif(struct aoetgt *t, struct aoeif *ifp)
 545{
 546	struct aoeif *e;
 547	struct net_device *nd;
 548	ulong n;
 549
 550	nd = ifp->nd;
 551	e = t->ifs + NAOEIFS - 1;
 552	n = (e - ifp) * sizeof *ifp;
 553	memmove(ifp, ifp+1, n);
 554	e->nd = NULL;
 555	dev_put(nd);
 556}
 557
 558static struct frame *
 559reassign_frame(struct frame *f)
 560{
 561	struct frame *nf;
 562	struct sk_buff *skb;
 563
 564	nf = newframe(f->t->d);
 565	if (!nf)
 566		return NULL;
 567	if (nf->t == f->t) {
 568		aoe_freetframe(nf);
 569		return NULL;
 570	}
 571
 572	skb = nf->skb;
 573	nf->skb = f->skb;
 574	nf->buf = f->buf;
 575	nf->iter = f->iter;
 576	nf->waited = 0;
 577	nf->waited_total = f->waited_total;
 578	nf->sent = f->sent;
 579	f->skb = skb;
 580
 581	return nf;
 582}
 583
 584static void
 585probe(struct aoetgt *t)
 586{
 587	struct aoedev *d;
 588	struct frame *f;
 589	struct sk_buff *skb;
 590	struct sk_buff_head queue;
 591	size_t n, m;
 592	int frag;
 593
 594	d = t->d;
 595	f = newtframe(d, t);
 596	if (!f) {
 597		pr_err("%s %pm for e%ld.%d: %s\n",
 598			"aoe: cannot probe remote address",
 599			t->addr,
 600			(long) d->aoemajor, d->aoeminor,
 601			"no frame available");
 602		return;
 603	}
 604	f->flags |= FFL_PROBE;
 605	ifrotate(t);
 606	f->iter.bi_size = t->d->maxbcnt ? t->d->maxbcnt : DEFAULTBCNT;
 607	ata_rw_frameinit(f);
 608	skb = f->skb;
 609	for (frag = 0, n = f->iter.bi_size; n > 0; ++frag, n -= m) {
 610		if (n < PAGE_SIZE)
 611			m = n;
 612		else
 613			m = PAGE_SIZE;
 614		skb_fill_page_desc(skb, frag, empty_page, 0, m);
 615	}
 616	skb->len += f->iter.bi_size;
 617	skb->data_len = f->iter.bi_size;
 618	skb->truesize += f->iter.bi_size;
 619
 620	skb = skb_clone(f->skb, GFP_ATOMIC);
 621	if (skb) {
 622		f->sent = ktime_get();
 623		__skb_queue_head_init(&queue);
 624		__skb_queue_tail(&queue, skb);
 625		aoenet_xmit(&queue);
 626	} else {
 627		dev_put(f->t->ifp->nd);
 628	}
 629}
 630
 631static long
 632rto(struct aoedev *d)
 633{
 634	long t;
 635
 636	t = 2 * d->rttavg >> RTTSCALE;
 637	t += 8 * d->rttdev >> RTTDSCALE;
 638	if (t == 0)
 639		t = 1;
 640
 641	return t;
 642}
 643
 644static void
 645rexmit_deferred(struct aoedev *d)
 646{
 647	struct aoetgt *t;
 648	struct frame *f;
 649	struct frame *nf;
 650	struct list_head *pos, *nx, *head;
 651	int since;
 652	int untainted;
 653
 654	count_targets(d, &untainted);
 655
 656	head = &d->rexmitq;
 657	list_for_each_safe(pos, nx, head) {
 658		f = list_entry(pos, struct frame, head);
 659		t = f->t;
 660		if (t->taint) {
 661			if (!(f->flags & FFL_PROBE)) {
 662				nf = reassign_frame(f);
 663				if (nf) {
 664					if (t->nout_probes == 0
 665					&& untainted > 0) {
 666						probe(t);
 667						t->nout_probes++;
 668					}
 669					list_replace(&f->head, &nf->head);
 670					pos = &nf->head;
 671					aoe_freetframe(f);
 672					f = nf;
 673					t = f->t;
 674				}
 675			} else if (untainted < 1) {
 676				/* don't probe w/o other untainted aoetgts */
 677				goto stop_probe;
 678			} else if (tsince_hr(f) < t->taint * rto(d)) {
 679				/* reprobe slowly when taint is high */
 680				continue;
 681			}
 682		} else if (f->flags & FFL_PROBE) {
 683stop_probe:		/* don't probe untainted aoetgts */
 684			list_del(pos);
 685			aoe_freetframe(f);
 686			/* leaving d->kicked, because this is routine */
 687			f->t->d->flags |= DEVFL_KICKME;
 688			continue;
 689		}
 690		if (t->nout >= t->maxout)
 691			continue;
 692		list_del(pos);
 693		t->nout++;
 694		if (f->flags & FFL_PROBE)
 695			t->nout_probes++;
 696		since = tsince_hr(f);
 697		f->waited += since;
 698		f->waited_total += since;
 699		resend(d, f);
 700	}
 701}
 702
 703/* An aoetgt accumulates demerits quickly, and successful
 704 * probing redeems the aoetgt slowly.
 705 */
 706static void
 707scorn(struct aoetgt *t)
 708{
 709	int n;
 710
 711	n = t->taint++;
 712	t->taint += t->taint * 2;
 713	if (n > t->taint)
 714		t->taint = n;
 715	if (t->taint > MAX_TAINT)
 716		t->taint = MAX_TAINT;
 717}
 718
 719static int
 720count_targets(struct aoedev *d, int *untainted)
 721{
 722	int i, good;
 723
 724	for (i = good = 0; i < d->ntargets && d->targets[i]; ++i)
 725		if (d->targets[i]->taint == 0)
 726			good++;
 727
 728	if (untainted)
 729		*untainted = good;
 730	return i;
 731}
 732
 733static void
 734rexmit_timer(struct timer_list *timer)
 735{
 736	struct aoedev *d;
 737	struct aoetgt *t;
 738	struct aoeif *ifp;
 739	struct frame *f;
 740	struct list_head *head, *pos, *nx;
 741	LIST_HEAD(flist);
 742	register long timeout;
 743	ulong flags, n;
 744	int i;
 745	int utgts;	/* number of aoetgt descriptors (not slots) */
 746	int since;
 747
 748	d = timer_container_of(d, timer, timer);
 749
 750	spin_lock_irqsave(&d->lock, flags);
 751
 752	/* timeout based on observed timings and variations */
 753	timeout = rto(d);
 754
 755	utgts = count_targets(d, NULL);
 756
 757	if (d->flags & (DEVFL_TKILL | DEVFL_DEAD)) {
 758		spin_unlock_irqrestore(&d->lock, flags);
 759		return;
 760	}
 761
 762	/* collect all frames to rexmit into flist */
 763	for (i = 0; i < NFACTIVE; i++) {
 764		head = &d->factive[i];
 765		list_for_each_safe(pos, nx, head) {
 766			f = list_entry(pos, struct frame, head);
 767			if (tsince_hr(f) < timeout)
 768				break;	/* end of expired frames */
 769			/* move to flist for later processing */
 770			list_move_tail(pos, &flist);
 771		}
 772	}
 773
 774	/* process expired frames */
 775	while (!list_empty(&flist)) {
 776		pos = flist.next;
 777		f = list_entry(pos, struct frame, head);
 778		since = tsince_hr(f);
 779		n = f->waited_total + since;
 780		n /= USEC_PER_SEC;
 781		if (aoe_deadsecs
 782		&& n > aoe_deadsecs
 783		&& !(f->flags & FFL_PROBE)) {
 784			/* Waited too long.  Device failure.
 785			 * Hang all frames on first hash bucket for downdev
 786			 * to clean up.
 787			 */
 788			list_splice(&flist, &d->factive[0]);
 789			d->flags |= DEVFL_DEAD;
 790			queue_work(aoe_wq, &d->work);
 791			goto out;
 792		}
 793
 794		t = f->t;
 795		n = f->waited + since;
 796		n /= USEC_PER_SEC;
 797		if (aoe_deadsecs && utgts > 0
 798		&& (n > aoe_deadsecs / utgts || n > HARD_SCORN_SECS))
 799			scorn(t); /* avoid this target */
 800
 801		if (t->maxout != 1) {
 802			t->ssthresh = t->maxout / 2;
 803			t->maxout = 1;
 804		}
 805
 806		if (f->flags & FFL_PROBE) {
 807			t->nout_probes--;
 808		} else {
 809			ifp = getif(t, f->skb->dev);
 810			if (ifp && ++ifp->lost > (t->nframes << 1)
 811			&& (ifp != t->ifs || t->ifs[1].nd)) {
 812				ejectif(t, ifp);
 813				ifp = NULL;
 814			}
 815		}
 816		list_move_tail(pos, &d->rexmitq);
 817		t->nout--;
 818	}
 819	rexmit_deferred(d);
 820
 821out:
 822	if ((d->flags & DEVFL_KICKME) && d->blkq) {
 823		d->flags &= ~DEVFL_KICKME;
 824		blk_mq_run_hw_queues(d->blkq, true);
 825	}
 826
 827	d->timer.expires = jiffies + TIMERTICK;
 828	add_timer(&d->timer);
 829
 830	spin_unlock_irqrestore(&d->lock, flags);
 831}
 832
 833static void
 834bufinit(struct buf *buf, struct request *rq, struct bio *bio)
 835{
 836	memset(buf, 0, sizeof(*buf));
 837	buf->rq = rq;
 838	buf->bio = bio;
 839	buf->iter = bio->bi_iter;
 840}
 841
 842static struct buf *
 843nextbuf(struct aoedev *d)
 844{
 845	struct request *rq;
 846	struct request_queue *q;
 847	struct aoe_req *req;
 848	struct buf *buf;
 849	struct bio *bio;
 850
 851	q = d->blkq;
 852	if (q == NULL)
 853		return NULL;	/* initializing */
 854	if (d->ip.buf)
 855		return d->ip.buf;
 856	rq = d->ip.rq;
 857	if (rq == NULL) {
 858		rq = list_first_entry_or_null(&d->rq_list, struct request,
 859						queuelist);
 860		if (rq == NULL)
 861			return NULL;
 862		list_del_init(&rq->queuelist);
 863		blk_mq_start_request(rq);
 864		d->ip.rq = rq;
 865		d->ip.nxbio = rq->bio;
 866
 867		req = blk_mq_rq_to_pdu(rq);
 868		req->nr_bios = 0;
 869		__rq_for_each_bio(bio, rq)
 870			req->nr_bios++;
 871	}
 872	buf = mempool_alloc(d->bufpool, GFP_ATOMIC);
 873	if (buf == NULL) {
 874		pr_err("aoe: nextbuf: unable to mempool_alloc!\n");
 875		return NULL;
 876	}
 877	bio = d->ip.nxbio;
 878	bufinit(buf, rq, bio);
 879	bio = bio->bi_next;
 880	d->ip.nxbio = bio;
 881	if (bio == NULL)
 882		d->ip.rq = NULL;
 883	return d->ip.buf = buf;
 884}
 885
 886/* enters with d->lock held */
 887void
 888aoecmd_work(struct aoedev *d)
 889{
 890	rexmit_deferred(d);
 891	while (aoecmd_ata_rw(d))
 892		;
 893}
 894
 895/* this function performs work that has been deferred until sleeping is OK
 896 */
 897void
 898aoecmd_sleepwork(struct work_struct *work)
 899{
 900	struct aoedev *d = container_of(work, struct aoedev, work);
 901
 902	if (d->flags & DEVFL_DEAD)
 903		aoedev_downdev(d);
 904
 905	if (d->flags & DEVFL_GDALLOC)
 906		aoeblk_gdalloc(d);
 907
 908	if (d->flags & DEVFL_NEWSIZE) {
 909		set_capacity_and_notify(d->gd, d->ssize);
 910
 911		spin_lock_irq(&d->lock);
 912		d->flags |= DEVFL_UP;
 913		d->flags &= ~DEVFL_NEWSIZE;
 914		spin_unlock_irq(&d->lock);
 915	}
 916}
 917
 918static void
 919ata_ident_fixstring(u16 *id, int ns)
 920{
 921	u16 s;
 922
 923	while (ns-- > 0) {
 924		s = *id;
 925		*id++ = s >> 8 | s << 8;
 926	}
 927}
 928
 929static void
 930ataid_complete(struct aoedev *d, struct aoetgt *t, unsigned char *id)
 931{
 932	u64 ssize;
 933	u16 n;
 934
 935	/* word 83: command set supported */
 936	n = get_unaligned_le16(&id[83 << 1]);
 937
 938	/* word 86: command set/feature enabled */
 939	n |= get_unaligned_le16(&id[86 << 1]);
 940
 941	if (n & (1<<10)) {	/* bit 10: LBA 48 */
 942		d->flags |= DEVFL_EXT;
 943
 944		/* word 100: number lba48 sectors */
 945		ssize = get_unaligned_le64(&id[100 << 1]);
 946
 947		/* set as in ide-disk.c:init_idedisk_capacity */
 948		d->geo.cylinders = ssize;
 949		d->geo.cylinders /= (255 * 63);
 950		d->geo.heads = 255;
 951		d->geo.sectors = 63;
 952	} else {
 953		d->flags &= ~DEVFL_EXT;
 954
 955		/* number lba28 sectors */
 956		ssize = get_unaligned_le32(&id[60 << 1]);
 957
 958		/* NOTE: obsolete in ATA 6 */
 959		d->geo.cylinders = get_unaligned_le16(&id[54 << 1]);
 960		d->geo.heads = get_unaligned_le16(&id[55 << 1]);
 961		d->geo.sectors = get_unaligned_le16(&id[56 << 1]);
 962	}
 963
 964	ata_ident_fixstring((u16 *) &id[10<<1], 10);	/* serial */
 965	ata_ident_fixstring((u16 *) &id[23<<1], 4);	/* firmware */
 966	ata_ident_fixstring((u16 *) &id[27<<1], 20);	/* model */
 967	memcpy(d->ident, id, sizeof(d->ident));
 968
 969	if (d->ssize != ssize)
 970		printk(KERN_INFO
 971			"aoe: %pm e%ld.%d v%04x has %llu sectors\n",
 972			t->addr,
 973			d->aoemajor, d->aoeminor,
 974			d->fw_ver, (long long)ssize);
 975	d->ssize = ssize;
 976	d->geo.start = 0;
 977	if (d->flags & (DEVFL_GDALLOC|DEVFL_NEWSIZE))
 978		return;
 979	if (d->gd != NULL)
 980		d->flags |= DEVFL_NEWSIZE;
 981	else
 982		d->flags |= DEVFL_GDALLOC;
 983	queue_work(aoe_wq, &d->work);
 984}
 985
 986static void
 987calc_rttavg(struct aoedev *d, struct aoetgt *t, int rtt)
 988{
 989	register long n;
 990
 991	n = rtt;
 992
 993	/* cf. Congestion Avoidance and Control, Jacobson & Karels, 1988 */
 994	n -= d->rttavg >> RTTSCALE;
 995	d->rttavg += n;
 996	if (n < 0)
 997		n = -n;
 998	n -= d->rttdev >> RTTDSCALE;
 999	d->rttdev += n;
1000
1001	if (!t || t->maxout >= t->nframes)
1002		return;
1003	if (t->maxout < t->ssthresh)
1004		t->maxout += 1;
1005	else if (t->nout == t->maxout && t->next_cwnd-- == 0) {
1006		t->maxout += 1;
1007		t->next_cwnd = t->maxout;
1008	}
1009}
1010
1011static struct aoetgt *
1012gettgt(struct aoedev *d, char *addr)
1013{
1014	struct aoetgt **t, **e;
1015
1016	t = d->targets;
1017	e = t + d->ntargets;
1018	for (; t < e && *t; t++)
1019		if (memcmp((*t)->addr, addr, sizeof((*t)->addr)) == 0)
1020			return *t;
1021	return NULL;
1022}
1023
1024static void
1025bvcpy(struct sk_buff *skb, struct bio *bio, struct bvec_iter iter, long cnt)
1026{
1027	int soff = 0;
1028	struct bio_vec bv;
1029
1030	iter.bi_size = cnt;
1031
1032	__bio_for_each_segment(bv, bio, iter, iter) {
1033		char *p = bvec_kmap_local(&bv);
1034		skb_copy_bits(skb, soff, p, bv.bv_len);
1035		kunmap_local(p);
1036		soff += bv.bv_len;
1037	}
1038}
1039
1040void
1041aoe_end_request(struct aoedev *d, struct request *rq, int fastfail)
1042{
1043	struct bio *bio;
1044	int bok;
1045	struct request_queue *q;
1046	blk_status_t err = BLK_STS_OK;
1047
1048	q = d->blkq;
1049	if (rq == d->ip.rq)
1050		d->ip.rq = NULL;
1051	do {
1052		bio = rq->bio;
1053		bok = !fastfail && !bio->bi_status;
1054		if (!bok)
1055			err = BLK_STS_IOERR;
1056	} while (blk_update_request(rq, bok ? BLK_STS_OK : BLK_STS_IOERR, bio->bi_iter.bi_size));
1057
1058	__blk_mq_end_request(rq, err);
1059
1060	/* cf. https://lore.kernel.org/lkml/20061031071040.GS14055@kernel.dk/ */
1061	if (!fastfail)
1062		blk_mq_run_hw_queues(q, true);
1063}
1064
1065static void
1066aoe_end_buf(struct aoedev *d, struct buf *buf)
1067{
1068	struct request *rq = buf->rq;
1069	struct aoe_req *req = blk_mq_rq_to_pdu(rq);
1070
1071	if (buf == d->ip.buf)
1072		d->ip.buf = NULL;
1073	mempool_free(buf, d->bufpool);
1074	if (--req->nr_bios == 0)
1075		aoe_end_request(d, rq, 0);
1076}
1077
1078static void
1079ktiocomplete(struct frame *f)
1080{
1081	struct aoe_hdr *hin, *hout;
1082	struct aoe_atahdr *ahin, *ahout;
1083	struct buf *buf;
1084	struct sk_buff *skb;
1085	struct aoetgt *t;
1086	struct aoeif *ifp;
1087	struct aoedev *d;
1088	long n;
1089	int untainted;
1090
1091	if (f == NULL)
1092		return;
1093
1094	t = f->t;
1095	d = t->d;
1096	skb = f->r_skb;
1097	buf = f->buf;
1098	if (f->flags & FFL_PROBE)
1099		goto out;
1100	if (!skb)		/* just fail the buf. */
1101		goto noskb;
1102
1103	hout = (struct aoe_hdr *) skb_mac_header(f->skb);
1104	ahout = (struct aoe_atahdr *) (hout+1);
1105
1106	hin = (struct aoe_hdr *) skb->data;
1107	skb_pull(skb, sizeof(*hin));
1108	ahin = (struct aoe_atahdr *) skb->data;
1109	skb_pull(skb, sizeof(*ahin));
1110	if (ahin->cmdstat & 0xa9) {	/* these bits cleared on success */
1111		pr_err("aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n",
1112			ahout->cmdstat, ahin->cmdstat,
1113			d->aoemajor, d->aoeminor);
1114noskb:		if (buf)
1115			buf->bio->bi_status = BLK_STS_IOERR;
1116		goto out;
1117	}
1118
1119	n = ahout->scnt << 9;
1120	switch (ahout->cmdstat) {
1121	case ATA_CMD_PIO_READ:
1122	case ATA_CMD_PIO_READ_EXT:
1123		if (skb->len < n) {
1124			pr_err("%s e%ld.%d.  skb->len=%d need=%ld\n",
1125				"aoe: runt data size in read from",
1126				(long) d->aoemajor, d->aoeminor,
1127			       skb->len, n);
1128			buf->bio->bi_status = BLK_STS_IOERR;
1129			break;
1130		}
1131		if (n > f->iter.bi_size) {
1132			pr_err_ratelimited("%s e%ld.%d.  bytes=%ld need=%u\n",
1133				"aoe: too-large data size in read from",
1134				(long) d->aoemajor, d->aoeminor,
1135				n, f->iter.bi_size);
1136			buf->bio->bi_status = BLK_STS_IOERR;
1137			break;
1138		}
1139		bvcpy(skb, f->buf->bio, f->iter, n);
1140		fallthrough;
1141	case ATA_CMD_PIO_WRITE:
1142	case ATA_CMD_PIO_WRITE_EXT:
1143		spin_lock_irq(&d->lock);
1144		ifp = getif(t, skb->dev);
1145		if (ifp)
1146			ifp->lost = 0;
1147		spin_unlock_irq(&d->lock);
1148		break;
1149	case ATA_CMD_ID_ATA:
1150		if (skb->len < 512) {
1151			pr_info("%s e%ld.%d.  skb->len=%d need=512\n",
1152				"aoe: runt data size in ataid from",
1153				(long) d->aoemajor, d->aoeminor,
1154				skb->len);
1155			break;
1156		}
1157		if (skb_linearize(skb))
1158			break;
1159		spin_lock_irq(&d->lock);
1160		ataid_complete(d, t, skb->data);
1161		spin_unlock_irq(&d->lock);
1162		break;
1163	default:
1164		pr_info("aoe: unrecognized ata command %2.2Xh for %d.%d\n",
1165			ahout->cmdstat,
1166			be16_to_cpu(get_unaligned(&hin->major)),
1167			hin->minor);
1168	}
1169out:
1170	spin_lock_irq(&d->lock);
1171	if (t->taint > 0
1172	&& --t->taint > 0
1173	&& t->nout_probes == 0) {
1174		count_targets(d, &untainted);
1175		if (untainted > 0) {
1176			probe(t);
1177			t->nout_probes++;
1178		}
1179	}
1180
1181	aoe_freetframe(f);
1182
1183	if (buf && --buf->nframesout == 0 && buf->iter.bi_size == 0)
1184		aoe_end_buf(d, buf);
1185
1186	spin_unlock_irq(&d->lock);
1187	aoedev_put(d);
1188	dev_kfree_skb(skb);
1189}
1190
1191/* Enters with iocq.lock held.
1192 * Returns true iff responses needing processing remain.
1193 */
1194static int
1195ktio(int id)
1196{
1197	struct frame *f;
1198	struct list_head *pos;
1199	int i;
1200	int actual_id;
1201
1202	for (i = 0; ; ++i) {
1203		if (i == MAXIOC)
1204			return 1;
1205		if (list_empty(&iocq[id].head))
1206			return 0;
1207		pos = iocq[id].head.next;
1208		list_del(pos);
1209		f = list_entry(pos, struct frame, head);
1210		spin_unlock_irq(&iocq[id].lock);
1211		ktiocomplete(f);
1212
1213		/* Figure out if extra threads are required. */
1214		actual_id = f->t->d->aoeminor % ncpus;
1215
1216		if (!kts[actual_id].active) {
1217			BUG_ON(id != 0);
1218			mutex_lock(&ktio_spawn_lock);
1219			if (!kts[actual_id].active
1220				&& aoe_ktstart(&kts[actual_id]) == 0)
1221				kts[actual_id].active = 1;
1222			mutex_unlock(&ktio_spawn_lock);
1223		}
1224		spin_lock_irq(&iocq[id].lock);
1225	}
1226}
1227
1228static int
1229kthread(void *vp)
1230{
1231	struct ktstate *k;
1232	DECLARE_WAITQUEUE(wait, current);
1233	int more;
1234
1235	k = vp;
1236	current->flags |= PF_NOFREEZE;
1237	set_user_nice(current, -10);
1238	complete(&k->rendez);	/* tell spawner we're running */
1239	do {
1240		spin_lock_irq(k->lock);
1241		more = k->fn(k->id);
1242		if (!more) {
1243			add_wait_queue(k->waitq, &wait);
1244			__set_current_state(TASK_INTERRUPTIBLE);
1245		}
1246		spin_unlock_irq(k->lock);
1247		if (!more) {
1248			schedule();
1249			remove_wait_queue(k->waitq, &wait);
1250		} else
1251			cond_resched();
1252	} while (!kthread_should_stop());
1253	complete(&k->rendez);	/* tell spawner we're stopping */
1254	return 0;
1255}
1256
1257void
1258aoe_ktstop(struct ktstate *k)
1259{
1260	kthread_stop(k->task);
1261	wait_for_completion(&k->rendez);
1262}
1263
1264int
1265aoe_ktstart(struct ktstate *k)
1266{
1267	struct task_struct *task;
1268
1269	init_completion(&k->rendez);
1270	task = kthread_run(kthread, k, "%s", k->name);
1271	if (task == NULL || IS_ERR(task))
1272		return -ENOMEM;
1273	k->task = task;
1274	wait_for_completion(&k->rendez); /* allow kthread to start */
1275	init_completion(&k->rendez);	/* for waiting for exit later */
1276	return 0;
1277}
1278
1279/* pass it off to kthreads for processing */
1280static void
1281ktcomplete(struct frame *f, struct sk_buff *skb)
1282{
1283	int id;
1284	ulong flags;
1285
1286	f->r_skb = skb;
1287	id = f->t->d->aoeminor % ncpus;
1288	spin_lock_irqsave(&iocq[id].lock, flags);
1289	if (!kts[id].active) {
1290		spin_unlock_irqrestore(&iocq[id].lock, flags);
1291		/* The thread with id has not been spawned yet,
1292		 * so delegate the work to the main thread and
1293		 * try spawning a new thread.
1294		 */
1295		id = 0;
1296		spin_lock_irqsave(&iocq[id].lock, flags);
1297	}
1298	list_add_tail(&f->head, &iocq[id].head);
1299	spin_unlock_irqrestore(&iocq[id].lock, flags);
1300	wake_up(&ktiowq[id]);
1301}
1302
1303struct sk_buff *
1304aoecmd_ata_rsp(struct sk_buff *skb)
1305{
1306	struct aoedev *d;
1307	struct aoe_hdr *h;
1308	struct frame *f;
1309	u32 n;
1310	ulong flags;
1311	char ebuf[128];
1312	u16 aoemajor;
1313
1314	h = (struct aoe_hdr *) skb->data;
1315	aoemajor = be16_to_cpu(get_unaligned(&h->major));
1316	d = aoedev_by_aoeaddr(aoemajor, h->minor, 0);
1317	if (d == NULL) {
1318		snprintf(ebuf, sizeof ebuf, "aoecmd_ata_rsp: ata response "
1319			"for unknown device %d.%d\n",
1320			aoemajor, h->minor);
1321		aoechr_error(ebuf);
1322		return skb;
1323	}
1324
1325	spin_lock_irqsave(&d->lock, flags);
1326
1327	n = be32_to_cpu(get_unaligned(&h->tag));
1328	f = getframe(d, n);
1329	if (f) {
1330		calc_rttavg(d, f->t, tsince_hr(f));
1331		f->t->nout--;
1332		if (f->flags & FFL_PROBE)
1333			f->t->nout_probes--;
1334	} else {
1335		f = getframe_deferred(d, n);
1336		if (f) {
1337			calc_rttavg(d, NULL, tsince_hr(f));
1338		} else {
1339			calc_rttavg(d, NULL, tsince(n));
1340			spin_unlock_irqrestore(&d->lock, flags);
1341			aoedev_put(d);
1342			snprintf(ebuf, sizeof(ebuf),
1343				 "%15s e%d.%d    tag=%08x@%08lx s=%pm d=%pm\n",
1344				 "unexpected rsp",
1345				 get_unaligned_be16(&h->major),
1346				 h->minor,
1347				 get_unaligned_be32(&h->tag),
1348				 jiffies,
1349				 h->src,
1350				 h->dst);
1351			aoechr_error(ebuf);
1352			return skb;
1353		}
1354	}
1355	aoecmd_work(d);
1356
1357	spin_unlock_irqrestore(&d->lock, flags);
1358
1359	ktcomplete(f, skb);
1360
1361	/*
1362	 * Note here that we do not perform an aoedev_put, as we are
1363	 * leaving this reference for the ktio to release.
1364	 */
1365	return NULL;
1366}
1367
1368void
1369aoecmd_cfg(ushort aoemajor, unsigned char aoeminor)
1370{
1371	struct sk_buff_head queue;
1372
1373	__skb_queue_head_init(&queue);
1374	aoecmd_cfg_pkts(aoemajor, aoeminor, &queue);
1375	aoenet_xmit(&queue);
1376}
1377
1378struct sk_buff *
1379aoecmd_ata_id(struct aoedev *d)
1380{
1381	struct aoe_hdr *h;
1382	struct aoe_atahdr *ah;
1383	struct frame *f;
1384	struct sk_buff *skb;
1385	struct aoetgt *t;
1386
1387	f = newframe(d);
1388	if (f == NULL)
1389		return NULL;
1390
1391	t = *d->tgt;
1392
1393	/* initialize the headers & frame */
1394	skb = f->skb;
1395	h = (struct aoe_hdr *) skb_mac_header(skb);
1396	ah = (struct aoe_atahdr *) (h+1);
1397	skb_put(skb, sizeof *h + sizeof *ah);
1398	memset(h, 0, skb->len);
1399	f->tag = aoehdr_atainit(d, t, h);
1400	fhash(f);
1401	t->nout++;
1402	f->waited = 0;
1403	f->waited_total = 0;
1404
1405	/* set up ata header */
1406	ah->scnt = 1;
1407	ah->cmdstat = ATA_CMD_ID_ATA;
1408	ah->lba3 = 0xa0;
1409
1410	dev_hold(t->ifp->nd);
1411	skb->dev = t->ifp->nd;
1412
1413	d->rttavg = RTTAVG_INIT;
1414	d->rttdev = RTTDEV_INIT;
1415	d->timer.function = rexmit_timer;
1416
1417	skb = skb_clone(skb, GFP_ATOMIC);
1418	if (skb)
1419		f->sent = ktime_get();
1420	else
1421		dev_put(t->ifp->nd);
1422
1423	return skb;
1424}
1425
1426static struct aoetgt **
1427grow_targets(struct aoedev *d)
1428{
1429	ulong oldn, newn;
1430	struct aoetgt **tt;
1431
1432	oldn = d->ntargets;
1433	newn = oldn * 2;
1434	tt = kcalloc(newn, sizeof(*d->targets), GFP_ATOMIC);
1435	if (!tt)
1436		return NULL;
1437	memmove(tt, d->targets, sizeof(*d->targets) * oldn);
1438	d->tgt = tt + (d->tgt - d->targets);
1439	kfree(d->targets);
1440	d->targets = tt;
1441	d->ntargets = newn;
1442
1443	return &d->targets[oldn];
1444}
1445
1446static struct aoetgt *
1447addtgt(struct aoedev *d, char *addr, ulong nframes)
1448{
1449	struct aoetgt *t, **tt, **te;
1450
1451	tt = d->targets;
1452	te = tt + d->ntargets;
1453	for (; tt < te && *tt; tt++)
1454		;
1455
1456	if (tt == te) {
1457		tt = grow_targets(d);
1458		if (!tt)
1459			goto nomem;
1460	}
1461	t = kzalloc(sizeof(*t), GFP_ATOMIC);
1462	if (!t)
1463		goto nomem;
1464	t->nframes = nframes;
1465	t->d = d;
1466	memcpy(t->addr, addr, sizeof t->addr);
1467	t->ifp = t->ifs;
1468	aoecmd_wreset(t);
1469	t->maxout = t->nframes / 2;
1470	INIT_LIST_HEAD(&t->ffree);
1471	return *tt = t;
1472
1473 nomem:
1474	pr_info("aoe: cannot allocate memory to add target\n");
1475	return NULL;
1476}
1477
1478static void
1479setdbcnt(struct aoedev *d)
1480{
1481	struct aoetgt **t, **e;
1482	int bcnt = 0;
1483
1484	t = d->targets;
1485	e = t + d->ntargets;
1486	for (; t < e && *t; t++)
1487		if (bcnt == 0 || bcnt > (*t)->minbcnt)
1488			bcnt = (*t)->minbcnt;
1489	if (bcnt != d->maxbcnt) {
1490		d->maxbcnt = bcnt;
1491		pr_info("aoe: e%ld.%d: setting %d byte data frames\n",
1492			d->aoemajor, d->aoeminor, bcnt);
1493	}
1494}
1495
1496static void
1497setifbcnt(struct aoetgt *t, struct net_device *nd, int bcnt)
1498{
1499	struct aoedev *d;
1500	struct aoeif *p, *e;
1501	int minbcnt;
1502
1503	d = t->d;
1504	minbcnt = bcnt;
1505	p = t->ifs;
1506	e = p + NAOEIFS;
1507	for (; p < e; p++) {
1508		if (p->nd == NULL)
1509			break;		/* end of the valid interfaces */
1510		if (p->nd == nd) {
1511			p->bcnt = bcnt;	/* we're updating */
1512			nd = NULL;
1513		} else if (minbcnt > p->bcnt)
1514			minbcnt = p->bcnt; /* find the min interface */
1515	}
1516	if (nd) {
1517		if (p == e) {
1518			pr_err("aoe: device setifbcnt failure; too many interfaces.\n");
1519			return;
1520		}
1521		dev_hold(nd);
1522		p->nd = nd;
1523		p->bcnt = bcnt;
1524	}
1525	t->minbcnt = minbcnt;
1526	setdbcnt(d);
1527}
1528
1529void
1530aoecmd_cfg_rsp(struct sk_buff *skb)
1531{
1532	struct aoedev *d;
1533	struct aoe_hdr *h;
1534	struct aoe_cfghdr *ch;
1535	struct aoetgt *t;
1536	ulong flags, aoemajor;
1537	struct sk_buff *sl;
1538	struct sk_buff_head queue;
1539	u16 n;
1540
1541	sl = NULL;
1542	h = (struct aoe_hdr *) skb_mac_header(skb);
1543	ch = (struct aoe_cfghdr *) (h+1);
1544
1545	/*
1546	 * Enough people have their dip switches set backwards to
1547	 * warrant a loud message for this special case.
1548	 */
1549	aoemajor = get_unaligned_be16(&h->major);
1550	if (aoemajor == 0xfff) {
1551		printk(KERN_ERR "aoe: Warning: shelf address is all ones.  "
1552			"Check shelf dip switches.\n");
1553		return;
1554	}
1555	if (aoemajor == 0xffff) {
1556		pr_info("aoe: e%ld.%d: broadcast shelf number invalid\n",
1557			aoemajor, (int) h->minor);
1558		return;
1559	}
1560	if (h->minor == 0xff) {
1561		pr_info("aoe: e%ld.%d: broadcast slot number invalid\n",
1562			aoemajor, (int) h->minor);
1563		return;
1564	}
1565
1566	n = be16_to_cpu(ch->bufcnt);
1567	if (n > aoe_maxout)	/* keep it reasonable */
1568		n = aoe_maxout;
1569
1570	d = aoedev_by_aoeaddr(aoemajor, h->minor, 1);
1571	if (d == NULL) {
1572		pr_info("aoe: device allocation failure\n");
1573		return;
1574	}
1575
1576	spin_lock_irqsave(&d->lock, flags);
1577
1578	t = gettgt(d, h->src);
1579	if (t) {
1580		t->nframes = n;
1581		if (n < t->maxout)
1582			aoecmd_wreset(t);
1583	} else {
1584		t = addtgt(d, h->src, n);
1585		if (!t)
1586			goto bail;
1587	}
1588	n = skb->dev->mtu;
1589	n -= sizeof(struct aoe_hdr) + sizeof(struct aoe_atahdr);
1590	n /= 512;
1591	if (n > ch->scnt)
1592		n = ch->scnt;
1593	n = n ? n * 512 : DEFAULTBCNT;
1594	setifbcnt(t, skb->dev, n);
1595
1596	/* don't change users' perspective */
1597	if (d->nopen == 0) {
1598		d->fw_ver = be16_to_cpu(ch->fwver);
1599		sl = aoecmd_ata_id(d);
1600	}
1601bail:
1602	spin_unlock_irqrestore(&d->lock, flags);
1603	aoedev_put(d);
1604	if (sl) {
1605		__skb_queue_head_init(&queue);
1606		__skb_queue_tail(&queue, sl);
1607		aoenet_xmit(&queue);
1608	}
1609}
1610
1611void
1612aoecmd_wreset(struct aoetgt *t)
1613{
1614	t->maxout = 1;
1615	t->ssthresh = t->nframes / 2;
1616	t->next_cwnd = t->nframes;
1617}
1618
1619void
1620aoecmd_cleanslate(struct aoedev *d)
1621{
1622	struct aoetgt **t, **te;
1623
1624	d->rttavg = RTTAVG_INIT;
1625	d->rttdev = RTTDEV_INIT;
1626	d->maxbcnt = 0;
1627
1628	t = d->targets;
1629	te = t + d->ntargets;
1630	for (; t < te && *t; t++)
1631		aoecmd_wreset(*t);
1632}
1633
1634void
1635aoe_failbuf(struct aoedev *d, struct buf *buf)
1636{
1637	if (buf == NULL)
1638		return;
1639	buf->iter.bi_size = 0;
1640	buf->bio->bi_status = BLK_STS_IOERR;
1641	if (buf->nframesout == 0)
1642		aoe_end_buf(d, buf);
1643}
1644
1645void
1646aoe_flush_iocq(void)
1647{
1648	int i;
1649
1650	for (i = 0; i < ncpus; i++) {
1651		if (kts[i].active)
1652			aoe_flush_iocq_by_index(i);
1653	}
1654}
1655
1656void
1657aoe_flush_iocq_by_index(int id)
1658{
1659	struct frame *f;
1660	struct aoedev *d;
1661	LIST_HEAD(flist);
1662	struct list_head *pos;
1663	struct sk_buff *skb;
1664	ulong flags;
1665
1666	spin_lock_irqsave(&iocq[id].lock, flags);
1667	list_splice_init(&iocq[id].head, &flist);
1668	spin_unlock_irqrestore(&iocq[id].lock, flags);
1669	while (!list_empty(&flist)) {
1670		pos = flist.next;
1671		list_del(pos);
1672		f = list_entry(pos, struct frame, head);
1673		d = f->t->d;
1674		skb = f->r_skb;
1675		spin_lock_irqsave(&d->lock, flags);
1676		if (f->buf) {
1677			f->buf->nframesout--;
1678			aoe_failbuf(d, f->buf);
1679		}
1680		aoe_freetframe(f);
1681		spin_unlock_irqrestore(&d->lock, flags);
1682		dev_kfree_skb(skb);
1683		aoedev_put(d);
1684	}
1685}
1686
1687int __init
1688aoecmd_init(void)
1689{
1690	void *p;
1691	int i;
1692	int ret;
1693
1694	/* get_zeroed_page returns page with ref count 1 */
1695	p = (void *) get_zeroed_page(GFP_KERNEL);
1696	if (!p)
1697		return -ENOMEM;
1698	empty_page = virt_to_page(p);
1699
1700	ncpus = num_online_cpus();
1701
1702	iocq = kcalloc(ncpus, sizeof(struct iocq_ktio), GFP_KERNEL);
1703	if (!iocq)
1704		return -ENOMEM;
1705
1706	kts = kcalloc(ncpus, sizeof(struct ktstate), GFP_KERNEL);
1707	if (!kts) {
1708		ret = -ENOMEM;
1709		goto kts_fail;
1710	}
1711
1712	ktiowq = kcalloc(ncpus, sizeof(wait_queue_head_t), GFP_KERNEL);
1713	if (!ktiowq) {
1714		ret = -ENOMEM;
1715		goto ktiowq_fail;
1716	}
1717
1718	for (i = 0; i < ncpus; i++) {
1719		INIT_LIST_HEAD(&iocq[i].head);
1720		spin_lock_init(&iocq[i].lock);
1721		init_waitqueue_head(&ktiowq[i]);
1722		snprintf(kts[i].name, sizeof(kts[i].name), "aoe_ktio%d", i);
1723		kts[i].fn = ktio;
1724		kts[i].waitq = &ktiowq[i];
1725		kts[i].lock = &iocq[i].lock;
1726		kts[i].id = i;
1727		kts[i].active = 0;
1728	}
1729	kts[0].active = 1;
1730	if (aoe_ktstart(&kts[0])) {
1731		ret = -ENOMEM;
1732		goto ktstart_fail;
1733	}
1734	return 0;
1735
1736ktstart_fail:
1737	kfree(ktiowq);
1738ktiowq_fail:
1739	kfree(kts);
1740kts_fail:
1741	kfree(iocq);
1742
1743	return ret;
1744}
1745
1746void
1747aoecmd_exit(void)
1748{
1749	int i;
1750
1751	for (i = 0; i < ncpus; i++)
1752		if (kts[i].active)
1753			aoe_ktstop(&kts[i]);
1754
1755	aoe_flush_iocq();
1756
1757	/* Free up the iocq and thread speicific configuration
1758	* allocated during startup.
1759	*/
1760	kfree(iocq);
1761	kfree(kts);
1762	kfree(ktiowq);
1763
1764	__free_page(empty_page);
1765	empty_page = NULL;
1766}