[PATCH] MSI: MSI/MSI-X API updates

Second half of MSI rewrite: fix the API and update documentation.  Split
enabling MSI and MSI-X to separate pci_enable_msi()/pci_disable_msi() and
pci_enable_msix()/pci_disable_msix() functions.  free_irq() no longer has
the side effect of freeing interrupt vectors (so a device driver can do
multiple request_irq()/free_irq() cycles on the same MSI/MSI_X vector).

From: Tom L. Nguyen <tom.l.nguyen@intel.com>
Signed-off-by: Roland Dreier <roland@topspin.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

2 files changed, 471 insertions, 323 deletions

diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 3b909ce268348f..d4c89d73e16dda 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -154,13 +154,25 @@ static void unmask_MSI_irq(unsigned int vector)
 
 static unsigned int startup_msi_irq_wo_maskbit(unsigned int vector)
 {
+	struct msi_desc *entry;
+	unsigned long flags;
+
+	spin_lock_irqsave(&msi_lock, flags);
+	entry = msi_desc[vector];
+	if (!entry || !entry->dev) {
+		spin_unlock_irqrestore(&msi_lock, flags);
+		return 0;
+	}
+	entry->msi_attrib.state = 1;	/* Mark it active */
+	spin_unlock_irqrestore(&msi_lock, flags);
+
 	return 0;	/* never anything pending */
 }
 
-static void pci_disable_msi(unsigned int vector);
+static void release_msi(unsigned int vector);
 static void shutdown_msi_irq(unsigned int vector)
 {
-	pci_disable_msi(vector);
+	release_msi(vector);
 }
 
 #define shutdown_msi_irq_wo_maskbit	shutdown_msi_irq
@@ -175,6 +187,18 @@ static void end_msi_irq_wo_maskbit(unsigned int vector)
 
 static unsigned int startup_msi_irq_w_maskbit(unsigned int vector)
 {
+	struct msi_desc *entry;
+	unsigned long flags;
+
+	spin_lock_irqsave(&msi_lock, flags);
+	entry = msi_desc[vector];
+	if (!entry || !entry->dev) {
+		spin_unlock_irqrestore(&msi_lock, flags);
+		return 0;
+	}
+	entry->msi_attrib.state = 1;	/* Mark it active */
+	spin_unlock_irqrestore(&msi_lock, flags);
+
 	unmask_MSI_irq(vector);
 	return 0;	/* never anything pending */
 }
@@ -196,7 +220,7 @@ static void end_msi_irq_w_maskbit(unsigned int vector)
  * which implement the MSI-X Capability Structure.
  */
 static struct hw_interrupt_type msix_irq_type = {
-	.typename	= "PCI MSI-X",
+	.typename	= "PCI-MSI-X",
 	.startup	= startup_msi_irq_w_maskbit,
 	.shutdown	= shutdown_msi_irq_w_maskbit,
 	.enable		= enable_msi_irq_w_maskbit,
@@ -212,7 +236,7 @@ static struct hw_interrupt_type msix_irq_type = {
  * Mask-and-Pending Bits.
  */
 static struct hw_interrupt_type msi_irq_w_maskbit_type = {
-	.typename	= "PCI MSI",
+	.typename	= "PCI-MSI",
 	.startup	= startup_msi_irq_w_maskbit,
 	.shutdown	= shutdown_msi_irq_w_maskbit,
 	.enable		= enable_msi_irq_w_maskbit,
@@ -228,7 +252,7 @@ static struct hw_interrupt_type msi_irq_w_maskbit_type = {
  * Mask-and-Pending Bits.
  */
 static struct hw_interrupt_type msi_irq_wo_maskbit_type = {
-	.typename	= "PCI MSI",
+	.typename	= "PCI-MSI",
 	.startup	= startup_msi_irq_wo_maskbit,
 	.shutdown	= shutdown_msi_irq_wo_maskbit,
 	.enable		= enable_msi_irq_wo_maskbit,
@@ -261,6 +285,7 @@ static void msi_address_init(struct msg_address *msi_address)
 	msi_address->lo_address.value |= (MSI_TARGET_CPU << MSI_TARGET_CPU_SHIFT);
 }
 
+static int msi_free_vector(struct pci_dev* dev, int vector, int reassign);
 static int assign_msi_vector(void)
 {
 	static int new_vector_avail = 1;
@@ -274,6 +299,8 @@ static int assign_msi_vector(void)
 	spin_lock_irqsave(&msi_lock, flags);
 
 	if (!new_vector_avail) {
+		int free_vector = 0;
+
 		/*
 	 	 * vector_irq[] = -1 indicates that this specific vector is:
 	 	 * - assigned for MSI (since MSI have no associated IRQ) or
@@ -290,13 +317,34 @@ static int assign_msi_vector(void)
 		for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
 			if (vector_irq[vector] != 0)
 				continue;
-			vector_irq[vector] = -1;
-			nr_released_vectors--;
+			free_vector = vector;
+			if (!msi_desc[vector])
+			      	break;
+			else
+				continue;
+		}
+		if (!free_vector) {
 			spin_unlock_irqrestore(&msi_lock, flags);
-			return vector;
+			return -EBUSY;
 		}
+		vector_irq[free_vector] = -1;
+		nr_released_vectors--;
 		spin_unlock_irqrestore(&msi_lock, flags);
-		return -EBUSY;
+		if (msi_desc[free_vector] != NULL) {
+			struct pci_dev *dev;
+			int tail;
+
+			/* free all linked vectors before re-assign */
+			do {
+				spin_lock_irqsave(&msi_lock, flags);
+				dev = msi_desc[free_vector]->dev;
+				tail = msi_desc[free_vector]->link.tail;
+				spin_unlock_irqrestore(&msi_lock, flags);
+				msi_free_vector(dev, tail, 1);
+			} while (free_vector != tail);
+		}
+
+		return free_vector;
 	}
 	vector = assign_irq_vector(AUTO_ASSIGN);
 	last_alloc_vector = vector;
@@ -329,6 +377,15 @@ static int msi_init(void)
 		printk(KERN_INFO "WARNING: MSI INIT FAILURE\n");
 		return status;
 	}
+	last_alloc_vector = assign_irq_vector(AUTO_ASSIGN);
+	if (last_alloc_vector < 0) {
+		pci_msi_enable = 0;
+		printk(KERN_INFO "WARNING: ALL VECTORS ARE BUSY\n");
+		status = -EBUSY;
+		return status;
+	}
+	vector_irq[last_alloc_vector] = 0;
+	nr_released_vectors++;
 	printk(KERN_INFO "MSI INIT SUCCESS\n");
 
 	return status;
@@ -421,7 +478,7 @@ static void disable_msi_mode(struct pci_dev *dev, int pos, int type)
 	}
 }
 
-static int msi_lookup_vector(struct pci_dev *dev)
+static int msi_lookup_vector(struct pci_dev *dev, int type)
 {
 	int vector;
 	unsigned long flags;
@@ -429,11 +486,11 @@ static int msi_lookup_vector(struct pci_dev *dev)
 	spin_lock_irqsave(&msi_lock, flags);
 	for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
 		if (!msi_desc[vector] || msi_desc[vector]->dev != dev ||
-			msi_desc[vector]->msi_attrib.entry_nr ||
+			msi_desc[vector]->msi_attrib.type != type ||
 			msi_desc[vector]->msi_attrib.default_vector != dev->irq)
-			continue;	/* not entry 0, skip */
+			continue;
 		spin_unlock_irqrestore(&msi_lock, flags);
-		/* This pre-assigned entry-0 MSI vector for this device
+		/* This pre-assigned MSI vector for this device
 		   already exits. Override dev->irq with this vector */
 		dev->irq = vector;
 		return 0;
@@ -448,10 +505,9 @@ void pci_scan_msi_device(struct pci_dev *dev)
 	if (!dev)
 		return;
 
-   	if (pci_find_capability(dev, PCI_CAP_ID_MSIX) > 0) {
-		nr_reserved_vectors++;
+   	if (pci_find_capability(dev, PCI_CAP_ID_MSIX) > 0)
 		nr_msix_devices++;
-	} else if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0)
+	else if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0)
 		nr_reserved_vectors++;
 }
 
@@ -473,18 +529,7 @@ static int msi_capability_init(struct pci_dev *dev)
 	u16 control;
 
    	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
-	if (!pos)
-		return -EINVAL;
-
 	pci_read_config_word(dev, msi_control_reg(pos), &control);
-	if (control & PCI_MSI_FLAGS_ENABLE)
-		return 0;
-
-	if (!msi_lookup_vector(dev)) {
-		/* Lookup Sucess */
-		enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
-		return 0;
-	}
 	/* MSI Entry Initialization */
 	if (!(entry = alloc_msi_entry()))
 		return -ENOMEM;
@@ -493,11 +538,14 @@ static int msi_capability_init(struct pci_dev *dev)
 		kmem_cache_free(msi_cachep, entry);
 		return -EBUSY;
 	}
+	entry->link.head = vector;
+	entry->link.tail = vector;
 	entry->msi_attrib.type = PCI_CAP_ID_MSI;
+	entry->msi_attrib.state = 0;			/* Mark it not active */
 	entry->msi_attrib.entry_nr = 0;
 	entry->msi_attrib.maskbit = is_mask_bit_support(control);
-	entry->msi_attrib.default_vector = dev->irq;
-	dev->irq = vector;	/* save default pre-assigned ioapic vector */
+	entry->msi_attrib.default_vector = dev->irq;	/* Save IOAPIC IRQ */
+	dev->irq = vector;
 	entry->dev = dev;
 	if (is_mask_bit_support(control)) {
 		entry->mask_base = msi_mask_bits_reg(pos,
@@ -545,237 +593,219 @@ static int msi_capability_init(struct pci_dev *dev)
  * @dev: pointer to the pci_dev data structure of MSI-X device function
  *
  * Setup the MSI-X capability structure of device funtion with a
- * single MSI-X vector. A return of zero indicates the successful setup
- * of an entry zero with the new MSI-X vector or non-zero for otherwise.
- * To request for additional MSI-X vectors, the device drivers are
- * required to utilize the following supported APIs:
- * 1) msi_alloc_vectors(...) for requesting one or more MSI-X vectors
- * 2) msi_free_vectors(...) for releasing one or more MSI-X vectors
- *    back to PCI subsystem before calling free_irq(...)
+ * single MSI-X vector. A return of zero indicates the successful setup of
+ * requested MSI-X entries with allocated vectors or non-zero for otherwise.
  **/
-static int msix_capability_init(struct pci_dev	*dev)
+static int msix_capability_init(struct pci_dev *dev,
+				struct msix_entry *entries, int nvec)
 {
-	struct msi_desc *entry;
+	struct msi_desc *head = NULL, *tail = NULL, *entry = NULL;
 	struct msg_address address;
 	struct msg_data data;
-	int vector = 0, pos, dev_msi_cap, i;
+	int vector, pos, i, j, nr_entries, temp = 0;
 	u32 phys_addr, table_offset;
-	u16 control;
+ 	u16 control;
 	u8 bir;
 	void *base;
 
    	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
-	if (!pos)
-		return -EINVAL;
-
 	/* Request & Map MSI-X table region */
  	pci_read_config_word(dev, msi_control_reg(pos), &control);
-	if (control & PCI_MSIX_FLAGS_ENABLE)
-		return 0;
-
-	if (!msi_lookup_vector(dev)) {
-		/* Lookup Sucess */
-		enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
-		return 0;
-	}
-
-	dev_msi_cap = multi_msix_capable(control);
+	nr_entries = multi_msix_capable(control);
  	pci_read_config_dword(dev, msix_table_offset_reg(pos),
  		&table_offset);
 	bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
 	phys_addr = pci_resource_start (dev, bir);
 	phys_addr += (u32)(table_offset & ~PCI_MSIX_FLAGS_BIRMASK);
 	if (!request_mem_region(phys_addr,
-		dev_msi_cap * PCI_MSIX_ENTRY_SIZE,
-		"MSI-X iomap Failure"))
+		nr_entries * PCI_MSIX_ENTRY_SIZE,
+		"MSI-X vector table"))
 		return -ENOMEM;
-	base = ioremap_nocache(phys_addr, dev_msi_cap * PCI_MSIX_ENTRY_SIZE);
-	if (base == NULL)
-		goto free_region;
-	/* MSI Entry Initialization */
-	entry = alloc_msi_entry();
-	if (!entry)
-		goto free_iomap;
-	if ((vector = get_msi_vector(dev)) < 0)
-		goto free_entry;
+	base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
+	if (base == NULL) {
+		release_mem_region(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
+		return -ENOMEM;
+	}
+	/* MSI-X Table Initialization */
+	for (i = 0; i < nvec; i++) {
+		entry = alloc_msi_entry();
+		if (!entry)
+			break;
+		if ((vector = get_msi_vector(dev)) < 0)
+			break;
 
-	entry->msi_attrib.type = PCI_CAP_ID_MSIX;
-	entry->msi_attrib.entry_nr = 0;
-	entry->msi_attrib.maskbit = 1;
-	entry->msi_attrib.default_vector = dev->irq;
-	dev->irq = vector;	/* save default pre-assigned ioapic vector */
-	entry->dev = dev;
-	entry->mask_base = (unsigned long)base;
-	/* Replace with MSI handler */
-	irq_handler_init(PCI_CAP_ID_MSIX, vector, 1);
-	/* Configure MSI-X capability structure */
-	msi_address_init(&address);
-	msi_data_init(&data, vector);
-	entry->msi_attrib.current_cpu = ((address.lo_address.u.dest_id >>
-				MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK);
-	writel(address.lo_address.value, base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
-	writel(address.hi_address, base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
-	writel(*(u32*)&data, base + PCI_MSIX_ENTRY_DATA_OFFSET);
-	/* Initialize all entries from 1 up to 0 */
-	for (i = 1; i < dev_msi_cap; i++) {
-		writel(0, base + i * PCI_MSIX_ENTRY_SIZE +
+ 		j = entries[i].entry;
+ 		entries[i].vector = vector;
+		entry->msi_attrib.type = PCI_CAP_ID_MSIX;
+ 		entry->msi_attrib.state = 0;		/* Mark it not active */
+		entry->msi_attrib.entry_nr = j;
+		entry->msi_attrib.maskbit = 1;
+		entry->msi_attrib.default_vector = dev->irq;
+		entry->dev = dev;
+		entry->mask_base = (unsigned long)base;
+		if (!head) {
+			entry->link.head = vector;
+			entry->link.tail = vector;
+			head = entry;
+		} else {
+			entry->link.head = temp;
+			entry->link.tail = tail->link.tail;
+			tail->link.tail = vector;
+			head->link.head = vector;
+		}
+		temp = vector;
+		tail = entry;
+		/* Replace with MSI-X handler */
+		irq_handler_init(PCI_CAP_ID_MSIX, vector, 1);
+		/* Configure MSI-X capability structure */
+		msi_address_init(&address);
+		msi_data_init(&data, vector);
+		entry->msi_attrib.current_cpu =
+			((address.lo_address.u.dest_id >>
+			MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK);
+		writel(address.lo_address.value,
+			base + j * PCI_MSIX_ENTRY_SIZE +
 			PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
-		writel(0, base + i * PCI_MSIX_ENTRY_SIZE +
+		writel(address.hi_address,
+			base + j * PCI_MSIX_ENTRY_SIZE +
 			PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
-		writel(0, base + i * PCI_MSIX_ENTRY_SIZE +
+		writel(*(u32*)&data,
+			base + j * PCI_MSIX_ENTRY_SIZE +
 			PCI_MSIX_ENTRY_DATA_OFFSET);
+		attach_msi_entry(entry, vector);
 	}
-	attach_msi_entry(entry, vector);
-	/* Set MSI enabled bits	 */
+	if (i != nvec) {
+		i--;
+		for (; i >= 0; i--) {
+			vector = (entries + i)->vector;
+			msi_free_vector(dev, vector, 0);
+			(entries + i)->vector = 0;
+		}
+		return -EBUSY;
+	}
+	/* Set MSI-X enabled bits */
 	enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
 
 	return 0;
-
-free_entry:
-	kmem_cache_free(msi_cachep, entry);
-free_iomap:
-	iounmap(base);
-free_region:
-	release_mem_region(phys_addr, dev_msi_cap * PCI_MSIX_ENTRY_SIZE);
-
-	return ((vector < 0) ? -EBUSY : -ENOMEM);
 }
 
 /**
- * pci_enable_msi - configure device's MSI(X) capability structure
- * @dev: pointer to the pci_dev data structure of MSI(X) device function
+ * pci_enable_msi - configure device's MSI capability structure
+ * @dev: pointer to the pci_dev data structure of MSI device function
  *
- * Setup the MSI/MSI-X capability structure of device function with
- * a single MSI(X) vector upon its software driver call to request for
- * MSI(X) mode enabled on its hardware device function. A return of zero
- * indicates the successful setup of an entry zero with the new MSI(X)
+ * Setup the MSI capability structure of device function with
+ * a single MSI vector upon its software driver call to request for
+ * MSI mode enabled on its hardware device function. A return of zero
+ * indicates the successful setup of an entry zero with the new MSI
  * vector or non-zero for otherwise.
  **/
 int pci_enable_msi(struct pci_dev* dev)
 {
-	int status = -EINVAL;
+	int pos, temp = dev->irq, status = -EINVAL;
+	u16 control;
 
 	if (!pci_msi_enable || !dev)
  		return status;
 
-	if (msi_init() < 0)
-		return -ENOMEM;
+	if ((status = msi_init()) < 0)
+		return status;
 
-	if ((status = msix_capability_init(dev)) == -EINVAL)
-		status = msi_capability_init(dev);
-	if (!status)
-		nr_reserved_vectors--;
+   	if (!(pos = pci_find_capability(dev, PCI_CAP_ID_MSI)))
+		return -EINVAL;
+
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	if (control & PCI_MSI_FLAGS_ENABLE)
+		return 0;			/* Already in MSI mode */
+
+	if (!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
+		/* Lookup Sucess */
+		unsigned long flags;
+
+		spin_lock_irqsave(&msi_lock, flags);
+		if (!vector_irq[dev->irq]) {
+			msi_desc[dev->irq]->msi_attrib.state = 0;
+			vector_irq[dev->irq] = -1;
+			nr_released_vectors--;
+			spin_unlock_irqrestore(&msi_lock, flags);
+			enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
+			return 0;
+		}
+		spin_unlock_irqrestore(&msi_lock, flags);
+		dev->irq = temp;
+	}
+	/* Check whether driver already requested for MSI-X vectors */
+   	if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)) > 0 &&
+		!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
+			printk(KERN_INFO "Can't enable MSI. Device already had MSI-X vectors assigned\n");
+			dev->irq = temp;
+			return -EINVAL;
+	}
+	status = msi_capability_init(dev);
+	if (!status) {
+   		if (!pos)
+			nr_reserved_vectors--;	/* Only MSI capable */
+		else if (nr_msix_devices > 0)
+			nr_msix_devices--;	/* Both MSI and MSI-X capable,
+						   but choose enabling MSI */
+	}
 
 	return status;
 }
 
-static int msi_free_vector(struct pci_dev* dev, int vector);
-static void pci_disable_msi(unsigned int vector)
+void pci_disable_msi(struct pci_dev* dev)
 {
-	int head, tail, type, default_vector;
 	struct msi_desc *entry;
-	struct pci_dev *dev;
+	int pos, default_vector;
+	u16 control;
 	unsigned long flags;
 
+   	if (!dev || !(pos = pci_find_capability(dev, PCI_CAP_ID_MSI)))
+		return;
+
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	if (!(control & PCI_MSI_FLAGS_ENABLE))
+		return;
+
 	spin_lock_irqsave(&msi_lock, flags);
-	entry = msi_desc[vector];
-	if (!entry || !entry->dev) {
+	entry = msi_desc[dev->irq];
+	if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) {
 		spin_unlock_irqrestore(&msi_lock, flags);
 		return;
 	}
-	dev = entry->dev;
-	type = entry->msi_attrib.type;
-	head = entry->link.head;
-	tail = entry->link.tail;
-	default_vector = entry->msi_attrib.default_vector;
-	spin_unlock_irqrestore(&msi_lock, flags);
-
-	disable_msi_mode(dev, pci_find_capability(dev, type), type);
-	/* Restore dev->irq to its default pin-assertion vector */
-	dev->irq = default_vector;
-	if (type == PCI_CAP_ID_MSIX && head != tail) {
-		/* Bad driver, which do not call msi_free_vectors before exit.
-		   We must do a cleanup here */
-		while (1) {
-			spin_lock_irqsave(&msi_lock, flags);
-			entry = msi_desc[vector];
-			head = entry->link.head;
-			tail = entry->link.tail;
-			spin_unlock_irqrestore(&msi_lock, flags);
-			if (tail == head)
-				break;
-			if (msi_free_vector(dev, entry->link.tail))
-				break;
-		}
+	if (entry->msi_attrib.state) {
+		spin_unlock_irqrestore(&msi_lock, flags);
+		printk(KERN_DEBUG "Driver[%d:%d:%d] unloaded wo doing free_irq on vector->%d\n",
+		dev->bus->number, PCI_SLOT(dev->devfn),	PCI_FUNC(dev->devfn),
+		dev->irq);
+		BUG_ON(entry->msi_attrib.state > 0);
+	} else {
+		vector_irq[dev->irq] = 0; /* free it */
+		nr_released_vectors++;
+		default_vector = entry->msi_attrib.default_vector;
+		spin_unlock_irqrestore(&msi_lock, flags);
+		/* Restore dev->irq to its default pin-assertion vector */
+		dev->irq = default_vector;
+		disable_msi_mode(dev, pci_find_capability(dev, PCI_CAP_ID_MSI),
+					PCI_CAP_ID_MSI);
 	}
 }
 
-static int msi_alloc_vector(struct pci_dev* dev, int head)
+static void release_msi(unsigned int vector)
 {
 	struct msi_desc *entry;
-	struct msg_address address;
-	struct msg_data data;
-	int i, offset, pos, dev_msi_cap, vector;
-	u32 low_address;
-	u16 control;
-	unsigned long base = 0L;
 	unsigned long flags;
 
 	spin_lock_irqsave(&msi_lock, flags);
-	entry = msi_desc[dev->irq];
-	if (!entry) {
-		spin_unlock_irqrestore(&msi_lock, flags);
-		return -EINVAL;
-	}
-	base = entry->mask_base;
+	entry = msi_desc[vector];
+	if (entry && entry->dev)
+		entry->msi_attrib.state = 0;	/* Mark it not active */
 	spin_unlock_irqrestore(&msi_lock, flags);
-
-   	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
- 	pci_read_config_word(dev, msi_control_reg(pos), &control);
-	dev_msi_cap = multi_msix_capable(control);
-	for (i = 1; i < dev_msi_cap; i++) {
-		if (!(low_address = readl(base + i * PCI_MSIX_ENTRY_SIZE)))
-			 break;
-	}
-	if (i >= dev_msi_cap)
-		return -EINVAL;
-
-	/* MSI Entry Initialization */
-	if (!(entry = alloc_msi_entry()))
-		return -ENOMEM;
-
-	if ((vector = get_new_vector()) < 0) {
-		kmem_cache_free(msi_cachep, entry);
-		return vector;
-	}
-	entry->msi_attrib.type = PCI_CAP_ID_MSIX;
-	entry->msi_attrib.entry_nr = i;
-	entry->msi_attrib.maskbit = 1;
-	entry->dev = dev;
-	entry->link.head = head;
-	entry->mask_base = base;
-	irq_handler_init(PCI_CAP_ID_MSIX, vector, 1);
-	/* Configure MSI-X capability structure */
-	msi_address_init(&address);
-	msi_data_init(&data, vector);
-	entry->msi_attrib.current_cpu = ((address.lo_address.u.dest_id >>
-				MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK);
-	offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
-	writel(address.lo_address.value, base + offset +
-		PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
-	writel(address.hi_address, base + offset +
-		PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
-	writel(*(u32*)&data, base + offset + PCI_MSIX_ENTRY_DATA_OFFSET);
-	writel(1, base + offset + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
-	attach_msi_entry(entry, vector);
-
-	return vector;
 }
 
-static int msi_free_vector(struct pci_dev* dev, int vector)
+static int msi_free_vector(struct pci_dev* dev, int vector, int reassign)
 {
 	struct msi_desc *entry;
-	int entry_nr, type;
+	int head, entry_nr, type;
 	unsigned long base = 0L;
 	unsigned long flags;
 
@@ -787,66 +817,177 @@ static int msi_free_vector(struct pci_dev* dev, int vector)
 	}
 	type = entry->msi_attrib.type;
 	entry_nr = entry->msi_attrib.entry_nr;
+	head = entry->link.head;
 	base = entry->mask_base;
-	if (entry->link.tail != entry->link.head) {
-		msi_desc[entry->link.head]->link.tail = entry->link.tail;
-		if (entry->link.tail)
-			msi_desc[entry->link.tail]->link.head = entry->link.head;
-	}
+	msi_desc[entry->link.head]->link.tail = entry->link.tail;
+	msi_desc[entry->link.tail]->link.head = entry->link.head;
 	entry->dev = NULL;
-	vector_irq[vector] = 0;
-	nr_released_vectors++;
+	if (!reassign) {
+		vector_irq[vector] = 0;
+		nr_released_vectors++;
+	}
 	msi_desc[vector] = NULL;
 	spin_unlock_irqrestore(&msi_lock, flags);
 
 	kmem_cache_free(msi_cachep, entry);
+
 	if (type == PCI_CAP_ID_MSIX) {
-		int offset;
+		if (!reassign)
+			writel(1, base +
+				entry_nr * PCI_MSIX_ENTRY_SIZE +
+				PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
+
+		if (head == vector) {
+			/*
+			 * Detect last MSI-X vector to be released.
+			 * Release the MSI-X memory-mapped table.
+			 */
+			int pos, nr_entries;
+			u32 phys_addr, table_offset;
+			u16 control;
+			u8 bir;
+
+   			pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
+			pci_read_config_word(dev, msi_control_reg(pos),
+				&control);
+			nr_entries = multi_msix_capable(control);
+			pci_read_config_dword(dev, msix_table_offset_reg(pos),
+				&table_offset);
+			bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
+			phys_addr = pci_resource_start (dev, bir);
+			phys_addr += (u32)(table_offset &
+				~PCI_MSIX_FLAGS_BIRMASK);
+			iounmap((void*)base);
+			release_mem_region(phys_addr,
+				nr_entries * PCI_MSIX_ENTRY_SIZE);
+		}
+	}
 
-		offset = entry_nr * PCI_MSIX_ENTRY_SIZE;
-		writel(1, base + offset + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
-		writel(0, base + offset + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
+	return 0;
+}
+
+static int reroute_msix_table(int head, struct msix_entry *entries, int *nvec)
+{
+	int vector = head, tail = 0;
+	int i = 0, j = 0, nr_entries = 0;
+	unsigned long base = 0L;
+	unsigned long flags;
+
+	spin_lock_irqsave(&msi_lock, flags);
+	while (head != tail) {
+		nr_entries++;
+		tail = msi_desc[vector]->link.tail;
+		if (entries[0].entry == msi_desc[vector]->msi_attrib.entry_nr)
+			j = vector;
+		vector = tail;
 	}
+	if (*nvec > nr_entries) {
+		spin_unlock_irqrestore(&msi_lock, flags);
+		*nvec = nr_entries;
+		return -EINVAL;
+	}
+	vector = ((j > 0) ? j : head);
+	for (i = 0; i < *nvec; i++) {
+		j = msi_desc[vector]->msi_attrib.entry_nr;
+		msi_desc[vector]->msi_attrib.state = 0;	/* Mark it not active */
+		vector_irq[vector] = -1;		/* Mark it busy */
+		nr_released_vectors--;
+		entries[i].vector = vector;
+		if (j != (entries + i)->entry) {
+			base = msi_desc[vector]->mask_base;
+			msi_desc[vector]->msi_attrib.entry_nr =
+				(entries + i)->entry;
+			writel( readl(base + j * PCI_MSIX_ENTRY_SIZE +
+				PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET), base +
+				(entries + i)->entry * PCI_MSIX_ENTRY_SIZE +
+				PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
+			writel(	readl(base + j * PCI_MSIX_ENTRY_SIZE +
+				PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET), base +
+				(entries + i)->entry * PCI_MSIX_ENTRY_SIZE +
+				PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
+			writel( (readl(base + j * PCI_MSIX_ENTRY_SIZE +
+				PCI_MSIX_ENTRY_DATA_OFFSET) & 0xff00) | vector,
+				base + (entries+i)->entry*PCI_MSIX_ENTRY_SIZE +
+				PCI_MSIX_ENTRY_DATA_OFFSET);
+		}
+		vector = msi_desc[vector]->link.tail;
+	}
+	spin_unlock_irqrestore(&msi_lock, flags);
 
 	return 0;
 }
 
 /**
- * msi_alloc_vectors - allocate additional MSI-X vectors
+ * pci_enable_msix - configure device's MSI-X capability structure
  * @dev: pointer to the pci_dev data structure of MSI-X device function
- * @vector: pointer to an array of new allocated MSI-X vectors
+ * @data: pointer to an array of MSI-X entries
  * @nvec: number of MSI-X vectors requested for allocation by device driver
  *
- * Allocate additional MSI-X vectors requested by device driver. A
- * return of zero indicates the successful setup of MSI-X capability
- * structure with new allocated MSI-X vectors or non-zero for otherwise.
+ * Setup the MSI-X capability structure of device function with the number
+ * of requested vectors upon its software driver call to request for
+ * MSI-X mode enabled on its hardware device function. A return of zero
+ * indicates the successful configuration of MSI-X capability structure
+ * with new allocated MSI-X vectors. A return of < 0 indicates a failure.
+ * Or a return of > 0 indicates that driver request is exceeding the number
+ * of vectors available. Driver should use the returned value to re-send
+ * its request.
  **/
-int msi_alloc_vectors(struct pci_dev* dev, int *vector, int nvec)
+int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec)
 {
-	struct msi_desc *entry;
-	int i, head, pos, vec, free_vectors, alloc_vectors;
-	int *vectors = (int *)vector;
+	int status, pos, nr_entries, free_vectors;
+	int i, j, temp;
 	u16 control;
 	unsigned long flags;
 
-	if (!pci_msi_enable || !dev)
+	if (!pci_msi_enable || !dev || !entries)
  		return -EINVAL;
 
+	if ((status = msi_init()) < 0)
+		return status;
+
    	if (!(pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)))
  		return -EINVAL;
 
- 	pci_read_config_word(dev, msi_control_reg(pos), &control);
-	if (nvec > multi_msix_capable(control))
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	if (control & PCI_MSIX_FLAGS_ENABLE)
+		return -EINVAL;			/* Already in MSI-X mode */
+
+	nr_entries = multi_msix_capable(control);
+	if (nvec > nr_entries)
 		return -EINVAL;
 
-	spin_lock_irqsave(&msi_lock, flags);
-	entry = msi_desc[dev->irq];
-	if (!entry || entry->dev != dev ||		/* legal call */
-	   entry->msi_attrib.type != PCI_CAP_ID_MSIX || /* must be MSI-X */
-	   entry->link.head != entry->link.tail) {	/* already multi */
-		spin_unlock_irqrestore(&msi_lock, flags);
+	/* Check for any invalid entries */
+	for (i = 0; i < nvec; i++) {
+		if (entries[i].entry >= nr_entries)
+			return -EINVAL;		/* invalid entry */
+		for (j = i + 1; j < nvec; j++) {
+			if (entries[i].entry == entries[j].entry)
+				return -EINVAL;	/* duplicate entry */
+		}
+	}
+	temp = dev->irq;
+	if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
+		/* Lookup Sucess */
+		nr_entries = nvec;
+		/* Reroute MSI-X table */
+		if (reroute_msix_table(dev->irq, entries, &nr_entries)) {
+			/* #requested > #previous-assigned */
+			dev->irq = temp;
+			return nr_entries;
+		}
+		dev->irq = temp;
+		enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
+		return 0;
+	}
+	/* Check whether driver already requested for MSI vector */
+   	if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 &&
+		!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
+		printk(KERN_INFO "Can't enable MSI-X. Device already had MSI vector assigned\n");
+		dev->irq = temp;
 		return -EINVAL;
 	}
+
+	spin_lock_irqsave(&msi_lock, flags);
 	/*
 	 * msi_lock is provided to ensure that enough vectors resources are
 	 * available before granting.
@@ -862,71 +1003,65 @@ int msi_alloc_vectors(struct pci_dev* dev, int *vector, int nvec)
 		free_vectors /= nr_msix_devices;
 	spin_unlock_irqrestore(&msi_lock, flags);
 
-	if (nvec > free_vectors)
-		return -EBUSY;
-
-	alloc_vectors = 0;
-	head = dev->irq;
-	for (i = 0; i < nvec; i++) {
-		if ((vec = msi_alloc_vector(dev, head)) < 0)
-			break;
-		*(vectors + i) = vec;
-		head = vec;
-		alloc_vectors++;
-	}
-	if (alloc_vectors != nvec) {
-		for (i = 0; i < alloc_vectors; i++) {
-			vec = *(vectors + i);
-			msi_free_vector(dev, vec);
-		}
-		spin_lock_irqsave(&msi_lock, flags);
-		msi_desc[dev->irq]->link.tail = msi_desc[dev->irq]->link.head;
-		spin_unlock_irqrestore(&msi_lock, flags);
-		return -EBUSY;
+	if (nvec > free_vectors) {
+		if (free_vectors > 0)
+			return free_vectors;
+		else
+			return -EBUSY;
 	}
-	if (nr_msix_devices > 0)
+
+	status = msix_capability_init(dev, entries, nvec);
+	if (!status && nr_msix_devices > 0)
 		nr_msix_devices--;
 
-	return 0;
+	return status;
 }
 
-/**
- * msi_free_vectors - reclaim MSI-X vectors to unused state
- * @dev: pointer to the pci_dev data structure of MSI-X device function
- * @vector: pointer to an array of released MSI-X vectors
- * @nvec: number of MSI-X vectors requested for release by device driver
- *
- * Reclaim MSI-X vectors released by device driver to unused state,
- * which may be used later on. A return of zero indicates the
- * success or non-zero for otherwise. Device driver should call this
- * before calling function free_irq.
- **/
-int msi_free_vectors(struct pci_dev* dev, int *vector, int nvec)
+void pci_disable_msix(struct pci_dev* dev)
 {
-	struct msi_desc *entry;
-	int i;
-	unsigned long flags;
+	int pos, temp;
+	u16 control;
 
-	if (!pci_msi_enable)
- 		return -EINVAL;
+   	if (!dev || !(pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)))
+		return;
 
-	spin_lock_irqsave(&msi_lock, flags);
-	entry = msi_desc[dev->irq];
-	if (!entry || entry->dev != dev ||
-	   	entry->msi_attrib.type != PCI_CAP_ID_MSIX ||
-		entry->link.head == entry->link.tail) {	/* Nothing to free */
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	if (!(control & PCI_MSIX_FLAGS_ENABLE))
+		return;
+
+	temp = dev->irq;
+	if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
+		int state, vector, head, tail = 0, warning = 0;
+		unsigned long flags;
+
+		vector = head = dev->irq;
+		spin_lock_irqsave(&msi_lock, flags);
+		while (head != tail) {
+			state = msi_desc[vector]->msi_attrib.state;
+			if (state)
+				warning = 1;
+			else {
+				vector_irq[vector] = 0; /* free it */
+				nr_released_vectors++;
+			}
+			tail = msi_desc[vector]->link.tail;
+			vector = tail;
+		}
 		spin_unlock_irqrestore(&msi_lock, flags);
-		return -EINVAL;
-	}
-	spin_unlock_irqrestore(&msi_lock, flags);
+		if (warning) {
+			dev->irq = temp;
+			printk(KERN_DEBUG "Driver[%d:%d:%d] unloaded wo doing free_irq on all vectors\n",
+			dev->bus->number, PCI_SLOT(dev->devfn),
+			PCI_FUNC(dev->devfn));
+			BUG_ON(warning > 0);
+		} else {
+			dev->irq = temp;
+			disable_msi_mode(dev,
+				pci_find_capability(dev, PCI_CAP_ID_MSIX),
+				PCI_CAP_ID_MSIX);
 
-	for (i = 0; i < nvec; i++) {
-		if (*(vector + i) == dev->irq)
-			continue;/* Don't free entry 0 if mistaken by driver */
-		msi_free_vector(dev, *(vector + i));
+		}
 	}
-
-	return 0;
 }
 
 /**
@@ -940,61 +1075,73 @@ int msi_free_vectors(struct pci_dev* dev, int *vector, int nvec)
  **/
 void msi_remove_pci_irq_vectors(struct pci_dev* dev)
 {
-	struct msi_desc *entry;
-	int type, temp;
+	int state, pos, temp;
 	unsigned long flags;
 
 	if (!pci_msi_enable || !dev)
  		return;
 
-   	if (!pci_find_capability(dev, PCI_CAP_ID_MSI)) {
-   		if (!pci_find_capability(dev, PCI_CAP_ID_MSIX))
-			return;
-	}
-	temp = dev->irq;
-	if (msi_lookup_vector(dev))
-		return;
-
-	spin_lock_irqsave(&msi_lock, flags);
-	entry = msi_desc[dev->irq];
-	if (!entry || entry->dev != dev) {
+	temp = dev->irq;		/* Save IOAPIC IRQ */
+   	if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSI)) > 0 &&
+		!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
+		spin_lock_irqsave(&msi_lock, flags);
+		state = msi_desc[dev->irq]->msi_attrib.state;
 		spin_unlock_irqrestore(&msi_lock, flags);
-		return;
+		if (state) {
+			printk(KERN_DEBUG "Driver[%d:%d:%d] unloaded wo doing free_irq on vector->%d\n",
+			dev->bus->number, PCI_SLOT(dev->devfn),
+			PCI_FUNC(dev->devfn), dev->irq);
+			BUG_ON(state > 0);
+		} else /* Release MSI vector assigned to this device */
+			msi_free_vector(dev, dev->irq, 0);
+		dev->irq = temp;		/* Restore IOAPIC IRQ */
 	}
-	type = entry->msi_attrib.type;
-	spin_unlock_irqrestore(&msi_lock, flags);
+   	if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)) > 0 &&
+		!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
+		int vector, head, tail = 0, warning = 0;
+		unsigned long base = 0L;
 
-	msi_free_vector(dev, dev->irq);
-	if (type == PCI_CAP_ID_MSIX) {
-		int i, pos, dev_msi_cap;
-		u32 phys_addr, table_offset;
-		u16 control;
-		u8 bir;
-
-   		pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
-		pci_read_config_word(dev, msi_control_reg(pos), &control);
-		dev_msi_cap = multi_msix_capable(control);
-		pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset);
-		bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
-		phys_addr = pci_resource_start (dev, bir);
-		phys_addr += (u32)(table_offset & ~PCI_MSIX_FLAGS_BIRMASK);
-		for (i = FIRST_DEVICE_VECTOR; i < NR_IRQS; i++) {
+		vector = head = dev->irq;
+		while (head != tail) {
 			spin_lock_irqsave(&msi_lock, flags);
-			if (!msi_desc[i] || msi_desc[i]->dev != dev) {
-				spin_unlock_irqrestore(&msi_lock, flags);
-				continue;
-			}
+			state = msi_desc[vector]->msi_attrib.state;
+			tail = msi_desc[vector]->link.tail;
+			base = msi_desc[vector]->mask_base;
 			spin_unlock_irqrestore(&msi_lock, flags);
-			msi_free_vector(dev, i);
+			if (state)
+				warning = 1;
+			else if (vector != head) /* Release MSI-X vector */
+				msi_free_vector(dev, vector, 0);
+			vector = tail;
+		}
+		msi_free_vector(dev, vector, 0);
+		if (warning) {
+			/* Force to release the MSI-X memory-mapped table */
+			u32 phys_addr, table_offset;
+			u16 control;
+			u8 bir;
+
+			pci_read_config_word(dev, msi_control_reg(pos),
+				&control);
+			pci_read_config_dword(dev, msix_table_offset_reg(pos),
+				&table_offset);
+			bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
+			phys_addr = pci_resource_start (dev, bir);
+			phys_addr += (u32)(table_offset &
+				~PCI_MSIX_FLAGS_BIRMASK);
+			iounmap((void*)base);
+			release_mem_region(phys_addr, PCI_MSIX_ENTRY_SIZE *
+				multi_msix_capable(control));
+			printk(KERN_DEBUG "Driver[%d:%d:%d] unloaded wo doing free_irq on all vectors\n",
+				dev->bus->number, PCI_SLOT(dev->devfn),
+				PCI_FUNC(dev->devfn));
+			BUG_ON(warning > 0);
 		}
-		writel(1, entry->mask_base + PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
-		iounmap((void*)entry->mask_base);
-		release_mem_region(phys_addr, dev_msi_cap * PCI_MSIX_ENTRY_SIZE);
+		dev->irq = temp;		/* Restore IOAPIC IRQ */
 	}
-	dev->irq = temp;
-	nr_reserved_vectors++;
 }
 
 EXPORT_SYMBOL(pci_enable_msi);
-EXPORT_SYMBOL(msi_alloc_vectors);
-EXPORT_SYMBOL(msi_free_vectors);
+EXPORT_SYMBOL(pci_disable_msi);
+EXPORT_SYMBOL(pci_enable_msix);
+EXPORT_SYMBOL(pci_disable_msix);
diff --git a/drivers/pci/msi.h b/drivers/pci/msi.h
index 524504fdd4f015..3a7a5285439306 100644
--- a/drivers/pci/msi.h
+++ b/drivers/pci/msi.h
@@ -140,7 +140,8 @@ struct msi_desc {
 	struct {
 		__u8	type	: 5; 	/* {0: unused, 5h:MSI, 11h:MSI-X} */
 		__u8	maskbit	: 1; 	/* mask-pending bit supported ?   */
-		__u8	reserved: 2; 	/* reserved			  */
+		__u8	state	: 1; 	/* {0: free, 1: busy}		  */
+		__u8	reserved: 1; 	/* reserved			  */
 		__u8	entry_nr;    	/* specific enabled entry 	  */
 		__u8	default_vector; /* default pre-assigned vector    */
 		__u8	current_cpu; 	/* current destination cpu	  */