pci.c 146 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9
/*
 *	PCI Bus Services, see include/linux/pci.h for further explanation.
 *
 *	Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter,
 *	David Mosberger-Tang
 *
 *	Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz>
 */

10
#include <linux/acpi.h>
Linus Torvalds's avatar
Linus Torvalds committed
11 12
#include <linux/kernel.h>
#include <linux/delay.h>
13
#include <linux/dmi.h>
Linus Torvalds's avatar
Linus Torvalds committed
14
#include <linux/init.h>
15 16
#include <linux/of.h>
#include <linux/of_pci.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
#include <linux/pci.h>
18
#include <linux/pm.h>
19
#include <linux/slab.h>
Linus Torvalds's avatar
Linus Torvalds committed
20 21
#include <linux/module.h>
#include <linux/spinlock.h>
Tim Schmielau's avatar
Tim Schmielau committed
22
#include <linux/string.h>
23
#include <linux/log2.h>
24
#include <linux/pci-aspm.h>
25
#include <linux/pm_wakeup.h>
26
#include <linux/interrupt.h>
27
#include <linux/device.h>
28
#include <linux/pm_runtime.h>
29
#include <linux/pci_hotplug.h>
30
#include <linux/vmalloc.h>
31
#include <linux/pci-ats.h>
32
#include <asm/setup.h>
33
#include <asm/dma.h>
34
#include <linux/aer.h>
35
#include "pci.h"
Linus Torvalds's avatar
Linus Torvalds committed
36

37 38 39 40 41
const char *pci_power_names[] = {
	"error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown",
};
EXPORT_SYMBOL_GPL(pci_power_names);

42 43 44 45 46 47
int isa_dma_bridge_buggy;
EXPORT_SYMBOL(isa_dma_bridge_buggy);

int pci_pci_problems;
EXPORT_SYMBOL(pci_pci_problems);

48 49
unsigned int pci_pm_d3_delay;

50 51 52 53 54 55 56 57 58 59 60 61 62
static void pci_pme_list_scan(struct work_struct *work);

static LIST_HEAD(pci_pme_list);
static DEFINE_MUTEX(pci_pme_list_mutex);
static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan);

struct pci_pme_device {
	struct list_head list;
	struct pci_dev *dev;
};

#define PME_TIMEOUT 1000 /* How long between PME checks */

63 64 65 66 67 68 69
static void pci_dev_d3_sleep(struct pci_dev *dev)
{
	unsigned int delay = dev->d3_delay;

	if (delay < pci_pm_d3_delay)
		delay = pci_pm_d3_delay;

70 71
	if (delay)
		msleep(delay);
72
}
Linus Torvalds's avatar
Linus Torvalds committed
73

74 75 76 77
#ifdef CONFIG_PCI_DOMAINS
int pci_domains_supported = 1;
#endif

78 79 80 81 82 83
#define DEFAULT_CARDBUS_IO_SIZE		(256)
#define DEFAULT_CARDBUS_MEM_SIZE	(64*1024*1024)
/* pci=cbmemsize=nnM,cbiosize=nn can override this */
unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;

84 85 86 87 88 89
#define DEFAULT_HOTPLUG_IO_SIZE		(256)
#define DEFAULT_HOTPLUG_MEM_SIZE	(2*1024*1024)
/* pci=hpmemsize=nnM,hpiosize=nn can override this */
unsigned long pci_hotplug_io_size  = DEFAULT_HOTPLUG_IO_SIZE;
unsigned long pci_hotplug_mem_size = DEFAULT_HOTPLUG_MEM_SIZE;

90 91 92
#define DEFAULT_HOTPLUG_BUS_SIZE	1
unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;

93
enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT;
94

95 96 97 98 99 100
/*
 * The default CLS is used if arch didn't set CLS explicitly and not
 * all pci devices agree on the same value.  Arch can override either
 * the dfl or actual value as it sees fit.  Don't forget this is
 * measured in 32-bit words, not bytes.
 */
Bill Pemberton's avatar
Bill Pemberton committed
101
u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2;
102 103
u8 pci_cache_line_size;

104 105 106 107 108 109
/*
 * If we set up a device for bus mastering, we need to check the latency
 * timer as certain BIOSes forget to set it properly.
 */
unsigned int pcibios_max_latency = 255;

110 111 112
/* If set, the PCIe ARI capability will not be used. */
static bool pcie_ari_disabled;

113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
/* Disable bridge_d3 for all PCIe ports */
static bool pci_bridge_d3_disable;
/* Force bridge_d3 for all PCIe ports */
static bool pci_bridge_d3_force;

static int __init pcie_port_pm_setup(char *str)
{
	if (!strcmp(str, "off"))
		pci_bridge_d3_disable = true;
	else if (!strcmp(str, "force"))
		pci_bridge_d3_force = true;
	return 1;
}
__setup("pcie_port_pm=", pcie_port_pm_setup);

Linus Torvalds's avatar
Linus Torvalds committed
128 129 130 131 132 133 134
/**
 * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children
 * @bus: pointer to PCI bus structure to search
 *
 * Given a PCI bus, returns the highest PCI bus number present in the set
 * including the given PCI bus and its list of child PCI buses.
 */
135
unsigned char pci_bus_max_busnr(struct pci_bus *bus)
Linus Torvalds's avatar
Linus Torvalds committed
136
{
137
	struct pci_bus *tmp;
Linus Torvalds's avatar
Linus Torvalds committed
138 139
	unsigned char max, n;

140
	max = bus->busn_res.end;
141 142
	list_for_each_entry(tmp, &bus->children, node) {
		n = pci_bus_max_busnr(tmp);
Ryan Desfosses's avatar
Ryan Desfosses committed
143
		if (n > max)
Linus Torvalds's avatar
Linus Torvalds committed
144 145 146 147
			max = n;
	}
	return max;
}
148
EXPORT_SYMBOL_GPL(pci_bus_max_busnr);
Linus Torvalds's avatar
Linus Torvalds committed
149

150 151 152
#ifdef CONFIG_HAS_IOMEM
void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar)
{
153 154
	struct resource *res = &pdev->resource[bar];

155 156 157
	/*
	 * Make sure the BAR is actually a memory resource, not an IO resource
	 */
158
	if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) {
159
		dev_warn(&pdev->dev, "can't ioremap BAR %d: %pR\n", bar, res);
160 161
		return NULL;
	}
162
	return ioremap_nocache(res->start, resource_size(res));
163 164
}
EXPORT_SYMBOL_GPL(pci_ioremap_bar);
165 166 167 168 169 170 171 172 173 174 175 176 177 178

void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar)
{
	/*
	 * Make sure the BAR is actually a memory resource, not an IO resource
	 */
	if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) {
		WARN_ON(1);
		return NULL;
	}
	return ioremap_wc(pci_resource_start(pdev, bar),
			  pci_resource_len(pdev, bar));
}
EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar);
179 180
#endif

181 182 183

static int __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn,
				   u8 pos, int cap, int *ttl)
184 185
{
	u8 id;
186 187 188
	u16 ent;

	pci_bus_read_config_byte(bus, devfn, pos, &pos);
189

190
	while ((*ttl)--) {
191 192 193
		if (pos < 0x40)
			break;
		pos &= ~3;
194 195 196
		pci_bus_read_config_word(bus, devfn, pos, &ent);

		id = ent & 0xff;
197 198 199 200
		if (id == 0xff)
			break;
		if (id == cap)
			return pos;
201
		pos = (ent >> 8);
202 203 204 205
	}
	return 0;
}

206 207 208 209 210 211 212 213
static int __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn,
			       u8 pos, int cap)
{
	int ttl = PCI_FIND_CAP_TTL;

	return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl);
}

214 215 216 217 218 219 220
int pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap)
{
	return __pci_find_next_cap(dev->bus, dev->devfn,
				   pos + PCI_CAP_LIST_NEXT, cap);
}
EXPORT_SYMBOL_GPL(pci_find_next_capability);

221 222
static int __pci_bus_find_cap_start(struct pci_bus *bus,
				    unsigned int devfn, u8 hdr_type)
Linus Torvalds's avatar
Linus Torvalds committed
223 224 225 226 227 228 229 230 231 232
{
	u16 status;

	pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status);
	if (!(status & PCI_STATUS_CAP_LIST))
		return 0;

	switch (hdr_type) {
	case PCI_HEADER_TYPE_NORMAL:
	case PCI_HEADER_TYPE_BRIDGE:
233
		return PCI_CAPABILITY_LIST;
Linus Torvalds's avatar
Linus Torvalds committed
234
	case PCI_HEADER_TYPE_CARDBUS:
235
		return PCI_CB_CAPABILITY_LIST;
Linus Torvalds's avatar
Linus Torvalds committed
236
	}
237 238

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
239 240 241
}

/**
242
 * pci_find_capability - query for devices' capabilities
Linus Torvalds's avatar
Linus Torvalds committed
243 244 245 246 247 248 249 250
 * @dev: PCI device to query
 * @cap: capability code
 *
 * Tell if a device supports a given PCI capability.
 * Returns the address of the requested capability structure within the
 * device's PCI configuration space or 0 in case the device does not
 * support it.  Possible values for @cap:
 *
251 252 253 254
 *  %PCI_CAP_ID_PM           Power Management
 *  %PCI_CAP_ID_AGP          Accelerated Graphics Port
 *  %PCI_CAP_ID_VPD          Vital Product Data
 *  %PCI_CAP_ID_SLOTID       Slot Identification
Linus Torvalds's avatar
Linus Torvalds committed
255
 *  %PCI_CAP_ID_MSI          Message Signalled Interrupts
256
 *  %PCI_CAP_ID_CHSWP        CompactPCI HotSwap
Linus Torvalds's avatar
Linus Torvalds committed
257 258 259 260 261
 *  %PCI_CAP_ID_PCIX         PCI-X
 *  %PCI_CAP_ID_EXP          PCI Express
 */
int pci_find_capability(struct pci_dev *dev, int cap)
{
262 263 264 265 266 267 268
	int pos;

	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
	if (pos)
		pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap);

	return pos;
Linus Torvalds's avatar
Linus Torvalds committed
269
}
270
EXPORT_SYMBOL(pci_find_capability);
Linus Torvalds's avatar
Linus Torvalds committed
271 272

/**
273
 * pci_bus_find_capability - query for devices' capabilities
Linus Torvalds's avatar
Linus Torvalds committed
274 275 276 277 278
 * @bus:   the PCI bus to query
 * @devfn: PCI device to query
 * @cap:   capability code
 *
 * Like pci_find_capability() but works for pci devices that do not have a
279
 * pci_dev structure set up yet.
Linus Torvalds's avatar
Linus Torvalds committed
280 281 282 283 284 285 286
 *
 * Returns the address of the requested capability structure within the
 * device's PCI configuration space or 0 in case the device does not
 * support it.
 */
int pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap)
{
287
	int pos;
Linus Torvalds's avatar
Linus Torvalds committed
288 289 290 291
	u8 hdr_type;

	pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type);

292 293 294 295 296
	pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & 0x7f);
	if (pos)
		pos = __pci_find_next_cap(bus, devfn, pos, cap);

	return pos;
Linus Torvalds's avatar
Linus Torvalds committed
297
}
298
EXPORT_SYMBOL(pci_bus_find_capability);
Linus Torvalds's avatar
Linus Torvalds committed
299 300

/**
301
 * pci_find_next_ext_capability - Find an extended capability
Linus Torvalds's avatar
Linus Torvalds committed
302
 * @dev: PCI device to query
303
 * @start: address at which to start looking (0 to start at beginning of list)
Linus Torvalds's avatar
Linus Torvalds committed
304 305
 * @cap: capability code
 *
306
 * Returns the address of the next matching extended capability structure
Linus Torvalds's avatar
Linus Torvalds committed
307
 * within the device's PCI configuration space or 0 if the device does
308 309
 * not support it.  Some capabilities can occur several times, e.g., the
 * vendor-specific capability, and this provides a way to find them all.
Linus Torvalds's avatar
Linus Torvalds committed
310
 */
311
int pci_find_next_ext_capability(struct pci_dev *dev, int start, int cap)
Linus Torvalds's avatar
Linus Torvalds committed
312 313
{
	u32 header;
314 315
	int ttl;
	int pos = PCI_CFG_SPACE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
316

317 318 319 320
	/* minimum 8 bytes per capability */
	ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;

	if (dev->cfg_size <= PCI_CFG_SPACE_SIZE)
Linus Torvalds's avatar
Linus Torvalds committed
321 322
		return 0;

323 324 325
	if (start)
		pos = start;

Linus Torvalds's avatar
Linus Torvalds committed
326 327 328 329 330 331 332 333 334 335 336
	if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
		return 0;

	/*
	 * If we have no capabilities, this is indicated by cap ID,
	 * cap version and next pointer all being 0.
	 */
	if (header == 0)
		return 0;

	while (ttl-- > 0) {
337
		if (PCI_EXT_CAP_ID(header) == cap && pos != start)
Linus Torvalds's avatar
Linus Torvalds committed
338 339 340
			return pos;

		pos = PCI_EXT_CAP_NEXT(header);
341
		if (pos < PCI_CFG_SPACE_SIZE)
Linus Torvalds's avatar
Linus Torvalds committed
342 343 344 345 346 347 348 349
			break;

		if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
			break;
	}

	return 0;
}
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
EXPORT_SYMBOL_GPL(pci_find_next_ext_capability);

/**
 * pci_find_ext_capability - Find an extended capability
 * @dev: PCI device to query
 * @cap: capability code
 *
 * Returns the address of the requested extended capability structure
 * within the device's PCI configuration space or 0 if the device does
 * not support it.  Possible values for @cap:
 *
 *  %PCI_EXT_CAP_ID_ERR		Advanced Error Reporting
 *  %PCI_EXT_CAP_ID_VC		Virtual Channel
 *  %PCI_EXT_CAP_ID_DSN		Device Serial Number
 *  %PCI_EXT_CAP_ID_PWR		Power Budgeting
 */
int pci_find_ext_capability(struct pci_dev *dev, int cap)
{
	return pci_find_next_ext_capability(dev, 0, cap);
}
370
EXPORT_SYMBOL_GPL(pci_find_ext_capability);
Linus Torvalds's avatar
Linus Torvalds committed
371

372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
static int __pci_find_next_ht_cap(struct pci_dev *dev, int pos, int ht_cap)
{
	int rc, ttl = PCI_FIND_CAP_TTL;
	u8 cap, mask;

	if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST)
		mask = HT_3BIT_CAP_MASK;
	else
		mask = HT_5BIT_CAP_MASK;

	pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos,
				      PCI_CAP_ID_HT, &ttl);
	while (pos) {
		rc = pci_read_config_byte(dev, pos + 3, &cap);
		if (rc != PCIBIOS_SUCCESSFUL)
			return 0;

		if ((cap & mask) == ht_cap)
			return pos;

392 393
		pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn,
					      pos + PCI_CAP_LIST_NEXT,
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
					      PCI_CAP_ID_HT, &ttl);
	}

	return 0;
}
/**
 * pci_find_next_ht_capability - query a device's Hypertransport capabilities
 * @dev: PCI device to query
 * @pos: Position from which to continue searching
 * @ht_cap: Hypertransport capability code
 *
 * To be used in conjunction with pci_find_ht_capability() to search for
 * all capabilities matching @ht_cap. @pos should always be a value returned
 * from pci_find_ht_capability().
 *
 * NB. To be 100% safe against broken PCI devices, the caller should take
 * steps to avoid an infinite loop.
 */
int pci_find_next_ht_capability(struct pci_dev *dev, int pos, int ht_cap)
{
	return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap);
}
EXPORT_SYMBOL_GPL(pci_find_next_ht_capability);

/**
 * pci_find_ht_capability - query a device's Hypertransport capabilities
 * @dev: PCI device to query
 * @ht_cap: Hypertransport capability code
 *
 * Tell if a device supports a given Hypertransport capability.
 * Returns an address within the device's PCI configuration space
 * or 0 in case the device does not support the request capability.
 * The address points to the PCI capability, of type PCI_CAP_ID_HT,
 * which has a Hypertransport capability matching @ht_cap.
 */
int pci_find_ht_capability(struct pci_dev *dev, int ht_cap)
{
	int pos;

	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
	if (pos)
		pos = __pci_find_next_ht_cap(dev, pos, ht_cap);

	return pos;
}
EXPORT_SYMBOL_GPL(pci_find_ht_capability);

Linus Torvalds's avatar
Linus Torvalds committed
441 442 443 444 445 446
/**
 * pci_find_parent_resource - return resource region of parent bus of given region
 * @dev: PCI device structure contains resources to be searched
 * @res: child resource record for which parent is sought
 *
 *  For given resource region of given device, return the resource
447
 *  region of parent bus the given region is contained in.
Linus Torvalds's avatar
Linus Torvalds committed
448
 */
Ryan Desfosses's avatar
Ryan Desfosses committed
449 450
struct resource *pci_find_parent_resource(const struct pci_dev *dev,
					  struct resource *res)
Linus Torvalds's avatar
Linus Torvalds committed
451 452
{
	const struct pci_bus *bus = dev->bus;
453
	struct resource *r;
Linus Torvalds's avatar
Linus Torvalds committed
454 455
	int i;

456
	pci_bus_for_each_resource(bus, r, i) {
Linus Torvalds's avatar
Linus Torvalds committed
457 458
		if (!r)
			continue;
459
		if (resource_contains(r, res)) {
460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478

			/*
			 * If the window is prefetchable but the BAR is
			 * not, the allocator made a mistake.
			 */
			if (r->flags & IORESOURCE_PREFETCH &&
			    !(res->flags & IORESOURCE_PREFETCH))
				return NULL;

			/*
			 * If we're below a transparent bridge, there may
			 * be both a positively-decoded aperture and a
			 * subtractively-decoded region that contain the BAR.
			 * We want the positively-decoded one, so this depends
			 * on pci_bus_for_each_resource() giving us those
			 * first.
			 */
			return r;
		}
Linus Torvalds's avatar
Linus Torvalds committed
479
	}
480
	return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
481
}
482
EXPORT_SYMBOL(pci_find_parent_resource);
Linus Torvalds's avatar
Linus Torvalds committed
483

484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507
/**
 * pci_find_resource - Return matching PCI device resource
 * @dev: PCI device to query
 * @res: Resource to look for
 *
 * Goes over standard PCI resources (BARs) and checks if the given resource
 * is partially or fully contained in any of them. In that case the
 * matching resource is returned, %NULL otherwise.
 */
struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res)
{
	int i;

	for (i = 0; i < PCI_ROM_RESOURCE; i++) {
		struct resource *r = &dev->resource[i];

		if (r->start && resource_contains(r, res))
			return r;
	}

	return NULL;
}
EXPORT_SYMBOL(pci_find_resource);

508 509 510 511 512 513 514 515 516
/**
 * pci_find_pcie_root_port - return PCIe Root Port
 * @dev: PCI device to query
 *
 * Traverse up the parent chain and return the PCIe Root Port PCI Device
 * for a given PCI Device.
 */
struct pci_dev *pci_find_pcie_root_port(struct pci_dev *dev)
{
517
	struct pci_dev *bridge, *highest_pcie_bridge = dev;
518 519 520 521 522 523 524

	bridge = pci_upstream_bridge(dev);
	while (bridge && pci_is_pcie(bridge)) {
		highest_pcie_bridge = bridge;
		bridge = pci_upstream_bridge(bridge);
	}

525 526
	if (pci_pcie_type(highest_pcie_bridge) != PCI_EXP_TYPE_ROOT_PORT)
		return NULL;
527

528
	return highest_pcie_bridge;
529 530 531
}
EXPORT_SYMBOL(pci_find_pcie_root_port);

532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
/**
 * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos
 * @dev: the PCI device to operate on
 * @pos: config space offset of status word
 * @mask: mask of bit(s) to care about in status word
 *
 * Return 1 when mask bit(s) in status word clear, 0 otherwise.
 */
int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask)
{
	int i;

	/* Wait for Transaction Pending bit clean */
	for (i = 0; i < 4; i++) {
		u16 status;
		if (i)
			msleep((1 << (i - 1)) * 100);

		pci_read_config_word(dev, pos, &status);
		if (!(status & mask))
			return 1;
	}

	return 0;
}

558
/**
Wei Yang's avatar
Wei Yang committed
559
 * pci_restore_bars - restore a device's BAR values (e.g. after wake-up)
560 561 562 563 564
 * @dev: PCI device to have its BARs restored
 *
 * Restore the BAR values for a given device, so as to make it
 * accessible by its driver.
 */
Ryan Desfosses's avatar
Ryan Desfosses committed
565
static void pci_restore_bars(struct pci_dev *dev)
566
{
567
	int i;
568

569
	for (i = 0; i < PCI_BRIDGE_RESOURCES; i++)
570
		pci_update_resource(dev, i);
571 572
}

573
static const struct pci_platform_pm_ops *pci_platform_pm;
574

575
int pci_set_platform_pm(const struct pci_platform_pm_ops *ops)
576
{
577
	if (!ops->is_manageable || !ops->set_state  || !ops->get_state ||
578
	    !ops->choose_state  || !ops->set_wakeup || !ops->need_resume)
579 580 581 582 583 584 585 586 587 588 589
		return -EINVAL;
	pci_platform_pm = ops;
	return 0;
}

static inline bool platform_pci_power_manageable(struct pci_dev *dev)
{
	return pci_platform_pm ? pci_platform_pm->is_manageable(dev) : false;
}

static inline int platform_pci_set_power_state(struct pci_dev *dev,
Ryan Desfosses's avatar
Ryan Desfosses committed
590
					       pci_power_t t)
591 592 593 594
{
	return pci_platform_pm ? pci_platform_pm->set_state(dev, t) : -ENOSYS;
}

595 596 597 598 599
static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev)
{
	return pci_platform_pm ? pci_platform_pm->get_state(dev) : PCI_UNKNOWN;
}

600 601 602 603 604
static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev)
{
	return pci_platform_pm ?
			pci_platform_pm->choose_state(dev) : PCI_POWER_ERROR;
}
Randy Dunlap's avatar
Randy Dunlap committed
605

606
static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable)
607 608
{
	return pci_platform_pm ?
609
			pci_platform_pm->set_wakeup(dev, enable) : -ENODEV;
610 611
}

612 613 614 615 616
static inline bool platform_pci_need_resume(struct pci_dev *dev)
{
	return pci_platform_pm ? pci_platform_pm->need_resume(dev) : false;
}

Linus Torvalds's avatar
Linus Torvalds committed
617
/**
618 619 620 621
 * pci_raw_set_power_state - Use PCI PM registers to set the power state of
 *                           given PCI device
 * @dev: PCI device to handle.
 * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
Linus Torvalds's avatar
Linus Torvalds committed
622
 *
623 624 625 626 627 628
 * RETURN VALUE:
 * -EINVAL if the requested state is invalid.
 * -EIO if device does not support PCI PM or its PM capabilities register has a
 * wrong version, or device doesn't support the requested state.
 * 0 if device already is in the requested state.
 * 0 if device's power state has been successfully changed.
Linus Torvalds's avatar
Linus Torvalds committed
629
 */
630
static int pci_raw_set_power_state(struct pci_dev *dev, pci_power_t state)
Linus Torvalds's avatar
Linus Torvalds committed
631
{
632
	u16 pmcsr;
633
	bool need_restore = false;
Linus Torvalds's avatar
Linus Torvalds committed
634

635 636 637 638
	/* Check if we're already there */
	if (dev->current_state == state)
		return 0;

639
	if (!dev->pm_cap)
640 641
		return -EIO;

642 643 644
	if (state < PCI_D0 || state > PCI_D3hot)
		return -EINVAL;

Linus Torvalds's avatar
Linus Torvalds committed
645
	/* Validate current state:
646
	 * Can enter D0 from any state, but if we can only go deeper
Linus Torvalds's avatar
Linus Torvalds committed
647 648
	 * to sleep if we're already in a low power state
	 */
649
	if (state != PCI_D0 && dev->current_state <= PCI_D3cold
650
	    && dev->current_state > state) {
651 652
		dev_err(&dev->dev, "invalid power transition (from state %d to %d)\n",
			dev->current_state, state);
Linus Torvalds's avatar
Linus Torvalds committed
653
		return -EINVAL;
654
	}
Linus Torvalds's avatar
Linus Torvalds committed
655 656

	/* check if this device supports the desired state */
657 658
	if ((state == PCI_D1 && !dev->d1_support)
	   || (state == PCI_D2 && !dev->d2_support))
659
		return -EIO;
Linus Torvalds's avatar
Linus Torvalds committed
660

661
	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
662

663
	/* If we're (effectively) in D3, force entire word to 0.
Linus Torvalds's avatar
Linus Torvalds committed
664 665 666
	 * This doesn't affect PME_Status, disables PME_En, and
	 * sets PowerState to 0.
	 */
667
	switch (dev->current_state) {
668 669 670 671 672 673
	case PCI_D0:
	case PCI_D1:
	case PCI_D2:
		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
		pmcsr |= state;
		break;
674 675
	case PCI_D3hot:
	case PCI_D3cold:
676 677
	case PCI_UNKNOWN: /* Boot-up */
		if ((pmcsr & PCI_PM_CTRL_STATE_MASK) == PCI_D3hot
678
		 && !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET))
679
			need_restore = true;
680 681
		/* Fall-through: force to D0 */
	default:
682
		pmcsr = 0;
683
		break;
Linus Torvalds's avatar
Linus Torvalds committed
684 685 686
	}

	/* enter specified state */
687
	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
Linus Torvalds's avatar
Linus Torvalds committed
688 689 690 691

	/* Mandatory power management transition delays */
	/* see PCI PM 1.1 5.6.1 table 18 */
	if (state == PCI_D3hot || dev->current_state == PCI_D3hot)
692
		pci_dev_d3_sleep(dev);
Linus Torvalds's avatar
Linus Torvalds committed
693
	else if (state == PCI_D2 || dev->current_state == PCI_D2)
694
		udelay(PCI_PM_D2_DELAY);
Linus Torvalds's avatar
Linus Torvalds committed
695

696 697 698
	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
	dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
	if (dev->current_state != state && printk_ratelimit())
699 700
		dev_info(&dev->dev, "Refused to change power state, currently in D%d\n",
			 dev->current_state);
701

702 703
	/*
	 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT
704 705 706 707 708 709 710 711 712 713 714 715 716 717
	 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning
	 * from D3hot to D0 _may_ perform an internal reset, thereby
	 * going to "D0 Uninitialized" rather than "D0 Initialized".
	 * For example, at least some versions of the 3c905B and the
	 * 3c556B exhibit this behaviour.
	 *
	 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave
	 * devices in a D3hot state at boot.  Consequently, we need to
	 * restore at least the BARs so that the device will be
	 * accessible to its driver.
	 */
	if (need_restore)
		pci_restore_bars(dev);

718
	if (dev->bus->self)
719 720
		pcie_aspm_pm_state_change(dev->bus->self);

Linus Torvalds's avatar
Linus Torvalds committed
721 722 723
	return 0;
}

724
/**
725
 * pci_update_current_state - Read power state of given device and cache it
726
 * @dev: PCI device to handle.
727
 * @state: State to cache in case the device doesn't have the PM capability
728 729 730 731 732 733 734
 *
 * The power state is read from the PMCSR register, which however is
 * inaccessible in D3cold.  The platform firmware is therefore queried first
 * to detect accessibility of the register.  In case the platform firmware
 * reports an incorrect state or the device isn't power manageable by the
 * platform at all, we try to detect D3cold by testing accessibility of the
 * vendor ID in config space.
735
 */
736
void pci_update_current_state(struct pci_dev *dev, pci_power_t state)
737
{
738 739 740 741
	if (platform_pci_get_power_state(dev) == PCI_D3cold ||
	    !pci_device_is_present(dev)) {
		dev->current_state = PCI_D3cold;
	} else if (dev->pm_cap) {
742 743
		u16 pmcsr;

744
		pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
745
		dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
746 747
	} else {
		dev->current_state = state;
748 749 750
	}
}

751 752 753 754 755 756 757 758 759 760 761 762 763
/**
 * pci_power_up - Put the given device into D0 forcibly
 * @dev: PCI device to power up
 */
void pci_power_up(struct pci_dev *dev)
{
	if (platform_pci_power_manageable(dev))
		platform_pci_set_power_state(dev, PCI_D0);

	pci_raw_set_power_state(dev, PCI_D0);
	pci_update_current_state(dev, PCI_D0);
}

764 765 766 767 768 769 770 771 772 773 774 775 776
/**
 * pci_platform_power_transition - Use platform to change device power state
 * @dev: PCI device to handle.
 * @state: State to put the device into.
 */
static int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state)
{
	int error;

	if (platform_pci_power_manageable(dev)) {
		error = platform_pci_set_power_state(dev, state);
		if (!error)
			pci_update_current_state(dev, state);
777
	} else
778
		error = -ENODEV;
779 780 781

	if (error && !dev->pm_cap) /* Fall back to PCI_D0 */
		dev->current_state = PCI_D0;
782 783 784 785

	return error;
}

786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807
/**
 * pci_wakeup - Wake up a PCI device
 * @pci_dev: Device to handle.
 * @ign: ignored parameter
 */
static int pci_wakeup(struct pci_dev *pci_dev, void *ign)
{
	pci_wakeup_event(pci_dev);
	pm_request_resume(&pci_dev->dev);
	return 0;
}

/**
 * pci_wakeup_bus - Walk given bus and wake up devices on it
 * @bus: Top bus of the subtree to walk.
 */
static void pci_wakeup_bus(struct pci_bus *bus)
{
	if (bus)
		pci_walk_bus(bus, pci_wakeup, NULL);
}

808 809 810 811 812 813 814
/**
 * __pci_start_power_transition - Start power transition of a PCI device
 * @dev: PCI device to handle.
 * @state: State to put the device into.
 */
static void __pci_start_power_transition(struct pci_dev *dev, pci_power_t state)
{
815
	if (state == PCI_D0) {
816
		pci_platform_power_transition(dev, PCI_D0);
817 818 819 820 821 822 823 824
		/*
		 * Mandatory power management transition delays, see
		 * PCI Express Base Specification Revision 2.0 Section
		 * 6.6.1: Conventional Reset.  Do not delay for
		 * devices powered on/off by corresponding bridge,
		 * because have already delayed for the bridge.
		 */
		if (dev->runtime_d3cold) {
825 826
			if (dev->d3cold_delay)
				msleep(dev->d3cold_delay);
827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859
			/*
			 * When powering on a bridge from D3cold, the
			 * whole hierarchy may be powered on into
			 * D0uninitialized state, resume them to give
			 * them a chance to suspend again
			 */
			pci_wakeup_bus(dev->subordinate);
		}
	}
}

/**
 * __pci_dev_set_current_state - Set current state of a PCI device
 * @dev: Device to handle
 * @data: pointer to state to be set
 */
static int __pci_dev_set_current_state(struct pci_dev *dev, void *data)
{
	pci_power_t state = *(pci_power_t *)data;

	dev->current_state = state;
	return 0;
}

/**
 * __pci_bus_set_current_state - Walk given bus and set current state of devices
 * @bus: Top bus of the subtree to walk.
 * @state: state to be set
 */
static void __pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state)
{
	if (bus)
		pci_walk_bus(bus, __pci_dev_set_current_state, &state);
860 861 862 863 864 865 866 867 868 869 870
}

/**
 * __pci_complete_power_transition - Complete power transition of a PCI device
 * @dev: PCI device to handle.
 * @state: State to put the device into.
 *
 * This function should not be called directly by device drivers.
 */
int __pci_complete_power_transition(struct pci_dev *dev, pci_power_t state)
{
871 872
	int ret;

873
	if (state <= PCI_D0)
874 875 876 877 878 879
		return -EINVAL;
	ret = pci_platform_power_transition(dev, state);
	/* Power off the bridge may power off the whole hierarchy */
	if (!ret && state == PCI_D3cold)
		__pci_bus_set_current_state(dev->subordinate, PCI_D3cold);
	return ret;
880 881 882
}
EXPORT_SYMBOL_GPL(__pci_complete_power_transition);

883 884 885 886 887
/**
 * pci_set_power_state - Set the power state of a PCI device
 * @dev: PCI device to handle.
 * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
 *
888
 * Transition a device to a new power state, using the platform firmware and/or
889 890 891 892 893 894
 * the device's PCI PM registers.
 *
 * RETURN VALUE:
 * -EINVAL if the requested state is invalid.
 * -EIO if device does not support PCI PM or its PM capabilities register has a
 * wrong version, or device doesn't support the requested state.
895
 * 0 if the transition is to D1 or D2 but D1 and D2 are not supported.
896
 * 0 if device already is in the requested state.
897
 * 0 if the transition is to D3 but D3 is not supported.
898 899 900 901
 * 0 if device's power state has been successfully changed.
 */
int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
{
902
	int error;
903 904

	/* bound the state we're entering */
905 906
	if (state > PCI_D3cold)
		state = PCI_D3cold;
907 908 909 910 911 912 913 914 915 916
	else if (state < PCI_D0)
		state = PCI_D0;
	else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev))
		/*
		 * If the device or the parent bridge do not support PCI PM,
		 * ignore the request if we're doing anything other than putting
		 * it into D0 (which would only happen on boot).
		 */
		return 0;

917 918 919 920
	/* Check if we're already there */
	if (dev->current_state == state)
		return 0;

921 922
	__pci_start_power_transition(dev, state);

923 924
	/* This device is quirked not to be put into D3, so
	   don't put it in D3 */
925
	if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3))
926
		return 0;
927

928 929 930 931 932 933
	/*
	 * To put device in D3cold, we put device into D3hot in native
	 * way, then put device into D3cold with platform ops
	 */
	error = pci_raw_set_power_state(dev, state > PCI_D3hot ?
					PCI_D3hot : state);
934

935 936
	if (!__pci_complete_power_transition(dev, state))
		error = 0;
937 938 939

	return error;
}
940
EXPORT_SYMBOL(pci_set_power_state);
941

Linus Torvalds's avatar
Linus Torvalds committed
942 943 944 945 946 947 948 949 950 951 952 953
/**
 * pci_choose_state - Choose the power state of a PCI device
 * @dev: PCI device to be suspended
 * @state: target sleep state for the whole system. This is the value
 *	that is passed to suspend() function.
 *
 * Returns PCI power state suitable for given device and given system
 * message.
 */

pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state)
{
954
	pci_power_t ret;
955

956
	if (!dev->pm_cap)
Linus Torvalds's avatar
Linus Torvalds committed
957 958
		return PCI_D0;

959 960 961
	ret = platform_pci_choose_state(dev);
	if (ret != PCI_POWER_ERROR)
		return ret;
962 963 964 965 966

	switch (state.event) {
	case PM_EVENT_ON:
		return PCI_D0;
	case PM_EVENT_FREEZE:
967 968
	case PM_EVENT_PRETHAW:
		/* REVISIT both freeze and pre-thaw "should" use D0 */
969
	case PM_EVENT_SUSPEND:
970
	case PM_EVENT_HIBERNATE:
971
		return PCI_D3hot;
Linus Torvalds's avatar
Linus Torvalds committed
972
	default:
973 974
		dev_info(&dev->dev, "unrecognized suspend event %d\n",
			 state.event);
Linus Torvalds's avatar
Linus Torvalds committed
975 976 977 978 979 980
		BUG();
	}
	return PCI_D0;
}
EXPORT_SYMBOL(pci_choose_state);

981 982
#define PCI_EXP_SAVE_REGS	7

983 984
static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev,
						       u16 cap, bool extended)
985 986 987
{
	struct pci_cap_saved_state *tmp;

988
	hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) {
989
		if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap)
990 991 992 993 994
			return tmp;
	}
	return NULL;
}

995 996 997 998 999 1000 1001 1002 1003 1004
struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap)
{
	return _pci_find_saved_cap(dev, cap, false);
}

struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap)
{
	return _pci_find_saved_cap(dev, cap, true);
}

1005 1006
static int pci_save_pcie_state(struct pci_dev *dev)
{
1007
	int i = 0;
1008 1009 1010
	struct pci_cap_saved_state *save_state;
	u16 *cap;

1011
	if (!pci_is_pcie(dev))
1012 1013
		return 0;

1014
	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1015
	if (!save_state) {
1016
		dev_err(&dev->dev, "buffer not found in %s\n", __func__);
1017 1018
		return -ENOMEM;
	}
1019

1020 1021 1022 1023 1024 1025 1026 1027
	cap = (u16 *)&save_state->cap.data[0];
	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]);
	pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]);
	pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]);
	pcie_capability_read_word(dev, PCI_EXP_RTCTL,  &cap[i++]);
	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]);
	pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]);
	pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]);
1028

1029 1030 1031 1032 1033
	return 0;
}

static void pci_restore_pcie_state(struct pci_dev *dev)
{
1034
	int i = 0;
1035 1036 1037 1038
	struct pci_cap_saved_state *save_state;
	u16 *cap;

	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1039
	if (!save_state)
1040 1041
		return;

1042 1043 1044 1045 1046 1047 1048 1049
	cap = (u16 *)&save_state->cap.data[0];
	pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]);
	pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]);
	pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]);
	pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]);
	pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]);
	pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]);
	pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]);
1050 1051
}

1052 1053 1054

static int pci_save_pcix_state(struct pci_dev *dev)
{
1055
	int pos;
1056 1057 1058
	struct pci_cap_saved_state *save_state;

	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1059
	if (!pos)
1060 1061
		return 0;

1062
	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1063
	if (!save_state) {
1064
		dev_err(&dev->dev, "buffer not found in %s\n", __func__);
1065 1066 1067
		return -ENOMEM;
	}

1068 1069
	pci_read_config_word(dev, pos + PCI_X_CMD,
			     (u16 *)save_state->cap.data);
1070

1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
	return 0;
}

static void pci_restore_pcix_state(struct pci_dev *dev)
{
	int i = 0, pos;
	struct pci_cap_saved_state *save_state;
	u16 *cap;

	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1082
	if (!save_state || !pos)
1083
		return;
1084
	cap = (u16 *)&save_state->cap.data[0];
1085 1086 1087 1088 1089

	pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]);
}


Linus Torvalds's avatar
Linus Torvalds committed
1090 1091 1092 1093
/**
 * pci_save_state - save the PCI configuration space of a device before suspending
 * @dev: - PCI device that we're dealing with
 */
Ryan Desfosses's avatar
Ryan Desfosses committed
1094
int pci_save_state(struct pci_dev *dev)
Linus Torvalds's avatar
Linus Torvalds committed
1095 1096 1097 1098
{
	int i;
	/* XXX: 100% dword access ok here? */
	for (i = 0; i < 16; i++)
1099
		pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]);
1100
	dev->state_saved = true;
1101 1102 1103

	i = pci_save_pcie_state(dev);
	if (i != 0)
1104
		return i;
1105 1106 1107

	i = pci_save_pcix_state(dev);
	if (i != 0)
1108
		return i;
1109

1110
	return pci_save_vc_state(dev);
Linus Torvalds's avatar
Linus Torvalds committed
1111
}
1112
EXPORT_SYMBOL(pci_save_state);
Linus Torvalds's avatar
Linus Torvalds committed
1113

1114
static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
1115
				     u32 saved_val, int retry, bool force)
1116 1117 1118 1119
{
	u32 val;

	pci_read_config_dword(pdev, offset, &val);
1120
	if (!force && val == saved_val)
1121 1122 1123
		return;

	for (;;) {
1124 1125
		dev_dbg(&pdev->dev, "restoring config space at offset %#x (was %#x, writing %#x)\n",
			offset, val, saved_val);
1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137
		pci_write_config_dword(pdev, offset, saved_val);
		if (retry-- <= 0)
			return;

		pci_read_config_dword(pdev, offset, &val);
		if (val == saved_val)
			return;

		mdelay(1);
	}
}

1138
static void pci_restore_config_space_range(struct pci_dev *pdev,
1139 1140
					   int start, int end, int retry,
					   bool force)
1141 1142 1143 1144 1145 1146
{
	int index;

	for (index = end; index >= start; index--)
		pci_restore_config_dword(pdev, 4 * index,
					 pdev->saved_config_space[index],
1147
					 retry, force);
1148 1149
}

1150 1151 1152
static void pci_restore_config_space(struct pci_dev *pdev)
{
	if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
1153
		pci_restore_config_space_range(pdev, 10, 15, 0, false);
1154
		/* Restore BARs before the command register. */
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
		pci_restore_config_space_range(pdev, 4, 9, 10, false);
		pci_restore_config_space_range(pdev, 0, 3, 0, false);
	} else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
		pci_restore_config_space_range(pdev, 12, 15, 0, false);

		/*
		 * Force rewriting of prefetch registers to avoid S3 resume
		 * issues on Intel PCI bridges that occur when these
		 * registers are not explicitly written.
		 */
		pci_restore_config_space_range(pdev, 9, 11, 0, true);
		pci_restore_config_space_range(pdev, 0, 8, 0, false);
1167
	} else {
1168
		pci_restore_config_space_range(pdev, 0, 15, 0, false);
1169 1170 1171
	}
}

1172
/**
Linus Torvalds's avatar
Linus Torvalds committed
1173 1174 1175
 * pci_restore_state - Restore the saved state of a PCI device
 * @dev: - PCI device that we're dealing with
 */
1176
void pci_restore_state(struct pci_dev *dev)
Linus Torvalds's avatar
Linus Torvalds committed
1177
{
1178
	if (!dev->state_saved)
1179
		return;
1180

1181 1182
	/* PCI Express register must be restored first */
	pci_restore_pcie_state(dev);
1183 1184
	pci_restore_pasid_state(dev);
	pci_restore_pri_state(dev);
1185
	pci_restore_ats_state(dev);
1186
	pci_restore_vc_state(dev);
1187

1188 1189
	pci_cleanup_aer_error_status_regs(dev);

1190
	pci_restore_config_space(dev);
1191

1192
	pci_restore_pcix_state(dev);
1193
	pci_restore_msi_state(dev);
1194 1195 1196

	/* Restore ACS and IOV configuration state */
	pci_enable_acs(dev);
Yu Zhao's avatar
Yu Zhao committed
1197
	pci_restore_iov_state(dev);
1198

1199
	dev->state_saved = false;
Linus Torvalds's avatar
Linus Torvalds committed
1200
}
1201
EXPORT_SYMBOL(pci_restore_state);
Linus Torvalds's avatar
Linus Torvalds committed
1202

1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
struct pci_saved_state {
	u32 config_space[16];
	struct pci_cap_saved_data cap[0];
};

/**
 * pci_store_saved_state - Allocate and return an opaque struct containing
 *			   the device saved state.
 * @dev: PCI device that we're dealing with
 *
1213
 * Return NULL if no state or error.
1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226
 */
struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev)
{
	struct pci_saved_state *state;
	struct pci_cap_saved_state *tmp;
	struct pci_cap_saved_data *cap;
	size_t size;

	if (!dev->state_saved)
		return NULL;

	size = sizeof(*state) + sizeof(struct pci_cap_saved_data);

1227
	hlist_for_each_entry(tmp, &dev->saved_cap_space, next)
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
		size += sizeof(struct pci_cap_saved_data) + tmp->cap.size;

	state = kzalloc(size, GFP_KERNEL);
	if (!state)
		return NULL;

	memcpy(state->config_space, dev->saved_config_space,
	       sizeof(state->config_space));

	cap = state->cap;
1238
	hlist_for_each_entry(tmp, &dev->saved_cap_space, next) {
1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253
		size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size;
		memcpy(cap, &tmp->cap, len);
		cap = (struct pci_cap_saved_data *)((u8 *)cap + len);
	}
	/* Empty cap_save terminates list */

	return state;
}
EXPORT_SYMBOL_GPL(pci_store_saved_state);

/**
 * pci_load_saved_state - Reload the provided save state into struct pci_dev.
 * @dev: PCI device that we're dealing with
 * @state: Saved state returned from pci_store_saved_state()
 */
1254 1255
int pci_load_saved_state(struct pci_dev *dev,
			 struct pci_saved_state *state)
1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
{
	struct pci_cap_saved_data *cap;

	dev->state_saved = false;

	if (!state)
		return 0;

	memcpy(dev->saved_config_space, state->config_space,
	       sizeof(state->config_space));

	cap = state->cap;
	while (cap->size) {
		struct pci_cap_saved_state *tmp;

1271
		tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended);
1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282
		if (!tmp || tmp->cap.size != cap->size)
			return -EINVAL;

		memcpy(tmp->cap.data, cap->data, tmp->cap.size);
		cap = (struct pci_cap_saved_data *)((u8 *)cap +
		       sizeof(struct pci_cap_saved_data) + cap->size);
	}

	dev->state_saved = true;
	return 0;
}
1283
EXPORT_SYMBOL_GPL(pci_load_saved_state);
1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300

/**
 * pci_load_and_free_saved_state - Reload the save state pointed to by state,
 *				   and free the memory allocated for it.
 * @dev: PCI device that we're dealing with
 * @state: Pointer to saved state returned from pci_store_saved_state()
 */
int pci_load_and_free_saved_state(struct pci_dev *dev,
				  struct pci_saved_state **state)
{
	int ret = pci_load_saved_state(dev, *state);
	kfree(*state);
	*state = NULL;
	return ret;
}
EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state);

1301 1302 1303 1304 1305
int __weak pcibios_enable_device(struct pci_dev *dev, int bars)
{
	return pci_enable_resources(dev, bars);
}

1306 1307 1308
static int do_pci_enable_device(struct pci_dev *dev, int bars)
{
	int err;
1309
	struct pci_dev *bridge;
1310 1311
	u16 cmd;
	u8 pin;
1312 1313 1314 1315

	err = pci_set_power_state(dev, PCI_D0);
	if (err < 0 && err != -EIO)
		return err;
1316 1317 1318 1319 1320

	bridge = pci_upstream_bridge(dev);
	if (bridge)
		pcie_aspm_powersave_config_link(bridge);

1321 1322 1323 1324 1325
	err = pcibios_enable_device(dev, bars);
	if (err < 0)
		return err;
	pci_fixup_device(pci_fixup_enable, dev);

1326 1327 1328
	if (dev->msi_enabled || dev->msix_enabled)
		return 0;

1329 1330 1331 1332 1333 1334 1335 1336
	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin);
	if (pin) {
		pci_read_config_word(dev, PCI_COMMAND, &cmd);
		if (cmd & PCI_COMMAND_INTX_DISABLE)
			pci_write_config_word(dev, PCI_COMMAND,
					      cmd & ~PCI_COMMAND_INTX_DISABLE);
	}

1337 1338 1339 1340
	return 0;
}

/**
1341
 * pci_reenable_device - Resume abandoned device
1342 1343 1344 1345 1346
 * @dev: PCI device to be resumed
 *
 *  Note this function is a backend of pci_default_resume and is not supposed
 *  to be called by normal code, write proper resume handler and use it instead.
 */
1347
int pci_reenable_device(struct pci_dev *dev)
1348
{
1349
	if (pci_is_enabled(dev))
1350 1351 1352
		return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1);
	return 0;
}
1353
EXPORT_SYMBOL(pci_reenable_device);