bootmem.c 15.2 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11
/*
 *  linux/mm/bootmem.c
 *
 *  Copyright (C) 1999 Ingo Molnar
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *
 *  simple boot-time physical memory area allocator and
 *  free memory collector. It's used to deal with reserved
 *  system memory and memory holes as well.
 */
#include <linux/init.h>
12
#include <linux/pfn.h>
Linus Torvalds's avatar
Linus Torvalds committed
13 14
#include <linux/bootmem.h>
#include <linux/module.h>
15 16

#include <asm/bug.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
#include <asm/io.h>
18
#include <asm/processor.h>
19

Linus Torvalds's avatar
Linus Torvalds committed
20 21 22 23 24 25 26 27 28 29
#include "internal.h"

/*
 * Access to this subsystem has to be serialized externally. (this is
 * true for the boot process anyway)
 */
unsigned long max_low_pfn;
unsigned long min_low_pfn;
unsigned long max_pfn;

30
static LIST_HEAD(bdata_list);
31 32 33 34 35 36 37 38
#ifdef CONFIG_CRASH_DUMP
/*
 * If we have booted due to a crash, max_pfn will be a very low value. We need
 * to know the amount of memory that the previous kernel used.
 */
unsigned long saved_max_pfn;
#endif

39 40
bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;

41 42 43 44 45 46 47 48 49 50 51 52 53
/*
 * Given an initialised bdata, it returns the size of the boot bitmap
 */
static unsigned long __init get_mapsize(bootmem_data_t *bdata)
{
	unsigned long mapsize;
	unsigned long start = PFN_DOWN(bdata->node_boot_start);
	unsigned long end = bdata->node_low_pfn;

	mapsize = ((end - start) + 7) / 8;
	return ALIGN(mapsize, sizeof(long));
}

Linus Torvalds's avatar
Linus Torvalds committed
54
/* return the number of _pages_ that will be allocated for the boot bitmap */
55
unsigned long __init bootmem_bootmap_pages(unsigned long pages)
Linus Torvalds's avatar
Linus Torvalds committed
56 57 58 59 60 61 62 63 64
{
	unsigned long mapsize;

	mapsize = (pages+7)/8;
	mapsize = (mapsize + ~PAGE_MASK) & PAGE_MASK;
	mapsize >>= PAGE_SHIFT;

	return mapsize;
}
65

66 67 68
/*
 * link bdata in order
 */
69
static void __init link_bootmem(bootmem_data_t *bdata)
70 71
{
	bootmem_data_t *ent;
72

73 74 75 76 77 78 79 80 81 82 83 84 85 86
	if (list_empty(&bdata_list)) {
		list_add(&bdata->list, &bdata_list);
		return;
	}
	/* insert in order */
	list_for_each_entry(ent, &bdata_list, list) {
		if (bdata->node_boot_start < ent->node_boot_start) {
			list_add_tail(&bdata->list, &ent->list);
			return;
		}
	}
	list_add_tail(&bdata->list, &bdata_list);
}

Linus Torvalds's avatar
Linus Torvalds committed
87 88 89
/*
 * Called once to set up the allocator itself.
 */
90
static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,
Linus Torvalds's avatar
Linus Torvalds committed
91 92
	unsigned long mapstart, unsigned long start, unsigned long end)
{
93
	unsigned long mapsize;
Linus Torvalds's avatar
Linus Torvalds committed
94

95
	mminit_validate_memmodel_limits(&start, &end);
96 97
	bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
	bdata->node_boot_start = PFN_PHYS(start);
Linus Torvalds's avatar
Linus Torvalds committed
98
	bdata->node_low_pfn = end;
99
	link_bootmem(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
100 101 102 103 104

	/*
	 * Initially all pages are reserved - setup_arch() has to
	 * register free RAM areas explicitly.
	 */
105
	mapsize = get_mapsize(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
106 107 108 109 110
	memset(bdata->node_bootmem_map, 0xff, mapsize);

	return mapsize;
}

111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250
unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
				unsigned long startpfn, unsigned long endpfn)
{
	return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);
}

unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
{
	max_low_pfn = pages;
	min_low_pfn = start;
	return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
}

static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
{
	struct page *page;
	unsigned long pfn;
	unsigned long i, count;
	unsigned long idx;
	unsigned long *map;
	int gofast = 0;

	BUG_ON(!bdata->node_bootmem_map);

	count = 0;
	/* first extant page of the node */
	pfn = PFN_DOWN(bdata->node_boot_start);
	idx = bdata->node_low_pfn - pfn;
	map = bdata->node_bootmem_map;
	/*
	 * Check if we are aligned to BITS_PER_LONG pages.  If so, we might
	 * be able to free page orders of that size at once.
	 */
	if (!(pfn & (BITS_PER_LONG-1)))
		gofast = 1;

	for (i = 0; i < idx; ) {
		unsigned long v = ~map[i / BITS_PER_LONG];

		if (gofast && v == ~0UL) {
			int order;

			page = pfn_to_page(pfn);
			count += BITS_PER_LONG;
			order = ffs(BITS_PER_LONG) - 1;
			__free_pages_bootmem(page, order);
			i += BITS_PER_LONG;
			page += BITS_PER_LONG;
		} else if (v) {
			unsigned long m;

			page = pfn_to_page(pfn);
			for (m = 1; m && i < idx; m<<=1, page++, i++) {
				if (v & m) {
					count++;
					__free_pages_bootmem(page, 0);
				}
			}
		} else {
			i += BITS_PER_LONG;
		}
		pfn += BITS_PER_LONG;
	}

	/*
	 * Now free the allocator bitmap itself, it's not
	 * needed anymore:
	 */
	page = virt_to_page(bdata->node_bootmem_map);
	idx = (get_mapsize(bdata) + PAGE_SIZE-1) >> PAGE_SHIFT;
	for (i = 0; i < idx; i++, page++)
		__free_pages_bootmem(page, 0);
	count += i;
	bdata->node_bootmem_map = NULL;

	return count;
}

unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
{
	register_page_bootmem_info_node(pgdat);
	return free_all_bootmem_core(pgdat->bdata);
}

unsigned long __init free_all_bootmem(void)
{
	return free_all_bootmem_core(NODE_DATA(0)->bdata);
}

static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr,
				     unsigned long size)
{
	unsigned long sidx, eidx;
	unsigned long i;

	BUG_ON(!size);

	/* out range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;
	/*
	 * round down end of usable mem, partially free pages are
	 * considered reserved.
	 */

	if (addr >= bdata->node_boot_start && addr < bdata->last_success)
		bdata->last_success = addr;

	/*
	 * Round up to index to the range.
	 */
	if (PFN_UP(addr) > PFN_DOWN(bdata->node_boot_start))
		sidx = PFN_UP(addr) - PFN_DOWN(bdata->node_boot_start);
	else
		sidx = 0;

	eidx = PFN_DOWN(addr + size - bdata->node_boot_start);
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);

	for (i = sidx; i < eidx; i++) {
		if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map)))
			BUG();
	}
}

void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
			      unsigned long size)
{
	free_bootmem_core(pgdat->bdata, physaddr, size);
}

void __init free_bootmem(unsigned long addr, unsigned long size)
{
	bootmem_data_t *bdata;
	list_for_each_entry(bdata, &bdata_list, list)
		free_bootmem_core(bdata, addr, size);
}

Linus Torvalds's avatar
Linus Torvalds committed
251 252 253 254 255
/*
 * Marks a particular physical memory range as unallocatable. Usable RAM
 * might be used for boot-time allocations - or it might get added
 * to the free page pool later on.
 */
256
static int __init can_reserve_bootmem_core(bootmem_data_t *bdata,
257
			unsigned long addr, unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
258
{
259
	unsigned long sidx, eidx;
Linus Torvalds's avatar
Linus Torvalds committed
260
	unsigned long i;
261 262 263 264 265 266 267

	BUG_ON(!size);

	/* out of range, don't hold other */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return 0;
268

Linus Torvalds's avatar
Linus Torvalds committed
269
	/*
270
	 * Round up to index to the range.
Linus Torvalds's avatar
Linus Torvalds committed
271
	 */
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

	eidx = PFN_UP(addr + size - bdata->node_boot_start);
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);

	for (i = sidx; i < eidx; i++) {
		if (test_bit(i, bdata->node_bootmem_map)) {
			if (flags & BOOTMEM_EXCLUSIVE)
				return -EBUSY;
		}
	}

	return 0;

}

static void __init reserve_bootmem_core(bootmem_data_t *bdata,
			unsigned long addr, unsigned long size, int flags)
{
	unsigned long sidx, eidx;
	unsigned long i;

Linus Torvalds's avatar
Linus Torvalds committed
298
	BUG_ON(!size);
299

300 301 302 303 304 305 306 307 308 309 310 311 312
	/* out of range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;

	/*
	 * Round up to index to the range.
	 */
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

313
	eidx = PFN_UP(addr + size - bdata->node_boot_start);
314 315
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
316

317
	for (i = sidx; i < eidx; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
318 319 320 321 322
		if (test_and_set_bit(i, bdata->node_bootmem_map)) {
#ifdef CONFIG_DEBUG_BOOTMEM
			printk("hm, page %08lx reserved twice.\n", i*PAGE_SIZE);
#endif
		}
323
	}
Linus Torvalds's avatar
Linus Torvalds committed
324 325
}

326 327
int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
				 unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
328
{
329
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
330

331 332 333 334 335 336
	ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
	if (ret < 0)
		return -ENOMEM;
	reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
	return 0;
}
337

338 339 340 341 342 343
#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
int __init reserve_bootmem(unsigned long addr, unsigned long size,
			    int flags)
{
	bootmem_data_t *bdata;
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
344

345 346 347 348
	list_for_each_entry(bdata, &bdata_list, list) {
		ret = can_reserve_bootmem_core(bdata, addr, size, flags);
		if (ret < 0)
			return ret;
Linus Torvalds's avatar
Linus Torvalds committed
349
	}
350 351 352 353
	list_for_each_entry(bdata, &bdata_list, list)
		reserve_bootmem_core(bdata, addr, size, flags);

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
354
}
355
#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
Linus Torvalds's avatar
Linus Torvalds committed
356 357 358 359 360 361 362 363 364 365 366 367 368 369

/*
 * We 'merge' subsequent allocations to save space. We might 'lose'
 * some fraction of a page if allocations cannot be satisfied due to
 * size constraints on boxes where there is physical RAM space
 * fragmentation - in these cases (mostly large memory boxes) this
 * is not a problem.
 *
 * On low memory boxes we get it right in 100% of the cases.
 *
 * alignment has to be a power of 2 value.
 *
 * NOTE:  This function is _not_ reentrant.
 */
370 371 372
static void * __init
alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
		unsigned long align, unsigned long goal, unsigned long limit)
Linus Torvalds's avatar
Linus Torvalds committed
373
{
374
	unsigned long areasize, preferred;
375
	unsigned long i, start = 0, incr, eidx, end_pfn;
Linus Torvalds's avatar
Linus Torvalds committed
376
	void *ret;
377 378
	unsigned long node_boot_start;
	void *node_bootmem_map;
Linus Torvalds's avatar
Linus Torvalds committed
379

380
	if (!size) {
381
		printk("alloc_bootmem_core(): zero-sized request\n");
Linus Torvalds's avatar
Linus Torvalds committed
382 383 384 385
		BUG();
	}
	BUG_ON(align & (align-1));

386 387 388 389
	/* on nodes without memory - bootmem_map is NULL */
	if (!bdata->node_bootmem_map)
		return NULL;

390 391 392 393 394 395 396 397 398 399 400 401 402
	/* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */
	node_boot_start = bdata->node_boot_start;
	node_bootmem_map = bdata->node_bootmem_map;
	if (align) {
		node_boot_start = ALIGN(bdata->node_boot_start, align);
		if (node_boot_start > bdata->node_boot_start)
			node_bootmem_map = (unsigned long *)bdata->node_bootmem_map +
			    PFN_DOWN(node_boot_start - bdata->node_boot_start)/BITS_PER_LONG;
	}

	if (limit && node_boot_start >= limit)
		return NULL;

403 404
	end_pfn = bdata->node_low_pfn;
	limit = PFN_DOWN(limit);
405 406 407
	if (limit && end_pfn > limit)
		end_pfn = limit;

408
	eidx = end_pfn - PFN_DOWN(node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
409 410 411 412 413

	/*
	 * We try to allocate bootmem pages above 'goal'
	 * first, then we try to allocate lower pages.
	 */
414 415
	preferred = 0;
	if (goal && PFN_DOWN(goal) < end_pfn) {
416 417
		if (goal > node_boot_start)
			preferred = goal - node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
418

419 420
		if (bdata->last_success > node_boot_start &&
			bdata->last_success - node_boot_start >= preferred)
421
			if (!limit || (limit && limit > bdata->last_success))
422
				preferred = bdata->last_success - node_boot_start;
423
	}
Linus Torvalds's avatar
Linus Torvalds committed
424

425
	preferred = PFN_DOWN(ALIGN(preferred, align));
426
	areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
427 428 429
	incr = align >> PAGE_SHIFT ? : 1;

restart_scan:
430
	for (i = preferred; i < eidx;) {
Linus Torvalds's avatar
Linus Torvalds committed
431
		unsigned long j;
432

433
		i = find_next_zero_bit(node_bootmem_map, eidx, i);
Linus Torvalds's avatar
Linus Torvalds committed
434
		i = ALIGN(i, incr);
435 436
		if (i >= eidx)
			break;
437
		if (test_bit(i, node_bootmem_map)) {
438
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
439
			continue;
440
		}
Linus Torvalds's avatar
Linus Torvalds committed
441 442 443
		for (j = i + 1; j < i + areasize; ++j) {
			if (j >= eidx)
				goto fail_block;
444
			if (test_bit(j, node_bootmem_map))
Linus Torvalds's avatar
Linus Torvalds committed
445 446 447 448 449 450
				goto fail_block;
		}
		start = i;
		goto found;
	fail_block:
		i = ALIGN(j, incr);
451 452
		if (i == j)
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
453 454
	}

455 456
	if (preferred > 0) {
		preferred = 0;
Linus Torvalds's avatar
Linus Torvalds committed
457 458 459 460 461
		goto restart_scan;
	}
	return NULL;

found:
462
	bdata->last_success = PFN_PHYS(start) + node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
463 464 465 466 467 468 469 470 471
	BUG_ON(start >= eidx);

	/*
	 * Is the next page of the previous allocation-end the start
	 * of this allocation's buffer? If yes then we can 'merge'
	 * the previous partial page with this allocation.
	 */
	if (align < PAGE_SIZE &&
	    bdata->last_offset && bdata->last_pos+1 == start) {
472
		unsigned long offset, remaining_size;
473
		offset = ALIGN(bdata->last_offset, align);
Linus Torvalds's avatar
Linus Torvalds committed
474
		BUG_ON(offset > PAGE_SIZE);
475
		remaining_size = PAGE_SIZE - offset;
Linus Torvalds's avatar
Linus Torvalds committed
476 477 478
		if (size < remaining_size) {
			areasize = 0;
			/* last_pos unchanged */
479 480
			bdata->last_offset = offset + size;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
481
					   offset + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
482 483
		} else {
			remaining_size = size - remaining_size;
484 485
			areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
486
					   offset + node_boot_start);
487
			bdata->last_pos = start + areasize - 1;
Linus Torvalds's avatar
Linus Torvalds committed
488 489 490 491 492 493
			bdata->last_offset = remaining_size;
		}
		bdata->last_offset &= ~PAGE_MASK;
	} else {
		bdata->last_pos = start + areasize - 1;
		bdata->last_offset = size & ~PAGE_MASK;
494
		ret = phys_to_virt(start * PAGE_SIZE + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
495 496 497 498 499
	}

	/*
	 * Reserve the area now:
	 */
500
	for (i = start; i < start + areasize; i++)
501
		if (unlikely(test_and_set_bit(i, node_bootmem_map)))
Linus Torvalds's avatar
Linus Torvalds committed
502 503 504 505 506
			BUG();
	memset(ret, 0, size);
	return ret;
}

507 508
void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
				      unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
509
{
510
	bootmem_data_t *bdata;
Linus Torvalds's avatar
Linus Torvalds committed
511 512
	void *ptr;

513
	list_for_each_entry(bdata, &bdata_list, list) {
514
		ptr = alloc_bootmem_core(bdata, size, align, goal, 0);
515 516 517
		if (ptr)
			return ptr;
	}
518 519
	return NULL;
}
Linus Torvalds's avatar
Linus Torvalds committed
520

521 522
void * __init __alloc_bootmem(unsigned long size, unsigned long align,
			      unsigned long goal)
523 524
{
	void *mem = __alloc_bootmem_nopanic(size,align,goal);
525

526 527
	if (mem)
		return mem;
Linus Torvalds's avatar
Linus Torvalds committed
528 529 530 531 532 533 534 535
	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of memory");
	return NULL;
}

536 537
void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
				   unsigned long align, unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
538 539 540
{
	void *ptr;

541
	ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
Linus Torvalds's avatar
Linus Torvalds committed
542
	if (ptr)
543
		return ptr;
Linus Torvalds's avatar
Linus Torvalds committed
544

545
	return __alloc_bootmem(size, align, goal);
Linus Torvalds's avatar
Linus Torvalds committed
546 547
}

548 549 550 551 552 553 554 555 556 557 558 559
#ifdef CONFIG_SPARSEMEM
void * __init alloc_bootmem_section(unsigned long size,
				    unsigned long section_nr)
{
	void *ptr;
	unsigned long limit, goal, start_nr, end_nr, pfn;
	struct pglist_data *pgdat;

	pfn = section_nr_to_pfn(section_nr);
	goal = PFN_PHYS(pfn);
	limit = PFN_PHYS(section_nr_to_pfn(section_nr + 1)) - 1;
	pgdat = NODE_DATA(early_pfn_to_nid(pfn));
560 561
	ptr = alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal,
				limit);
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578

	if (!ptr)
		return NULL;

	start_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr)));
	end_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr) + size));
	if (start_nr != section_nr || end_nr != section_nr) {
		printk(KERN_WARNING "alloc_bootmem failed on section %ld.\n",
		       section_nr);
		free_bootmem_core(pgdat->bdata, __pa(ptr), size);
		ptr = NULL;
	}

	return ptr;
}
#endif

579 580 581 582 583 584 585 586 587 588 589 590
void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
				   unsigned long align, unsigned long goal)
{
	void *ptr;

	ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
	if (ptr)
		return ptr;

	return __alloc_bootmem_nopanic(size, align, goal);
}

591 592 593
#ifndef ARCH_LOW_ADDRESS_LIMIT
#define ARCH_LOW_ADDRESS_LIMIT	0xffffffffUL
#endif
594

595 596
void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
				  unsigned long goal)
597
{
598
	bootmem_data_t *bdata;
599 600
	void *ptr;

601
	list_for_each_entry(bdata, &bdata_list, list) {
602 603
		ptr = alloc_bootmem_core(bdata, size, align, goal,
					ARCH_LOW_ADDRESS_LIMIT);
604 605 606
		if (ptr)
			return ptr;
	}
607 608 609 610 611 612 613 614 615 616 617 618

	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "low bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of low memory");
	return NULL;
}

void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
				       unsigned long align, unsigned long goal)
{
619 620
	return alloc_bootmem_core(pgdat->bdata, size, align, goal,
				ARCH_LOW_ADDRESS_LIMIT);
621
}