zram_drv.c 19.9 KB
Newer Older
1
/*
2
 * Compressed RAM block device
3
 *
4
 * Copyright (C) 2008, 2009, 2010  Nitin Gupta
Minchan Kim's avatar
Minchan Kim committed
5
 *               2012, 2013 Minchan Kim
6 7 8 9 10 11 12 13 14
 *
 * This code is released using a dual license strategy: BSD/GPL
 * You can choose the licence that better fits your requirements.
 *
 * Released under the terms of 3-clause BSD License
 * Released under the terms of GNU General Public License Version 2.0
 *
 */

15
#define KMSG_COMPONENT "zram"
16 17
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

18 19 20 21
#ifdef CONFIG_ZRAM_DEBUG
#define DEBUG
#endif

22 23
#include <linux/module.h>
#include <linux/kernel.h>
24
#include <linux/bio.h>
25 26 27 28 29 30
#include <linux/bitops.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>
#include <linux/device.h>
#include <linux/genhd.h>
#include <linux/highmem.h>
31
#include <linux/slab.h>
32 33 34 35
#include <linux/lzo.h>
#include <linux/string.h>
#include <linux/vmalloc.h>

36
#include "zram_drv.h"
37 38

/* Globals */
39
static int zram_major;
40
static struct zram *zram_devices;
41 42

/* Module params (documentation at end) */
43
static unsigned int num_devices = 1;
44

45 46 47 48 49 50 51 52 53 54 55
#define ZRAM_ATTR_RO(name)						\
static ssize_t zram_attr_##name##_show(struct device *d,		\
				struct device_attribute *attr, char *b)	\
{									\
	struct zram *zram = dev_to_zram(d);				\
	return sprintf(b, "%llu\n",					\
		(u64)atomic64_read(&zram->stats.name));			\
}									\
static struct device_attribute dev_attr_##name =			\
	__ATTR(name, S_IRUGO, zram_attr_##name##_show, NULL);

56 57 58 59 60
static inline int init_done(struct zram *zram)
{
	return zram->meta != NULL;
}

61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
static inline struct zram *dev_to_zram(struct device *dev)
{
	return (struct zram *)dev_to_disk(dev)->private_data;
}

static ssize_t disksize_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct zram *zram = dev_to_zram(dev);

	return sprintf(buf, "%llu\n", zram->disksize);
}

static ssize_t initstate_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
77
	u32 val;
78 79
	struct zram *zram = dev_to_zram(dev);

80 81 82
	down_read(&zram->init_lock);
	val = init_done(zram);
	up_read(&zram->init_lock);
83

84
	return sprintf(buf, "%u\n", val);
85 86 87 88 89 90 91 92
}

static ssize_t orig_data_size_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct zram *zram = dev_to_zram(dev);

	return sprintf(buf, "%llu\n",
93
		(u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
94 95 96 97 98 99 100 101 102 103
}

static ssize_t mem_used_total_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val = 0;
	struct zram *zram = dev_to_zram(dev);
	struct zram_meta *meta = zram->meta;

	down_read(&zram->init_lock);
104
	if (init_done(zram))
105 106 107 108 109 110
		val = zs_get_total_size_bytes(meta->mem_pool);
	up_read(&zram->init_lock);

	return sprintf(buf, "%llu\n", val);
}

Minchan Kim's avatar
Minchan Kim committed
111
/* flag operations needs meta->tb_lock */
Minchan Kim's avatar
Minchan Kim committed
112
static int zram_test_flag(struct zram_meta *meta, u32 index,
113
			enum zram_pageflags flag)
114
{
Minchan Kim's avatar
Minchan Kim committed
115
	return meta->table[index].flags & BIT(flag);
116 117
}

Minchan Kim's avatar
Minchan Kim committed
118
static void zram_set_flag(struct zram_meta *meta, u32 index,
119
			enum zram_pageflags flag)
120
{
Minchan Kim's avatar
Minchan Kim committed
121
	meta->table[index].flags |= BIT(flag);
122 123
}

Minchan Kim's avatar
Minchan Kim committed
124
static void zram_clear_flag(struct zram_meta *meta, u32 index,
125
			enum zram_pageflags flag)
126
{
Minchan Kim's avatar
Minchan Kim committed
127
	meta->table[index].flags &= ~BIT(flag);
128 129
}

130 131 132 133 134 135 136 137 138 139 140
static inline int is_partial_io(struct bio_vec *bvec)
{
	return bvec->bv_len != PAGE_SIZE;
}

/*
 * Check if request is within bounds and aligned on zram logical blocks.
 */
static inline int valid_io_request(struct zram *zram, struct bio *bio)
{
	u64 start, end, bound;
141

142
	/* unaligned request */
143 144
	if (unlikely(bio->bi_iter.bi_sector &
		     (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
145
		return 0;
146
	if (unlikely(bio->bi_iter.bi_size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
147 148
		return 0;

149 150
	start = bio->bi_iter.bi_sector;
	end = start + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
151 152
	bound = zram->disksize >> SECTOR_SHIFT;
	/* out of range range */
153
	if (unlikely(start >= bound || end > bound || start > end))
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
		return 0;

	/* I/O request is valid */
	return 1;
}

static void zram_meta_free(struct zram_meta *meta)
{
	zs_destroy_pool(meta->mem_pool);
	kfree(meta->compress_workmem);
	free_pages((unsigned long)meta->compress_buffer, 1);
	vfree(meta->table);
	kfree(meta);
}

static struct zram_meta *zram_meta_alloc(u64 disksize)
{
	size_t num_pages;
	struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
	if (!meta)
		goto out;

	meta->compress_workmem = kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL);
	if (!meta->compress_workmem)
		goto free_meta;

	meta->compress_buffer =
		(void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
	if (!meta->compress_buffer) {
		pr_err("Error allocating compressor buffer space\n");
		goto free_workmem;
	}

	num_pages = disksize >> PAGE_SHIFT;
	meta->table = vzalloc(num_pages * sizeof(*meta->table));
	if (!meta->table) {
		pr_err("Error allocating zram address table\n");
		goto free_buffer;
	}

	meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM);
	if (!meta->mem_pool) {
		pr_err("Error creating memory pool\n");
		goto free_table;
	}

Minchan Kim's avatar
Minchan Kim committed
200
	rwlock_init(&meta->tb_lock);
201
	mutex_init(&meta->buffer_lock);
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
	return meta;

free_table:
	vfree(meta->table);
free_buffer:
	free_pages((unsigned long)meta->compress_buffer, 1);
free_workmem:
	kfree(meta->compress_workmem);
free_meta:
	kfree(meta);
	meta = NULL;
out:
	return meta;
}

static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
{
	if (*offset + bvec->bv_len >= PAGE_SIZE)
		(*index)++;
	*offset = (*offset + bvec->bv_len) % PAGE_SIZE;
}

224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
static int page_zero_filled(void *ptr)
{
	unsigned int pos;
	unsigned long *page;

	page = (unsigned long *)ptr;

	for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
		if (page[pos])
			return 0;
	}

	return 1;
}

239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
static void handle_zero_page(struct bio_vec *bvec)
{
	struct page *page = bvec->bv_page;
	void *user_mem;

	user_mem = kmap_atomic(page);
	if (is_partial_io(bvec))
		memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
	else
		clear_page(user_mem);
	kunmap_atomic(user_mem);

	flush_dcache_page(page);
}

Minchan Kim's avatar
Minchan Kim committed
254
/* NOTE: caller should hold meta->tb_lock with write-side */
255
static void zram_free_page(struct zram *zram, size_t index)
256
{
Minchan Kim's avatar
Minchan Kim committed
257 258
	struct zram_meta *meta = zram->meta;
	unsigned long handle = meta->table[index].handle;
259

260
	if (unlikely(!handle)) {
261 262 263 264
		/*
		 * No memory is allocated for zero filled pages.
		 * Simply clear zero page flag.
		 */
Minchan Kim's avatar
Minchan Kim committed
265 266
		if (zram_test_flag(meta, index, ZRAM_ZERO)) {
			zram_clear_flag(meta, index, ZRAM_ZERO);
267
			atomic64_dec(&zram->stats.zero_pages);
268 269 270 271
		}
		return;
	}

Minchan Kim's avatar
Minchan Kim committed
272
	zs_free(meta->mem_pool, handle);
273

274 275
	atomic64_sub(meta->table[index].size, &zram->stats.compr_data_size);
	atomic64_dec(&zram->stats.pages_stored);
276

Minchan Kim's avatar
Minchan Kim committed
277 278
	meta->table[index].handle = 0;
	meta->table[index].size = 0;
279 280
}

281
static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
282
{
283 284 285
	int ret = LZO_E_OK;
	size_t clen = PAGE_SIZE;
	unsigned char *cmem;
Minchan Kim's avatar
Minchan Kim committed
286
	struct zram_meta *meta = zram->meta;
Minchan Kim's avatar
Minchan Kim committed
287 288 289 290 291 292
	unsigned long handle;
	u16 size;

	read_lock(&meta->tb_lock);
	handle = meta->table[index].handle;
	size = meta->table[index].size;
293

Minchan Kim's avatar
Minchan Kim committed
294
	if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
Minchan Kim's avatar
Minchan Kim committed
295
		read_unlock(&meta->tb_lock);
296
		clear_page(mem);
297 298
		return 0;
	}
299

Minchan Kim's avatar
Minchan Kim committed
300
	cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
Minchan Kim's avatar
Minchan Kim committed
301
	if (size == PAGE_SIZE)
302
		copy_page(mem, cmem);
303
	else
Minchan Kim's avatar
Minchan Kim committed
304
		ret = lzo1x_decompress_safe(cmem, size,	mem, &clen);
Minchan Kim's avatar
Minchan Kim committed
305
	zs_unmap_object(meta->mem_pool, handle);
Minchan Kim's avatar
Minchan Kim committed
306
	read_unlock(&meta->tb_lock);
307

308 309 310
	/* Should NEVER happen. Return bio error if it does. */
	if (unlikely(ret != LZO_E_OK)) {
		pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
311
		atomic64_inc(&zram->stats.failed_reads);
312
		return ret;
313
	}
314

315
	return 0;
316 317
}

318 319
static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
			  u32 index, int offset, struct bio *bio)
320 321
{
	int ret;
322 323
	struct page *page;
	unsigned char *user_mem, *uncmem = NULL;
Minchan Kim's avatar
Minchan Kim committed
324
	struct zram_meta *meta = zram->meta;
325 326
	page = bvec->bv_page;

Minchan Kim's avatar
Minchan Kim committed
327
	read_lock(&meta->tb_lock);
Minchan Kim's avatar
Minchan Kim committed
328 329
	if (unlikely(!meta->table[index].handle) ||
			zram_test_flag(meta, index, ZRAM_ZERO)) {
Minchan Kim's avatar
Minchan Kim committed
330
		read_unlock(&meta->tb_lock);
331
		handle_zero_page(bvec);
332 333
		return 0;
	}
Minchan Kim's avatar
Minchan Kim committed
334
	read_unlock(&meta->tb_lock);
335

336 337
	if (is_partial_io(bvec))
		/* Use  a temporary buffer to decompress the page */
338 339 340 341
		uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);

	user_mem = kmap_atomic(page);
	if (!is_partial_io(bvec))
342 343 344 345 346 347 348
		uncmem = user_mem;

	if (!uncmem) {
		pr_info("Unable to allocate temp memory\n");
		ret = -ENOMEM;
		goto out_cleanup;
	}
349

350
	ret = zram_decompress_page(zram, uncmem, index);
351
	/* Should NEVER happen. Return bio error if it does. */
352
	if (unlikely(ret != LZO_E_OK))
353
		goto out_cleanup;
354

355 356 357 358 359 360 361 362 363 364 365
	if (is_partial_io(bvec))
		memcpy(user_mem + bvec->bv_offset, uncmem + offset,
				bvec->bv_len);

	flush_dcache_page(page);
	ret = 0;
out_cleanup:
	kunmap_atomic(user_mem);
	if (is_partial_io(bvec))
		kfree(uncmem);
	return ret;
366 367 368 369
}

static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
			   int offset)
370
{
371
	int ret = 0;
372
	size_t clen;
373
	unsigned long handle;
374
	struct page *page;
375
	unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
Minchan Kim's avatar
Minchan Kim committed
376
	struct zram_meta *meta = zram->meta;
377
	bool locked = false;
378

379
	page = bvec->bv_page;
Minchan Kim's avatar
Minchan Kim committed
380
	src = meta->compress_buffer;
381

382 383 384 385 386
	if (is_partial_io(bvec)) {
		/*
		 * This is a partial IO. We need to read the full page
		 * before to write the changes.
		 */
387
		uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
388 389 390 391
		if (!uncmem) {
			ret = -ENOMEM;
			goto out;
		}
392
		ret = zram_decompress_page(zram, uncmem, index);
393
		if (ret)
394 395 396
			goto out;
	}

397 398
	mutex_lock(&meta->buffer_lock);
	locked = true;
399
	user_mem = kmap_atomic(page);
400

401
	if (is_partial_io(bvec)) {
402 403
		memcpy(uncmem + offset, user_mem + bvec->bv_offset,
		       bvec->bv_len);
404 405 406
		kunmap_atomic(user_mem);
		user_mem = NULL;
	} else {
407
		uncmem = user_mem;
408
	}
409 410

	if (page_zero_filled(uncmem)) {
411
		kunmap_atomic(user_mem);
412
		/* Free memory associated with this sector now. */
Minchan Kim's avatar
Minchan Kim committed
413
		write_lock(&zram->meta->tb_lock);
414
		zram_free_page(zram, index);
Minchan Kim's avatar
Minchan Kim committed
415 416
		zram_set_flag(meta, index, ZRAM_ZERO);
		write_unlock(&zram->meta->tb_lock);
417

418
		atomic64_inc(&zram->stats.zero_pages);
419 420
		ret = 0;
		goto out;
421
	}
422

423
	ret = lzo1x_1_compress(uncmem, PAGE_SIZE, src, &clen,
Minchan Kim's avatar
Minchan Kim committed
424
			       meta->compress_workmem);
425 426 427 428 429
	if (!is_partial_io(bvec)) {
		kunmap_atomic(user_mem);
		user_mem = NULL;
		uncmem = NULL;
	}
430

431 432
	if (unlikely(ret != LZO_E_OK)) {
		pr_err("Compression failed! err=%d\n", ret);
433
		goto out;
434
	}
435

436 437
	if (unlikely(clen > max_zpage_size)) {
		clen = PAGE_SIZE;
438 439 440
		src = NULL;
		if (is_partial_io(bvec))
			src = uncmem;
441
	}
442

Minchan Kim's avatar
Minchan Kim committed
443
	handle = zs_malloc(meta->mem_pool, clen);
444
	if (!handle) {
445 446
		pr_info("Error allocating memory for compressed page: %u, size=%zu\n",
			index, clen);
447 448
		ret = -ENOMEM;
		goto out;
449
	}
Minchan Kim's avatar
Minchan Kim committed
450
	cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
451

452
	if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
453
		src = kmap_atomic(page);
454
		copy_page(cmem, src);
455
		kunmap_atomic(src);
456 457 458
	} else {
		memcpy(cmem, src, clen);
	}
459

Minchan Kim's avatar
Minchan Kim committed
460
	zs_unmap_object(meta->mem_pool, handle);
461

462 463 464 465
	/*
	 * Free memory associated with this sector
	 * before overwriting unused sectors.
	 */
Minchan Kim's avatar
Minchan Kim committed
466
	write_lock(&zram->meta->tb_lock);
467 468
	zram_free_page(zram, index);

Minchan Kim's avatar
Minchan Kim committed
469 470
	meta->table[index].handle = handle;
	meta->table[index].size = clen;
Minchan Kim's avatar
Minchan Kim committed
471
	write_unlock(&zram->meta->tb_lock);
472

473
	/* Update stats */
474 475
	atomic64_add(clen, &zram->stats.compr_data_size);
	atomic64_inc(&zram->stats.pages_stored);
476
out:
477 478
	if (locked)
		mutex_unlock(&meta->buffer_lock);
479 480 481
	if (is_partial_io(bvec))
		kfree(uncmem);

482
	if (ret)
483
		atomic64_inc(&zram->stats.failed_writes);
484
	return ret;
485 486 487
}

static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
488
			int offset, struct bio *bio)
489
{
490
	int ret;
491
	int rw = bio_data_dir(bio);
492

493 494
	if (rw == READ) {
		atomic64_inc(&zram->stats.num_reads);
495
		ret = zram_bvec_read(zram, bvec, index, offset, bio);
496 497
	} else {
		atomic64_inc(&zram->stats.num_writes);
498
		ret = zram_bvec_write(zram, bvec, index, offset);
499
	}
500 501

	return ret;
502 503
}

Minchan Kim's avatar
Minchan Kim committed
504
static void zram_reset_device(struct zram *zram, bool reset_capacity)
505
{
506 507 508
	size_t index;
	struct zram_meta *meta;

509
	down_write(&zram->init_lock);
510
	if (!init_done(zram)) {
511
		up_write(&zram->init_lock);
512
		return;
513
	}
514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530

	meta = zram->meta;
	/* Free all pages that are still in this zram device */
	for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) {
		unsigned long handle = meta->table[index].handle;
		if (!handle)
			continue;

		zs_free(meta->mem_pool, handle);
	}

	zram_meta_free(zram->meta);
	zram->meta = NULL;
	/* Reset stats */
	memset(&zram->stats, 0, sizeof(zram->stats));

	zram->disksize = 0;
Minchan Kim's avatar
Minchan Kim committed
531 532
	if (reset_capacity)
		set_capacity(zram->disk, 0);
533
	up_write(&zram->init_lock);
534 535 536 537 538 539 540 541 542 543 544 545 546 547 548
}

static ssize_t disksize_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	u64 disksize;
	struct zram_meta *meta;
	struct zram *zram = dev_to_zram(dev);

	disksize = memparse(buf, NULL);
	if (!disksize)
		return -EINVAL;

	disksize = PAGE_ALIGN(disksize);
	meta = zram_meta_alloc(disksize);
549 550
	if (!meta)
		return -ENOMEM;
551

552
	down_write(&zram->init_lock);
553
	if (init_done(zram)) {
554
		zram_meta_free(meta);
555
		up_write(&zram->init_lock);
556 557 558 559
		pr_info("Cannot change disksize for initialized device\n");
		return -EBUSY;
	}

560
	zram->meta = meta;
561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578
	zram->disksize = disksize;
	set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
	up_write(&zram->init_lock);

	return len;
}

static ssize_t reset_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int ret;
	unsigned short do_reset;
	struct zram *zram;
	struct block_device *bdev;

	zram = dev_to_zram(dev);
	bdev = bdget_disk(zram->disk, 0);

579 580 581
	if (!bdev)
		return -ENOMEM;

582
	/* Do not reset an active device! */
583 584 585 586
	if (bdev->bd_holders) {
		ret = -EBUSY;
		goto out;
	}
587 588 589

	ret = kstrtou16(buf, 10, &do_reset);
	if (ret)
590
		goto out;
591

592 593 594 595
	if (!do_reset) {
		ret = -EINVAL;
		goto out;
	}
596 597

	/* Make sure all pending I/O is finished */
598
	fsync_bdev(bdev);
599
	bdput(bdev);
600

Minchan Kim's avatar
Minchan Kim committed
601
	zram_reset_device(zram, true);
602
	return len;
603 604 605 606

out:
	bdput(bdev);
	return ret;
607 608
}

609
static void __zram_make_request(struct zram *zram, struct bio *bio)
610
{
611
	int offset;
612
	u32 index;
613 614
	struct bio_vec bvec;
	struct bvec_iter iter;
615

616 617 618
	index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
	offset = (bio->bi_iter.bi_sector &
		  (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
619

620
	bio_for_each_segment(bvec, bio, iter) {
621 622
		int max_transfer_size = PAGE_SIZE - offset;

623
		if (bvec.bv_len > max_transfer_size) {
624 625 626 627 628 629
			/*
			 * zram_bvec_rw() can only make operation on a single
			 * zram page. Split the bio vector.
			 */
			struct bio_vec bv;

630
			bv.bv_page = bvec.bv_page;
631
			bv.bv_len = max_transfer_size;
632
			bv.bv_offset = bvec.bv_offset;
633

634
			if (zram_bvec_rw(zram, &bv, index, offset, bio) < 0)
635 636
				goto out;

637
			bv.bv_len = bvec.bv_len - max_transfer_size;
638
			bv.bv_offset += max_transfer_size;
639
			if (zram_bvec_rw(zram, &bv, index + 1, 0, bio) < 0)
640 641
				goto out;
		} else
642
			if (zram_bvec_rw(zram, &bvec, index, offset, bio) < 0)
643 644
				goto out;

645
		update_position(&index, &offset, &bvec);
646
	}
647 648 649

	set_bit(BIO_UPTODATE, &bio->bi_flags);
	bio_endio(bio, 0);
650
	return;
651 652 653 654 655 656

out:
	bio_io_error(bio);
}

/*
657
 * Handler function for all zram I/O requests.
658
 */
659
static void zram_make_request(struct request_queue *queue, struct bio *bio)
660
{
661
	struct zram *zram = queue->queuedata;
662

663
	down_read(&zram->init_lock);
664
	if (unlikely(!init_done(zram)))
665
		goto error;
666

667
	if (!valid_io_request(zram, bio)) {
668
		atomic64_inc(&zram->stats.invalid_io);
669
		goto error;
670 671
	}

672
	__zram_make_request(zram, bio);
673
	up_read(&zram->init_lock);
674

675
	return;
676 677

error:
678
	up_read(&zram->init_lock);
679
	bio_io_error(bio);
680 681
}

Nitin Gupta's avatar
Nitin Gupta committed
682 683
static void zram_slot_free_notify(struct block_device *bdev,
				unsigned long index)
684
{
685
	struct zram *zram;
686
	struct zram_meta *meta;
687

688
	zram = bdev->bd_disk->private_data;
689
	meta = zram->meta;
690

691 692 693 694
	write_lock(&meta->tb_lock);
	zram_free_page(zram, index);
	write_unlock(&meta->tb_lock);
	atomic64_inc(&zram->stats.notify_free);
695 696
}

697 698
static const struct block_device_operations zram_devops = {
	.swap_slot_free_notify = zram_slot_free_notify,
699
	.owner = THIS_MODULE
700 701
};

702 703 704 705 706 707 708
static DEVICE_ATTR(disksize, S_IRUGO | S_IWUSR,
		disksize_show, disksize_store);
static DEVICE_ATTR(initstate, S_IRUGO, initstate_show, NULL);
static DEVICE_ATTR(reset, S_IWUSR, NULL, reset_store);
static DEVICE_ATTR(orig_data_size, S_IRUGO, orig_data_size_show, NULL);
static DEVICE_ATTR(mem_used_total, S_IRUGO, mem_used_total_show, NULL);

709 710
ZRAM_ATTR_RO(num_reads);
ZRAM_ATTR_RO(num_writes);
711 712
ZRAM_ATTR_RO(failed_reads);
ZRAM_ATTR_RO(failed_writes);
713 714 715 716 717
ZRAM_ATTR_RO(invalid_io);
ZRAM_ATTR_RO(notify_free);
ZRAM_ATTR_RO(zero_pages);
ZRAM_ATTR_RO(compr_data_size);

718 719 720 721 722 723
static struct attribute *zram_disk_attrs[] = {
	&dev_attr_disksize.attr,
	&dev_attr_initstate.attr,
	&dev_attr_reset.attr,
	&dev_attr_num_reads.attr,
	&dev_attr_num_writes.attr,
724 725
	&dev_attr_failed_reads.attr,
	&dev_attr_failed_writes.attr,
726 727 728 729 730 731 732 733 734 735 736 737 738
	&dev_attr_invalid_io.attr,
	&dev_attr_notify_free.attr,
	&dev_attr_zero_pages.attr,
	&dev_attr_orig_data_size.attr,
	&dev_attr_compr_data_size.attr,
	&dev_attr_mem_used_total.attr,
	NULL,
};

static struct attribute_group zram_disk_attr_group = {
	.attrs = zram_disk_attrs,
};

739
static int create_device(struct zram *zram, int device_id)
740
{
741
	int ret = -ENOMEM;
742

743
	init_rwsem(&zram->init_lock);
744

745 746
	zram->queue = blk_alloc_queue(GFP_KERNEL);
	if (!zram->queue) {
747 748
		pr_err("Error allocating disk queue for device %d\n",
			device_id);
749
		goto out;
750 751
	}

752 753
	blk_queue_make_request(zram->queue, zram_make_request);
	zram->queue->queuedata = zram;
754 755

	 /* gendisk structure */
756 757
	zram->disk = alloc_disk(1);
	if (!zram->disk) {
758
		pr_warn("Error allocating disk structure for device %d\n",
759
			device_id);
760
		goto out_free_queue;
761 762
	}

763 764 765 766 767 768
	zram->disk->major = zram_major;
	zram->disk->first_minor = device_id;
	zram->disk->fops = &zram_devops;
	zram->disk->queue = zram->queue;
	zram->disk->private_data = zram;
	snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
769

770
	/* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
771
	set_capacity(zram->disk, 0);
772 773
	/* zram devices sort of resembles non-rotational disks */
	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
774 775 776 777
	/*
	 * To ensure that we always get PAGE_SIZE aligned
	 * and n*PAGE_SIZED sized I/O requests.
	 */
778
	blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
779 780
	blk_queue_logical_block_size(zram->disk->queue,
					ZRAM_LOGICAL_BLOCK_SIZE);
781 782
	blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
	blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
783

784
	add_disk(zram->disk);
785

786 787 788
	ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj,
				&zram_disk_attr_group);
	if (ret < 0) {
789
		pr_warn("Error creating sysfs group");
790
		goto out_free_disk;
791 792
	}

793
	zram->meta = NULL;
794
	return 0;
795

796 797 798 799 800
out_free_disk:
	del_gendisk(zram->disk);
	put_disk(zram->disk);
out_free_queue:
	blk_cleanup_queue(zram->queue);
801 802
out:
	return ret;
803 804
}

805
static void destroy_device(struct zram *zram)
806
{
807 808 809
	sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
			&zram_disk_attr_group);

810 811
	del_gendisk(zram->disk);
	put_disk(zram->disk);
812

813
	blk_cleanup_queue(zram->queue);
814 815
}

816
static int __init zram_init(void)
817
{
818
	int ret, dev_id;
819

820
	if (num_devices > max_num_devices) {
821
		pr_warn("Invalid value for num_devices: %u\n",
822
				num_devices);
823 824
		ret = -EINVAL;
		goto out;
825 826
	}

827 828
	zram_major = register_blkdev(0, "zram");
	if (zram_major <= 0) {
829
		pr_warn("Unable to get major number\n");
830 831
		ret = -EBUSY;
		goto out;
832 833 834
	}

	/* Allocate the device array and initialize each one */
835
	zram_devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL);
836
	if (!zram_devices) {
837 838 839
		ret = -ENOMEM;
		goto unregister;
	}
840

841
	for (dev_id = 0; dev_id < num_devices; dev_id++) {
842
		ret = create_device(&zram_devices[dev_id], dev_id);
843
		if (ret)
844
			goto free_devices;
845 846
	}

847 848
	pr_info("Created %u device(s) ...\n", num_devices);

849
	return 0;
850

851
free_devices:
852
	while (dev_id)
853 854
		destroy_device(&zram_devices[--dev_id]);
	kfree(zram_devices);
855
unregister:
856
	unregister_blkdev(zram_major, "zram");
857
out:
858 859 860
	return ret;
}

861
static void __exit zram_exit(void)
862 863
{
	int i;
864
	struct zram *zram;
865

866
	for (i = 0; i < num_devices; i++) {
867
		zram = &zram_devices[i];
868

869
		destroy_device(zram);
Minchan Kim's avatar
Minchan Kim committed
870 871 872 873 874
		/*
		 * Shouldn't access zram->disk after destroy_device
		 * because destroy_device already released zram->disk.
		 */
		zram_reset_device(zram, false);
875 876
	}

877
	unregister_blkdev(zram_major, "zram");
878

879
	kfree(zram_devices);
880 881 882
	pr_debug("Cleanup done!\n");
}

883 884
module_init(zram_init);
module_exit(zram_exit);
885

886 887 888
module_param(num_devices, uint, 0);
MODULE_PARM_DESC(num_devices, "Number of zram devices");

889 890
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
891
MODULE_DESCRIPTION("Compressed RAM Block Device");