zram_drv.c 30.6 KB
Newer Older
1
/*
2
 * Compressed RAM block device
3
 *
4
 * Copyright (C) 2008, 2009, 2010  Nitin Gupta
Minchan Kim's avatar
Minchan Kim committed
5
 *               2012, 2013 Minchan Kim
6 7 8 9 10 11 12 13 14
 *
 * This code is released using a dual license strategy: BSD/GPL
 * You can choose the licence that better fits your requirements.
 *
 * Released under the terms of 3-clause BSD License
 * Released under the terms of GNU General Public License Version 2.0
 *
 */

15
#define KMSG_COMPONENT "zram"
16 17
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

18 19 20 21
#ifdef CONFIG_ZRAM_DEBUG
#define DEBUG
#endif

22 23
#include <linux/module.h>
#include <linux/kernel.h>
24
#include <linux/bio.h>
25 26 27 28 29 30
#include <linux/bitops.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>
#include <linux/device.h>
#include <linux/genhd.h>
#include <linux/highmem.h>
31
#include <linux/slab.h>
32 33
#include <linux/string.h>
#include <linux/vmalloc.h>
34
#include <linux/err.h>
35

36
#include "zram_drv.h"
37 38

/* Globals */
39
static int zram_major;
40
static struct zram *zram_devices;
41
static const char *default_compressor = "lzo";
42 43

/* Module params (documentation at end) */
44
static unsigned int num_devices = 1;
45

46 47 48 49 50 51 52 53 54
static inline void deprecated_attr_warn(const char *name)
{
	pr_warn_once("%d (%s) Attribute %s (and others) will be removed. %s\n",
			task_pid_nr(current),
			current->comm,
			name,
			"See zram documentation.");
}

55
#define ZRAM_ATTR_RO(name)						\
56
static ssize_t name##_show(struct device *d,		\
57 58 59
				struct device_attribute *attr, char *b)	\
{									\
	struct zram *zram = dev_to_zram(d);				\
60 61
									\
	deprecated_attr_warn(__stringify(name));			\
62
	return scnprintf(b, PAGE_SIZE, "%llu\n",			\
63 64
		(u64)atomic64_read(&zram->stats.name));			\
}									\
65
static DEVICE_ATTR_RO(name);
66

67
static inline bool init_done(struct zram *zram)
68
{
69
	return zram->disksize;
70 71
}

72 73 74 75 76 77 78 79 80 81
static inline struct zram *dev_to_zram(struct device *dev)
{
	return (struct zram *)dev_to_disk(dev)->private_data;
}

static ssize_t disksize_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct zram *zram = dev_to_zram(dev);

82
	return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
83 84 85 86 87
}

static ssize_t initstate_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
88
	u32 val;
89 90
	struct zram *zram = dev_to_zram(dev);

91 92 93
	down_read(&zram->init_lock);
	val = init_done(zram);
	up_read(&zram->init_lock);
94

95
	return scnprintf(buf, PAGE_SIZE, "%u\n", val);
96 97 98 99 100 101 102
}

static ssize_t orig_data_size_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct zram *zram = dev_to_zram(dev);

103
	deprecated_attr_warn("orig_data_size");
104
	return scnprintf(buf, PAGE_SIZE, "%llu\n",
105
		(u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
106 107 108 109 110 111 112 113
}

static ssize_t mem_used_total_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val = 0;
	struct zram *zram = dev_to_zram(dev);

114
	deprecated_attr_warn("mem_used_total");
115
	down_read(&zram->init_lock);
116 117
	if (init_done(zram)) {
		struct zram_meta *meta = zram->meta;
118
		val = zs_get_total_pages(meta->mem_pool);
119
	}
120 121
	up_read(&zram->init_lock);

122
	return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
123 124
}

125 126 127 128 129 130 131 132 133 134
static ssize_t max_comp_streams_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	int val;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
	val = zram->max_comp_streams;
	up_read(&zram->init_lock);

135
	return scnprintf(buf, PAGE_SIZE, "%d\n", val);
136 137
}

138 139 140 141 142 143
static ssize_t mem_limit_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val;
	struct zram *zram = dev_to_zram(dev);

144
	deprecated_attr_warn("mem_limit");
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169
	down_read(&zram->init_lock);
	val = zram->limit_pages;
	up_read(&zram->init_lock);

	return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
}

static ssize_t mem_limit_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	u64 limit;
	char *tmp;
	struct zram *zram = dev_to_zram(dev);

	limit = memparse(buf, &tmp);
	if (buf == tmp) /* no chars parsed, invalid input */
		return -EINVAL;

	down_write(&zram->init_lock);
	zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
	up_write(&zram->init_lock);

	return len;
}

Minchan Kim's avatar
Minchan Kim committed
170 171 172 173 174 175
static ssize_t mem_used_max_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val = 0;
	struct zram *zram = dev_to_zram(dev);

176
	deprecated_attr_warn("mem_used_max");
Minchan Kim's avatar
Minchan Kim committed
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
	down_read(&zram->init_lock);
	if (init_done(zram))
		val = atomic_long_read(&zram->stats.max_used_pages);
	up_read(&zram->init_lock);

	return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
}

static ssize_t mem_used_max_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int err;
	unsigned long val;
	struct zram *zram = dev_to_zram(dev);

	err = kstrtoul(buf, 10, &val);
	if (err || val != 0)
		return -EINVAL;

	down_read(&zram->init_lock);
197 198
	if (init_done(zram)) {
		struct zram_meta *meta = zram->meta;
Minchan Kim's avatar
Minchan Kim committed
199 200
		atomic_long_set(&zram->stats.max_used_pages,
				zs_get_total_pages(meta->mem_pool));
201
	}
Minchan Kim's avatar
Minchan Kim committed
202 203 204 205 206
	up_read(&zram->init_lock);

	return len;
}

207 208 209 210 211
static ssize_t max_comp_streams_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int num;
	struct zram *zram = dev_to_zram(dev);
Minchan Kim's avatar
Minchan Kim committed
212
	int ret;
213

Minchan Kim's avatar
Minchan Kim committed
214 215 216
	ret = kstrtoint(buf, 0, &num);
	if (ret < 0)
		return ret;
217 218
	if (num < 1)
		return -EINVAL;
Minchan Kim's avatar
Minchan Kim committed
219

220 221
	down_write(&zram->init_lock);
	if (init_done(zram)) {
Minchan Kim's avatar
Minchan Kim committed
222
		if (!zcomp_set_max_streams(zram->comp, num)) {
223
			pr_info("Cannot change max compression streams\n");
Minchan Kim's avatar
Minchan Kim committed
224 225 226
			ret = -EINVAL;
			goto out;
		}
227
	}
Minchan Kim's avatar
Minchan Kim committed
228

229
	zram->max_comp_streams = num;
Minchan Kim's avatar
Minchan Kim committed
230 231
	ret = len;
out:
232
	up_write(&zram->init_lock);
Minchan Kim's avatar
Minchan Kim committed
233
	return ret;
234 235
}

236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
static ssize_t comp_algorithm_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	size_t sz;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
	sz = zcomp_available_show(zram->compressor, buf);
	up_read(&zram->init_lock);

	return sz;
}

static ssize_t comp_algorithm_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	struct zram *zram = dev_to_zram(dev);
	down_write(&zram->init_lock);
	if (init_done(zram)) {
		up_write(&zram->init_lock);
		pr_info("Can't change algorithm for initialized device\n");
		return -EBUSY;
	}
	strlcpy(zram->compressor, buf, sizeof(zram->compressor));
	up_write(&zram->init_lock);
	return len;
}

Minchan Kim's avatar
Minchan Kim committed
264
/* flag operations needs meta->tb_lock */
Minchan Kim's avatar
Minchan Kim committed
265
static int zram_test_flag(struct zram_meta *meta, u32 index,
266
			enum zram_pageflags flag)
267
{
268
	return meta->table[index].value & BIT(flag);
269 270
}

Minchan Kim's avatar
Minchan Kim committed
271
static void zram_set_flag(struct zram_meta *meta, u32 index,
272
			enum zram_pageflags flag)
273
{
274
	meta->table[index].value |= BIT(flag);
275 276
}

Minchan Kim's avatar
Minchan Kim committed
277
static void zram_clear_flag(struct zram_meta *meta, u32 index,
278
			enum zram_pageflags flag)
279
{
280 281 282 283 284 285 286 287 288 289 290 291 292 293
	meta->table[index].value &= ~BIT(flag);
}

static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)
{
	return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);
}

static void zram_set_obj_size(struct zram_meta *meta,
					u32 index, size_t size)
{
	unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;

	meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;
294 295
}

296 297 298 299 300 301 302 303
static inline int is_partial_io(struct bio_vec *bvec)
{
	return bvec->bv_len != PAGE_SIZE;
}

/*
 * Check if request is within bounds and aligned on zram logical blocks.
 */
304 305
static inline int valid_io_request(struct zram *zram,
		sector_t start, unsigned int size)
306
{
307
	u64 end, bound;
308

309
	/* unaligned request */
310
	if (unlikely(start & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
311
		return 0;
312
	if (unlikely(size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
313 314
		return 0;

315
	end = start + (size >> SECTOR_SHIFT);
316 317
	bound = zram->disksize >> SECTOR_SHIFT;
	/* out of range range */
318
	if (unlikely(start >= bound || end > bound || start > end))
319 320 321 322 323 324
		return 0;

	/* I/O request is valid */
	return 1;
}

325
static void zram_meta_free(struct zram_meta *meta, u64 disksize)
326
{
327 328 329 330 331 332 333 334 335 336 337 338 339
	size_t num_pages = disksize >> PAGE_SHIFT;
	size_t index;

	/* Free all pages that are still in this zram device */
	for (index = 0; index < num_pages; index++) {
		unsigned long handle = meta->table[index].handle;

		if (!handle)
			continue;

		zs_free(meta->mem_pool, handle);
	}

340 341 342 343 344
	zs_destroy_pool(meta->mem_pool);
	vfree(meta->table);
	kfree(meta);
}

345
static struct zram_meta *zram_meta_alloc(int device_id, u64 disksize)
346 347
{
	size_t num_pages;
348
	char pool_name[8];
349
	struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
350

351
	if (!meta)
352
		return NULL;
353 354 355 356 357

	num_pages = disksize >> PAGE_SHIFT;
	meta->table = vzalloc(num_pages * sizeof(*meta->table));
	if (!meta->table) {
		pr_err("Error allocating zram address table\n");
358
		goto out_error;
359 360
	}

361 362
	snprintf(pool_name, sizeof(pool_name), "zram%d", device_id);
	meta->mem_pool = zs_create_pool(pool_name, GFP_NOIO | __GFP_HIGHMEM);
363 364
	if (!meta->mem_pool) {
		pr_err("Error creating memory pool\n");
365
		goto out_error;
366 367 368 369
	}

	return meta;

370
out_error:
371 372
	vfree(meta->table);
	kfree(meta);
373
	return NULL;
374 375
}

376 377 378 379 380 381 382 383 384 385 386 387
static inline bool zram_meta_get(struct zram *zram)
{
	if (atomic_inc_not_zero(&zram->refcount))
		return true;
	return false;
}

static inline void zram_meta_put(struct zram *zram)
{
	atomic_dec(&zram->refcount);
}

388 389 390 391 392 393 394
static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
{
	if (*offset + bvec->bv_len >= PAGE_SIZE)
		(*index)++;
	*offset = (*offset + bvec->bv_len) % PAGE_SIZE;
}

395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
static int page_zero_filled(void *ptr)
{
	unsigned int pos;
	unsigned long *page;

	page = (unsigned long *)ptr;

	for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
		if (page[pos])
			return 0;
	}

	return 1;
}

410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
static void handle_zero_page(struct bio_vec *bvec)
{
	struct page *page = bvec->bv_page;
	void *user_mem;

	user_mem = kmap_atomic(page);
	if (is_partial_io(bvec))
		memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
	else
		clear_page(user_mem);
	kunmap_atomic(user_mem);

	flush_dcache_page(page);
}

425 426 427 428 429 430

/*
 * To protect concurrent access to the same index entry,
 * caller should hold this table index entry's bit_spinlock to
 * indicate this index entry is accessing.
 */
431
static void zram_free_page(struct zram *zram, size_t index)
432
{
Minchan Kim's avatar
Minchan Kim committed
433 434
	struct zram_meta *meta = zram->meta;
	unsigned long handle = meta->table[index].handle;
435

436
	if (unlikely(!handle)) {
437 438 439 440
		/*
		 * No memory is allocated for zero filled pages.
		 * Simply clear zero page flag.
		 */
Minchan Kim's avatar
Minchan Kim committed
441 442
		if (zram_test_flag(meta, index, ZRAM_ZERO)) {
			zram_clear_flag(meta, index, ZRAM_ZERO);
443
			atomic64_dec(&zram->stats.zero_pages);
444 445 446 447
		}
		return;
	}

Minchan Kim's avatar
Minchan Kim committed
448
	zs_free(meta->mem_pool, handle);
449

450 451
	atomic64_sub(zram_get_obj_size(meta, index),
			&zram->stats.compr_data_size);
452
	atomic64_dec(&zram->stats.pages_stored);
453

Minchan Kim's avatar
Minchan Kim committed
454
	meta->table[index].handle = 0;
455
	zram_set_obj_size(meta, index, 0);
456 457
}

458
static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
459
{
460
	int ret = 0;
461
	unsigned char *cmem;
Minchan Kim's avatar
Minchan Kim committed
462
	struct zram_meta *meta = zram->meta;
Minchan Kim's avatar
Minchan Kim committed
463
	unsigned long handle;
Minchan Kim's avatar
Minchan Kim committed
464
	size_t size;
Minchan Kim's avatar
Minchan Kim committed
465

466
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
Minchan Kim's avatar
Minchan Kim committed
467
	handle = meta->table[index].handle;
468
	size = zram_get_obj_size(meta, index);
469

Minchan Kim's avatar
Minchan Kim committed
470
	if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
471
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
472
		clear_page(mem);
473 474
		return 0;
	}
475

Minchan Kim's avatar
Minchan Kim committed
476
	cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
Minchan Kim's avatar
Minchan Kim committed
477
	if (size == PAGE_SIZE)
478
		copy_page(mem, cmem);
479
	else
480
		ret = zcomp_decompress(zram->comp, cmem, size, mem);
Minchan Kim's avatar
Minchan Kim committed
481
	zs_unmap_object(meta->mem_pool, handle);
482
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
483

484
	/* Should NEVER happen. Return bio error if it does. */
485
	if (unlikely(ret)) {
486 487
		pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
		return ret;
488
	}
489

490
	return 0;
491 492
}

493
static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
494
			  u32 index, int offset)
495 496
{
	int ret;
497 498
	struct page *page;
	unsigned char *user_mem, *uncmem = NULL;
Minchan Kim's avatar
Minchan Kim committed
499
	struct zram_meta *meta = zram->meta;
500 501
	page = bvec->bv_page;

502
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
Minchan Kim's avatar
Minchan Kim committed
503 504
	if (unlikely(!meta->table[index].handle) ||
			zram_test_flag(meta, index, ZRAM_ZERO)) {
505
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
506
		handle_zero_page(bvec);
507 508
		return 0;
	}
509
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
510

511 512
	if (is_partial_io(bvec))
		/* Use  a temporary buffer to decompress the page */
513 514 515 516
		uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);

	user_mem = kmap_atomic(page);
	if (!is_partial_io(bvec))
517 518 519 520 521 522 523
		uncmem = user_mem;

	if (!uncmem) {
		pr_info("Unable to allocate temp memory\n");
		ret = -ENOMEM;
		goto out_cleanup;
	}
524

525
	ret = zram_decompress_page(zram, uncmem, index);
526
	/* Should NEVER happen. Return bio error if it does. */
527
	if (unlikely(ret))
528
		goto out_cleanup;
529

530 531 532 533 534 535 536 537 538 539 540
	if (is_partial_io(bvec))
		memcpy(user_mem + bvec->bv_offset, uncmem + offset,
				bvec->bv_len);

	flush_dcache_page(page);
	ret = 0;
out_cleanup:
	kunmap_atomic(user_mem);
	if (is_partial_io(bvec))
		kfree(uncmem);
	return ret;
541 542
}

Minchan Kim's avatar
Minchan Kim committed
543 544 545
static inline void update_used_max(struct zram *zram,
					const unsigned long pages)
{
546
	unsigned long old_max, cur_max;
Minchan Kim's avatar
Minchan Kim committed
547 548 549 550 551 552 553 554 555 556 557

	old_max = atomic_long_read(&zram->stats.max_used_pages);

	do {
		cur_max = old_max;
		if (pages > cur_max)
			old_max = atomic_long_cmpxchg(
				&zram->stats.max_used_pages, cur_max, pages);
	} while (old_max != cur_max);
}

558 559
static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
			   int offset)
560
{
561
	int ret = 0;
562
	size_t clen;
563
	unsigned long handle;
564
	struct page *page;
565
	unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
Minchan Kim's avatar
Minchan Kim committed
566
	struct zram_meta *meta = zram->meta;
567
	struct zcomp_strm *zstrm;
568
	bool locked = false;
Minchan Kim's avatar
Minchan Kim committed
569
	unsigned long alloced_pages;
570

571
	page = bvec->bv_page;
572 573 574 575 576
	if (is_partial_io(bvec)) {
		/*
		 * This is a partial IO. We need to read the full page
		 * before to write the changes.
		 */
577
		uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
578 579 580 581
		if (!uncmem) {
			ret = -ENOMEM;
			goto out;
		}
582
		ret = zram_decompress_page(zram, uncmem, index);
583
		if (ret)
584 585 586
			goto out;
	}

587
	zstrm = zcomp_strm_find(zram->comp);
588
	locked = true;
589
	user_mem = kmap_atomic(page);
590

591
	if (is_partial_io(bvec)) {
592 593
		memcpy(uncmem + offset, user_mem + bvec->bv_offset,
		       bvec->bv_len);
594 595 596
		kunmap_atomic(user_mem);
		user_mem = NULL;
	} else {
597
		uncmem = user_mem;
598
	}
599 600

	if (page_zero_filled(uncmem)) {
601 602
		if (user_mem)
			kunmap_atomic(user_mem);
603
		/* Free memory associated with this sector now. */
604
		bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
605
		zram_free_page(zram, index);
Minchan Kim's avatar
Minchan Kim committed
606
		zram_set_flag(meta, index, ZRAM_ZERO);
607
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
608

609
		atomic64_inc(&zram->stats.zero_pages);
610 611
		ret = 0;
		goto out;
612
	}
613

614
	ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen);
615 616 617 618 619
	if (!is_partial_io(bvec)) {
		kunmap_atomic(user_mem);
		user_mem = NULL;
		uncmem = NULL;
	}
620

621
	if (unlikely(ret)) {
622
		pr_err("Compression failed! err=%d\n", ret);
623
		goto out;
624
	}
625
	src = zstrm->buffer;
626 627
	if (unlikely(clen > max_zpage_size)) {
		clen = PAGE_SIZE;
628 629
		if (is_partial_io(bvec))
			src = uncmem;
630
	}
631

Minchan Kim's avatar
Minchan Kim committed
632
	handle = zs_malloc(meta->mem_pool, clen);
633
	if (!handle) {
634 635
		pr_info("Error allocating memory for compressed page: %u, size=%zu\n",
			index, clen);
636 637
		ret = -ENOMEM;
		goto out;
638
	}
639

Minchan Kim's avatar
Minchan Kim committed
640 641
	alloced_pages = zs_get_total_pages(meta->mem_pool);
	if (zram->limit_pages && alloced_pages > zram->limit_pages) {
642 643 644 645 646
		zs_free(meta->mem_pool, handle);
		ret = -ENOMEM;
		goto out;
	}

Minchan Kim's avatar
Minchan Kim committed
647 648
	update_used_max(zram, alloced_pages);

Minchan Kim's avatar
Minchan Kim committed
649
	cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
650

651
	if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
652
		src = kmap_atomic(page);
653
		copy_page(cmem, src);
654
		kunmap_atomic(src);
655 656 657
	} else {
		memcpy(cmem, src, clen);
	}
658

659 660
	zcomp_strm_release(zram->comp, zstrm);
	locked = false;
Minchan Kim's avatar
Minchan Kim committed
661
	zs_unmap_object(meta->mem_pool, handle);
662

663 664 665 666
	/*
	 * Free memory associated with this sector
	 * before overwriting unused sectors.
	 */
667
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
668 669
	zram_free_page(zram, index);

Minchan Kim's avatar
Minchan Kim committed
670
	meta->table[index].handle = handle;
671 672
	zram_set_obj_size(meta, index, clen);
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
673

674
	/* Update stats */
675 676
	atomic64_add(clen, &zram->stats.compr_data_size);
	atomic64_inc(&zram->stats.pages_stored);
677
out:
678
	if (locked)
679
		zcomp_strm_release(zram->comp, zstrm);
680 681
	if (is_partial_io(bvec))
		kfree(uncmem);
682
	return ret;
683 684 685
}

static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
686
			int offset, int rw)
687
{
688
	unsigned long start_time = jiffies;
689
	int ret;
690

691 692 693
	generic_start_io_acct(rw, bvec->bv_len >> SECTOR_SHIFT,
			&zram->disk->part0);

694 695
	if (rw == READ) {
		atomic64_inc(&zram->stats.num_reads);
696
		ret = zram_bvec_read(zram, bvec, index, offset);
697 698
	} else {
		atomic64_inc(&zram->stats.num_writes);
699
		ret = zram_bvec_write(zram, bvec, index, offset);
700
	}
701

702 703
	generic_end_io_acct(rw, &zram->disk->part0, start_time);

704 705 706 707 708 709 710
	if (unlikely(ret)) {
		if (rw == READ)
			atomic64_inc(&zram->stats.failed_reads);
		else
			atomic64_inc(&zram->stats.failed_writes);
	}

711
	return ret;
712 713
}

Joonsoo Kim's avatar
Joonsoo Kim committed
714 715 716 717 718 719 720 721 722
/*
 * zram_bio_discard - handler on discard request
 * @index: physical block index in PAGE_SIZE units
 * @offset: byte offset within physical block
 */
static void zram_bio_discard(struct zram *zram, u32 index,
			     int offset, struct bio *bio)
{
	size_t n = bio->bi_iter.bi_size;
723
	struct zram_meta *meta = zram->meta;
Joonsoo Kim's avatar
Joonsoo Kim committed
724 725 726 727 728 729 730 731 732 733 734 735

	/*
	 * zram manages data in physical block size units. Because logical block
	 * size isn't identical with physical block size on some arch, we
	 * could get a discard request pointing to a specific offset within a
	 * certain physical block.  Although we can handle this request by
	 * reading that physiclal block and decompressing and partially zeroing
	 * and re-compressing and then re-storing it, this isn't reasonable
	 * because our intent with a discard request is to save memory.  So
	 * skipping this logical block is appropriate here.
	 */
	if (offset) {
736
		if (n <= (PAGE_SIZE - offset))
Joonsoo Kim's avatar
Joonsoo Kim committed
737 738
			return;

739
		n -= (PAGE_SIZE - offset);
Joonsoo Kim's avatar
Joonsoo Kim committed
740 741 742 743
		index++;
	}

	while (n >= PAGE_SIZE) {
744
		bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
Joonsoo Kim's avatar
Joonsoo Kim committed
745
		zram_free_page(zram, index);
746
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
747
		atomic64_inc(&zram->stats.notify_free);
Joonsoo Kim's avatar
Joonsoo Kim committed
748 749 750 751 752
		index++;
		n -= PAGE_SIZE;
	}
}

753
static void zram_reset_device(struct zram *zram)
754
{
755 756 757 758
	struct zram_meta *meta;
	struct zcomp *comp;
	u64 disksize;

759
	down_write(&zram->init_lock);
760 761 762

	zram->limit_pages = 0;

763
	if (!init_done(zram)) {
764
		up_write(&zram->init_lock);
765
		return;
766
	}
767

768 769 770 771 772 773 774 775 776 777 778 779 780 781 782
	meta = zram->meta;
	comp = zram->comp;
	disksize = zram->disksize;
	/*
	 * Refcount will go down to 0 eventually and r/w handler
	 * cannot handle further I/O so it will bail out by
	 * check zram_meta_get.
	 */
	zram_meta_put(zram);
	/*
	 * We want to free zram_meta in process context to avoid
	 * deadlock between reclaim path and any other locks.
	 */
	wait_event(zram->io_done, atomic_read(&zram->refcount) == 0);

783 784 785
	/* Reset stats */
	memset(&zram->stats, 0, sizeof(zram->stats));
	zram->disksize = 0;
786
	zram->max_comp_streams = 1;
787 788
	set_capacity(zram->disk, 0);

789
	up_write(&zram->init_lock);
790 791 792
	/* I/O operation under all of CPU are done so let's free */
	zram_meta_free(meta, disksize);
	zcomp_destroy(comp);
793 794 795 796 797 798
}

static ssize_t disksize_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	u64 disksize;
799
	struct zcomp *comp;
800 801
	struct zram_meta *meta;
	struct zram *zram = dev_to_zram(dev);
802
	int err;
803 804 805 806 807 808

	disksize = memparse(buf, NULL);
	if (!disksize)
		return -EINVAL;

	disksize = PAGE_ALIGN(disksize);
809
	meta = zram_meta_alloc(zram->disk->first_minor, disksize);
810 811
	if (!meta)
		return -ENOMEM;
812

813
	comp = zcomp_create(zram->compressor, zram->max_comp_streams);
814
	if (IS_ERR(comp)) {
815 816
		pr_info("Cannot initialise %s compressing backend\n",
				zram->compressor);
817 818
		err = PTR_ERR(comp);
		goto out_free_meta;
819 820
	}

821
	down_write(&zram->init_lock);
822
	if (init_done(zram)) {
823
		pr_info("Cannot change disksize for initialized device\n");
824
		err = -EBUSY;
825
		goto out_destroy_comp;
826 827
	}

828 829
	init_waitqueue_head(&zram->io_done);
	atomic_set(&zram->refcount, 1);
830
	zram->meta = meta;
831
	zram->comp = comp;
832 833 834
	zram->disksize = disksize;
	set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
	up_write(&zram->init_lock);
835 836 837 838 839 840 841 842

	/*
	 * Revalidate disk out of the init_lock to avoid lockdep splat.
	 * It's okay because disk's capacity is protected by init_lock
	 * so that revalidate_disk always sees up-to-date capacity.
	 */
	revalidate_disk(zram->disk);

843
	return len;
844

845 846 847 848
out_destroy_comp:
	up_write(&zram->init_lock);
	zcomp_destroy(comp);
out_free_meta:
849
	zram_meta_free(meta, disksize);
850
	return err;
851 852 853 854 855 856 857 858 859 860 861 862 863
}

static ssize_t reset_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int ret;
	unsigned short do_reset;
	struct zram *zram;
	struct block_device *bdev;

	zram = dev_to_zram(dev);
	bdev = bdget_disk(zram->disk, 0);

864 865 866
	if (!bdev)
		return -ENOMEM;

867
	mutex_lock(&bdev->bd_mutex);
868
	/* Do not reset an active device! */
869
	if (bdev->bd_openers) {
870 871 872
		ret = -EBUSY;
		goto out;
	}
873 874 875

	ret = kstrtou16(buf, 10, &do_reset);
	if (ret)
876
		goto out;
877

878 879 880 881
	if (!do_reset) {
		ret = -EINVAL;
		goto out;
	}
882 883

	/* Make sure all pending I/O is finished */
884
	fsync_bdev(bdev);
885 886 887 888
	zram_reset_device(zram);

	mutex_unlock(&bdev->bd_mutex);
	revalidate_disk(zram->disk);
889
	bdput(bdev);
890 891

	return len;
892 893

out:
894
	mutex_unlock(&bdev->bd_mutex);
895 896
	bdput(bdev);
	return ret;
897 898
}

899
static void __zram_make_request(struct zram *zram, struct bio *bio)
900
{
901
	int offset, rw;
902
	u32 index;
903 904
	struct bio_vec bvec;
	struct bvec_iter iter;
905

906 907 908
	index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
	offset = (bio->bi_iter.bi_sector &
		  (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
909

Joonsoo Kim's avatar
Joonsoo Kim committed
910 911 912 913 914 915
	if (unlikely(bio->bi_rw & REQ_DISCARD)) {
		zram_bio_discard(zram, index, offset, bio);
		bio_endio(bio, 0);
		return;
	}

916
	rw = bio_data_dir(bio);
917
	bio_for_each_segment(bvec, bio, iter) {
918 919
		int max_transfer_size = PAGE_SIZE - offset;

920
		if (bvec.bv_len > max_transfer_size) {
921 922 923 924 925 926
			/*
			 * zram_bvec_rw() can only make operation on a single
			 * zram page. Split the bio vector.
			 */
			struct bio_vec bv;

927
			bv.bv_page = bvec.bv_page;
928
			bv.bv_len = max_transfer_size;
929
			bv.bv_offset = bvec.bv_offset;
930

931