zram_drv.c 31.3 KB
Newer Older
1
/*
2
 * Compressed RAM block device
3
 *
4
 * Copyright (C) 2008, 2009, 2010  Nitin Gupta
Minchan Kim's avatar
Minchan Kim committed
5
 *               2012, 2013 Minchan Kim
6 7 8 9 10 11 12 13 14
 *
 * This code is released using a dual license strategy: BSD/GPL
 * You can choose the licence that better fits your requirements.
 *
 * Released under the terms of 3-clause BSD License
 * Released under the terms of GNU General Public License Version 2.0
 *
 */

15
#define KMSG_COMPONENT "zram"
16 17 18 19
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/module.h>
#include <linux/kernel.h>
20
#include <linux/bio.h>
21 22 23 24 25 26
#include <linux/bitops.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>
#include <linux/device.h>
#include <linux/genhd.h>
#include <linux/highmem.h>
27
#include <linux/slab.h>
28 29
#include <linux/string.h>
#include <linux/vmalloc.h>
30
#include <linux/err.h>
31
#include <linux/idr.h>
32

33
#include "zram_drv.h"
34

35
static DEFINE_IDR(zram_index_idr);
36
static int zram_major;
37
static const char *default_compressor = "lzo";
38 39

/* Module params (documentation at end) */
40
static unsigned int num_devices = 1;
41

42 43 44 45 46 47 48 49 50
static inline void deprecated_attr_warn(const char *name)
{
	pr_warn_once("%d (%s) Attribute %s (and others) will be removed. %s\n",
			task_pid_nr(current),
			current->comm,
			name,
			"See zram documentation.");
}

51
#define ZRAM_ATTR_RO(name)						\
52
static ssize_t name##_show(struct device *d,				\
53 54 55
				struct device_attribute *attr, char *b)	\
{									\
	struct zram *zram = dev_to_zram(d);				\
56 57
									\
	deprecated_attr_warn(__stringify(name));			\
58
	return scnprintf(b, PAGE_SIZE, "%llu\n",			\
59 60
		(u64)atomic64_read(&zram->stats.name));			\
}									\
61
static DEVICE_ATTR_RO(name);
62

63
static inline bool init_done(struct zram *zram)
64
{
65
	return zram->disksize;
66 67
}

68 69 70 71 72
static inline struct zram *dev_to_zram(struct device *dev)
{
	return (struct zram *)dev_to_disk(dev)->private_data;
}

73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
static ssize_t compact_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	unsigned long nr_migrated;
	struct zram *zram = dev_to_zram(dev);
	struct zram_meta *meta;

	down_read(&zram->init_lock);
	if (!init_done(zram)) {
		up_read(&zram->init_lock);
		return -EINVAL;
	}

	meta = zram->meta;
	nr_migrated = zs_compact(meta->mem_pool);
	atomic64_add(nr_migrated, &zram->stats.num_migrated);
	up_read(&zram->init_lock);

	return len;
}

94 95 96 97 98
static ssize_t disksize_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct zram *zram = dev_to_zram(dev);

99
	return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
100 101 102 103 104
}

static ssize_t initstate_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
105
	u32 val;
106 107
	struct zram *zram = dev_to_zram(dev);

108 109 110
	down_read(&zram->init_lock);
	val = init_done(zram);
	up_read(&zram->init_lock);
111

112
	return scnprintf(buf, PAGE_SIZE, "%u\n", val);
113 114 115 116 117 118 119
}

static ssize_t orig_data_size_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct zram *zram = dev_to_zram(dev);

120
	deprecated_attr_warn("orig_data_size");
121
	return scnprintf(buf, PAGE_SIZE, "%llu\n",
122
		(u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
123 124 125 126 127 128 129 130
}

static ssize_t mem_used_total_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val = 0;
	struct zram *zram = dev_to_zram(dev);

131
	deprecated_attr_warn("mem_used_total");
132
	down_read(&zram->init_lock);
133 134
	if (init_done(zram)) {
		struct zram_meta *meta = zram->meta;
135
		val = zs_get_total_pages(meta->mem_pool);
136
	}
137 138
	up_read(&zram->init_lock);

139
	return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
140 141
}

142 143 144 145 146 147 148 149 150 151
static ssize_t max_comp_streams_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	int val;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
	val = zram->max_comp_streams;
	up_read(&zram->init_lock);

152
	return scnprintf(buf, PAGE_SIZE, "%d\n", val);
153 154
}

155 156 157 158 159 160
static ssize_t mem_limit_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val;
	struct zram *zram = dev_to_zram(dev);

161
	deprecated_attr_warn("mem_limit");
162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
	down_read(&zram->init_lock);
	val = zram->limit_pages;
	up_read(&zram->init_lock);

	return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
}

static ssize_t mem_limit_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	u64 limit;
	char *tmp;
	struct zram *zram = dev_to_zram(dev);

	limit = memparse(buf, &tmp);
	if (buf == tmp) /* no chars parsed, invalid input */
		return -EINVAL;

	down_write(&zram->init_lock);
	zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
	up_write(&zram->init_lock);

	return len;
}

Minchan Kim's avatar
Minchan Kim committed
187 188 189 190 191 192
static ssize_t mem_used_max_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val = 0;
	struct zram *zram = dev_to_zram(dev);

193
	deprecated_attr_warn("mem_used_max");
Minchan Kim's avatar
Minchan Kim committed
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
	down_read(&zram->init_lock);
	if (init_done(zram))
		val = atomic_long_read(&zram->stats.max_used_pages);
	up_read(&zram->init_lock);

	return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
}

static ssize_t mem_used_max_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int err;
	unsigned long val;
	struct zram *zram = dev_to_zram(dev);

	err = kstrtoul(buf, 10, &val);
	if (err || val != 0)
		return -EINVAL;

	down_read(&zram->init_lock);
214 215
	if (init_done(zram)) {
		struct zram_meta *meta = zram->meta;
Minchan Kim's avatar
Minchan Kim committed
216 217
		atomic_long_set(&zram->stats.max_used_pages,
				zs_get_total_pages(meta->mem_pool));
218
	}
Minchan Kim's avatar
Minchan Kim committed
219 220 221 222 223
	up_read(&zram->init_lock);

	return len;
}

224 225 226 227 228
static ssize_t max_comp_streams_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int num;
	struct zram *zram = dev_to_zram(dev);
Minchan Kim's avatar
Minchan Kim committed
229
	int ret;
230

Minchan Kim's avatar
Minchan Kim committed
231 232 233
	ret = kstrtoint(buf, 0, &num);
	if (ret < 0)
		return ret;
234 235
	if (num < 1)
		return -EINVAL;
Minchan Kim's avatar
Minchan Kim committed
236

237 238
	down_write(&zram->init_lock);
	if (init_done(zram)) {
Minchan Kim's avatar
Minchan Kim committed
239
		if (!zcomp_set_max_streams(zram->comp, num)) {
240
			pr_info("Cannot change max compression streams\n");
Minchan Kim's avatar
Minchan Kim committed
241 242 243
			ret = -EINVAL;
			goto out;
		}
244
	}
Minchan Kim's avatar
Minchan Kim committed
245

246
	zram->max_comp_streams = num;
Minchan Kim's avatar
Minchan Kim committed
247 248
	ret = len;
out:
249
	up_write(&zram->init_lock);
Minchan Kim's avatar
Minchan Kim committed
250
	return ret;
251 252
}

253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
static ssize_t comp_algorithm_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	size_t sz;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
	sz = zcomp_available_show(zram->compressor, buf);
	up_read(&zram->init_lock);

	return sz;
}

static ssize_t comp_algorithm_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	struct zram *zram = dev_to_zram(dev);
	down_write(&zram->init_lock);
	if (init_done(zram)) {
		up_write(&zram->init_lock);
		pr_info("Can't change algorithm for initialized device\n");
		return -EBUSY;
	}
	strlcpy(zram->compressor, buf, sizeof(zram->compressor));
	up_write(&zram->init_lock);
	return len;
}

Minchan Kim's avatar
Minchan Kim committed
281
/* flag operations needs meta->tb_lock */
Minchan Kim's avatar
Minchan Kim committed
282
static int zram_test_flag(struct zram_meta *meta, u32 index,
283
			enum zram_pageflags flag)
284
{
285
	return meta->table[index].value & BIT(flag);
286 287
}

Minchan Kim's avatar
Minchan Kim committed
288
static void zram_set_flag(struct zram_meta *meta, u32 index,
289
			enum zram_pageflags flag)
290
{
291
	meta->table[index].value |= BIT(flag);
292 293
}

Minchan Kim's avatar
Minchan Kim committed
294
static void zram_clear_flag(struct zram_meta *meta, u32 index,
295
			enum zram_pageflags flag)
296
{
297 298 299 300 301 302 303 304 305 306 307 308 309 310
	meta->table[index].value &= ~BIT(flag);
}

static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)
{
	return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);
}

static void zram_set_obj_size(struct zram_meta *meta,
					u32 index, size_t size)
{
	unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;

	meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;
311 312
}

313 314 315 316 317 318 319 320
static inline int is_partial_io(struct bio_vec *bvec)
{
	return bvec->bv_len != PAGE_SIZE;
}

/*
 * Check if request is within bounds and aligned on zram logical blocks.
 */
321 322
static inline int valid_io_request(struct zram *zram,
		sector_t start, unsigned int size)
323
{
324
	u64 end, bound;
325

326
	/* unaligned request */
327
	if (unlikely(start & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
328
		return 0;
329
	if (unlikely(size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
330 331
		return 0;

332
	end = start + (size >> SECTOR_SHIFT);
333 334
	bound = zram->disksize >> SECTOR_SHIFT;
	/* out of range range */
335
	if (unlikely(start >= bound || end > bound || start > end))
336 337 338 339 340 341
		return 0;

	/* I/O request is valid */
	return 1;
}

342
static void zram_meta_free(struct zram_meta *meta, u64 disksize)
343
{
344 345 346 347 348 349 350 351 352 353 354 355 356
	size_t num_pages = disksize >> PAGE_SHIFT;
	size_t index;

	/* Free all pages that are still in this zram device */
	for (index = 0; index < num_pages; index++) {
		unsigned long handle = meta->table[index].handle;

		if (!handle)
			continue;

		zs_free(meta->mem_pool, handle);
	}

357 358 359 360 361
	zs_destroy_pool(meta->mem_pool);
	vfree(meta->table);
	kfree(meta);
}

362
static struct zram_meta *zram_meta_alloc(int device_id, u64 disksize)
363 364
{
	size_t num_pages;
365
	char pool_name[8];
366
	struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
367

368
	if (!meta)
369
		return NULL;
370 371 372 373 374

	num_pages = disksize >> PAGE_SHIFT;
	meta->table = vzalloc(num_pages * sizeof(*meta->table));
	if (!meta->table) {
		pr_err("Error allocating zram address table\n");
375
		goto out_error;
376 377
	}

378 379
	snprintf(pool_name, sizeof(pool_name), "zram%d", device_id);
	meta->mem_pool = zs_create_pool(pool_name, GFP_NOIO | __GFP_HIGHMEM);
380 381
	if (!meta->mem_pool) {
		pr_err("Error creating memory pool\n");
382
		goto out_error;
383 384 385 386
	}

	return meta;

387
out_error:
388 389
	vfree(meta->table);
	kfree(meta);
390
	return NULL;
391 392
}

393 394 395 396 397 398 399 400 401 402 403 404
static inline bool zram_meta_get(struct zram *zram)
{
	if (atomic_inc_not_zero(&zram->refcount))
		return true;
	return false;
}

static inline void zram_meta_put(struct zram *zram)
{
	atomic_dec(&zram->refcount);
}

405 406 407 408 409 410 411
static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
{
	if (*offset + bvec->bv_len >= PAGE_SIZE)
		(*index)++;
	*offset = (*offset + bvec->bv_len) % PAGE_SIZE;
}

412 413 414 415 416 417 418 419 420 421 422 423 424 425 426
static int page_zero_filled(void *ptr)
{
	unsigned int pos;
	unsigned long *page;

	page = (unsigned long *)ptr;

	for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
		if (page[pos])
			return 0;
	}

	return 1;
}

427 428 429 430 431 432 433 434 435 436 437 438 439 440 441
static void handle_zero_page(struct bio_vec *bvec)
{
	struct page *page = bvec->bv_page;
	void *user_mem;

	user_mem = kmap_atomic(page);
	if (is_partial_io(bvec))
		memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
	else
		clear_page(user_mem);
	kunmap_atomic(user_mem);

	flush_dcache_page(page);
}

442 443 444 445 446 447

/*
 * To protect concurrent access to the same index entry,
 * caller should hold this table index entry's bit_spinlock to
 * indicate this index entry is accessing.
 */
448
static void zram_free_page(struct zram *zram, size_t index)
449
{
Minchan Kim's avatar
Minchan Kim committed
450 451
	struct zram_meta *meta = zram->meta;
	unsigned long handle = meta->table[index].handle;
452

453
	if (unlikely(!handle)) {
454 455 456 457
		/*
		 * No memory is allocated for zero filled pages.
		 * Simply clear zero page flag.
		 */
Minchan Kim's avatar
Minchan Kim committed
458 459
		if (zram_test_flag(meta, index, ZRAM_ZERO)) {
			zram_clear_flag(meta, index, ZRAM_ZERO);
460
			atomic64_dec(&zram->stats.zero_pages);
461 462 463 464
		}
		return;
	}

Minchan Kim's avatar
Minchan Kim committed
465
	zs_free(meta->mem_pool, handle);
466

467 468
	atomic64_sub(zram_get_obj_size(meta, index),
			&zram->stats.compr_data_size);
469
	atomic64_dec(&zram->stats.pages_stored);
470

Minchan Kim's avatar
Minchan Kim committed
471
	meta->table[index].handle = 0;
472
	zram_set_obj_size(meta, index, 0);
473 474
}

475
static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
476
{
477
	int ret = 0;
478
	unsigned char *cmem;
Minchan Kim's avatar
Minchan Kim committed
479
	struct zram_meta *meta = zram->meta;
Minchan Kim's avatar
Minchan Kim committed
480
	unsigned long handle;
Minchan Kim's avatar
Minchan Kim committed
481
	size_t size;
Minchan Kim's avatar
Minchan Kim committed
482

483
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
Minchan Kim's avatar
Minchan Kim committed
484
	handle = meta->table[index].handle;
485
	size = zram_get_obj_size(meta, index);
486

Minchan Kim's avatar
Minchan Kim committed
487
	if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
488
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
489
		clear_page(mem);
490 491
		return 0;
	}
492

Minchan Kim's avatar
Minchan Kim committed
493
	cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
Minchan Kim's avatar
Minchan Kim committed
494
	if (size == PAGE_SIZE)
495
		copy_page(mem, cmem);
496
	else
497
		ret = zcomp_decompress(zram->comp, cmem, size, mem);
Minchan Kim's avatar
Minchan Kim committed
498
	zs_unmap_object(meta->mem_pool, handle);
499
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
500

501
	/* Should NEVER happen. Return bio error if it does. */
502
	if (unlikely(ret)) {
503 504
		pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
		return ret;
505
	}
506

507
	return 0;
508 509
}

510
static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
511
			  u32 index, int offset)
512 513
{
	int ret;
514 515
	struct page *page;
	unsigned char *user_mem, *uncmem = NULL;
Minchan Kim's avatar
Minchan Kim committed
516
	struct zram_meta *meta = zram->meta;
517 518
	page = bvec->bv_page;

519
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
Minchan Kim's avatar
Minchan Kim committed
520 521
	if (unlikely(!meta->table[index].handle) ||
			zram_test_flag(meta, index, ZRAM_ZERO)) {
522
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
523
		handle_zero_page(bvec);
524 525
		return 0;
	}
526
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
527

528 529
	if (is_partial_io(bvec))
		/* Use  a temporary buffer to decompress the page */
530 531 532 533
		uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);

	user_mem = kmap_atomic(page);
	if (!is_partial_io(bvec))
534 535 536 537 538 539 540
		uncmem = user_mem;

	if (!uncmem) {
		pr_info("Unable to allocate temp memory\n");
		ret = -ENOMEM;
		goto out_cleanup;
	}
541

542
	ret = zram_decompress_page(zram, uncmem, index);
543
	/* Should NEVER happen. Return bio error if it does. */
544
	if (unlikely(ret))
545
		goto out_cleanup;
546

547 548 549 550 551 552 553 554 555 556 557
	if (is_partial_io(bvec))
		memcpy(user_mem + bvec->bv_offset, uncmem + offset,
				bvec->bv_len);

	flush_dcache_page(page);
	ret = 0;
out_cleanup:
	kunmap_atomic(user_mem);
	if (is_partial_io(bvec))
		kfree(uncmem);
	return ret;
558 559
}

Minchan Kim's avatar
Minchan Kim committed
560 561 562
static inline void update_used_max(struct zram *zram,
					const unsigned long pages)
{
563
	unsigned long old_max, cur_max;
Minchan Kim's avatar
Minchan Kim committed
564 565 566 567 568 569 570 571 572 573 574

	old_max = atomic_long_read(&zram->stats.max_used_pages);

	do {
		cur_max = old_max;
		if (pages > cur_max)
			old_max = atomic_long_cmpxchg(
				&zram->stats.max_used_pages, cur_max, pages);
	} while (old_max != cur_max);
}

575 576
static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
			   int offset)
577
{
578
	int ret = 0;
579
	size_t clen;
580
	unsigned long handle;
581
	struct page *page;
582
	unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
Minchan Kim's avatar
Minchan Kim committed
583
	struct zram_meta *meta = zram->meta;
584
	struct zcomp_strm *zstrm;
585
	bool locked = false;
Minchan Kim's avatar
Minchan Kim committed
586
	unsigned long alloced_pages;
587

588
	page = bvec->bv_page;
589 590 591 592 593
	if (is_partial_io(bvec)) {
		/*
		 * This is a partial IO. We need to read the full page
		 * before to write the changes.
		 */
594
		uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
595 596 597 598
		if (!uncmem) {
			ret = -ENOMEM;
			goto out;
		}
599
		ret = zram_decompress_page(zram, uncmem, index);
600
		if (ret)
601 602 603
			goto out;
	}

604
	zstrm = zcomp_strm_find(zram->comp);
605
	locked = true;
606
	user_mem = kmap_atomic(page);
607

608
	if (is_partial_io(bvec)) {
609 610
		memcpy(uncmem + offset, user_mem + bvec->bv_offset,
		       bvec->bv_len);
611 612 613
		kunmap_atomic(user_mem);
		user_mem = NULL;
	} else {
614
		uncmem = user_mem;
615
	}
616 617

	if (page_zero_filled(uncmem)) {
618 619
		if (user_mem)
			kunmap_atomic(user_mem);
620
		/* Free memory associated with this sector now. */
621
		bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
622
		zram_free_page(zram, index);
Minchan Kim's avatar
Minchan Kim committed
623
		zram_set_flag(meta, index, ZRAM_ZERO);
624
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
625

626
		atomic64_inc(&zram->stats.zero_pages);
627 628
		ret = 0;
		goto out;
629
	}
630

631
	ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen);
632 633 634 635 636
	if (!is_partial_io(bvec)) {
		kunmap_atomic(user_mem);
		user_mem = NULL;
		uncmem = NULL;
	}
637

638
	if (unlikely(ret)) {
639
		pr_err("Compression failed! err=%d\n", ret);
640
		goto out;
641
	}
642
	src = zstrm->buffer;
643 644
	if (unlikely(clen > max_zpage_size)) {
		clen = PAGE_SIZE;
645 646
		if (is_partial_io(bvec))
			src = uncmem;
647
	}
648

Minchan Kim's avatar
Minchan Kim committed
649
	handle = zs_malloc(meta->mem_pool, clen);
650
	if (!handle) {
651 652
		pr_info("Error allocating memory for compressed page: %u, size=%zu\n",
			index, clen);
653 654
		ret = -ENOMEM;
		goto out;
655
	}
656

Minchan Kim's avatar
Minchan Kim committed
657 658
	alloced_pages = zs_get_total_pages(meta->mem_pool);
	if (zram->limit_pages && alloced_pages > zram->limit_pages) {
659 660 661 662 663
		zs_free(meta->mem_pool, handle);
		ret = -ENOMEM;
		goto out;
	}

Minchan Kim's avatar
Minchan Kim committed
664 665
	update_used_max(zram, alloced_pages);

Minchan Kim's avatar
Minchan Kim committed
666
	cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
667

668
	if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
669
		src = kmap_atomic(page);
670
		copy_page(cmem, src);
671
		kunmap_atomic(src);
672 673 674
	} else {
		memcpy(cmem, src, clen);
	}
675

676 677
	zcomp_strm_release(zram->comp, zstrm);
	locked = false;
Minchan Kim's avatar
Minchan Kim committed
678
	zs_unmap_object(meta->mem_pool, handle);
679

680 681 682 683
	/*
	 * Free memory associated with this sector
	 * before overwriting unused sectors.
	 */
684
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
685 686
	zram_free_page(zram, index);

Minchan Kim's avatar
Minchan Kim committed
687
	meta->table[index].handle = handle;
688 689
	zram_set_obj_size(meta, index, clen);
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
690

691
	/* Update stats */
692 693
	atomic64_add(clen, &zram->stats.compr_data_size);
	atomic64_inc(&zram->stats.pages_stored);
694
out:
695
	if (locked)
696
		zcomp_strm_release(zram->comp, zstrm);
697 698
	if (is_partial_io(bvec))
		kfree(uncmem);
699
	return ret;
700 701 702
}

static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
703
			int offset, int rw)
704
{
705
	unsigned long start_time = jiffies;
706
	int ret;
707

708 709 710
	generic_start_io_acct(rw, bvec->bv_len >> SECTOR_SHIFT,
			&zram->disk->part0);

711 712
	if (rw == READ) {
		atomic64_inc(&zram->stats.num_reads);
713
		ret = zram_bvec_read(zram, bvec, index, offset);
714 715
	} else {
		atomic64_inc(&zram->stats.num_writes);
716
		ret = zram_bvec_write(zram, bvec, index, offset);
717
	}
718

719 720
	generic_end_io_acct(rw, &zram->disk->part0, start_time);

721 722 723 724 725 726 727
	if (unlikely(ret)) {
		if (rw == READ)
			atomic64_inc(&zram->stats.failed_reads);
		else
			atomic64_inc(&zram->stats.failed_writes);
	}

728
	return ret;
729 730
}

Joonsoo Kim's avatar
Joonsoo Kim committed
731 732 733 734 735 736 737 738 739
/*
 * zram_bio_discard - handler on discard request
 * @index: physical block index in PAGE_SIZE units
 * @offset: byte offset within physical block
 */
static void zram_bio_discard(struct zram *zram, u32 index,
			     int offset, struct bio *bio)
{
	size_t n = bio->bi_iter.bi_size;
740
	struct zram_meta *meta = zram->meta;
Joonsoo Kim's avatar
Joonsoo Kim committed
741 742 743 744 745 746 747 748 749 750 751 752

	/*
	 * zram manages data in physical block size units. Because logical block
	 * size isn't identical with physical block size on some arch, we
	 * could get a discard request pointing to a specific offset within a
	 * certain physical block.  Although we can handle this request by
	 * reading that physiclal block and decompressing and partially zeroing
	 * and re-compressing and then re-storing it, this isn't reasonable
	 * because our intent with a discard request is to save memory.  So
	 * skipping this logical block is appropriate here.
	 */
	if (offset) {
753
		if (n <= (PAGE_SIZE - offset))
Joonsoo Kim's avatar
Joonsoo Kim committed
754 755
			return;

756
		n -= (PAGE_SIZE - offset);
Joonsoo Kim's avatar
Joonsoo Kim committed
757 758 759 760
		index++;
	}

	while (n >= PAGE_SIZE) {
761
		bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
Joonsoo Kim's avatar
Joonsoo Kim committed
762
		zram_free_page(zram, index);
763
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
764
		atomic64_inc(&zram->stats.notify_free);
Joonsoo Kim's avatar
Joonsoo Kim committed
765 766 767 768 769
		index++;
		n -= PAGE_SIZE;
	}
}

770
static void zram_reset_device(struct zram *zram)
771
{
772 773 774 775
	struct zram_meta *meta;
	struct zcomp *comp;
	u64 disksize;

776
	down_write(&zram->init_lock);
777 778 779

	zram->limit_pages = 0;

780
	if (!init_done(zram)) {
781
		up_write(&zram->init_lock);
782
		return;
783
	}
784

785 786 787 788 789 790 791 792 793 794 795 796 797 798 799
	meta = zram->meta;
	comp = zram->comp;
	disksize = zram->disksize;
	/*
	 * Refcount will go down to 0 eventually and r/w handler
	 * cannot handle further I/O so it will bail out by
	 * check zram_meta_get.
	 */
	zram_meta_put(zram);
	/*
	 * We want to free zram_meta in process context to avoid
	 * deadlock between reclaim path and any other locks.
	 */
	wait_event(zram->io_done, atomic_read(&zram->refcount) == 0);

800 801 802
	/* Reset stats */
	memset(&zram->stats, 0, sizeof(zram->stats));
	zram->disksize = 0;
803
	zram->max_comp_streams = 1;
804

805
	set_capacity(zram->disk, 0);
806
	part_stat_set_all(&zram->disk->part0, 0);
807

808
	up_write(&zram->init_lock);
809 810 811
	/* I/O operation under all of CPU are done so let's free */
	zram_meta_free(meta, disksize);
	zcomp_destroy(comp);
812 813 814 815 816 817
}

static ssize_t disksize_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	u64 disksize;
818
	struct zcomp *comp;
819 820
	struct zram_meta *meta;
	struct zram *zram = dev_to_zram(dev);
821
	int err;
822 823 824 825 826 827

	disksize = memparse(buf, NULL);
	if (!disksize)
		return -EINVAL;

	disksize = PAGE_ALIGN(disksize);
828
	meta = zram_meta_alloc(zram->disk->first_minor, disksize);
829 830
	if (!meta)
		return -ENOMEM;
831

832
	comp = zcomp_create(zram->compressor, zram->max_comp_streams);
833
	if (IS_ERR(comp)) {
834 835
		pr_info("Cannot initialise %s compressing backend\n",
				zram->compressor);
836 837
		err = PTR_ERR(comp);
		goto out_free_meta;
838 839
	}

840
	down_write(&zram->init_lock);
841
	if (init_done(zram)) {
842
		pr_info("Cannot change disksize for initialized device\n");
843
		err = -EBUSY;
844
		goto out_destroy_comp;
845 846
	}

847 848
	init_waitqueue_head(&zram->io_done);
	atomic_set(&zram->refcount, 1);
849
	zram->meta = meta;
850
	zram->comp = comp;
851 852 853
	zram->disksize = disksize;
	set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
	up_write(&zram->init_lock);
854 855 856 857 858 859 860 861

	/*
	 * Revalidate disk out of the init_lock to avoid lockdep splat.
	 * It's okay because disk's capacity is protected by init_lock
	 * so that revalidate_disk always sees up-to-date capacity.
	 */
	revalidate_disk(zram->disk);

862
	return len;
863

864 865 866 867
out_destroy_comp:
	up_write(&zram->init_lock);
	zcomp_destroy(comp);
out_free_meta:
868
	zram_meta_free(meta, disksize);
869
	return err;
870 871 872 873 874 875 876 877 878 879 880 881 882
}

static ssize_t reset_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int ret;
	unsigned short do_reset;
	struct zram *zram;
	struct block_device *bdev;

	zram = dev_to_zram(dev);
	bdev = bdget_disk(zram->disk, 0);

883 884 885
	if (!bdev)
		return -ENOMEM;

886
	mutex_lock(&bdev->bd_mutex);
887
	/* Do not reset an active device! */
888
	if (bdev->bd_openers) {
889 890 891
		ret = -EBUSY;
		goto out;
	}
892 893 894

	ret = kstrtou16(buf, 10, &do_reset);
	if (ret)
895
		goto out;
896

897 898 899 900
	if (!do_reset) {
		ret = -EINVAL;
		goto out;
	}
901 902

	/* Make sure all pending I/O is finished */
903
	fsync_bdev(bdev);
904 905 906 907
	zram_reset_device(zram);

	mutex_unlock(&bdev->bd_mutex);
	revalidate_disk(zram->disk);
908
	bdput(bdev);
909 910

	return len;
911 912

out:
913
	mutex_unlock(&bdev->bd_mutex);
914 915
	bdput(bdev);
	return ret;
916 917
}

918
static void __zram_make_request(struct zram *zram, struct bio *bio)
919
{
920
	int offset, rw;
921
	u32 index;
922 923
	struct bio_vec bvec;
	struct bvec_iter iter;
924

925 926 927
	index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
	offset = (bio->bi_iter.bi_sector &
		  (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
928

Joonsoo Kim's avatar
Joonsoo Kim committed
929 930 931 932 933 934
	if (unlikely(bio->bi_rw & REQ_DISCARD)) {
		zram_bio_discard(zram, index, offset, bio);
		bio_endio(bio, 0);
		return;
	}

935
	rw = bio_data_dir(bio);
936
	bio_for_each_segment(bvec, bio, iter) {
937 938
		int max_transfer_size = PAGE_SIZE - offset;

Kent Overstreet's avatar