zram_drv.c 27.5 KB
Newer Older
1
/*
2
 * Compressed RAM block device
3
 *
4
 * Copyright (C) 2008, 2009, 2010  Nitin Gupta
Minchan Kim's avatar
Minchan Kim committed
5
 *               2012, 2013 Minchan Kim
6 7 8 9 10 11 12 13 14
 *
 * This code is released using a dual license strategy: BSD/GPL
 * You can choose the licence that better fits your requirements.
 *
 * Released under the terms of 3-clause BSD License
 * Released under the terms of GNU General Public License Version 2.0
 *
 */

15
#define KMSG_COMPONENT "zram"
16 17
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

18 19 20 21
#ifdef CONFIG_ZRAM_DEBUG
#define DEBUG
#endif

22 23
#include <linux/module.h>
#include <linux/kernel.h>
24
#include <linux/bio.h>
25 26 27 28 29 30
#include <linux/bitops.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>
#include <linux/device.h>
#include <linux/genhd.h>
#include <linux/highmem.h>
31
#include <linux/slab.h>
32 33
#include <linux/string.h>
#include <linux/vmalloc.h>
34
#include <linux/err.h>
35

36
#include "zram_drv.h"
37 38

/* Globals */
39
static int zram_major;
40
static struct zram *zram_devices;
41
static const char *default_compressor = "lzo";
42 43

/* Module params (documentation at end) */
44
static unsigned int num_devices = 1;
45

46 47 48 49 50
#define ZRAM_ATTR_RO(name)						\
static ssize_t zram_attr_##name##_show(struct device *d,		\
				struct device_attribute *attr, char *b)	\
{									\
	struct zram *zram = dev_to_zram(d);				\
51
	return scnprintf(b, PAGE_SIZE, "%llu\n",			\
52 53 54 55 56
		(u64)atomic64_read(&zram->stats.name));			\
}									\
static struct device_attribute dev_attr_##name =			\
	__ATTR(name, S_IRUGO, zram_attr_##name##_show, NULL);

57 58 59 60 61
static inline int init_done(struct zram *zram)
{
	return zram->meta != NULL;
}

62 63 64 65 66 67 68 69 70 71
static inline struct zram *dev_to_zram(struct device *dev)
{
	return (struct zram *)dev_to_disk(dev)->private_data;
}

static ssize_t disksize_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct zram *zram = dev_to_zram(dev);

72
	return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
73 74 75 76 77
}

static ssize_t initstate_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
78
	u32 val;
79 80
	struct zram *zram = dev_to_zram(dev);

81 82 83
	down_read(&zram->init_lock);
	val = init_done(zram);
	up_read(&zram->init_lock);
84

85
	return scnprintf(buf, PAGE_SIZE, "%u\n", val);
86 87 88 89 90 91 92
}

static ssize_t orig_data_size_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct zram *zram = dev_to_zram(dev);

93
	return scnprintf(buf, PAGE_SIZE, "%llu\n",
94
		(u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
95 96 97 98 99 100 101 102 103
}

static ssize_t mem_used_total_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val = 0;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
104 105
	if (init_done(zram)) {
		struct zram_meta *meta = zram->meta;
106
		val = zs_get_total_pages(meta->mem_pool);
107
	}
108 109
	up_read(&zram->init_lock);

110
	return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
111 112
}

113 114 115 116 117 118 119 120 121 122
static ssize_t max_comp_streams_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	int val;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
	val = zram->max_comp_streams;
	up_read(&zram->init_lock);

123
	return scnprintf(buf, PAGE_SIZE, "%d\n", val);
124 125
}

126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
static ssize_t mem_limit_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
	val = zram->limit_pages;
	up_read(&zram->init_lock);

	return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
}

static ssize_t mem_limit_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	u64 limit;
	char *tmp;
	struct zram *zram = dev_to_zram(dev);

	limit = memparse(buf, &tmp);
	if (buf == tmp) /* no chars parsed, invalid input */
		return -EINVAL;

	down_write(&zram->init_lock);
	zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
	up_write(&zram->init_lock);

	return len;
}

Minchan Kim's avatar
Minchan Kim committed
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
static ssize_t mem_used_max_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val = 0;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
	if (init_done(zram))
		val = atomic_long_read(&zram->stats.max_used_pages);
	up_read(&zram->init_lock);

	return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
}

static ssize_t mem_used_max_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int err;
	unsigned long val;
	struct zram *zram = dev_to_zram(dev);

	err = kstrtoul(buf, 10, &val);
	if (err || val != 0)
		return -EINVAL;

	down_read(&zram->init_lock);
183 184
	if (init_done(zram)) {
		struct zram_meta *meta = zram->meta;
Minchan Kim's avatar
Minchan Kim committed
185 186
		atomic_long_set(&zram->stats.max_used_pages,
				zs_get_total_pages(meta->mem_pool));
187
	}
Minchan Kim's avatar
Minchan Kim committed
188 189 190 191 192
	up_read(&zram->init_lock);

	return len;
}

193 194 195 196 197
static ssize_t max_comp_streams_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int num;
	struct zram *zram = dev_to_zram(dev);
Minchan Kim's avatar
Minchan Kim committed
198
	int ret;
199

Minchan Kim's avatar
Minchan Kim committed
200 201 202
	ret = kstrtoint(buf, 0, &num);
	if (ret < 0)
		return ret;
203 204
	if (num < 1)
		return -EINVAL;
Minchan Kim's avatar
Minchan Kim committed
205

206 207
	down_write(&zram->init_lock);
	if (init_done(zram)) {
Minchan Kim's avatar
Minchan Kim committed
208
		if (!zcomp_set_max_streams(zram->comp, num)) {
209
			pr_info("Cannot change max compression streams\n");
Minchan Kim's avatar
Minchan Kim committed
210 211 212
			ret = -EINVAL;
			goto out;
		}
213
	}
Minchan Kim's avatar
Minchan Kim committed
214

215
	zram->max_comp_streams = num;
Minchan Kim's avatar
Minchan Kim committed
216 217
	ret = len;
out:
218
	up_write(&zram->init_lock);
Minchan Kim's avatar
Minchan Kim committed
219
	return ret;
220 221
}

222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
static ssize_t comp_algorithm_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	size_t sz;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
	sz = zcomp_available_show(zram->compressor, buf);
	up_read(&zram->init_lock);

	return sz;
}

static ssize_t comp_algorithm_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	struct zram *zram = dev_to_zram(dev);
	down_write(&zram->init_lock);
	if (init_done(zram)) {
		up_write(&zram->init_lock);
		pr_info("Can't change algorithm for initialized device\n");
		return -EBUSY;
	}
	strlcpy(zram->compressor, buf, sizeof(zram->compressor));
	up_write(&zram->init_lock);
	return len;
}

Minchan Kim's avatar
Minchan Kim committed
250
/* flag operations needs meta->tb_lock */
Minchan Kim's avatar
Minchan Kim committed
251
static int zram_test_flag(struct zram_meta *meta, u32 index,
252
			enum zram_pageflags flag)
253
{
254
	return meta->table[index].value & BIT(flag);
255 256
}

Minchan Kim's avatar
Minchan Kim committed
257
static void zram_set_flag(struct zram_meta *meta, u32 index,
258
			enum zram_pageflags flag)
259
{
260
	meta->table[index].value |= BIT(flag);
261 262
}

Minchan Kim's avatar
Minchan Kim committed
263
static void zram_clear_flag(struct zram_meta *meta, u32 index,
264
			enum zram_pageflags flag)
265
{
266 267 268 269 270 271 272 273 274 275 276 277 278 279
	meta->table[index].value &= ~BIT(flag);
}

static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)
{
	return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);
}

static void zram_set_obj_size(struct zram_meta *meta,
					u32 index, size_t size)
{
	unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;

	meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;
280 281
}

282 283 284 285 286 287 288 289 290 291 292
static inline int is_partial_io(struct bio_vec *bvec)
{
	return bvec->bv_len != PAGE_SIZE;
}

/*
 * Check if request is within bounds and aligned on zram logical blocks.
 */
static inline int valid_io_request(struct zram *zram, struct bio *bio)
{
	u64 start, end, bound;
293

294
	/* unaligned request */
295 296
	if (unlikely(bio->bi_iter.bi_sector &
		     (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
297
		return 0;
298
	if (unlikely(bio->bi_iter.bi_size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
299 300
		return 0;

301 302
	start = bio->bi_iter.bi_sector;
	end = start + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
303 304
	bound = zram->disksize >> SECTOR_SHIFT;
	/* out of range range */
305
	if (unlikely(start >= bound || end > bound || start > end))
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
		return 0;

	/* I/O request is valid */
	return 1;
}

static void zram_meta_free(struct zram_meta *meta)
{
	zs_destroy_pool(meta->mem_pool);
	vfree(meta->table);
	kfree(meta);
}

static struct zram_meta *zram_meta_alloc(u64 disksize)
{
	size_t num_pages;
	struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
	if (!meta)
		goto out;

	num_pages = disksize >> PAGE_SHIFT;
	meta->table = vzalloc(num_pages * sizeof(*meta->table));
	if (!meta->table) {
		pr_err("Error allocating zram address table\n");
330
		goto free_meta;
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
	}

	meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM);
	if (!meta->mem_pool) {
		pr_err("Error creating memory pool\n");
		goto free_table;
	}

	return meta;

free_table:
	vfree(meta->table);
free_meta:
	kfree(meta);
	meta = NULL;
out:
	return meta;
}

static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
{
	if (*offset + bvec->bv_len >= PAGE_SIZE)
		(*index)++;
	*offset = (*offset + bvec->bv_len) % PAGE_SIZE;
}

357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
static int page_zero_filled(void *ptr)
{
	unsigned int pos;
	unsigned long *page;

	page = (unsigned long *)ptr;

	for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
		if (page[pos])
			return 0;
	}

	return 1;
}

372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
static void handle_zero_page(struct bio_vec *bvec)
{
	struct page *page = bvec->bv_page;
	void *user_mem;

	user_mem = kmap_atomic(page);
	if (is_partial_io(bvec))
		memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
	else
		clear_page(user_mem);
	kunmap_atomic(user_mem);

	flush_dcache_page(page);
}

387 388 389 390 391 392

/*
 * To protect concurrent access to the same index entry,
 * caller should hold this table index entry's bit_spinlock to
 * indicate this index entry is accessing.
 */
393
static void zram_free_page(struct zram *zram, size_t index)
394
{
Minchan Kim's avatar
Minchan Kim committed
395 396
	struct zram_meta *meta = zram->meta;
	unsigned long handle = meta->table[index].handle;
397

398
	if (unlikely(!handle)) {
399 400 401 402
		/*
		 * No memory is allocated for zero filled pages.
		 * Simply clear zero page flag.
		 */
Minchan Kim's avatar
Minchan Kim committed
403 404
		if (zram_test_flag(meta, index, ZRAM_ZERO)) {
			zram_clear_flag(meta, index, ZRAM_ZERO);
405
			atomic64_dec(&zram->stats.zero_pages);
406 407 408 409
		}
		return;
	}

Minchan Kim's avatar
Minchan Kim committed
410
	zs_free(meta->mem_pool, handle);
411

412 413
	atomic64_sub(zram_get_obj_size(meta, index),
			&zram->stats.compr_data_size);
414
	atomic64_dec(&zram->stats.pages_stored);
415

Minchan Kim's avatar
Minchan Kim committed
416
	meta->table[index].handle = 0;
417
	zram_set_obj_size(meta, index, 0);
418 419
}

420
static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
421
{
422
	int ret = 0;
423
	unsigned char *cmem;
Minchan Kim's avatar
Minchan Kim committed
424
	struct zram_meta *meta = zram->meta;
Minchan Kim's avatar
Minchan Kim committed
425
	unsigned long handle;
Minchan Kim's avatar
Minchan Kim committed
426
	size_t size;
Minchan Kim's avatar
Minchan Kim committed
427

428
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
Minchan Kim's avatar
Minchan Kim committed
429
	handle = meta->table[index].handle;
430
	size = zram_get_obj_size(meta, index);
431

Minchan Kim's avatar
Minchan Kim committed
432
	if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
433
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
434
		clear_page(mem);
435 436
		return 0;
	}
437

Minchan Kim's avatar
Minchan Kim committed
438
	cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
Minchan Kim's avatar
Minchan Kim committed
439
	if (size == PAGE_SIZE)
440
		copy_page(mem, cmem);
441
	else
442
		ret = zcomp_decompress(zram->comp, cmem, size, mem);
Minchan Kim's avatar
Minchan Kim committed
443
	zs_unmap_object(meta->mem_pool, handle);
444
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
445

446
	/* Should NEVER happen. Return bio error if it does. */
447
	if (unlikely(ret)) {
448 449
		pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
		return ret;
450
	}
451

452
	return 0;
453 454
}

455 456
static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
			  u32 index, int offset, struct bio *bio)
457 458
{
	int ret;
459 460
	struct page *page;
	unsigned char *user_mem, *uncmem = NULL;
Minchan Kim's avatar
Minchan Kim committed
461
	struct zram_meta *meta = zram->meta;
462 463
	page = bvec->bv_page;

464
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
Minchan Kim's avatar
Minchan Kim committed
465 466
	if (unlikely(!meta->table[index].handle) ||
			zram_test_flag(meta, index, ZRAM_ZERO)) {
467
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
468
		handle_zero_page(bvec);
469 470
		return 0;
	}
471
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
472

473 474
	if (is_partial_io(bvec))
		/* Use  a temporary buffer to decompress the page */
475 476 477 478
		uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);

	user_mem = kmap_atomic(page);
	if (!is_partial_io(bvec))
479 480 481 482 483 484 485
		uncmem = user_mem;

	if (!uncmem) {
		pr_info("Unable to allocate temp memory\n");
		ret = -ENOMEM;
		goto out_cleanup;
	}
486

487
	ret = zram_decompress_page(zram, uncmem, index);
488
	/* Should NEVER happen. Return bio error if it does. */
489
	if (unlikely(ret))
490
		goto out_cleanup;
491

492 493 494 495 496 497 498 499 500 501 502
	if (is_partial_io(bvec))
		memcpy(user_mem + bvec->bv_offset, uncmem + offset,
				bvec->bv_len);

	flush_dcache_page(page);
	ret = 0;
out_cleanup:
	kunmap_atomic(user_mem);
	if (is_partial_io(bvec))
		kfree(uncmem);
	return ret;
503 504
}

Minchan Kim's avatar
Minchan Kim committed
505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
static inline void update_used_max(struct zram *zram,
					const unsigned long pages)
{
	int old_max, cur_max;

	old_max = atomic_long_read(&zram->stats.max_used_pages);

	do {
		cur_max = old_max;
		if (pages > cur_max)
			old_max = atomic_long_cmpxchg(
				&zram->stats.max_used_pages, cur_max, pages);
	} while (old_max != cur_max);
}

520 521
static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
			   int offset)
522
{
523
	int ret = 0;
524
	size_t clen;
525
	unsigned long handle;
526
	struct page *page;
527
	unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
Minchan Kim's avatar
Minchan Kim committed
528
	struct zram_meta *meta = zram->meta;
529
	struct zcomp_strm *zstrm;
530
	bool locked = false;
Minchan Kim's avatar
Minchan Kim committed
531
	unsigned long alloced_pages;
532

533
	page = bvec->bv_page;
534 535 536 537 538
	if (is_partial_io(bvec)) {
		/*
		 * This is a partial IO. We need to read the full page
		 * before to write the changes.
		 */
539
		uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
540 541 542 543
		if (!uncmem) {
			ret = -ENOMEM;
			goto out;
		}
544
		ret = zram_decompress_page(zram, uncmem, index);
545
		if (ret)
546 547 548
			goto out;
	}

549
	zstrm = zcomp_strm_find(zram->comp);
550
	locked = true;
551
	user_mem = kmap_atomic(page);
552

553
	if (is_partial_io(bvec)) {
554 555
		memcpy(uncmem + offset, user_mem + bvec->bv_offset,
		       bvec->bv_len);
556 557 558
		kunmap_atomic(user_mem);
		user_mem = NULL;
	} else {
559
		uncmem = user_mem;
560
	}
561 562

	if (page_zero_filled(uncmem)) {
563
		kunmap_atomic(user_mem);
564
		/* Free memory associated with this sector now. */
565
		bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
566
		zram_free_page(zram, index);
Minchan Kim's avatar
Minchan Kim committed
567
		zram_set_flag(meta, index, ZRAM_ZERO);
568
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
569

570
		atomic64_inc(&zram->stats.zero_pages);
571 572
		ret = 0;
		goto out;
573
	}
574

575
	ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen);
576 577 578 579 580
	if (!is_partial_io(bvec)) {
		kunmap_atomic(user_mem);
		user_mem = NULL;
		uncmem = NULL;
	}
581

582
	if (unlikely(ret)) {
583
		pr_err("Compression failed! err=%d\n", ret);
584
		goto out;
585
	}
586
	src = zstrm->buffer;
587 588
	if (unlikely(clen > max_zpage_size)) {
		clen = PAGE_SIZE;
589 590
		if (is_partial_io(bvec))
			src = uncmem;
591
	}
592

Minchan Kim's avatar
Minchan Kim committed
593
	handle = zs_malloc(meta->mem_pool, clen);
594
	if (!handle) {
595 596
		pr_info("Error allocating memory for compressed page: %u, size=%zu\n",
			index, clen);
597 598
		ret = -ENOMEM;
		goto out;
599
	}
600

Minchan Kim's avatar
Minchan Kim committed
601 602
	alloced_pages = zs_get_total_pages(meta->mem_pool);
	if (zram->limit_pages && alloced_pages > zram->limit_pages) {
603 604 605 606 607
		zs_free(meta->mem_pool, handle);
		ret = -ENOMEM;
		goto out;
	}

Minchan Kim's avatar
Minchan Kim committed
608 609
	update_used_max(zram, alloced_pages);

Minchan Kim's avatar
Minchan Kim committed
610
	cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
611

612
	if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
613
		src = kmap_atomic(page);
614
		copy_page(cmem, src);
615
		kunmap_atomic(src);
616 617 618
	} else {
		memcpy(cmem, src, clen);
	}
619

620 621
	zcomp_strm_release(zram->comp, zstrm);
	locked = false;
Minchan Kim's avatar
Minchan Kim committed
622
	zs_unmap_object(meta->mem_pool, handle);
623

624 625 626 627
	/*
	 * Free memory associated with this sector
	 * before overwriting unused sectors.
	 */
628
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
629 630
	zram_free_page(zram, index);

Minchan Kim's avatar
Minchan Kim committed
631
	meta->table[index].handle = handle;
632 633
	zram_set_obj_size(meta, index, clen);
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
634

635
	/* Update stats */
636 637
	atomic64_add(clen, &zram->stats.compr_data_size);
	atomic64_inc(&zram->stats.pages_stored);
638
out:
639
	if (locked)
640
		zcomp_strm_release(zram->comp, zstrm);
641 642
	if (is_partial_io(bvec))
		kfree(uncmem);
643
	return ret;
644 645 646
}

static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
647
			int offset, struct bio *bio)
648
{
649
	int ret;
650
	int rw = bio_data_dir(bio);
651

652 653
	if (rw == READ) {
		atomic64_inc(&zram->stats.num_reads);
654
		ret = zram_bvec_read(zram, bvec, index, offset, bio);
655 656
	} else {
		atomic64_inc(&zram->stats.num_writes);
657
		ret = zram_bvec_write(zram, bvec, index, offset);
658
	}
659

660 661 662 663 664 665 666
	if (unlikely(ret)) {
		if (rw == READ)
			atomic64_inc(&zram->stats.failed_reads);
		else
			atomic64_inc(&zram->stats.failed_writes);
	}

667
	return ret;
668 669
}

Joonsoo Kim's avatar
Joonsoo Kim committed
670 671 672 673 674 675 676 677 678
/*
 * zram_bio_discard - handler on discard request
 * @index: physical block index in PAGE_SIZE units
 * @offset: byte offset within physical block
 */
static void zram_bio_discard(struct zram *zram, u32 index,
			     int offset, struct bio *bio)
{
	size_t n = bio->bi_iter.bi_size;
679
	struct zram_meta *meta = zram->meta;
Joonsoo Kim's avatar
Joonsoo Kim committed
680 681 682 683 684 685 686 687 688 689 690 691

	/*
	 * zram manages data in physical block size units. Because logical block
	 * size isn't identical with physical block size on some arch, we
	 * could get a discard request pointing to a specific offset within a
	 * certain physical block.  Although we can handle this request by
	 * reading that physiclal block and decompressing and partially zeroing
	 * and re-compressing and then re-storing it, this isn't reasonable
	 * because our intent with a discard request is to save memory.  So
	 * skipping this logical block is appropriate here.
	 */
	if (offset) {
692
		if (n <= (PAGE_SIZE - offset))
Joonsoo Kim's avatar
Joonsoo Kim committed
693 694
			return;

695
		n -= (PAGE_SIZE - offset);
Joonsoo Kim's avatar
Joonsoo Kim committed
696 697 698 699
		index++;
	}

	while (n >= PAGE_SIZE) {
700
		bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
Joonsoo Kim's avatar
Joonsoo Kim committed
701
		zram_free_page(zram, index);
702
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
703
		atomic64_inc(&zram->stats.notify_free);
Joonsoo Kim's avatar
Joonsoo Kim committed
704 705 706 707 708
		index++;
		n -= PAGE_SIZE;
	}
}

Minchan Kim's avatar
Minchan Kim committed
709
static void zram_reset_device(struct zram *zram, bool reset_capacity)
710
{
711 712 713
	size_t index;
	struct zram_meta *meta;

714
	down_write(&zram->init_lock);
715 716 717

	zram->limit_pages = 0;

718
	if (!init_done(zram)) {
719
		up_write(&zram->init_lock);
720
		return;
721
	}
722 723 724 725 726 727 728 729 730 731 732

	meta = zram->meta;
	/* Free all pages that are still in this zram device */
	for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) {
		unsigned long handle = meta->table[index].handle;
		if (!handle)
			continue;

		zs_free(meta->mem_pool, handle);
	}

733
	zcomp_destroy(zram->comp);
734 735
	zram->max_comp_streams = 1;

736 737 738 739 740 741
	zram_meta_free(zram->meta);
	zram->meta = NULL;
	/* Reset stats */
	memset(&zram->stats, 0, sizeof(zram->stats));

	zram->disksize = 0;
742
	if (reset_capacity)
Minchan Kim's avatar
Minchan Kim committed
743
		set_capacity(zram->disk, 0);
744

745
	up_write(&zram->init_lock);
746 747 748 749 750 751 752 753

	/*
	 * Revalidate disk out of the init_lock to avoid lockdep splat.
	 * It's okay because disk's capacity is protected by init_lock
	 * so that revalidate_disk always sees up-to-date capacity.
	 */
	if (reset_capacity)
		revalidate_disk(zram->disk);
754 755 756 757 758 759
}

static ssize_t disksize_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	u64 disksize;
760
	struct zcomp *comp;
761 762
	struct zram_meta *meta;
	struct zram *zram = dev_to_zram(dev);
763
	int err;
764 765 766 767 768 769 770

	disksize = memparse(buf, NULL);
	if (!disksize)
		return -EINVAL;

	disksize = PAGE_ALIGN(disksize);
	meta = zram_meta_alloc(disksize);
771 772
	if (!meta)
		return -ENOMEM;
773

774
	comp = zcomp_create(zram->compressor, zram->max_comp_streams);
775
	if (IS_ERR(comp)) {
776 777
		pr_info("Cannot initialise %s compressing backend\n",
				zram->compressor);
778 779
		err = PTR_ERR(comp);
		goto out_free_meta;
780 781
	}

782
	down_write(&zram->init_lock);
783
	if (init_done(zram)) {
784
		pr_info("Cannot change disksize for initialized device\n");
785
		err = -EBUSY;
786
		goto out_destroy_comp;
787 788
	}

789
	zram->meta = meta;
790
	zram->comp = comp;
791 792 793
	zram->disksize = disksize;
	set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
	up_write(&zram->init_lock);
794 795 796 797 798 799 800 801

	/*
	 * Revalidate disk out of the init_lock to avoid lockdep splat.
	 * It's okay because disk's capacity is protected by init_lock
	 * so that revalidate_disk always sees up-to-date capacity.
	 */
	revalidate_disk(zram->disk);

802
	return len;
803

804 805 806 807
out_destroy_comp:
	up_write(&zram->init_lock);
	zcomp_destroy(comp);
out_free_meta:
808 809
	zram_meta_free(meta);
	return err;
810 811 812 813 814 815 816 817 818 819 820 821 822
}

static ssize_t reset_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int ret;
	unsigned short do_reset;
	struct zram *zram;
	struct block_device *bdev;

	zram = dev_to_zram(dev);
	bdev = bdget_disk(zram->disk, 0);

823 824 825
	if (!bdev)
		return -ENOMEM;

826
	/* Do not reset an active device! */
827 828 829 830
	if (bdev->bd_holders) {
		ret = -EBUSY;
		goto out;
	}
831 832 833

	ret = kstrtou16(buf, 10, &do_reset);
	if (ret)
834
		goto out;
835

836 837 838 839
	if (!do_reset) {
		ret = -EINVAL;
		goto out;
	}
840 841

	/* Make sure all pending I/O is finished */
842
	fsync_bdev(bdev);
843
	bdput(bdev);
844

Minchan Kim's avatar
Minchan Kim committed
845
	zram_reset_device(zram, true);
846
	return len;
847 848 849 850

out:
	bdput(bdev);
	return ret;
851 852
}

853
static void __zram_make_request(struct zram *zram, struct bio *bio)
854
{
855
	int offset;
856
	u32 index;
857 858
	struct bio_vec bvec;
	struct bvec_iter iter;
859

860 861 862
	index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
	offset = (bio->bi_iter.bi_sector &
		  (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
863

Joonsoo Kim's avatar
Joonsoo Kim committed
864 865 866 867 868 869
	if (unlikely(bio->bi_rw & REQ_DISCARD)) {
		zram_bio_discard(zram, index, offset, bio);
		bio_endio(bio, 0);
		return;
	}

870
	bio_for_each_segment(bvec, bio, iter) {
871 872
		int max_transfer_size = PAGE_SIZE - offset;

873
		if (bvec.bv_len > max_transfer_size) {
874 875 876 877 878 879
			/*
			 * zram_bvec_rw() can only make operation on a single
			 * zram page. Split the bio vector.
			 */
			struct bio_vec bv;

880
			bv.bv_page = bvec.bv_page;
881
			bv.bv_len = max_transfer_size;
882
			bv.bv_offset = bvec.bv_offset;
883

884
			if (zram_bvec_rw(zram, &bv, index, offset, bio) < 0)
885 886
				goto out;

887
			bv.bv_len = bvec.bv_len - max_transfer_size;
888
			bv.bv_offset += max_transfer_size;
889
			if (zram_bvec_rw(zram, &bv, index + 1, 0, bio) < 0)
890 891
				goto out;
		} else
892
			if (zram_bvec_rw(zram, &bvec, index, offset, bio) < 0)
893 894
				goto out;

895
		update_position(&index, &offset, &bvec);
896
	}
897 898 899

	set_bit(BIO_UPTODATE, &bio->bi_flags);
	bio_endio(bio, 0);
900
	return;
901 902 903 904 905 906

out:
	bio_io_error(bio);
}

/*
907
 * Handler function for all zram I/O requests.
908
 */
909
static void zram_make_request(struct request_queue *queue, struct bio *bio)
910
{
911
	struct zram *zram = queue->queuedata;
912

913
	down_read(&zram->init_lock);
914
	if (unlikely(!init_done(zram)))
915
		goto error;
916

917
	if (!valid_io_request(zram, bio)) {
918
		atomic64_inc(&zram->stats.invalid_io);
919
		goto error;
920 921
	}

922
	__zram_make_request(zram, bio);
923
	up_read(&zram->init_lock);
924

925
	return;
926 927

error:
928
	up_read(&zram->init_lock);
929
	bio_io_error(bio);
930 931
}

Nitin Gupta's avatar
Nitin Gupta committed
932 933
static void zram_slot_free_notify(struct block_device *bdev,
				unsigned long index)
934
{
935
	struct zram *zram;
936
	struct zram_meta *meta;
937

938
	zram = bdev->bd_disk->private_data;
939
	meta = zram->meta;
940

941
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
942
	zram_free_page(zram, index);
943
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
944
	atomic64_inc(&zram->stats.notify_free);
945