zram_drv.c 28.5 KB
Newer Older
1
/*
2
 * Compressed RAM block device
3
 *
4
 * Copyright (C) 2008, 2009, 2010  Nitin Gupta
Minchan Kim's avatar
Minchan Kim committed
5
 *               2012, 2013 Minchan Kim
6 7 8 9 10 11 12 13 14
 *
 * This code is released using a dual license strategy: BSD/GPL
 * You can choose the licence that better fits your requirements.
 *
 * Released under the terms of 3-clause BSD License
 * Released under the terms of GNU General Public License Version 2.0
 *
 */

15
#define KMSG_COMPONENT "zram"
16 17
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

18 19 20 21
#ifdef CONFIG_ZRAM_DEBUG
#define DEBUG
#endif

22 23
#include <linux/module.h>
#include <linux/kernel.h>
24
#include <linux/bio.h>
25 26 27 28 29 30
#include <linux/bitops.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>
#include <linux/device.h>
#include <linux/genhd.h>
#include <linux/highmem.h>
31
#include <linux/slab.h>
32 33
#include <linux/string.h>
#include <linux/vmalloc.h>
34
#include <linux/err.h>
35

36
#include "zram_drv.h"
37 38

/* Globals */
39
static int zram_major;
40
static struct zram *zram_devices;
41
static const char *default_compressor = "lzo";
42 43

/* Module params (documentation at end) */
44
static unsigned int num_devices = 1;
45

46 47 48 49 50
#define ZRAM_ATTR_RO(name)						\
static ssize_t zram_attr_##name##_show(struct device *d,		\
				struct device_attribute *attr, char *b)	\
{									\
	struct zram *zram = dev_to_zram(d);				\
51
	return scnprintf(b, PAGE_SIZE, "%llu\n",			\
52 53 54 55 56
		(u64)atomic64_read(&zram->stats.name));			\
}									\
static struct device_attribute dev_attr_##name =			\
	__ATTR(name, S_IRUGO, zram_attr_##name##_show, NULL);

57 58 59 60 61
static inline int init_done(struct zram *zram)
{
	return zram->meta != NULL;
}

62 63 64 65 66 67 68 69 70 71
static inline struct zram *dev_to_zram(struct device *dev)
{
	return (struct zram *)dev_to_disk(dev)->private_data;
}

static ssize_t disksize_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct zram *zram = dev_to_zram(dev);

72
	return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
73 74 75 76 77
}

static ssize_t initstate_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
78
	u32 val;
79 80
	struct zram *zram = dev_to_zram(dev);

81 82 83
	down_read(&zram->init_lock);
	val = init_done(zram);
	up_read(&zram->init_lock);
84

85
	return scnprintf(buf, PAGE_SIZE, "%u\n", val);
86 87 88 89 90 91 92
}

static ssize_t orig_data_size_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct zram *zram = dev_to_zram(dev);

93
	return scnprintf(buf, PAGE_SIZE, "%llu\n",
94
		(u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
95 96 97 98 99 100 101 102 103
}

static ssize_t mem_used_total_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val = 0;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
104 105
	if (init_done(zram)) {
		struct zram_meta *meta = zram->meta;
106
		val = zs_get_total_pages(meta->mem_pool);
107
	}
108 109
	up_read(&zram->init_lock);

110
	return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
111 112
}

113 114 115 116 117 118 119 120 121 122
static ssize_t max_comp_streams_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	int val;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
	val = zram->max_comp_streams;
	up_read(&zram->init_lock);

123
	return scnprintf(buf, PAGE_SIZE, "%d\n", val);
124 125
}

126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
static ssize_t mem_limit_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
	val = zram->limit_pages;
	up_read(&zram->init_lock);

	return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
}

static ssize_t mem_limit_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	u64 limit;
	char *tmp;
	struct zram *zram = dev_to_zram(dev);

	limit = memparse(buf, &tmp);
	if (buf == tmp) /* no chars parsed, invalid input */
		return -EINVAL;

	down_write(&zram->init_lock);
	zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
	up_write(&zram->init_lock);

	return len;
}

Minchan Kim's avatar
Minchan Kim committed
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
static ssize_t mem_used_max_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	u64 val = 0;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
	if (init_done(zram))
		val = atomic_long_read(&zram->stats.max_used_pages);
	up_read(&zram->init_lock);

	return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
}

static ssize_t mem_used_max_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int err;
	unsigned long val;
	struct zram *zram = dev_to_zram(dev);

	err = kstrtoul(buf, 10, &val);
	if (err || val != 0)
		return -EINVAL;

	down_read(&zram->init_lock);
183 184
	if (init_done(zram)) {
		struct zram_meta *meta = zram->meta;
Minchan Kim's avatar
Minchan Kim committed
185 186
		atomic_long_set(&zram->stats.max_used_pages,
				zs_get_total_pages(meta->mem_pool));
187
	}
Minchan Kim's avatar
Minchan Kim committed
188 189 190 191 192
	up_read(&zram->init_lock);

	return len;
}

193 194 195 196 197
static ssize_t max_comp_streams_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int num;
	struct zram *zram = dev_to_zram(dev);
Minchan Kim's avatar
Minchan Kim committed
198
	int ret;
199

Minchan Kim's avatar
Minchan Kim committed
200 201 202
	ret = kstrtoint(buf, 0, &num);
	if (ret < 0)
		return ret;
203 204
	if (num < 1)
		return -EINVAL;
Minchan Kim's avatar
Minchan Kim committed
205

206 207
	down_write(&zram->init_lock);
	if (init_done(zram)) {
Minchan Kim's avatar
Minchan Kim committed
208
		if (!zcomp_set_max_streams(zram->comp, num)) {
209
			pr_info("Cannot change max compression streams\n");
Minchan Kim's avatar
Minchan Kim committed
210 211 212
			ret = -EINVAL;
			goto out;
		}
213
	}
Minchan Kim's avatar
Minchan Kim committed
214

215
	zram->max_comp_streams = num;
Minchan Kim's avatar
Minchan Kim committed
216 217
	ret = len;
out:
218
	up_write(&zram->init_lock);
Minchan Kim's avatar
Minchan Kim committed
219
	return ret;
220 221
}

222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
static ssize_t comp_algorithm_show(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	size_t sz;
	struct zram *zram = dev_to_zram(dev);

	down_read(&zram->init_lock);
	sz = zcomp_available_show(zram->compressor, buf);
	up_read(&zram->init_lock);

	return sz;
}

static ssize_t comp_algorithm_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	struct zram *zram = dev_to_zram(dev);
	down_write(&zram->init_lock);
	if (init_done(zram)) {
		up_write(&zram->init_lock);
		pr_info("Can't change algorithm for initialized device\n");
		return -EBUSY;
	}
	strlcpy(zram->compressor, buf, sizeof(zram->compressor));
	up_write(&zram->init_lock);
	return len;
}

Minchan Kim's avatar
Minchan Kim committed
250
/* flag operations needs meta->tb_lock */
Minchan Kim's avatar
Minchan Kim committed
251
static int zram_test_flag(struct zram_meta *meta, u32 index,
252
			enum zram_pageflags flag)
253
{
254
	return meta->table[index].value & BIT(flag);
255 256
}

Minchan Kim's avatar
Minchan Kim committed
257
static void zram_set_flag(struct zram_meta *meta, u32 index,
258
			enum zram_pageflags flag)
259
{
260
	meta->table[index].value |= BIT(flag);
261 262
}

Minchan Kim's avatar
Minchan Kim committed
263
static void zram_clear_flag(struct zram_meta *meta, u32 index,
264
			enum zram_pageflags flag)
265
{
266 267 268 269 270 271 272 273 274 275 276 277 278 279
	meta->table[index].value &= ~BIT(flag);
}

static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)
{
	return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);
}

static void zram_set_obj_size(struct zram_meta *meta,
					u32 index, size_t size)
{
	unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;

	meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;
280 281
}

282 283 284 285 286 287 288 289
static inline int is_partial_io(struct bio_vec *bvec)
{
	return bvec->bv_len != PAGE_SIZE;
}

/*
 * Check if request is within bounds and aligned on zram logical blocks.
 */
290 291
static inline int valid_io_request(struct zram *zram,
		sector_t start, unsigned int size)
292
{
293
	u64 end, bound;
294

295
	/* unaligned request */
296
	if (unlikely(start & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
297
		return 0;
298
	if (unlikely(size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
299 300
		return 0;

301
	end = start + (size >> SECTOR_SHIFT);
302 303
	bound = zram->disksize >> SECTOR_SHIFT;
	/* out of range range */
304
	if (unlikely(start >= bound || end > bound || start > end))
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
		return 0;

	/* I/O request is valid */
	return 1;
}

static void zram_meta_free(struct zram_meta *meta)
{
	zs_destroy_pool(meta->mem_pool);
	vfree(meta->table);
	kfree(meta);
}

static struct zram_meta *zram_meta_alloc(u64 disksize)
{
	size_t num_pages;
	struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
	if (!meta)
		goto out;

	num_pages = disksize >> PAGE_SHIFT;
	meta->table = vzalloc(num_pages * sizeof(*meta->table));
	if (!meta->table) {
		pr_err("Error allocating zram address table\n");
329
		goto free_meta;
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
	}

	meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM);
	if (!meta->mem_pool) {
		pr_err("Error creating memory pool\n");
		goto free_table;
	}

	return meta;

free_table:
	vfree(meta->table);
free_meta:
	kfree(meta);
	meta = NULL;
out:
	return meta;
}

static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
{
	if (*offset + bvec->bv_len >= PAGE_SIZE)
		(*index)++;
	*offset = (*offset + bvec->bv_len) % PAGE_SIZE;
}

356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
static int page_zero_filled(void *ptr)
{
	unsigned int pos;
	unsigned long *page;

	page = (unsigned long *)ptr;

	for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
		if (page[pos])
			return 0;
	}

	return 1;
}

371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
static void handle_zero_page(struct bio_vec *bvec)
{
	struct page *page = bvec->bv_page;
	void *user_mem;

	user_mem = kmap_atomic(page);
	if (is_partial_io(bvec))
		memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
	else
		clear_page(user_mem);
	kunmap_atomic(user_mem);

	flush_dcache_page(page);
}

386 387 388 389 390 391

/*
 * To protect concurrent access to the same index entry,
 * caller should hold this table index entry's bit_spinlock to
 * indicate this index entry is accessing.
 */
392
static void zram_free_page(struct zram *zram, size_t index)
393
{
Minchan Kim's avatar
Minchan Kim committed
394 395
	struct zram_meta *meta = zram->meta;
	unsigned long handle = meta->table[index].handle;
396

397
	if (unlikely(!handle)) {
398 399 400 401
		/*
		 * No memory is allocated for zero filled pages.
		 * Simply clear zero page flag.
		 */
Minchan Kim's avatar
Minchan Kim committed
402 403
		if (zram_test_flag(meta, index, ZRAM_ZERO)) {
			zram_clear_flag(meta, index, ZRAM_ZERO);
404
			atomic64_dec(&zram->stats.zero_pages);
405 406 407 408
		}
		return;
	}

Minchan Kim's avatar
Minchan Kim committed
409
	zs_free(meta->mem_pool, handle);
410

411 412
	atomic64_sub(zram_get_obj_size(meta, index),
			&zram->stats.compr_data_size);
413
	atomic64_dec(&zram->stats.pages_stored);
414

Minchan Kim's avatar
Minchan Kim committed
415
	meta->table[index].handle = 0;
416
	zram_set_obj_size(meta, index, 0);
417 418
}

419
static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
420
{
421
	int ret = 0;
422
	unsigned char *cmem;
Minchan Kim's avatar
Minchan Kim committed
423
	struct zram_meta *meta = zram->meta;
Minchan Kim's avatar
Minchan Kim committed
424
	unsigned long handle;
Minchan Kim's avatar
Minchan Kim committed
425
	size_t size;
Minchan Kim's avatar
Minchan Kim committed
426

427
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
Minchan Kim's avatar
Minchan Kim committed
428
	handle = meta->table[index].handle;
429
	size = zram_get_obj_size(meta, index);
430

Minchan Kim's avatar
Minchan Kim committed
431
	if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
432
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
433
		clear_page(mem);
434 435
		return 0;
	}
436

Minchan Kim's avatar
Minchan Kim committed
437
	cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
Minchan Kim's avatar
Minchan Kim committed
438
	if (size == PAGE_SIZE)
439
		copy_page(mem, cmem);
440
	else
441
		ret = zcomp_decompress(zram->comp, cmem, size, mem);
Minchan Kim's avatar
Minchan Kim committed
442
	zs_unmap_object(meta->mem_pool, handle);
443
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
444

445
	/* Should NEVER happen. Return bio error if it does. */
446
	if (unlikely(ret)) {
447 448
		pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
		return ret;
449
	}
450

451
	return 0;
452 453
}

454
static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
455
			  u32 index, int offset)
456 457
{
	int ret;
458 459
	struct page *page;
	unsigned char *user_mem, *uncmem = NULL;
Minchan Kim's avatar
Minchan Kim committed
460
	struct zram_meta *meta = zram->meta;
461 462
	page = bvec->bv_page;

463
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
Minchan Kim's avatar
Minchan Kim committed
464 465
	if (unlikely(!meta->table[index].handle) ||
			zram_test_flag(meta, index, ZRAM_ZERO)) {
466
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
467
		handle_zero_page(bvec);
468 469
		return 0;
	}
470
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
471

472 473
	if (is_partial_io(bvec))
		/* Use  a temporary buffer to decompress the page */
474 475 476 477
		uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);

	user_mem = kmap_atomic(page);
	if (!is_partial_io(bvec))
478 479 480 481 482 483 484
		uncmem = user_mem;

	if (!uncmem) {
		pr_info("Unable to allocate temp memory\n");
		ret = -ENOMEM;
		goto out_cleanup;
	}
485

486
	ret = zram_decompress_page(zram, uncmem, index);
487
	/* Should NEVER happen. Return bio error if it does. */
488
	if (unlikely(ret))
489
		goto out_cleanup;
490

491 492 493 494 495 496 497 498 499 500 501
	if (is_partial_io(bvec))
		memcpy(user_mem + bvec->bv_offset, uncmem + offset,
				bvec->bv_len);

	flush_dcache_page(page);
	ret = 0;
out_cleanup:
	kunmap_atomic(user_mem);
	if (is_partial_io(bvec))
		kfree(uncmem);
	return ret;
502 503
}

Minchan Kim's avatar
Minchan Kim committed
504 505 506 507 508 509 510 511 512 513 514 515 516 517 518
static inline void update_used_max(struct zram *zram,
					const unsigned long pages)
{
	int old_max, cur_max;

	old_max = atomic_long_read(&zram->stats.max_used_pages);

	do {
		cur_max = old_max;
		if (pages > cur_max)
			old_max = atomic_long_cmpxchg(
				&zram->stats.max_used_pages, cur_max, pages);
	} while (old_max != cur_max);
}

519 520
static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
			   int offset)
521
{
522
	int ret = 0;
523
	size_t clen;
524
	unsigned long handle;
525
	struct page *page;
526
	unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
Minchan Kim's avatar
Minchan Kim committed
527
	struct zram_meta *meta = zram->meta;
528
	struct zcomp_strm *zstrm;
529
	bool locked = false;
Minchan Kim's avatar
Minchan Kim committed
530
	unsigned long alloced_pages;
531

532
	page = bvec->bv_page;
533 534 535 536 537
	if (is_partial_io(bvec)) {
		/*
		 * This is a partial IO. We need to read the full page
		 * before to write the changes.
		 */
538
		uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
539 540 541 542
		if (!uncmem) {
			ret = -ENOMEM;
			goto out;
		}
543
		ret = zram_decompress_page(zram, uncmem, index);
544
		if (ret)
545 546 547
			goto out;
	}

548
	zstrm = zcomp_strm_find(zram->comp);
549
	locked = true;
550
	user_mem = kmap_atomic(page);
551

552
	if (is_partial_io(bvec)) {
553 554
		memcpy(uncmem + offset, user_mem + bvec->bv_offset,
		       bvec->bv_len);
555 556 557
		kunmap_atomic(user_mem);
		user_mem = NULL;
	} else {
558
		uncmem = user_mem;
559
	}
560 561

	if (page_zero_filled(uncmem)) {
562 563
		if (user_mem)
			kunmap_atomic(user_mem);
564
		/* Free memory associated with this sector now. */
565
		bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
566
		zram_free_page(zram, index);
Minchan Kim's avatar
Minchan Kim committed
567
		zram_set_flag(meta, index, ZRAM_ZERO);
568
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
569

570
		atomic64_inc(&zram->stats.zero_pages);
571 572
		ret = 0;
		goto out;
573
	}
574

575
	ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen);
576 577 578 579 580
	if (!is_partial_io(bvec)) {
		kunmap_atomic(user_mem);
		user_mem = NULL;
		uncmem = NULL;
	}
581

582
	if (unlikely(ret)) {
583
		pr_err("Compression failed! err=%d\n", ret);
584
		goto out;
585
	}
586
	src = zstrm->buffer;
587 588
	if (unlikely(clen > max_zpage_size)) {
		clen = PAGE_SIZE;
589 590
		if (is_partial_io(bvec))
			src = uncmem;
591
	}
592

Minchan Kim's avatar
Minchan Kim committed
593
	handle = zs_malloc(meta->mem_pool, clen);
594
	if (!handle) {
595 596
		pr_info("Error allocating memory for compressed page: %u, size=%zu\n",
			index, clen);
597 598
		ret = -ENOMEM;
		goto out;
599
	}
600

Minchan Kim's avatar
Minchan Kim committed
601 602
	alloced_pages = zs_get_total_pages(meta->mem_pool);
	if (zram->limit_pages && alloced_pages > zram->limit_pages) {
603 604 605 606 607
		zs_free(meta->mem_pool, handle);
		ret = -ENOMEM;
		goto out;
	}

Minchan Kim's avatar
Minchan Kim committed
608 609
	update_used_max(zram, alloced_pages);

Minchan Kim's avatar
Minchan Kim committed
610
	cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
611

612
	if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
613
		src = kmap_atomic(page);
614
		copy_page(cmem, src);
615
		kunmap_atomic(src);
616 617 618
	} else {
		memcpy(cmem, src, clen);
	}
619

620 621
	zcomp_strm_release(zram->comp, zstrm);
	locked = false;
Minchan Kim's avatar
Minchan Kim committed
622
	zs_unmap_object(meta->mem_pool, handle);
623

624 625 626 627
	/*
	 * Free memory associated with this sector
	 * before overwriting unused sectors.
	 */
628
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
629 630
	zram_free_page(zram, index);

Minchan Kim's avatar
Minchan Kim committed
631
	meta->table[index].handle = handle;
632 633
	zram_set_obj_size(meta, index, clen);
	bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
634

635
	/* Update stats */
636 637
	atomic64_add(clen, &zram->stats.compr_data_size);
	atomic64_inc(&zram->stats.pages_stored);
638
out:
639
	if (locked)
640
		zcomp_strm_release(zram->comp, zstrm);
641 642
	if (is_partial_io(bvec))
		kfree(uncmem);
643
	return ret;
644 645 646
}

static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
647
			int offset, int rw)
648
{
649
	int ret;
650

651 652
	if (rw == READ) {
		atomic64_inc(&zram->stats.num_reads);
653
		ret = zram_bvec_read(zram, bvec, index, offset);
654 655
	} else {
		atomic64_inc(&zram->stats.num_writes);
656
		ret = zram_bvec_write(zram, bvec, index, offset);
657
	}
658

659 660 661 662 663 664 665
	if (unlikely(ret)) {
		if (rw == READ)
			atomic64_inc(&zram->stats.failed_reads);
		else
			atomic64_inc(&zram->stats.failed_writes);
	}

666
	return ret;
667 668
}

Joonsoo Kim's avatar
Joonsoo Kim committed
669 670 671 672 673 674 675 676 677
/*
 * zram_bio_discard - handler on discard request
 * @index: physical block index in PAGE_SIZE units
 * @offset: byte offset within physical block
 */
static void zram_bio_discard(struct zram *zram, u32 index,
			     int offset, struct bio *bio)
{
	size_t n = bio->bi_iter.bi_size;
678
	struct zram_meta *meta = zram->meta;
Joonsoo Kim's avatar
Joonsoo Kim committed
679 680 681 682 683 684 685 686 687 688 689 690

	/*
	 * zram manages data in physical block size units. Because logical block
	 * size isn't identical with physical block size on some arch, we
	 * could get a discard request pointing to a specific offset within a
	 * certain physical block.  Although we can handle this request by
	 * reading that physiclal block and decompressing and partially zeroing
	 * and re-compressing and then re-storing it, this isn't reasonable
	 * because our intent with a discard request is to save memory.  So
	 * skipping this logical block is appropriate here.
	 */
	if (offset) {
691
		if (n <= (PAGE_SIZE - offset))
Joonsoo Kim's avatar
Joonsoo Kim committed
692 693
			return;

694
		n -= (PAGE_SIZE - offset);
Joonsoo Kim's avatar
Joonsoo Kim committed
695 696 697 698
		index++;
	}

	while (n >= PAGE_SIZE) {
699
		bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
Joonsoo Kim's avatar
Joonsoo Kim committed
700
		zram_free_page(zram, index);
701
		bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
702
		atomic64_inc(&zram->stats.notify_free);
Joonsoo Kim's avatar
Joonsoo Kim committed
703 704 705 706 707
		index++;
		n -= PAGE_SIZE;
	}
}

Minchan Kim's avatar
Minchan Kim committed
708
static void zram_reset_device(struct zram *zram, bool reset_capacity)
709
{
710 711 712
	size_t index;
	struct zram_meta *meta;

713
	down_write(&zram->init_lock);
714 715 716

	zram->limit_pages = 0;

717
	if (!init_done(zram)) {
718
		up_write(&zram->init_lock);
719
		return;
720
	}
721 722 723 724 725 726 727 728 729 730 731

	meta = zram->meta;
	/* Free all pages that are still in this zram device */
	for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) {
		unsigned long handle = meta->table[index].handle;
		if (!handle)
			continue;

		zs_free(meta->mem_pool, handle);
	}

732
	zcomp_destroy(zram->comp);
733 734
	zram->max_comp_streams = 1;

735 736 737 738 739 740
	zram_meta_free(zram->meta);
	zram->meta = NULL;
	/* Reset stats */
	memset(&zram->stats, 0, sizeof(zram->stats));

	zram->disksize = 0;
741
	if (reset_capacity)
Minchan Kim's avatar
Minchan Kim committed
742
		set_capacity(zram->disk, 0);
743

744
	up_write(&zram->init_lock);
745 746 747 748 749 750 751 752

	/*
	 * Revalidate disk out of the init_lock to avoid lockdep splat.
	 * It's okay because disk's capacity is protected by init_lock
	 * so that revalidate_disk always sees up-to-date capacity.
	 */
	if (reset_capacity)
		revalidate_disk(zram->disk);
753 754 755 756 757 758
}

static ssize_t disksize_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	u64 disksize;
759
	struct zcomp *comp;
760 761
	struct zram_meta *meta;
	struct zram *zram = dev_to_zram(dev);
762
	int err;
763 764 765 766 767 768 769

	disksize = memparse(buf, NULL);
	if (!disksize)
		return -EINVAL;

	disksize = PAGE_ALIGN(disksize);
	meta = zram_meta_alloc(disksize);
770 771
	if (!meta)
		return -ENOMEM;
772

773
	comp = zcomp_create(zram->compressor, zram->max_comp_streams);
774
	if (IS_ERR(comp)) {
775 776
		pr_info("Cannot initialise %s compressing backend\n",
				zram->compressor);
777 778
		err = PTR_ERR(comp);
		goto out_free_meta;
779 780
	}

781
	down_write(&zram->init_lock);
782
	if (init_done(zram)) {
783
		pr_info("Cannot change disksize for initialized device\n");
784
		err = -EBUSY;
785
		goto out_destroy_comp;
786 787
	}

788
	zram->meta = meta;
789
	zram->comp = comp;
790 791 792
	zram->disksize = disksize;
	set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
	up_write(&zram->init_lock);
793 794 795 796 797 798 799 800

	/*
	 * Revalidate disk out of the init_lock to avoid lockdep splat.
	 * It's okay because disk's capacity is protected by init_lock
	 * so that revalidate_disk always sees up-to-date capacity.
	 */
	revalidate_disk(zram->disk);

801
	return len;
802

803 804 805 806
out_destroy_comp:
	up_write(&zram->init_lock);
	zcomp_destroy(comp);
out_free_meta:
807 808
	zram_meta_free(meta);
	return err;
809 810 811 812 813 814 815 816 817 818 819 820 821
}

static ssize_t reset_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t len)
{
	int ret;
	unsigned short do_reset;
	struct zram *zram;
	struct block_device *bdev;

	zram = dev_to_zram(dev);
	bdev = bdget_disk(zram->disk, 0);

822 823 824
	if (!bdev)
		return -ENOMEM;

825
	/* Do not reset an active device! */
826 827 828 829
	if (bdev->bd_holders) {
		ret = -EBUSY;
		goto out;
	}
830 831 832

	ret = kstrtou16(buf, 10, &do_reset);
	if (ret)
833
		goto out;
834

835 836 837 838
	if (!do_reset) {
		ret = -EINVAL;
		goto out;
	}
839 840

	/* Make sure all pending I/O is finished */
841
	fsync_bdev(bdev);
842
	bdput(bdev);
843

Minchan Kim's avatar
Minchan Kim committed
844
	zram_reset_device(zram, true);
845
	return len;
846 847 848 849

out:
	bdput(bdev);
	return ret;
850 851
}

852
static void __zram_make_request(struct zram *zram, struct bio *bio)
853
{
854
	int offset, rw;
855
	u32 index;
856 857
	struct bio_vec bvec;
	struct bvec_iter iter;
858

859 860 861
	index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
	offset = (bio->bi_iter.bi_sector &
		  (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
862

Joonsoo Kim's avatar
Joonsoo Kim committed
863 864 865 866 867 868
	if (unlikely(bio->bi_rw & REQ_DISCARD)) {
		zram_bio_discard(zram, index, offset, bio);
		bio_endio(bio, 0);
		return;
	}

869
	rw = bio_data_dir(bio);
870
	bio_for_each_segment(bvec, bio, iter) {
871 872
		int max_transfer_size = PAGE_SIZE - offset;

873
		if (bvec.bv_len > max_transfer_size) {
874 875 876 877 878 879
			/*
			 * zram_bvec_rw() can only make operation on a single
			 * zram page. Split the bio vector.
			 */
			struct bio_vec bv;

880
			bv.bv_page = bvec.bv_page;
881
			bv.bv_len = max_transfer_size;
882
			bv.bv_offset = bvec.bv_offset;
883

884
			if (zram_bvec_rw(zram, &bv, index, offset, rw) < 0)
885 886
				goto out;

887
			bv.bv_len = bvec.bv_len - max_transfer_size;
888
			bv.bv_offset += max_transfer_size;
889
			if (zram_bvec_rw(zram, &bv, index + 1, 0, rw) < 0)
890 891
				goto out;
		} else
892
			if (zram_bvec_rw(zram, &bvec, index, offset, rw) < 0)
893 894
				goto out;

895
		update_position(&index, &offset, &bvec);
896
	}
897 898 899

	set_bit(BIO_UPTODATE, &bio->bi_flags);
	bio_endio(bio, 0);
900
	return;
901 902 903 904 905 906

out:
	bio_io_error(bio);
}

/*
907
 * Handler function for all zram I/O requests.
908
 */
909
static void zram_make_request(struct request_queue *queue, struct bio *bio)
910
{
911
	struct zram *zram = queue->queuedata;
912

913
	down_read(&zram->init_lock);
914
	if (unlikely(!init_done(zram)))
915
		goto error;
916

917 918
	if (!valid_io_request(zram, bio->bi_iter.bi_sector,
					bio->bi_iter.bi_size)) {
919
		atomic64_inc(&zram->stats.invalid_io);
920
		goto error;
921 922
	}

923
	__zram_make_request(zram, bio);
924
	up_read(&zram->init_lock);
925

926
	return;
927 928

error:
929
	up_read(&zram->init_lock);
930
	bio_io_error(bio);
931 932
}

Nitin Gupta's avatar
Nitin Gupta committed
933 934
static void zram_slot_free_notify(struct block_device *bdev,
				unsigned long index)
935
{
936
	struct zram *zram;
937
	struct zram_meta *meta;
938

939
	zram = bdev->bd_disk->private_data;
940
	meta = zram->meta;
941

942
	bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
943
	zram_free_page(zram, index);
Weijie Yang's avatar