tcrypt.c 51.4 KB
Newer Older
1
/*
Linus Torvalds's avatar
Linus Torvalds committed
2 3 4 5 6 7 8
 * Quick & dirty crypto testing module.
 *
 * This will only exist until we have a better testing mechanism
 * (e.g. a char device).
 *
 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
 * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
9
 * Copyright (c) 2007 Nokia Siemens Networks
Linus Torvalds's avatar
Linus Torvalds committed
10
 *
11 12 13 14 15 16 17
 * Updated RFC4106 AES-GCM testing.
 *    Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
 *             Adrian Hoban <adrian.hoban@intel.com>
 *             Gabriele Paoloni <gabriele.paoloni@intel.com>
 *             Tadeusz Struk (tadeusz.struk@intel.com)
 *             Copyright (c) 2010, Intel Corporation.
 *
Linus Torvalds's avatar
Linus Torvalds committed
18 19
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
20
 * Software Foundation; either version 2 of the License, or (at your option)
Linus Torvalds's avatar
Linus Torvalds committed
21 22 23 24
 * any later version.
 *
 */

25 26
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

27
#include <crypto/aead.h>
28
#include <crypto/hash.h>
Herbert Xu's avatar
Herbert Xu committed
29
#include <crypto/skcipher.h>
30
#include <linux/err.h>
31
#include <linux/fips.h>
Linus Torvalds's avatar
Linus Torvalds committed
32
#include <linux/init.h>
33
#include <linux/gfp.h>
Linus Torvalds's avatar
Linus Torvalds committed
34
#include <linux/module.h>
35
#include <linux/scatterlist.h>
Linus Torvalds's avatar
Linus Torvalds committed
36 37
#include <linux/string.h>
#include <linux/moduleparam.h>
38
#include <linux/jiffies.h>
39 40
#include <linux/timex.h>
#include <linux/interrupt.h>
Linus Torvalds's avatar
Linus Torvalds committed
41 42 43
#include "tcrypt.h"

/*
44
 * Need slab memory for testing (size in number of pages).
Linus Torvalds's avatar
Linus Torvalds committed
45
 */
46
#define TVMEMSIZE	4
Linus Torvalds's avatar
Linus Torvalds committed
47 48

/*
49
* Used by test_cipher_speed()
Linus Torvalds's avatar
Linus Torvalds committed
50 51 52 53
*/
#define ENCRYPT 1
#define DECRYPT 0

54 55
#define MAX_DIGEST_SIZE		64

56 57 58 59 60
/*
 * return a string with the driver name
 */
#define get_driver_name(tfm_type, tfm) crypto_tfm_alg_driver_name(tfm_type ## _tfm(tfm))

61 62 63
/*
 * Used by test_cipher_speed()
 */
64
static unsigned int sec;
65

66 67
static char *alg = NULL;
static u32 type;
68
static u32 mask;
Linus Torvalds's avatar
Linus Torvalds committed
69
static int mode;
70
static char *tvmem[TVMEMSIZE];
Linus Torvalds's avatar
Linus Torvalds committed
71 72

static char *check[] = {
73
	"des", "md5", "des3_ede", "rot13", "sha1", "sha224", "sha256", "sm3",
74 75
	"blowfish", "twofish", "serpent", "sha384", "sha512", "md4", "aes",
	"cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea",
76
	"khazad", "wp512", "wp384", "wp256", "tnepres", "xeta",  "fcrypt",
77
	"camellia", "seed", "salsa20", "rmd128", "rmd160", "rmd256", "rmd320",
78 79
	"lzo", "cts", "zlib", "sha3-224", "sha3-256", "sha3-384", "sha3-512",
	NULL
Linus Torvalds's avatar
Linus Torvalds committed
80 81
};

82 83
static inline int do_one_aead_op(struct aead_request *req, int ret)
{
84
	struct crypto_wait *wait = req->base.data;
85

86
	return crypto_wait_req(ret, wait);
87 88
}

89
static int test_aead_jiffies(struct aead_request *req, int enc,
90
				int blen, int secs)
91 92 93 94 95
{
	unsigned long start, end;
	int bcount;
	int ret;

96
	for (start = jiffies, end = start + secs * HZ, bcount = 0;
97 98
	     time_before(jiffies, end); bcount++) {
		if (enc)
99
			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
100
		else
101
			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
102 103 104 105 106 107

		if (ret)
			return ret;
	}

	printk("%d operations in %d seconds (%ld bytes)\n",
108
	       bcount, secs, (long)bcount * blen);
109 110 111 112 113 114 115 116 117 118 119 120
	return 0;
}

static int test_aead_cycles(struct aead_request *req, int enc, int blen)
{
	unsigned long cycles = 0;
	int ret = 0;
	int i;

	/* Warm-up run. */
	for (i = 0; i < 4; i++) {
		if (enc)
121
			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
122
		else
123
			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
124 125 126 127 128 129 130 131 132 133 134

		if (ret)
			goto out;
	}

	/* The real thing. */
	for (i = 0; i < 8; i++) {
		cycles_t start, end;

		start = get_cycles();
		if (enc)
135
			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
136
		else
137
			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
		end = get_cycles();

		if (ret)
			goto out;

		cycles += end - start;
	}

out:
	if (ret == 0)
		printk("1 operation in %lu cycles (%d bytes)\n",
		       (cycles + 4) / 8, blen);

	return ret;
}

154
static u32 block_sizes[] = { 16, 64, 256, 1024, 8192, 0 };
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
static u32 aead_sizes[] = { 16, 64, 256, 512, 1024, 2048, 4096, 8192, 0 };

#define XBUFSIZE 8
#define MAX_IVLEN 32

static int testmgr_alloc_buf(char *buf[XBUFSIZE])
{
	int i;

	for (i = 0; i < XBUFSIZE; i++) {
		buf[i] = (void *)__get_free_page(GFP_KERNEL);
		if (!buf[i])
			goto err_free_buf;
	}

	return 0;

err_free_buf:
	while (i-- > 0)
		free_page((unsigned long)buf[i]);

	return -ENOMEM;
}

static void testmgr_free_buf(char *buf[XBUFSIZE])
{
	int i;

	for (i = 0; i < XBUFSIZE; i++)
		free_page((unsigned long)buf[i]);
}

static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE],
			unsigned int buflen)
{
	int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE;
	int k, rem;

	if (np > XBUFSIZE) {
		rem = PAGE_SIZE;
		np = XBUFSIZE;
196 197
	} else {
		rem = buflen % PAGE_SIZE;
198
	}
199

200
	sg_init_table(sg, np + 1);
201 202
	np--;
	for (k = 0; k < np; k++)
203
		sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE);
204

205
	sg_set_buf(&sg[k + 1], xbuf[k], rem);
206 207
}

208
static void test_aead_speed(const char *algo, int enc, unsigned int secs,
209 210 211 212 213 214 215 216 217 218 219 220 221
			    struct aead_speed_template *template,
			    unsigned int tcount, u8 authsize,
			    unsigned int aad_size, u8 *keysize)
{
	unsigned int i, j;
	struct crypto_aead *tfm;
	int ret = -ENOMEM;
	const char *key;
	struct aead_request *req;
	struct scatterlist *sg;
	struct scatterlist *sgout;
	const char *e;
	void *assoc;
222
	char *iv;
223 224 225 226 227
	char *xbuf[XBUFSIZE];
	char *xoutbuf[XBUFSIZE];
	char *axbuf[XBUFSIZE];
	unsigned int *b_size;
	unsigned int iv_len;
228
	struct crypto_wait wait;
229

230 231 232 233
	iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
	if (!iv)
		return;

234 235
	if (aad_size >= PAGE_SIZE) {
		pr_err("associate data length (%u) too big\n", aad_size);
236
		goto out_noxbuf;
237 238
	}

239 240 241 242 243 244 245 246 247 248 249 250
	if (enc == ENCRYPT)
		e = "encryption";
	else
		e = "decryption";

	if (testmgr_alloc_buf(xbuf))
		goto out_noxbuf;
	if (testmgr_alloc_buf(axbuf))
		goto out_noaxbuf;
	if (testmgr_alloc_buf(xoutbuf))
		goto out_nooutbuf;

251
	sg = kmalloc(sizeof(*sg) * 9 * 2, GFP_KERNEL);
252 253
	if (!sg)
		goto out_nosg;
254
	sgout = &sg[9];
255

256
	tfm = crypto_alloc_aead(algo, 0, 0);
257 258 259 260

	if (IS_ERR(tfm)) {
		pr_err("alg: aead: Failed to load transform for %s: %ld\n", algo,
		       PTR_ERR(tfm));
261
		goto out_notfm;
262 263
	}

264
	crypto_init_wait(&wait);
265 266 267
	printk(KERN_INFO "\ntesting speed of %s (%s) %s\n", algo,
			get_driver_name(crypto_aead, tfm), e);

268 269 270 271
	req = aead_request_alloc(tfm, GFP_KERNEL);
	if (!req) {
		pr_err("alg: aead: Failed to allocate request for %s\n",
		       algo);
272
		goto out_noreq;
273 274
	}

275
	aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
276
				  crypto_req_done, &wait);
277

278 279 280 281 282
	i = 0;
	do {
		b_size = aead_sizes;
		do {
			assoc = axbuf[0];
283
			memset(assoc, 0xff, aad_size);
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303

			if ((*keysize + *b_size) > TVMEMSIZE * PAGE_SIZE) {
				pr_err("template (%u) too big for tvmem (%lu)\n",
				       *keysize + *b_size,
					TVMEMSIZE * PAGE_SIZE);
				goto out;
			}

			key = tvmem[0];
			for (j = 0; j < tcount; j++) {
				if (template[j].klen == *keysize) {
					key = template[j].key;
					break;
				}
			}
			ret = crypto_aead_setkey(tfm, key, *keysize);
			ret = crypto_aead_setauthsize(tfm, authsize);

			iv_len = crypto_aead_ivsize(tfm);
			if (iv_len)
304
				memset(iv, 0xff, iv_len);
305 306 307 308 309 310 311 312 313 314 315 316 317 318

			crypto_aead_clear_flags(tfm, ~0);
			printk(KERN_INFO "test %u (%d bit key, %d byte blocks): ",
					i, *keysize * 8, *b_size);


			memset(tvmem[0], 0xff, PAGE_SIZE);

			if (ret) {
				pr_err("setkey() failed flags=%x\n",
						crypto_aead_get_flags(tfm));
				goto out;
			}

319
			sg_init_aead(sg, xbuf,
320
				    *b_size + (enc ? 0 : authsize));
321

322
			sg_init_aead(sgout, xoutbuf,
323 324
				    *b_size + (enc ? authsize : 0));

325 326 327
			sg_set_buf(&sg[0], assoc, aad_size);
			sg_set_buf(&sgout[0], assoc, aad_size);

328 329 330
			aead_request_set_crypt(req, sg, sgout,
					       *b_size + (enc ? 0 : authsize),
					       iv);
331
			aead_request_set_ad(req, aad_size);
332

333 334 335
			if (secs)
				ret = test_aead_jiffies(req, enc, *b_size,
							secs);
336 337 338 339 340 341 342 343 344 345 346 347 348 349
			else
				ret = test_aead_cycles(req, enc, *b_size);

			if (ret) {
				pr_err("%s() failed return code=%d\n", e, ret);
				break;
			}
			b_size++;
			i++;
		} while (*b_size);
		keysize++;
	} while (*keysize);

out:
350 351
	aead_request_free(req);
out_noreq:
352
	crypto_free_aead(tfm);
353
out_notfm:
354 355 356 357 358 359 360 361
	kfree(sg);
out_nosg:
	testmgr_free_buf(xoutbuf);
out_nooutbuf:
	testmgr_free_buf(axbuf);
out_noaxbuf:
	testmgr_free_buf(xbuf);
out_noxbuf:
362
	kfree(iv);
363
}
364

365 366 367 368 369 370 371 372 373 374 375 376 377
static void test_hash_sg_init(struct scatterlist *sg)
{
	int i;

	sg_init_table(sg, TVMEMSIZE);
	for (i = 0; i < TVMEMSIZE; i++) {
		sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
		memset(tvmem[i], 0xff, PAGE_SIZE);
	}
}

static inline int do_one_ahash_op(struct ahash_request *req, int ret)
{
378
	struct crypto_wait *wait = req->base.data;
379

380
	return crypto_wait_req(ret, wait);
381 382
}

383 384 385 386
struct test_mb_ahash_data {
	struct scatterlist sg[TVMEMSIZE];
	char result[64];
	struct ahash_request *req;
387
	struct crypto_wait wait;
388 389
	char *xbuf[XBUFSIZE];
};
390 391

static void test_mb_ahash_speed(const char *algo, unsigned int sec,
392
				struct hash_speed *speed)
393
{
394
	struct test_mb_ahash_data *data;
395
	struct crypto_ahash *tfm;
396
	unsigned long start, end;
397
	unsigned long cycles;
398 399 400 401 402 403
	unsigned int i, j, k;
	int ret;

	data = kzalloc(sizeof(*data) * 8, GFP_KERNEL);
	if (!data)
		return;
404 405 406 407 408

	tfm = crypto_alloc_ahash(algo, 0, 0);
	if (IS_ERR(tfm)) {
		pr_err("failed to load transform for %s: %ld\n",
			algo, PTR_ERR(tfm));
409
		goto free_data;
410
	}
411

412
	for (i = 0; i < 8; ++i) {
413 414
		if (testmgr_alloc_buf(data[i].xbuf))
			goto out;
415

416
		crypto_init_wait(&data[i].wait);
417

418 419
		data[i].req = ahash_request_alloc(tfm, GFP_KERNEL);
		if (!data[i].req) {
420 421
			pr_err("alg: hash: Failed to allocate request for %s\n",
			       algo);
422
			goto out;
423 424
		}

425 426
		ahash_request_set_callback(data[i].req, 0, crypto_req_done,
					   &data[i].wait);
427
		test_hash_sg_init(data[i].sg);
428 429
	}

430 431
	pr_info("\ntesting speed of multibuffer %s (%s)\n", algo,
		get_driver_name(crypto_ahash, tfm));
432 433

	for (i = 0; speed[i].blen != 0; i++) {
434 435 436 437
		/* For some reason this only tests digests. */
		if (speed[i].blen != speed[i].plen)
			continue;

438
		if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
439 440 441
			pr_err("template (%u) too big for tvmem (%lu)\n",
			       speed[i].blen, TVMEMSIZE * PAGE_SIZE);
			goto out;
442 443 444 445 446
		}

		if (speed[i].klen)
			crypto_ahash_setkey(tfm, tvmem[0], speed[i].klen);

447 448 449
		for (k = 0; k < 8; k++)
			ahash_request_set_crypt(data[k].req, data[k].sg,
						data[k].result, speed[i].blen);
450

451 452
		pr_info("test%3u "
			"(%5u byte blocks,%5u bytes per update,%4u updates): ",
453 454 455
			i, speed[i].blen, speed[i].plen,
			speed[i].blen / speed[i].plen);

456 457 458 459
		start = get_cycles();

		for (k = 0; k < 8; k++) {
			ret = crypto_ahash_digest(data[k].req);
460 461
			if (ret == -EINPROGRESS) {
				ret = 0;
462
				continue;
463
			}
464 465 466 467

			if (ret)
				break;

468
			crypto_req_done(&data[k].req->base, 0);
469 470
		}

471
		for (j = 0; j < k; j++) {
472 473
			struct crypto_wait *wait = &data[j].wait;
			int wait_ret;
474

475 476 477
			wait_ret = crypto_wait_req(-EINPROGRESS, wait);
			if (wait_ret)
				ret = wait_ret;
478 479
		}

480 481 482 483 484 485 486 487 488
		end = get_cycles();
		cycles = end - start;
		pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
			cycles, cycles / (8 * speed[i].blen));

		if (ret) {
			pr_err("At least one hashing failed ret=%d\n", ret);
			break;
		}
489 490 491 492
	}

out:
	for (k = 0; k < 8; ++k)
493 494
		ahash_request_free(data[k].req);

495
	for (k = 0; k < 8; ++k)
496 497 498 499 500 501
		testmgr_free_buf(data[k].xbuf);

	crypto_free_ahash(tfm);

free_data:
	kfree(data);
502 503
}

504
static int test_ahash_jiffies_digest(struct ahash_request *req, int blen,
505
				     char *out, int secs)
506 507 508 509 510
{
	unsigned long start, end;
	int bcount;
	int ret;

511
	for (start = jiffies, end = start + secs * HZ, bcount = 0;
512 513 514 515 516 517 518
	     time_before(jiffies, end); bcount++) {
		ret = do_one_ahash_op(req, crypto_ahash_digest(req));
		if (ret)
			return ret;
	}

	printk("%6u opers/sec, %9lu bytes/sec\n",
519
	       bcount / secs, ((long)bcount * blen) / secs);
520 521 522 523 524

	return 0;
}

static int test_ahash_jiffies(struct ahash_request *req, int blen,
525
			      int plen, char *out, int secs)
526 527 528 529 530 531
{
	unsigned long start, end;
	int bcount, pcount;
	int ret;

	if (plen == blen)
532
		return test_ahash_jiffies_digest(req, blen, out, secs);
533

534
	for (start = jiffies, end = start + secs * HZ, bcount = 0;
535
	     time_before(jiffies, end); bcount++) {
536
		ret = do_one_ahash_op(req, crypto_ahash_init(req));
537 538 539 540 541 542 543 544 545 546 547 548 549 550
		if (ret)
			return ret;
		for (pcount = 0; pcount < blen; pcount += plen) {
			ret = do_one_ahash_op(req, crypto_ahash_update(req));
			if (ret)
				return ret;
		}
		/* we assume there is enough space in 'out' for the result */
		ret = do_one_ahash_op(req, crypto_ahash_final(req));
		if (ret)
			return ret;
	}

	pr_cont("%6u opers/sec, %9lu bytes/sec\n",
551
		bcount / secs, ((long)bcount * blen) / secs);
552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604

	return 0;
}

static int test_ahash_cycles_digest(struct ahash_request *req, int blen,
				    char *out)
{
	unsigned long cycles = 0;
	int ret, i;

	/* Warm-up run. */
	for (i = 0; i < 4; i++) {
		ret = do_one_ahash_op(req, crypto_ahash_digest(req));
		if (ret)
			goto out;
	}

	/* The real thing. */
	for (i = 0; i < 8; i++) {
		cycles_t start, end;

		start = get_cycles();

		ret = do_one_ahash_op(req, crypto_ahash_digest(req));
		if (ret)
			goto out;

		end = get_cycles();

		cycles += end - start;
	}

out:
	if (ret)
		return ret;

	pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
		cycles / 8, cycles / (8 * blen));

	return 0;
}

static int test_ahash_cycles(struct ahash_request *req, int blen,
			     int plen, char *out)
{
	unsigned long cycles = 0;
	int i, pcount, ret;

	if (plen == blen)
		return test_ahash_cycles_digest(req, blen, out);

	/* Warm-up run. */
	for (i = 0; i < 4; i++) {
605
		ret = do_one_ahash_op(req, crypto_ahash_init(req));
606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623
		if (ret)
			goto out;
		for (pcount = 0; pcount < blen; pcount += plen) {
			ret = do_one_ahash_op(req, crypto_ahash_update(req));
			if (ret)
				goto out;
		}
		ret = do_one_ahash_op(req, crypto_ahash_final(req));
		if (ret)
			goto out;
	}

	/* The real thing. */
	for (i = 0; i < 8; i++) {
		cycles_t start, end;

		start = get_cycles();

624
		ret = do_one_ahash_op(req, crypto_ahash_init(req));
625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650
		if (ret)
			goto out;
		for (pcount = 0; pcount < blen; pcount += plen) {
			ret = do_one_ahash_op(req, crypto_ahash_update(req));
			if (ret)
				goto out;
		}
		ret = do_one_ahash_op(req, crypto_ahash_final(req));
		if (ret)
			goto out;

		end = get_cycles();

		cycles += end - start;
	}

out:
	if (ret)
		return ret;

	pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
		cycles / 8, cycles / (8 * blen));

	return 0;
}

Herbert Xu's avatar
Herbert Xu committed
651 652
static void test_ahash_speed_common(const char *algo, unsigned int secs,
				    struct hash_speed *speed, unsigned mask)
653 654
{
	struct scatterlist sg[TVMEMSIZE];
655
	struct crypto_wait wait;
656 657
	struct ahash_request *req;
	struct crypto_ahash *tfm;
658
	char *output;
659 660
	int i, ret;

Herbert Xu's avatar
Herbert Xu committed
661
	tfm = crypto_alloc_ahash(algo, 0, mask);
662 663 664 665 666 667
	if (IS_ERR(tfm)) {
		pr_err("failed to load transform for %s: %ld\n",
		       algo, PTR_ERR(tfm));
		return;
	}

668 669 670
	printk(KERN_INFO "\ntesting speed of async %s (%s)\n", algo,
			get_driver_name(crypto_ahash, tfm));

671 672 673
	if (crypto_ahash_digestsize(tfm) > MAX_DIGEST_SIZE) {
		pr_err("digestsize(%u) > %d\n", crypto_ahash_digestsize(tfm),
		       MAX_DIGEST_SIZE);
674 675 676 677 678 679 680 681 682 683
		goto out;
	}

	test_hash_sg_init(sg);
	req = ahash_request_alloc(tfm, GFP_KERNEL);
	if (!req) {
		pr_err("ahash request allocation failure\n");
		goto out;
	}

684
	crypto_init_wait(&wait);
685
	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
686
				   crypto_req_done, &wait);
687

688 689 690 691
	output = kmalloc(MAX_DIGEST_SIZE, GFP_KERNEL);
	if (!output)
		goto out_nomem;

692 693 694 695 696 697 698 699 700 701 702 703 704
	for (i = 0; speed[i].blen != 0; i++) {
		if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
			pr_err("template (%u) too big for tvmem (%lu)\n",
			       speed[i].blen, TVMEMSIZE * PAGE_SIZE);
			break;
		}

		pr_info("test%3u "
			"(%5u byte blocks,%5u bytes per update,%4u updates): ",
			i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);

		ahash_request_set_crypt(req, sg, output, speed[i].plen);

705
		if (secs)
706
			ret = test_ahash_jiffies(req, speed[i].blen,
707
						 speed[i].plen, output, secs);
708 709 710 711 712 713 714 715 716 717
		else
			ret = test_ahash_cycles(req, speed[i].blen,
						speed[i].plen, output);

		if (ret) {
			pr_err("hashing failed ret=%d\n", ret);
			break;
		}
	}

718 719 720
	kfree(output);

out_nomem:
721 722 723 724 725 726
	ahash_request_free(req);

out:
	crypto_free_ahash(tfm);
}

Herbert Xu's avatar
Herbert Xu committed
727 728 729 730 731 732 733 734 735 736 737 738
static void test_ahash_speed(const char *algo, unsigned int secs,
			     struct hash_speed *speed)
{
	return test_ahash_speed_common(algo, secs, speed, 0);
}

static void test_hash_speed(const char *algo, unsigned int secs,
			    struct hash_speed *speed)
{
	return test_ahash_speed_common(algo, secs, speed, CRYPTO_ALG_ASYNC);
}

Herbert Xu's avatar
Herbert Xu committed
739
static inline int do_one_acipher_op(struct skcipher_request *req, int ret)
740
{
741
	struct crypto_wait *wait = req->base.data;
742

743
	return crypto_wait_req(ret, wait);
744 745
}

Herbert Xu's avatar
Herbert Xu committed
746
static int test_acipher_jiffies(struct skcipher_request *req, int enc,
747
				int blen, int secs)
748 749 750 751 752
{
	unsigned long start, end;
	int bcount;
	int ret;

753
	for (start = jiffies, end = start + secs * HZ, bcount = 0;
754 755 756
	     time_before(jiffies, end); bcount++) {
		if (enc)
			ret = do_one_acipher_op(req,
Herbert Xu's avatar
Herbert Xu committed
757
						crypto_skcipher_encrypt(req));
758 759
		else
			ret = do_one_acipher_op(req,
Herbert Xu's avatar
Herbert Xu committed
760
						crypto_skcipher_decrypt(req));
761 762 763 764 765 766

		if (ret)
			return ret;
	}

	pr_cont("%d operations in %d seconds (%ld bytes)\n",
767
		bcount, secs, (long)bcount * blen);
768 769 770
	return 0;
}

Herbert Xu's avatar
Herbert Xu committed
771
static int test_acipher_cycles(struct skcipher_request *req, int enc,
772 773 774 775 776 777 778 779 780 781
			       int blen)
{
	unsigned long cycles = 0;
	int ret = 0;
	int i;

	/* Warm-up run. */
	for (i = 0; i < 4; i++) {
		if (enc)
			ret = do_one_acipher_op(req,
Herbert Xu's avatar
Herbert Xu committed
782
						crypto_skcipher_encrypt(req));
783 784
		else
			ret = do_one_acipher_op(req,
Herbert Xu's avatar
Herbert Xu committed
785
						crypto_skcipher_decrypt(req));
786 787 788 789 790 791 792 793 794 795 796 797

		if (ret)
			goto out;
	}

	/* The real thing. */
	for (i = 0; i < 8; i++) {
		cycles_t start, end;

		start = get_cycles();
		if (enc)
			ret = do_one_acipher_op(req,
Herbert Xu's avatar
Herbert Xu committed
798
						crypto_skcipher_encrypt(req));
799 800
		else
			ret = do_one_acipher_op(req,
Herbert Xu's avatar
Herbert Xu committed
801
						crypto_skcipher_decrypt(req));
802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817
		end = get_cycles();

		if (ret)
			goto out;

		cycles += end - start;
	}

out:
	if (ret == 0)
		pr_cont("1 operation in %lu cycles (%d bytes)\n",
			(cycles + 4) / 8, blen);

	return ret;
}

Herbert Xu's avatar
Herbert Xu committed
818 819 820
static void test_skcipher_speed(const char *algo, int enc, unsigned int secs,
				struct cipher_speed_template *template,
				unsigned int tcount, u8 *keysize, bool async)
821
{
822
	unsigned int ret, i, j, k, iv_len;
823
	struct crypto_wait wait;
824 825
	const char *key;
	char iv[128];
Herbert Xu's avatar
Herbert Xu committed
826 827
	struct skcipher_request *req;
	struct crypto_skcipher *tfm;
828 829 830 831 832 833 834 835
	const char *e;
	u32 *b_size;

	if (enc == ENCRYPT)
		e = "encryption";
	else
		e = "decryption";

836
	crypto_init_wait(&wait);
837

Herbert Xu's avatar
Herbert Xu committed
838
	tfm = crypto_alloc_skcipher(algo, 0, async ? 0 : CRYPTO_ALG_ASYNC);
839 840 841 842 843 844 845

	if (IS_ERR(tfm)) {
		pr_err("failed to load transform for %s: %ld\n", algo,
		       PTR_ERR(tfm));
		return;
	}

846
	pr_info("\ntesting speed of async %s (%s) %s\n", algo,
Herbert Xu's avatar
Herbert Xu committed
847
			get_driver_name(crypto_skcipher, tfm), e);
848

Herbert Xu's avatar
Herbert Xu committed
849
	req = skcipher_request_alloc(tfm, GFP_KERNEL);
850 851 852 853 854 855
	if (!req) {
		pr_err("tcrypt: skcipher: Failed to allocate request for %s\n",
		       algo);
		goto out;
	}

Herbert Xu's avatar
Herbert Xu committed
856
	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
857
				      crypto_req_done, &wait);
858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886

	i = 0;
	do {
		b_size = block_sizes;

		do {
			struct scatterlist sg[TVMEMSIZE];

			if ((*keysize + *b_size) > TVMEMSIZE * PAGE_SIZE) {
				pr_err("template (%u) too big for "
				       "tvmem (%lu)\n", *keysize + *b_size,
				       TVMEMSIZE * PAGE_SIZE);
				goto out_free_req;
			}

			pr_info("test %u (%d bit key, %d byte blocks): ", i,
				*keysize * 8, *b_size);

			memset(tvmem[0], 0xff, PAGE_SIZE);

			/* set key, plain text and IV */
			key = tvmem[0];
			for (j = 0; j < tcount; j++) {
				if (template[j].klen == *keysize) {
					key = template[j].key;
					break;
				}
			}

Herbert Xu's avatar
Herbert Xu committed
887
			crypto_skcipher_clear_flags(tfm, ~0);
888

Herbert Xu's avatar
Herbert Xu committed
889
			ret = crypto_skcipher_setkey(tfm, key, *keysize);
890 891
			if (ret) {
				pr_err("setkey() failed flags=%x\n",
Herbert Xu's avatar
Herbert Xu committed
892
					crypto_skcipher_get_flags(tfm));
893 894 895
				goto out_free_req;
			}

896
			k = *keysize + *b_size;
897 898
			sg_init_table(sg, DIV_ROUND_UP(k, PAGE_SIZE));

899 900
			if (k > PAGE_SIZE) {
				sg_set_buf(sg, tvmem[0] + *keysize,
901
				   PAGE_SIZE - *keysize);
902 903 904 905 906 907 908 909 910 911 912 913
				k -= PAGE_SIZE;
				j = 1;
				while (k > PAGE_SIZE) {
					sg_set_buf(sg + j, tvmem[j], PAGE_SIZE);
					memset(tvmem[j], 0xff, PAGE_SIZE);
					j++;
					k -= PAGE_SIZE;
				}
				sg_set_buf(sg + j, tvmem[j], k);
				memset(tvmem[j], 0xff, k);
			} else {
				sg_set_buf(sg, tvmem[0] + *keysize, *b_size);
914 915
			}

Herbert Xu's avatar
Herbert Xu committed
916
			iv_len = crypto_skcipher_ivsize(tfm);
917 918 919
			if (iv_len)
				memset(&iv, 0xff, iv_len);

Herbert Xu's avatar
Herbert Xu committed
920
			skcipher_request_set_crypt(req, sg, sg, *b_size, iv);
921

922
			if (secs)
923
				ret = test_acipher_jiffies(req, enc,
924
							   *b_size, secs);
925 926 927 928 929 930
			else
				ret = test_acipher_cycles(req, enc,
							  *b_size);

			if (ret) {
				pr_err("%s() failed flags=%x\n", e,
Herbert Xu's avatar
Herbert Xu committed
931
				       crypto_skcipher_get_flags(tfm));
932 933 934 935 936 937 938 939 940
				break;
			}
			b_size++;
			i++;
		} while (*b_size);
		keysize++;
	} while (*keysize);

out_free_req:
Herbert Xu's avatar
Herbert Xu committed
941
	skcipher_request_free(req);
942
out:
Herbert Xu's avatar
Herbert Xu committed
943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959
	crypto_free_skcipher(tfm);
}

static void test_acipher_speed(const char *algo, int enc, unsigned int secs,
			       struct cipher_speed_template *template,
			       unsigned int tcount, u8 *keysize)
{
	return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
				   true);
}

static void test_cipher_speed(const char *algo, int enc, unsigned int secs,
			      struct cipher_speed_template *template,
			      unsigned int tcount, u8 *keysize)
{
	return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
				   false);
960 961
}

962
static void test_available(void)
Linus Torvalds's avatar
Linus Torvalds committed
963 964
{
	char **name = check;
965

Linus Torvalds's avatar
Linus Torvalds committed
966 967
	while (*name) {
		printk("alg %s ", *name);
968
		printk(crypto_has_alg(*name, 0, 0) ?
969
		       "found\n" : "not found\n");
Linus Torvalds's avatar
Linus Torvalds committed
970
		name++;
971
	}
Linus Torvalds's avatar
Linus Torvalds committed
972 973
}

974 975
static inline int tcrypt_test(const char *alg)
{
976 977
	int ret;

978 979
	pr_debug("testing %s\n", alg);

980 981 982 983 984
	ret = alg_test(alg, alg, 0, 0);
	/* non-fips algs return -EINVAL in fips mode */
	if (fips_enabled && ret == -EINVAL)
		ret = 0;
	return ret;
985 986
}

987
static int do_test(const char *alg, u32 type, u32 mask, int m)
988 989
{
	int i;
990
	int ret = 0;
991 992

	switch (m) {
Linus Torvalds's avatar
Linus Torvalds committed
993
	case 0:
994 995 996 997 998 999 1000
		if (alg) {
			if (!crypto_has_alg(alg, type,
					    mask ?: CRYPTO_ALG_TYPE_MASK))
				ret = -ENOENT;
			break;
		}

1001
		for (i = 1; i < 200; i++)
1002
			ret += do_test(NULL, 0, 0, i);
Linus Torvalds's avatar
Linus Torvalds committed
1003 1004 1005
		break;

	case 1:
1006
		ret += tcrypt_test("md5");
Linus Torvalds's avatar
Linus Torvalds committed
1007 1008 1009
		break;

	case 2:
1010
		ret += tcrypt_test("sha1");
Linus Torvalds's avatar
Linus Torvalds committed
1011 1012 1013
		break;

	case 3:
1014 1015
		ret += tcrypt_test("ecb(des)");
		ret += tcrypt_test("cbc(des)");
1016
		ret += tcrypt_test("ctr(des)");
Linus Torvalds's avatar
Linus Torvalds committed
1017 1018 1019
		break;

	case 4:
1020 1021
		ret += tcrypt_test("ecb(des3_ede)");
		ret += tcrypt_test("cbc(des3_ede)");
1022
		ret += tcrypt_test("ctr(des3_ede)");
Linus Torvalds's avatar
Linus Torvalds committed
1023 1024 1025
		break;

	case 5:
1026
		ret += tcrypt_test("md4");
Linus Torvalds's avatar
Linus Torvalds committed
1027
		break;
1028

Linus Torvalds's avatar
Linus Torvalds committed
1029
	case 6:
1030
		ret += tcrypt_test("sha256");
Linus Torvalds's avatar
Linus Torvalds committed
1031
		break;
1032

Linus Torvalds's avatar
Linus Torvalds committed
1033
	case 7:
1034 1035
		ret += tcrypt_test("ecb(blowfish)");
		ret += tcrypt_test("cbc(blowfish)");
1036
		ret += tcrypt_test("ctr(blowfish)");
Linus Torvalds's avatar
Linus Torvalds committed
1037 1038 1039
		break;

	case 8:
1040 1041
		ret += tcrypt_test("ecb(twofish)");
		ret += tcrypt_test("cbc(twofish)");
1042
		ret += tcrypt_test("ctr(twofish)");
1043
		ret += tcrypt_test("lrw(twofish)");
1044
		ret += tcrypt_test("xts(twofish)");
Linus Torvalds's avatar
Linus Torvalds committed
1045
		break;
1046

Linus Torvalds's avatar
Linus Torvalds committed
1047
	case 9:
1048
		ret += tcrypt_test("ecb(serpent)");
1049 1050
		ret += tcrypt_test("cbc(serpent)");
		ret += tcrypt_test("ctr(serpent)");
1051
		ret += tcrypt_test("lrw(serpent)");
1052
		ret += tcrypt_test("xts(serpent)");
Linus Torvalds's avatar
Linus Torvalds committed
1053 1054 1055
		break;

	case 10:
1056 1057 1058 1059 1060 1061
		ret += tcrypt_test("ecb(aes)");
		ret += tcrypt_test("cbc(aes)");
		ret += tcrypt_test("lrw(aes)");
		ret += tcrypt_test("xts(aes)");
		ret += tcrypt_test("ctr(aes)");
		ret += tcrypt_test("rfc3686(ctr(aes))");
Linus Torvalds's avatar
Linus Torvalds committed
1062 1063 1064
		break;

	case 11:
1065
		ret += tcrypt_test("sha384");
Linus Torvalds's avatar
Linus Torvalds committed
1066
		break;
1067

Linus Torvalds's avatar
Linus Torvalds committed
1068
	case 12:
1069
		ret += tcrypt_test("sha512");
Linus Torvalds's avatar
Linus Torvalds committed
1070 1071 1072
		break;

	case 13:
1073
		ret += tcrypt_test("deflate");
Linus Torvalds's avatar
Linus Torvalds committed
1074 1075 1076
		break;

	case 14:
1077
		ret += tcrypt_test("ecb(cast5)");
1078 1079
		ret += tcrypt_test("cbc(cast5)");
		ret += tcrypt_test("ctr(cast5)");
Linus Torvalds's avatar
Linus Torvalds committed
1080 1081 1082
		break;

	case 15:
1083
		ret += tcrypt_test("ecb(cast6)");
1084 1085 1086 1087
		ret += tcrypt_test("cbc(cast6)");
		ret += tcrypt_test("ctr(cast6)");
		ret += tcrypt_test("lrw(cast6)");
		ret += tcrypt_test("xts(cast6)");
Linus Torvalds's avatar
Linus Torvalds committed
1088 1089 1090
		break;

	case 16:
1091
		ret += tcrypt_test("ecb(arc4)");
Linus Torvalds's avatar
Linus Torvalds committed
1092 1093 1094
		break;

	case 17:
1095
		ret += tcrypt_test("michael_mic");
Linus Torvalds's avatar
Linus Torvalds committed
1096 1097 1098
		break;

	case 18:
1099
		ret += tcrypt_test("crc32c");
Linus Torvalds's avatar
Linus Torvalds committed
1100 1101 1102
		break;

	case 19:
1103
		ret += tcrypt_test("ecb(tea)");
Linus Torvalds's avatar
Linus Torvalds committed
1104 1105 1106
		break;

	case 20:
1107
		ret += tcrypt_test("ecb(xtea)");
Linus Torvalds's avatar
Linus Torvalds committed
1108 1109 1110
		break;

	case 21:
1111
		ret += tcrypt_test("ecb(khazad)");
Linus Torvalds's avatar
Linus Torvalds committed
1112 1113 1114
		break;

	case 22:
1115
		ret += tcrypt_test("wp512");
Linus Torvalds's avatar
Linus Torvalds committed
1116 1117 1118
		break;

	case 23:
1119
		ret += tcrypt_test("wp384");
Linus Torvalds's avatar
Linus Torvalds committed
1120 1121 1122
		break;

	case 24:
1123
		ret += tcrypt_test("wp256");
Linus Torvalds's avatar
Linus Torvalds committed
1124 1125 1126
		break;

	case 25:
1127
		ret += tcrypt_test("ecb(tnepres)");
Linus Torvalds's avatar
Linus Torvalds committed
1128 1129 1130
		break;

	case 26:
1131 1132
		ret += tcrypt_test("ecb(anubis)");
		ret += tcrypt_test("cbc(anubis)");
Linus Torvalds's avatar
Linus Torvalds committed
1133 1134 1135
		break;

	case 27:
1136
		ret += tcrypt_test("tgr192");
Linus Torvalds's avatar
Linus Torvalds committed
1137 1138 1139
		break;

	case 28:
1140
		ret += tcrypt_test("tgr160");
Linus Torvalds's avatar
Linus Torvalds committed
1141 1142 1143
		break;

	case 29:
1144
		ret += tcrypt_test("tgr128");
Linus Torvalds's avatar
Linus Torvalds committed
1145
		break;
1146

1147
	case 30:
1148
		ret += tcrypt_test("ecb(xeta)");
1149
		break;
Linus Torvalds's avatar
Linus Torvalds committed
1150

1151
	case 31:
1152
		ret += tcrypt_test("pcbc(fcrypt)");
1153 1154
		break;

1155
	case 32:
1156 1157
		ret += tcrypt_test("ecb(camellia)");
		ret += tcrypt_test("cbc(camellia)");
1158 1159 1160
		ret += tcrypt_test("ctr(camellia)");
		ret += tcrypt_test("lrw(camellia)");
		ret += tcrypt_test("xts(camellia)");
1161
		break;
1162

1163
	case 33:
1164
		ret += tcrypt_test("sha224");
1165
		break;
1166

1167
	case 34:
1168
		ret += tcrypt_test("salsa20");
1169 1170
		break;

1171
	case 35:
1172
		ret += tcrypt_test("gcm(aes)");
1173 1174
		break;

1175
	case 36:
1176
		ret += tcrypt_test("lzo");
1177 1178
		break;

1179
	case 37:
1180
		ret += tcrypt_test("ccm(aes)");
1181 1182
		break;

1183
	case 38:
1184
		ret += tcrypt_test("cts(cbc(aes))");
1185 1186
		break;

1187
        case 39:
1188
		ret += tcrypt_test("rmd128");
1189 1190 1191
		break;

        case 40:
1192
		ret += tcrypt_test("rmd160");
1193 1194
		break;

1195
	case 41:
1196
		ret += tcrypt_test("rmd256");
1197 1198 1199
		break;

	case 42:
1200
		ret += tcrypt_test("rmd320");
1201 1202 1203
		break;

	case 43:
1204
		ret += tcrypt_test("ecb(seed)");
1205 1206
		break;

1207
	case 44: