cryptocop.c

linux内核源码

		if (!tctx) {
			DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
			failed = -EINVAL;
			goto error_cleanup;
		}
		if (tcfg->inject_ix > operation->tfrm_op.outlen){
			DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
			failed = -EINVAL;
			goto error_cleanup;
		}
		switch (tctx->init.alg){
		case cryptocop_alg_mem2mem:
			if (cipher_ctx.tcfg != NULL){
				DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
				failed = -EINVAL;
				goto error_cleanup;
			}
			/* mem2mem is handled as a NULL cipher. */
			cipher_ctx.cbcmode = 0;
			cipher_ctx.decrypt = 0;
			cipher_ctx.blocklength = 1;
			cipher_ctx.ciph_conf = 0;
			cipher_ctx.unit_no = src_dma;
			cipher_ctx.tcfg = tcfg;
			cipher_ctx.tctx = tctx;
			break;
		case cryptocop_alg_des:
		case cryptocop_alg_3des:
		case cryptocop_alg_aes:
			/* cipher */
			if (cipher_ctx.tcfg != NULL){
				DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
				failed = -EINVAL;
				goto error_cleanup;
			}
			cipher_ctx.tcfg = tcfg;
			cipher_ctx.tctx = tctx;
			if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
				cipher_ctx.decrypt = 1;
			}
			switch (tctx->init.cipher_mode) {
			case cryptocop_cipher_mode_ecb:
				cipher_ctx.cbcmode = 0;
				break;
			case cryptocop_cipher_mode_cbc:
				cipher_ctx.cbcmode = 1;
				break;
			default:
				DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
				failed = -EINVAL;
				goto error_cleanup;
			}
			DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
			switch (tctx->init.alg){
			case cryptocop_alg_des:
				cipher_ctx.ciph_conf = 0;
				cipher_ctx.unit_no = src_des;
				cipher_ctx.blocklength = DES_BLOCK_LENGTH;
				break;
			case cryptocop_alg_3des:
				cipher_ctx.ciph_conf = 1;
				cipher_ctx.unit_no = src_des;
				cipher_ctx.blocklength = DES_BLOCK_LENGTH;
				break;
			case cryptocop_alg_aes:
				cipher_ctx.ciph_conf = 2;
				cipher_ctx.unit_no = src_aes;
				cipher_ctx.blocklength = AES_BLOCK_LENGTH;
				break;
			default:
				panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
			}
			(*int_op)->tdes_mode = tctx->init.tdes_mode;
			break;
		case cryptocop_alg_md5:
		case cryptocop_alg_sha1:
			/* digest */
			if (digest_ctx.tcfg != NULL){
				DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
				failed = -EINVAL;
				goto error_cleanup;
			}
			digest_ctx.tcfg = tcfg;
			digest_ctx.tctx = tctx;
			digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
			switch (tctx->init.alg){
			case cryptocop_alg_md5:
				digest_ctx.blocklength = MD5_BLOCK_LENGTH;
				digest_ctx.unit_no = src_md5;
				digest_ctx.hash_conf = 1; /* 1 => MD-5 */
				break;
			case cryptocop_alg_sha1:
				digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
				digest_ctx.unit_no = src_sha1;
				digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
				break;
			default:
				panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
			}
			break;
		case cryptocop_alg_csum:
			/* digest */
			if (csum_ctx.tcfg != NULL){
				DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
				failed = -EINVAL;
				goto error_cleanup;
			}
			(*int_op)->csum_mode = tctx->init.csum_mode;
			csum_ctx.tcfg = tcfg;
			csum_ctx.tctx = tctx;
			break;
		default:
			/* no algorithm. */
			DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
			failed = -EINVAL;
			goto error_cleanup;
		}
		tcfg = tcfg->next;
	}
	/* Download key if a cipher is used. */
	if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
		struct cryptocop_dma_desc  *key_desc = NULL;

		failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
		if (failed) {
			DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
			goto error_cleanup;
		}
		current_out_cdesc->next = key_desc;
		current_out_cdesc = key_desc;
		indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);

		/* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
		if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
			struct cryptocop_dma_desc  *iv_desc = NULL;

			DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));

			failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
			if (failed) {
				DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
				goto error_cleanup;
			}
			current_out_cdesc->next = iv_desc;
			current_out_cdesc = iv_desc;
			indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
		}
	}

	/* Process descriptors. */
	odsc = operation->tfrm_op.desc;
	while (odsc) {
		struct cryptocop_desc_cfg   *dcfg = odsc->cfg;
		struct strcop_meta_out      meta_out = {0};
		size_t                      desc_len = odsc->length;
		int                         active_count, eop_needed_count;

		output_tc = NULL;

		DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));

		while (dcfg) {
			struct cryptocop_tfrm_ctx  *tc = NULL;

			DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
			/* Get the local context for the transform and mark it as the output unit if it produces output. */
			if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
				tc = &digest_ctx;
			} else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
				tc = &cipher_ctx;
			} else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
				tc = &csum_ctx;
			}
			if (!tc) {
				DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
				failed = -EINVAL;
				goto error_cleanup;
			}
			if (tc->done) {
				DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
				failed = -EINVAL;
				goto error_cleanup;
			}
			if (!tc->active) {
				tc->start_ix = indata_ix;
				tc->active = 1;
			}

			tc->previous_src = tc->current_src;
			tc->prev_src = tc->curr_src;
			/* Map source unit id to DMA source config. */
			switch (dcfg->src){
			case cryptocop_source_dma:
				tc->current_src = src_dma;
				break;
			case cryptocop_source_des:
				tc->current_src = src_des;
				break;
			case cryptocop_source_3des:
				tc->current_src = src_des;
				break;
			case cryptocop_source_aes:
				tc->current_src = src_aes;
				break;
			case cryptocop_source_md5:
			case cryptocop_source_sha1:
			case cryptocop_source_csum:
			case cryptocop_source_none:
			default:
				/* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
				 */
				DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
				failed = -EINVAL;
				goto error_cleanup;
			}
			if (tc->current_src != src_dma) {
				/* Find the unit we are sourcing from. */
				if (digest_ctx.unit_no == tc->current_src){
					tc->curr_src = &digest_ctx;
				} else if (cipher_ctx.unit_no == tc->current_src){
					tc->curr_src = &cipher_ctx;
				} else if (csum_ctx.unit_no == tc->current_src){
					tc->curr_src = &csum_ctx;
				}
				if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
					DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
					failed = -EINVAL;
					goto error_cleanup;
				}
			} else {
				tc->curr_src = NULL;
			}

			/* Detect source switch. */
			DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
			if (tc->active && (tc->current_src != tc->previous_src)) {
				/* Only allow source switch when both the old source unit and the new one have
				 * no pending data to process (i.e. the consumed length must be a multiple of the
				 * transform blocklength). */
				/* Note: if the src == NULL we are actually sourcing from DMA out. */
				if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
				    ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
				{
					DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
					failed = -EINVAL;
					goto error_cleanup;
				}
			}
			/* Detect unit deactivation. */
			if (dcfg->last) {
				/* Length check of this is handled below. */
				tc->done = 1;
			}
			dcfg = dcfg->next;
		} /* while (dcfg) */
		DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));

		if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
			DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
			failed = -EINVAL;
			goto error_cleanup;
		}
		if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
			DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
			failed = -EINVAL;
			goto error_cleanup;
		}
		if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
			DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
			failed = -EINVAL;
			goto error_cleanup;
		}

		/* Update consumed and produced lengths.

		   The consumed length accounting here is actually cheating.  If a unit source from DMA (or any
		   other unit that process data in blocks of one octet) it is correct, but if it source from a
		   block processing unit, i.e. a cipher, it will be temporarily incorrect at some times.  However
		   since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
		   unit has processed an exact multiple of its block length the end result will be correct.
		   Beware that if the source change restriction change this code will need to be (much) reworked.
		*/
		DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));

		if (csum_ctx.active) {
			csum_ctx.consumed += desc_len;
			if (csum_ctx.done) {
				csum_ctx.produced = 2;
			}
			DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
		}
		if (digest_ctx.active) {
			digest_ctx.consumed += desc_len;
			if (digest_ctx.done) {
				if (digest_ctx.unit_no == src_md5) {
					digest_ctx.produced = MD5_STATE_LENGTH;
				} else {
					digest_ctx.produced = SHA1_STATE_LENGTH;
				}
			}
			DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
		}
		if (cipher_ctx.active) {
			/* Ciphers are allowed only to source from DMA out.  That is filtered above. */
			assert(cipher_ctx.current_src == src_dma);
			cipher_ctx.consumed += desc_len;
			cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
			if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
				cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
			}
			DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
		}

		/* Setup the DMA out descriptors. */
		/* Configure the metadata. */
		active_count = 0;
		eop_needed_count = 0;
		if (cipher_ctx.active) {
			++active_count;
			if (cipher_ctx.unit_no == src_dma){
				/* mem2mem */
				meta_out.ciphsel = src_none;
			} else {
				meta_out.ciphsel = cipher_ctx.current_src;
			}
			meta_out.ciphconf = cipher_ctx.ciph_conf;
			meta_out.cbcmode = cipher_ctx.cbcmode;
			meta_out.decrypt = cipher_ctx.decrypt;
			DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
			if (cipher_ctx.done) ++eop_needed_count;
		} else {
			meta_out.ciphsel = src_none;
		}

		if (digest_ctx.active) {
			++active_count;
			meta_out.hashsel = digest_ctx.current_src;
			meta_out.hashconf = digest_ctx.hash_conf;
			meta_out.hashmode = 0; /* Explicit mode is not used here. */
			DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
			if (digest_ctx.done) {
				assert(digest_ctx.pad_descs == NULL);
				failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
				if (failed) {
					DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
					goto error_cleanup;
				}
			}
		} else {
			meta_out.hashsel = src_none;
		}

		if (csum_ctx.active) {
			++active_count;
			meta_out.csumsel = csum_ctx.current_src;
			if (csum_ctx.done) {
				assert(csum_ctx.pad_descs == NULL);
				failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
				if (failed) {
					DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
					goto error_cleanup;
				}
			}
		} else {
			meta_out.csumsel = src_none;
		}
		DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
		/* Setup DMA out descriptors for the indata. */
		failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
		if (failed) {
			DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
			goto error_cleanup;
		}
		/* Setup out EOP.  If there are active units that are not done here they cannot get an EOP
		 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
		 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
		 */
		assert(active_count >= eop_needed_count);
		assert((eop_needed_count == 0) || (eop_needed_count == 1));
		if (eop_needed_count) {
			/* This means that the bulk operation (cipeher/m2m) is terminated. */
			if (active_count > 1) {
				/* Use zero length EOP descriptor. */
				struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
				struct strcop_meta_out    ed_mo = {0};
				if (!ed) {
					DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
					failed = -ENOMEM;
					goto error_cleanup;
				}

				assert(cipher_ctx.active && cipher_ctx.done);

				if (cipher_ctx.unit_no == src_dma){
					/* mem2mem */
					ed_mo.ciphsel = src_none;
				} else {
					ed_mo.ciphsel = cipher_ctx.current_src;
				}
				ed_mo.ciphconf = cipher_ctx.ciph_conf;
				ed_mo.cbcmode = cipher_ctx.cbcmode;
				ed_mo.decrypt = cipher_ctx.decrypt;

				ed->free_buf = NULL;
				ed->dma_descr->wait = 1;
				ed->dma_descr->out_eop = 1;

				ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
				ed->dma_descr->after = ed->dma_descr->buf;
				ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);