cryptocop.c

linux内核源码

	cdesc->dma_descr->wait = 1;
	cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed.  EOP is ok here since the padded unit is the only one active. */
	cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
	cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;

	cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
	*pad_desc = cdesc;

	return 0;

 error_cleanup:
	if (cdesc) free_cdesc(cdesc);
	return -1;
}


static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
{
	struct cryptocop_dma_desc  *key_desc = alloc_cdesc(alloc_flag);
	struct strcop_meta_out     mo = {0};

	DEBUG(printk("setup_key_dl_desc\n"));

	if (!key_desc) {
		DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
		return -ENOMEM;
	}

	/* Download key. */
	if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
		/* Precook the AES decrypt key. */
		if (!tc->tctx->dec_key_set){
			get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
			tc->tctx->dec_key_set = 1;
		}
		key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
		key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
	} else {
		key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
		key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
	}
	/* Setup metadata. */
	mo.dlkey = 1;
	switch (tc->tctx->init.keylen) {
	case 64:
		mo.decrypt = 0;
		mo.hashmode = 0;
		break;
	case 128:
		mo.decrypt = 0;
		mo.hashmode = 1;
		break;
	case 192:
		mo.decrypt = 1;
		mo.hashmode = 0;
		break;
	case 256:
		mo.decrypt = 1;
		mo.hashmode = 1;
		break;
	default:
		break;
	}
	mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
	key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);

	key_desc->dma_descr->out_eop = 1;
	key_desc->dma_descr->wait = 1;
	key_desc->dma_descr->intr = 0;

	*kd = key_desc;
	return 0;
}

static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
{
	struct cryptocop_dma_desc  *iv_desc = alloc_cdesc(alloc_flag);
	struct strcop_meta_out     mo = {0};

	DEBUG(printk("setup_cipher_iv_desc\n"));

	if (!iv_desc) {
		DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
		return -ENOMEM;
	}
	/* Download IV. */
	iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
	iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;

	/* Setup metadata. */
	mo.hashsel = mo.csumsel = src_none;
	mo.ciphsel = src_dma;
	mo.ciphconf = tc->ciph_conf;
	mo.cbcmode = tc->cbcmode;

	iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);

	iv_desc->dma_descr->out_eop = 0;
	iv_desc->dma_descr->wait = 1;
	iv_desc->dma_descr->intr = 0;

	*id = iv_desc;
	return 0;
}

/* Map the ouput length of the transform to operation output starting on the inject index. */
static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
{
	int                        err = 0;
	struct cryptocop_dma_desc  head = {0};
	struct cryptocop_dma_desc  *outdesc = &head;
	size_t                     iov_offset = 0;
	size_t                     out_ix = 0;
	int                        outiov_ix = 0;
	struct strcop_meta_in      mi = {0};

	size_t                     out_length = tc->produced;
	int                        rem_length;
	int                        dlength;

	assert(out_length != 0);
	if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
		DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
		return -EINVAL;
	}
	/* Traverse the out iovec until the result inject index is reached. */
	while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
		out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
		outiov_ix++;
	}
	if (outiov_ix >= operation->tfrm_op.outcount){
		DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
		return -EINVAL;
	}
	iov_offset = tc->tcfg->inject_ix - out_ix;
	mi.dmasel = tc->unit_no;

	/* Setup the output descriptors. */
	while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
		outdesc->next = alloc_cdesc(alloc_flag);
		if (!outdesc->next) {
			DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
			err = -ENOMEM;
			goto error_cleanup;
		}
		outdesc = outdesc->next;
		rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
		dlength = (out_length < rem_length) ? out_length : rem_length;

		DEBUG(printk("create_input_descriptors:\n"
			     "outiov_ix=%d, rem_length=%d, dlength=%d\n"
			     "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
			     "outcount=%d, outiov_ix=%d\n",
			     outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));

		outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
		outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
		outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);

		out_length -= dlength;
		iov_offset += dlength;
		if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
			iov_offset = 0;
			++outiov_ix;
		}
	}
	if (out_length > 0){
		DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
		err = -EINVAL;
		goto error_cleanup;
	}
	/* Set sync in last descriptor. */
	mi.sync = 1;
	outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);

	*id = head.next;
	return 0;

 error_cleanup:
	while (head.next) {
		outdesc = head.next->next;
		free_cdesc(head.next);
		head.next = outdesc;
	}
	return err;
}


static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
{
	while (desc_len != 0) {
		struct cryptocop_dma_desc  *cdesc;
		int                        rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
		int                        dlength = (desc_len < rem_length) ? desc_len : rem_length;

		cdesc = alloc_cdesc(alloc_flag);
		if (!cdesc) {
			DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
			return -ENOMEM;
		}
		(*current_out_cdesc)->next = cdesc;
		(*current_out_cdesc) = cdesc;

		cdesc->free_buf = NULL;

		cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
		cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;

		desc_len -= dlength;
		*iniov_offset += dlength;
		assert(desc_len >= 0);
		if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
			*iniov_offset = 0;
			++(*iniov_ix);
			if (*iniov_ix > operation->tfrm_op.incount) {
				DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
				return  -EINVAL;
			}
		}
		cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
	} /* while (desc_len != 0) */
	/* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
	(*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */

	return 0;
}


static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
{
	DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
	if (tc->tcfg) {
		int                        failed = 0;
		struct cryptocop_dma_desc  *idescs = NULL;
		DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
		if (tc->pad_descs) {
			DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
			while (tc->pad_descs) {
				DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
				(*current_out_cdesc)->next = tc->pad_descs;
				tc->pad_descs = tc->pad_descs->next;
				(*current_out_cdesc) = (*current_out_cdesc)->next;
			}
		}

		/* Setup and append output descriptors to DMA in list. */
		if (tc->unit_no == src_dma){
			/* mem2mem.  Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
			struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
			unsigned int start_ix = tc->start_ix;
			while (start_ix){
				unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
				(*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
				if (!(*current_in_cdesc)->next){
					DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
					return -ENOMEM;
				}
				(*current_in_cdesc) = (*current_in_cdesc)->next;
				(*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
				(*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
				(*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
				start_ix -= desclen;
			}
			mi.sync = 1;
			(*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
		}

		failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
		if (failed){
			DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
			return failed;
		}
		DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
		while (idescs) {
			DEBUG(printk("append descriptor 0x%p\n", idescs));
			(*current_in_cdesc)->next = idescs;
			idescs = idescs->next;
			(*current_in_cdesc) = (*current_in_cdesc)->next;
		}
	}
	return 0;
}



static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
{
	struct cryptocop_session *sess;
	struct cryptocop_transform_ctx *tctx;

	struct cryptocop_tfrm_ctx digest_ctx = {
		.previous_src = src_none,
		.current_src = src_none,
		.start_ix = 0,
		.requires_padding = 1,
		.strict_block_length = 0,
		.hash_conf = 0,
		.hash_mode = 0,
		.ciph_conf = 0,
		.cbcmode = 0,
		.decrypt = 0,
		.consumed = 0,
		.produced = 0,
		.pad_descs = NULL,
		.active = 0,
		.done = 0,
		.prev_src = NULL,
		.curr_src = NULL,
		.tcfg = NULL};
	struct cryptocop_tfrm_ctx cipher_ctx = {
		.previous_src = src_none,
		.current_src = src_none,
		.start_ix = 0,
		.requires_padding = 0,
		.strict_block_length = 1,
		.hash_conf = 0,
		.hash_mode = 0,
		.ciph_conf = 0,
		.cbcmode = 0,
		.decrypt = 0,
		.consumed = 0,
		.produced = 0,
		.pad_descs = NULL,
		.active = 0,
		.done = 0,
		.prev_src = NULL,
		.curr_src = NULL,
		.tcfg = NULL};
	struct cryptocop_tfrm_ctx csum_ctx = {
		.previous_src = src_none,
		.current_src = src_none,
		.start_ix = 0,
		.blocklength = 2,
		.requires_padding = 1,
		.strict_block_length = 0,
		.hash_conf = 0,
		.hash_mode = 0,
		.ciph_conf = 0,
		.cbcmode = 0,
		.decrypt = 0,
		.consumed = 0,
		.produced = 0,
		.pad_descs = NULL,
		.active = 0,
		.done = 0,
		.tcfg = NULL,
		.prev_src = NULL,
		.curr_src = NULL,
		.unit_no = src_csum};
	struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;

	unsigned int indata_ix = 0;

	/* iovec accounting. */
	int iniov_ix = 0;
	int iniov_offset = 0;

	/* Operation descriptor cfg traversal pointer. */
	struct cryptocop_desc *odsc;

	int failed = 0;
	/* List heads for allocated descriptors. */
	struct cryptocop_dma_desc out_cdesc_head = {0};
	struct cryptocop_dma_desc in_cdesc_head = {0};

	struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
	struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;

	struct cryptocop_tfrm_ctx *output_tc = NULL;
	void                      *iop_alloc_ptr;

	assert(operation != NULL);
	assert(int_op != NULL);

	DEBUG(printk("cryptocop_setup_dma_list: start\n"));
	DEBUG(print_cryptocop_operation(operation));

	sess = get_session(operation->sid);
	if (!sess) {
		DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
		failed = -EINVAL;
		goto error_cleanup;
	}
	iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
	if (!iop_alloc_ptr) {
		DEBUG_API(printk("cryptocop_setup_dma_list:  kmalloc cryptocop_int_operation\n"));
		failed = -ENOMEM;
		goto error_cleanup;
	}
	(*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
	DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
	(*int_op)->alloc_ptr = iop_alloc_ptr;
	DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));

	(*int_op)->sid = operation->sid;
	(*int_op)->cdesc_out = NULL;
	(*int_op)->cdesc_in = NULL;
	(*int_op)->tdes_mode = cryptocop_3des_ede;
	(*int_op)->csum_mode = cryptocop_csum_le;
	(*int_op)->ddesc_out = NULL;
	(*int_op)->ddesc_in = NULL;

	/* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
	if (!tcfg) {
		DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
		failed = -EINVAL;
		goto error_cleanup;
	}
	while (tcfg) {
		tctx = get_transform_ctx(sess, tcfg->tid);