cryptocop.c

linux内核源码

	spin_lock_irqsave(&cryptocop_sessions_lock, flags);
	sess = cryptocop_sessions;
	while (sess && (sess->sid != sid)){
		sess = sess->next;
	}
	spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);

	return sess;
}

static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
{
	struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;

	DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
	assert(sess != NULL);
	while (tc && tc->init.tid != tid){
		DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
		tc = tc->next;
	}
	DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
	return tc;
}



/* The AES s-transform matrix (s-box). */
static const u8 aes_sbox[256] = {
	99,  124, 119, 123, 242, 107, 111, 197, 48,  1,   103, 43,  254, 215, 171, 118,
	202, 130, 201, 125, 250, 89,  71,  240, 173, 212, 162, 175, 156, 164, 114, 192,
	183, 253, 147, 38,  54,  63,  247, 204, 52,  165, 229, 241, 113, 216, 49,  21,
	4,   199, 35,  195, 24,  150, 5,   154, 7,   18,  128, 226, 235, 39,  178, 117,
	9,   131, 44,  26,  27,  110, 90,  160, 82,  59,  214, 179, 41,  227, 47,  132,
	83,  209, 0,   237, 32,  252, 177, 91,  106, 203, 190, 57,  74,  76,  88,  207,
	208, 239, 170, 251, 67,  77,  51,  133, 69,  249, 2,   127, 80,  60,  159, 168,
	81,  163, 64,  143, 146, 157, 56,  245, 188, 182, 218, 33,  16,  255, 243, 210,
	205, 12,  19,  236, 95,  151, 68,  23,  196, 167, 126, 61,  100, 93,  25,  115,
	96,  129, 79,  220, 34,  42,  144, 136, 70,  238, 184, 20,  222, 94,  11,  219,
	224, 50,  58,  10,  73,  6,   36,  92,  194, 211, 172, 98,  145, 149, 228, 121,
	231, 200, 55,  109, 141, 213, 78,  169, 108, 86,  244, 234, 101, 122, 174, 8,
	186, 120, 37,  46,  28,  166, 180, 198, 232, 221, 116, 31,  75,  189, 139, 138,
	112, 62,  181, 102, 72,  3,   246, 14,  97,  53,  87,  185, 134, 193, 29,  158,
	225, 248, 152, 17,  105, 217, 142, 148, 155, 30,  135, 233, 206, 85,  40,  223,
	140, 161, 137, 13,  191, 230, 66,  104, 65,  153, 45,  15,  176, 84,  187, 22
};

/* AES has a 32 bit word round constants for each round in the
 * key schedule.  round_constant[i] is really Rcon[i+1] in FIPS187.
 */
static u32 round_constant[11] = {
	0x01000000, 0x02000000, 0x04000000, 0x08000000,
	0x10000000, 0x20000000, 0x40000000, 0x80000000,
	0x1B000000, 0x36000000, 0x6C000000
};

/* Apply the s-box to each of the four occtets in w. */
static u32 aes_ks_subword(const u32 w)
{
	u8 bytes[4];

	*(u32*)(&bytes[0]) = w;
	bytes[0] = aes_sbox[bytes[0]];
	bytes[1] = aes_sbox[bytes[1]];
	bytes[2] = aes_sbox[bytes[2]];
	bytes[3] = aes_sbox[bytes[3]];
	return *(u32*)(&bytes[0]);
}

/* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
 * (Note that AES words are 32 bit long)
 *
 * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
 * word temp
 * i = 0
 * while (i < Nk) {
 *   w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
 *   i = i + 1
 * }
 * i = Nk
 *
 * while (i < (Nb * (Nr + 1))) {
 *   temp = w[i - 1]
 *   if ((i mod Nk) == 0) {
 *     temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
 *   }
 *   else if ((Nk > 6) && ((i mod Nk) == 4)) {
 *     temp = SubWord(temp)
 *   }
 *   w[i] = w[i - Nk] xor temp
 * }
 * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
 * SubWord(t) applies the AES s-box individually to each octet
 * in a 32 bit word.
 *
 * For AES Nk can have the values 4, 6, and 8 (corresponding to
 * values for Nr of 10, 12, and 14).  Nb is always 4.
 *
 * To construct w[i], w[i - 1] and w[i - Nk] must be
 * available.  Consequently we must keep a state of the last Nk words
 * to be able to create the last round keys.
 */
static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength)
{
	u32 temp;
	u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
	u8  w_last_ix;
	int i;
	u8  nr, nk;

	switch (keylength){
	case 128:
		nk = 4;
		nr = 10;
		break;
	case 192:
		nk = 6;
		nr = 12;
		break;
	case 256:
		nk = 8;
		nr = 14;
		break;
	default:
		panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
	};

	/* Need to do host byte order correction here since key is byte oriented and the
	 * kx algorithm is word (u32) oriented. */
	for (i = 0; i < nk; i+=1) {
		w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
	}

	i = (int)nk;
	w_last_ix = i - 1;
	while (i < (4 * (nr + 2))) {
		temp = w_ring[w_last_ix];
		if (!(i % nk)) {
			/* RotWord(temp) */
			temp = (temp << 8) | (temp >> 24);
			temp = aes_ks_subword(temp);
			temp ^= round_constant[i/nk - 1];
		} else if ((nk > 6) && ((i % nk) == 4)) {
			temp = aes_ks_subword(temp);
		}
		w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
		temp ^= w_ring[w_last_ix];
		w_ring[w_last_ix] = temp;

		/* We need the round keys for round Nr+1 and Nr+2 (round key
		 * Nr+2 is the round key beyond the last one used when
		 * encrypting).  Rounds are numbered starting from 0, Nr=10
		 * implies 11 rounds are used in encryption/decryption.
		 */
		if (i >= (4 * nr)) {
			/* Need to do host byte order correction here, the key
			 * is byte oriented. */
			*(u32*)dec_key = cpu_to_be32(temp);
			dec_key += 4;
		}
		++i;
	}
}


/**** Job/operation management. ****/

int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
{
	return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
}

int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
{
	return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
}

int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
{
	return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
}

static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
{
	int                           ret;
	struct cryptocop_prio_job     *pj = NULL;
	unsigned long int             flags;

	DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));

	if (!operation || !operation->cb){
		DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
		return -EINVAL;
	}

	if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
		DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
		return ret;
	}
	assert(pj != NULL);

	spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
	list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
	spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);

	/* Make sure a job is running */
	cryptocop_start_job();
	return 0;
}

static void cryptocop_do_tasklet(unsigned long unused);
DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);

static void cryptocop_do_tasklet(unsigned long unused)
{
	struct list_head             *node;
	struct cryptocop_prio_job    *pj = NULL;
	unsigned long                flags;

	DEBUG(printk("cryptocop_do_tasklet: entering\n"));

	do {
		spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
		if (!list_empty(&cryptocop_completed_jobs)){
			node = cryptocop_completed_jobs.next;
			list_del(node);
			pj = list_entry(node, struct cryptocop_prio_job, node);
		} else {
			pj = NULL;
		}
		spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
		if (pj) {
			assert(pj->oper != NULL);

			/* Notify consumer of operation completeness. */
			DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));

			pj->oper->operation_status = 0; /* Job is completed. */
			pj->oper->cb(pj->oper, pj->oper->cb_data);
			delete_internal_operation(pj->iop);
			kfree(pj);
		}
	} while (pj != NULL);

	DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
}

static irqreturn_t
dma_done_interrupt(int irq, void *dev_id, struct pt_regs * regs)
{
	struct cryptocop_prio_job *done_job;
	reg_dma_rw_ack_intr ack_intr = {
		.data = 1,
	};

	REG_WR (dma, regi_dma9, rw_ack_intr, ack_intr);

	DEBUG(printk("cryptocop DMA done\n"));

	spin_lock(&running_job_lock);
	if (cryptocop_running_job == NULL){
		printk("stream co-processor got interrupt when not busy\n");
		spin_unlock(&running_job_lock);
		return IRQ_HANDLED;
	}
	done_job = cryptocop_running_job;
	cryptocop_running_job = NULL;
	spin_unlock(&running_job_lock);

	/* Start processing a job. */
	if (!spin_trylock(&cryptocop_process_lock)){
		DEBUG(printk("cryptocop irq handler, not starting a job\n"));
	} else {
		cryptocop_start_job();
		spin_unlock(&cryptocop_process_lock);
	}

	done_job->oper->operation_status = 0; /* Job is completed. */
	if (done_job->oper->fast_callback){
		/* This operation wants callback from interrupt. */
		done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
		delete_internal_operation(done_job->iop);
		kfree(done_job);
	} else {
		spin_lock(&cryptocop_completed_jobs_lock);
		list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
		spin_unlock(&cryptocop_completed_jobs_lock);
		tasklet_schedule(&cryptocop_tasklet);
	}

	DEBUG(printk("cryptocop leave irq handler\n"));
	return IRQ_HANDLED;
}


/* Setup interrupts and DMA channels. */
static int init_cryptocop(void)
{
	unsigned long          flags;
	reg_intr_vect_rw_mask  intr_mask;
	reg_dma_rw_cfg         dma_cfg = {.en = 1};
	reg_dma_rw_intr_mask   intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
	reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
	reg_strcop_rw_cfg      strcop_cfg = {
		.ipend = regk_strcop_little,
		.td1 = regk_strcop_e,
		.td2 = regk_strcop_d,
		.td3 = regk_strcop_e,
		.ignore_sync = 0,
		.en = 1
	};

	if (request_irq(DMA9_INTR_VECT, dma_done_interrupt, 0, "stream co-processor DMA", NULL)) panic("request_irq stream co-processor irq dma9");

	(void)crisv32_request_dma(8, "strcop", DMA_PANIC_ON_ERROR, 0, dma_strp);
	(void)crisv32_request_dma(9, "strcop", DMA_PANIC_ON_ERROR, 0, dma_strp);

	local_irq_save(flags);

	/* Reset and enable the cryptocop. */
	strcop_cfg.en = 0;
	REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
	strcop_cfg.en = 1;
	REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);

	/* Enable DMA9 interrupt */
	intr_mask = REG_RD(intr_vect, regi_irq, rw_mask);
	intr_mask.dma9 = 1;
	REG_WR(intr_vect, regi_irq, rw_mask, intr_mask);

	/* Enable DMAs. */
	REG_WR(dma, regi_dma9, rw_cfg, dma_cfg); /* input DMA */
	REG_WR(dma, regi_dma8, rw_cfg, dma_cfg); /* output DMA */

	/* Set up wordsize = 4 for DMAs. */
	DMA_WR_CMD (regi_dma8, regk_dma_set_w_size4);
	DMA_WR_CMD (regi_dma9, regk_dma_set_w_size4);

	/* Enable interrupts. */
	REG_WR(dma, regi_dma9, rw_intr_mask, intr_mask_in);

	/* Clear intr ack. */
	REG_WR(dma, regi_dma9, rw_ack_intr, ack_intr);

	local_irq_restore(flags);

	return 0;
}

/* Free used cryptocop hw resources (interrupt and DMA channels). */
static void release_cryptocop(void)
{
	unsigned long          flags;
	reg_intr_vect_rw_mask  intr_mask;
	reg_dma_rw_cfg         dma_cfg = {.en = 0};
	reg_dma_rw_intr_mask   intr_mask_in = {0};
	reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };

	local_irq_save(flags);

	/* Clear intr ack. */
	REG_WR(dma, regi_dma9, rw_ack_intr, ack_intr);

	/* Disable DMA9 interrupt */
	intr_mask = REG_RD(intr_vect, regi_irq, rw_mask);
	intr_mask.dma9 = 0;
	REG_WR(intr_vect, regi_irq, rw_mask, intr_mask);

	/* Disable DMAs. */
	REG_WR(dma, regi_dma9, rw_cfg, dma_cfg); /* input DMA */
	REG_WR(dma, regi_dma8, rw_cfg, dma_cfg); /* output DMA */

	/* Disable interrupts. */
	REG_WR(dma, regi_dma9, rw_intr_mask, intr_mask_in);

	local_irq_restore(flags);

	free_irq(DMA9_INTR_VECT, NULL);

	(void)crisv32_free_dma(8);
	(void)crisv32_free_dma(9);
}


/* Init job queue. */
static int cryptocop_job_queue_init(void)
{
	int i;

	INIT_LIST_HEAD(&cryptocop_completed_jobs);

	for (i = 0; i < cryptocop_prio_no_prios; i++){
		cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
		INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
	}
	return 0;
}


static void cryptocop_job_queue_close(void)
{
	struct list_head               *node, *tmp;
	struct cryptocop_prio_job      *pj = NULL;
	unsigned long int              process_flags, flags;
	int                            i;

	/* FIXME: This is as yet untested code. */

	/* Stop strcop from getting an operation