iscsi_tcp.c

linux 内核源代码

		if (tcp_conn->in.padding)
			tcp_conn->in_progress = IN_PROGRESS_PAD_RECV;
		else if (conn->datadgst_en)
			tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
		else
			tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
		tcp_conn->data_copied = 0;
	}

	if (tcp_conn->in_progress == IN_PROGRESS_PAD_RECV &&
	    tcp_conn->in.copy) {
		int copylen = min(tcp_conn->in.padding - tcp_conn->data_copied,
				  tcp_conn->in.copy);

		tcp_conn->in.copy -= copylen;
		tcp_conn->in.offset += copylen;
		tcp_conn->data_copied += copylen;

		if (tcp_conn->data_copied != tcp_conn->in.padding)
			tcp_conn->in_progress = IN_PROGRESS_PAD_RECV;
		else if (conn->datadgst_en)
			tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
		else
			tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
		tcp_conn->data_copied = 0;
	}

	debug_tcp("f, processed %d from out of %d padding %d\n",
	       tcp_conn->in.offset - offset, (int)len, tcp_conn->in.padding);
	BUG_ON(tcp_conn->in.offset - offset > len);

	if (tcp_conn->in.offset - offset != len) {
		debug_tcp("continue to process %d bytes\n",
		       (int)len - (tcp_conn->in.offset - offset));
		goto more;
	}

nomore:
	processed = tcp_conn->in.offset - offset;
	BUG_ON(processed == 0);
	return processed;

again:
	processed = tcp_conn->in.offset - offset;
	debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
	          processed, (int)len, (int)rd_desc->count);
	BUG_ON(processed == 0);
	BUG_ON(processed > len);

	conn->rxdata_octets += processed;
	return processed;
}

static void
iscsi_tcp_data_ready(struct sock *sk, int flag)
{
	struct iscsi_conn *conn = sk->sk_user_data;
	read_descriptor_t rd_desc;

	read_lock(&sk->sk_callback_lock);

	/*
	 * Use rd_desc to pass 'conn' to iscsi_tcp_data_recv.
	 * We set count to 1 because we want the network layer to
	 * hand us all the skbs that are available. iscsi_tcp_data_recv
	 * handled pdus that cross buffers or pdus that still need data.
	 */
	rd_desc.arg.data = conn;
	rd_desc.count = 1;
	tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);

	read_unlock(&sk->sk_callback_lock);
}

static void
iscsi_tcp_state_change(struct sock *sk)
{
	struct iscsi_tcp_conn *tcp_conn;
	struct iscsi_conn *conn;
	struct iscsi_session *session;
	void (*old_state_change)(struct sock *);

	read_lock(&sk->sk_callback_lock);

	conn = (struct iscsi_conn*)sk->sk_user_data;
	session = conn->session;

	if ((sk->sk_state == TCP_CLOSE_WAIT ||
	     sk->sk_state == TCP_CLOSE) &&
	    !atomic_read(&sk->sk_rmem_alloc)) {
		debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
		iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
	}

	tcp_conn = conn->dd_data;
	old_state_change = tcp_conn->old_state_change;

	read_unlock(&sk->sk_callback_lock);

	old_state_change(sk);
}

/**
 * iscsi_write_space - Called when more output buffer space is available
 * @sk: socket space is available for
 **/
static void
iscsi_write_space(struct sock *sk)
{
	struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;

	tcp_conn->old_write_space(sk);
	debug_tcp("iscsi_write_space: cid %d\n", conn->id);
	scsi_queue_work(conn->session->host, &conn->xmitwork);
}

static void
iscsi_conn_set_callbacks(struct iscsi_conn *conn)
{
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	struct sock *sk = tcp_conn->sock->sk;

	/* assign new callbacks */
	write_lock_bh(&sk->sk_callback_lock);
	sk->sk_user_data = conn;
	tcp_conn->old_data_ready = sk->sk_data_ready;
	tcp_conn->old_state_change = sk->sk_state_change;
	tcp_conn->old_write_space = sk->sk_write_space;
	sk->sk_data_ready = iscsi_tcp_data_ready;
	sk->sk_state_change = iscsi_tcp_state_change;
	sk->sk_write_space = iscsi_write_space;
	write_unlock_bh(&sk->sk_callback_lock);
}

static void
iscsi_conn_restore_callbacks(struct iscsi_tcp_conn *tcp_conn)
{
	struct sock *sk = tcp_conn->sock->sk;

	/* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
	write_lock_bh(&sk->sk_callback_lock);
	sk->sk_user_data    = NULL;
	sk->sk_data_ready   = tcp_conn->old_data_ready;
	sk->sk_state_change = tcp_conn->old_state_change;
	sk->sk_write_space  = tcp_conn->old_write_space;
	sk->sk_no_check	 = 0;
	write_unlock_bh(&sk->sk_callback_lock);
}

/**
 * iscsi_send - generic send routine
 * @sk: kernel's socket
 * @buf: buffer to write from
 * @size: actual size to write
 * @flags: socket's flags
 */
static inline int
iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
{
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	struct socket *sk = tcp_conn->sock;
	int offset = buf->sg.offset + buf->sent, res;

	/*
	 * if we got use_sg=0 or are sending something we kmallocd
	 * then we did not have to do kmap (kmap returns page_address)
	 *
	 * if we got use_sg > 0, but had to drop down, we do not
	 * set clustering so this should only happen for that
	 * slab case.
	 */
	if (buf->use_sendmsg)
		res = sock_no_sendpage(sk, sg_page(&buf->sg), offset, size, flags);
	else
		res = tcp_conn->sendpage(sk, sg_page(&buf->sg), offset, size, flags);

	if (res >= 0) {
		conn->txdata_octets += res;
		buf->sent += res;
		return res;
	}

	tcp_conn->sendpage_failures_cnt++;
	if (res == -EAGAIN)
		res = -ENOBUFS;
	else
		iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
	return res;
}

/**
 * iscsi_sendhdr - send PDU Header via tcp_sendpage()
 * @conn: iscsi connection
 * @buf: buffer to write from
 * @datalen: lenght of data to be sent after the header
 *
 * Notes:
 *	(Tx, Fast Path)
 **/
static inline int
iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
{
	int flags = 0; /* MSG_DONTWAIT; */
	int res, size;

	size = buf->sg.length - buf->sent;
	BUG_ON(buf->sent + size > buf->sg.length);
	if (buf->sent + size != buf->sg.length || datalen)
		flags |= MSG_MORE;

	res = iscsi_send(conn, buf, size, flags);
	debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
	if (res >= 0) {
		if (size != res)
			return -EAGAIN;
		return 0;
	}

	return res;
}

/**
 * iscsi_sendpage - send one page of iSCSI Data-Out.
 * @conn: iscsi connection
 * @buf: buffer to write from
 * @count: remaining data
 * @sent: number of bytes sent
 *
 * Notes:
 *	(Tx, Fast Path)
 **/
static inline int
iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
	       int *count, int *sent)
{
	int flags = 0; /* MSG_DONTWAIT; */
	int res, size;

	size = buf->sg.length - buf->sent;
	BUG_ON(buf->sent + size > buf->sg.length);
	if (size > *count)
		size = *count;
	if (buf->sent + size != buf->sg.length || *count != size)
		flags |= MSG_MORE;

	res = iscsi_send(conn, buf, size, flags);
	debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
		  size, buf->sent, *count, *sent, res);
	if (res >= 0) {
		*count -= res;
		*sent += res;
		if (size != res)
			return -EAGAIN;
		return 0;
	}

	return res;
}

static inline void
iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn,
		      struct iscsi_tcp_cmd_task *tcp_ctask)
{
	crypto_hash_init(&tcp_conn->tx_hash);
	tcp_ctask->digest_count = 4;
}

/**
 * iscsi_solicit_data_cont - initialize next Data-Out
 * @conn: iscsi connection
 * @ctask: scsi command task
 * @r2t: R2T info
 * @left: bytes left to transfer
 *
 * Notes:
 *	Initialize next Data-Out within this R2T sequence and continue
 *	to process next Scatter-Gather element(if any) of this SCSI command.
 *
 *	Called under connection lock.
 **/
static void
iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
			struct iscsi_r2t_info *r2t, int left)
{
	struct iscsi_data *hdr;
	int new_offset;

	hdr = &r2t->dtask.hdr;
	memset(hdr, 0, sizeof(struct iscsi_data));
	hdr->ttt = r2t->ttt;
	hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
	r2t->solicit_datasn++;
	hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
	memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
	hdr->itt = ctask->hdr->itt;
	hdr->exp_statsn = r2t->exp_statsn;
	new_offset = r2t->data_offset + r2t->sent;
	hdr->offset = cpu_to_be32(new_offset);
	if (left > conn->max_xmit_dlength) {
		hton24(hdr->dlength, conn->max_xmit_dlength);
		r2t->data_count = conn->max_xmit_dlength;
	} else {
		hton24(hdr->dlength, left);
		r2t->data_count = left;
		hdr->flags = ISCSI_FLAG_CMD_FINAL;
	}
	conn->dataout_pdus_cnt++;

	iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
			   sizeof(struct iscsi_hdr));

	if (iscsi_buf_left(&r2t->sendbuf))
		return;

	iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
	r2t->sg += 1;
}

static void iscsi_set_padding(struct iscsi_tcp_cmd_task *tcp_ctask,
			      unsigned long len)
{
	tcp_ctask->pad_count = len & (ISCSI_PAD_LEN - 1);
	if (!tcp_ctask->pad_count)
		return;

	tcp_ctask->pad_count = ISCSI_PAD_LEN - tcp_ctask->pad_count;
	debug_scsi("write padding %d bytes\n", tcp_ctask->pad_count);
	set_bit(XMSTATE_BIT_W_PAD, &tcp_ctask->xmstate);
}

/**
 * iscsi_tcp_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
 * @conn: iscsi connection
 * @ctask: scsi command task
 * @sc: scsi command
 **/
static void
iscsi_tcp_cmd_init(struct iscsi_cmd_task *ctask)
{
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;

	BUG_ON(__kfifo_len(tcp_ctask->r2tqueue));
	tcp_ctask->xmstate = 1 << XMSTATE_BIT_CMD_HDR_INIT;
}

/**
 * iscsi_tcp_mtask_xmit - xmit management(immediate) task
 * @conn: iscsi connection
 * @mtask: task management task
 *
 * Notes:
 *	The function can return -EAGAIN in which case caller must
 *	call it again later, or recover. '0' return code means successful
 *	xmit.
 *
 *	Management xmit state machine consists of these states:
 *		XMSTATE_BIT_IMM_HDR_INIT - calculate digest of PDU Header
 *		XMSTATE_BIT_IMM_HDR      - PDU Header xmit in progress
 *		XMSTATE_BIT_IMM_DATA     - PDU Data xmit in progress
 *		XMSTATE_VALUE_IDLE       - management PDU is done
 **/
static int
iscsi_tcp_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
{
	struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
	int rc;

	debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
		conn->id, tcp_mtask->xmstate, mtask->itt);

	if (test_bit(XMSTATE_BIT_IMM_HDR_INIT, &tcp_mtask->xmstate)) {
		iscsi_buf_init_iov(&tcp_mtask->headbuf, (char*)mtask->hdr,
				   sizeof(struct iscsi_hdr));

		if (mtask->data_count) {
			set_bit(XMSTATE_BIT_IMM_DATA, &tcp_mtask->xmstate);
			iscsi_buf_init_iov(&tcp_mtask->sendbuf,
					   (char*)mtask->data,
					   mtask->data_count);
		}

		if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
		    conn->stop_stage != STOP_CONN_RECOVER &&
		    conn->hdrdgst_en)
			iscsi_hdr_digest(conn, &tcp_mtask->headbuf,
					(u8*)tcp_mtask->hdrext);

		tcp_mtask->sent = 0;
		clear_bit(XMSTATE_BIT_IMM_HDR_INIT, &tcp_mtask->xmstate);
		set_bit(XMSTATE_BIT_IMM_HDR, &tcp_mtask->xmstate);
	}

	if (test_bit(XMSTATE_BIT_IMM_HDR, &tcp_mtask->xmstate)) {
		rc = iscsi_sendhdr(conn, &tcp_mtask->headbuf,
				   mtask->data_count);
		if (rc)
			return rc;
		clear_bit(XMSTATE_BIT_IMM_HDR, &tcp_mtask->xmstate);
	}

	if (test_and_clear_bit(XMSTATE_BIT_IMM_DATA, &tcp_mtask->xmstate)) {
		BUG_ON(!mtask->data_count);
		/* FIXME: implement.
		 * Virtual buffer could be spreaded across multiple pages...
		 */
		do {
			int rc;

			rc = iscsi_sendpage(conn, &tcp_mtask->sendbuf,
					&mtask->data_count, &tcp_mtask->sent);
			if (rc) {
				set_bit(XMSTATE_BIT_IMM_DATA, &tcp_mtask->xmstate);
				return rc;
			}
		} while (mtask->data_count);
	}

	BUG_ON(tcp_mtask->xmstate != XMSTATE_VALUE_IDLE);
	if (mtask->hdr->itt == RESERVED_ITT) {
		struct iscsi_session *session = conn->session;

		spin_lock_bh(&session->lock);
		list_del(&conn->mtask->running);
		__kfifo_put(session->mgmtpool.queue, (void*)&conn->mtask,
			    sizeof(void*));
		spin_unlock_bh(&session->lock);
	}
	return 0;
}

static int
iscsi_send_cmd_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
{
	struct scsi_cmnd *sc = ctask->sc;
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
	int rc = 0;

	if (test_bit(XMSTATE_BIT_CMD_HDR_INIT, &tcp_ctask->xmstate)) {
		tcp_ctask->sent = 0;
		tcp_ctask->sg_count = 0;
		tcp_ctask->exp_datasn = 0;

		if (sc->sc_data_direction == DMA_TO_DEVICE) {
			struct scatterlist *sg = scsi_sglist(sc);

			iscsi_buf_init_sg(&tcp_ctask->sendbuf, sg);
			tcp_ctask->sg = sg + 1;
			tcp_ctask->bad_sg = sg + scsi_sg_count(sc);

			debug_scsi("cmd [itt 0x%x total %d imm_data %d "
				   "unsol count %d, unsol offset %d]\n",
				   ctask->itt, scsi_bufflen(sc),
				   ctask->imm_count, ctask->unsol_count,
				   ctask->unsol_offset);
		}

		iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)ctask->hdr,
				  sizeof(struct iscsi_hdr));

		if (conn->hdrdgst_en)
			iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
					 (u8*)tcp_ctask->hdrext);
		clear_bit(XMSTATE_BIT_CMD_HDR_INIT, &tcp_ctask->xmstate);