fast_interrupt.s

C++ 编写的EROS RTOS

#ifdef COUNT_PHASES	
	incl	EXT(fast_ok);
#endif
	/*   This is a cheap hack: if receiver IS accepting a string,
	     then low word of %ECX must be nonzero.  If receiver is NOT
	     accepting a string, then value of low word of ECX is a
	     don't care.  In either case, a zero value in the low
	     word of %ECX means no string transmission.  Take
	     advantage of this! */
	
	/*   If recipient rcv length is zero, no need to worry: */
	cmpw	$0x0,92(%EDI)		/* recip ECX */
	jz	Done_String_Xmit
	
	/*   invTy::rcvSpec::sndSPec::keyData */
	/*   If recipient is not accepting a string, no need to worry */
	testl	$0x00f00000,84(%EDI)	/* recip EBX */
	jz	Done_String_Xmit

	/*   This is a cheap hack: if sender IS sending a string,
	     then low word of %EDX must be nonzero.  If sender is NOT
	     sending a string, then value of low word of EDX is a
	     don't care.  In either case, a zero value in the low
	     word of %EDX means no string transmission.  Take
	     advantage of this! */

	cmpw	$0x0,%dx
	jz	Done_Empty_String_Xmit
	
	/*   If DX nonzero, we still might not be sending a string */
	testl	$0x000f0000,%EBX
	jz	Done_Empty_String_Xmit


	/*   Sender is sending a string, and receiver is receiving it. 
	     Figure out who has shorter length, and put that value in 
	     in %ecx as xfer count: */
#if 0
	jmp	Standard_Gate_Path	/* UNTIL DEBUGGED */
#endif
	
	xorl	%ecx,%ecx
	movw	%dx,%cx
	cmpw	%cx,92(%EDI)
	jae	have_count
	
	movw	92(%EDI),%cx
have_count:

#ifdef DISPLAY
	movl	$0x8F438F48,0x000b8F98	/* "HC" */
#endif
#ifdef COUNT_PHASES	
	incl	EXT(move_string)
#endif
	
	/* recipient must have valid mapping table */
	cmpl	$0,64(%EDI)
	jz      Standard_Gate_Path

#ifdef NEWFANGLED_PATH
	/* bounds check the recipient string */
	movl	68(%EDI),%EDX			/* recip str ptr */
	leal	(%EDX,%ECX),%EBX		/* top of dest buffer */
	jb	Standard_Gate_Path	/* send str within kernel space */
	cmpl	$0xc0000000,%EBX
	jae	Standard_Gate_Path	/* send str within kernel space */

	/* For the send string portion of the test, we must first
	   validate that the legal bounds are not exceeded by
	   computing the end address and checking it's magnitude.  */

	addl	%ESI,%ECX
	jb	Standard_Gate_Path	/* send str within kernel space */
	cmpl	$0xc0000000,%ECX
	jae	Standard_Gate_Path	/* send str within kernel space */

	/* Attempt to make the paging protection checks work for us. You 
	   might think that the right strategy was to switch to the target 
	   space and test away, but it proves not.  Turns out that we will 
	   end up taking extra TLB misses if we do that, because after the 
	   string copy we still need to copy the keys.

	   Instead, we map the destination buffer into sender space, and 
	   probe that way. */

	/* Dest ptr now in %EDX and has valid range. Limit of send
	   str in %ECX. Limit of dest str in %EBX, but we're about to 
	   discard that. Compute address of dest ptr PDE: */
	
	shrl	$20,%edx
	andl	$0xffc,%edx
	addl	64(%EDI),%EDX		/* compute DEST PDE address */
	
	/* Figure out where to copy that PDE in sender mapping table */
	movl	64(%ESP),%EBX		/* sender mapping table pointer */
	
	/* Copy two PDE's to the kernel mapping PDE's */
	movl	(%EDX),%EAX		/* load PDE */
	orb	$0x20,%AL		/* suppress PDE accessed WB */
	movl	%EAX,0xff0(%EBX)	/* save it */
	movl	4(%EDX),%EAX		/* load 2nd PDE */
	orb	$0x20,%AL		/* suppress PDE accessed WB */
	movl	%EAX,0xff4(%EBX)	/* save it */

	/* Reload dest ptr */
	movl	68(%EDI),%EDX
	
	/* convert that to a kernel page address under the new mapping */
	andl	$0x003ff000,%EDX
	orl	$0x3F000000,%EDX

	movl	%ESI,%EAX
	/* Src ptr now in %EAX, dest in %EDX, Src limit in %ECX. Round
	   src and dest down to page boundaries */
	andl	$0xfffff000,%EAX

	/* We are about to validate the send and receive strings.
	   In so doing, we will bash almost all of the available
	   registers, and we will need to have the stack pointer
	   pointing someplace that permits us to take exceptions.
	   Stash it in %EBX for now. */

	movl	$1,EXT(TrapDepth)
	movl	$1,EXT(_3IRQ.DisableDepth)
	movl	%ESP,%EBX
	movl	$EXT(InterruptStackTop),%esp

#ifdef COUNT_PHASES	
	incl	EXT(src_ok)
#endif

validate_loop:
Fast_Send_Probe:
	cmpb	$0,0x40000000(%EAX)
	addl	$4096,%EAX
	invlpg	(%EDX)			/* invalidate dest TLB entry */
Fast_Rcv_Probe:
	andb	$0xff,(%EDX)
	addl	$4096,%EDX
	cmpl	%EAX,%ECX
	ja	validate_loop
	
#ifdef COUNT_PHASES	
	incl	EXT(dest_ok)
#endif
	/* now we know we have a valid copy. ESI still holds source 
	   string pointer.  Re-obtain %ECX: */
	subl	%ESI,%ECX
	movw	%CX,92(%EDI)		/* write rcv string length */
	/* Adjust %ESI for kernel mapping */
	addl	$0x40000000,%ESI
	
	/* Move recip ctxt ptr (EDI) to a safe place and load dest ptr */
	movl	%EDI,%EAX
	movl	68(%EDI),%EDI
	/* convert that to a kernel address under the new mapping */
	andl	$0x003fffff,%EDI
	orl	$0x3F000000,%EDI

	/* int	$0x33 */
	movl	%EBX,%ESP
#else
	/* For the send string portion of the test, we must first
	   validate that the legal bounds are not exceeded by
	   computing the end address and checking it's magnitude.  */

	movl	%ESI,%EAX
	addl	%ECX,%EAX
	jb	Standard_Gate_Path	/* send str within kernel space */
	cmpl	$0xc0000000,%EAX
	jae	Standard_Gate_Path	/* send str within kernel space */

	.globl	EXT(Fast_Send_Probe)

	/* Checking the various string parameters is going to smash
	   far more than we would like.  The send string validation
	   is the easier of the two:.

	   For your edification (and amusement), here's the register usage
	   convention for the send string probe:	

	   EBP:	pointer to invoked key -- we keep this in preference to
	        keeping the recipient context because we aren't done with
	        it yet.  Given this, we can reload EDI when the time comes.
	
           ESI:	initially the string base address, rounded down to nearest 
	        page boundary. used to increment through following loop.
	
           EAX:	pointer to end (exclusive) of sender string.

	   Note that we are careful not to blast ECX (the length), 
	   EDI (rcv context) and EBP (inv key ptr) during the send probe,
	   because we will need them during the receive probe. 

	   ESI already holds the send string pointer, so we don't need
	   to load that.

	   The send probe is less complex, because the page fault
	   handler deals with recovery when we need to.
	 */

	andl	$0xfffff000,%ESI	/* round down to page boundary */

	/* Arrange that if we page fault we land on correct stack */
	movl	$1,EXT(TrapDepth)
	movl	$1,EXT(_3IRQ.DisableDepth)
	
	movl	%esp,%ebx		/* save stack ptr */
	movl	$EXT(InterruptStackTop),%esp
	
LEXT(Fast_Send_Probe)
	cmpb	$0,0x40000000(%ESI)	/* probe this send page using
					   the corresponding address in
					   the KERNEL data segment */
	addl	$4096,%ESI
	cmpl	%ESI,%EAX		/* check termination */
	ja	Fast_Send_Probe

#ifdef DISPLAY
	movl	$0x8F538F53,0x000b8F98	/* "SS" */
#endif
#ifdef COUNT_PHASES	
	incl	EXT(src_ok)
#endif

	/* We need %EBX back, and won't fault from here down,
	   so restore %ESP from it. */

	movl	%EBX,%ESP

#if 0
	jmp	Standard_Gate_Path
#endif	
	
	/* So much for the easy part.  Now to probe the receive string.
	   This validation will smash essentially ALL of the registers.
	
	   For your edification (and amusement), here's the register usage
	   convention for the send string probe:	

	   EDI:	pointer to recipient context.
	
           ESI:	initially the receive string base address, rounded down to
		nearest page boundary. used to increment through following 
	        loop.

	   EAX:	KVA at which we are mapping current page -- used to zap 
	   stale TLB entries.  We either need to map sender or receiver
	   TLB entries, and it's easier to zap receiver entries.

	   EBX:	pointer to kernel page table entry into which we will map
	   the receive string mapping entries.  This iterates.

	   ECX:	pointer to end (exclusive) of receiver string.
	
	   EDX: pointer to page directory entry, page directory entry
	   contents, page table entry, page table entry content.

	   EBP: used as a late temporary in preference to EDI.  It is cheaper
	   to recompute proper EBP value than to lose EDI.
	
	   Note that we are careful not to blast ECX (the length, which is
	   currently a computed value that hasn't been saved anywhere) in a
	   way that is hard to recover -- we can recover it by subtracting
	   out the recipient start address, which we will need to reload
	   in any case.

	   We are also careful not to blast EBP, which is needed to reload
	   the recipient context.
	 */

	movl	68(%EDI),%ESI			/* recip str ptr */
	movl	$KVA_RCVBUF,%EAX		/* kern mapping addr */
	movl	EXT(_3PTE.kern_rcvbuf),%EBX	/* kern mapping pte */
	addl	%ESI,%ECX			/* end of string */
	jb	Standard_Gate_Path	/* send str within kernel space */
	cmpl	$0xc0000000,%ESI
	jae	Standard_Gate_Path	/* send str within kernel space */
	
	andl	$0xfffff000,%ESI		/* round dwn to pg bndry */

Fast_Rcv_Probe:	
	/* find page directory entry */
	movl	%ESI,%EDX
	shrl	$0x20,%EDX
	andl	$0xffc,%EDX			/* clean low bits */
	addl	64(%EDI),%EDX			/* compute PDE address */
	movl	(%EDX),%EDX			/* load PDE */
	/* watch out for large pages! */
	andb	$0x87,%DL			/* PS, U, W, P */
	cmpb	$0x7,%DL
	jne	Standard_Gate_Path

	andl	$0xfffff000,%EDX		/* extract page table addr */
	movl	%ESI,%EBP
	shrl	$10,%EBP
	andl	$0xffc,%EBP			/* clean low bits */
	addl	%EDX,%EBP			/* compute PTE address */
load_pte:	
	movl	(%EBP),%EDX			/* load PTE */
	
	andb	$0x7,%DL			/* U, W, P */
	cmpb	$0x7,%DL
	jne	Standard_Gate_Path

	orl	$0x6,(%EBP)			/* set dirty, accessed */
	movl	(%EBP),%EDX			/* reload -- we dorked it */
	movl	%EDX,(%EBX)			/* save to kpte */
	invlpg	(%EAX)				/* invalidate that mapping */
#if 0
	addl	$4096,%ESI			/* nxt rcv page */
	cmpl	%ESI,%ECX			/* done? */
	ja	do_string_move			/* no -- loop */
	addl	$4096,%EAX			/* next kern addr */
	addl	$4,%EBX				/* next kpte */
	jmp	Fast_Rcv_Probe			/* no -- loop */
#else
	addl	$4096,%ESI			/* nxt rcv page */
	cmpl	%ESI,%ECX			/* done? */
	jbe	do_string_move			/* yes -- quit */
	addl	$4096,%EAX			/* next kern addr */
	addl	$4,%EBX				/* next kpte */

	/* can often avoid recomputing PTE*, so try checking
	   for PTE overflow.  Bump the PTE* and check if the address
	   is congruent mod 4096 to zero. */
	addl	$4,%EBP
	testl	$0xfff,%EBP
	jnz	load_pte
	/* we overran - bite the bullet */	
	jmp	Fast_Rcv_Probe
#endif	

do_string_move:	
#ifdef COUNT_PHASES	
	incl	EXT(dest_ok)
#endif
#ifdef DISPLAY
	movl	$0x8F538F52,0x000b8F98	/* "RS" */
#endif
	/* We've now made a fine mess of just about everything, but
	   if we make it to this point there is nothing further that
	   can prevent the entire IPC from succeeding.  Go ahead and
	   transfer the string:
	 */

	subl	68(%EDI),%ECX		/* recover true length */
	movw	%CX,92(%EDI)		/* write rcv string length */

	movl	%EDI,%EAX		/* someplace we won't step on */
	movl	72(%ESP),%ESI		/* send str ptr */
	addl	$0x40000000,%ESI	/* kern seg adjust */
	movl	68(%EDI),%EDI		/* rcv str ptr */
	andl	$0xfff,%EDI
	addl	$KVA_RCVBUF,%EDI	/* rcv buf kern addr */
#endif

#if 0
	jmp	Standard_Gate_Path
#endif	
	
	/* BLOCK MOVE (YAY!) */

	addl	%ECX,EXT(totmov)
#if defined(FAST_BLOCK_MOVE) && 0
	/* always precache the first,last byte */
	cmpb	$0,(%EDI)
	cmpb	$0,0xffffffff(%EDI,%ECX)
#endif

	/* Special case if src, dest, and len align advantageously */
	cmp	$0x10,%ECX		/* if short move, test doesn't pay */
	jbe	byte_move
	
	movl	%ESI,%EDX
	orl	%EDI,%EDX
	orl	%ECX,%EDX
	testl	$0x3,%EDX
	jnz	byte_move

	shrl	$2,%ECX
#ifdef FAST_BLOCK_MOVE
	/* Pentium and family are NOT write allocate.  It is therefore
	   faster to run things by hand than it is to use movsl. On both
	   Pentium and later, this way proves to be just as fast anyway.
	   On 486 it doesn't lose by very much.  
	
	   Cache line size is 32 bytes or 8 words. Move things along until
	   we hit a cache line boundary */

#if 0
	testl	$0x1f,%EDI
	jz	move_with_prefetch

find_cache_line_start:
	movl	(%ESI),%EDX
	addl	$4,%ESI
	decl	%ECX
	movl	%EDX,(%EDI)
	addl	$4,%EDI
	
	testl	%ECX,%ECX
	jz	move_done

	testl	$0x1f,%EDI
	jnz	find_cache_line_start
#endif
	
	/* At or above 2048 bytes it's better to just do the block move */
	cmpl	$512,%ECX
	jae	finish_move
	
move_with_prefetch:
	/* there is more to move, but not necessarily a cache line full.
	   fetch the cache line and then do the escape test */
#if 0
	cmpb	$0,(%EDI)
#endif
	cmpl	$8,%ECX
	jl	finish_move

	/* there are at least 8 words to move -- move them by hand */
	movl	(%ESI),%EDX
	movl	4(%ESI),%EBX
	movl	%EDX,(%EDI)
	movl	%EBX,4(%EDI)
	movl	8(%ESI),%EDX
	movl	12(%ESI),%EBX
	movl	%EDX,8(%EDI)
	movl	%EBX,12(%EDI)
	movl	16(%ESI),%EDX
	movl	20(%ESI),%EBX
	movl	%EDX,16(%EDI)
	movl	%EBX,20(%EDI)
	movl	24(%ESI),%EDX
	movl	28(%ESI),%EBX
	movl	%EDX,24(%EDI)
	movl	%EBX,28(%EDI)

	subl	$8,%ECX
	addl	$32,%EDI
	addl	$32,%ESI
	jnz	move_with_prefetch
	
finish_move:
#endif
	cld
	rep
	movsl
	jmp move_done
	
byte_move:
	cld
	rep
	movsb

move_done:	
#ifdef COUNT_PHASES	
	incl	EXT(copy_ok)
#endif	
#ifdef DISPLAY
	movl	$0x8F508F43,0x000b8F98	/* "CP" */
#endif
#if 0
	jmp	Standard_Gate_Path
#endif	

	/* Begin cleanup -- reload essential registers */
	movl	76(%ESP),%EBP		/* send key ndx */
	movl	%EAX,%EDI		/* recover rcv cntxt ptr */
	movl	88(%ESP),%EDX		/* send keys */
	shll	$4,%EBP	        /* offset into key registers (mul * 16) */
	movl	84(%ESP),%EBX	/* send ctrl block */
	addl	36(%ESP),%EBP	/* add base address of gen keys node */