fasttrap_isa.c

Sun Solaris 10 中的 DTrace 组件的源代码。请参看: http://www.sun.com/software/solaris/observability.jsp

/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only.
 * See the file usr/src/LICENSING.NOTICE in this distribution or
 * http://www.opensolaris.org/license/ for details.
 */

#pragma ident	"@(#)fasttrap_isa.c	1.15	04/11/17 SMI"

#include <sys/fasttrap_isa.h>
#include <sys/fasttrap_impl.h>
#include <sys/dtrace.h>
#include <sys/dtrace_impl.h>
#include <sys/cmn_err.h>
#include <sys/frame.h>
#include <sys/stack.h>
#include <sys/sysmacros.h>
#include <sys/trap.h>

#include <v9/sys/machpcb.h>
#include <v9/sys/privregs.h>

/*
 * Lossless User-Land Tracing on SPARC
 * -----------------------------------
 *
 * The Basic Idea
 *
 * The most important design constraint is, of course, correct execution of
 * the user thread above all else. The next most important goal is rapid
 * execution. We combine execution of instructions in user-land with
 * emulation of certain instructions in the kernel to aim for complete
 * correctness and maximal performance.
 *
 * We take advantage of the split PC/NPC architecture to speed up logical
 * single-stepping; when we copy an instruction out to the scratch space in
 * the ulwp_t structure (held in the %g7 register on SPARC), we can
 * effectively single step by setting the PC to our scratch space and leaving
 * the NPC alone. This executes the replaced instruction and then continues
 * on without having to reenter the kernel as with single- stepping. The
 * obvious caveat is for instructions whose execution is PC dependant --
 * branches, call and link instructions (call and jmpl), and the rdpc
 * instruction. These instructions cannot be executed in the manner described
 * so they must be emulated in the kernel.
 *
 * Emulation for this small set of instructions if fairly simple; the most
 * difficult part being emulating branch conditions.
 *
 *
 * A Cache Heavy Portfolio
 *
 * It's important to note at this time that copying an instruction out to the
 * ulwp_t scratch space in user-land is rather complicated. SPARC has
 * separate data and instruction caches so any writes to the D$ (using a
 * store instruction for example) aren't necessarily reflected in the I$.
 * The flush instruction can be used to synchronize the two and must be used
 * for any self-modifying code, but the flush instruction only applies to the
 * primary address space (the absence of a flusha analogue to the flush
 * instruction that accepts an ASI argument is an obvious omission from SPARC
 * v9 where the notion of the alternate address space was introduced on
 * SPARC). To correctly copy out the instruction we must use a block store
 * that doesn't allocate in the D$ and ensures synchronization with the I$;
 * see dtrace_blksuword32() for the implementation  (this function uses
 * ASI_BLK_COMMIT_S to write a block through the secondary ASI in the manner
 * described). Refer to the UltraSPARC I/II manual for details on the
 * ASI_BLK_COMMIT_S ASI.
 *
 *
 * Return Subtleties
 *
 * When we're firing a return probe we need to expose the value returned by
 * the function being traced. Since the function can set the return value
 * in its last instruction, we need to fire the return probe only _after_
 * the effects of the instruction are apparent. For instructions that we
 * emulate, we can call dtrace_probe() after we've performed the emulation;
 * for instructions that we execute after we return to user-land, we set
 * %pc to the instruction we copied out (as described above) and set %npc
 * to a trap instruction stashed in the ulwp_t structure. After the traced
 * instruction is executed, the trap instruction returns control to the
 * kernel where we can fire the return probe.
 *
 * This need for a second trap in cases where we execute the traced
 * instruction makes it all the more important to emulate the most common
 * instructions to avoid the second trip in and out of the kernel.
 *
 *
 * Making it Fast
 *
 * Since copying out an instruction is neither simple nor inexpensive for the
 * CPU, we should attempt to avoid doing it in as many cases as possible.
 * Since function entry and return are usually the most interesting probe
 * sites, we attempt to tune the performance of the fasttrap provider around
 * instructions typically in those places.
 *
 * Looking at a bunch of functions in libraries and executables reveals that
 * most functions begin with either a save or a sethi (to setup a larger
 * argument to the save) and end with a restore or an or (in the case of leaf
 * functions). To try to improve performance, we emulate all of these
 * instructions in the kernel.
 *
 * The save and restore instructions are a little tricky since they perform
 * register window maniplulation. Rather than trying to tinker with the
 * register windows from the kernel, we emulate the implicit add that takes
 * place as part of those instructions and set the %pc to point to a simple
 * save or restore we've hidden in the ulwp_t structure. If we're in a return
 * probe so want to make it seem as though the tracepoint has been completely
 * executed we need to remember that we've pulled this trick with restore and
 * pull registers from the previous window (the one that we'll switch to once
 * the simple store instruction is executed) rather than the current one. This
 * is why in the case of emulating a restore we set the DTrace CPU flag
 * CPU_DTRACE_FAKERESTORE before calling dtrace_probe() for the return probes
 * (see fasttrap_return_common()).
 */

#define	OP(x)		((x) >> 30)
#define	OP2(x)		(((x) >> 22) & 0x07)
#define	OP3(x)		(((x) >> 19) & 0x3f)
#define	RCOND(x)	(((x) >> 25) & 0x07)
#define	COND(x)		(((x) >> 25) & 0x0f)
#define	A(x)		(((x) >> 29) & 0x01)
#define	I(x)		(((x) >> 13) & 0x01)
#define	RD(x)		(((x) >> 25) & 0x1f)
#define	RS1(x)		(((x) >> 14) & 0x1f)
#define	RS2(x)		(((x) >> 0) & 0x1f)
#define	CC(x)		(((x) >> 20) & 0x03)
#define	DISP16(x)	((((x) >> 6) & 0xc000) | ((x) & 0x3fff))
#define	DISP22(x)	((x) & 0x3fffff)
#define	DISP19(x)	((x) & 0x7ffff)
#define	DISP30(x)	((x) & 0x3fffffff)
#define	SW_TRAP(x)	((x) & 0x7f)

#define	OP3_OR		0x02
#define	OP3_RD		0x28
#define	OP3_JMPL	0x38
#define	OP3_RETURN	0x39
#define	OP3_TCC		0x3a
#define	OP3_SAVE	0x3c
#define	OP3_RESTORE	0x3d

#define	OP3_PREFETCH	0x2d
#define	OP3_CASA	0x3c
#define	OP3_PREFETCHA	0x3d
#define	OP3_CASXA	0x3e

#define	OP2_ILLTRAP	0x0
#define	OP2_BPcc	0x1
#define	OP2_Bicc	0x2
#define	OP2_BPr		0x3
#define	OP2_SETHI	0x4
#define	OP2_FBPfcc	0x5
#define	OP2_FBfcc	0x6

#define	R_G0		0
#define	R_O0		8
#define	R_SP		14
#define	R_I0		24
#define	R_I1		25
#define	R_I2		26
#define	R_I3		27

/*
 * Check the comment in fasttrap.h when changing these offsets or adding
 * new instructions.
 */
#define	FASTTRAP_OFF_SAVE	64
#define	FASTTRAP_OFF_RESTORE	68
#define	FASTTRAP_OFF_FTRET	72
#define	FASTTRAP_OFF_RETURN	76

#define	BREAKPOINT_INSTR	0x91d02001	/* ta 1 */

/*
 * Tunable to let users turn off the fancy save instruction optimization.
 * If a program is non-ABI compliant, there's a possibility that the save
 * instruction optimization could cause an error.
 */
int fasttrap_optimize_save = 1;

static uint64_t
fasttrap_anarg(struct regs *rp, int argno)
{
	uint64_t value;

	if (argno < 6)
		return ((&rp->r_o0)[argno]);

	if (curproc->p_model == DATAMODEL_NATIVE) {
		struct frame *fr = (struct frame *)(rp->r_sp + STACK_BIAS);

		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
		value = dtrace_fulword(&fr->fr_argd[argno]);
		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR |
		    CPU_DTRACE_BADALIGN);
	} else {
		struct frame32 *fr = (struct frame32 *)rp->r_sp;

		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
		value = dtrace_fuword32(&fr->fr_argd[argno]);
		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR |
		    CPU_DTRACE_BADALIGN);
	}

	return (value);
}

static ulong_t fasttrap_getreg(struct regs *, uint_t);
static void fasttrap_putreg(struct regs *, uint_t, ulong_t);

int
fasttrap_probe(struct regs *rp)
{
	dtrace_probe(fasttrap_probe_id,
	    rp->r_o0, rp->r_o1, rp->r_o2, rp->r_o3, rp->r_o4);

	rp->r_pc = rp->r_npc;
	rp->r_npc = rp->r_pc + 4;

	return (0);
}

static void
fasttrap_usdt_args(fasttrap_probe_t *probe, struct regs *rp, int argc,
    uintptr_t *argv)
{
	int i, x, cap = MIN(argc, probe->ftp_nargs);

	if (curproc->p_model == DATAMODEL_NATIVE) {
		struct frame *fr = (struct frame *)(rp->r_sp + STACK_BIAS);
		uintptr_t v;

		for (i = 0; i < cap; i++) {
			x = probe->ftp_argmap[i];

			if (x < 6)
				argv[i] = (&rp->r_o0)[x];
			else if (fasttrap_fulword(&fr->fr_argd[x], &v) != 0)
				argv[i] = 0;
		}

	} else {
		struct frame32 *fr = (struct frame32 *)rp->r_sp;
		uint32_t v;

		for (i = 0; i < cap; i++) {
			x = probe->ftp_argmap[i];

			if (x < 6)
				argv[i] = (&rp->r_o0)[x];
			else if (fasttrap_fuword32(&fr->fr_argd[x], &v) != 0)
				argv[i] = 0;
		}
	}

	for (; i < argc; i++) {
		argv[i] = 0;
	}
}

static void
fasttrap_return_common(struct regs *rp, uintptr_t pc, pid_t pid,
    uint_t fake_restore)
{
	fasttrap_tracepoint_t *tp;
	fasttrap_bucket_t *bucket;
	fasttrap_id_t *id;
	kmutex_t *pid_mtx;
	dtrace_icookie_t cookie;

	pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
	mutex_enter(pid_mtx);
	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];

	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
		if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
		    !tp->ftt_prov->ftp_defunct)
			break;
	}

	/*
	 * Don't sweat it if we can't find the tracepoint again; unlike
	 * when we're in fasttrap_pid_probe(), finding the tracepoint here
	 * is not essential to the correct execution of the process.
	 */
	if (tp == NULL || tp->ftt_retids == NULL) {
		mutex_exit(pid_mtx);
		return;
	}

	for (id = tp->ftt_retids; id != NULL; id = id->fti_next) {
		fasttrap_probe_t *probe = id->fti_probe;

		if (probe->ftp_type == DTFTP_POST_OFFSETS) {
			if (probe->ftp_argmap == NULL) {
				dtrace_probe(probe->ftp_id, rp->r_o0, rp->r_o1,
				    rp->r_o2, rp->r_o3, rp->r_o4);
			} else {
				uintptr_t t[5];

				fasttrap_usdt_args(probe, rp,
				    sizeof (t) / sizeof (t[0]), t);

				dtrace_probe(probe->ftp_id, t[0], t[1],
				    t[2], t[3], t[4]);
			}
			continue;
		}

		/*
		 * If this isn't a restore instruction then we must
		 * be in leaf content (non-leaf functions can't return
		 * without executing a restore). If we're in leaf context
		 * and the %npc is still within this function, then we
		 * must have misidentified a jmpl as a tail-call when it
		 * is, in fact, part of a jump table. It would be nice to
		 * remove this tracepoint, but this is neither the time
		 * nor the place.
		 */
		if (tp->ftt_type != FASTTRAP_T_RESTORE &&
		    rp->r_npc - probe->ftp_faddr < probe->ftp_fsize)
			continue;

		/*
		 * It's possible for a function to branch to the delay slot
		 * of an instruction that we've identified as a return site.
		 * We can dectect this spurious return probe activation by
		 * observing that in this case %npc will be %pc + 4 and %npc
		 * will be inside the current function (unless the user is
		 * doing _crazy_ instruction picking in which case there's
		 * very little we can do). The second check is important
		 * in case the last instructions of a function make a tail
		 * call to the function located immediately subsequent.
		 */
		if (rp->r_npc == rp->r_pc + 4 &&
		    rp->r_npc - probe->ftp_faddr < probe->ftp_fsize)
			continue;

		/*
		 * The first argument is the offset of return tracepoint
		 * in the function; the remaining arguments are the return
		 * values.
		 *
		 * If fake_restore is set, we need to pull the return values
		 * out of the %i's rather than the %o's -- a little trickier.
		 */
		if (!fake_restore) {
			dtrace_probe(probe->ftp_id, pc - probe->ftp_faddr,
			    rp->r_o0, rp->r_o1, rp->r_o2, rp->r_o3);
		} else {
			uintptr_t arg0 = fasttrap_getreg(rp, R_I0);
			uintptr_t arg1 = fasttrap_getreg(rp, R_I1);
			uintptr_t arg2 = fasttrap_getreg(rp, R_I2);
			uintptr_t arg3 = fasttrap_getreg(rp, R_I3);

			cookie = dtrace_interrupt_disable();
			DTRACE_CPUFLAG_SET(CPU_DTRACE_FAKERESTORE);
			dtrace_probe(probe->ftp_id, pc - probe->ftp_faddr,
			    arg0, arg1, arg2, arg3);
			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_FAKERESTORE);
			dtrace_interrupt_enable(cookie);
		}
	}

	mutex_exit(pid_mtx);
}

int
fasttrap_pid_probe(struct regs *rp)
{
	proc_t *p = curproc;
	fasttrap_tracepoint_t *tp, tp_local;
	fasttrap_id_t *id;
	pid_t pid;
	uintptr_t pc = rp->r_pc;
	uintptr_t npc = rp->r_npc;
	uintptr_t orig_pc = pc;
	fasttrap_bucket_t *bucket;
	kmutex_t *pid_mtx;
	uint_t fake_restore = 0;
	dtrace_icookie_t cookie;

	/*
	 * It's possible that a user (in a veritable orgy of bad planning)
	 * could redirect this thread's flow of control before it reached the
	 * return probe fasttrap. In this case we need to kill the process
	 * since it's in a unrecoverable state.
	 */
	if (curthread->t_dtrace_step) {
		ASSERT(curthread->t_dtrace_on);
		fasttrap_sigtrap(p, curthread, pc);
		return (0);
	}

	/*
	 * Clear all user tracing flags.
	 */
	curthread->t_dtrace_ft = 0;
	curthread->t_dtrace_pc = 0;
	curthread->t_dtrace_npc = 0;
	curthread->t_dtrace_scrpc = 0;
	curthread->t_dtrace_astpc = 0;

	/*
	 * Treat a child created by a call to vfork(2) as if it were its
	 * parent. We know that there's only one thread of control in such a
	 * process: this one.
	 */
	while (p->p_flag & SVFORK) {
		p = p->p_parent;
	}

	pid = p->p_pid;
	pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
	mutex_enter(pid_mtx);
	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];

	/*
	 * Lookup the tracepoint that the process just hit.
	 */
	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
		if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
		    !tp->ftt_prov->ftp_defunct)
			break;
	}

	/*
	 * If we couldn't find a matching tracepoint, either a tracepoint has
	 * been inserted without using the pid<pid> ioctl interface (see
	 * fasttrap_ioctl), or somehow we have mislaid this tracepoint.
	 */
	if (tp == NULL) {
		mutex_exit(pid_mtx);
		return (-1);
	}

	for (id = tp->ftt_ids; id != NULL; id = id->fti_next) {
		fasttrap_probe_t *probe = id->fti_probe;
		int isentry;
		/*
		 * We note that this was an entry probe to help ustack() find
		 * the first caller.
		 */
		if ((isentry = (probe->ftp_type == DTFTP_ENTRY)) != 0) {
			cookie = dtrace_interrupt_disable();
			DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY);
		}
		dtrace_probe(probe->ftp_id, rp->r_o0, rp->r_o1, rp->r_o2,
		    rp->r_o3, rp->r_o4);
		if (isentry) {
			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY);
			dtrace_interrupt_enable(cookie);
		}
	}

	/*
	 * We're about to do a bunch of work so we cache a local copy of
	 * the tracepoint to emulate the instruction, and then find the
	 * tracepoint again later if we need to light up any return probes.
	 */
	tp_local = *tp;
	mutex_exit(pid_mtx);
	tp = &tp_local;

	/*
	 * We emulate certain types of instructions do ensure correctness
	 * (in the case of position dependent instructions) or optimize
	 * common cases. The rest we have the thread execute back in user-
	 * land.
	 */
	switch (tp->ftt_type) {
	case FASTTRAP_T_SAVE:
	{
		int32_t imm;

		/*
		 * This an optimization to let us handle function entry
		 * probes more efficiently. Many functions begin with a save
		 * instruction that follows the pattern:
		 *	save	%sp, <imm>, %sp
		 *
		 * Meanwhile, we've stashed the instruction:
		 *	save	%g1, %g0, %sp
		 *
		 * off of %g7, so all we have to do is stick the right value
		 * into %g1 and reset %pc to point to the instruction we've
		 * cleverly hidden (%npc should not be touched).
		 */

		imm = tp->ftt_instr << 19;
		imm >>= 19;
		rp->r_g1 = rp->r_sp + imm;
		pc = rp->r_g7 + FASTTRAP_OFF_SAVE;
		break;
	}

	case FASTTRAP_T_RESTORE:
	{
		ulong_t value;
		uint_t rd;

		/*