smpktlib.c

VxWorks操作系统内核源代码

/* smPktLib.c - VxWorks shared packet protocol library */

/* Copyright 1984-2002 Wind River Systems, Inc. */

/*
modification history
--------------------
02m,03may02,mas  cache flush and volatile fix (SPR 68334); bridge flush fix
		 (SPR 68844)
02l,24oct01,mas  doc update (SPR 71149)
02k,09oct01,mas  smPktSend: added error checking (SPR 6004), limited interrupt
		 retries (SPR 25341), removed all FAST modifiers, code cleanup
02j,17oct00,sn   replaced NULL with 0 so htonl works
02i,21feb99,jdi  doc: listed errnos.
02h,26aug93,kdl  smPktBroadcast() now checks smPktSllPut() status before 
		 interrupting destination CPU  (SPR #2441).
02g,13nov92,jcf  removed potential bus error during attach.
02f,02aug92,kdl  Uncommented include of "copyright_wrs.h".
02e,24jul92,elh  Moved heartbeat from anchor to header.
02d,23jun92,elh  general cleanup and documentation.
02c,02jun92,elh  the tree shuffle
02b,27may92,elh	 Made completely independant, changed to use
		 offsets, general cleanup.
02a,14may92,pme  Split to keep only packet passing routines.
01e,20feb92,elh  Added in ntohl, and htonl for 960, Added USE_OFFSET.
		 Extracted and renamed OS specific calls & moved to
		 smUtilLib.  Also removed function pointers
		 shMemIntGenFunc and shMemHwTasFunc.
		 made TAS_CHECKS 10 TAS_TRIES 5000.
		 modified parameters passed to intGen function.
01d,10feb92,kdl+ Changed shMemDetach to flush input queue.
	    elh  Removed references to OK_WAS{EMPTY, NT_EMPTY}.
		 Made shMemSend return an ERROR if packet too large.
		 Changed shMemSend to return silently (no
		 interrupts generated) if sending to self.
		 Misc code review changes.
01c,05feb92,elh  ansified.
01b,27jan92,elh	 added masterCpu to shared memory anchor.
		 changed shMemBeat to take pAnchor as argument.
		 changed arguments to shMemIsAlive.
		 changed shMemSetup to probe memory.
		 changed copyright.
01a,15aug90,kdl	 written.
*/

/*
DESCRIPTION

This library contains routines which allow multiple processors
to communicate over a backplane using shared memory.  All
services for initializing and managing the shared memory are
provided.

Data is sent between CPU's via shared memory "packets".  A packet
is simply a data buffer (of configurable size) with a header
that allows it to be properly identified and manipulated as a node
in a linked list of packets.


SHARED MEMORY MASTER CPU

One CPU node acts as the shared memory master.  This CPU initializes
the shared memory area and sets up the shared memory anchor.  These
steps are performed by the master calling the smPktSetup() routine.
This routine should be called only by the shared memory master CPU,
and it should only be called once for a given shared memory region.
(It is, however, possible to maintain and use multiple separate
shared memory regions.)

Once smPktSetup() has completed successfully, there is little functional
difference between the master CPU and other CPU's using shared memory,
except that the master is responsible for maintaining the heartbeat in
the shared memory packet region header.


SHARED MEMORY ANCHOR

The shared memory anchor is a small data structure which is at
a predetermined location, to allow all CPU's using shared memory to
find it.  The shared memory anchor contains the base offset to the
actual shared memory region which will be used for inter-processor
message passing.  (This allows the master CPU to dynamically allocate
the shared memory region.)

The shared memory anchor does not have to be defined as part of the
shared memory region which will be used for message passing.  However,
it must be located in a similar address space; the address translation
constants which CPU boards use to convert local addresses to bus addresses
must apply equally to the anchor and the regular shared memory region.


ATTACHING TO SHARED MEMORY

Each CPU, master or non-master, which will use the shared memory region
must attach itself to the shared memory.  The shared memory region must
have already been initialized by the master CPU calling smPktSetup().

The first step in attaching to shared memory is for each CPU to allocate and
initialize a shared memory packet descriptor (SM_PKT_DESC).  This structure 
describes the individual CPU's attachment to the shared memory region 
and is used in all subsequent shared memory calls to identify which 
shared memory region is being used.  Since the shared memory descriptor 
is used only by the local CPU, it is not necessary for the descriptor 
itself to be located in shared memory.  In fact, it is preferable for 
the descriptor to be allocated from the CPU's local memory, since local 
memory is usually more efficient to access.

The shared memory packet descriptor is initialized by calling the 
smPktInit() routine.  This routine takes a number of parameters which 
specify the characteristics of this CPU and its access to shared memory.

After the shared memory packet descriptor has been initialized, the CPU may
attach itself to the shared memory region.  This is done by calling the
smPktAttach() routine.

When smPktAttach() is called, it checks that the shared memory anchor 
contains the special ready value and that the heartbeat is incrementing.
If either of these conditions is not met, the routine will check 
periodically until either the ready value and incrementing heartbeat 
are recognized or a time limit is reached.  For non-master CPU's, 
this limit may be set by changing the global variable, smAliveTimeout.  
The limit is expressed in seconds, with 600 seconds (10 minutes) the 
default.  The master CPU will only wait 5 beat periods for a recognized 
heartbeat, since it is the CPU responsible for initializing shared memory 
and incrementing the heartbeat and therefore should not have to wait.  
If the time limit is reached before a valid ready value and heartbeat 
is seen, ERROR is returned and errno is set to S_smPktLib_DOWN .

Once the CPU has attached itself to the shared memory region, it may
send packets to other CPU's or receive packets which have been sent to it.
(Attempts to perform any shared memory operations without first attaching
successfully will return ERROR, with errno set to S_smPktLib_NOT_ATTACHED .)


SENDING PACKETS

To send a packet to another CPU, an application task must first obtain
a shared memory packet from the pool of free packets.  This is
done by calling smPktFreeGet(), as follows:
\cs
	status = smPktFreeGet (pSmPktDesc, &pPkt);
\ce
In this example, <pSmPktDesc> is the address of this CPU's shared memory packet
descriptor.  If status value returned by smPktFreeGet is OK, the address
of the obtained packet will be placed in <pPkt>.  If the packet address is
NULL, no free packets were available.

Once a packet has been obtained, the application task must copy the data
to be sent into the "data" field of the packet structure.  The maximum
number of bytes which may be copied to the packet's data buffer is
<maxPktBytes>, as specified by the master CPU during smPktSetup.  This
limit may be determined by reading a field in the shared memory packet 
descriptor.

The application task may set the "type" field in the packet header (SM_PKT_HDR)
to indicate the specific type of packet being sent.  This field is not used
by smPktLib and may therefore be given any value.

To send the completed packet, the application task calls smPktSend(),
as follows:
\cs
	status = smPktSend (pSmPktDesc, destCpu, pPkt);
\ce
Here, <destCpu> is the number of the destination CPU, and <pPkt> is the
address of the packet to be sent.  If smPktSend() returns a status of OK,
the packet was successfully queued for the destination CPU.

If the destination CPU did not previously have any input packets queued to
it smPktSend() will call the user-provided routine, smUtilIntGen(), to 
interrupt the destination CPU to notify it that a 
packet is available.  See "Interrupts," below, for more information.

If the destination CPU is not attached, ERROR is returned and errno is set
to S_smPktLib_DEST_NOT_ATTACHED .  If the specified destination cpu
number is out of range (i.e. less than zero or greater than the maximum
specified during smPktSetup), ERROR is returned and errno is set to
S_smPktLib_INVALID_CPU_NUMBER .

If the first interrupt fails, a delay of one tick is made and another
interrupt is attempted.  This is repeated up to smPktMaxIntRetries attempts or
until the interrupt is successful.  If not successful, ERROR is returned
and errno is set to S_smPktLib_INCOMPLETE_BROADCAST so that the caller
does not try to remove the packet and place it on the free list.  Doing
so would put the packet in both an input list and the free list!

BROADCAST PACKETS

In some circumstances, it may be desirable to send the same data to
all CPU nodes using the shared memory region.  This may be accomplished
by using a special "broadcast" mode of smPktSend().  This option sends
a copy of the same packet data to each attached CPU (except the sender).

To send a broadcast message, a CPU must first obtain a free shared memory
packet using smPktFreeGet() as usual.  The data area of the packet
is then filled with the message to be sent, again the same as for sending
to a single destination.

Broadcast mode is indicated during smPktSend() by a special value of the
<destCpu> (destination CPU) parameter.  This parameter is set to
SM_BROADCAST, rather than a particular CPU number.

When a broadcast message is sent, a separate packet is obtained for each
attached CPU, the data from the original packet is copied to it, and
the packet is queued to a particular CPU.  Therefore, there must be
sufficient free packets to send one to each CPU.  Broadcast packets are
sent to destination CPU's in cpu-number order.

If there are not enough free packets to send a copy to each CPU,
as many as possible will be sent, but ERROR is returned and errno is
set to S_smPktLib_INCOMPLETE_BROADCAST .  (If there are insufficient
free packets, the original packet passed during smPktSend() will be
sent to a destination CPU, to provide as complete a broadcast as
possible.)

Broadcast packets are received in the same manner as any other packets.


RECEIVING PACKETS

Packets are received by calling the smPktRecv() routine.  This routine
will normally be called by an interrupt handler, in response to a
an interrupt generated by a remote CPU sending a packet.  The smPktRecv()
routine may also be called periodically, in a polling fashion, to check
for received packets in systems which do not use interrupts to notify
the receiving CPU.

To receive a packet, smPktRecv is called as follows:
\cs
	status = smPktRecv (pSmPktDesc, &pPkt);
\ce
If the returned status is OK, <pPkt> will contain either the address of
a received packet, or NULL if no input packets were available.  A returned
status of ERROR indicates that an error occurred while attempting to
obtain received packets.

A sending CPU will interrupt the destination CPU only if there
were no packets previously queued for the destination CPU.  It is
therefore important that an interrupt handler which receives packets
call smPktRecv again after each packet is received, to check for
additional packets.

After a packet has been received, it must be explicitly freed by calling
smPktFreePut().


DETACHING FROM SHARED MEMORY

The attachment of a CPU to shared memory may be ended by calling
smPktDetach().  This routine will mark the calling CPU as no longer
attached.  After this, other CPU's may no longer send packets to it.
The CPU may re-attach itself to the shared memory region by later
calling smPktAttach().  When re-attaching, the original shared
memory descriptor may be re-used if the CPU's configuration remains
the same, or new values may be specified via smPktInit().


INTERRUPTS

When a packet is sent to a CPU, there must be some method for that CPU
to be informed that an input packet is available.  The preferred method
is for the sending CPU to be able to interrupt the receiving CPU.  This
will be highly dependent on the specific hardware being used.  

Two types of interrupts are supported, mailbox interrupts and vme
bus interrupts.  Mailbox interrupts are the first preferred
method (SM_INT_MAILBOX), followed by vme bus interrupts (SM_INT_BUS).
If interrupts cannot be used, a polling scheme may be employed
(SM_INT_NONE), but this is much less efficient.

When a CPU initailizes its shared memory packet descriptor via the
smPktInit() call, it passes in an interrupt type as well as three
interrupt arguments.  This describes how the cpu wishes to be notified
of incomming packets.  The interrupt types recognized by this library
are listed in smLib.h.  These values may be obtained for any attached CPU by
calling smPktCpuInfoGet().

The default interrupt method for a particular target is defined by the
configuration parameters: SM_INT_TYPE, SM_INT_ARG1, SM_INT_ARG2, SM_INT_ARG3 .

When a CPU sends a packet to a destination CPU which did not previously
have any input packets queued to it, the sending CPU will 
interrupt the destination CPU. 

A handler routine which executes in response to such an interrupt
must call smPktRecv() to obtain the input packet.  Since the interrupt
is generated only for the first packet queued when the input list was
previously empty, it is important that the handler routine then call
smPktRecv additional times until no more packets are available.

If it is not possible to use interrupts to notify receiving CPU's, a 
polling method may be used.  The simplest method is for the recieving CPU to
repeatedly call smPktRecv() to check for input packets.  In this case,
no notification routine is used.


OBTAINING STATUS INFORMATION

Two routines are provided to obtain current status information about
the shared memory region and individual CPU's:

The smPktInfoGet() routine gets status information which applies to the
the shared memory region as a whole.  It takes as a parameter a pointer
to a special information structure (SM_PKT_INFO), which it fills before
returning.  

The smPktCpuInfoGet() routine obtains status information for a single
CPU.  The number of the CPU must be specified during the call to
smPktCpuInfoGet().  A CPU number of NONE (-1) indicates that information
on the calling CPU should be returned.  The routine takes as a parameter
a pointer to a special CPU information structure (SM_PKT_CPU_INFO), 
which it fills before returning.  

INTERNAL

This file runs under only vxWorks.  SunOS is no longer supported.

Pointers into shared memory are declared 'volatile' so that compiler
optimization does not disrupt ordering of I/O operations.

Reading of locking semaphores after updates to shared memory but before
release of the semaphores is to cause the flushing of any external bus
bridge read and write FIFOs.  Failure to do so can result in erroneous
reads of shared memory and subsequent deadlock conditions.
*/

#ifdef UNIX
#undef INET
#endif

/* includes */

#include "vxWorks.h"
#include "sysLib.h"
#include "taskLib.h"
#include "cacheLib.h"
#include "smPktLib.h"
#include "smLib.h"
#include "smUtilLib.h"

/* defines */

#define	SM_LOCK_TAKE(lockLocalAdrs,tasRoutine,numTries,pOldLvl) \
		(smLockTake (lockLocalAdrs, tasRoutine, numTries, pOldLvl))

#define	SM_LOCK_GIVE(lockLocalAdrs,tasClearRoutine, oldLvl) \
		(smLockGive (lockLocalAdrs, tasClearRoutine, oldLvl))

#define DEFAULT_TAS_TRIES	5000	/* default tries for test-and-set */

#ifndef SM_PKT_MAX_INT_RETRIES
# define SM_PKT_MAX_INT_RETRIES 5   /* max #retries of interrupt per pkt */
#endif

/* Globals */

int	smPktMemSizeDefault 	= DEFAULT_MEM_SIZE;	/* memory size */
int 	smPktMaxBytesDefault 	= DEFAULT_PKT_SIZE; 	/* max pkt size */
int 	smPktMaxInputDefault	= DEFAULT_PKTS_MAX;	/* max input pkts */
int 	smPktMaxCpusDefault     = DEFAULT_CPUS_MAX;	/* max num CPU's */
int	smPktTasTries 	  	= DEFAULT_TAS_TRIES;
				/* times to try test-and-set */
int     smPktMaxIntRetries      = SM_PKT_MAX_INT_RETRIES;
                                /* max #int retries to attempt per packet */

/* Forward References */

#ifdef __STDC__

LOCAL STATUS smPktSllGet
    (
    SM_SLL_LIST volatile * listLocalAdrs,	/* local addr of packet list */
    int                    baseAddr,		/* addr conversion constant */
    FUNCPTR                tasRoutine,		/* test-and-set routine addr */
    FUNCPTR                tasClearRoutine,	/* test-and-set routine addr */
    SM_SLL_NODE **	   pNodeLocalAdrs	/* location to put node addr */
    );

LOCAL STATUS smPktSllPut
    (
    SM_SLL_LIST volatile * listLocalAdrs,	/* local addr of packet list */
    int			   base,		/* base address */
    FUNCPTR		   tasRoutine,		/* test-and-set routine addr */
    FUNCPTR		   tasClearRoutine,	/* test-and-set routine addr */
    SM_SLL_NODE volatile * nodeLocalAdrs,	/* local addr of node */
    BOOL *		   pListWasEmpty	/* set to true if adding to
                                                 * empty list */
    );

LOCAL STATUS smPktBroadcast
    (
    SM_PKT_DESC *	pSmPktDesc,	/* sh mem pkt descriptor */
    volatile SM_PKT *	pPktOrig	/* ptr to original packet */
    );

#else	/* __STDC__ */

LOCAL STATUS smPktSllGet ();
LOCAL STATUS smPktSllPut ();
LOCAL STATUS smPktBroadcast ();