mom_mach.c

openPBS的开放源代码

/*
*         OpenPBS (Portable Batch System) v2.3 Software License
* 
* Copyright (c) 1999-2000 Veridian Information Solutions, Inc.
* All rights reserved.
* 
* ---------------------------------------------------------------------------
* For a license to use or redistribute the OpenPBS software under conditions
* other than those described below, or to purchase support for this software,
* please contact Veridian Systems, PBS Products Department ("Licensor") at:
* 
*    www.OpenPBS.org  [phone]                  [e-mail]
*                        877 902-4PBS (US toll-free)
* ---------------------------------------------------------------------------
* 
* This license covers use of the OpenPBS v2.3 software (the "Software") at
* your site or location, and, for certain users, redistribution of the
* Software to other sites and locations.  Use and redistribution of
* OpenPBS v2.3 in source and binary forms, with or without modification,
* are permitted provided that all of the following conditions are met.
* After December 31, 2001, only conditions 3-6 must be met:
* 
* 1. Commercial and/or non-commercial use of the Software is permitted
*    provided a current software registration is on file at www.OpenPBS.org.
*    If use of this software contributes to a publication, product, or
*    service, proper attribution must be given; see www.OpenPBS.org/credit.html
* 
* 2. Redistribution in any form is only permitted for non-commercial,
*    non-profit purposes.  There can be no charge for the Software or any
*    software incorporating the Software.  Further, there can be no
*    expectation of revenue generated as a consequence of redistributing
*    the Software.
* 
* 3. Any Redistribution of source code must retain the above copyright notice
*    and the acknowledgment contained in paragraph 6, this list of conditions
*    and the disclaimer contained in paragraph 7.
* 
* 4. Any Redistribution in binary form must reproduce the above copyright
*    notice and the acknowledgment contained in paragraph 6, this list of
*    conditions and the disclaimer contained in paragraph 7 in the
*    documentation and/or other materials provided with the distribution.
* 
* 5. Redistributions in any form must be accompanied by information on how to
*    obtain complete source code for the OpenPBS software and any
*    modifications and/or additions to the OpenPBS software.  The source code
*    must either be included in the distribution or be available for no more
*    than the cost of distribution plus a nominal fee, and all modifications
*    and additions to the Software must be freely redistributable by any party
*    (including Licensor) without restriction.
* 
* 6. All advertising materials mentioning features or use of the Software must
*    display the following acknowledgment:
* 
*     "This product includes software developed by NASA Ames Research Center,
*     Lawrence Livermore National Laboratory, and Veridian Information 
*     Solutions, Inc.
*     Visit www.OpenPBS.org for OpenPBS software support,
*     products, and information."
* 
* 7. DISCLAIMER OF WARRANTY
* 
* THIS SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND. ANY EXPRESS
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT
* ARE EXPRESSLY DISCLAIMED.
* 
* IN NO EVENT SHALL VERIDIAN CORPORATION, ITS AFFILIATED COMPANIES, OR THE
* U.S. GOVERNMENT OR ANY OF ITS AGENCIES BE LIABLE FOR ANY DIRECT OR INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
* 
* This license will be governed by the laws of the Commonwealth of Virginia,
* without reference to its choice of law rules.
*/
#define SGI_ZOMBIE_WRONG 1
#define COMPLEX_MEM_CALC 0

#include <pbs_config.h>

#include <assert.h>
#include <limits.h>
#include <stdio.h>
#include <unistd.h>
#include <dirent.h>
#include <fcntl.h>
#include <errno.h>
#include <strings.h>
#include <pwd.h>
#include <mntent.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/procfs.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/quota.h>
#include <sys/statfs.h>
#include <sys/sysmacros.h>
#include <sys/sysmp.h>
#include <sys/resource.h>
#if MOM_CHECKPOINT == 1
#include <ckpt.h>
#endif	/* MOM_CHECKPOINT */
#if	NODEMASK != 0
#include <sys/pmo.h>
#include <sys/syssgi.h>
#include <sys/nodemask.h>
#endif	/* NODEMASK */

#include "pbs_error.h"
#include "portability.h"
#include "list_link.h"
#include "server_limits.h"
#include "attribute.h"
#include "resource.h"
#include "job.h"
#include "log.h"
#include "mom_mach.h"
#include "resmon.h"
#include "../rm_dep.h"

/*
**	System dependent code to gather information for the resource
**	monitor for a Silicon Graphics (SGI) machine.
**
**	Resources known by this code:
**		cput		cpu time for a pid or session
**		mem		memory size for a pid or session in KB
**		resi		resident memory size for a pid or session in KB
**		sessions	list of sessions in the system
**		pids		list of pids in a session
**		nsessions	number of sessions in the system
**		nusers		number of users in the system
**		totmem		total memory size in KB
**		availmem	available memory size in KB
**		ncpus		number of cpus
**		physmem		physical memory size in KB
**		size		size of a file or filesystem in KB
**		idletime	seconds of idle time
**		walltime	wall clock time for a pid
**		loadave		current load average
**		quota		quota information (sizes in KB)
*/

static char ident[] = "@(#) sgi/$RCSfile: mom_mach.c,v $ $Revision: 2.12.4.6 $";

#ifndef TRUE
#define FALSE	0
#define TRUE	1
#endif	/* TRUE */

#if COMPLEX_MEM_CALC==1
static	char		procfs[] = "/proc";
static	char		procfmts[] = "/proc/%s";
#else
static	char		procfs[] = "/proc/pinfo";
static	char		procfmts[] = "/proc/pinfo/%s";
#endif /* COMPLEX_MEM_CALC */
static	DIR		*pdir = NULL;
static	int		pagesize;
static	int		kfd = -1;

#define	TBL_INC		200		/* initial proc table */
#define MAPNUM		512		/* max number of mem segs */

static	int		nproc = 0;
static	int		max_proc = 0;
struct	proc_info {			/* structure used to hold proc info */
	prpsinfo_t	info;
#if COMPLEX_MEM_CALC==1
	int		map_num;
	prmap_sgi_t	*map;
#endif /* COMPLEX_MEM_CALC */
};
static	struct	proc_info	*proc_array = NULL;
static  time_t			 sampletime;

extern	char	*ret_string;
time_t		wait_time = 10;
extern	char	extra_parm[];
extern	char	no_parm[];

extern	struct	pbs_err_to_txt	pbs_err_to_txt[];
extern	time_t			time_now;
extern	time_t			last_scan;

/*
** external functions and data
*/
extern	struct	config		*search A_((struct config *, char *));
extern	struct	rm_attribute	*momgetattr A_((char *));
extern	int			rm_errno;
extern	unsigned	int	reqnum;
extern	double	cputfactor;
extern	double	wallfactor;

/*
** local functions and data
*/
static char	*resi		A_((struct rm_attribute *attrib));
static char	*totmem		A_((struct rm_attribute *attrib));
static char	*availmem	A_((struct rm_attribute *attrib));
static char	*physmem	A_((struct rm_attribute *attrib));
static char	*ncpus		A_((struct rm_attribute *attrib));
static char	*walltime	A_((struct rm_attribute *attrib));
static char	*quota		A_((struct rm_attribute *attrib));

extern char	*loadave	A_((struct rm_attribute *attrib));
extern char	*nullproc	A_((struct rm_attribute *attrib));


/*
** local resource array
*/
struct	config	dependent_config[] = {
	{ "resi",	resi },
	{ "totmem",	totmem },
	{ "availmem",	availmem },
	{ "physmem",	physmem },
	{ "ncpus",	ncpus },
	{ "loadave",	loadave },
	{ "walltime",	walltime },
	{ "quota",	quota },
	{ NULL,		nullproc },
};
	
off_t	kern_addr[] = {
	-1,			/* KSYM_PHYS */
	-1,			/* KSYM_LOAD */
};

#define	KSYM_PHYS	0
#define	KSYM_LOAD	1

void
dep_initialize()
{
	int	 i;
	char	*id = "dep_initialize";
	static	char	mem[] = "/dev/kmem";

	pagesize = getpagesize();

	if ((pdir = opendir(procfs)) == NULL) {
		log_err(errno, id, "opendir");
		return;
	}

	kern_addr[KSYM_PHYS] = SEEKLIMIT & sysmp(MP_KERNADDR, MPKA_PHYSMEM);
	kern_addr[KSYM_LOAD] = SEEKLIMIT & sysmp(MP_KERNADDR, MPKA_AVENRUN);

	if ((kfd = open(mem, O_RDONLY)) == -1) {
		log_err(errno, id, mem);
		return;
	}

	/* insure /dev/kmem closed on exec */

	if ((i = fcntl(kfd,  F_GETFD)) == -1) {
		log_err(errno, id, "F_GETFD");
	}
	i |= FD_CLOEXEC;
	if (fcntl(kfd, F_SETFD, i) == -1) {
		log_err(errno, id, "F_SETFD");
	}

	return;
}

void
dep_cleanup()
{
	char	*id = "dep_cleanup";

	log_record(PBSEVENT_SYSTEM, 0, id, "dependent cleanup");
	if (pdir) {
		closedir(pdir);
		pdir = NULL;
	}
	if (kfd != -1)
		close(kfd);
}

/*
**	Don't need any periodic procsessing.
*/
void
end_proc()
{
	return;
}

/*
 * Internal size decoding routine.
 *
 *	Accepts a resource pointer and a pointer to the unsigned 64 bit integer
 *	to receive the decoded value.  It returns a PBS error code, and the
 *	decoded value in the unsigned long integer.
 *
 *		sizeof(word) = sizeof(int)
 */

static int getsize(pres, ret)
    resource		*pres;
    rlim64_t		*ret;
{
	rlim64_t	value;

	if (pres->rs_value.at_type != ATR_TYPE_SIZE)
		return (PBSE_ATTRTYPE);
	value = pres->rs_value.at_val.at_size.atsv_num;
	if (pres->rs_value.at_val.at_size.atsv_units ==
	    ATR_SV_WORDSZ) {
		value *= sizeof(int);
	}
	*ret = value << pres->rs_value.at_val.at_size.atsv_shift;

	return (PBSE_NONE);
}

/*
 * Internal time decoding routine.
 *
 *	Accepts a resource pointer and a pointer to the unsigned long integer
 *	to receive the decoded value.  It returns a PBS error code, and the
 *	decoded value of time in seconds in the unsigned long integer.
 */

static int getlong(pres, ret)
    resource		*pres;
    unsigned long	*ret;
{

	if (pres->rs_value.at_type != ATR_TYPE_LONG)
		return (PBSE_ATTRTYPE);
	if (pres->rs_value.at_val.at_long < 0)
	        return (PBSE_BADATVAL);
	*ret = pres->rs_value.at_val.at_long;

	return (PBSE_NONE);
}

/*
 * Internal time decoding routine.
 *
 *	Accepts a resource pointer and a pointer to the unsigned long integer
 *	to receive the decoded value.  It returns a PBS error code, and the
 *	decoded value of time in seconds in the unsigned long integer.
 */

static int gettime(pres, ret)
    resource		*pres;
    unsigned long	*ret;
{

	if (pres->rs_value.at_type != ATR_TYPE_LONG)
		return (PBSE_ATTRTYPE);
	if (pres->rs_value.at_val.at_long < 0)
	        return (PBSE_BADATVAL);
	*ret = pres->rs_value.at_val.at_long;

	return (PBSE_NONE);
}

/*
 * Time decoding macro.  Accepts a timestruc_t pointer.  Returns unsigned long
 * time in seconds, rounded.
 */

#define tv(val) (ulong)((val).tv_sec + ((val).tv_nsec + 500000000)/1000000000)

/*
**      Scan a list of tasks and return true if one of them matches
**      the process (sid or pid) represented by *psp.
*/
static
int
injob(phead, psp)
    list_head 	*phead;
    prpsinfo_t	*psp;
{
	task		*ptask;
	pid_t		key;

	key = (psp->pr_sid == 0) ? psp->pr_pid : psp->pr_sid;
	for (ptask = (task *)GET_NEXT(*phead);
			ptask;
			ptask = (task *)GET_NEXT(ptask->ti_jobtask)) {
		if (ptask->ti_qs.ti_sid <= 1)
			continue;
		if (ptask->ti_qs.ti_sid == key)
			return TRUE;
	}
	return FALSE;
}

/*
 * Internal session cpu time decoding routine.
 *
 *	Accepts a job pointer.  Returns the sum of all cpu time
 *	consumed for all tasks executed by the job, in seconds.
 */
static unsigned long cput_sum(pjob)
	job	*pjob;
{
	char			*id = "cput_ses";
	int			i;
	ulong			cputime;
	int			nps = 0;
	list_head		*phead;
	ulong			proctime;
	prpsinfo_t		*pi;

	phead = &pjob->ji_tasks;
	cputime = 0;
	for (i=0; i<nproc; i++) {
		pi = &proc_array[i].info;

		if (!injob(phead, pi))
			continue;

		nps++;
		proctime = tv(pi->pr_time) + tv(pi->pr_ctime);
		cputime += proctime;
		DBPRT(("%s: ses %d pid %d pcput %lu cputime %lu\n", id, pi->pr_sid, pi->pr_pid, proctime, cputime))
	}

	if (nps == 0)
		pjob->ji_flags |= MOM_NO_PROC;

	return ((unsigned long)((double)cputime * cputfactor));
}

/*
 * Return TRUE if any process in the job is over limit for cputime usage.
 */
static int overcpu_proc(phead, limit)
    list_head		*phead;
    unsigned long	limit;
{
	ulong		cputime;
	int		i;
	prpsinfo_t	*pi;

	for (i=0; i<nproc; i++) {
		pi = &proc_array[i].info;

		if (!injob(phead, pi))
			continue;

		cputime = tv(pi->pr_time) + tv(pi->pr_ctime);

		if ((ulong)((double)cputime * cputfactor) > limit)
			return (TRUE);
        }

	return (FALSE);
}

/*
 * Internal session memory usage function.
 *
 *	Returns the total number of bytes of address space (virtual mem)
 *      consumed by all current tasks within the list of tasks.
 */
static rlim64_t mem_sum(phead)
    list_head	*phead;
{
	static	char		id[] = "mem_sum";
	int			i;
	rlim64_t		segadd;
	prpsinfo_t		*pi;

	DBPRT(("%s: entered pagesize %d\n", id, pagesize))

	segadd = 0;
	for (i=0; i<nproc; i++) {
		pi = &proc_array[i].info;

		if (!injob(phead, pi))
			continue;

		DBPRT(("%s: %s(%d:%d) mem %llu\n",
			id, pi->pr_fname, pi->pr_sid, pi->pr_pid,
			(rlim64_t)((rlim64_t)pi->pr_size * (rlim64_t)pagesize)))

		segadd += (rlim64_t)((rlim64_t)pi->pr_size*(rlim64_t)pagesize);
	}
	DBPRT(("%s: total mem %llu\n\n", id, segadd))
	return (segadd);
}

#if COMPLEX_MEM_CALC==1
/*
 * Internal session mem (workingset) size function.  COMPLEX CALC VERSION
 *
 *	Returns in a 64 bit intege the number of bytes used by session
 */
static rlim64_t resi_sum(phead)
    list_head		*phead;
{
	static	char		id[] = "resi_sum";
	rlim64_t		resisize, resisub;
	int			num, i, j;
	prpsinfo_t		*pi;
	prmap_sgi_t		*mp;
	u_long			lastseg, nbps;

	DBPRT(("%s: entered pagesize %d\n", id, pagesize))

	resisize = 0;
	lastseg = 99999;
	nbps = (pagesize / sizeof(uint_t)) * pagesize;
				/* sysmacros.h says "4Meg" ...hmmm */

	for (i=0; i<nproc; i++) {
		pi = &proc_array[i].info;

		if (!injob(phead, pi))
                        continue;

		DBPRT(("%s: %s(%d:%d) rss %llu\n",
			id, pi->pr_fname, pi->pr_sid, pi->pr_pid,
			(rlim64_t)((rlim64_t)pi->pr_rssize *
					(rlim64_t)pagesize)))

		resisub = 0;
		num = proc_array[i].map_num;
		mp = proc_array[i].map;
		for (j=0; j<num; j++, mp++) {
			u_long	cnt = mp->pr_mflags >> MA_REFCNT_SHIFT;
			u_long	end = (u_long)mp->pr_vaddr + mp->pr_size - 1;
			u_long	seg1 = (u_long)mp->pr_vaddr / nbps;
			u_long	seg2 = end / nbps;
			rlim64_t	numseg = seg2 - seg1;

			if (lastseg != seg2)
				numseg++;
			lastseg = seg2;
			numseg = numseg*pagesize/cnt;
			numseg += mp->pr_wsize*pagesize/MA_WSIZE_FRAC/cnt;
			resisub += numseg;
			DBPRT(("%s: %d\t%lluk\t%lluk\n", id, j,
				numseg/1024, resisub/1024))
		}
		resisize += resisub;
		DBPRT(("%s: %s subtotal rss %llu\n", id,
			pi->pr_fname, resisub))
        }
	DBPRT(("%s: total rss %llu\n\n", id, resisize))

	return (resisize);
}
#else	/* COMPLEX_MEM_CALC == 0 */
/*
 * Internal session mem (workingset) size function.  SIMPLE CALC VERSION
 *
 *	Returns in a 64 bit intege the number of bytes used by session
 */
static rlim64_t resi_sum(phead)
    list_head		*phead;
{
	static	char		id[] = "resi_sum";
	int			i;
	rlim64_t		resisize, resisub;
	prpsinfo_t		*pi;

	DBPRT(("%s: entered pagesize %d\n", id, pagesize))

	resisize = 0;
	for (i=0; i<nproc; i++) {
		pi = &proc_array[i].info;

		if (!injob(phead, pi))
                        continue;

		DBPRT(("%s: %s(%d:%d) rss %llu\n",
			id, pi->pr_fname, pi->pr_sid, pi->pr_pid,
			(rlim64_t)((rlim64_t)pi->pr_rssize *
					(rlim64_t)pagesize)))
		resisize += (rlim64_t)((rlim64_t)pagesize * pi->pr_rssize);
	}
	DBPRT(("%s: total rss %llu\n\n", id, resisize))
	return (resisize);
}
#endif /* COMPLEX_MEM_CALC */

/*
 * Return TRUE if any process in the session is over limit for Vir memory usage.
 */
static int overmem_proc(phead, limit)
    list_head		*phead;
    rlim64_t		limit;