helper.sh

linux集群服务器软件代码包

#!/bin/bash
#
# Copyright (C) 2004 Andrew Beekhof <[e-mail]>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#

[[ $0 != *helper.sh ]] || {
	echo >&2 "helper.sh expects to be sourced!"
	exit 1
}

# no more exit below here, use return instead; gives more flexibility ...

required_nodes=0

test_type=`basename $0`
test_nodes=1
dump_dir=~/ha-tests

HALIB_DIR=/usr/lib/heartbeat
HAVAR_DIR=/var/lib/heartbeat
HAINIT_DIR=/etc/init.d

INIT_USER=root
CRMD_USER=hacluster
ADMIN_USER=root

test_node_1=""
test_node_2=""
test_node_3=""

#CRM_OPTS="-VVVVVV"

CRM_ERR_SHUTDOWN=0

OUTPUT_STDERR=0
OUTPUT_STDOUT=0
OUTPUT_CMDS=0
OUTPUT_ECHO=1

repeats=1
iteration=0

done=0

ip_rsc_1=192.168.9.161
ip_rsc_2=192.168.9.162

logfile=/var/log/messages
local_version=0
crm_only=0

if [ ! -z "$CRM_LOCAL_OPTS" ]; then
    echo "Checking $CRM_LOCAL_OPTS for local options"
    if [ -f "$CRM_LOCAL_OPTS" ]; then
	. "$CRM_LOCAL_OPTS"
    else
	echo "File $CRM_LOCAL_OPTS does not exist"	
    fi
else
	echo "Hint: Define CRM_LOCAL_OPTS=/config/file/name.path to automatically set options for your environment"	
fi

while test "$done" = "0"; do
    case "$1" in
	--debug   | -x)	set -x; shift;;
	--silent  | -s)	OUTPUT_ECHO=0; shift;;
	--stdout  | -o)	OUTPUT_STDOUT=1; shift;;
	--stderr  | -e)	OUTPUT_STDERR=1; shift;;
	--verbose | -v)	OUTPUT_CMDS=1; shift;;

	--crm-only | -C) crm_only=1; shift;;
	--local | -L)	local_version=1; shift;;

	--logfile | -l)	logfile=${2?no logfile??}  || return 1;  shift 2;;
	--dumpdir | -d)	dump_dir=${2?no dumpdir??} || return 1; shift 2;;

	--repeat  | -r)	repeats=${2?no max repetitions??} || return 1; shift 2;;

	--init    | -i)	INIT_USER=${2?no init user??} || return 1; shift 2;;
	--admin   | -a)	ADMIN_USER=${2?no admin??}    || return 1; shift 2;;
	--crmd    | -c)	CRMD_USER=${2?no crmd user??} || return 1; shift 2;;

	--node1   | -1)	test_node_1=${2?no node1??}   || return 1; shift 2;;
	--node2   | -2)	test_node_2=${2?no node2??}   || return 1; shift 2;;
	--node3   | -3)	test_node_3=${2?no node2??}   || return 1; shift 2;;

	--ip1)	ip_rsc_1=${2?no ip1??}                || return 1; shift 2;;
	--ip2)	ip_rsc_2=${2?no ip2??}                || return 1; shift 2;;
 	
	""|--) done=1;;
	*) echo "unknown option: $1"; return 1;;
    esac
done

function test_init() {
    export date_str=`date "+%Y-%b-%d_%H:%M"`
    export test_dump_dir="$dump_dir/$test_type/$date_str"
    mkdir -p "$test_dump_dir"
    echo Test dir: $test_dump_dir
    crm_test_log_start=$(stat -L -c %s $logfile)
}

function remote_cmd() {
    user=$1
    host=$2

    shift 
    shift
    
    if [ $local_version = "1" ]; then
	su - ${user} -c "$*"
    else
	`which ssh` ${user}@${host} "$*"
    fi
    return $?;
}

function start_node() {
    ha_node=$1
    do_cmd echo "wait for HA to start on $ha_node"
    crm_log_pos=$(stat -L -c %s $logfile)
    do_cmd remote_cmd $INIT_USER $ha_node $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" &
    do_cmd ${testdir}/testutils.pl -l ${logfile} -p $crm_log_pos --search -a -m 100 \
	-s "${ha_node} ccm(.*) info: Hostname: ${ha_node}" \
	-s "${ha_node} cib(.*) Hostname: ${ha_node}" \
	-s "${ha_node} heartbeat(.*) info: Starting(.*)lrmd" \
	-e "${ha_node} heartbeat(.*)Client(.*) respawning too fast"
    cts_assert "Startup of Heartbeat on ${ha_node} failed."
    
#----
    
    do_cmd echo "Check if CRMd was started too"
    do_cmd remote_cmd $INIT_USER $ha_node "killall -INT crmd"
    if [ $? = 1 ]; then 
	crm_log_pos=$(stat -L -c %s $logfile)
	do_cmd echo "wait for CRMd to start on $ha_node"
	do_cmd remote_cmd $CRMD_USER $ha_node $HALIB_DIR/crmd "$CRM_OPTS" "2>&1 >/dev/null" &
    fi

    do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search  -a -m 20 \
	-s "${ha_node} crmd(.*) info:(.*)FSA Hostname: ${ha_node}"
    cts_assert "CRMd startup on ${ha_node} failed."
}

function stop_node() 
{
    a_node=$1; shift;
    down_cmd=$1; shift;
    do_cmd echo Shutdown $a_node
    crm_log_pos=$(stat -L -c %s $logfile)

    if [ $crm_only = 1 ]; then
	do_cmd remote_cmd $CRMD_USER $a_node $HALIB_DIR/crmadmin -K $a_node
    else
	do_cmd remote_cmd $INIT_USER $a_node $HALIB_DIR/heartbeat -k
    fi
    do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -t 120 \
	-s "${a_node} crmd(.*)State transition" \
	-s "${a_node} crmd(.*)State transition (.*) \-> S_STOPPING" \
	-s "${a_node} crmd(.*)\[crmd\] stopped" \
	-e "${a_node} crmd(.*)Timer I_TERMINATE just popped"
    cts_assert "Shutdown of ${a_node} failed."
}

function is_state() {
    ret=1
    host=$1
    state=$2

    output=`remote_cmd $ADMIN_USER $host $HALIB_DIR/crmadmin -S $host | grep $state`

    if [ ! -z "$output" ]; then
	echo "$host is in $state"
	return 0
    fi
    echo "$host is NOT in $state"
    return $ret
}

function is_running() {
    rsc=$1
    askhost=$2
    host=$3

    if [ -z $host ]; then
	output=`remote_cmd $ADMIN_USER $askhost $HALIB_DIR/crmadmin -W $rsc | grep -v $HALIB_DIR/crmadmin`
	output=`echo $output | grep -v NOT`
    else
	output=`remote_cmd $ADMIN_USER $askhost $HALIB_DIR/crmadmin -W $rsc | grep -v $HALIB_DIR/crmadmin`
	output=`echo $output | grep $host`
    fi

    if [ -z "$output" ]; then
	if [ -z $host ]; then
	    echo "$rsc is NOT running"
	else
	    echo "$rsc is NOT running on host $host"
	fi
	return 1
    fi

    if [ -z $host ]; then
	echo "$rsc is running"
    else
	echo "$rsc is running on host $host"
    fi

    return 0
}

function is_dc() {
    host=$1
    output=`remote_cmd $ADMIN_USER $host $HALIB_DIR/crmadmin -S $host | grep -v S_PENDING | grep -v S_NOT_DC | grep -v S_ELECTION | grep -v S_RECOVERY`

    if [ ! -z "$output" ]; then 
	echo "$host is DC"
	return 0
    fi

    echo "$host is NOT DC"
    remote_cmd $ADMIN_USER $host $HALIB_DIR/crmadmin -S $host

    return 1
}	

function wait_for_state() {
    state=$1
    max=$2
    host=$3
    target=$4

    sleep 1

    if [ "$host" = "DC" ]; then
	host=`remote_cmd $ADMIN_USER $target $HALIB_DIR/crmadmin -D`
	host=${host#*: }
    fi

    count=1
    until is_state $host $state; do
	if [ $count -gt $max ]; then
	    echo "Attempt to reach $state on $host failed"
	    return 1
	fi
	count=`expr $count + 1`
	sleep 1
    done
    echo "$state reached on $host"
    return 0
}


function make_node()
{
    host=$1
    uname=$2
    type=$3
    uuid=`uuidgen`
    
    if [ -z $type ]; then
	type="member"
    fi
    
    node_xml="'<node id=\"${uuid}\" uname=\"${uname}\" description=\"test node: ${uname}\" type=\"${type}\"/>'";
    
    remote_cmd $ADMIN_USER $host "$HALIB_DIR/cibadmin -C -o node -X $node_xml"

}

function make_node_local()
{
    host=$1
    uname=$2
    type=$3
    uuid=`uuidgen`
    
    if [ -z $type ]; then
	type="member"
    fi
    
    node_xml="'<node id=\"${uuid}\" uname=\"${uname}\" description=\"test node: ${uname}\" type=\"${type}\"/>'";
    
    remote_cmd $ADMIN_USER $host "$HALIB_DIR/cibadmin -l -C -o node -X $node_xml"
}

function make_incarnation()
{
    host=$1; shift
    id=$1; shift
    class=$1; shift
    type=$1; shift
    max=$1; shift
    priority=$1; shift
    version=$1; shift
    stopfail=$1; shift

    args=$*

    uuid=`uuidgen`
    node_xml="'<incarnation id=\\\"${id}\\\" incarnation_node_max=\\\"1\\\" incarnation_max=\\\"${max}\\\">"

    if [ ! -z $priority -a "x$priority" != "x-" ]; then
	node_xml="$node_xml priority=\\\"${priority}\\\""
    fi

    node_xml="$node_xml <resource id=\\\"child_${id}\\\" class=\\\"${class}\\\" type=\\\"${type}\\\""

    if [ ! -z $version -a "x$version" != "x-" ]; then
	node_xml="$node_xml version=\\\"${version}\\\""
    fi
    if [ ! -z $stopfail -a "x$stopfail" != "x-" ]; then
	node_xml="$node_xml on_stopfail=\\\"${stopfail}\\\""
    fi
    node_xml="$node_xml >";

    if [ ! -z "$args" -a "x$args" != "x-" ]; then
	node_xml="$node_xml <instance_attributes><rsc_parameters>${args}</rsc_parameters></instance_attributes>"
    fi
    node_xml="$node_xml </resource>'";
    node_xml="$node_xml </incarnation>";
    
    do_cmd remote_cmd ${ADMIN_USER} ${host} "$HALIB_DIR/cibadmin -VVVV -C -o resource -X $node_xml"