helper.sh

linux集群服务器软件代码包

# 2>&1 > /dev/null
}

function make_resource()
{
    host=$1
    id=$2
    class=$3
    type=$4
    priority=$5
    version=$6
    stopfail=$7

    shift 7
    
    args=$*

    uuid=`uuidgen`
    
    node_xml="<resource id=\\\"${id}\\\" class=\\\"${class}\\\" type=\\\"${type}\\\""

    if [ ! -z $version -a "x$version" != "x-" ]; then
	node_xml="$node_xml version=\\\"${version}\\\""
    fi
    if [ ! -z $priority -a "x$priority" != "x-" ]; then
	node_xml="$node_xml priority=\\\"${priority}\\\""
    fi
    if [ ! -z $stopfail -a "x$stopfail" != "x-" ]; then
	node_xml="$node_xml on_stopfail=\\\"${stopfail}\\\""
    fi
    node_xml="$node_xml >";
    if [ ! -z "$args" -a "x$args" != "x-" ]; then
	node_xml="$node_xml <instance_attributes><rsc_parameters>${args}</rsc_parameters></instance_attributes>"
    fi
    node_xml="$node_xml </resource>";
    
    do_cmd remote_cmd ${ADMIN_USER} ${host} "$HALIB_DIR/cibadmin -VVVV -C -o resource -X \"$node_xml\""
# 2>&1 > /dev/null
}

function make_constraint_adv()
{
    host=$1
    shift
    node_xml=$*

    node_xml=`echo $node_xml | tr '\n' ' '`

    do_cmd remote_cmd ${ADMIN_USER} ${host} "$HALIB_DIR/cibadmin -C -o constraint -VVVV -X \"$node_xml\""
}

function make_constraint()
{
    host=$1
    rsc=$2
    result=$3
    uuid1=`uuidgen`
    uuid2=`uuidgen`
    
    node_xml="<rsc_location id=\\\"${uuid1}\\\" rsc=\\\"${rsc}\\\"><rule id=\\\"${uuid2}\\\" result=\\\"${result}\\\"/></rsc_location>"

    make_constraint_adv $host "$node_xml"
}

function cts_assert() {
    do_cts_assert $? 0 "Assert failed - " $*
}

function cts_assert_false() {
    do_cts_assert $? 1 "AssertFalse failed - " $*
}

function do_cts_assert() {
    result=$1
    expected=$2
    shift
    shift
    do_err=0

    if [ $result -ne $expected ]; then
	do_cmd echo "Expected rc ($expected) != $result"
	do_err=1
    else
	for a_host in $node_list; do
	    remote_cmd ${ADMIN_USER} ${a_host} "ls -al ${HAVAR_DIR}/cores/*/core*" 2>/dev/null 
	    if [ $? -ne 1 ]; then
		do_cmd echo "Core file(s) found on $a_host"
		do_err=1
	    fi
	done
    fi
    
    if [ $do_err = 1 ]; then
	err $*
    fi
}

function err() {
    do_cmd echo Dumping test nodes to $dump_dir
    node_num=1
    for a_host in $node_list; do
	dump_node $node_num $a_host
	node_num=`expr $node_num + 1`
    done

    crm_test_log_end=$(stat -L -c %s $logfile)
    do_cmd echo "ERROR: $*"
    do_cmd echo "test $test_type: FAILED"
    echo "see ${test_dump_dir}/test.txt for details..."
    echo "ERROR: $*" > $test_dump_dir/error.txt
    $HALIB_DIR/crmtest/testutils.pl --dump -p $crm_test_log_start -ep $crm_test_log_end -l $logfile > $test_dump_dir/cluster.log

    exit 1
}

function dump_node() {

    a_node_index=$1
    a_node=$2

    dump_file="/tmp/${a_node_index}-${a_node}.tgz"

    remote_cmd $ADMIN_USER ${a_node} "stat -L -c %s $logfile > /tmp/crm/.logend"
    remote_cmd ${ADMIN_USER} ${a_node} "ps axf > /tmp/crm/processes"
    remote_cmd ${ADMIN_USER} ${a_node} "$HALIB_DIR/crmtest/testutils.pl --dump -pf /tmp/crm/.logstart  -l $logfile > /tmp/crm/cluster-test.log"
#    `which ssh` ${ADMIN_USER}@${a_node} "$HALIB_DIR/crmtest/testutils.pl --dump -pf /tmp/crm/.logstart -epf /tmp/crm/.logend -l $logfile > /tmp/crm/cluster-test.log"

    remote_cmd ${ADMIN_USER} ${a_node} "ln -s /tmp/crm /tmp/crm-${a_node_index}"
    remote_cmd ${ADMIN_USER} ${a_node} "ln -s ${HAVAR_DIR}/cores /tmp/crm"
    remote_cmd ${ADMIN_USER} ${a_node} "cp ${HAVAR_DIR}/crm/cib.xml /tmp/crm"
    remote_cmd ${ADMIN_USER} ${a_node} "tar -zhcvf ${dump_file} /tmp/crm-${a_node_index}/ "
    remote_cmd ${ADMIN_USER} ${a_node} "rm -f /tmp/crm-${a_node_index}"

    cd $test_dump_dir
    if [ $local_version = 1 ]; then
	cp -r ${dump_file} .
    else 
	scp -r ${ADMIN_USER} ${a_node}:${dump_file} .
    fi
    tar zxvf ${a_node_index}-${a_node}.tgz
    cd -
    remote_cmd ${ADMIN_USER} ${a_node} "rm ${dump_file}"

    $HALIB_DIR/crmtest/extract.logs ${test_dump_dir} node-${a_node_index} ${test_dump_dir}/tmp/crm-${a_node_index}/cluster-test.log
    
}

function do_cmd() {

    cmdline=$*
    old_OUTPUT_STDOUT=$OUTPUT_STDOUT
    cmd=$1


    if [ $OUTPUT_STDERR -eq 1 -o $OUTPUT_STDOUT -eq 1 -o $OUTPUT_CMDS -eq 1 ]; then
	
	if [ $cmd = "remote_cmd" ]; then
	    user=$2
	    host=$3
	    shift 3
	    echo `date` ": Running '$*' as ${user}@${host}" | tee -a $test_dump_dir/test.txt;

	elif [ $cmd = "wait_for_state" ]; then
	    host=$4
	    if [ -z $host ]; then
		host="the DC"
	    fi
	    echo `date` ": Waiting for state $2 on $host (max $3 attempts)" | tee -a $test_dump_dir/test.txt;
	    
	elif [ $cmd = "${HALIB_DIR}/crmtest/testutils.pl" ]; then
	    echo `date` ": Searching '$cmdline'" | tee -a $test_dump_dir/test.txt;
	    
	elif [ $cmd = "echo" ]; then
	    echo -n ""
#	    if [ $OUTPUT_CMDS -eq 1 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0  ]; then
#		echo -n `date` ": "
#		$cmdline
#	    elif [ $OUTPUT_ECHO -eq 1 -a $OUTPUT_CMDS -eq 0 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0  ]; then
#		$cmdline		
#	    fi
	else
	    echo `date` ": Running '$cmdline'" | tee -a $test_dump_dir/test.txt;
	fi
    fi

    if [ $cmd = "echo" ]; then
	    if [ $OUTPUT_CMDS -eq 1 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0  ]; then
		echo -n `date` ": " | tee -a $test_dump_dir/test.txt
		echo -n -e "\033[01;32m"
		$cmdline | tee -a $test_dump_dir/test.txt
		echo -n -e "\033[00m"
	    elif [ $OUTPUT_ECHO -eq 1 -a $OUTPUT_CMDS -eq 0 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_STDERR -eq 0  ]; then
		echo -n `date` ": " | tee -a $test_dump_dir/test.txt
		echo -n -e "\033[01;32m"
		$cmdline | tee -a $test_dump_dir/test.txt
		echo -n -e "\033[00m"
	    fi

    elif [ $OUTPUT_STDERR -eq 1 -a $OUTPUT_STDOUT -eq 1 ]; then
	$cmdline 2>&1 >> $test_dump_dir/test.txt
	rc=$?
    elif [ $OUTPUT_STDERR -eq 0 -a $OUTPUT_STDOUT -eq 0 ]; then
	$cmdline 2>&1 >/dev/null
	rc=$?
    elif [ $OUTPUT_STDOUT -eq 0 ]; then
	$cmdline >/dev/null 2>> $test_dump_dir/test.txt
	rc=$?
    elif [ $OUTPUT_STDERR -eq 0 ]; then
	$cmdline 2>/dev/null >> $test_dump_dir/test.txt
	rc=$?
    else
	$cmdline 2>&1 >> $test_dump_dir/test.txt
	rc=$?
    fi

    if [ $cmd = "${HALIB_DIR}/crmtest/testutils.pl" ]; then
	# for the logs...
	if [ $OUTPUT_STDERR -eq 0 -a $OUTPUT_STDOUT -eq 0 -a $OUTPUT_CMDS -eq 0 ]; then
	    echo `date` ": Searching '$cmdline'" >> $test_dump_dir/test.txt;
	fi
	$cmdline -t 1 -v 2>&1 >> $test_dump_dir/test.txt

	if [ $rc -lt 0 ]; then
	    echo `date` ": Searching '$cmdline'"
	    $cmdline
	fi
    fi

    OUTPUT_STDOUT=$old_OUTPUT_STDOUT
#    echo Result: $rc
    return $rc
}

function crm-cleanup() {
    
    if [ -z $1 ]; then
	test_init
    fi

    for a_host in $node_list; do
	crm-cleanup-node $a_host $1
    done
}

function crm-cleanup-node() {
    ha_host=$1
    do_cmd echo Cleaning up on $ha_host

# be *very* sure everything has stopped
    do_cmd remote_cmd $INIT_USER $ha_host "killall -q9 heartbeat ccm lrmd crmd crmadmin cibadmin ha_logd stonithd cib pengine tengine"

# resources too
    do_cmd remote_cmd $INIT_USER $ha_host "/etc/ha.d/resource.d/IPaddr $ip_rsc_1 stop" 2>&1 > /dev/null
    do_cmd remote_cmd $INIT_USER $ha_host "/etc/ha.d/resource.d/IPaddr $ip_rsc_2 stop" 2>&1 > /dev/null

    if [ -z $2 ]; then
# make *sure* theres nothing left over from last time
	do_cmd remote_cmd $INIT_USER $ha_host "rm -f $HAVAR_DIR/crm/cib*.xml"

	remote_cmd ${INIT_USER} ${ha_host} "rm -f ${HAVAR_DIR}/cores/*/*" 2>/dev/null 
	remote_cmd ${INIT_USER} ${ha_host} "rm -rf /tmp/crm/* /tmp/crm/.log*" 2>/dev/null 

	if [ `dirname $logfile` = "/tmp/crm" ]; then
	    do_cmd remote_cmd $INIT_USER $ha_host "/etc/init.d/syslog restart"
	fi
	remote_cmd $ADMIN_USER $ha_host "stat -L -c %s $logfile > /tmp/crm/.logstart"
	do_cmd remote_cmd $ADMIN_USER $ha_host "logger -i -p local7.info __crmtest_${test_type}_log_mark__"
	do_cmd sleep 2
	do_cmd remote_cmd $ADMIN_USER $ha_host "echo 1 > /proc/sys/kernel/core_uses_pid"
	
# zero out logs, maybe
    fi
}

function mark_log() 
{
    export crm_log_pos=$(stat -L -c %s $logfile)
}

function gres() 
{
    if [ $# -lt "3" ] 
	then 
	echo Usage: gres pattern replacement files
	return 1
    fi
    pattern=$1
    replacement=$2
    shift; shift;
    filelist=$*
    A="`echo | tr '\012' '\001' `"
    
    for file in $filelist; do
	
	fname=`basename $file`
	dname=`dirname $file`
	
	is_backup=${fname##.gres.}
	if [ "$is_backup" = "$fname" ]; then
	    grep -q "$pattern" "$file"
	    isthere=$?
	    
	    if [ $isthere = 0 ]; then
#		echo Processing $file...
		cp "$file" "$dname/.gres.$fname" && sed -e "s$A$pattern$A$replacement$A" "$dname/.gres.$fname" > "$file"
	    fi
#    else
#	echo "Ignoring backup of $is_backup in $dname"
	fi
    done
}

# successfully sourced
if [ $local_version = 1 ]; then
    test_node_1=`uname -n`
    test_node_2=""
    test_node_3=""
    echo "Set test_node_1=\"$test_node_1\""
    echo "Unset test_node_2 and test_node_3"
fi

node_list="$test_node_1 $test_node_2 $test_node_3"
node_count=0
for a_host in $node_list; do
    node_count=`expr $node_count + 1`
done

if [ $node_count -lt $required_nodes ]; then
    do_cmd echo "$required_nodes required for this test.  You specified only $node_list" 
    do_cmd echo "test: FAILED" 
    return false
fi

true