3node-rapid.sh.in

linux集群服务器软件代码包

#!/bin/bash
#
# Copyright (C) 2004 Andrew Beekhof <[e-mail]>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#

required_nodes=3
testdir=@libdir@/heartbeat/crmtest
. ${testdir}/helper.sh || exit 1

CRM_ERR_SHUTDOWN=0
test_nodes=3

while [ $iteration -lt $repeats ]; do
    iteration=`expr $iteration + 1`
    echo -ne "\033]0;$test_type : Iteration $iteration of $repeats\007"
    echo "########### Begining iteration $iteration of $repeats ###########"
(
# make *sure* theres nothing left over from last time
crm-cleanup

#----

do_cmd echo "wait for HA to start on ${test_node_1}"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $INIT_USER $test_node_1 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" 
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \
    -s "${test_node_1} ccm(.*) info: Hostname: ${test_node_1}" \
    -s "${test_node_1} heartbeat(.*) info: Starting(.*)lrmd" \
    -e "${test_node_1} heartbeat(.*)Client(.*) respawning too fast"
cts_assert "Startup of Heartbeat on ${test_node_1} failed."

#----

do_cmd echo "wait for HA to start on ${test_node_2}"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $INIT_USER $test_node_2 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" 
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \
    -s "${test_node_2} ccm(.*) info: Hostname: ${test_node_2}" \
    -s "${test_node_2} heartbeat(.*) info: Starting(.*)lrmd" \
    -e "${test_node_2} heartbeat(.*)Client(.*) respawning too fast"
cts_assert "Startup of Heartbeat on ${test_node_2} failed."

#----

do_cmd echo "wait for HA to start on ${test_node_3}"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $INIT_USER $test_node_3 $HALIB_DIR/heartbeat -M "2>&1 >/dev/null" 
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 500 \
    -s "${test_node_3} ccm(.*) info: Hostname: ${test_node_3}" \
    -s "${test_node_3} heartbeat(.*) info: Starting(.*)lrmd" \
    -e "${test_node_3} heartbeat(.*)Client(.*) respawning too fast"
cts_assert "Startup of Heartbeat on ${test_node_3} failed."

#----

do_cmd echo "wait for CRMd to start on ${test_node_1}, ${test_node_2} and ${test_node_3}"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" &
do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" &
do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmd -VVVV "2>&1 >/dev/null" &

do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search  -a -m 3000 \
    -s "${test_node_1} crmd(.*) info:(.*)FSA Hostname: ${test_node_1}"
cts_assert "CRMd startup on ${test_node_1} failed."

do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search  -a -m 3000 \
    -s "${test_node_2} crmd(.*) info:(.*)FSA Hostname: ${test_node_2}"
cts_assert "CRMd startup on ${test_node_2} failed."

do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search  -a -m 3000 \
    -s "${test_node_3} crmd(.*) info:(.*)FSA Hostname: ${test_node_3}"
cts_assert "CRMd startup on ${test_node_3} failed."

do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search  -a -m 25000 \
    -s "${test_node_1} crmd(.*) State transition (.*) \-> \"S_IDLE\"" \
    -s "${test_node_2} crmd(.*) State transition (.*) \-> \"S_NOT_DC\"" \
    -s "${test_node_3} crmd(.*) State transition (.*) \-> \"S_NOT_DC\""
cts_assert "CRMd did not start up into correct modes."

#----

do_cmd echo Various sanity checks - stage 1
do_cmd is_dc $test_node_1 $test_node_1
cts_assert "$test_node_1 is supposed to be the DC"

do_cmd wait_for_state S_IDLE 3 DC $test_node_1 
cts_assert "S_IDLE not reached on the DC!"

#----

do_cmd echo Create the first constraint and wait for S_IDLE
rsc=rsc1
uuid1=`uuidgen`
uuid2=`uuidgen`
uuid3=`uuidgen`
uuid4=`uuidgen`
node_xml="'<rsc_location id=\"${uuid1}\" rsc=\"${rsc}\">
	        <rule id=\"${uuid2}\" result=\"can\"/>
		<rule id=\"${uuid3}\" score=\"1000\" boolean_op=\"or\">
		  <expression attribute=\"uname\" operation=\"eq\" value=\"${test_node_1}\"/>
		</rule>
		<rule id=\"${uuid4}\" score=\"100\" boolean_op=\"or\">
		   <expression attribute=\"uname\" operation=\"eq\" value=\"${test_node_2}\"/>
		</rule>
      </rsc_location>'"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_constraint_adv $test_node_1 $node_xml
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
    -s "crmd(.*) State transition (.*) \-> \"S_IDLE\""
cts_assert Adding constraint1 did not pass

#----

do_cmd echo Create the second constraint and wait for S_IDLE
rsc=rsc2
uuid1=`uuidgen`
uuid2=`uuidgen`
uuid3=`uuidgen`
uuid4=`uuidgen`
node_xml="'<rsc_location id=\"${uuid1}\" rsc=\"${rsc}\">
	        <rule id=\"${uuid2}\" result=\"can\"/>
		<rule id=\"${uuid3}\" score=\"1000\" boolean_op=\"or\">
		   <expression attribute=\"uname\" operation=\"eq\" value=\"${test_node_2}\"/>
		</rule>
		<rule id=\"${uuid4}\" score=\"100\" boolean_op=\"or\">
		   <expression attribute=\"uname\" operation=\"eq\" value=\"${test_node_1}\"/>
		</rule>
	   </rsc_location>'"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_constraint_adv $test_node_1 $node_xml
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 1500 \
    -s "crmd(.*) State transition (.*) \-> \"S_IDLE\""
cts_assert Adding constraint2 did not pass

#----

do_cmd echo Create the first resource and wait for S_IDLE after start
args="<nvpair name=\"1\" value=\"${ip_rsc_1}\"/>"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_resource $test_node_1 rsc1 heartbeat IPaddr - - - $args
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \
    -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \
    -s "crmd(.*) Performing op start(.*) on rsc1" \
    -s "crmd(.*) Resource state: rsc1(.*) after start"
cts_assert Adding rsc1 did not pass

#----

do_cmd echo Create the second resource and wait for S_IDLE after start
args="<nvpair name=\"1\" value=\"${ip_rsc_2}\"/>"
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd make_resource $test_node_1 rsc2 heartbeat IPaddr - - - $args
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 2500 \
    -s "crmd(.*) State transition (.*) \-> \"S_IDLE\"" \
    -s "crmd(.*) Performing op start(.*) on rsc2" \
    -s "crmd(.*) Resource state: rsc2(.*) after start"
cts_assert Adding rsc2 did not pass

#----

do_cmd echo Various sanity checks - stage 2
do_cmd is_dc $test_node_1 $test_node_1
cts_assert "$test_node_1 is supposed to be the DC"

do_cmd wait_for_state S_IDLE 3 DC $test_node_1 
cts_assert "S_IDLE not reached on the DC!"

do_cmd is_running rsc1 $test_node_1 $test_node_1
cts_assert "rsc1 NOT running on $test_node_1"

do_cmd is_running rsc2 $test_node_1 $test_node_2
cts_assert "rsc2 NOT running on $test_node_2"

#----

do_cmd echo Shutdown $test_node_1
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_1 $HALIB_DIR/crmadmin -K $test_node_1 &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
    -s "crmd(.*) State transition (.*) \-> \"S_ELECTION\"" \
    -s "${test_node_1} crmd(.*) State transition (.*) \-> \"S_NOT_DC\""
cts_assert "Transition of the DC away from ${test_node_1} failed."
#    -s "crmd(.*) State transition (.*) \->" \

do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
    -s "${test_node_1} crmd(.*)State transition \"S_NOT_DC\" \-> \"S_STOPPING\"" \
    -s "${test_node_1} crmd(.*)\[crmd\] stopped" \
    -e "${test_node_1} crmd(.*)Timer I_TERMINATE just popped"
cts_assert "Shutdown of ${test_node_1} failed"

do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
    -s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" 
cts_assert "No DC Declared"

do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
    -s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" \
    -s "crmd(.*)State transition(.*) \-> \"S_IDLE\""
cts_assert "DC not idle"

#----

do_cmd echo Various sanity checks - stage 3
do_cmd is_dc $test_node_2 $test_node_2
cts_assert "$test_node_2 is supposed to be the DC"

do_cmd wait_for_state S_IDLE 3 $test_node_2 
cts_assert "S_IDLE not reached on $test_node_2!"

do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
    -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_2}"
do_cmd is_running rsc1 $test_node_2 $test_node_2
cts_assert "rsc1 NOT running on $test_node_2"

do_cmd is_running rsc2 $test_node_2 $test_node_2
cts_assert "rsc2 NOT running on $test_node_2"

#----

do_cmd echo Shutdown $test_node_2
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_2 $HALIB_DIR/crmadmin -K $test_node_2 &
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
    -s "crmd(.*) State transition (.*) \-> \"S_ELECTION\"" \
    -s "${test_node_2} crmd(.*) State transition (.*) \-> \"S_NOT_DC\""
cts_assert "Transition of the DC away from ${test_node_2} failed."
#    -s "crmd(.*) State transition(.*) \->" \

do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
    -s "${test_node_2} crmd(.*)State transition" \
    -s "${test_node_2} crmd(.*)State transition \"S_NOT_DC\" \-> \"S_STOPPING\"" \
    -s "${test_node_2} crmd(.*)\[crmd\] stopped" \
    -e "${test_node_2} crmd(.*)Timer I_TERMINATE just popped"
cts_assert "Shutdown of ${test_node_2} failed"

do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 11000 \
    -s "crmd(.*)State transition(.*) \-> \"S_INTEGRATION\"" \
    -s "crmd(.*)State transition(.*) \-> \"S_IDLE\""
cts_assert "No DC Declared"

#----

do_cmd echo Various sanity checks - stage 4
do_cmd is_dc $test_node_3 $test_node_3
cts_assert "$test_node_3 is supposed to be the DC"

do_cmd wait_for_state S_IDLE 3 $test_node_3 
cts_assert "S_IDLE not reached on $test_node_3!"

do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
    -s "crmd(.*) Resource state: rsc1(.*) after start] on ${test_node_3}"
do_cmd is_running rsc1 $test_node_3 $test_node_3
cts_assert "rsc1 NOT running on $test_node_3"

do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 10000 \
    -s "crmd(.*) Resource state: rsc2(.*) after start] on ${test_node_3}"
do_cmd is_running rsc2 $test_node_3 $test_node_3
cts_assert "rsc2 NOT running on $test_node_3"

#----

do_cmd echo Shutdown $test_node_3
crm_log_pos=$(stat -L -c %s $logfile)
do_cmd remote_cmd $CRMD_USER $test_node_3 $HALIB_DIR/crmadmin -K $test_node_3 
do_cmd ${testdir}/testutils.pl -p $crm_log_pos -l ${logfile} --search -a -m 5000 \
    -s "${test_node_3} crmd(.*)State transition (.*) \-> \"S_PENDING\"" \
    -s "${test_node_3} crmd(.*)State transition \"S_PENDING\" \-> \"S_STOPPING\"" \
    -s "${test_node_3} crmd(.*)\[crmd\] stopped" 
cts_assert "Shutdown of ${test_node_3} failed."

#----

do_cmd remote_cmd $CRMD_USER $test_node_3 killall -9 crmd

echo "test: PASSED"
)
done