ndbrestarts.cpp

mysql-5.0.22.tar.gz源码包

/* Copyright (C) 2003 MySQL AB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

#include <NdbRestarts.hpp>
#include <NDBT.hpp>
#include <string.h>
#include <NdbSleep.h>
#include <kernel/ndb_limits.h>
#include <signaldata/DumpStateOrd.hpp>
#include <NdbEnv.h>


int restartRandomNodeGraceful(NdbRestarter&, const NdbRestarts::NdbRestart*);
int restartRandomNodeAbort(NdbRestarter&, const NdbRestarts::NdbRestart*);
int restartRandomNodeError(NdbRestarter&, const NdbRestarts::NdbRestart*);
int restartRandomNodeInitial(NdbRestarter&, const NdbRestarts::NdbRestart*);
int restartNFDuringNR(NdbRestarter&, const NdbRestarts::NdbRestart*);
int restartMasterNodeError(NdbRestarter&, const NdbRestarts::NdbRestart*);
int twoNodeFailure(NdbRestarter&, const NdbRestarts::NdbRestart*);
int fiftyPercentFail(NdbRestarter&, const NdbRestarts::NdbRestart*);
int twoMasterNodeFailure(NdbRestarter&, const NdbRestarts::NdbRestart*);
int restartAllNodesGracfeul(NdbRestarter&, const NdbRestarts::NdbRestart*);
int restartAllNodesAbort(NdbRestarter&, const NdbRestarts::NdbRestart*);
int restartAllNodesError9999(NdbRestarter&, const NdbRestarts::NdbRestart*);
int fiftyPercentStopAndWait(NdbRestarter&, const NdbRestarts::NdbRestart*);
int restartNodeDuringLCP(NdbRestarter& _restarter,
			 const NdbRestarts::NdbRestart* _restart);
int stopOnError(NdbRestarter&, const NdbRestarts::NdbRestart*);
int getRandomNodeId(NdbRestarter& _restarter);

/**
 * Define list of restarts
 *  - name of restart
 *  - function perfoming the restart
 *  - required number of nodes
 *  - ...
 *  - arg1, used depending of restart
 *  - arg2, used depending of restart
 */

const NdbRestarts::NdbRestart NdbRestarts::m_restarts[] = {
  /*********************************************************
   *
   *  NODE RESTARTS with 1 node restarted
   *
   *********************************************************/
  /** 
   * Restart a randomly selected node
   * with graceful shutdown
   */ 
  NdbRestart("RestartRandomNode", 
	     NODE_RESTART,
	     restartRandomNodeGraceful,
	     2),
  /** 
   * Restart a randomly selected node
   * with immediate(abort) shutdown
   */ 
  NdbRestart("RestartRandomNodeAbort", 
	     NODE_RESTART,
	     restartRandomNodeAbort,
	     2),
  /** 
   * Restart a randomly selected node
   * with  error insert
   *
   */ 
  NdbRestart("RestartRandomNodeError",
	     NODE_RESTART,
	     restartRandomNodeError,
	     2),
  /**
   * Restart the master node
   * with  error insert
   */ 
  NdbRestart("RestartMasterNodeError",
	     NODE_RESTART,
	     restartMasterNodeError,
	     2),
  /**
   * Restart a randomly selected node without fileystem
   *
   */ 
  NdbRestart("RestartRandomNodeInitial",
	     NODE_RESTART,
	     restartRandomNodeInitial,
	     2),
  /**
   * Restart a randomly selected node and then 
   * crash it while restarting
   *
   */    
  NdbRestart("RestartNFDuringNR",
	     NODE_RESTART,
	     restartNFDuringNR,
	     2),   

  /**
   * Set StopOnError and crash the node by sending
   * SYSTEM_ERROR to it
   *
   */    
  NdbRestart("StopOnError",
	     NODE_RESTART,
	     stopOnError,
	     1),  

  /*********************************************************
   *
   *  MULTIPLE NODE RESTARTS with more than 1 node
   *
   *********************************************************/
  /**
   * 2 nodes restart, select nodes to restart randomly and restart 
   * with a small random delay between restarts
   */ 
  NdbRestart("TwoNodeFailure",
	     MULTIPLE_NODE_RESTART,
	     twoNodeFailure,
	     4),
  /**
   * 2 nodes restart, select master nodes and restart with 
   * a small random delay between restarts 
   */ 
  
  NdbRestart("TwoMasterNodeFailure",
	     MULTIPLE_NODE_RESTART,
	     twoMasterNodeFailure,
	     4),

  NdbRestart("FiftyPercentFail",
	     MULTIPLE_NODE_RESTART,
	     fiftyPercentFail,
	     2),

  /*********************************************************
   *
   *  SYSTEM RESTARTS
   *
   *********************************************************/
  /**
   * Restart all nodes with graceful shutdown
   *
   */ 
  
  NdbRestart("RestartAllNodes",
	     SYSTEM_RESTART,
	     restartAllNodesGracfeul,
	     1),
  /**
   * Restart all nodes immediately without
   * graful shutdown
   */ 
  NdbRestart("RestartAllNodesAbort",
	     SYSTEM_RESTART,
	     restartAllNodesAbort,
	     1),
  /**
   * Restart all nodes with error insert 9999
   * TODO! We can later add more errors like 9998, 9997 etc.
   */ 
  NdbRestart("RestartAllNodesError9999",
	     SYSTEM_RESTART,
	     restartAllNodesError9999,
	     1),
  /**
   * Stop 50% of all nodes with error insert 9999
   * Wait for a random number of minutes
   * Stop the rest of the nodes and then start all again
   */ 
  NdbRestart("FiftyPercentStopAndWait",
	     SYSTEM_RESTART,
	     fiftyPercentStopAndWait,
	     2),
  /** 
   * Restart a master node during LCP with error inserts.
   */ 
  NdbRestart("RestartNodeDuringLCP", 
	     NODE_RESTART,
	     restartNodeDuringLCP,
	     2),
};

const int NdbRestarts::m_NoOfRestarts = sizeof(m_restarts) / sizeof(NdbRestart);


const NdbRestarts::NdbErrorInsert NdbRestarts::m_errors[] = {
  NdbErrorInsert("Error9999", 9999)
};

const int NdbRestarts::m_NoOfErrors = sizeof(m_errors) / sizeof(NdbErrorInsert);

NdbRestarts::NdbRestart::NdbRestart(const char* _name,
				    NdbRestartType _type,
				    restartFunc* _func,
				    int _requiredNodes,
				    int _arg1){
  m_name = _name;
  m_type = _type;
  m_restartFunc = _func;
  m_numRequiredNodes = _requiredNodes;
  //  m_arg1 = arg1;
}


int NdbRestarts::getNumRestarts(){
  return m_NoOfRestarts;
}

const NdbRestarts::NdbRestart* NdbRestarts::getRestart(int _num){
  if (_num >= m_NoOfRestarts)
    return NULL;

  return &m_restarts[_num];
}

const NdbRestarts::NdbRestart* NdbRestarts::getRestart(const char* _name){
  for(int i = 0; i < m_NoOfRestarts; i++){
    if (strcmp(m_restarts[i].m_name, _name) == 0){
      return &m_restarts[i];
    }
  }
  g_err << "The restart \""<< _name << "\" not found in NdbRestarts" << endl;
  return NULL;
}


int NdbRestarts::executeRestart(const NdbRestarts::NdbRestart* _restart, 
				unsigned int _timeout){
  // Check that there are enough nodes in the cluster
  // for this test
  NdbRestarter restarter;
  if (_restart->m_numRequiredNodes > restarter.getNumDbNodes()){
    g_err << "This test requires " << _restart->m_numRequiredNodes << " nodes "
	  << "there are only "<< restarter.getNumDbNodes() <<" nodes in cluster" 
	  << endl;
    return NDBT_OK;
  }
  if (restarter.waitClusterStarted(120) != 0){
    // If cluster is not started when we shall peform restart
    // the restart can not be executed and the test fails
    return NDBT_FAILED;
  }
  
  int res = _restart->m_restartFunc(restarter, _restart);

  // Sleep a little waiting for nodes to react to command
  NdbSleep_SecSleep(2);

  if  (_timeout == 0){
    // If timeout == 0 wait for ever
    while(restarter.waitClusterStarted(60) != 0)
      g_err << "Cluster is not started after restart. Waiting 60s more..." 
	    << endl;
  } else {
    if (restarter.waitClusterStarted(_timeout) != 0){
      g_err<<"Cluster failed to start" << endl;
      res = NDBT_FAILED; 
    }
  }

  return res;
} 

int NdbRestarts::executeRestart(int _num,
				unsigned int _timeout){
  const NdbRestarts::NdbRestart* r = getRestart(_num);
  if (r == NULL)
    return NDBT_FAILED;

  int res = executeRestart(r, _timeout);
  return res;
}

int NdbRestarts::executeRestart(const char* _name,
				unsigned int _timeout){
  const NdbRestarts::NdbRestart* r = getRestart(_name);
  if (r == NULL)
    return NDBT_FAILED;

  int res = executeRestart(r, _timeout);
  return res;
}

void NdbRestarts::listRestarts(NdbRestartType _type){
  for(int i = 0; i < m_NoOfRestarts; i++){
    if (m_restarts[i].m_type == _type)
      ndbout << " " << m_restarts[i].m_name << ", min " 
	     << m_restarts[i].m_numRequiredNodes 
	     << " nodes"<< endl;
  }  
}

void NdbRestarts::listRestarts(){
  ndbout << "NODE RESTARTS" << endl;
  listRestarts(NODE_RESTART);
  ndbout << "MULTIPLE NODE RESTARTS" << endl;
  listRestarts(MULTIPLE_NODE_RESTART);
  ndbout << "SYSTEM RESTARTS" << endl;
  listRestarts(SYSTEM_RESTART);  
}

NdbRestarts::NdbErrorInsert::NdbErrorInsert(const char* _name,
					    int _errorNo){
  
  m_name = _name;
  m_errorNo = _errorNo;
}

int NdbRestarts::getNumErrorInserts(){
  return m_NoOfErrors;
}

const NdbRestarts::NdbErrorInsert* NdbRestarts::getError(int _num){
  if (_num >= m_NoOfErrors)
    return NULL;
  return &m_errors[_num];
}

const NdbRestarts::NdbErrorInsert* NdbRestarts::getRandomError(){
  int randomId = myRandom48(m_NoOfErrors);
  return &m_errors[randomId];
}



/**
 *
 * IMPLEMENTATION OF THE DIFFERENT RESTARTS
 * Each function should perform it's action
 * and the returned NDBT_OK or NDBT_FAILED
 *
 */


#define CHECK(b, m) { int _xx = b; if (!(_xx)) { \
  ndbout << "ERR: "<< m \
           << "   " << "File: " << __FILE__ \
           << " (Line: " << __LINE__ << ")" << "- " << _xx << endl; \
  return NDBT_FAILED; } }



int restartRandomNodeGraceful(NdbRestarter& _restarter, 
			      const NdbRestarts::NdbRestart* _restart){

  myRandom48Init(NdbTick_CurrentMillisecond());
  int randomId = myRandom48(_restarter.getNumDbNodes());
  int nodeId = _restarter.getDbNodeId(randomId);
  
  g_info << _restart->m_name << ": node = "<<nodeId << endl;

  CHECK(_restarter.restartOneDbNode(nodeId) == 0, 
	"Could not restart node "<<nodeId);

  return NDBT_OK;
}

int restartRandomNodeAbort(NdbRestarter& _restarter, 
			      const NdbRestarts::NdbRestart* _restart){

  myRandom48Init(NdbTick_CurrentMillisecond());
  int randomId = myRandom48(_restarter.getNumDbNodes());
  int nodeId = _restarter.getDbNodeId(randomId);
  
  g_info << _restart->m_name << ": node = "<<nodeId << endl;

  CHECK(_restarter.restartOneDbNode(nodeId, false, false, true) == 0, 
	"Could not restart node "<<nodeId);

  return NDBT_OK;
}

int restartRandomNodeError(NdbRestarter& _restarter, 
			   const NdbRestarts::NdbRestart* _restart){

  myRandom48Init(NdbTick_CurrentMillisecond());
  int randomId = myRandom48(_restarter.getNumDbNodes());
  int nodeId = _restarter.getDbNodeId(randomId);
  
  ndbout << _restart->m_name << ": node = "<<nodeId << endl;

  CHECK(_restarter.insertErrorInNode(nodeId, 9999) == 0, 
	"Could not restart node "<<nodeId);

  return NDBT_OK;
}

int restartMasterNodeError(NdbRestarter& _restarter, 
			   const NdbRestarts::NdbRestart* _restart){

  int nodeId = _restarter.getDbNodeId(0);
  
  g_info << _restart->m_name << ": node = "<<nodeId << endl;

  CHECK(_restarter.insertErrorInNode(nodeId, 39999) == 0, 
	"Could not restart node "<<nodeId);

  return NDBT_OK;
}

int restartRandomNodeInitial(NdbRestarter& _restarter, 
			     const NdbRestarts::NdbRestart* _restart){

  myRandom48Init(NdbTick_CurrentMillisecond());
  int randomId = myRandom48(_restarter.getNumDbNodes());
  int nodeId = _restarter.getDbNodeId(randomId);
  
  g_info << _restart->m_name << ": node = "<<nodeId << endl;

  CHECK(_restarter.restartOneDbNode(nodeId, true) == 0,
	"Could not restart node "<<nodeId);

  return NDBT_OK;
}

int twoNodeFailure(NdbRestarter& _restarter, 
		   const NdbRestarts::NdbRestart* _restart){

  myRandom48Init(NdbTick_CurrentMillisecond());
  int randomId = myRandom48(_restarter.getNumDbNodes());
  int nodeId = _restarter.getDbNodeId(randomId);  
  g_info << _restart->m_name << ": node = "<< nodeId << endl;

  CHECK(_restarter.insertErrorInNode(nodeId, 9999) == 0,