lustre_config.in

来自「lustre 1.6.5 source code」· IN 代码 · 共 1,221 行 · 第 1/3 页

#!/bin/bash

# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:

#
# lustre_config - format and set up multiple lustre servers from a csv file
#
# This script is used to parse each line of a spreadsheet (csv file) and 
# execute remote commands to format (mkfs.lustre) every Lustre target 
# that will be part of the Lustre cluster.
# 
# In addition, it can also verify the network connectivity and hostnames in 
# the cluster, configure Linux MD/LVM devices and produce High-Availability
# software configurations for Heartbeat or CluManager.
#
################################################################################

# Usage
usage() {
    cat >&2 <<EOF

Usage:  `basename $0` [options] <csv file>

    This script is used to format and set up multiple lustre servers from a
    csv file.

    Options:
    -h          help and examples
    -a          select all the nodes from the csv file to operate on
    -w hostname,hostname,...
                select the specified list of nodes (separated by commas) to
                operate on rather than all the nodes in the csv file
    -x hostname,hostname,...
                exclude the specified list of nodes (separated by commas)
    -t HAtype   produce High-Availability software configurations
                The argument following -t is used to indicate the High-
                Availability software type. The HA software types which 
                are currently supported are: hbv1 (Heartbeat version 1)
                and hbv2 (Heartbeat version 2).
    -n          no net - don't verify network connectivity and hostnames
                in the cluster
    -d          configure Linux MD/LVM devices before formatting the
                Lustre targets
    -f          force-format the Lustre targets using --reformat option
    -m          no fstab change - don't modify /etc/fstab to add the new
                Lustre targets
                If using this option, then the value of "mount options"
                item in the csv file will be passed to mkfs.lustre, else
                the value will be added into the /etc/fstab.
    -u          upgrade Lustre targets from 1.4 to 1.6
    -v          verbose mode
    csv file    a spreadsheet that contains configuration parameters
                (separated by commas) for each target in a Lustre cluster

EOF
    exit 1
}

# Samples 
sample() {
    cat <<EOF

This script is used to parse each line of a spreadsheet (csv file) and 
execute remote commands to format (mkfs.lustre) every Lustre target 
that will be part of the Lustre cluster.

It can also optionally: 
 * upgrade Lustre targets from 1.4 to 1.6
 * verify the network connectivity and hostnames in the cluster
 * configure Linux MD/LVM devices
 * modify /etc/modprobe.conf to add Lustre networking info
 * add the Lustre server info to /etc/fstab
 * produce configurations for Heartbeat or CluManager.

There are 5 kinds of line formats in the csv file. They represent the following 
targets:
1) Linux MD device
The format is:
hostname,MD,md name,operation mode,options,raid level,component devices

hostname            hostname of the node in the cluster
MD                  marker of MD device line
md name             MD device name, e.g. /dev/md0
operation mode      create or remove, default is create
options             a "catchall" for other mdadm options, e.g. "-c 128"
raid level          raid level: 0,1,4,5,6,10,linear and multipath
component devices   block devices to be combined into the MD device
                    Multiple devices are separated by space or by using
                    shell expansions, e.g. "/dev/sd{a,b,c}"

2) Linux LVM PV (Physical Volume)
The format is:
hostname,PV,pv names,operation mode,options

hostname            hostname of the node in the cluster
PV                  marker of PV line
pv names            devices or loopback files to be initialized for later
                    use by LVM or to be wiped the label, e.g. /dev/sda
                    Multiple devices or files are separated by space or by
                    using shell expansions, e.g. "/dev/sd{a,b,c}"
operation mode      create or remove, default is create
options             a "catchall" for other pvcreate/pvremove options
                    e.g. "-vv"

3) Linux LVM VG (Volume Group)
The format is:
hostname,VG,vg name,operation mode,options,pv paths

hostname            hostname of the node in the cluster
VG                  marker of VG line
vg name             name of the volume group, e.g. ost_vg
operation mode      create or remove, default is create
options             a "catchall" for other vgcreate/vgremove options
                    e.g. "-s 32M"
pv paths            physical volumes to construct this VG, required by
                    create mode
                    Multiple PVs are separated by space or by using
                    shell expansions, e.g. "/dev/sd[k-m]1"

4) Linux LVM LV (Logical Volume)
The format is:
hostname,LV,lv name,operation mode,options,lv size,vg name

hostname            hostname of the node in the cluster
LV                  marker of LV line
lv name             name of the logical volume to be created (optional)
                    or path of the logical volume to be removed (required
                    by remove mode)
operation mode      create or remove, default is create
options             a "catchall" for other lvcreate/lvremove options
                    e.g. "-i 2 -I 128"
lv size             size [kKmMgGtT] to be allocated for the new LV
                    Default unit is megabytes.
vg name             name of the VG in which the new LV will be created

5) Lustre target
The format is:
hostname,module_opts,device name,mount point,device type,fsname,mgs nids,index,
format options,mkfs options,mount options,failover nids

hostname            hostname of the node in the cluster, must match "uname -n"
module_opts         Lustre networking module options
device name         Lustre target (block device or loopback file)
mount point         Lustre target mount point
device type         Lustre target type (mgs, mdt, ost, mgs|mdt, mdt|mgs)
fsname              Lustre filesystem name, should be limited to 8 characters 
                    Default is "lustre".
mgs nids            NID(s) of remote mgs node, required for mdt and ost targets
                    If this item is not given for an mdt, it is assumed that
                    the mdt will also be an mgs, according to mkfs.lustre.
index               Lustre target index
format options      a "catchall" contains options to be passed to mkfs.lustre
                    "--device-size", "--param", etc. all goes into this item.
mkfs options        format options to be wrapped with --mkfsoptions="" and
                    passed to mkfs.lustre
mount options       If this script is invoked with "-m" option, then the value of
                    this item will be wrapped with --mountfsoptions="" and passed
                    to mkfs.lustre, else the value will be added into /etc/fstab.
failover nids       NID(s) of failover partner node

All the NIDs in one node are delimited by commas (','). When multiple nodes are
specified, they are delimited by a colon (':').

Items left blank will be set to defaults.

Example 1 - Simple, with combo MGS/MDT:
-------------------------------------------------------------------------------
# combo mdt/mgs
lustre-mgs,options lnet networks=tcp,/tmp/mgs,/mnt/mgs,mgs|mdt,,,,--device-size=10240

# ost0
lustre-ost,options lnet networks=tcp,/tmp/ost0,/mnt/ost0,ost,,lustre-mgs@tcp0,,--device-size=10240

# ost1
lustre-ost,options lnet networks=tcp,/tmp/ost1,/mnt/ost1,ost,,lustre-mgs@tcp0,,--device-size=10240
-------------------------------------------------------------------------------

Example 2 - Separate MGS/MDT, two networks interfaces:
-------------------------------------------------------------------------------
# mgs
lustre-mgs1,options lnet 'networks="tcp,elan"',/dev/sda,/mnt/mgs,mgs,,,,--quiet --param="sys.timeout=50",,"defaults,noauto","lustre-mgs2,2@elan"

# mdt
lustre-mdt1,options lnet 'networks="tcp,elan"',/dev/sdb,/mnt/mdt,mdt,lustre2,"lustre-mgs1,1@elan:lustre-mgs2,2@elan",,--quiet --param="lov.stripesize=4194304",-J size=16,"defaults,noauto",lustre-mdt2

# ost
lustre-ost1,options lnet 'networks="tcp,elan"',/dev/sdc,/mnt/ost,ost,lustre2,"lustre-mgs1,1@elan:lustre-mgs2,2@elan",,--quiet,-I 512,"defaults,noauto",lustre-ost2
-------------------------------------------------------------------------------

Example 3 - with combo MGS/MDT failover pair and OST failover pair:
-------------------------------------------------------------------------------
# combo mgs/mdt
lustre-mgs1,options lnet networks=tcp,/tmp/mgs,/mnt/mgs,mgs|mdt,,,,--quiet --device-size=10240,,,lustre-mgs2@tcp0

# combo mgs/mdt backup (--noformat)
lustre-mgs2,options lnet networks=tcp,/tmp/mgs,/mnt/mgs,mgs|mdt,,,,--quiet --device-size=10240 --noformat,,,lustre-mgs1@tcp0

# ost
lustre-ost1,options lnet networks=tcp,/tmp/ost1,/mnt/ost1,ost,,"lustre-mgs1@tcp0:lustre-mgs2@tcp0",,--quiet --device-size=10240,,,lustre-ost2@tcp0

# ost backup (--noformat) (note different device name)
lustre-ost2,options lnet networks=tcp,/tmp/ost2,/mnt/ost2,ost,,"lustre-mgs1@tcp0:lustre-mgs2@tcp0",,--quiet --device-size=10240 --noformat,,,lustre-ost1@tcp0
-------------------------------------------------------------------------------

Example 4 - Configure Linux MD/LVM devices before formatting Lustre targets:
-------------------------------------------------------------------------------
# MD device on mgsnode
mgsnode,MD,/dev/md0,,-q,1,/dev/sda1 /dev/sdb1

# MD/LVM devices on ostnode
ostnode,MD,/dev/md0,,-q -c 128,5,"/dev/sd{a,b,c}"
ostnode,MD,/dev/md1,,-q -c 128,5,"/dev/sd{d,e,f}"
ostnode,PV,/dev/md0 /dev/md1
ostnode,VG,ost_vg,,-s 32M,/dev/md0 /dev/md1
ostnode,LV,ost0,,-i 2 -I 128,300G,ost_vg
ostnode,LV,ost1,,-i 2 -I 128,300G,ost_vg

# combo mgs/mdt
mgsnode,options lnet networks=tcp,/dev/md0,/mnt/mgs,mgs|mdt,,,,--quiet

# ost0
ostnode,options lnet networks=tcp,/dev/ost_vg/ost0,/mnt/ost0,ost,,mgsnode,,--quiet

# ost1
ostnode,options lnet networks=tcp,/dev/ost_vg/ost1,/mnt/ost1,ost,,mgsnode,,--quiet
-------------------------------------------------------------------------------

EOF
    exit 0
}

# Get the library of functions
. @scriptlibdir@/lc_common

#***************************** Global variables *****************************#
declare -a MGS_NODENAME             # node names of the MGS servers
declare -a MGS_IDX                  # indexes of MGSs in the global arrays
declare -i MGS_NUM                  # number of MGS servers in the cluster
declare -i INIT_IDX

declare -a NODE_NAMES               # node names in the failover group
declare -a TARGET_OPTS              # target services in one failover group

# All the items in the csv file
declare -a HOST_NAME MODULE_OPTS DEVICE_NAME MOUNT_POINT DEVICE_TYPE FS_NAME
declare -a MGS_NIDS INDEX FORMAT_OPTIONS MKFS_OPTIONS MOUNT_OPTIONS FAILOVERS

# Heartbeat software requires that node names in the configuration directive
# must (normally) match the "uname -n" of that machine. Since the value of the
# "failover nids" field in the csv file is the NID(s) of failover partner node,
# we have to figure out the corresponding hostname of that node.
declare -a FAILOVERS_NAMES

VERIFY_CONNECT=true
CONFIG_MD_LVM=false
MODIFY_FSTAB=true
UPGRADE_TARGET=false
VERBOSE_OUTPUT=false
# Get and check the positional parameters
while getopts "aw:x:t:ndfmuhv" OPTION; do
    case $OPTION in
    a)
        [ -z "${SPECIFIED_NODELIST}" ] && [ -z "${EXCLUDED_NODELIST}" ] \
        && USE_ALLNODES=true
        NODELIST_OPT="${NODELIST_OPT} -a"
        ;;
    w)
        USE_ALLNODES=false
        SPECIFIED_NODELIST=$OPTARG
        NODELIST_OPT="${NODELIST_OPT} -w ${SPECIFIED_NODELIST}"
        ;;
    x)
        USE_ALLNODES=false
        EXCLUDED_NODELIST=$OPTARG
        NODELIST_OPT="${NODELIST_OPT} -x ${EXCLUDED_NODELIST}"
        ;;
    t)
        HATYPE_OPT=$OPTARG
        if [ "${HATYPE_OPT}" != "${HBVER_HBV1}" ] \
        && [ "${HATYPE_OPT}" != "${HBVER_HBV2}" ] \
        && [ "${HATYPE_OPT}" != "${HATYPE_CLUMGR}" ]; then
            echo >&2 $"`basename $0`: Invalid HA software type" \
                      "- ${HATYPE_OPT}!"
            usage
        fi
        ;;
    n)
        VERIFY_CONNECT=false
        ;;
    d)
        CONFIG_MD_LVM=true
        ;;
    f)
        REFORMAT_OPTION=$"--reformat "
        ;;
    m)
        MODIFY_FSTAB=false
        ;;
    u)
        UPGRADE_TARGET=true 
        ;;
    h)
        sample
        ;;
    v)
        VERBOSE_OPT=$" -v"
        VERBOSE_OUTPUT=true
        ;;
    ?)
        usage 
    esac
done

# Toss out the parameters we've already processed
shift  `expr $OPTIND - 1`

# Here we expect the csv file
if [ $# -eq 0 ]; then
    echo >&2 $"`basename $0`: Missing csv file!"
    usage
fi

# Check the items required for OSTs, MDTs and MGS
#
# When formatting an OST, the following items: hostname, module_opts,
# device name, device type and mgs nids, cannot have null value.
#
# When formatting an MDT or MGS, the following items: hostname,
# module_opts, device name and device type, cannot have null value.
check_item() {
    # Check argument
    if [ $# -eq 0 ]; then
        echo >&2 $"`basename $0`: check_item() error: Missing argument"\
                  "for function check_item()!"
        return 1
    fi

    declare -i i=$1

    # Check hostname, module_opts, device name and device type
    if [ -z "${HOST_NAME[i]}" ]||[ -z "${MODULE_OPTS[i]}" ]\
    ||[ -z "${DEVICE_NAME[i]}" ]||[ -z "${DEVICE_TYPE[i]}" ]; then
        echo >&2 $"`basename $0`: check_item() error: Some required"\
                  "item has null value! Check hostname, module_opts,"\
                  "device name and device type!"
        return 1
    fi

    # Check mgs nids
    if [ "${DEVICE_TYPE[i]}" = "ost" ]&&[ -z "${MGS_NIDS[i]}" ]; then
        echo >&2 $"`basename $0`: check_item() error: OST's mgs nids"\
                  "item has null value!"
        return 1
    fi

    # Check mount point
    if [ -z "${MOUNT_POINT[i]}" ]; then
        echo >&2 $"`basename $0`: check_item() error: mount"\
                  "point item of target ${DEVICE_NAME[i]} has null value!"
        return 1
    fi

    return 0
}

# Get the number of MGS nodes in the cluster
get_mgs_num() {
    INIT_IDX=0
    MGS_NUM=${#MGS_NODENAME[@]}
    [ -z "${MGS_NODENAME[0]}" ] && let "INIT_IDX += 1" \
    && let "MGS_NUM += 1"
}

# is_mgs_node hostname
# Verify whether @hostname is a MGS node
is_mgs_node() {
    local host_name=$1
    declare -i i

    get_mgs_num
    for ((i = ${INIT_IDX}; i < ${MGS_NUM}; i++)); do
        [ "${MGS_NODENAME[i]}" = "${host_name}" ] && return 0
    done

    return 1
}

# Check whether the MGS nodes are in the same failover group
check_mgs_group() {
    declare -i i
    declare -i j
    declare -i idx
    local mgs_node

    get_mgs_num
    for ((i = ${INIT_IDX}; i < ${MGS_NUM}; i++)); do
        mgs_node=${MGS_NODENAME[i]}
        for ((j = ${INIT_IDX}; j < ${MGS_NUM}; j++)); do
          [ "${MGS_NODENAME[j]}" = "${mgs_node}" ] && continue 1

          idx=${MGS_IDX[j]}
          if [ "${FAILOVERS_NAMES[idx]#*$mgs_node*}" = "${FAILOVERS_NAMES[idx]}" ]
          then
            echo >&2 $"`basename $0`: check_mgs_group() error:"\
            "MGS node ${mgs_node} is not in the ${HOST_NAME[idx]}"\
            "failover group!"
            return 1