lustre_config.in

来自「lustre 1.6.5 source code」· IN 代码 · 共 1,221 行 · 第 1/3 页

        # Convert IP addresses in NIDs to hostnames
        FAILOVERS_NAMES[idx]=$(ip2hostname_multi_node ${FAILOVERS[idx]})
        if [ ${PIPESTATUS[0]} -ne 0 ]; then
            echo >&2 "${FAILOVERS_NAMES[idx]}"
            return 1
        fi

        # Check some required items for formatting target
        if ! check_item $idx; then
            echo >&2 $"`basename $0`: check_item() error:"\
                  "Occurred on line ${line_num} in ${CSV_FILE}."
            return 1    
        fi

        idx=${idx}+1
    done

    return 0
}

# check_lnet_connect hostname_index mgs_hostname
# Check whether the target node can contact the MGS node @mgs_hostname
# If @mgs_hostname is null, then it means the primary MGS node
check_lnet_connect() {
    declare -i i=$1
    local mgs_node=$2

    local COMMAND RET_STR
    local mgs_prim_nids
    local nids_str=
    local mgs_nid 
    local ping_mgs

    # Execute remote command to check that 
    # this node can contact the MGS node
    verbose_output "Checking lnet connectivity between" \
    "${HOST_NAME[i]} and the MGS node ${mgs_node}"
    mgs_prim_nids=`echo ${MGS_NIDS[i]} | awk -F: '{print $1}'`

    if [ -z "${mgs_node}" -o $MGS_NUM -eq 1 ]; then
        nids_str=${mgs_prim_nids}    # nids of primary MGS node
        if [ -z "${nids_str}" ]; then
            echo >&2 $"`basename $0`: check_lnet_connect() error:"\
            "Check the mgs nids item of host ${HOST_NAME[i]}!"\
            "Missing nids of the primary MGS node!"
            return 1
        fi
    else
        # Get the corresponding NID(s) of the MGS node ${mgs_node}
        # from the "mgs nids" field
        nids_str=$(get_mgs_nids ${mgs_node} ${MGS_NIDS[i]})
        if [ ${PIPESTATUS[0]} -ne 0 ]; then
            echo >&2 "${nids_str}"
            return 1
        fi
    fi

    ping_mgs=false
    for mgs_nid in ${nids_str//,/ }
    do
        COMMAND=$"${LCTL} ping ${mgs_nid} 5 || echo failed 2>&1"
        RET_STR=$(${REMOTE} ${HOST_NAME[i]} "${COMMAND}" 2>&1)
        if [ ${PIPESTATUS[0]} -eq 0 -a "${RET_STR}" = "${RET_STR#*failed*}" ]
        then
            # This node can contact the MGS node
            verbose_output "${HOST_NAME[i]} can contact the MGS" \
            "node ${mgs_node} by using nid \"${mgs_nid}\"!"
            ping_mgs=true
            break
        fi
    done

    if ! ${ping_mgs}; then
        echo >&2 "`basename $0`: check_lnet_connect() error:" \
        "${HOST_NAME[i]} cannot contact the MGS node ${mgs_node}"\
        "with nids - \"${nids_str}\"! Check ${LCTL} command!"
        return 1
    fi

    return 0
}

# Start lnet network in the cluster node and check that 
# this node can contact the MGS node
check_lnet() {
    if ! ${VERIFY_CONNECT}; then
        return 0
    fi

    # Check argument
    if [ $# -eq 0 ]; then
        echo >&2 $"`basename $0`: check_lnet() error: Missing"\
              "argument for function check_lnet()!"
        return 1
    fi

    declare -i i=$1
    declare -i j
    local COMMAND RET_STR

    # Execute remote command to start lnet network
    verbose_output "Starting lnet network in ${HOST_NAME[i]}"
    COMMAND="PATH=\$PATH:/sbin:/usr/sbin modprobe lnet; ${LCTL} network up 2>&1"
    RET_STR=$(${REMOTE} ${HOST_NAME[i]} "${COMMAND}" 2>&1)
    if [ ${PIPESTATUS[0]} -ne 0 -o "${RET_STR}" = "${RET_STR#*LNET configured*}" ]
    then
        echo >&2 "`basename $0`: check_lnet() error: remote" \
                 "${HOST_NAME[i]} error: ${RET_STR}"
        return 1
    fi

    if is_mgs_node ${HOST_NAME[i]}; then
        return 0
    fi

    # Execute remote command to check that 
    # this node can contact the MGS node
    for ((j = 0; j < ${MGS_NUM}; j++)); do
        if ! check_lnet_connect $i ${MGS_NODENAME[j]}; then
            return 1
        fi
    done

    return 0
}

# Start lnet network in the MGS node
start_mgs_lnet() {
    declare -i i
    declare -i idx
    local COMMAND

    if [ -z "${MGS_NODENAME[0]}" -a  -z "${MGS_NODENAME[1]}" ]; then
        if ${USE_ALLNODES}; then
            verbose_output "There is no MGS target in the ${CSV_FILE} file."
        else
            verbose_output "There is no MGS target in the node list \"${NODES_TO_USE}\"."
        fi
        return 0
    fi

    for ((i = ${INIT_IDX}; i < ${MGS_NUM}; i++)); do
        # Execute remote command to add lnet options lines to 
        # the MGS node's modprobe.conf/modules.conf
        idx=${MGS_IDX[i]}
        COMMAND=$"echo \"${MODULE_OPTS[${idx}]}\"|${MODULE_CONFIG}"
        verbose_output "Adding lnet module options to ${MGS_NODENAME[i]}"
        ${REMOTE} ${MGS_NODENAME[i]} "${COMMAND}" >&2 
        if [ ${PIPESTATUS[0]} -ne 0 ]; then
            echo >&2 "`basename $0`: start_mgs_lnet() error:"\
                 "Failed to execute remote command to" \
                 "add module options to ${MGS_NODENAME[i]}!"\
                 "Check ${MODULE_CONFIG}!"
            return 1
        fi

        # Start lnet network in the MGS node
        if ! check_lnet ${idx}; then
            return 1    
        fi
    done

    return 0
}

# Execute remote command to add lnet options lines to remote nodes'
# modprobe.conf/modules.conf and format(mkfs.lustre) Lustre targets
mass_config() {
    local COMMAND
    declare -a REMOTE_PID 
    declare -a REMOTE_CMD 
    declare -i pid_num=0
    declare -i i=0

    if [ ${#HOST_NAME[@]} -eq 0 ]; then
        verbose_output "There are no lustre targets specified."
        return 0
    fi

    if ! $UPGRADE_TARGET; then
        # Start lnet network in the MGS node
        start_mgs_lnet || return 1    
    fi

    for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do
        # Construct the command line of mkfs.lustre
        if ! construct_mkfs_cmdline $i; then
            return 1    
        fi

        # create the mount point on the node
        COMMAND="mkdir -p ${MOUNT_POINT[i]}"
        verbose_output "Creating the mount point ${MOUNT_POINT[i]} on" \
                       "${HOST_NAME[i]}"
        ${REMOTE} ${HOST_NAME[i]} "${COMMAND}" >&2 
        if [ ${PIPESTATUS[0]} -ne 0 ]; then
            echo >&2 "`basename $0`: mass_config() error:"\
                 "Failed to execute remote command to"\
                 "create the mountpoint on ${HOST_NAME[i]}!"
            return 1
        fi

        if ! $UPGRADE_TARGET && ! is_mgs_node ${HOST_NAME[i]}; then
            # Execute remote command to add lnet options lines to 
            # modprobe.conf/modules.conf
            COMMAND=$"echo \"${MODULE_OPTS[i]}\"|${MODULE_CONFIG}"
            verbose_output "Adding lnet module options to" \
                       "${HOST_NAME[i]}"
            ${REMOTE} ${HOST_NAME[i]} "${COMMAND}" >&2 
            if [ ${PIPESTATUS[0]} -ne 0 ]; then
                echo >&2 "`basename $0`: mass_config() error:"\
                     "Failed to execute remote command to"\
                     "add module options to ${HOST_NAME[i]}!"
                return 1
            fi

            # Check lnet networks
            if ! check_lnet $i; then
                return 1    
            fi
        fi

        # Execute remote command to format or upgrade Lustre target
        local OP
        $UPGRADE_TARGET && OP="Upgrading" || OP="Formatting"
        verbose_output "$OP Lustre target ${DEVICE_NAME[i]} on ${HOST_NAME[i]}..."

        COMMAND="$EXPORT_PATH $MKFS_CMD"
        REMOTE_CMD[$pid_num]="$REMOTE ${HOST_NAME[i]} \"$COMMAND\""
        verbose_output "$OP command line is: ${REMOTE_CMD[$pid_num]}"

        $REMOTE ${HOST_NAME[i]} "$COMMAND" &
        REMOTE_PID[$pid_num]=$!
        let pid_num=$pid_num+1
        sleep 1
    done

    # Wait for the exit status of the background remote command
    verbose_output "Waiting for the return of the remote command..."
    fail_exit_status=false
    for ((pid_num = 0; pid_num < ${#REMOTE_PID[@]}; pid_num++)); do
        wait ${REMOTE_PID[${pid_num}]}
        if [ ${PIPESTATUS[0]} -ne 0 ]; then
            echo >&2 "`basename $0`: mass_config() error: Failed"\
            "to execute \"${REMOTE_CMD[${pid_num}]}\"!"
            fail_exit_status=true
        fi
    done

    if ${fail_exit_status}; then
        return 1
    fi    

    verbose_output "Success on all Lustre targets!"
    return 0
}

# get_mntopts hostname device_name failovers
# Construct the mount options of Lustre target @device_name in host @hostname
get_mntopts() {
    local host_name=$1
    local device_name=$2
    local failovers=$3
    local mnt_opts=
    local ret_str

    [ -n "${failovers}" ] && mnt_opts=defaults,noauto || mnt_opts=defaults

    # Execute remote command to check whether the device
    # is a block device or not
    ret_str=$(${REMOTE} ${host_name} \
            "[ -b ${device_name} ] && echo block || echo loop" 2>&1)
    if [ ${PIPESTATUS[0]} -ne 0 -a -n "${ret_str}" ]; then
        echo "`basename $0`: get_mntopts() error:" \
        "remote command to ${host_name} error: ${ret_str}"
        return 1
    fi

    if [ -z "${ret_str}" ]; then
        echo "`basename $0`: get_mntopts() error: remote error:" \
        "No results from remote!" \
        "Check network connectivity between the local host and ${host_name}!"
        return 1
    fi

    [ "${ret_str}" != "${ret_str#*loop}" ] && mnt_opts=${mnt_opts},loop

    echo ${mnt_opts}
    return 0
}

# Execute remote command to modify /etc/fstab to add the new Lustre targets
modify_fstab() {
    declare -i i
    local mntent mntopts device_name
    local COMMAND

    if ! ${MODIFY_FSTAB}; then
        return 0    
    fi

    for ((i = 0; i < ${#HOST_NAME[@]}; i++)); do
        verbose_output "Modify /etc/fstab of host ${HOST_NAME[i]}"\
                   "to add Lustre target ${DEVICE_NAME[i]}"
        mntent=${DEVICE_NAME[i]}"\t\t"${MOUNT_POINT[i]}"\t\t"${FS_TYPE}

        # Get mount options
        if [ -n "${MOUNT_OPTIONS[i]}" ]; then
            # The mount options already specified in the csv file.
            mntopts=${MOUNT_OPTIONS[i]}
        else
            mntopts=$(get_mntopts ${HOST_NAME[i]} ${DEVICE_NAME[i]}\
                    ${FAILOVERS[i]})
            if [ ${PIPESTATUS[0]} -ne 0 ]; then
                echo >&2 "${mntopts}"
                return 1
            fi
        fi

        mntent=${mntent}"\t"${mntopts}"\t"0" "0
        verbose_output "`echo -e ${mntent}`"

        # Execute remote command to modify /etc/fstab
        device_name=${DEVICE_NAME[i]//\//\\/}
        COMMAND=". @scriptlibdir@/lc_common; \
                sed -i \"/^${device_name}\t/d\" \$(fcanon /etc/fstab); \
                echo -e \"${mntent}\" >> \$(fcanon /etc/fstab)"
        ${REMOTE} ${HOST_NAME[i]} "${COMMAND}" >&2
        if [ ${PIPESTATUS[0]} -ne 0 ]; then
            echo >&2 "`basename $0`: modify_fstab() error:"\
            "Failed to modify /etc/fstab of host ${HOST_NAME[i]}"\
            "to add Lustre target ${DEVICE_NAME[i]}!"
            return 1
        fi
    done

    return 0
}

# Main flow
# Check the csv file
if ! check_file $1; then
    exit 1 
fi

# Get the list of nodes to be operated on
NODES_TO_USE=$(get_nodelist)
[ ${PIPESTATUS[0]} -ne 0 ] && echo >&2 "${NODES_TO_USE}" && exit 1

# Check the node list
check_nodelist ${NODES_TO_USE} || exit 1

if ${VERIFY_CONNECT}; then
# Check the network connectivity and hostnames
    echo "`basename $0`: Checking the cluster network connectivity"\
         "and hostnames..."
    if ! ${VERIFY_CLUSTER_NET} ${NODELIST_OPT} ${VERBOSE_OPT} ${CSV_FILE}; then
        exit 1
    fi
    echo "`basename $0`: Check the cluster network connectivity"\
         "and hostnames OK!"
    echo
fi

if $CONFIG_MD_LVM && ! $UPGRADE_TARGET; then
# Configure Linux MD/LVM devices
    echo "`basename $0`: Configuring Linux MD/LVM devices..."
    if ! ${SCRIPT_CONFIG_MD} ${NODELIST_OPT} ${VERBOSE_OPT} ${CSV_FILE}; then
        exit 1
    fi

    if ! ${SCRIPT_CONFIG_LVM} ${NODELIST_OPT} ${VERBOSE_OPT} ${CSV_FILE}; then
        exit 1
    fi
    echo "`basename $0`: Configure Linux MD/LVM devices OK!"
    echo
fi

# Configure the Lustre cluster
echo "`basename $0`: ******** Lustre cluster configuration START ********"
if ! get_items ${CSV_FILE}; then
    exit 1
fi

if ! check_mgs; then
    exit 1
fi

if ! mass_config; then
    exit 1
fi

if ! modify_fstab; then
    exit 1
fi

# Produce HA software's configuration files
if ! config_ha; then
    rm -rf ${TMP_DIRS}
    exit 1
fi

echo "`basename $0`: ******** Lustre cluster configuration END **********"

exit 0