mpdman.py

fortran并行计算包

                    msg['procinfo'].append((socket.gethostname(),self.clientPgm,clientPid))
                if self.numConndWithIO >= self.numWithIO:
                    self.ring.rhsSock.send_dict_msg(msg)  # forward it on
                else:
                    self.holdingJobgoLoop1 = msg
        elif msg['cmd'] == 'jobgo_loop_2':
            if self.myRank != 0:
                self.ring.rhsSock.send_dict_msg(msg)  # forward it on
            if not self.singinitPID:
                self.handshake_sock_man_end.send_char_msg('go\n')
                self.handshake_sock_man_end.close()
            self.jobStarted = 1
        elif msg['cmd'] == 'info_from_parent_in_tree':
            if int(msg['to_rank']) == self.myRank:
                self.parentIfhn = msg['parent_ifhn']
                self.parentPort = msg['parent_port']
                self.parentStdoutSock = MPDSock(name='stdout_ro_parent')
                self.parentStdoutSock.connect((self.parentIfhn,self.parentPort))
                msgToSend = { 'cmd' : 'child_in_stdout_tree', 'from_rank' : self.myRank }
                self.parentStdoutSock.send_dict_msg(msgToSend)
                self.parentStderrSock = MPDSock(name='stderr_ro_parent')
                self.parentStderrSock.connect((self.parentIfhn,self.parentPort))
                msgToSend = { 'cmd' : 'child_in_stderr_tree', 'from_rank' : self.myRank }
                self.parentStderrSock.send_dict_msg(msgToSend)
            else:
                self.ring.rhsSock.send_dict_msg(msg)
        elif msg['cmd'] == 'end_barrier_loop_1':
            if self.myRank == 0:
                msgToSend = { 'cmd' : 'end_barrier_loop_2' }
                self.ring.rhsSock.send_dict_msg(msgToSend)
            else:
                if self.numDone >= self.numWithIO:
                    if self.ring.rhsSock:
                        self.ring.rhsSock.send_dict_msg(msg)
                else:
                    self.holdingEndBarrierLoop1 = 1
        elif msg['cmd'] == 'end_barrier_loop_2':
            self.endBarrierDone = 1
            if self.myRank != 0:
                self.ring.rhsSock.send_dict_msg(msg)
        elif msg['cmd'] == 'pmi_barrier_loop_1':
            if self.myRank == 0:
                msgToSend = { 'cmd' : 'pmi_barrier_loop_2' }
                self.ring.rhsSock.send_dict_msg(msgToSend)
                if self.doingBNR:    ## BNR
                    pmiMsgToSend = 'cmd=client_bnr_fence_out\n'
                    self.pmiSock.send_char_msg(pmiMsgToSend)
                    sleep(0.1)  # minor pause before intr
                    os.kill(clientPid,signal.SIGUSR1)
                else:
                    if self.pmiSock:
                        pmiMsgToSend = 'cmd=barrier_out\n'
                        self.pmiSock.send_char_msg(pmiMsgToSend)
            else:
                self.holdingPMIBarrierLoop1 = 1
                if self.pmiBarrierInRecvd:
                    self.ring.rhsSock.send_dict_msg(msg)
        elif msg['cmd'] == 'pmi_barrier_loop_2':
            self.pmiBarrierInRecvd = 0
            self.holdingPMIBarrierLoop1 = 0
            if self.myRank != 0:
                self.ring.rhsSock.send_dict_msg(msg)
                if self.doingBNR:    ## BNR
                    pmiMsgToSend = 'cmd=client_bnr_fence_out\n'
                    self.pmiSock.send_char_msg(pmiMsgToSend)
                    sleep(0.1)  # minor pause before intr
                    os.kill(clientPid,signal.SIGUSR1)
                else:
                    if self.pmiSock:
                        pmiMsgToSend = 'cmd=barrier_out\n'
                        self.pmiSock.send_char_msg(pmiMsgToSend)
        elif msg['cmd'] == 'pmi_get':
            if msg['from_rank'] == self.myRank:
                if self.pmiSock:  # may have disappeared in early shutdown
                    pmiMsgToSend = 'cmd=get_result rc=-1 key="%s"\n' % msg['key']
                    self.pmiSock.send_char_msg(pmiMsgToSend)
            else:
                key = msg['key']
                kvsname = msg['kvsname']
                if self.KVSs.has_key(kvsname)  and  self.KVSs[kvsname].has_key(key):
                    value = self.KVSs[kvsname][key]
                    msgToSend = { 'cmd' : 'response_to_pmi_get', 'value' : value,
                                  'to_rank' : msg['from_rank'] }
                    self.ring.rhsSock.send_dict_msg(msgToSend)
                else:
                    self.ring.rhsSock.send_dict_msg(msg)
        elif msg['cmd'] == 'pmi_getbyidx':
            if msg['from_rank'] == self.myRank:
                if self.pmiSock:  # may have disappeared in early shutdown
                    self.KVSs[self.default_kvsname].update(msg['kvs'])
                    if self.KVSs[self.default_kvsname].keys():
                        key = self.KVSs[self.default_kvsname].keys()[0]
                        val = self.KVSs[self.default_kvsname][key]
                        pmiMsgToSend = 'cmd=getbyidx_results rc=0 nextidx=1 key=%s val=%s\n' % \
                                       (key,val)
                    else:
                        pmiMsgToSend = 'cmd=getbyidx_results rc=-2 reason=no_more_keyvals\n'
                    self.pmiSock.send_char_msg(pmiMsgToSend)
            else:
                msg['kvs'].update(self.KVSs[self.default_kvsname])
                self.ring.rhsSock.send_dict_msg(msg)
        elif msg['cmd'] == 'response_to_pmi_get':
            if msg['to_rank'] == self.myRank:
                if self.pmiSock:  # may have disappeared in early shutdown
                    pmiMsgToSend = 'cmd=get_result rc=0 value=%s\n' % (msg['value'])
                    self.pmiSock.send_char_msg(pmiMsgToSend)
            else:
                self.ring.rhsSock.send_dict_msg(msg)
        elif msg['cmd'] == 'signal':
            if msg['signo'] == 'SIGINT':
                if not self.gdb:
                    self.jobEndingEarly = 1
                for s in self.spawnedChildSocks:
                    s.send_dict_msg(msg)
                if self.myRank != 0:
                    if self.ring.rhsSock:  # still alive ?
                        self.ring.rhsSock.send_dict_msg(msg)
                    if self.gdb:
                        os.kill(clientPid,signal.SIGINT)
                    else:
                        try:
                            pgrp = clientPid * (-1)   # neg Pid -> group
                            os.kill(pgrp,signal.SIGKILL)   # may be reaped by sighandler
                        except:
                            pass
            elif msg['signo'] == 'SIGKILL':
                self.jobEndingEarly = 1
                for s in self.spawnedChildSocks:
                    s.send_dict_msg(msg)
                if self.myRank != 0:
                    if self.ring.rhsSock:  # still alive ?
                        self.ring.rhsSock.send_dict_msg(msg)
                    if self.gdb:
                        os.kill(clientPid,signal.SIGUSR1)   # tell gdb driver to kill all
                    else:
                        try:
                            pgrp = clientPid * (-1)   # neg Pid -> group
                            os.kill(pgrp,signal.SIGKILL)   # may be reaped by sighandler
                        except:
                            pass
            elif msg['signo'] == 'SIGTSTP':
                if msg['dest'] != self.myId:
                    self.ring.rhsSock.send_dict_msg(msg)
                    try:
                        pgrp = clientPid * (-1)   # neg Pid -> group
                        os.kill(pgrp,signal.SIGTSTP)   # may be reaped by sighandler
                    except:
                        pass
            elif msg['signo'] == 'SIGCONT':
                if msg['dest'] != self.myId:
                    self.ring.rhsSock.send_dict_msg(msg)
                    try:
                        pgrp = clientPid * (-1)   # neg Pid -> group
                        os.kill(pgrp,signal.SIGCONT)   # may be reaped by sighandler
                    except:
                        pass
        elif msg['cmd'] == 'client_exit_status':
            if self.myRank == 0:
                if self.conSock:
                    self.conSock.send_dict_msg(msg,errprint=0)
            else:
                if self.ring.rhsSock:
                    self.ring.rhsSock.send_dict_msg(msg)
        elif msg['cmd'] == 'collective_abort':
            self.jobEndingEarly = 1
            if msg['src'] != self.myId:
                if self.ring.rhsSock:  # still alive ?
                    self.ring.rhsSock.send_dict_msg(msg)
            if self.conSock:
                msgToSend = { 'cmd' : 'job_aborted_early', 'jobid' : self.jobid,
                              'rank' : msg['rank'], 
                              'exit_status' : msg['exit_status'] }
                self.conSock.send_dict_msg(msgToSend,errprint=0)
            try:
                pgrp = clientPid * (-1)   # neg Pid -> group
                os.kill(pgrp,signal.SIGKILL)   # may be reaped by sighandler
            except:
                pass
        elif msg['cmd'] == 'startup_status':
            if msg['rc'] < 0:
                self.jobEndingEarly = 1
                try:
                    pgrp = clientPid * (-1)   # neg Pid -> group
                    os.kill(pgrp,signal.SIGKILL)   # may be reaped by sighandler
                except:
                    pass
            ##### RMB if msg['src'] == self.myId:
            if self.myRank == 0:
                if self.conSock:
                    self.conSock.send_dict_msg(msg,errprint=0)
            else:
                if msg['src'] != self.myId  and  self.ring.rhsSock:  # rhs still alive ?
                    self.ring.rhsSock.send_dict_msg(msg)
        elif msg['cmd'] == 'stdin_from_user':
            if msg['src'] != self.myId:
                self.ring.rhsSock.send_dict_msg(msg)
                if in_stdinRcvrs(self.myRank,self.stdinDest):
                    if msg.has_key('eof'):
                        if self.subproc:    # must close subproc's file (not just the fd)
                            self.subproc.stdin.close()
                        else:
                            os.close(self.fd_write_cli_stdin)
                    else:
                        os.write(self.fd_write_cli_stdin,msg['line'])
        elif msg['cmd'] == 'stdin_dest':
            if msg['src'] != self.myId:
                self.stdinDest = msg['stdin_procs']
                self.ring.rhsSock.send_dict_msg(msg)
        elif msg['cmd'] == 'interrupt_peer_with_msg':    ## BNR
            if int(msg['torank']) == self.myRank:
                if self.pmiSock:  # may have disappeared in early shutdown
                    pmiMsgToSend = '%s\n' % (msg['msg'])
                    self.pmiSock.send_char_msg(pmiMsgToSend)
                    sleep(0.1)  # minor pause before intr
                    os.kill(clientPid,signal.SIGUSR1)
            else:
                self.ring.rhsSock.send_dict_msg(msg)
        elif msg['cmd'] == 'tv_ready':
            self.tvReady = 1
            if self.myRank != 0:
                msg['src'] = self.myId
                self.ring.rhsSock.send_dict_msg(msg)
                if self.pmiSock:    # should be valid sock if running tv
                    pmiMsgToSend = 'cmd=tv_ready\n'
                    self.pmiSock.send_char_msg(pmiMsgToSend)
        else:
            mpd_print(1, 'unexpected msg recvd on lhsSock :%s:' % msg )

    def handle_rhs_input(self,sock):
        msg = self.ring.rhsSock.recv_dict_msg()  #### NOT USING msg; should I ?
        mpd_print(0000, 'rhs died' )
        self.streamHandler.del_handler(self.ring.rhsSock)
        self.ring.rhsSock.close()
        self.ring.rhsSock = 0
    def handle_cli_stdout_input(self,sock):
        line = mpd_read_nbytes(sock,1024)  # sock is self.fd_read_cli_stdout
        if not line:
            if self.subproc:    # must close subproc's file (not just the fd)
                self.subproc.stdout.close()
            else:
                self.streamHandler.del_handler(self.fd_read_cli_stdout)
                os.close(self.fd_read_cli_stdout)
            self.numDone += 1
            if self.numDone >= self.numWithIO:
                if self.parentStdoutSock:
                    self.parentStdoutSock.close()
                    self.parentStdoutSock = 0
                if self.parentStderrSock:
                    self.parentStderrSock.close()
                    self.parentStderrSock = 0
        else:
            if self.parentStdoutSock:
                if self.lineLabelFmt:
                    lineLabel = self.create_line_label(self.lineLabelFmt,self.spawned)
                    splitLine = line.split('\n',1024)
                    if self.startStdoutLineLabel:
                        line = lineLabel