inviso_tool_suite.erl

来自「OTP是开放电信平台的简称」· ERL 代码 · 共 968 行 · 第 1/3 页

    ?l [{file,{TraceCaseFileNameInit,[]}},
	{file,{TraceCaseFileName1,[{'ProcessName',inviso_tool_test_proc}]}},
	{binary,Bin1},
	{file,{TraceCaseFileName2,[]}},
	{binary,Bin2}]=AutostartData1,
    ?l "inviso:tpl(math,module_info,0,[]).\n"=binary_to_list(Bin1),
    ?l "inviso:tpl(math,sin,1,[]).\n"=binary_to_list(Bin2),

    %% Try to activate a faulty tracecase. We shall get the same history as before.
    ?l ok=inviso_tool:atc(tracecase3,id3,[]),
    ?l ok=poll(inviso_tool,get_activities,[],{ok,[]},10),
    ?l {ok,{AutostartData1,NodeResults3}}=
	inviso_tool:get_autostart_data(Nodes,{dependency,infinity}),

    %% Now deactivate a trace case.
    ?l inviso_tool:dtc(tracecase1,id1),
    %% Check that the now deactivated trace case is not part of autostart data
    %% if requested from the tool.
    ?l {ok,{AutostartData2,_NodeResults}}=
	inviso_tool:get_autostart_data(Nodes,{dependency,infinity}),
    ?l [{file,{TraceCaseFileNameInit,[]}},
	{binary,Bin1},
	{file,{TraceCaseFileName2,[]}},
	{binary,Bin2}]=AutostartData2,
    %% Now tracing shall be removed since we deactivated tracecase1.
    ?l lists:foreach(fun(P)->
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[P,flags]],
					{flags,[]},
					10),
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[{math,module_info,1},traced]],
					{traced,false},
					10)
		     end,
		     TestProcs),

    %% Suspend the ANode again and check that when it is reactivated that
    %% tracecase1 is not redone at all. We use an ets table with a counter that is
    %% incremented every time the tracecase1 is executed.
    ?l {ok,[{ANode,ok}]}=rpc:call(CNode,inviso,suspend,[[ANode],testagain]),
    ?l [{counter,SideEffectCounter1}]=rpc:call(ANode,ets,lookup,[test_proc_tab,counter]),
    ?l true=(SideEffectCounter1>0),
    ?l ok=inviso_tool:reactivate(ANode),
    ?l timer:sleep(3000),                    % The delay in tracecase2.
    ?l ok=poll(inviso_tool,get_activities,[],{ok,[]},10),
    %% Now the reactivation is done, check that tracecase1 was not redone at ANode.
    ?l [{counter,SideEffectCounter1}]=rpc:call(ANode,ets,lookup,[test_proc_tab,counter]),

    %% Deactivate tracecase2.
    ?l another_return_value=inviso_tool:sync_dtc(tracecase2,id2),
    %% Immediately check the autostart data (again!). This time we want to see
    %% that the two inviso commands have been joined since there is no tracecase
    %% in between.
    ?l {ok,{AutostartData3,NodeResults}}=
	inviso_tool:get_autostart_data(Nodes,{dependency,infinity}),
    ?l [{file,{TraceCaseFileNameInit,[]}},
	{binary,Bin3}]=AutostartData3,
    ?l "inviso:tpl(math,module_info,0,[]).\ninviso:tpl(math,sin,1,[]).\n"=
	binary_to_list(Bin3),

    %% Check that a deactivating tracecase is not redone at a reactivating node.
    ?l inviso_tool:sync_atc(tracecase5,id5,[]), % Updates the counter.
    %% Yet again suspend the node when we know that tracecase5 has been executed.
    ?l {ok,[{ANode,ok}]}=rpc:call(CNode,inviso,suspend,[[ANode],testagain2]),
    ?l timer:sleep(100),                     % Subscription reaches the tool.
    ?l [{counter,SideEffectCounter2A}]=rpc:call(ANode,ets,lookup,[test_proc_tab,counter]),
    ?l [BNode|_]=OtherNodes,
    ?l [{counter,SideEffectCounter2B}]=rpc:call(BNode,ets,lookup,[test_proc_tab,counter]),
    ?l ok=inviso_tool:dtc(tracecase5,id5),   % In here there is a 2000 ms delay!
    ?l ok=inviso_tool:reactivate(ANode),
    %% Check that the reactivator is done, but that the tracecase remains. The
    %% reactivator should be done pretty quickly since there are no delays in the
    %% still active tracecases.
    ?l ok=poll(inviso_tool,
	       get_activities,
	       [],
	       {ok,[{tracecases,[{{tracecase5,id5},deactivating}]}]},
	       10),
    ?l ok=poll(inviso_tool,get_activities,[],{ok,[]},30),
    ?l [{counter,SideEffectCounter2A}]=rpc:call(ANode,ets,lookup,[test_proc_tab,counter]),
    ?l SideEffectCounter2B1=SideEffectCounter2B+1,
    ?l [{counter,SideEffectCounter2B1}]=rpc:call(BNode,ets,lookup,[test_proc_tab,counter]),

    ?l stop_inviso_tool_session(CNode,1,Nodes),

    %% Check that you can not start trace cases when the session is stopped.
    ?l {error,no_session}=inviso_tool:atc(tracecase2,id3,[]),
    %% But the history shall be there to retrieve.
    ?l {ok,{AutostartData3,NodeResults}}=
	inviso_tool:get_autostart_data(Nodes,{dependency,infinity}),
    ?l {ok,{inactive,running}}=inviso_tool:get_node_status(ANode),

    ?l stop_inviso_tool(CNode,Nodes),
    ok.
%% -----------------------------------------------------------------------------

%% This test case tests mainly that reconnect and reinitiations of a node works.
dist_reconnect(doc) -> [""];
dist_reconnect(suite) -> [];
dist_reconnect(Config) when list(Config) ->
    RemoteNodes=get_remotenodes_config(Config),
    [RegExpNode|OtherNodes]=RemoteNodes,
    CNode=node(),
    Nodes=RemoteNodes,
    DataDir=?config(data_dir,Config),
    PrivDir=?config(priv_dir,Config),
    Opts=[{regexp_node,RegExpNode},
	  {tdg,{?MODULE,tdg,[PrivDir]}},
	  {tc_def_file,?TC_DEF_FILE},
	  {initial_tcs,[{tracecase_init,[]}]},
	  {dir,DataDir}],                    % This is where we find tracecases.
    ?l start_inviso_tool(Nodes,CNode,Opts), 
    ?l start_inviso_tool_session(CNode,[],1,Nodes),
    %% Start a test process at every node with a runtime component.
    ?l lists:foreach(fun(N)->spawn(N,?MODULE,test_proc_init,[]) end,Nodes),
    %% Find the pids of the test processes.
    ?l TestProcs=lists:map(fun(N)->rpc:call(N,erlang,whereis,[inviso_tool_test_proc]) end,
			   Nodes),
    ?l true=(1=<length(TestProcs)),
    ?l ok=inviso_tool:atc(tracecase1,id1,[{'ProcessName',inviso_tool_test_proc}]),
    ?l ok=poll(inviso_tool,get_activities,[],{ok,[]},20),

    %% Now we want to crash a node. Lets choose the RegExp node, then we can test
    %% that regexp expansion is done elsewhere too.
    ?l test_server:stop_node(RegExpNode),
    ?l ok=poll(net_adm,ping,[RegExpNode],pang,10),
    %% Make time for the monitoring signal to reach inviso_c and for inviso_c to
    %% inform its subscribers e.g inviso_tool.
    ?l timer:sleep(100),
    ?l {_,[{_Node,Modules}]}=
	inviso_tool_lib:expand_module_names([node()],"application.*",[]),
    ?l NrOfModules=length(Modules),
    %% This also checks that regexp expansion can be made at another node
    %% than RexExpNode.
    ?l {ok,NodeResults1}=inviso_tool:inviso(tp,["application.*",module_info,0,[]]),
    ?l true=check_noderesults(OtherNodes,
			      fun({_N,{ok,Ints}}) when list(Ints) ->
				      NrOfModules=lists:sum(Ints),
				      true;
				 (_) ->
				      false
			      end,
			      NodeResults1),

    %% Do some faulty tests on the dead node.
    ?l {ok,{ok,[{RegExpNode,{error,down}}]}}=
	inviso_tool:reinitiate_session([RegExpNode]),
    ?l {ok,down}=inviso_tool:get_node_status(RegExpNode),

    %% Now it is time to restart the crashed node and reconnect it and then
    %% finally reinitiate it.
    ?l RegExpNodeString=atom_to_list(RegExpNode),
    ?l {match,Pos,1}=regexp:first_match(RegExpNodeString,"@"),
    ?l RegExpNodeName=list_to_atom(lists:sublist(RegExpNodeString,Pos-1)),
    ?l test_server:start_node(RegExpNodeName,peer,[]),
    ?l ok=poll(net_adm,ping,[RegExpNode],pong,20),
    ?l SuiteDir=filename:dirname(code:which(?MODULE)),
    ?l true=rpc:call(RegExpNode,code,add_patha,[SuiteDir]),
    ?l ok=rpc:call(RegExpNode,application,start,[runtime_tools]),
    ?l ok=poll(rpc,call,[RegExpNode,erlang,whereis,[inviso_rt]],undefined,20),
    ?l {ok,down}=inviso_tool:get_node_status(RegExpNode),

    %% Restart the test process.
    ?l spawn(RegExpNode,?MODULE,test_proc_init,[]),
    ?l ok=poll(rpc,
	       call,
	       [RegExpNode,erlang,whereis,[inviso_tool_test_proc]],
	       fun(P) when pid(P) -> true;
		  (undefined) -> false
	       end,
	       10),
    ?l TPid=rpc:call(RegExpNode,erlang,whereis,[inviso_tool_test_proc]),
    ?l {ok,[{RegExpNode,{ok,{inactive,running}}}]}=inviso_tool:reconnect_nodes([RegExpNode]),
    %% Try to reconnect the node again and an unknown node.
    ?l UnknownNode='unknown@nonexistant',
    ?l {ok,[{RegExpNode,{error,already_connected}},{UnknownNode,{error,unknown_node}}]}=
	inviso_tool:reconnect_nodes([RegExpNode,UnknownNode]),
    ?l {ok,{ok,[{RegExpNode,_Result}]}}=inviso_tool:reinitiate_session([RegExpNode]),
    ?l ok=poll(rpc,
	       call,
	       [RegExpNode,erlang,trace_info,[TPid,flags]],
	       {flags,[call]},
	       10),
    ?l {ok,{ok,[{RegExpNode,{error,already_in_session}},{UnknownNode,{error,unknown_node}}]}}=
	inviso_tool:reinitiate_session([RegExpNode,UnknownNode]),

    %% Suspend RegExpNode and test that it can not be reinitiated.
    ?l {ok,[{RegExpNode,ok}]}=rpc:call(CNode,inviso,suspend,[[RegExpNode],yetatest]),
    ?l {ok,[{RegExpNode,{error,already_connected}}]}=inviso_tool:reconnect_nodes([RegExpNode]),
    ?l {ok,{ok,[{RegExpNode,{error,suspended}}]}}=inviso_tool:reinitiate_session([RegExpNode]),

    %% Now we start a tracecase that will never terminate. We then reactivate the
    %% suspended node. Then there will be a running reactivator and a running
    %% tracecase to kill.
    ?l ok=inviso_tool:atc(tracecase4,id4,[]), % This one will not terminate.
    ?l ok=inviso_tool:reactivate(RegExpNode),
    ?l ok=poll(inviso_tool,
	       get_activities,
	       [],
	       fun({ok,L}) when length(L)==2 -> true;
		  (_) -> false
	       end,
	       20),
    %% Now the reactivator and the tracecase shall be stuck(!)
    ?l {links,ToolLinks}=process_info(whereis(inviso_tool),links),
    ?l [P1,P2]=lists:foldl(fun(P,Acc)->case process_info(P,initial_call) of
					   {initial_call,{inviso_tool,_,_}} ->
					       [P|Acc];
					   _ ->
					       Acc
				       end
			   end,
			   [],
			   ToolLinks),
    ?l stop_inviso_tool_session(CNode,1,Nodes),
    %% Check that the processes are killed.
    ?l ok=poll(erlang,process_info,[P1],undefined,10),
    ?l ok=poll(erlang,process_info,[P2],undefined,10),
    ?l stop_inviso_tool(CNode,Nodes),
    ok.
%% -----------------------------------------------------------------------------

%% This test tests that we can adopt a running inviso runtime component and
%% mark it as tracing-running.
dist_adopt(doc) -> [""];
dist_adopt(suite) -> [];
dist_adopt(Config) when list(Config) ->
    RemoteNodes=get_remotenodes_config(Config),
    [RegExpNode|_]=RemoteNodes,
    CNode=node(),
    Nodes=RemoteNodes,
    DataDir=?config(data_dir,Config),
    PrivDir=?config(priv_dir,Config),

    %% Then first start runtime components at different nodes for us to
    %% later adopt.
    ?l {ok,_IPid}=inviso:start(),
    ?l {ok,NodeResults1}=inviso:add_nodes(Nodes,a_tag,[{dependency,infinity}]),
    ?l true=check_noderesults(Nodes,{ok,new},NodeResults1),
    ?l [ANode|OtherNodes]=Nodes,
    ?l {ok,[{ANode,_LogResult}]}=
	inviso:init_tracing([ANode],
			    [{trace,{file,filename:join(PrivDir,"dist_adopt_adoptednode.log")}},
			     {ti,{file,filename:join(PrivDir,"dist_adopt_adoptednode.ti")}}]),
    ?l inviso:stop(),
    ?l timer:sleep(100),
    ?l ok=poll(erlang,whereis,[inviso_c],undefined,10),
    ?l lists:foreach(fun(N)->true=(is_pid(rpc:call(N,erlang,whereis,[inviso_rt]))) end,
		     Nodes),

    %% Now start the tool and watch it adopt the runtimes.
    Opts=[{regexp_node,RegExpNode},
	  {tdg,{?MODULE,tdg,[PrivDir]}},
	  {tc_def_file,?TC_DEF_FILE},
	  {initial_tcs,[{tracecase_init,[]}]},
	  {dir,DataDir}],                    % This is where we find tracecases.
    ?l Options=[{nodes,Nodes},{c_node,CNode}|Opts],
    ?l {ok,_Pid}=inviso_tool:start(Options),
    ?l ok=poll(erlang,whereis,[inviso_tool],fun(X)->true=is_pid(X) end,10),
    ?l io:format("LoopData:~p~n",[inviso_tool:get_loopdata()]),
    ?l {ok,{1,InvisoReturn}}=inviso_tool:start_session([]),
    ?l io:format("Invisoreturn:~p~n",[InvisoReturn]),
    %% Now check that all nodes are tracing.
    ?l lists:foreach(fun(N)->ok=poll(rpc,
				     call,
				     [CNode,inviso,get_status,[[N]]],
				     fun({ok,[{_N,{ok,{tracing,running}}}]})->true;
					(_) ->false
				     end,
				     10)
		     end,
		     Nodes),

    %% At this point all nodes shall be tracing. However the initial tracecase
    %% shall not have been executed at ANode since it was adopted by the tool.
    ?l {traced,false}=rpc:call(ANode,erlang,trace_info,[{lists,module_info,1},traced]),
    ?l lists:foreach(fun(N)->
			     {traced,local}=
				 rpc:call(N,erlang,trace_info,[{lists,module_info,1},traced])
		     end,
		     OtherNodes),
    ?l stop_inviso_tool_session(CNode,1,Nodes),
    ?l [BNode|_]=OtherNodes,
    %% Since nodes are not cleared the pattern still be there.
    ?l {traced,local}=rpc:call(BNode,erlang,trace_info,[{lists,module_info,1},traced]),
    ?l start_inviso_tool_session(CNode,[],2,Nodes),
    ?l stop_inviso_tool_session(CNode,2,Nodes),
    ?l {ok,_NodeResults}=inviso_tool:stop(),
    ?l ok=poll(erlang,whereis,[inviso_tool],undefined,10),
    ?l ok=poll(rpc,call,[CNode,erlang,whereis,[inviso_c]],undefined,10),

    ok.
%% -----------------------------------------------------------------------------

%% This test tests that saving and restoring a history works.
dist_history(doc) -> [""];
dist_history(suite) -> [];
dist_history(Config) when list(Config) ->		     
    RemoteNodes=get_remotenodes_config(Config),
    [RegExpNode|_]=RemoteNodes,
    CNode=RegExpNode,                       % We use a remote control component.
    Nodes=RemoteNodes,
    DataDir=?config(data_dir,Config),
    PrivDir=?config(priv_dir,Config),

    %% Start up the tool and a couple of inviso runtimes.
    Opts=[{regexp_node,RegExpNode},
	  {tdg,{?MODULE,tdg,[PrivDir]}},
	  {tc_def_file,?TC_DEF_FILE},
	  {initial_tcs,[{tracecase_init,[]}]},
	  {dir,DataDir}],                    % This is where we find tracecases.
    ?l start_inviso_tool(Nodes,CNode,Opts), 
    ?l start_inviso_tool_session(CNode,[],1,Nodes),
    %% Start a test process at every node with a runtime component.
    ?l lists:foreach(fun(N)->spawn(N,?MODULE,test_proc_init,[]) end,Nodes),

    %% Activate tracing of the test process.
    ?l ok=inviso_tool:atc(tracecase1,id1,[{'ProcessName',inviso_tool_test_proc}]),
    ?l ok=poll(inviso_tool,get_activities,[],{ok,[]},10),
    ?l TestProcs=lists:map(fun(N)->rpc:call(N,erlang,whereis,[inviso_tool_test_proc]) end,