/*-------------------------------------------------------------- * Copyright (c) 2006-2011 * SAS Institute Inc., Cary, NC, USA. All Rights Reserved. * * SAS AUTOCALL MACRO LIBRARY * * NAME: rdcspawn.sas * SUPPORT: stchri * PURPOSE: Macro for Risk Dimensions to distribute a * project run on MULTIPLE REMOTE machines using the * SAS Connect Spawner. * * USAGE: %rdcspawn( env=, project=, portfolio=, * envLibname=, netPathEnv=, netPathTemp=, * connectData=, * blocks=, splitby=, roptions= ); * * env : sasname for risk analysis environment. * proj : name of project to run. * port : name of portfolio to use. * envLibname : libname used when creating environment. * netPathEnv : full pathname for the environment * Must be a shared/network path accessible * by all nodes in the Network! * netPathTemp : full pathname for the block output * Must be a shared/network path accessible * by all nodes in the Network! * connectData= : Data set with SAS Connect info. Must contain * the following columns: * HOST, PORT, CPUS, USERID, PASSWORD. * Note that password can be encrypted with * proc pwencode. * blocks= : number of blocks to break either the * Market States or the Instruments into. * Defaults to -1, indicating to use as many as * available cpus in the Network. * splitby= : states or instruments * rpoptions= : any additional runproject statement * options * * NOTES: 1. This macro assumes that all machines in the network can see * a common network (shared) drive using the same name. This area * is where the Risk Environment should be set up and passed * as parameters 'netPathEnv' and 'netPathTemp'. * * 2. All data sets, catalogs, environments in project * must use the fully qualified libref passed above * ('envLibname' parameter). * EXAMPLE: * * * %rdcspawn( connectData=nodes, * env=myenv, proj=myproject, port=myportfolio, * envLibref=mylibref, * netPathEnv='/nfs/risk/stuff/', netPathTemp='/temp', * blocks=5, * rpoptions=date = '22Jan98'd * options=(alpha=0.01 epsilon=1.e-2 outall) * ); * * %rdcspawn( connectData=nodes, * env=myenv, proj=myproject, port=myportfolio, * envLibref=mylibref, * netPathEnv='/nfs/risk/stuff/', netPathTemp='/temp', * splitby=instruments * ); * *-------------------------------------------------------------*/ %macro rdcspawn( env =, proj =, port =, envLibname =, netPathEnv =, netPathTemp =, connectData =, blocks =-1, splitby =states, rpoptions =); %if &connectData eq %str() %then %do; %put ERROR: The CONNECTDATA parameter is required for the RDCSPAWN macro.; %return; %end; %if &envLibname eq %str() %then %do; %put ERROR: The ENVLIBNAME parameter is required for the RDCSPAWN macro.; %return; %end; %if &netPathEnv eq %str() %then %do; %put ERROR: The NETPATHENV parameter is required for the RDCSPAWN macro.; %return; %end; %if &netPathTemp eq %str() %then %do; %put ERROR: The NETPATHTEMP parameter is required for the RDCSPAWN macro.; %return; %end; %if &env eq %str() %then %do; %put ERROR: The ENV parameter is required for the RDCSPAWN macro.; %return; %end; %if &proj eq %str() %then %do; %put ERROR: The PROJ parameter is required for the RDCSPAWN macro.; %return; %end; %if &port eq %str() %then %do; %put ERROR: The PORT parameter is required for the RDCSPAWN macro.; %return; %end; /***** determine what type of job splitting are we doing *****/ %if &splitby ^= states %then %do; %let part= portpart; %let projpart= ags; %let partslib= aggregationlib; %end; %else %do; %let part= analpart; %let projpart= values; %let partslib= valueslib; %end; /***** run asynchronous loop *****/ %rdcAsynchSubmitLoop(&blocks, &connectData, blocks); /***** merge resultant files back together *****/ libname &envLibname &netPathEnv; libname rskparts &netPathTemp; proc risk; setoptions nobacksave; environment open=&envLibname..&env; runproject &proj portfolio = &port( &partslib = rskparts, %if &splitby ^= states %then %do; n_portparts= &blocks %end; %else %do; n_analparts= &blocks %end; ) endpart = results &rpoptions ; run; %mend; /*--------------------------------------------------------------------------- ** ** rdcAsynchSubmitLoop ** ** This macro will operate an asynchronous remote signon/submission/signoff ** state machine. As soon as remote hosts respond, they will receive jobs ** until all jobs are executed. When a job on a remote host finishes, the ** next available job is sent to that host until all the jobs are exhausted. ** **-------------------------------------------------------------------------*/ %macro rdcAsynchSubmitLoop(LoopCount, ConnectData, MacroNameToHoldLoopCount); %local cnt; /* Loop counter */ %local maxHosts; /* Maximum number of hosts */ %local badHosts; /* Number of hosts where signon failed */ %local gridJobCount; /* Number of jobs */ %local gridJobsComplete; /* Number of jobs completed */ %local iHost; /* Loop id of host we are evaluating */ /** Calculate the number of jobs. Reset the number we have completed ***/ %let gridJobCount = &LoopCount; %let gridJobsComplete = 0; /*** Set the sleep interval ***/ %let gridNumSecs=1; %let gridUnits=1; /***** get Max hosts from connectData *****/ %let maxHosts=0; data _hosts; set &connectData end= LAST; retain ihost 0; do i=1 to CPUS; ihost = ihost + 1; output; end; if LAST then do; call symput("maxHosts", ihost); end; drop i CPUS; run; proc print data=_hosts; run; %put RDCSPAWN: Maximum number of nodes is &maxHosts; %if &gridJobCount le 0 %then %do; %let gridJobCount= &maxHosts; %end; %else %if &maxHosts > &gridJobCount %then %do; %put RDCSPAWN: Reducing MaxHosts to number requested: &gridJobCount; %let maxHosts = &gridJobCount; %end; /*** Set the passed-in macro to the number of blocks being created. ***/ data _null_; call symput("&MacroNameToHoldLoopCount", "&maxHosts"); run; /*--------------------------------------------------------------------- ** Initialize the grid state for all instances to indicate a signon ** is needed. ** ** Grid host state can be the following: ** 0 - awaiting signon submission ** 1 - signon occurred, awaiting completion ** 2 - signon completed, awaiting init rsubmit completion ** 3 - signon completed, awaiting job rsubmit completion ** 9 - signon retries failed - host eliminated ** ** Signon states can be the following (per SAS/CONNECT): ** 0 - signon completed successfully ** 1 - signon failed ** 2 - already signed on to current session ** 3 - signon is in progress ** ** Rsubmit states can be the following (per SAS/CONNECT): ** 0 - rsubmit completed successfully ** 1 - rsubmit failed ** 2 - rsubmit is in progress ** ** ** Grid job state can be the following: ** 0 - awaiting submission ** 1 - rsubmit occurred, awaiting rsubmit completion ** 2 - rsubmit completed, done **-------------------------------------------------------------------*/ %do cnt=1 %to &maxHosts; %global SignonState&cnt; %global gridRsubmitState&cnt; %let gridHost&cnt = ""; %let gridHostRetry&cnt = 2; %let gridHostState&cnt = 0; /* State = awaiting submission */ %let gridHostJob&cnt = 0; %let SignonState&cnt = 3; /* State = In progress */ %let gridSignonRetry&cnt = 2; %let gridRsubmitState&cnt = 3; /* State = In progress */ %end; %do cnt=1 %to &gridJobCount; %let gridJobRetry&cnt = 2; %let gridJobState&cnt = 0; /* State = awaiting submission */ %end; /*--------------------------------------------------------------------- ** Loop waiting for all instances to finish, provided there are hosts ** to do the work on. **-------------------------------------------------------------------*/ %let iHost = 1; %let badHosts = 0; %do %while ((&maxHosts ne 0) and (&gridJobsComplete ne &gridJobCount) and (&badHosts ne &maxHosts)); /*----------------------------------------------------------------- ** If the host state indicates it has not signed on, signon ** asynchronously to the grid and set the state to waiting for ** signon to complete. **---------------------------------------------------------------*/ %if (&&gridHostState&iHost eq 0) %then %do; %if (&&gridHostRetry&iHost ne 0) %then %do; data _null_; set _hosts(where=(iHost=&iHost)); call symput('__Port', trim(left(PORT))); call symput('__Host', trim(left(HOST))); call symput('__User', trim(left(USERID))); call symput('__Password', trim(left(PASSWORD))); run; %let Host&iHost = &__Host &__Port; %let gridHost&iHost = &iHost (&&Host&iHost); %put RDCSPAWN: Performing signon for host &&gridHost&iHost; signon Host&iHost user="&__user" password="&__password" cmacvar=SignonState&iHost; %let gridHostState&iHost = 1; %let gridHostRetry&iHost = %eval(&&gridHostRetry&iHost - 1); %let gridHostJobs&iHost = 0; %end; %else %do; %put RDCSPAWN: Signon retry count for instance &&gridHost&iHost exceeded; %let gridHostState&iHost = 9; %let badHosts = %eval(&badHosts + 1); %end; %end; /*----------------------------------------------------------------- ** If the host state indicates that the signon has been ** done, but has not completed, check to see if the signon has ** finished through the cmacvar value. If the signon has ** finished, initialize the host and update the host state to ** waiting for the init rsubmit to finish. If the signon failed, ** reset the state to try to signon again. **---------------------------------------------------------------*/ %else %if (&&gridHostState&iHost eq 1) %then %do; /*------------------------------------------------------------- ** If the signon worked, INITIALIZE HOST **-----------------------------------------------------------*/ %if (&&SignonState&iHost eq 0 or &&SignonState&iHost eq 2) %then %do; %put RDCSPAWN: Performing initialization for host &&gridHost&iHost; /*** push macro needed vars to remote host ***/ %syslput REM_JOBCOUNT = &gridJobCount /remote = Host&iHost; %syslput REM_HOSTNUM = &iHost /remote = Host&iHost; %syslput REM_HOSTNAME = &&gridHost&iHost /remote = Host&iHost; %syslput rem_env= &env /remote = Host&iHost; %syslput rem_project= &proj /remote = Host&iHost; %syslput rem_netPathEnv= &netPathEnv /remote = Host&iHost; %syslput rem_envLibname= &envLibname /remote = Host&iHost; %syslput rem_part= &part /remote = Host&iHost; %syslput rem_projpart= &projpart /remote = Host&iHost; %syslput rem_netPathTemp= &netPathTemp /remote = Host&iHost; %syslput rem_portfolio= &port /remote = Host&iHost; %if &rpoptions ne %str() %then %do; %syslput rem_options= &rpoptions /remote = Host&iHost; %end; %else %do; /* Make sure rem_options is defined! */ %syslput rem_options= /remote = Host&iHost; %end; %let gridHostState&iHost = 2; %end; /*------------------------------------------------------------- ** If the signon failed, set the host to retry job **-----------------------------------------------------------*/ %else %if (&&SignonState&iHost ne 3) %then %do; %put RDCSPAWN: Signon failed for host &&gridHost&iHost - retrying; %let gridHostState&iHost = 0; %end; %end; /*----------------------------------------------------------------- ** If the host state indicates that the signon and init has ** been done, but has not completed, check to see if the init ** has finished through the cmacvar value. If the init has ** finished, find the next job to submit, setup the macros, and ** update the host state to waiting for the job rsubmit to ** finish. If the init failed, reset the state to try to init ** again. **---------------------------------------------------------------*/ %else %if (&&gridHostState&iHost eq 2) %then %do; /*--------------------------------------------------------- ** Find the next job iteration to do **-------------------------------------------------------*/ %let jobNum = 0; %do job = 1 %to &gridJobCount; %if (&&gridJobState&job eq 0) %then %do; %let jobNum = &job; %let job = &gridJobCount; %end; %end; /*--------------------------------------------------------- ** If a job is left to start... **-------------------------------------------------------*/ %if (&jobNum > 0) %then %do; %put RDCSPAWN: Submitting remote job &jobNum on host &&gridHost&iHost; %let gridHostJob&iHost = &jobNum; /*----------------------------------------------------- ** Submit the code to be executed after setting the ** job ID on the remote computer **---------------------------------------------------*/ %syslput REM_JOBNUM = &jobNum /remote = Host&iHost; /*----------------------------------------------------- ** Turn off redundant data set creation where applicable **---------------------------------------------------*/ %if &part=portpart and &jobNum>1 %then %do; %let mopts=options=(nosimstate nosimrfs noshocks nomodcov); %end; %else %do; %let mopts=; %end; %syslput moreopts=&mopts /remote = Host&iHost; rsubmit remote=Host&iHost connectwait=no cmacvar=gridRsubmitState&iHost; libname &rem_envLibname &rem_netPathEnv; libname rskparts &rem_netPathTemp; proc risk; setoptions nobacksave; environment open=&rem_envLibname..&rem_env; runproject &rem_project portfolio = &rem_portfolio endpart = &rem_projpart &rem_part =(&REM_JOBNUM of &REM_JOBCOUNT) partlib=rskparts &rem_options &moreopts ; run; endrsubmit; /*----------------------------------------------------- ** Update the job and host states **---------------------------------------------------*/ %let gridHostState&iHost = 3; %let gridJobState&jobNum = 1; %end; %end; /*----------------------------------------------------------------- ** If the host state indicates that the signon and rsubmit ** has occurred, check the rsubmit cmacvar to see if the code ** has completed. If it has, set the state indicating ** that this job has completed, and increment the completed ** processing count. **---------------------------------------------------------------*/ %else %if (&&gridHostState&iHost eq 3) %then %do; %if (&&gridRsubmitState&iHost ne 2) %then %do; /*--------------------------------------------------------- ** Get the job number executing on this host **-------------------------------------------------------*/ %let jobNum = &&gridHostJob&iHost; /*--------------------------------------------------------- ** If the rsubmit was successful, set the job as ** complete and increment the complete count **-------------------------------------------------------*/ %if (&&gridRsubmitState&iHost eq 0) %then %do; %let gridJobState&jobNum = 2; %let gridJobsComplete = %eval(&gridJobsComplete + 1); %put RDCSPAWN: Job &jobNum on host &&gridHost&iHost finished.; %end; /*--------------------------------------------------------- ** If the rsubmit failed, set the job to be retried **-------------------------------------------------------*/ %else %do; %put RDCSPAWN: Rsubmit failed for job &jobNum - retrying; %let gridJobState&jobNum = 0; %end; /*--------------------------------------------------------- ** Set host & rsubmit state to be able to submit next job. **-------------------------------------------------------*/ %let gridHostState&iHost = 2; %let gridRsubmitState&iHost = 0; %end; %end; /*----------------------------------------------------------------- ** Increment the host ID. **---------------------------------------------------------------*/ %let iHost = %eval(&iHost + 1); /*----------------------------------------------------------------- ** If we have gone through all hosts once... **---------------------------------------------------------------*/ %if (&iHost > &maxHosts) %then %do; /*------------------------------------------------------------- ** Reset the host id back to the beginning **-----------------------------------------------------------*/ %let iHost = 1; /*------------------------------------------------------------- ** Wait for 1 second **-----------------------------------------------------------*/ %syscall sleep(gridNumSecs,gridUnits); %end; %end; /*--------------------------------------------------------------------- ** Signoff all sessions **-------------------------------------------------------------------*/ signoff _all_; %mend rdcAsynchSubmitLoop;