/* --------------------------------------------------------------------------- $Revision: 1.1.2.19.2.1 $ $Date: 2017/01/11 13:02:05 $ NAME: cxt_exec_norm.sas TYPE: DESCRIPTION: Macro to prepare normalization input parameters and call Normalization rest api. SAS VERSION: 9.4 INPUT: OUTPUT: Specify data sets, files, reports, graphs, html, e-mail, pages, macro variables, slists, return codes, etc. CALLS: CALLED BY (optional): PRECONDITIONS (optional): POSTCONDITIONS (optional): HISTORY: userid yyyymmdd defectid desc sinshd 20140808 ------------------------------------------------------------------------------ Copyright (c) 2005-2015, SAS Institute Inc., Cary, NC, USA, All Rights Reserved ------------------------------------------------------------------------------ */ %macro cxt_exec_norm ; %******************************************************************************; %* Normalization preparation ; %******************************************************************************; %* initialise count_norm_hist variable ; %global count_norm_hist; %let count_norm_hist=; %cxt_prep_norm_params(); %if &retcode. eq 1 %then %goto ERROREXIT; %******************************************************************************; %* Execute Normalization Post Method ; %******************************************************************************; %* initialse NormJobId variable ; %global NormJobId; %let NormJobId=; %* Define filenames ; %let headerin=hdrin ; %let headerout= hdrout ; filename &headerin temp; filename &headerout temp; %* Create header input file ; data _null_; file &headerin. encoding="utf-8" recfm=f lrecl=1; /*put 'Accept: application/xml';*/ put 'Accept: application/vnd.sas.marketing.acs.normalization.jobstatus+xml'; putlog 'Accept: application/vnd.sas.marketing.acs.normalization.jobstatus+xml'; run; %* set http POST request variables; %let requestMethod=%nrquote(POST); %let infile=Postin; %let outfile= Postout; /*%let requestCT =%nrquote(application/json);*/ %let requestCT =%nrquote(application/vnd.sas.marketing.acs.normalization.jobcontrol+json); %let requestURL =%nrquote(%sysfunc(trim(&ACS_NORM_URL.))) ; %let proxy_host=%nrquote(&proxy_host); %let proxy_port=%nrquote(&proxy_port); %let proxy_auth=%nrquote(&proxy_auth); filename &infile temp; filename &outfile temp; /* Changes Java timestamps to SAS datetime see sas note - http://support.sas.com/kb/39/499.html */ %* set end_dttm = end_dttm + 1 for normalization as normalization is done till end_dttm - 1ms ; data EXTRACT_DATERANGE ; set &EXTRACT_DATERANGE.; NONHR_START_DTTM=dhms('01jan1970'd,0,0, NONHR_START_DTTM/1000); NONHR_END_DTTM=dhms('01jan1970'd,0,0, (NONHR_END_DTTM+1)/1000); HR_START_DTTM=dhms('01jan1970'd,0,0, HR_START_DTTM/1000); HR_END_DTTM=dhms('01jan1970'd,0,0, (HR_END_DTTM+1)/1000); run; %* prepare the POST in file ; data _null_; file &infile. encoding="utf-8"; set EXTRACT_DATERANGE; nonHRStart='"nonHourlyStartTime":"' || put(NONHR_START_DTTM,IS8601DT.) || '",'; nonHREnd='"nonHourlyEndTime":"' || put(NONHR_END_DTTM,IS8601DT.) || '",'; hrStart='"hourlyStartTime":"' || put(HR_START_DTTM,IS8601DT.) || '",'; hrEnd='"hourlyEndTime":"' || put(HR_END_DTTM,IS8601DT.) || '"'; put '{"deleteOutput": true,'; put nonHRStart; put nonHREnd; put hrStart; put hrEnd; put '}'; /* write normalization range to log */ putlog '{"deleteOutput": true,'; putlog nonHRStart; putlog nonHREnd; putlog hrStart; putlog hrEnd; putlog '}'; run; %let service_auth=%nrquote(&ACS_SERVICE_AUTH); %let RetryAttemptNo=0; %HTTPTRYAGAIN: %* call PROC HTTTP POST; %cxt_HTTPRequest(infile=&infile, outfile=&outfile, headerin=&headerin, headerout=&headerout, requestCT=&requestCT, requestURL=&requestURL, requestMethod=&requestMethod, service_auth=&service_auth, proxy_host=&proxy_host, proxy_port=&proxy_port, proxy_auth=&proxy_auth ); %if &retcode=1 %then %do; %cxt_echofile_tolog(fileRefs=&infile &outfile &headerin &headerout); /*%goto ERROREXIT;*/ %goto HTTPERROR; %end; %* When PROC HTTP POST is successful then read the header out file to check the status of the execution; %cxt_HTTPReadHeader(Action=ACS_NORM_POST,hdrout=&headerout,outfile=&outfile); %if &retcode=1 %then %do; %cxt_echofile_tolog(fileRefs=&infile &outfile &headerin &headerout); /*%goto ERROREXIT;*/ %goto HTTPERROR; %end; %* Default the job status wait time to 5 sec; %if %symexist(CXT_NORM_JOB_STATUS_WAIT_SEC) = 0 %then %do; %let CXT_NORM_JOB_STATUS_WAIT_SEC=5; %end; %******************************************************************************; %* Execute Get Job Status REST to get the status of Normalization Job %******************************************************************************; %if %length(&NormJobId) > 0 %then %do; %* loop till the job_state = SUCCEEDED; %let requestMethod =%nrquote(GET) ; %let outfile= jobStat; %let requestCT =%nrquote(application/xml); %let requestURL = %nrquote(%sysfunc(trim(&ACS_NORM_URL.))/&NormJobId.); filename &outfile temp; %* set the GET Request Status Map file name ; data _null_; statusMapfile =strip(symget('cxtcnfg_libPath')) || '/NormJobStatus.map'; call symput('GstatusMapfile', statusMapfile ); run; %* check if the statusMap file exists else create it ; %if %sysfunc(fileexist(&GstatusMapfile)) eq 0 %then %do; %cxt_MakeStatusMapFile(mapName=&GstatusMapfile); %end; %* Read the outout file to get the job progress ; filename GMap "&GstatusMapfile"; libname &outfile xmlv2 xmlmap=GMap; %let job_status_tbl=work.job_status; %let file_urls_tbl=work.Downloaditemreps; %let RetryAttemptNo=0; %HTTPTRYAGAIN2: %let GetStatus=0; %do %while (&GetStatus= 0); %* initialise job progress parameters ; %let job_state=; %* call PROC HTTTP GET; %cxt_HTTPRequest( outfile=&outfile, headerin=&headerin, headerout=&headerout, requestCT=&requestCT, requestURL=&requestURL, requestMethod=&requestMethod, service_auth=&service_auth, proxy_host=&proxy_host, proxy_port=&proxy_port, proxy_auth=&proxy_auth ); %if &retcode. eq 1 %then %do; %cxt_echofile_tolog(fileRefs=&infile &outfile &headerin &headerout); /*%goto ERROREXIT;*/ %goto HTTPERROR; %end; %* When PROC HTTP GET is successful then read the header out file to check the status of the execution; %cxt_HTTPReadHeader(Action=ACS_GET_JOB_STATUS,hdrout=&headerout,outfile=&outfile); %if &retcode=1 %then %do; %cxt_echofile_tolog(fileRefs=&infile &outfile &headerin &headerout); /*%goto ERROREXIT;*/ %goto HTTPERROR; %end; %* read the out xml file generated by GET request; proc copy in=&outfile out=work; run; %* check if there are errors in reading the response file ; %if &SYSERR. > 4 %then %do; %*write the files to log ; %cxt_echofile_tolog(fileRefs=&infile &outfile &headerin &headerout); %* do a retry attempt ; %goto HTTPERROR; %end; /* check if job_status table has record in it ,if not error out */ %if %cxt_get_observation_count(indsn=&job_status_tbl.) > 0 %then %do; %* read job_state ; proc sql noprint; select upcase(job_state) into :job_state from &job_status_tbl.; quit; %end; %else %do; %let retcode=1; %let errmsg=%sysfunc(sasmsg(&msg_dset,_cxa_norm_18_err,noquote,&job_status_tbl)); %let typemsg = %sysfunc(sasmsg(&msg_dset,_cxaerrmsg_label,noquote)); %goto ERROREXIT; %end; %* when job_state = COMPLETED then normaliztion is done ; %if &job_state = COMPLETED %then %do; /* check if "work.Downloaditemreps" table has records , if not error out */ %if %cxt_get_observation_count(indsn=&file_urls_tbl.) > 0 %then %do; %* set GetStatus=1 to complete the loop ; /* Extract file Names from urls */ /* S1154294 S1155026 work around % to %25 */ data &cxtwork_libref..Downloaditemreps; length url $5000; set &file_urls_tbl.; /*url=tranwrd(url,"%","%25");*/ fileNmExt=kscan(path,-1,'/'); fileNm=kscan(fileNmExt,1,'.'); run; %let GetStatus=1; %let Norm_done=%sysfunc(sasmsg(&msg_dset,_cxa_norm_07_msg,noquote)); %put &Norm_done.; %end; %else %do; %cxt_echofile_tolog(fileRefs=&infile &outfile &headerin &headerout); %let retcode=1; %let errmsg=%sysfunc(sasmsg(&msg_dset,_cxa_norm_18_err,noquote,&file_urls_tbl)); %let typemsg = %sysfunc(sasmsg(&msg_dset,_cxaerrmsg_label,noquote)); %goto ERROREXIT; %end; %end; %* when job_state = PENDING then normaliztion is in preparation ; %else %if &job_state = PENDING %then %do; %let Norm_prep_msg=%sysfunc(sasmsg(&msg_dset,_cxa_norm_09_msg,noquote)); %put &Norm_prep_msg.; %* Wait before checking the status again ? ; data _null_; call sleep(&CXT_NORM_JOB_STATUS_WAIT_SEC.,1); run; %end; %* when job_state = RUNNING then normaliztion is running; %else %if &job_state = RUNNING %then %do; %let Norm_get_done=%sysfunc(sasmsg(&msg_dset,_cxa_norm_08_msg,noquote)); %put &Norm_get_done.; %* Wait before checking the status again ? ; data _null_; call sleep(&CXT_NORM_JOB_STATUS_WAIT_SEC.,1); run; %end; %* when job_state = FAILED or KILLED then report failure and exit ; %else %if ( &job_state = FAILED or &job_state = KILLED or &job_state = CANCELED ) %then %do; %cxt_echofile_tolog(fileRefs=&infile &outfile &headerin &headerout); %let retcode=1; %let errmsg=%sysfunc(sasmsg(&msg_dset,_cxa_norm_11_err,noquote,&job_state)); %let typemsg = %sysfunc(sasmsg(&msg_dset,_cxaerrmsg_label,noquote)); %goto ERROREXIT; %end; %end;/*loop end */ %end; /* if NormJobID is not set then this indicates some issues in running normalization */ %else %do; %cxt_echofile_tolog(fileRefs=&infile &outfile &headerin &headerout); %let retcode=1; %let errmsg=%sysfunc(sasmsg(&msg_dset,_cxa_norm_17_err,noquote)); %let typemsg = %sysfunc(sasmsg(&msg_dset,_cxaerrmsg_label,noquote)); %end; %goto HTTPSUCCESS; /* on http errors retry http call */ %HTTPERROR: /* if retry attempts are left then try again */ %if &RetryAttemptNo. < &CXT_HTTP_MAX_RETRY_ATTEMPTS. %then %do; /* reset retcode */ %let retcode=0; %* reset errors 1. reset syserr 2. reset syscc 3. Reset SYNTAXCHECK OBS REPLACE DMSSYNCHK options - added for proc copy failure errors ; data _null_; run; %let syscc=0; OPTIONS OBS=MAX REPLACE NOSYNTAXCHECK NODMSSYNCHK; /*increment RetryAttemptNo */ %let RetryAttemptNo=%sysevalf(&RetryAttemptNo + 1); /* do some rest */ data _null_; call sleep(&CXT_HTTP_RETRY_WAIT_SEC.,1); run; %let msgtype = %sysfunc(sasmsg(&msg_dset,_cxanotemsg_label,noquote)); %let msgdesc = %sysfunc(sasmsg(&msg_dset,_cxa_norm_20_note,noquote,&RetryAttemptNo.)); %put &msgtype. &msgdesc.; %if &requestMethod = POST %then %goto HTTPTRYAGAIN; %else %goto HTTPTRYAGAIN2; %end; %HTTPSUCCESS: %ERROREXIT: %mend;