I'm Chris's colleague that was trying to figure this out originally :-). The main reason that I didn't want to use a proc or data step was because I wanted to use the macro as a function as Peter described. And the reason that I need this macro in the first place is because we can have data stored in either SPDS or Hadoop, and there are limitations about tables in Hadoop that are not there in SPDS (primarily that you can't apply an index to a table stored in Hadoop through SAS/Access), so we need to behave differently for the two storage methods. Here is my final version of the getEngine macro (in case anyone cares):

/* ****************************************************************** */
/* SAS Macro: getEngine */
/* ------------------------------------------------------------------ */
/* Description: */
/* Given a libref as input, returns the engine of the library. */
/* This is a macro function, so it should be used in a format similar */
/* to these examples: */
/* In a data step: */
/* engine = "%getEngine(sashelp)"; */
/* In a %let: */
/* %let engine = %getEngine(sashelp); */
/* In a %if: */
/* %if (%getEngine(sashelp) eq V9) %then */
/* ------------------------------------------------------------------ */
/* Parameters: */
/* libref - libref of the library. If the libref is blank or */
/* unassigned, then a blank value will be returned from */
/* the macro. */
/* ------------------------------------------------------------------ */
/* History */
/* 04JUN2013 1.0 Production version */
/* ------------------------------------------------------------------ */
/* Copyright (c) 2013 SAS Institute, Inc Cary, NC USA */
/* All rights reserved */
/**********************************************************************/

%macro getEngine(libref);

%let _engine =;

/* in case the parameter is a libref.tablename, pull */
/* off just the libref */
%let libref = %upcase(%scan(&libref, 1, %str(.)));

/* open the vlibnam view */
%let _dsid = %sysfunc(open(sashelp.vlibnam(where=(libname="&libref"))));

/* if the view opened successfully then fetch the first record */
/* and get the value of the engine variable */
%if (&_dsid gt 0) %then
%do;

%let _rc = %sysfunc(fetch(&_dsid));
%if (&_rc eq 0) %then
%do;
%let _engineVarnum = %sysfunc(varnum(&_dsid, ENGINE));
%let _engine = %sysfunc(getvarc(&_dsid, &_engineVarnum));
%end; /* if - fetch was successful */

%let _dsid = %sysfunc(close(&_dsid));

%end; /* if - view opened */

/* return engine value to calling program */
&_engine

%mend getEngine;