534 lines
15 KiB
C
534 lines
15 KiB
C
|
/* See COPYING.txt for the full license governing this code. */
|
||
|
|
/**
|
||
|
|
* \file testharness.c
|
||
|
|
*
|
||
|
|
* Source file for the test harness.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include <stdlib.h>
|
||
|
|
#include <SDL_test.h>
|
||
|
|
#include <SDL.h>
|
||
|
|
#include <SDL_assert.h>
|
||
|
|
#include "SDL_visualtest_harness_argparser.h"
|
||
|
|
#include "SDL_visualtest_process.h"
|
||
|
|
#include "SDL_visualtest_variators.h"
|
||
|
|
#include "SDL_visualtest_screenshot.h"
|
||
|
|
#include "SDL_visualtest_mischelper.h"
|
||
|
|
|
||
|
|
#if defined(__WIN32__) && !defined(__CYGWIN__)
|
||
|
|
#include <direct.h>
|
||
|
|
#elif defined(__WIN32__) && defined(__CYGWIN__)
|
||
|
|
#include <signal.h>
|
||
|
|
#elif defined(__LINUX__)
|
||
|
|
#include <sys/stat.h>
|
||
|
|
#include <sys/types.h>
|
||
|
|
#include <signal.h>
|
||
|
|
#else
|
||
|
|
#error "Unsupported platform"
|
||
|
|
#endif
|
||
|
|
|
||
|
|
/** Code for the user event triggered when a new action is to be executed */
|
||
|
|
#define ACTION_TIMER_EVENT 0
|
||
|
|
/** Code for the user event triggered when the maximum timeout is reached */
|
||
|
|
#define KILL_TIMER_EVENT 1
|
||
|
|
/** FPS value used for delays in the action loop */
|
||
|
|
#define ACTION_LOOP_FPS 10
|
||
|
|
|
||
|
|
/** Value returned by RunSUTAndTest() when the test has passed */
|
||
|
|
#define TEST_PASSED 1
|
||
|
|
/** Value returned by RunSUTAndTest() when the test has failed */
|
||
|
|
#define TEST_FAILED 0
|
||
|
|
/** Value returned by RunSUTAndTest() on a fatal error */
|
||
|
|
#define TEST_ERROR -1
|
||
|
|
|
||
|
|
static SDL_ProcessInfo pinfo;
|
||
|
|
static SDL_ProcessExitStatus sut_exitstatus;
|
||
|
|
static SDLVisualTest_HarnessState state;
|
||
|
|
static SDLVisualTest_Variator variator;
|
||
|
|
static SDLVisualTest_ActionNode* current; /* the current action being performed */
|
||
|
|
static SDL_TimerID action_timer, kill_timer;
|
||
|
|
|
||
|
|
/* returns a char* to be passed as the format argument of a printf-style function. */
|
||
|
|
static char*
|
||
|
|
usage()
|
||
|
|
{
|
||
|
|
return "Usage: \n%s --sutapp xyz"
|
||
|
|
" [--sutargs abc | --parameter-config xyz.parameters"
|
||
|
|
" [--variator exhaustive|random]"
|
||
|
|
" [--num-variations N] [--no-launch]] [--timeout hh:mm:ss]"
|
||
|
|
" [--action-config xyz.actions]"
|
||
|
|
" [--output-dir /path/to/output]"
|
||
|
|
" [--verify-dir /path/to/verify]"
|
||
|
|
" or --config app.config";
|
||
|
|
}
|
||
|
|
|
||
|
|
/* register Ctrl+C handlers */
|
||
|
|
#if defined(__LINUX__) || defined(__CYGWIN__)
|
||
|
|
static void
|
||
|
|
CtrlCHandlerCallback(int signum)
|
||
|
|
{
|
||
|
|
SDL_Event event;
|
||
|
|
SDLTest_Log("Ctrl+C received");
|
||
|
|
event.type = SDL_QUIT;
|
||
|
|
SDL_PushEvent(&event);
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
static Uint32
|
||
|
|
ActionTimerCallback(Uint32 interval, void* param)
|
||
|
|
{
|
||
|
|
SDL_Event event;
|
||
|
|
SDL_UserEvent userevent;
|
||
|
|
Uint32 next_action_time;
|
||
|
|
|
||
|
|
/* push an event to handle the action */
|
||
|
|
userevent.type = SDL_USEREVENT;
|
||
|
|
userevent.code = ACTION_TIMER_EVENT;
|
||
|
|
userevent.data1 = ¤t->action;
|
||
|
|
userevent.data2 = NULL;
|
||
|
|
|
||
|
|
event.type = SDL_USEREVENT;
|
||
|
|
event.user = userevent;
|
||
|
|
SDL_PushEvent(&event);
|
||
|
|
|
||
|
|
/* calculate the new interval and return it */
|
||
|
|
if(current->next)
|
||
|
|
next_action_time = current->next->action.time - current->action.time;
|
||
|
|
else
|
||
|
|
{
|
||
|
|
next_action_time = 0;
|
||
|
|
action_timer = 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
current = current->next;
|
||
|
|
return next_action_time;
|
||
|
|
}
|
||
|
|
|
||
|
|
static Uint32
|
||
|
|
KillTimerCallback(Uint32 interval, void* param)
|
||
|
|
{
|
||
|
|
SDL_Event event;
|
||
|
|
SDL_UserEvent userevent;
|
||
|
|
|
||
|
|
userevent.type = SDL_USEREVENT;
|
||
|
|
userevent.code = KILL_TIMER_EVENT;
|
||
|
|
userevent.data1 = NULL;
|
||
|
|
userevent.data2 = NULL;
|
||
|
|
|
||
|
|
event.type = SDL_USEREVENT;
|
||
|
|
event.user = userevent;
|
||
|
|
SDL_PushEvent(&event);
|
||
|
|
|
||
|
|
kill_timer = 0;
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
static int
|
||
|
|
ProcessAction(SDLVisualTest_Action* action, int* sut_running, char* args)
|
||
|
|
{
|
||
|
|
if(!action || !sut_running)
|
||
|
|
return TEST_ERROR;
|
||
|
|
|
||
|
|
switch(action->type)
|
||
|
|
{
|
||
|
|
case SDL_ACTION_KILL:
|
||
|
|
SDLTest_Log("Action: Kill SUT");
|
||
|
|
if(SDL_IsProcessRunning(&pinfo) == 1 &&
|
||
|
|
!SDL_KillProcess(&pinfo, &sut_exitstatus))
|
||
|
|
{
|
||
|
|
SDLTest_LogError("SDL_KillProcess() failed");
|
||
|
|
return TEST_ERROR;
|
||
|
|
}
|
||
|
|
*sut_running = 0;
|
||
|
|
break;
|
||
|
|
|
||
|
|
case SDL_ACTION_QUIT:
|
||
|
|
SDLTest_Log("Action: Quit SUT");
|
||
|
|
if(SDL_IsProcessRunning(&pinfo) == 1 &&
|
||
|
|
!SDL_QuitProcess(&pinfo, &sut_exitstatus))
|
||
|
|
{
|
||
|
|
SDLTest_LogError("SDL_QuitProcess() failed");
|
||
|
|
return TEST_FAILED;
|
||
|
|
}
|
||
|
|
*sut_running = 0;
|
||
|
|
break;
|
||
|
|
|
||
|
|
case SDL_ACTION_LAUNCH:
|
||
|
|
{
|
||
|
|
char* path;
|
||
|
|
char* args;
|
||
|
|
SDL_ProcessInfo action_process;
|
||
|
|
SDL_ProcessExitStatus ps;
|
||
|
|
|
||
|
|
path = action->extra.process.path;
|
||
|
|
args = action->extra.process.args;
|
||
|
|
if(args)
|
||
|
|
{
|
||
|
|
SDLTest_Log("Action: Launch process: %s with arguments: %s",
|
||
|
|
path, args);
|
||
|
|
}
|
||
|
|
else
|
||
|
|
SDLTest_Log("Action: Launch process: %s", path);
|
||
|
|
if(!SDL_LaunchProcess(path, args, &action_process))
|
||
|
|
{
|
||
|
|
SDLTest_LogError("SDL_LaunchProcess() failed");
|
||
|
|
return TEST_ERROR;
|
||
|
|
}
|
||
|
|
|
||
|
|
/* small delay so that the process can do its job */
|
||
|
|
SDL_Delay(1000);
|
||
|
|
|
||
|
|
if(SDL_IsProcessRunning(&action_process) > 0)
|
||
|
|
{
|
||
|
|
SDLTest_LogError("Process %s took too long too complete."
|
||
|
|
" Force killing...", action->extra);
|
||
|
|
if(!SDL_KillProcess(&action_process, &ps))
|
||
|
|
{
|
||
|
|
SDLTest_LogError("SDL_KillProcess() failed");
|
||
|
|
return TEST_ERROR;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
|
||
|
|
case SDL_ACTION_SCREENSHOT:
|
||
|
|
{
|
||
|
|
char path[MAX_PATH_LEN], hash[33];
|
||
|
|
|
||
|
|
SDLTest_Log("Action: Take screenshot");
|
||
|
|
/* can't take a screenshot if the SUT isn't running */
|
||
|
|
if(SDL_IsProcessRunning(&pinfo) != 1)
|
||
|
|
{
|
||
|
|
SDLTest_LogError("SUT has quit.");
|
||
|
|
*sut_running = 0;
|
||
|
|
return TEST_FAILED;
|
||
|
|
}
|
||
|
|
|
||
|
|
/* file name for the screenshot image */
|
||
|
|
SDLVisualTest_HashString(args, hash);
|
||
|
|
SDL_snprintf(path, MAX_PATH_LEN, "%s/%s", state.output_dir, hash);
|
||
|
|
if(!SDLVisualTest_ScreenshotProcess(&pinfo, path))
|
||
|
|
{
|
||
|
|
SDLTest_LogError("SDLVisualTest_ScreenshotProcess() failed");
|
||
|
|
return TEST_ERROR;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
|
||
|
|
case SDL_ACTION_VERIFY:
|
||
|
|
{
|
||
|
|
int ret;
|
||
|
|
|
||
|
|
SDLTest_Log("Action: Verify screenshot");
|
||
|
|
ret = SDLVisualTest_VerifyScreenshots(args, state.output_dir,
|
||
|
|
state.verify_dir);
|
||
|
|
|
||
|
|
if(ret == -1)
|
||
|
|
{
|
||
|
|
SDLTest_LogError("SDLVisualTest_VerifyScreenshots() failed");
|
||
|
|
return TEST_ERROR;
|
||
|
|
}
|
||
|
|
else if(ret == 0)
|
||
|
|
{
|
||
|
|
SDLTest_Log("Verification failed: Images were not equal.");
|
||
|
|
return TEST_FAILED;
|
||
|
|
}
|
||
|
|
else if(ret == 1)
|
||
|
|
SDLTest_Log("Verification successful.");
|
||
|
|
else
|
||
|
|
{
|
||
|
|
SDLTest_Log("Verfication skipped.");
|
||
|
|
return TEST_FAILED;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
|
||
|
|
default:
|
||
|
|
SDLTest_LogError("Invalid action type");
|
||
|
|
return TEST_ERROR;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
return TEST_PASSED;
|
||
|
|
}
|
||
|
|
|
||
|
|
static int
|
||
|
|
RunSUTAndTest(char* sutargs, int variation_num)
|
||
|
|
{
|
||
|
|
int success, sut_running, return_code;
|
||
|
|
char hash[33];
|
||
|
|
SDL_Event event;
|
||
|
|
|
||
|
|
return_code = TEST_PASSED;
|
||
|
|
|
||
|
|
if(!sutargs)
|
||
|
|
{
|
||
|
|
SDLTest_LogError("sutargs argument cannot be NULL");
|
||
|
|
return_code = TEST_ERROR;
|
||
|
|
goto runsutandtest_cleanup_generic;
|
||
|
|
}
|
||
|
|
|
||
|
|
SDLVisualTest_HashString(sutargs, hash);
|
||
|
|
SDLTest_Log("Hash: %s", hash);
|
||
|
|
|
||
|
|
success = SDL_LaunchProcess(state.sutapp, sutargs, &pinfo);
|
||
|
|
if(!success)
|
||
|
|
{
|
||
|
|
SDLTest_Log("Could not launch SUT.");
|
||
|
|
return_code = TEST_ERROR;
|
||
|
|
goto runsutandtest_cleanup_generic;
|
||
|
|
}
|
||
|
|
SDLTest_Log("SUT launch successful.");
|
||
|
|
SDLTest_Log("Process will be killed in %d milliseconds", state.timeout);
|
||
|
|
sut_running = 1;
|
||
|
|
|
||
|
|
/* launch the timers */
|
||
|
|
SDLTest_Log("Performing actions..");
|
||
|
|
current = state.action_queue.front;
|
||
|
|
action_timer = 0;
|
||
|
|
kill_timer = 0;
|
||
|
|
if(current)
|
||
|
|
{
|
||
|
|
action_timer = SDL_AddTimer(current->action.time, ActionTimerCallback, NULL);
|
||
|
|
if(!action_timer)
|
||
|
|
{
|
||
|
|
SDLTest_LogError("SDL_AddTimer() failed");
|
||
|
|
return_code = TEST_ERROR;
|
||
|
|
goto runsutandtest_cleanup_timer;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
kill_timer = SDL_AddTimer(state.timeout, KillTimerCallback, NULL);
|
||
|
|
if(!kill_timer)
|
||
|
|
{
|
||
|
|
SDLTest_LogError("SDL_AddTimer() failed");
|
||
|
|
return_code = TEST_ERROR;
|
||
|
|
goto runsutandtest_cleanup_timer;
|
||
|
|
}
|
||
|
|
|
||
|
|
/* the timer stops running if the actions queue is empty, and the
|
||
|
|
SUT stops running if it crashes or if we encounter a KILL/QUIT action */
|
||
|
|
while(sut_running)
|
||
|
|
{
|
||
|
|
/* process the actions by using an event queue */
|
||
|
|
while(SDL_PollEvent(&event))
|
||
|
|
{
|
||
|
|
if(event.type == SDL_USEREVENT)
|
||
|
|
{
|
||
|
|
if(event.user.code == ACTION_TIMER_EVENT)
|
||
|
|
{
|
||
|
|
SDLVisualTest_Action* action;
|
||
|
|
|
||
|
|
action = (SDLVisualTest_Action*)event.user.data1;
|
||
|
|
|
||
|
|
switch(ProcessAction(action, &sut_running, sutargs))
|
||
|
|
{
|
||
|
|
case TEST_PASSED:
|
||
|
|
break;
|
||
|
|
|
||
|
|
case TEST_FAILED:
|
||
|
|
return_code = TEST_FAILED;
|
||
|
|
goto runsutandtest_cleanup_timer;
|
||
|
|
break;
|
||
|
|
|
||
|
|
default:
|
||
|
|
SDLTest_LogError("ProcessAction() failed");
|
||
|
|
return_code = TEST_ERROR;
|
||
|
|
goto runsutandtest_cleanup_timer;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else if(event.user.code == KILL_TIMER_EVENT)
|
||
|
|
{
|
||
|
|
SDLTest_LogError("Maximum timeout reached. Force killing..");
|
||
|
|
return_code = TEST_FAILED;
|
||
|
|
goto runsutandtest_cleanup_timer;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else if(event.type == SDL_QUIT)
|
||
|
|
{
|
||
|
|
SDLTest_LogError("Received QUIT event. Testharness is quitting..");
|
||
|
|
return_code = TEST_ERROR;
|
||
|
|
goto runsutandtest_cleanup_timer;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
SDL_Delay(1000/ACTION_LOOP_FPS);
|
||
|
|
}
|
||
|
|
|
||
|
|
SDLTest_Log("SUT exit code was: %d", sut_exitstatus.exit_status);
|
||
|
|
if(sut_exitstatus.exit_status == 0)
|
||
|
|
{
|
||
|
|
return_code = TEST_PASSED;
|
||
|
|
goto runsutandtest_cleanup_timer;
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
return_code = TEST_FAILED;
|
||
|
|
goto runsutandtest_cleanup_timer;
|
||
|
|
}
|
||
|
|
|
||
|
|
return_code = TEST_ERROR;
|
||
|
|
goto runsutandtest_cleanup_generic;
|
||
|
|
|
||
|
|
runsutandtest_cleanup_timer:
|
||
|
|
if(action_timer && !SDL_RemoveTimer(action_timer))
|
||
|
|
{
|
||
|
|
SDLTest_Log("SDL_RemoveTimer() failed");
|
||
|
|
return_code = TEST_ERROR;
|
||
|
|
}
|
||
|
|
|
||
|
|
if(kill_timer && !SDL_RemoveTimer(kill_timer))
|
||
|
|
{
|
||
|
|
SDLTest_Log("SDL_RemoveTimer() failed");
|
||
|
|
return_code = TEST_ERROR;
|
||
|
|
}
|
||
|
|
/* runsutandtest_cleanup_process: */
|
||
|
|
if(SDL_IsProcessRunning(&pinfo) && !SDL_KillProcess(&pinfo, &sut_exitstatus))
|
||
|
|
{
|
||
|
|
SDLTest_Log("SDL_KillProcess() failed");
|
||
|
|
return_code = TEST_ERROR;
|
||
|
|
}
|
||
|
|
runsutandtest_cleanup_generic:
|
||
|
|
return return_code;
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Entry point for testharness */
|
||
|
|
int
|
||
|
|
main(int argc, char* argv[])
|
||
|
|
{
|
||
|
|
int i, passed, return_code, failed;
|
||
|
|
|
||
|
|
/* freeing resources, linux style! */
|
||
|
|
return_code = 0;
|
||
|
|
|
||
|
|
if(argc < 2)
|
||
|
|
{
|
||
|
|
SDLTest_Log(usage(), argv[0]);
|
||
|
|
goto cleanup_generic;
|
||
|
|
}
|
||
|
|
|
||
|
|
#if defined(__LINUX__) || defined(__CYGWIN__)
|
||
|
|
signal(SIGINT, CtrlCHandlerCallback);
|
||
|
|
#endif
|
||
|
|
|
||
|
|
/* parse arguments */
|
||
|
|
if(!SDLVisualTest_ParseHarnessArgs(argv + 1, &state))
|
||
|
|
{
|
||
|
|
SDLTest_Log(usage(), argv[0]);
|
||
|
|
return_code = 1;
|
||
|
|
goto cleanup_generic;
|
||
|
|
}
|
||
|
|
SDLTest_Log("Parsed harness arguments successfully.");
|
||
|
|
|
||
|
|
/* initialize SDL */
|
||
|
|
if(SDL_Init(SDL_INIT_TIMER) == -1)
|
||
|
|
{
|
||
|
|
SDLTest_LogError("SDL_Init() failed.");
|
||
|
|
SDLVisualTest_FreeHarnessState(&state);
|
||
|
|
return_code = 1;
|
||
|
|
goto cleanup_harness_state;
|
||
|
|
}
|
||
|
|
|
||
|
|
/* create an output directory if none exists */
|
||
|
|
#if defined(__LINUX__) || defined(__CYGWIN__)
|
||
|
|
mkdir(state.output_dir, 0777);
|
||
|
|
#elif defined(__WIN32__)
|
||
|
|
_mkdir(state.output_dir);
|
||
|
|
#else
|
||
|
|
#error "Unsupported platform"
|
||
|
|
#endif
|
||
|
|
|
||
|
|
/* test with sutargs */
|
||
|
|
if(SDL_strlen(state.sutargs))
|
||
|
|
{
|
||
|
|
SDLTest_Log("Running: %s %s", state.sutapp, state.sutargs);
|
||
|
|
if(!state.no_launch)
|
||
|
|
{
|
||
|
|
switch(RunSUTAndTest(state.sutargs, 0))
|
||
|
|
{
|
||
|
|
case TEST_PASSED:
|
||
|
|
SDLTest_Log("Status: PASSED");
|
||
|
|
break;
|
||
|
|
|
||
|
|
case TEST_FAILED:
|
||
|
|
SDLTest_Log("Status: FAILED");
|
||
|
|
break;
|
||
|
|
|
||
|
|
case TEST_ERROR:
|
||
|
|
SDLTest_LogError("Some error occurred while testing.");
|
||
|
|
return_code = 1;
|
||
|
|
goto cleanup_sdl;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if(state.sut_config.num_options > 0)
|
||
|
|
{
|
||
|
|
char* variator_name = state.variator_type == SDL_VARIATOR_RANDOM ?
|
||
|
|
"RANDOM" : "EXHAUSTIVE";
|
||
|
|
if(state.num_variations > 0)
|
||
|
|
SDLTest_Log("Testing SUT with variator: %s for %d variations",
|
||
|
|
variator_name, state.num_variations);
|
||
|
|
else
|
||
|
|
SDLTest_Log("Testing SUT with variator: %s and ALL variations",
|
||
|
|
variator_name);
|
||
|
|
/* initialize the variator */
|
||
|
|
if(!SDLVisualTest_InitVariator(&variator, &state.sut_config,
|
||
|
|
state.variator_type, 0))
|
||
|
|
{
|
||
|
|
SDLTest_LogError("Could not initialize variator");
|
||
|
|
return_code = 1;
|
||
|
|
goto cleanup_sdl;
|
||
|
|
}
|
||
|
|
|
||
|
|
/* iterate through all the variations */
|
||
|
|
passed = 0;
|
||
|
|
failed = 0;
|
||
|
|
for(i = 0; state.num_variations > 0 ? (i < state.num_variations) : 1; i++)
|
||
|
|
{
|
||
|
|
char* args = SDLVisualTest_GetNextVariation(&variator);
|
||
|
|
if(!args)
|
||
|
|
break;
|
||
|
|
SDLTest_Log("\nVariation number: %d\nArguments: %s", i + 1, args);
|
||
|
|
|
||
|
|
if(!state.no_launch)
|
||
|
|
{
|
||
|
|
switch(RunSUTAndTest(args, i + 1))
|
||
|
|
{
|
||
|
|
case TEST_PASSED:
|
||
|
|
SDLTest_Log("Status: PASSED");
|
||
|
|
passed++;
|
||
|
|
break;
|
||
|
|
|
||
|
|
case TEST_FAILED:
|
||
|
|
SDLTest_Log("Status: FAILED");
|
||
|
|
failed++;
|
||
|
|
break;
|
||
|
|
|
||
|
|
case TEST_ERROR:
|
||
|
|
SDLTest_LogError("Some error occurred while testing.");
|
||
|
|
goto cleanup_variator;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if(!state.no_launch)
|
||
|
|
{
|
||
|
|
/* report stats */
|
||
|
|
SDLTest_Log("Testing complete.");
|
||
|
|
SDLTest_Log("%d/%d tests passed.", passed, passed + failed);
|
||
|
|
}
|
||
|
|
goto cleanup_variator;
|
||
|
|
}
|
||
|
|
|
||
|
|
goto cleanup_sdl;
|
||
|
|
|
||
|
|
cleanup_variator:
|
||
|
|
SDLVisualTest_FreeVariator(&variator);
|
||
|
|
cleanup_sdl:
|
||
|
|
SDL_Quit();
|
||
|
|
cleanup_harness_state:
|
||
|
|
SDLVisualTest_FreeHarnessState(&state);
|
||
|
|
cleanup_generic:
|
||
|
|
return return_code;
|
||
|
|
}
|