[apple/security.git] / regressions / test / testmore.c

/*
 * Copyright (c) 2005-2007 Apple Inc. All Rights Reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 *
 * testmore.c
 */

#include <fcntl.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/time.h>
#include <unistd.h>
#include <AvailabilityMacros.h>

#include "testmore.h"
#include "testenv.h"

static int test_num = 0;
static int test_fails = 0;
static int test_cases = 0;
static const char *test_plan_file;
static int test_plan_line=0;

const char *test_directive = NULL;
const char *test_reason = NULL;

static void fprint_string(FILE *file, CFStringRef string) {
    UInt8 buf[256];
    CFRange range = { .location = 0 };
    range.length = CFStringGetLength(string);
    while (range.length > 0) {
        CFIndex bytesUsed = 0;
        CFIndex converted = CFStringGetBytes(string, range, kCFStringEncodingUTF8, 0, false, buf, sizeof(buf), &bytesUsed);
        fwrite(buf, 1, bytesUsed, file);
        range.length -= converted;
        range.location += converted;
    }
}

static void cffprint(FILE *file, CFStringRef fmt, ...) CF_FORMAT_FUNCTION(2,0);

static void cffprint(FILE *file, CFStringRef fmt, ...) {
    va_list args;
    va_start(args, fmt);
    CFStringRef line = CFStringCreateWithFormatAndArguments(NULL, NULL, fmt, args);
    va_end(args);
    fprint_string(file, line);
    CFRelease(line);
}

void test_skip(const char *reason, int how_many, int unless)
{
    if (unless)
        return;

    int done;
    for (done = 0; done < how_many; ++done)
        test_ok(1, NULL, "skip", reason, __FILE__, __LINE__, NULL);
}

void test_bail_out(const char *reason, const char *file, unsigned line)
{
    printf("BAIL OUT! (%s at line %u) %s\n", file, line, reason);
    fflush(stdout);
    exit(255);
}

void test_plan_skip_all(const char *reason)
{
    if (test_num < test_cases)
    {
        test_skip(reason, test_cases - test_num, 0);
    }
}

static int test_plan_exit(void)
{
    int status = 0;
    fflush(stdout);

    if (!test_num)
    {
        if (test_cases)
        {
            fprintf(stderr, "%s:%u: warning: No tests run!\n", test_plan_file, test_plan_line);
            status = 255;
        }
        else
        {
            fprintf(stderr, "%s:%u: error: Looks like your test died before it could "
                    "output anything.\n", test_plan_file, test_plan_line);
            status = 255;
        }
    }
    else if (test_num < test_cases)
    {
        fprintf(stderr, "%s:%u: warning: Looks like you planned %d tests but only ran %d.\n",
               test_plan_file, test_plan_line, test_cases, test_num);
        status = test_fails + test_cases - test_num;
    }
    else if (test_num > test_cases)
    {
        fprintf(stderr, "%s:%u: warning: Looks like you planned %d tests but ran %d extra.\n",
               test_plan_file, test_plan_line, test_cases, test_num - test_cases);
        status = test_fails;
    }
    else if (test_fails)
    {
        fprintf(stderr, "%s:%u: error: Looks like you failed %d tests of %d.\n",
               test_plan_file, test_plan_line, test_fails, test_cases);
        status = test_fails;
    }

    fflush(stderr);
    
    /* reset the test plan */
    test_num = 0;
    test_fails = 0;
    test_cases = 0;

    return status;
}

void test_plan_tests(int count, const char *file, unsigned line)
{
#if 0
    if (atexit(test_plan_exit) < 0)
    {
        fprintf(stderr, "failed to setup atexit handler: %s\n",
                strerror(errno));
        fflush(stderr);
        exit(255);
    }
#endif

	if (test_cases)
    {
        fprintf(stderr,
                "%s:%u: error: You tried to plan twice!\n",
                file, line);
        
        fflush(stderr);
        exit(255);
    }
    else
	{
        if (!count)
        {
            fprintf(stderr, "%s:%u: warning: You said to run 0 tests!  You've got to run "
                    "something.\n", file, line);
            fflush(stderr);
            exit(255);
        }

        test_plan_file=file;
        test_plan_line=line;
        
        test_cases = count;
		fprintf(stderr, "%s:%u: note: 1..%d\n", file, line, test_cases);
		fflush(stdout);
	}
}

int
test_diag(const char *directive, const char *reason,
	const char *file, unsigned line, const char *fmt, ...)
{
	int is_todo = directive && !strcmp(directive, "TODO");
	va_list args;

	va_start(args, fmt);

	if (is_todo)
	{
		fputs("# ", stdout);
		if (fmt)
			vprintf(fmt, args);
		fputs("\n", stdout);
		fflush(stdout);
	}
	else
	{
		fflush(stdout);
		fputs("# ", stderr);
		if (fmt)
			vfprintf(stderr, fmt, args);
		fputs("\n", stderr);
		fflush(stderr);
	}

	va_end(args);

	return 1;
}

int
test_ok(int passed, __attribute((cf_consumed)) CFStringRef description, const char *directive,
	const char *reason, const char *file, unsigned line,
	const char *fmt, ...)
{
	int is_todo = !passed && directive && !strcmp(directive, "TODO");
	int is_setup = directive && !is_todo && !strcmp(directive, "SETUP");

	if (is_setup)
	{
		if (!passed)
		{
			fflush(stdout);
            cffprint(stderr, CFSTR("# SETUP not ok%s%@%s%s\n"),
				   description ? " - " : "",
				   description ? description : CFSTR(""),
				   reason ? " - " : "",
				   reason ? reason : "");
		}
	}
	else
	{
		if (!test_cases)
		{
			atexit((void(*)(void))test_plan_exit);
			fprintf(stderr, "You tried to run a test without a plan!  "
					"Gotta have a plan. at %s line %u\n", file, line);
			fflush(stderr);
			exit(255);
		}

		++test_num;
		if (test_num > test_cases || (!passed && !is_todo))
			++test_fails;

        /* We only print this when a test fail, unless verbose is enabled */
        if ((!passed) || (test_verbose > 0)) {
            cffprint(stderr, CFSTR("%s:%u: note: %sok %d%s%@%s%s%s%s\n"),
                      file, line, passed ? "" : "not ", test_num,
                      description ? " - " : "",
                      description ? description : CFSTR(""),
                      directive ? " # " : "",
                      directive ? directive : "",
                      reason ? " " : "",
                      reason ? reason : "");
        }
    }

    if (passed)
		fflush(stdout);
	else
    {
		va_list args;

		va_start(args, fmt);

		if (is_todo)
		{
/* Enable this to output TODO as warning */
#if 0             
			printf("%s:%d: warning: Failed (TODO) test\n", file, line);
			if (fmt)
				vprintf(fmt, args);
#endif
			fflush(stdout);
		}
        else
		{
			fflush(stdout);
			fprintf(stderr, "%s:%d: error: Failed test\n", file, line);
			if (fmt)
				vfprintf(stderr, fmt, args);
			fflush(stderr);
		}

		va_end(args);
    }

    if (description)
        CFRelease(description);

    return passed;
}


const char *
sec_errstr(int err)
{
#if 1
	static int bufnum = 0;
    static char buf[2][20];
	bufnum = bufnum ? 0 : 1;
    sprintf(buf[bufnum], "0x%X", err);
    return buf[bufnum];
#else /* !1 */
    if (err >= errSecErrnoBase && err <= errSecErrnoLimit)
        return strerror(err - 100000);

#ifdef MAC_OS_X_VERSION_10_4
    /* AvailabilityMacros.h would only define this if we are on a
       Tiger or later machine. */
    extern const char *cssmErrorString(long);
    return cssmErrorString(err);
#else /* !defined(MAC_OS_X_VERSION_10_4) */
    extern const char *_ZN8Security15cssmErrorStringEl(long);
    return _ZN8Security15cssmErrorStringEl(err);
#endif /* MAC_OS_X_VERSION_10_4 */
#endif /* !1 */
}

/* run one test, described by test, return info in test struct */
int run_one_test(struct one_test_s *test, int argc, char * const *argv)
{
    struct timeval start, stop;
    
    if(test->entry==NULL) {
        printf("%s:%d: error: wtf?\n", __FILE__, __LINE__);
        return -1;
    }
    
    gettimeofday(&start, NULL);
    test->entry(argc, argv);
    gettimeofday(&stop, NULL);
    
    
    /* this may overflow... */
    test->duration=(stop.tv_sec-start.tv_sec)*1000+(stop.tv_usec/1000)-(start.tv_usec/1000);
    test->failed_tests=test_fails;

    return test_plan_exit();
 };
Commit	Line	Data
427c49bc A	1	/*
	2	* Copyright (c) 2005-2007 Apple Inc. All Rights Reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. Please obtain a copy of the License at
	10	* http://www.opensource.apple.com/apsl/ and read it before using this
	11	* file.
	12	*
	13	* The Original Code and all software distributed under the License are
	14	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	15	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	16	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	18	* Please see the License for the specific language governing rights and
	19	* limitations under the License.
	20	*
	21	* @APPLE_LICENSE_HEADER_END@
	22	*
	23	* testmore.c
	24	*/
	25
	26	#include <fcntl.h>
	27	#include <stdarg.h>
	28	#include <stdlib.h>
	29	#include <string.h>
	30	#include <sys/stat.h>
	31	#include <sys/types.h>
	32	#include <sys/time.h>
	33	#include <unistd.h>
	34	#include <AvailabilityMacros.h>
	35
	36	#include "testmore.h"
	37	#include "testenv.h"
	38
	39	static int test_num = 0;
	40	static int test_fails = 0;
	41	static int test_cases = 0;
	42	static const char *test_plan_file;
	43	static int test_plan_line=0;
	44
	45	const char *test_directive = NULL;
	46	const char *test_reason = NULL;
	47
	48	static void fprint_string(FILE *file, CFStringRef string) {
	49	UInt8 buf[256];
	50	CFRange range = { .location = 0 };
	51	range.length = CFStringGetLength(string);
	52	while (range.length > 0) {
	53	CFIndex bytesUsed = 0;
	54	CFIndex converted = CFStringGetBytes(string, range, kCFStringEncodingUTF8, 0, false, buf, sizeof(buf), &bytesUsed);
	55	fwrite(buf, 1, bytesUsed, file);
	56	range.length -= converted;
	57	range.location += converted;
	58	}
	59	}
	60
	61	static void cffprint(FILE *file, CFStringRef fmt, ...) CF_FORMAT_FUNCTION(2,0);
	62
	63	static void cffprint(FILE *file, CFStringRef fmt, ...) {
	64	va_list args;
65	va_start(args, fmt);
66	CFStringRef line = CFStringCreateWithFormatAndArguments(NULL, NULL, fmt, args);
67	va_end(args);
68	fprint_string(file, line);
69	CFRelease(line);
70	}
71
72	void test_skip(const char *reason, int how_many, int unless)
73	{
74	if (unless)
75	return;
76
77	int done;
78	for (done = 0; done < how_many; ++done)
79	test_ok(1, NULL, "skip", reason, __FILE__, __LINE__, NULL);
80	}
81
82	void test_bail_out(const char reason, const char file, unsigned line)
83	{
84	printf("BAIL OUT! (%s at line %u) %s\n", file, line, reason);
85	fflush(stdout);
86	exit(255);
87	}
88
89	void test_plan_skip_all(const char *reason)
90	{
91	if (test_num < test_cases)
92	{
93	test_skip(reason, test_cases - test_num, 0);
94	}
95	}
96
97	static int test_plan_exit(void)
98	{
99	int status = 0;
100	fflush(stdout);
101
102	if (!test_num)
103	{
104	if (test_cases)
105	{
106	fprintf(stderr, "%s:%u: warning: No tests run!\n", test_plan_file, test_plan_line);
107	status = 255;
108	}
109	else
110	{
111	fprintf(stderr, "%s:%u: error: Looks like your test died before it could "
112	"output anything.\n", test_plan_file, test_plan_line);
113	status = 255;
114	}
115	}
116	else if (test_num < test_cases)
117	{
118	fprintf(stderr, "%s:%u: warning: Looks like you planned %d tests but only ran %d.\n",
119	test_plan_file, test_plan_line, test_cases, test_num);
120	status = test_fails + test_cases - test_num;
121	}
122	else if (test_num > test_cases)
123	{
124	fprintf(stderr, "%s:%u: warning: Looks like you planned %d tests but ran %d extra.\n",
125	test_plan_file, test_plan_line, test_cases, test_num - test_cases);
126	status = test_fails;
127	}
128	else if (test_fails)
129	{
130	fprintf(stderr, "%s:%u: error: Looks like you failed %d tests of %d.\n",
131	test_plan_file, test_plan_line, test_fails, test_cases);
132	status = test_fails;
133	}
134
135	fflush(stderr);
136
137	/* reset the test plan */
138	test_num = 0;
139	test_fails = 0;
140	test_cases = 0;
141
142	return status;
143	}
144
145	void test_plan_tests(int count, const char *file, unsigned line)
146	{
147	#if 0
148	if (atexit(test_plan_exit) < 0)
149	{
150	fprintf(stderr, "failed to setup atexit handler: %s\n",
151	strerror(errno));
152	fflush(stderr);
153	exit(255);
154	}
155	#endif
156
157	if (test_cases)
158	{
159	fprintf(stderr,
160	"%s:%u: error: You tried to plan twice!\n",
161	file, line);
162
163	fflush(stderr);
164	exit(255);
165	}
166	else
167	{
168	if (!count)
169	{
170	fprintf(stderr, "%s:%u: warning: You said to run 0 tests! You've got to run "
171	"something.\n", file, line);
172	fflush(stderr);
173	exit(255);
174	}
175
176	test_plan_file=file;
177	test_plan_line=line;
178
179	test_cases = count;
180	fprintf(stderr, "%s:%u: note: 1..%d\n", file, line, test_cases);
181	fflush(stdout);
182	}
183	}
184
185	int
186	test_diag(const char directive, const char reason,
187	const char file, unsigned line, const char fmt, ...)
188	{
189	int is_todo = directive && !strcmp(directive, "TODO");
190	va_list args;
191
192	va_start(args, fmt);
193
194	if (is_todo)
195	{
196	fputs("# ", stdout);
197	if (fmt)
198	vprintf(fmt, args);
199	fputs("\n", stdout);
200	fflush(stdout);
201	}
202	else
203	{
204	fflush(stdout);
205	fputs("# ", stderr);
206	if (fmt)
207	vfprintf(stderr, fmt, args);
208	fputs("\n", stderr);
209	fflush(stderr);
210	}
211
212	va_end(args);
213
214	return 1;
215	}
216
217	int
218	test_ok(int passed, __attribute((cf_consumed)) CFStringRef description, const char *directive,
219	const char reason, const char file, unsigned line,
220	const char *fmt, ...)
221	{
222	int is_todo = !passed && directive && !strcmp(directive, "TODO");
223	int is_setup = directive && !is_todo && !strcmp(directive, "SETUP");
224
225	if (is_setup)
226	{
227	if (!passed)
228	{
229	fflush(stdout);
230	cffprint(stderr, CFSTR("# SETUP not ok%s%@%s%s\n"),
231	description ? " - " : "",
232	description ? description : CFSTR(""),
233	reason ? " - " : "",
234	reason ? reason : "");
235	}
236	}
237	else
238	{
239	if (!test_cases)
240	{
241	atexit((void(*)(void))test_plan_exit);
242	fprintf(stderr, "You tried to run a test without a plan! "
243	"Gotta have a plan. at %s line %u\n", file, line);
244	fflush(stderr);
245	exit(255);
246	}
247
248	++test_num;
249	if (test_num > test_cases \|\| (!passed && !is_todo))
250	++test_fails;
251
252	/* We only print this when a test fail, unless verbose is enabled */
253	if ((!passed) \|\| (test_verbose > 0)) {
254	cffprint(stderr, CFSTR("%s:%u: note: %sok %d%s%@%s%s%s%s\n"),
255	file, line, passed ? "" : "not ", test_num,
256	description ? " - " : "",
257	description ? description : CFSTR(""),
258	directive ? " # " : "",
259	directive ? directive : "",
260	reason ? " " : "",
261	reason ? reason : "");
262	}
263	}
264
265	if (passed)
266	fflush(stdout);
267	else
268	{
269	va_list args;
270
271	va_start(args, fmt);
272
273	if (is_todo)
274	{
275	/* Enable this to output TODO as warning */
276	#if 0
277	printf("%s:%d: warning: Failed (TODO) test\n", file, line);
278	if (fmt)
279	vprintf(fmt, args);
280	#endif
281	fflush(stdout);
282	}
283	else
284	{
285	fflush(stdout);
286	fprintf(stderr, "%s:%d: error: Failed test\n", file, line);
287	if (fmt)
288	vfprintf(stderr, fmt, args);
289	fflush(stderr);
290	}
291
292	va_end(args);
293	}
294
295	if (description)
296	CFRelease(description);
297
298	return passed;
299	}
300
301
302	const char *
303	sec_errstr(int err)
304	{
305	#if 1
306	static int bufnum = 0;
307	static char buf[2][20];
308	bufnum = bufnum ? 0 : 1;
309	sprintf(buf[bufnum], "0x%X", err);
310	return buf[bufnum];
311	#else /* !1 */
312	if (err >= errSecErrnoBase && err <= errSecErrnoLimit)
313	return strerror(err - 100000);
314
315	#ifdef MAC_OS_X_VERSION_10_4
316	/* AvailabilityMacros.h would only define this if we are on a
317	Tiger or later machine. */
318	extern const char *cssmErrorString(long);
319	return cssmErrorString(err);
320	#else /* !defined(MAC_OS_X_VERSION_10_4) */
321	extern const char *_ZN8Security15cssmErrorStringEl(long);
322	return _ZN8Security15cssmErrorStringEl(err);
323	#endif /* MAC_OS_X_VERSION_10_4 */
324	#endif /* !1 */
325	}
326
327	/* run one test, described by test, return info in test struct */
328	int run_one_test(struct one_test_s test, int argc, char const *argv)
329	{
330	struct timeval start, stop;
331
332	if(test->entry==NULL) {
333	printf("%s:%d: error: wtf?\n", __FILE__, __LINE__);
334	return -1;
335	}
336
337	gettimeofday(&start, NULL);
338	test->entry(argc, argv);
339	gettimeofday(&stop, NULL);
340
341
342	/* this may overflow... */
343	test->duration=(stop.tv_sec-start.tv_sec)*1000+(stop.tv_usec/1000)-(start.tv_usec/1000);
344	test->failed_tests=test_fails;
345
346	return test_plan_exit();
347	};