2 * xnu_quick_test - this tool will do a quick test of every (well, to be
3 * honest most) system calls we support in xnu.
5 * WARNING - this is not meant to be a full regression test of all the
6 * system calls. The intent is to have a quick test of each system call that
7 * can be run very easily and quickly when a new kerenl is built.
9 * This tool is meant to grow as we find xnu problems that could have be
10 * caught before we submit to a build train. So please add more tests and
11 * make the existing ones better. Some of the original tests are nothing
12 * more than place holders and quite lame. Just keep in mind that the tool
13 * should run as fast as possible. If it gets too slow then most people
14 * will stop running it.
16 * LP64 testing tip - when adding or modifying tests, keep in mind the
17 * variants in the LP64 world. If xnu gets passed a structure the varies in
18 * size between 32 and 64-bit processes, try to test that a field in the
19 * sructure contains valid data. For example if we know foo structure
26 * And if we know what another_int should contain then test for the known
27 * value since it's offset will vary depending on whether the calling process
30 * NOTE - we have several workarounds and test exceptions for some
31 * outstanding bugs in xnu. All the workarounds are marked with "todo" and
32 * some comments noting the radar number of the offending bug. Do a seach
33 * for "todo" in the source files for this project to locate which tests have
34 * known failures. And please tag any new exceptions you find with "todo"
35 * in the comment and the radar number of the bug.
47 #include <sys/mount.h>
48 #include <sys/param.h>
49 #include <sys/select.h>
51 #include <sys/syslimits.h>
52 #include <sys/types.h>
53 #include <sys/ucred.h>
55 #include <mach-o/ldsyms.h>
56 #include <mach-o/loader.h>
57 #include <mach-o/arch.h>
63 /* our table of tests to run */
64 struct test_entry g_tests
[] =
66 {1, &syscall_test
, NULL
, "syscall"},
67 {1, &fork_wait4_exit_test
, NULL
, "fork, wait4, exit"},
68 {1, &read_write_test
, NULL
, "fsync, ftruncate, lseek, pread, pwrite, read, readv, truncate, write, writev"},
69 {1, &open_close_test
, NULL
, "close, fpathconf, fstat, open, pathconf"},
70 {1, &link_stat_unlink_test
, NULL
, "link, stat, unlink"},
71 {1, &chdir_fchdir_test
, NULL
, "chdir, fchdir"},
72 {1, &access_chmod_fchmod_test
, NULL
, "access, chmod, fchmod"},
73 {1, &chown_fchown_lchown_lstat_symlink_test
, NULL
, "chown, fchown, lchown, lstat, readlink, symlink"},
74 {1, &fs_stat_tests
, NULL
, "fstatfs, getfsstat, statfs, fstatfs64, getfsstat64, statfs64"},
75 #if !TARGET_OS_EMBEDDED
76 {1, &statfs_32bit_inode_tests
, NULL
, "32-bit inode versions: fstatfs, getfsstat, statfs"},
78 {1, &getpid_getppid_pipe_test
, NULL
, "getpid, getppid, pipe"},
79 {1, &uid_tests
, NULL
, "getauid, gettid, getuid, geteuid, issetugid, setaudit_addr, seteuid, settid, settid_with_pid, setuid"},
80 {1, &mkdir_rmdir_umask_test
, NULL
, "mkdir, rmdir, umask"},
81 {1, &mknod_sync_test
, NULL
, "mknod, sync"},
82 {1, &socket2_tests
, NULL
, "fsync, getsockopt, poll, select, setsockopt, socketpair"},
83 {1, &socket_tests
, NULL
, "accept, bind, connect, getpeername, getsockname, listen, socket, recvmsg, sendmsg, sendto, sendfile"},
84 {1, &chflags_fchflags_test
, NULL
, "chflags, fchflags"},
85 {1, &execve_kill_vfork_test
, NULL
, "kill, vfork, execve, posix_spawn"},
86 {1, &groups_test
, NULL
, "getegid, getgid, getgroups, setegid, setgid, setgroups"},
87 {1, &dup_test
, NULL
, "dup, dup2, getdtablesize"},
88 {1, &getrusage_test
, NULL
, "getrusage"},
89 {1, &signals_test
, NULL
, "getitimer, setitimer, sigaction, sigpending, sigprocmask, sigsuspend, sigwait"},
90 {1, &acct_test
, NULL
, "acct"},
91 {1, &ioctl_test
, NULL
, "ioctl"},
92 {1, &chroot_test
, NULL
, "chroot"},
93 {1, &memory_tests
, NULL
, "madvise, mincore, minherit, mlock, mlock, mmap, mprotect, msync, munmap"},
94 {1, &process_group_test
, NULL
, "getpgrp, getpgid, getsid, setpgid, setpgrp, setsid"},
95 {1, &fcntl_test
, NULL
, "fcntl"},
96 {1, &getlogin_setlogin_test
, NULL
, "getlogin, setlogin"},
97 {1, &getpriority_setpriority_test
, NULL
, "getpriority, setpriority"},
98 {1, &time_tests
, NULL
, "futimes, gettimeofday, settimeofday, utimes"},
99 {1, &rename_test
, NULL
, "rename, stat"},
100 {1, &locking_test
, NULL
, "flock"},
101 {1, &mkfifo_test
, NULL
, "mkfifo, read, write"},
102 {1, "actl_test
, NULL
, "quotactl"},
103 {1, &limit_tests
, NULL
, "getrlimit, setrlimit"},
104 {1, &directory_tests
, NULL
, "getattrlist, getdirentriesattr, setattrlist"},
105 #if !TARGET_OS_EMBEDDED
106 {1, &getdirentries_test
, NULL
, "getdirentries"},
107 {1, &exchangedata_test
, NULL
, "exchangedata"},
109 {1, &searchfs_test
, NULL
, "searchfs"},
110 {1, &sema2_tests
, NULL
, "sem_close, sem_open, sem_post, sem_trywait, sem_unlink, sem_wait"},
111 {1, &sema_tests
, NULL
, "semctl, semget, semop"},
112 {1, &bsd_shm_tests
, NULL
, "shm_open, shm_unlink"},
113 {1, &shm_tests
, NULL
, "shmat, shmctl, shmdt, shmget"},
114 {1, &xattr_tests
, NULL
, "fgetxattr, flistxattr, fremovexattr, fsetxattr, getxattr, listxattr, removexattr, setxattr"},
115 {1, &aio_tests
, NULL
, "aio_cancel, aio_error, aio_read, aio_return, aio_suspend, aio_write, fcntl, lio_listio"},
116 {1, &kqueue_tests
, NULL
, "kevent, kqueue"},
117 {1, &message_queue_tests
, NULL
, "msgctl, msgget, msgrcv, msgsnd"},
118 {1, &data_exec_tests
, NULL
, "data/stack execution"},
119 {1, &machvm_tests
, NULL
, "Mach VM calls"},
120 {1, &commpage_data_tests
, NULL
, "Commpage data"},
121 #if defined(i386) || defined(__x86_64__)
122 {1, &atomic_fifo_queue_test
, NULL
, "OSAtomicFifoEnqueue, OSAtomicFifoDequeue"},
124 {1, &sched_tests
, NULL
, "Scheduler tests"},
125 #if TARGET_OS_EMBEDDED
126 {1, &content_protection_test
, NULL
, "Content protection tests"},
128 {1, &pipes_test
, NULL
, "Pipes tests"},
129 {1, &kaslr_test
, NULL
, "KASLR tests"},
130 {0, NULL
, NULL
, "last one"}
133 static void create_target_directory( const char * the_targetp
);
134 static void list_all_tests( void );
135 static void mark_tests_to_run( long my_start
, long my_end
);
136 static int parse_tests_to_run( int argc
, const char * argv
[], int * indexp
);
137 static void usage( void );
138 #if !TARGET_OS_EMBEDDED
139 static int setgroups_if_single_user(void);
141 static const char *current_arch( void );
144 long g_max_failures
= 0;
145 int g_skip_setuid_tests
= 0;
146 const char * g_cmd_namep
;
147 char g_target_path
[ PATH_MAX
];
148 int g_is_single_user
= 0;
149 int g_testbots_active
= 0;
151 int main( int argc
, const char * argv
[] )
155 int my_tests_count
, i
;
158 int list_the_tests
= 0;
159 const char * my_targetp
;
160 time_t my_start_time
, my_end_time
;
161 struct stat my_stat_buf
;
164 const char * sudo_uid_env
;
166 const char * sudo_gid_env
;
167 sranddev( ); /* set up seed for our random name generator */
168 g_cmd_namep
= argv
[0];
170 /* make sure SIGCHLD is not ignored, so wait4 calls work */
171 signal(SIGCHLD
, SIG_DFL
);
173 /* NOTE - code in create_target_directory will append '/' if it is necessary */
174 my_targetp
= getenv("TMPDIR");
175 if ( my_targetp
== NULL
)
178 /* make sure we are running as root */
179 if ( ( getuid() != 0 ) || ( geteuid() != 0 ) ) {
180 printf( "Test must be run as root\n", g_cmd_namep
);
184 sudo_uid_env
= getenv("SUDO_UID");
185 if ( sudo_uid_env
) {
186 sudo_uid
= strtol(sudo_uid_env
, NULL
, 10);
189 /* switch real uid to a non_root user, while keeping effective uid as root */
190 if ( sudo_uid
!= 0 ) {
191 setreuid( sudo_uid
, 0 );
194 /* Default to 501 if no sudo uid found */
198 /* restore the gid if run through sudo */
199 sudo_gid_env
= getenv("SUDO_GID");
200 if ( sudo_gid_env
) {
201 sudo_gid
= strtol(sudo_gid_env
, NULL
, 10);
204 if ( getgid() == 0 ) {
206 if ( sudo_gid
!= 0 ) {
210 /* Default to 20 if no sudo gid found */
215 /* parse input parameters */
216 for ( i
= 1; i
< argc
; i
++ ) {
217 if ( strcmp( argv
[i
], "-u" ) == 0 ) {
220 if ( strcmp( argv
[i
], "-t" ) == 0 ||
221 strcmp( argv
[i
], "-target" ) == 0 ) {
223 printf( "invalid target parameter \n" );
226 /* verify our target directory exists */
227 my_targetp
= argv
[i
];
228 err
= stat( my_targetp
, &my_stat_buf
);
229 if ( err
!= 0 || S_ISDIR(my_stat_buf
.st_mode
) == 0 ) {
230 printf( "invalid target path \n" );
232 printf( "stat call failed with error %d - \"%s\" \n", errno
, strerror( errno
) );
238 if ( strcmp( argv
[i
], "-f" ) == 0 ||
239 strcmp( argv
[i
], "-failures" ) == 0 ) {
241 printf( "invalid failures parameter \n" );
245 /* get our max number of failures */
246 g_max_failures
= strtol( argv
[i
], NULL
, 10 );
249 if ( strcmp( argv
[i
], "-l" ) == 0 ||
250 strcmp( argv
[i
], "-list" ) == 0 ) {
251 /* list all the tests this tool will do.
256 if ( strcmp( argv
[i
], "-r" ) == 0 ||
257 strcmp( argv
[i
], "-run" ) == 0 ) {
259 printf( "invalid run tests parameter \n" );
263 /* get which tests to run */
264 if ( parse_tests_to_run( argc
, argv
, &i
) != 0 ) {
265 printf( "invalid run tests parameter \n" );
270 if ( strcmp( argv
[i
], "-s" ) == 0 ||
271 strcmp( argv
[i
], "-skip" ) == 0 ) {
272 /* set that want to skip the setuid related tests - this is useful for debgugging since since I can't
273 * get setuid tests to work while in gdb.
275 g_skip_setuid_tests
= 1;
278 if ( strcmp( argv
[i
], "-testbot" ) == 0 ) {
279 g_testbots_active
= 1;
282 printf( "invalid argument \"%s\" \n", argv
[i
] );
289 /* Check if we are running under testbots */
290 #if RUN_UNDER_TESTBOTS
291 g_testbots_active
= 1;
293 /* Code added to run xnu_quick_test under testbots */
294 if ( g_testbots_active
== 1 ) {
295 printf("[TEST] xnu_quick_test \n"); /* Declare the beginning of test suite */
298 #if !TARGET_OS_EMBEDDED
299 /* Populate groups list if we're in single user mode */
300 if (setgroups_if_single_user()) {
304 if ( list_the_tests
!= 0 ) {
309 /* build a test target directory that we use as our path to create any test
310 * files and directories.
312 create_target_directory( my_targetp
);
313 printf( "Will allow %ld failures before testing is aborted \n", g_max_failures
);
315 my_start_time
= time( NULL
);
316 printf( "\nBegin testing - %s \n", ctime_r( &my_start_time
, &my_buffer
[0] ) );
317 printf( "Current architecture is %s\n", current_arch() );
319 /* Code added to run xnu_quick_test under testbots */
321 /* run each test that is marked to run in our table until we complete all of them or
322 * hit the maximum number of failures.
324 my_tests_count
= (sizeof( g_tests
) / sizeof( g_tests
[0] ));
325 for ( i
= 0; i
< (my_tests_count
- 1); i
++ ) {
327 test_entryp my_testp
;
329 my_testp
= &g_tests
[i
];
330 if ( my_testp
->test_run_it
== 0 || my_testp
->test_routine
== NULL
)
333 if ( g_testbots_active
== 1 ) {
334 printf("[BEGIN] %s \n", my_testp
->test_infop
);
337 printf( "test #%d - %s \n", (i
+ 1), my_testp
->test_infop
);
339 my_err
= my_testp
->test_routine( my_testp
->test_input
);
341 printf("\t--> FAILED \n");
342 printf("SysCall %s failed", my_testp
->test_infop
);
343 printf("Result %d", my_err
);
345 if ( my_failures
> g_max_failures
) {
346 printf( "\n Reached the maximum number of failures - Aborting xnu_quick_test. \n" );
347 /* Code added to run xnu_quick_test under testbots */
348 if ( g_testbots_active
== 1 ) {
349 printf("[FAIL] %s \n", my_testp
->test_infop
);
351 goto exit_this_routine
;
353 /* Code added to run xnu_quick_test under testbots */
354 if ( g_testbots_active
== 1 ) {
355 printf("\n[FAIL] %s \n", my_testp
->test_infop
);
359 /* Code added to run xnu_quick_test under testbots */
360 if ( g_testbots_active
== 1 ) {
361 printf("[PASS] %s \n", my_testp
->test_infop
);
366 my_end_time
= time( NULL
);
367 printf( "\nEnd testing - %s \n", ctime_r( &my_end_time
, &my_buffer
[0] ) );
369 /* clean up our test directory */
370 rmdir( &g_target_path
[0] );
372 /* exit non zero if there are any failures */
373 return my_failures
!= 0;
378 * parse_tests_to_run - parse the -run argument parameters. the run argument tells us which tests the user
379 * wants to run. we accept ranges (example 1 - 44) and runs of tests (example 2, 33, 34, 100) or a mix of
380 * both (example 1, 44 - 100, 200, 250)
382 static int parse_tests_to_run( int argc
, const char * argv
[], int * indexp
)
384 int my_tests_count
, is_range
= 0, i
;
387 long my_first_test_number
, my_last_test_number
;
388 char my_buffer
[ 128 ];
390 /* set tests table to not run any tests then go back and set the specific tests the caller asked for */
391 my_tests_count
= (sizeof( g_tests
) / sizeof( g_tests
[0] ));
392 for ( i
= 0; i
< (my_tests_count
- 1); i
++ ) {
393 g_tests
[ i
].test_run_it
= 0;
396 for ( i
= *indexp
; i
< argc
; i
++ ) {
398 if ( strlen( my_ptr
) > 1 && *my_ptr
== '-' && isalpha( *(my_ptr
+ 1) ) ) {
399 /* we have hit a new argument - need to make sure caller uses this argument on the next
400 * pass through its parse loop (which will bump the index value so we want to be one less
401 * than the actual index).
407 if ( strlen( my_ptr
) == 1 && *my_ptr
== '-' ) {
408 /* we are dealing with a range of tests, for example: 33 - 44 */
413 if ( strlen( my_ptr
) > (sizeof( my_buffer
) - 1) ) {
414 printf( "-run argument has too many test parameters (max of %lu characters) \n", sizeof( my_buffer
) );
417 /* get a local copy of the parameter string to work with - break range into two strings */
418 strcpy( &my_buffer
[0], my_ptr
);
420 my_temp_ptr
= strrchr( &my_buffer
[0], '-' );
421 if ( my_temp_ptr
!= NULL
) {
422 /* we are dealing with a range of tests with no white space, for example: 33-44 or 33- 44 */
423 my_temp_ptr
= strrchr( &my_buffer
[0], '-' );
425 my_first_test_number
= strtol( &my_buffer
[0], NULL
, 10 );
426 if ( *(my_temp_ptr
+ 1) == 0x00 ) {
427 /* we hit the case where the range indicator is at the end of one string, for example: 33- */
431 my_last_test_number
= strtol( (my_temp_ptr
+ 1), NULL
, 10 );
432 if ( my_first_test_number
< 1 || my_first_test_number
> my_last_test_number
) {
433 printf( "-run argument has invalid range parmeters \n" );
436 mark_tests_to_run( my_first_test_number
, my_last_test_number
);
442 /* should be the second part of the test number range */
443 my_last_test_number
= strtol( &my_buffer
[0], NULL
, 10 );
444 if ( my_first_test_number
< 1 || my_first_test_number
> my_last_test_number
) {
445 printf( "-run argument has invalid range parmeters \n" );
449 mark_tests_to_run( my_first_test_number
, my_last_test_number
);
454 my_first_test_number
= strtol( &my_buffer
[0], NULL
, 10 );
455 if ( my_first_test_number
< 1 ) {
456 printf( "-run argument has invalid test number parameter \n" );
459 mark_tests_to_run( my_first_test_number
, my_first_test_number
);
467 } /* parse_tests_to_run */
470 static void create_target_directory( const char * the_targetp
)
474 if ( strlen( the_targetp
) > (sizeof(g_target_path
) - 1) ) {
475 printf( "target path too long - \"%s\" \n", the_targetp
);
484 sprintf( &my_name
[0], "xnu_quick_test-%d", my_rand
);
485 if ( (strlen( &my_name
[0] ) + strlen( the_targetp
) + 2) > PATH_MAX
) {
486 printf( "target path plus our test directory name is too long: \n" );
487 printf( "target path - \"%s\" \n", the_targetp
);
488 printf( "test directory name - \"%s\" \n", &my_name
[0] );
492 /* append generated directory name onto our path */
493 g_target_path
[0] = 0x00;
494 strcat( &g_target_path
[0], the_targetp
);
495 if ( g_target_path
[ (strlen(the_targetp
) - 1) ] != '/' ) {
496 strcat( &g_target_path
[0], "/" );
498 strcat( &g_target_path
[0], &my_name
[0] );
500 /* try to create the test directory */
501 err
= mkdir( &g_target_path
[0], (S_IRWXU
| S_IRWXG
| S_IROTH
) );
506 if ( EEXIST
!= err
) {
507 printf( "test directory creation failed - \"%s\" \n", &g_target_path
[0] );
508 printf( "mkdir call failed with error %d - \"%s\" \n", errno
, strerror( err
) );
512 printf( "created test directory at \"%s\" \n", &g_target_path
[0] );
514 } /* create_target_directory */
517 static void mark_tests_to_run( long my_start
, long my_end
)
519 int my_tests_count
, i
;
521 my_tests_count
= (sizeof( g_tests
) / sizeof( g_tests
[0] ));
522 my_end
= (my_end
< (my_tests_count
- 1)) ? my_end
: (my_tests_count
- 1);
523 for ( i
= (my_start
- 1); i
< my_end
; i
++ ) {
524 g_tests
[ i
].test_run_it
= 1; /* run this test */
527 } /* mark_tests_to_run */
530 static void usage( void )
534 /* skip past full path and just show the tool name */
535 my_ptr
= strrchr( g_cmd_namep
, '/' );
536 if ( my_ptr
!= NULL
) {
540 printf( "\nUSAGE: %s -target TARGET_PATH \n\n", (my_ptr
!= NULL
) ? my_ptr
: g_cmd_namep
);
541 printf( "\t -f[ailures] MAX_FAILS_ALLOWED # number of test cases that may fail before we give up. defaults to 0 \n" );
542 printf( "\t -l[ist] # list all the tests this tool performs \n" );
543 printf( "\t -r[un] 1, 3, 10 - 19 # run specific tests. enter individual test numbers and/or range of numbers. use -list to list tests. \n" );
544 printf( "\t -s[kip] # skip setuid tests \n" );
545 printf( "\t -t[arget] TARGET_PATH # path to directory where tool will create test files. defaults to \"/tmp/\" \n" );
546 printf( "\t -testbot # output results in CoreOS TestBot compatible format \n" );
547 printf( "\nexamples: \n" );
548 printf( "--- Place all test files and directories at the root of volume \"test_vol\" --- \n" );
549 printf( "%s -t /Volumes/test_vol/ \n", (my_ptr
!= NULL
) ? my_ptr
: g_cmd_namep
);
551 printf( "--- Run the tool for tests 10 thru 15, test 18 and test 20 --- \n" );
552 printf( "%s -r 10-15, 18, 20 \n", (my_ptr
!= NULL
) ? my_ptr
: g_cmd_namep
);
558 #if !TARGET_OS_EMBEDDED
559 /* This is a private API between Libinfo, Libc, and the DirectoryService daemon.
560 * Since we are trying to determine if an external provider will back group
561 * lookups, we can use this, without relying on additional APIs or tools
562 * that might not work yet */
563 extern int _ds_running(void);
567 setgroups_if_single_user(void)
571 gid_t gids
[NUM_GROUPS
];
573 if (!_ds_running()) {
574 printf("In single-user mode.\n");
575 g_is_single_user
= 1;
577 /* We skip 'nobody' and 'anyone' */
580 for (i
= 0; i
< NUM_GROUPS
; i
++) {
586 gids
[i
] = grp
->gr_gid
;
591 /* Only succeed if we find at least NUM_GROUPS */
592 if (i
== NUM_GROUPS
) {
593 retval
= setgroups(NUM_GROUPS
, gids
);
595 getgroups(NUM_GROUPS
, gids
);
596 printf("After single-user hack, groups are: ");
597 for (i
= 0; i
< NUM_GROUPS
; i
++) {
598 printf("%d, ", gids
[i
]);
602 printf("Setgroups failed.\n");
605 printf("Couldn't get sufficient number of groups.\n");
608 printf("Not in single user mode.\n");
617 static const char *current_arch( void )
619 cpu_type_t cputype
= _mh_execute_header
.cputype
;
620 cpu_subtype_t cpusubtype
= _mh_execute_header
.cpusubtype
;
622 const NXArchInfo
*arch
= NXGetArchInfoFromCpuType(cputype
, cpusubtype
);
631 #undef printf /* this makes the "-l" output easier to read */
632 static void list_all_tests( void )
634 int i
, my_tests_count
;
636 my_tests_count
= (sizeof( g_tests
) / sizeof( g_tests
[0] ));
637 printf( "\nList of all tests this tool performs... \n" );
639 for ( i
= 0; i
< (my_tests_count
- 1); i
++ ) {
640 printf( " %d \t %s \n", (i
+ 1), g_tests
[ i
].test_infop
);
644 } /* list_all_tests */