From 937356ff807fe25e0d85d70c3cbba86240340294 Mon Sep 17 00:00:00 2001 From: Apple Date: Wed, 15 Nov 2017 06:18:32 +0000 Subject: [PATCH] copyfile-146.50.5.tar.gz --- copyfile.3 | 14 + copyfile.c | 354 ++++++++++++++++++++++++- copyfile.h | 2 + copyfile.xcodeproj/project.pbxproj | 147 +++++++++++ copyfile_test/main.c | 65 +++++ copyfile_test/sparse_test.c | 402 +++++++++++++++++++++++++++++ copyfile_test/sparse_test.h | 13 + copyfile_test/systemx.c | 68 +++++ copyfile_test/systemx.h | 15 ++ copyfile_test/test_utils.c | 148 +++++++++++ copyfile_test/test_utils.h | 119 +++++++++ 11 files changed, 1336 insertions(+), 11 deletions(-) create mode 100644 copyfile_test/main.c create mode 100644 copyfile_test/sparse_test.c create mode 100644 copyfile_test/sparse_test.h create mode 100644 copyfile_test/systemx.c create mode 100644 copyfile_test/systemx.h create mode 100644 copyfile_test/test_utils.c create mode 100644 copyfile_test/test_utils.h diff --git a/copyfile.3 b/copyfile.3 index 4233ce9..ceb898b 100644 --- a/copyfile.3 +++ b/copyfile.3 @@ -201,6 +201,15 @@ supported, see below for more information. (This is only applicable for the .Fn copyfile function.) +.It Dv COPYFILE_DATA_SPARSE +Copy a file sparsely. +This requires that the source and destination file systems support sparse files with hole sizes +at least as large as their block sizes. +This also requires that the source file is sparse, and for +.Fn fcopyfile +the source file descriptor's offset be a multiple of the minimum hole size. +If COPYFILE_DATA is also specified, this will fall back to a full copy +if sparse copying cannot be performed for any reason; otherwise, an error is returned. .It Dv COPYFILE_NOFOLLOW This is a convenience macro, equivalent to .Dv (COPYFILE_NOFOLLOW_DST | COPYFILE_NOFOLLOW_SRC) . @@ -610,6 +619,11 @@ was a negative number. A memory allocation failed. .It Bq Er ENOTSUP The source file was not a directory, symbolic link, or regular file. +.It Bq Er ENOTSUP +COPYFILE_CLONE_FORCE was specified and file cloning is not supported. +.It Bq Er ENOTSUP +COPYFILE_DATA_SPARSE was specified, sparse copying is not supported, +and COPYFILE_DATA was not specified. .It Bq Er ECANCELED The copy was cancelled by callback. .It Bq Er EEXIST diff --git a/copyfile.c b/copyfile.c index 91e4d9a..f07b86d 100644 --- a/copyfile.c +++ b/copyfile.c @@ -649,7 +649,7 @@ copytree(copyfile_state_t s) goto done; } - flags = s->flags & (COPYFILE_ALL | COPYFILE_NOFOLLOW | COPYFILE_VERBOSE | COPYFILE_EXCL | COPYFILE_CLONE); + flags = s->flags & (COPYFILE_ALL | COPYFILE_NOFOLLOW | COPYFILE_VERBOSE | COPYFILE_EXCL | COPYFILE_CLONE | COPYFILE_DATA_SPARSE); paths[0] = src = s->src; dst = s->dst; @@ -1422,7 +1422,7 @@ static int copyfile_internal(copyfile_state_t s, copyfile_flags_t flags) * the non-meta data portion of the file. We attempt to * remove (via unlink) the destination file if we fail. */ - if (COPYFILE_DATA & flags) + if ((COPYFILE_DATA|COPYFILE_DATA_SPARSE) & flags) { if ((ret = copyfile_data(s)) < 0) { @@ -2025,6 +2025,297 @@ static copyfile_flags_t copyfile_check(copyfile_state_t s) return ret; } +/* + * Attempt to copy the data section of a file sparsely. + * Requires that the source and destination file systems support sparse files. + * Also requires that the source file descriptor's offset is a multiple of the smaller of the + * source and destination file systems' block size. + * In practice, this means that we refuse to perform copies that are only partially sparse. + * Returns 0 if the source sparse file was copied, -1 on an unrecoverable error that + * callers should propagate, and ENOTSUP where this routine refuses to copy the source file. + * In this final case, callers are free to attempt a full copy. + */ +static int copyfile_data_sparse(copyfile_state_t s, size_t input_blk_size, size_t output_blk_size) +{ + int src_fd = s->src_fd, dst_fd = s->dst_fd, rc = 0; + off_t src_start, dst_start, src_size = s->sb.st_size; + off_t first_hole_offset, next_hole_offset, current_src_offset, next_src_offset; + ssize_t nread; + size_t iosize = MIN(input_blk_size, output_blk_size); + copyfile_callback_t status = s->statuscb; + char *bp = NULL; + bool use_punchhole = true; + errno = 0; + + // Sanity checks. + if (!(s->flags & COPYFILE_DATA_SPARSE)) { + // Don't attempt this unless the right flags are passed. + return ENOTSUP; + } else if (src_size <= 0) { + // The file size of our source is invalid; there's nothing to copy. + errno = EINVAL; + goto error_exit; + } + + // Since a major underlying filesystem requires that holes are block-aligned, + // we only punch holes if we can guarantee that all holes from the source can + // be holes in the destination, which requires that the source filesystem's block size + // be an integral multiple of the destination filesystem's block size. + if (input_blk_size % output_blk_size != 0) { + use_punchhole = false; + } + + // Get the starting src/dest file descriptor offsets. + src_start = lseek(src_fd, 0, SEEK_CUR); + dst_start = lseek(dst_fd, 0, SEEK_CUR); + if (src_start < 0 || src_start >= src_size || dst_start < 0) { + /* + * Invalid starting source/destination offset: + * Either < 0 which is plainly invalid (lseek may have failed), + * or > EOF which means that the copy operation is undefined, + * as by definition there is no data past EOF. + */ + if (errno == 0) { + errno = EINVAL; + } + copyfile_warn("Invalid file descriptor offset, cannot perform a sparse copy"); + goto error_exit; + } else if (src_start != (off_t) roundup(src_start, iosize) || + dst_start != (off_t) roundup(dst_start, iosize)) { + // If the starting offset isn't a multiple of the iosize, we can't do an entire sparse copy. + // Fall back to copyfile_data(), which will perform a full copy from the starting position. + return ENOTSUP; + } + + // Make sure that there is at least one hole in this [part of the] file. + first_hole_offset = lseek(src_fd, src_start, SEEK_HOLE); + if (first_hole_offset == -1 || first_hole_offset == src_size) { + /* + * Either an error occurred, the src starting position is EOF, or there are no + * holes in this [portion of the] source file. Regardless, we rewind the source file + * and return ENOTSUP so copyfile_data() can attempt a full copy. + */ + if (lseek(src_fd, src_start, SEEK_SET) == -1) { + goto error_exit; + } + return ENOTSUP; + } + + // We are ready to begin copying. + // First, truncate the destination file to zero out any existing contents. + // Then, truncate it again to its eventual size. + if (ftruncate(dst_fd, dst_start) == -1) { + copyfile_warn("Could not zero destination file before copy"); + goto error_exit; + } else if (ftruncate(dst_fd, dst_start + src_size - src_start) == -1) { + copyfile_warn("Could not set destination file size before copy"); + goto error_exit; + } + + // Set the source's offset to the first data section. + current_src_offset = lseek(src_fd, src_start, SEEK_DATA); + if (current_src_offset == -1) { + if (errno == ENXIO) { + // There are no more data sections in the file, so there's nothing to copy. + goto set_total_copied; + } + goto error_exit; + } + + // Now, current_src_offset points at the start of src's first data region. + // Update dst_fd to point to the same offset (respecting its start). + if (lseek(dst_fd, dst_start + current_src_offset - src_start, SEEK_SET) == -1) { + copyfile_warn("failed to set dst to first data section"); + goto error_exit; + } + + // Allocate a temporary buffer to copy data sections into. + bp = malloc(iosize); + if (bp == NULL) { + copyfile_warn("No memory for copy buffer"); + goto error_exit; + } + + /* + * Performing a sparse copy: + * While our source fd points to a data section (and is < EOF), read iosize bytes in. + * Then, write those bytes to the dest fd, using the same iosize. + * Finally, update our source and dest fds to point to the next data section. + */ + while ((nread = read(src_fd, bp, iosize)) > 0) { + ssize_t nwritten; + size_t left = nread; + void *ptr = bp; + int loop = 0; + + while (left > 0) { + nwritten = write(dst_fd, ptr, left); + switch (nwritten) { + case 0: + if (++loop > 5) { + copyfile_warn("writing to output %d times resulted in 0 bytes written", loop); + errno = EAGAIN; + goto error_exit; + } + break; + case -1: + copyfile_warn("writing to output file failed"); + if (status) { + int rv = (*status)(COPYFILE_COPY_DATA, COPYFILE_ERR, s, s->src, s->dst, s->ctx); + if (rv == COPYFILE_SKIP) { // Skip the data copy + errno = 0; + goto exit; + } else if (rv == COPYFILE_CONTINUE) { // Retry the write + errno = 0; + continue; + } + } + // If we get here, we either have no callback or it didn't tell us to continue. + goto error_exit; + break; + default: + left -= nwritten; + ptr = ((char*)ptr) + nwritten; + loop = 0; + break; + } + s->totalCopied += nwritten; + if (status) { + int rv = (*status)(COPYFILE_COPY_DATA, COPYFILE_PROGRESS, s, s->src, s->dst, s->ctx); + if (rv == COPYFILE_QUIT) { + errno = ECANCELED; + goto error_exit; + } + } + } + current_src_offset += nread; + + // Find the next area of src_fd to copy. + // Since data sections can be any length, we need see if current_src_offset points + // at a hole. + // If we get ENXIO, we're done copying (the last part of the file was a data section). + // If this is not a hole, we do not need to alter current_src_offset yet. + // If this is a hole, then we need to look for the next data section. + next_hole_offset = lseek(src_fd, current_src_offset, SEEK_HOLE); + if (next_hole_offset == -1) { + if (errno == ENXIO) { + break; // We're done copying data sections. + } + copyfile_warn("unable to find next hole in file during copy"); + goto error_exit; + } else if (next_hole_offset != current_src_offset) { + // Keep copying this data section (we must rewind src_fd to current_src_offset). + if (lseek(src_fd, current_src_offset, SEEK_SET) == -1) { + goto error_exit; + } + continue; + } + + // If we get here, we need to find the next data section to copy. + next_src_offset = lseek(src_fd, current_src_offset, SEEK_DATA); + if (next_src_offset == -1) { + if (errno == ENXIO) { + // There are no more data sections in this file, so we're done with the copy. + break; + } + + copyfile_warn("unable to advance src to next data section"); + goto error_exit; + } + + // Advance the dst_fd to match (taking into account where it started). + if (lseek(dst_fd, dst_start + (next_src_offset - src_start), SEEK_SET) == -1) { + copyfile_warn("unable to advance dst to next data section"); + goto error_exit; + } + + current_src_offset = next_src_offset; + } + if (nread < 0) { + copyfile_warn("error %d reading from %s", errno, s->src ? s->src : "(null src)"); + goto error_exit; + } + + // Punch holes where possible if needed. + if (use_punchhole) { + struct fpunchhole punchhole_args; + off_t hole_start = first_hole_offset, hole_end; + bool trailing_hole = true; + + // First, reset the source and destination file descriptors. + if (lseek(src_fd, src_start, SEEK_SET) == -1 || lseek(dst_fd, dst_start, SEEK_SET) == -1) { + copyfile_warn("unable to reset file descriptors to punch holes"); + // We have still copied the data, so there's no need to return an error here. + goto set_total_copied; + } + + // Now, find holes in the source (first_hole_offset already points to a source hole), + // determining their length by the presence of a data section. + while ((hole_end = lseek(src_fd, hole_start + (off_t) iosize, SEEK_DATA)) != -1) { + memset(&punchhole_args, 0, sizeof(punchhole_args)); + + // Fix up the offset and length for the destination file. + punchhole_args.fp_offset = hole_start - src_start + dst_start; + punchhole_args.fp_length = hole_end - hole_start; + if (fcntl(dst_fd, F_PUNCHHOLE, &punchhole_args) == -1) { + copyfile_warn("unable to punch hole in destination file, offset %lld length %lld", + hole_start - src_start + dst_start, hole_end - hole_start); + goto set_total_copied; + } + + // Now, find the start of the next hole. + hole_start = lseek(src_fd, hole_end, SEEK_HOLE); + if (hole_start == -1 || hole_start == src_size) { + // No more holes (or lseek failed), so break. + trailing_hole = false; + break; + } + } + + if ((hole_end == -1 || hole_start == -1) && errno != ENXIO) { + // A call to lseek() failed. Hole punching is best effort, so exit. + copyfile_warn("lseek during hole punching failed"); + goto set_total_copied; + } + + // We will still have a trailing hole to punch if the last lseek(SEEK_HOLE) succeeded. + if (trailing_hole) { + // Since we can only punch iosize-aligned holes, we must make sure the last hole + // is iosize-aligned. Unfortunately, no good truncate macros are in scope here, + // so we must round down the end of the trailing hole to an iosize boundary ourselves. + hole_end = (src_size % iosize == 0) ? src_size : roundup(src_size, iosize) - iosize; + + memset(&punchhole_args, 0, sizeof(punchhole_args)); + punchhole_args.fp_offset = hole_start - src_start + dst_start; + punchhole_args.fp_length = hole_end - hole_start; + if (fcntl(dst_fd, F_PUNCHHOLE, &punchhole_args) == -1) { + copyfile_warn("unable to punch trailing hole in destination file, offset %lld", + hole_start - src_start + dst_start); + goto set_total_copied; + } + } + } + +set_total_copied: + // Since we don't know in advance how many bytes we're copying, we advance this number + // as we copy, but to match copyfile_data() we set it here to the amount of bytes that would + // have been transferred in a full copy. + s->totalCopied = src_size - src_start; + +exit: + if (bp) { + free(bp); + bp = NULL; + } + + return rc; + +error_exit: + s->err = errno; + rc = -1; + goto exit; +} + /* * Attempt to copy the data section of a file. Using blockisize * is not necessarily the fastest -- it might be desirable to @@ -2037,8 +2328,8 @@ static int copyfile_data(copyfile_state_t s) char *bp = 0; ssize_t nread; int ret = 0; - size_t iBlocksize = 0; - size_t oBlocksize = 0; + size_t iBlocksize = 0, iMinblocksize = 0; + size_t oBlocksize = 0, oMinblocksize = 0; // If 0, we don't support sparse copying. const size_t onegig = 1 << 30; struct statfs sfs; copyfile_callback_t status = s->statuscb; @@ -2063,6 +2354,7 @@ static int copyfile_data(copyfile_state_t s) iBlocksize = s->sb.st_blksize; } else { iBlocksize = sfs.f_iosize; + iMinblocksize = sfs.f_bsize; } /* Work-around for 6453525, limit blocksize to 1G */ @@ -2070,21 +2362,58 @@ static int copyfile_data(copyfile_state_t s) iBlocksize = onegig; } - if ((bp = malloc(iBlocksize)) == NULL) - return -1; - - if (fstatfs(s->dst_fd, &sfs) == -1 || sfs.f_iosize == 0) { + if (fstatfs(s->dst_fd, &sfs) == -1) { + oBlocksize = iBlocksize; + } else if (sfs.f_iosize == 0) { oBlocksize = iBlocksize; + oMinblocksize = sfs.f_bsize; } else { + oMinblocksize = sfs.f_bsize; oBlocksize = sfs.f_iosize; if (oBlocksize > onegig) oBlocksize = onegig; } + s->totalCopied = 0; + + // If requested, attempt a sparse copy. + if (s->flags & COPYFILE_DATA_SPARSE) { + // Check if the source & destination volumes both support sparse files. + long min_hole_size = MIN(fpathconf(s->src_fd, _PC_MIN_HOLE_SIZE), + fpathconf(s->dst_fd, _PC_MIN_HOLE_SIZE)); + + // If holes are supported on both the source and dest volumes, make sure our min_hole_size + // is reasonable: if it's smaller than the source/dest block size, + // our copy performance will suffer (and we may not create sparse files). + if (iMinblocksize > 0 && oMinblocksize > 0 && (size_t) min_hole_size >= iMinblocksize + && (size_t) min_hole_size >= oMinblocksize) { + // Do the copy. + ret = copyfile_data_sparse(s, iMinblocksize, oMinblocksize); + + // If we returned an error, exit gracefully. + // If sparse copying is not supported, we try full copying if allowed by our caller. + if (ret == 0) { + goto exit; + } else if (ret != ENOTSUP) { + goto exit; + } + ret = 0; + } + + // Make sure we're allowed to perform non-sparse copying. + if (!(s->flags & COPYFILE_DATA)) { + ret = -1; + errno = ENOTSUP; + goto exit; + } + } + + if ((bp = malloc(iBlocksize)) == NULL) + return -1; + blen = iBlocksize; - s->totalCopied = 0; - /* If supported, do preallocation for Xsan / HFS volumes */ + /* If supported, do preallocation for Xsan / HFS / apfs volumes */ #ifdef F_PREALLOCATE { fstore_t fst; @@ -2846,6 +3175,9 @@ struct {char *s; int v;} opts[] = { COPYFILE_OPTION(VERBOSE) COPYFILE_OPTION(RECURSIVE) COPYFILE_OPTION(DEBUG) + COPYFILE_OPTION(CLONE) + COPYFILE_OPTION(CLONE_FORCE) + COPYFILE_OPTION(DATA_SPARSE) {NULL, 0} }; @@ -2892,7 +3224,7 @@ int main(int c, char *v[]) */ -#define offsetof(type, member) ((size_t)(&((type *)0)->member)) +#define offsetof(type, member) __builtin_offsetof(type, member) #define XATTR_MAXATTRLEN (16*1024*1024) diff --git a/copyfile.h b/copyfile.h index e071fef..efea9dd 100644 --- a/copyfile.h +++ b/copyfile.h @@ -106,6 +106,8 @@ typedef int (*copyfile_callback_t)(int, int, copyfile_state_t, const char *, con #define COPYFILE_RUN_IN_PLACE (1<<26) +#define COPYFILE_DATA_SPARSE (1<<27) + #define COPYFILE_VERBOSE (1<<30) #define COPYFILE_RECURSE_ERROR 0 diff --git a/copyfile.xcodeproj/project.pbxproj b/copyfile.xcodeproj/project.pbxproj index 87bc638..37eb53f 100644 --- a/copyfile.xcodeproj/project.pbxproj +++ b/copyfile.xcodeproj/project.pbxproj @@ -7,7 +7,13 @@ objects = { /* Begin PBXBuildFile section */ + 721D4F071EA95283000F0555 /* copyfile.c in Sources */ = {isa = PBXBuildFile; fileRef = FCCE17C1135A658F002CEE6D /* copyfile.c */; }; + 721D4F081EA95290000F0555 /* xattr_flags.c in Sources */ = {isa = PBXBuildFile; fileRef = 72406E621676C3C80099568B /* xattr_flags.c */; }; 72406E631676C3C80099568B /* xattr_flags.c in Sources */ = {isa = PBXBuildFile; fileRef = 72406E621676C3C80099568B /* xattr_flags.c */; }; + 726EE9DB1E9423E50017A5B9 /* main.c in Sources */ = {isa = PBXBuildFile; fileRef = 726EE9DA1E9423E50017A5B9 /* main.c */; }; + 726EE9E01E9425160017A5B9 /* sparse_test.c in Sources */ = {isa = PBXBuildFile; fileRef = 726EE9DE1E9425160017A5B9 /* sparse_test.c */; }; + 726EE9E41E946B320017A5B9 /* systemx.c in Sources */ = {isa = PBXBuildFile; fileRef = 726EE9E21E946B320017A5B9 /* systemx.c */; }; + 726EE9E61E946D590017A5B9 /* test_utils.c in Sources */ = {isa = PBXBuildFile; fileRef = 726EE9E51E946D590017A5B9 /* test_utils.c */; }; 72B4C0F41676C47D00C13E05 /* copyfile_private.h in Headers */ = {isa = PBXBuildFile; fileRef = 72B4C0F31676C47D00C13E05 /* copyfile_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; 72EAA3B016A72F4500833E98 /* xattr_flags.h in Headers */ = {isa = PBXBuildFile; fileRef = 72EAA3AF16A72F4500833E98 /* xattr_flags.h */; settings = {ATTRIBUTES = (Public, ); }; }; 86EF9F0A1834018C00AAB3F3 /* xattr_properties.h in Headers */ = {isa = PBXBuildFile; fileRef = 86EF9F091834018C00AAB3F3 /* xattr_properties.h */; settings = {ATTRIBUTES = (Private, ); }; }; @@ -15,9 +21,30 @@ FCCE17C4135A658F002CEE6D /* copyfile.h in Headers */ = {isa = PBXBuildFile; fileRef = FCCE17C2135A658F002CEE6D /* copyfile.h */; settings = {ATTRIBUTES = (Public, ); }; }; /* End PBXBuildFile section */ +/* Begin PBXCopyFilesBuildPhase section */ + 726EE9D61E9423E50017A5B9 /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; +/* End PBXCopyFilesBuildPhase section */ + /* Begin PBXFileReference section */ 3F1EFD4C185C4EB400D1C970 /* copyfile.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; name = copyfile.xcconfig; path = xcodescripts/copyfile.xcconfig; sourceTree = ""; }; + 721D4F051EA95008000F0555 /* libcopyfile.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libcopyfile.tbd; path = Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/usr/lib/system/libcopyfile.tbd; sourceTree = DEVELOPER_DIR; }; 72406E621676C3C80099568B /* xattr_flags.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = xattr_flags.c; sourceTree = ""; }; + 726EE9D81E9423E50017A5B9 /* copyfile_test */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = copyfile_test; sourceTree = BUILT_PRODUCTS_DIR; }; + 726EE9DA1E9423E50017A5B9 /* main.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = main.c; sourceTree = ""; }; + 726EE9DE1E9425160017A5B9 /* sparse_test.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = sparse_test.c; sourceTree = ""; }; + 726EE9DF1E9425160017A5B9 /* sparse_test.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = sparse_test.h; sourceTree = ""; }; + 726EE9E11E9427B40017A5B9 /* test_utils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = test_utils.h; sourceTree = ""; }; + 726EE9E21E946B320017A5B9 /* systemx.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = systemx.c; sourceTree = ""; }; + 726EE9E31E946B320017A5B9 /* systemx.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = systemx.h; sourceTree = ""; }; + 726EE9E51E946D590017A5B9 /* test_utils.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = test_utils.c; sourceTree = ""; }; 72B4C0F31676C47D00C13E05 /* copyfile_private.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = copyfile_private.h; sourceTree = ""; }; 72EAA3AF16A72F4500833E98 /* xattr_flags.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = xattr_flags.h; sourceTree = ""; }; 861E1C14180F0AF900E65B9A /* xattr_name_with_flags.3 */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = xattr_name_with_flags.3; sourceTree = ""; }; @@ -29,6 +56,13 @@ /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ + 726EE9D51E9423E50017A5B9 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; FCCE17B8135A6444002CEE6D /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -39,6 +73,28 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + 721D4F041EA95007000F0555 /* Frameworks */ = { + isa = PBXGroup; + children = ( + 721D4F051EA95008000F0555 /* libcopyfile.tbd */, + ); + name = Frameworks; + sourceTree = ""; + }; + 726EE9D91E9423E50017A5B9 /* copyfile_test */ = { + isa = PBXGroup; + children = ( + 726EE9DA1E9423E50017A5B9 /* main.c */, + 726EE9DE1E9425160017A5B9 /* sparse_test.c */, + 726EE9DF1E9425160017A5B9 /* sparse_test.h */, + 726EE9E51E946D590017A5B9 /* test_utils.c */, + 726EE9E11E9427B40017A5B9 /* test_utils.h */, + 726EE9E21E946B320017A5B9 /* systemx.c */, + 726EE9E31E946B320017A5B9 /* systemx.h */, + ); + path = copyfile_test; + sourceTree = ""; + }; FCCE17AB135A5FFB002CEE6D = { isa = PBXGroup; children = ( @@ -51,7 +107,9 @@ FCCE17C1135A658F002CEE6D /* copyfile.c */, 86EF9F091834018C00AAB3F3 /* xattr_properties.h */, FCCE17C2135A658F002CEE6D /* copyfile.h */, + 726EE9D91E9423E50017A5B9 /* copyfile_test */, FCCE17BC135A6444002CEE6D /* Products */, + 721D4F041EA95007000F0555 /* Frameworks */, ); sourceTree = ""; usesTabs = 1; @@ -60,6 +118,7 @@ isa = PBXGroup; children = ( FCCE17BB135A6444002CEE6D /* libcopyfile.dylib */, + 726EE9D81E9423E50017A5B9 /* copyfile_test */, ); name = Products; sourceTree = ""; @@ -81,6 +140,23 @@ /* End PBXHeadersBuildPhase section */ /* Begin PBXNativeTarget section */ + 726EE9D71E9423E50017A5B9 /* copyfile_test */ = { + isa = PBXNativeTarget; + buildConfigurationList = 726EE9DD1E9423E50017A5B9 /* Build configuration list for PBXNativeTarget "copyfile_test" */; + buildPhases = ( + 726EE9D41E9423E50017A5B9 /* Sources */, + 726EE9D51E9423E50017A5B9 /* Frameworks */, + 726EE9D61E9423E50017A5B9 /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = copyfile_test; + productName = copyfile_test; + productReference = 726EE9D81E9423E50017A5B9 /* copyfile_test */; + productType = "com.apple.product-type.tool"; + }; FCCE17BA135A6444002CEE6D /* copyfile */ = { isa = PBXNativeTarget; buildConfigurationList = FCCE17BE135A6444002CEE6D /* Build configuration list for PBXNativeTarget "copyfile" */; @@ -106,6 +182,12 @@ isa = PBXProject; attributes = { ORGANIZATIONNAME = "Apple Inc."; + TargetAttributes = { + 726EE9D71E9423E50017A5B9 = { + CreatedOnToolsVersion = 9.0; + ProvisioningStyle = Automatic; + }; + }; }; buildConfigurationList = FCCE17B0135A5FFB002CEE6D /* Build configuration list for PBXProject "copyfile" */; compatibilityVersion = "Xcode 3.2"; @@ -120,6 +202,7 @@ projectRoot = ""; targets = ( FCCE17BA135A6444002CEE6D /* copyfile */, + 726EE9D71E9423E50017A5B9 /* copyfile_test */, ); }; /* End PBXProject section */ @@ -142,6 +225,19 @@ /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ + 726EE9D41E9423E50017A5B9 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 721D4F081EA95290000F0555 /* xattr_flags.c in Sources */, + 721D4F071EA95283000F0555 /* copyfile.c in Sources */, + 726EE9DB1E9423E50017A5B9 /* main.c in Sources */, + 726EE9E61E946D590017A5B9 /* test_utils.c in Sources */, + 726EE9E41E946B320017A5B9 /* systemx.c in Sources */, + 726EE9E01E9425160017A5B9 /* sparse_test.c in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; FCCE17B7135A6444002CEE6D /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -154,6 +250,49 @@ /* End PBXSourcesBuildPhase section */ /* Begin XCBuildConfiguration section */ + 726EE9DC1E9423E50017A5B9 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGN_IDENTITY = "-"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(SDKROOT)/usr/lib/system", + ); + MACOSX_DEPLOYMENT_TARGET = 10.13; + MTL_ENABLE_DEBUG_INFO = NO; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + }; + name = Release; + }; FCCE17B3135A5FFB002CEE6D /* Release */ = { isa = XCBuildConfiguration; baseConfigurationReference = 3F1EFD4C185C4EB400D1C970 /* copyfile.xcconfig */; @@ -236,6 +375,14 @@ /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ + 726EE9DD1E9423E50017A5B9 /* Build configuration list for PBXNativeTarget "copyfile_test" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 726EE9DC1E9423E50017A5B9 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; FCCE17B0135A5FFB002CEE6D /* Build configuration list for PBXProject "copyfile" */ = { isa = XCConfigurationList; buildConfigurations = ( diff --git a/copyfile_test/main.c b/copyfile_test/main.c new file mode 100644 index 0000000..616dbe3 --- /dev/null +++ b/copyfile_test/main.c @@ -0,0 +1,65 @@ +// +// main.c +// copyfile_test +// + +#include +#include +#include +#include +#include +#include +#include + +#include "sparse_test.h" +#include "test_utils.h" + +#define DISK_IMAGE_SIZE_MB 512 + +#if TARGET_OS_OSX +#define TEST_DIR MOUNT_PATH +#define USING_DISK_IMAGE 1 +#else +#define TEST_DIR "/tmp/copyfile_test" +#define USING_DISK_IMAGE 0 +#endif // TARGET_OS_OSX + +#define MIN_BLOCKSIZE_B 512 +#define DEFAULT_BLOCKSIZE_B 4096 +#define MAX_BLOCKSIZE_B 16384 + +int main(__unused int argc, __unused const char * argv[]) { + bool failed = false; + struct statfs stb; + + // Create a disk image to run our tests in. + if (USING_DISK_IMAGE) { + disk_image_create(APFS_FSTYPE, DISK_IMAGE_SIZE_MB); + } else { + (void)removefile(TEST_DIR, NULL, REMOVEFILE_RECURSIVE); + assert_no_err(mkdir(TEST_DIR, 0777)); + } + + // Make sure the test directory exists, is apfs formatted, + // and that we have a sane block size. + assert_no_err(statfs(TEST_DIR, &stb)); + assert_no_err(memcmp(stb.f_fstypename, APFS_FSTYPE, sizeof(APFS_FSTYPE))); + if (stb.f_bsize < MIN_BLOCKSIZE_B || stb.f_bsize > MAX_BLOCKSIZE_B) { + stb.f_bsize = DEFAULT_BLOCKSIZE_B; + } + + // Run our tests. + sranddev(); + failed |= do_sparse_test(TEST_DIR, stb.f_bsize); + failed |= do_sparse_recursive_test(TEST_DIR, stb.f_bsize); + failed |= do_fcopyfile_offset_test(TEST_DIR, stb.f_bsize); + + // Cleanup the disk image we ran our tests on. + if (USING_DISK_IMAGE) { + disk_image_destroy(false); + } else { + (void)removefile(TEST_DIR, NULL, REMOVEFILE_RECURSIVE); + } + + return failed ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/copyfile_test/sparse_test.c b/copyfile_test/sparse_test.c new file mode 100644 index 0000000..ccced99 --- /dev/null +++ b/copyfile_test/sparse_test.c @@ -0,0 +1,402 @@ +// +// sparse_test.c +// copyfile_test +// + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../copyfile.h" +#include "sparse_test.h" +#include "test_utils.h" +#include "systemx.h" + +#define OPEN_FLAGS O_CREAT|O_TRUNC|O_RDWR +#define OPEN_PERM 0666 +#define MKDIR_PERM 0777 +#define NAME_MOD 999 + +/* + * Copy the file pointed to by src_fd (and orig_name) to copy_name, + * using copyfile()/fcopyfile() and COPYFILE_DATA. If do_sparse, also pass COPYFILE_DATA_SPARSE. + * Before copying, rewind src_fd to start_off. + */ +static bool test_copy(int src_fd, char* orig_name, char* copy_name, bool do_sparse, off_t start_off) { + struct stat orig_sb, copy_sb; + int copy_fd; + bool result = true; + copyfile_state_t cpf_state; + + // Get ready for the test. + memset(&orig_sb, 0, sizeof(orig_sb)); + memset(©_sb, 0, sizeof(copy_sb)); + assert_with_errno((cpf_state = copyfile_state_alloc()) != NULL); + assert_with_errno(lseek(src_fd, start_off, SEEK_SET) == start_off); + + // First, verify copyfile(). + copyfile_flags_t flags = COPYFILE_ALL; + if (do_sparse) { + flags |= COPYFILE_DATA_SPARSE; + } + assert_no_err(copyfile(orig_name, copy_name, cpf_state, flags)); + + // The file was (hopefully) copied. Now, we must verify three things: + // 1. If (do_sparse), verify that the copy is a sparse file. + // For now, let's approximate this by testing that the sizes of the two files are equal. + // 2. The copyfile_state_t for the copy returns that all bytes were copied. + // 3. The copy and the source have identical contents. + + // 1. The copy is a sparse file. + // 2. The copyfile_state_t for the copy returns that all bytes were copied. + assert_no_err(stat(orig_name, &orig_sb)); + assert_no_err(stat(copy_name, ©_sb)); + result &= verify_copy_sizes(&orig_sb, ©_sb, cpf_state, do_sparse, 0); + + // 3. The copy and the source have identical contents. + result &= verify_copy_contents(orig_name, copy_name); + + // Post-test cleanup. + assert_no_err(copyfile_state_free(cpf_state)); + assert_no_err(removefile(copy_name, NULL, REMOVEFILE_RECURSIVE)); + memset(&orig_sb, 0, sizeof(struct stat)); + memset(©_sb, 0, sizeof(struct stat)); + + // Next, verify fcopyfile(). + // Make an fd for the destination. + assert_with_errno((copy_fd = open(copy_name, OPEN_FLAGS, OPEN_PERM)) > 0); + + // Call fcopyfile(). + assert_with_errno((cpf_state = copyfile_state_alloc()) != NULL); + assert_no_err(fcopyfile(src_fd, copy_fd, cpf_state, flags)); + + // 1. The copy is a sparse file (if start_off is a multiple of the block size). + // 2. The copyfile_state_t for the copy returns that all bytes were copied. + assert_no_err(fstat(src_fd, &orig_sb)); + assert_no_err(fstat(copy_fd, ©_sb)); + result &= verify_copy_sizes(&orig_sb, ©_sb, cpf_state, + start_off % copy_sb.st_blksize ? false : do_sparse, start_off); + + // 3. The copy and the source have identical contents. + if (start_off == 0) { + result &= verify_copy_contents(orig_name, copy_name); + } + + // Post-test cleanup. + assert_no_err(copyfile_state_free(cpf_state)); + assert_no_err(removefile(copy_name, NULL, REMOVEFILE_RECURSIVE)); + assert_no_err(close(copy_fd)); + + return result; +} + + +// Sparse file creation functions. +// Each take the source file descriptor pointing at the beginning of the file and the block size. +// Each return the offset we should return the fd to before any copying should be performed. +typedef off_t (*creator_func)(int, off_t); + +static off_t write_start_and_end_holes(int fd, off_t block_size) { + assert_with_errno(pwrite(fd, "j", 1, block_size) == 1); + assert_no_err(ftruncate(fd, 3 * block_size)); + + assert_no_err(create_hole_in_fd(fd, 0, block_size)); + assert_no_err(create_hole_in_fd(fd, 2 * block_size, block_size)); + return 0; +} + +static off_t write_end_hole(int fd, off_t block_size) { + assert_with_errno(pwrite(fd, "n", 1, 0) == 1); + assert_no_err(ftruncate(fd, 16 * block_size)); + + assert_no_err(create_hole_in_fd(fd, block_size, 15 * block_size)); + return 0; +} + +static off_t write_start_hole(int fd, off_t block_size) { + assert_with_errno(pwrite(fd, "p", 1, 16 * block_size) == 1); + + assert_no_err(create_hole_in_fd(fd, 0, 16 * block_size)); + return 0; +} + +static off_t write_middle_hole(int fd, off_t block_size) { + assert_with_errno(pwrite(fd, "k", 1, 0) == 1); + assert_with_errno(pwrite(fd, "k", 1, 4 * block_size) == 1); + assert_no_err(ftruncate(fd, 5 * block_size)); + + assert_no_err(create_hole_in_fd(fd, block_size, 3 * block_size)); + return 0; +} + +static off_t write_start_and_middle_holes(int fd, off_t block_size) { + assert_with_errno(pwrite(fd, "l", 1, 16 * block_size) == 1); + assert_with_errno(pwrite(fd, "l", 1, 32 * block_size) == 1); + + assert_no_err(create_hole_in_fd(fd, 0, 16 * block_size)); + assert_no_err(create_hole_in_fd(fd, 17 * block_size, 15 * block_size)); + return 0; +} + +static off_t write_middle_and_end_holes(int fd, off_t block_size) { + assert_with_errno(pwrite(fd, "m", 1, 0) == 1); + assert_with_errno(pwrite(fd, "m", 1, 16 * block_size) == 1); + assert_no_err(ftruncate(fd, 32 * block_size)); + + assert_no_err(create_hole_in_fd(fd, block_size, 15 * block_size)); + assert_no_err(create_hole_in_fd(fd, 17 * block_size, 15 * block_size)); + return 0; +} + +static off_t write_no_sparse(int fd, __unused off_t block_size) { + assert_with_errno(pwrite(fd, "z", 1, 0) == 1); + return 0; +} + +static off_t write_sparse_odd_offset(int fd, off_t block_size) { + assert_with_errno(pwrite(fd, "q", 1, block_size) == 1); + + assert_no_err(create_hole_in_fd(fd, 0, block_size)); + // Return with the fd pointing at offset 1. + assert_with_errno(lseek(fd, 1, SEEK_SET) == 1); + return 1; +} + +static off_t write_sparse_bs_offset(int fd, off_t block_size) { + assert_with_errno(pwrite(fd, "a", 1, block_size) == 1); + assert_with_errno(pwrite(fd, "b", 1, 2 * block_size) == 1); + + assert_no_err(create_hole_in_fd(fd, 0, block_size)); + // Return with the fd pointing at block_size. + assert_with_errno(lseek(fd, block_size, SEEK_SET) == block_size); + return block_size; +} + +static off_t write_diff_adj_holes(int fd, off_t block_size) { + assert_with_errno(pwrite(fd, "w", 1, 0)); + assert_with_errno(pwrite(fd, "w", 1, 3 * block_size)); + assert_no_err(ftruncate(fd, 4 * block_size)); + + assert_no_err(create_hole_in_fd(fd, block_size, block_size)); + assert_no_err(create_hole_in_fd(fd, 2 * block_size, block_size)); + return 0; +} + +typedef struct { + creator_func func; // pointer to function to create a sparse file + const char * name; // null terminated string +} sparse_test_func; + +#define NUM_TEST_FUNCTIONS 10 +sparse_test_func test_functions[NUM_TEST_FUNCTIONS] = { + {write_start_and_end_holes, "start_and_end_holes"}, + {write_middle_hole, "middle_hole"}, + {write_start_and_middle_holes, "start_and_middle_holes"}, + {write_middle_and_end_holes, "middle_and_end_holes"}, + {write_end_hole, "end_hole"}, + {write_start_hole, "start_hole"}, + {write_no_sparse, "no_sparse"}, + {write_sparse_odd_offset, "write_sparse_odd_offset"}, + {write_sparse_bs_offset, "write_sparse_bs_offset"}, + {write_diff_adj_holes, "write_diff_adj_holes"} +}; + +bool do_sparse_test(const char* apfs_test_directory, size_t block_size) { + int fd, test_file_id; + char out_name[BSIZE_B], sparse_copy_name[BSIZE_B], full_copy_name[BSIZE_B]; + bool success = true, sub_test_success; + off_t start_off; + + for (size_t sub_test = 0; sub_test < NUM_TEST_FUNCTIONS; sub_test++) { + printf("START [%s]\n", test_functions[sub_test].name); + sub_test_success = false; + + // Make new names for this file and its copies. + test_file_id = rand() % NAME_MOD; + create_test_file_name(apfs_test_directory, "sparse", test_file_id, out_name); + create_test_file_name(apfs_test_directory, "copy_sparse", test_file_id, sparse_copy_name); + create_test_file_name(apfs_test_directory, "copy_full", test_file_id, full_copy_name); + + // Create the test file. + fd = open(out_name, OPEN_FLAGS, OPEN_PERM); + assert_with_errno(fd >= 0); + + // Write to the test file, making it sparse. + start_off = test_functions[sub_test].func(fd, (off_t) block_size); + assert_no_err(fsync(fd)); + + // Make sure that a sparse copy is successful. + sub_test_success = test_copy(fd, out_name, sparse_copy_name, true, start_off); + if (sub_test_success) { + // Make sure that a full copy is successful. + sub_test_success = test_copy(fd, out_name, full_copy_name, false, start_off); + } + + // Report the result on stdout. + if (!sub_test_success) { + printf("FAIL [%s]\n", test_functions[sub_test].name); + success = false; + } else { + printf("PASS [%s]\n", test_functions[sub_test].name); + } + + // Cleanup for the next test. + assert_no_err(close(fd)); + (void)removefile(out_name, NULL, 0); + (void)removefile(sparse_copy_name, NULL, 0); + (void)removefile(full_copy_name, NULL, 0); + memset(out_name, 0, BSIZE_B); + memset(sparse_copy_name, 0, BSIZE_B); + memset(full_copy_name, 0, BSIZE_B); + } + + return success ? EXIT_SUCCESS : EXIT_FAILURE; +} + +bool do_sparse_recursive_test(const char *apfs_test_directory, size_t block_size) { + int exterior_file_src_fd, interior_file_src_fd, test_file_id; + char exterior_dir_src[BSIZE_B] = {0}, interior_dir_src[BSIZE_B] = {0}, exterior_dir_dst[BSIZE_B] = {0}, interior_dir_dst[BSIZE_B] = {0}; + char exterior_file_src[BSIZE_B] = {0}, interior_file_src[BSIZE_B] = {0}, exterior_file_dst[BSIZE_B] = {0}, interior_file_dst[BSIZE_B] = {0}; + struct stat exterior_file_src_sb, interior_file_src_sb, exterior_file_dst_sb, interior_file_dst_sb; + bool success = true; + + printf("START [sparse_recursive]\n"); + + // Get ready for the test. + memset(&exterior_file_src_sb, 0, sizeof(exterior_file_src_sb)); + memset(&interior_file_src_sb, 0, sizeof(interior_file_src_sb)); + memset(&exterior_file_dst_sb, 0, sizeof(exterior_file_dst_sb)); + memset(&interior_file_dst_sb, 0, sizeof(interior_file_dst_sb)); + + // Construct our source layout. + assert_with_errno(snprintf(exterior_dir_src, BSIZE_B, "%s/recursive_src", apfs_test_directory) > 0); + assert_with_errno(snprintf(interior_dir_src, BSIZE_B, "%s/interior", exterior_dir_src) > 0); + + assert_no_err(mkdir(exterior_dir_src, MKDIR_PERM)); + assert_no_err(mkdir(interior_dir_src, MKDIR_PERM)); + + test_file_id = rand() % NAME_MOD; + create_test_file_name(exterior_dir_src, "exterior_sparse_file", test_file_id, exterior_file_src); + create_test_file_name(interior_dir_src, "interior_sparse_file", test_file_id, interior_file_src); + + // Create the actual test files. + exterior_file_src_fd = open(exterior_file_src, OPEN_FLAGS, OPEN_PERM); + assert_with_errno(exterior_file_src_fd >= 0); + write_start_and_end_holes(exterior_file_src_fd, block_size); + + interior_file_src_fd = open(interior_file_src, OPEN_FLAGS, OPEN_PERM); + assert_with_errno(interior_file_src_fd >= 0); + write_middle_hole(interior_file_src_fd, block_size); + + // Now, recursively copy our folder using sparse data copying. + assert_with_errno(snprintf(exterior_dir_dst, BSIZE_B, "%s/recursive_dst", apfs_test_directory) > 0); + assert_no_err(copyfile(exterior_dir_src, exterior_dir_dst, NULL, COPYFILE_ALL|COPYFILE_RECURSIVE|COPYFILE_DATA_SPARSE)); + + // The files were (hopefully) copied. Now, we must verify three things: + // 1. Verify that the copy is a sparse file. + // For now, let's approximate this by testing that the sizes of the two files are equal. + // 2. The copy and the source have identical contents. + + // First, construct our destination layout. + assert_with_errno(snprintf(exterior_dir_dst, BSIZE_B, "%s/recursive_dst", apfs_test_directory) > 0); + create_test_file_name(exterior_dir_dst, "exterior_sparse_file", test_file_id, exterior_file_dst); + assert_with_errno(snprintf(interior_dir_dst, BSIZE_B, "%s/interior", exterior_dir_dst) > 0); + create_test_file_name(interior_dir_dst, "interior_sparse_file", test_file_id, interior_file_dst); + + // 1. The copy is a sparse file. + assert_no_err(fstat(exterior_file_src_fd, &exterior_file_src_sb)); + assert_no_err(stat(exterior_file_dst, &exterior_file_dst_sb)); + + assert_no_err(fstat(interior_file_src_fd, &interior_file_src_sb)); + assert_no_err(stat(interior_file_dst, &interior_file_dst_sb)); + + success &= verify_copy_sizes(&exterior_file_src_sb, &exterior_file_dst_sb, NULL, true, 0); + success &= verify_copy_sizes(&interior_file_src_sb, &interior_file_dst_sb, NULL, true, 0); + + // 2. The copy and the source have identical contents. + success &= verify_copy_contents(exterior_file_src, exterior_file_dst); + success &= verify_copy_contents(interior_file_src, interior_file_dst); + + if (success) { + printf("PASS [sparse_recursive]\n"); + } else { + printf("FAIL [sparse_recursive]\n"); + } + + assert_no_err(close(interior_file_src_fd)); + assert_no_err(close(exterior_file_src_fd)); + (void)removefile(exterior_dir_src, NULL, REMOVEFILE_RECURSIVE); + (void)removefile(exterior_dir_dst, NULL, REMOVEFILE_RECURSIVE); + + return success ? EXIT_SUCCESS : EXIT_FAILURE; +} + +bool do_fcopyfile_offset_test(const char *apfs_test_directory, size_t block_size) { + int src_fd, sparse_copy_fd, full_copy_fd, test_file_id; + char out_name[BSIZE_B], sparse_copy_name[BSIZE_B], full_copy_name[BSIZE_B]; + bool success = true; + + printf("START [fcopyfile_offset]\n"); + + // Make new names for this file and its copies. + test_file_id = rand() % NAME_MOD; + + create_test_file_name(apfs_test_directory, "foff_sparse", test_file_id, out_name); + create_test_file_name(apfs_test_directory, "foff_copy_sparse", test_file_id, sparse_copy_name); + create_test_file_name(apfs_test_directory, "foff_copy_full", test_file_id, full_copy_name); + + // Create the test file. + src_fd = open(out_name, OPEN_FLAGS, OPEN_PERM); + assert_with_errno(src_fd >= 0); + // This writes 5 * block_size bytes. + assert_with_errno(lseek(src_fd, write_middle_hole(src_fd, block_size), SEEK_SET) == 0); + + // Create a sparse copy using fcopyfile(). + sparse_copy_fd = open(sparse_copy_name, OPEN_FLAGS, OPEN_PERM); + assert_with_errno(sparse_copy_fd >= 0); + + // Seek the sparse copy to a non-zero offset. + assert_with_errno(lseek(sparse_copy_fd, block_size, SEEK_SET) == (off_t) block_size); + // Write into the sparse copy a different byte. + assert_with_errno(pwrite(sparse_copy_fd, "z", 1, block_size) == 1); + + // Run fcopyfile(). + assert_no_err(fcopyfile(src_fd, sparse_copy_fd, NULL, COPYFILE_ALL|COPYFILE_DATA_SPARSE)); + + // Check that the source matches the copy at the appropriate region. + success &= verify_fd_contents(src_fd, 0, sparse_copy_fd, block_size, 4 * block_size); + + // Now, repeat the same procedure with a full copy. + assert_with_errno(lseek(src_fd, 0, SEEK_SET) == 0); + full_copy_fd = open(full_copy_name, OPEN_FLAGS, OPEN_PERM); + assert_with_errno(full_copy_name >= 0); + + assert_with_errno(lseek(full_copy_fd, block_size, SEEK_SET) == (off_t) block_size); + assert_with_errno(pwrite(full_copy_fd, "r", 1, block_size) == 1); + + // Run fcopyfile(). + assert_no_err(fcopyfile(src_fd, full_copy_fd, NULL, COPYFILE_ALL)); + + // Check that the source matches the copy at the appropriate region. + success &= verify_fd_contents(src_fd, 0, full_copy_fd, block_size, 4 * block_size); + + if (success) { + printf("PASS [fcopyfile_offset]\n"); + } else { + printf("FAIL [fcopyfile_offset]\n"); + } + + assert_no_err(close(full_copy_fd)); + assert_no_err(close(sparse_copy_fd)); + assert_no_err(close(src_fd)); + (void)removefile(full_copy_name, NULL, 0); + (void)removefile(sparse_copy_name, NULL, 0); + (void)removefile(out_name, NULL, 0); + + return success ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/copyfile_test/sparse_test.h b/copyfile_test/sparse_test.h new file mode 100644 index 0000000..414007e --- /dev/null +++ b/copyfile_test/sparse_test.h @@ -0,0 +1,13 @@ +// +// sparse_test.h +// copyfile_test +// + +#ifndef sparse_test_h +#define sparse_test_h + +bool do_sparse_test(const char *apfs_test_directory, size_t block_size); +bool do_sparse_recursive_test(const char *apfs_test_directory, size_t block_size); +bool do_fcopyfile_offset_test(const char *apfs_test_directory, size_t block_size); + +#endif /* sparse_test_h */ diff --git a/copyfile_test/systemx.c b/copyfile_test/systemx.c new file mode 100644 index 0000000..2b08bb1 --- /dev/null +++ b/copyfile_test/systemx.c @@ -0,0 +1,68 @@ +// +// systemx.c +// copyfile_test +// Stolen from the test routines from the apfs project. +// +#include +#include +#include +#include +#include +#include +#include + +#include "systemx.h" +#include "test_utils.h" + + +int __attribute__((sentinel)) systemx(const char *prog, ...) +{ + const char *args[64]; + + va_list ap; + + const char **parg = args; + + *parg++ = basename((char *)prog); + + va_start(ap, prog); + + bool quiet = false, quiet_stderr = false; + + while ((*parg = va_arg(ap, char *))) { + if (*parg == SYSTEMX_QUIET) { + quiet = true; + } else if (*parg == SYSTEMX_QUIET_STDERR) + quiet_stderr = true; + else + ++parg; + } + + va_end(ap); + + posix_spawn_file_actions_t facts, *pfacts = NULL; + + if (quiet || quiet_stderr) { + posix_spawn_file_actions_init(&facts); + if (quiet) + posix_spawn_file_actions_addopen(&facts, STDOUT_FILENO, "/dev/null", O_APPEND, 0); + if (quiet_stderr) + posix_spawn_file_actions_addopen(&facts, STDERR_FILENO, "/dev/null", O_APPEND, 0); + pfacts = &facts; + } + + pid_t pid; + assert_no_err(errno = posix_spawn(&pid, prog, pfacts, NULL, + (char * const *)args, NULL)); + + if (pfacts) + posix_spawn_file_actions_destroy(pfacts); + + int status; + assert(ignore_eintr(waitpid(pid, &status, 0), -1) == pid); + + if (WIFEXITED(status)) + return WEXITSTATUS(status); + else + return -1; +} diff --git a/copyfile_test/systemx.h b/copyfile_test/systemx.h new file mode 100644 index 0000000..3ef7388 --- /dev/null +++ b/copyfile_test/systemx.h @@ -0,0 +1,15 @@ +// +// systemx.h +// copyfile_test +// Stolen from the test routines from the apfs project. +// + +#ifndef systemx_h_ +#define systemx_h_ + +#define SYSTEMX_QUIET ((void *)1) +#define SYSTEMX_QUIET_STDERR ((void *)2) + +int __attribute__((sentinel)) systemx(const char *prog, ...); + +#endif /* systemx_h_ */ diff --git a/copyfile_test/test_utils.c b/copyfile_test/test_utils.c new file mode 100644 index 0000000..97c9efe --- /dev/null +++ b/copyfile_test/test_utils.c @@ -0,0 +1,148 @@ +// +// test_utils.c +// copyfile_test +// + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test_utils.h" +#include "systemx.h" + +bool verify_fd_contents(int orig_fd, off_t orig_pos, int copy_fd, off_t copy_pos, size_t length) { + // Read *length* contents of the two fds and make sure they compare as equal. + // Don't alter the position of either fd. + char orig_contents[length], copy_contents[length]; + bool equal; + + assert(orig_fd > 0 && copy_fd > 0); + memset(orig_contents, 0, length); + memset(copy_contents, 0, length); + + // Read the contents into our temporary buffers, and call memcmp(). + errno = 0; + ssize_t pread_res = pread(orig_fd, orig_contents, length, 0); + assert_with_errno(pread_res == (off_t) length); + assert_with_errno(pread(copy_fd, copy_contents, length, copy_pos) >= 0); + equal = (memcmp(orig_contents, copy_contents, length) == 0); + if (!equal) { + printf("original fd (%lld - %lld) did not match copy (%lld - %lld)\n", + orig_pos, orig_pos + length, copy_pos, copy_pos + length); + + // Find the first non-matching byte and print it out. + for (size_t bad_off = 0; bad_off < length; bad_off++) { + if (orig_contents[bad_off] != copy_contents[bad_off]) { + printf("first mismatch is at offset %zu, original 0x%llx COPY 0x%llx\n", + bad_off, orig_contents[bad_off], copy_contents[bad_off]); + break; + } + } + } + + return equal; +} + +bool verify_copy_contents(const char *orig_name, const char *copy_name) { + // Verify that the copy and the source have identical contents. + // Here, we just call out to 'diff' to do the work for us. + int rc = systemx(DIFF_PATH, orig_name, copy_name, SYSTEMX_QUIET, SYSTEMX_QUIET_STDERR, NULL); + if (rc != 0) { + printf("%s and %s are not identical: diff returned %d\n", orig_name, copy_name, rc); + exit(1); + } + + return !rc; +} + +bool verify_copy_sizes(struct stat *orig_sb, struct stat *copy_sb, copyfile_state_t cpf_state, + bool do_sparse, off_t src_start) { + off_t cpf_bytes_copied, blocks_offset; + bool result = true; + + // If requested, verify that the copy is a sparse file. + if (do_sparse) { + if (orig_sb->st_size - src_start != copy_sb->st_size) { + printf("original size - offset (%zd) != copy size (%zd)\n", + orig_sb->st_size - src_start, copy_sb->st_size); + result = false; + } + + blocks_offset = src_start / orig_sb->st_blksize; + if (orig_sb->st_blocks - blocks_offset < copy_sb->st_blocks) { + printf("original blocks - offset (%zd) < copy blocks (%zd)\n", + orig_sb->st_blocks - blocks_offset, copy_sb->st_blocks); + result = false; + } + } + + // Verify that the copyfile_state_t for the copy returns that all bytes were copied. + if (cpf_state) { + assert_no_err(copyfile_state_get(cpf_state, COPYFILE_STATE_COPIED, &cpf_bytes_copied)); + if (orig_sb->st_size - src_start != cpf_bytes_copied) { + printf("original size - start (%zd) != copied bytes (%zd)\n", + orig_sb->st_size - src_start, cpf_bytes_copied); + result = false; + } + } + + return result; +} + +int create_hole_in_fd(int fd, off_t offset, off_t length) { + struct fpunchhole punchhole_args = { + .fp_flags = 0, + .reserved = 0, + .fp_offset = offset, + .fp_length = length + }; + + return fcntl(fd, F_PUNCHHOLE, &punchhole_args); +} + + +void create_test_file_name(const char *dir, const char *postfix, int id, char *string_out) { + // Make a name for this new file and put it in out_name, which should be BSIZE_B bytes. + assert_with_errno(snprintf(string_out, BSIZE_B, "%s/testfile-%d.%s", dir, id, postfix) > 0); +} + +void disk_image_create(const char *fstype, size_t size_in_mb) { + char size[BSIZE_B]; + + // Set up good default values. + if (!fstype) { + fstype = DEFAULT_FSTYPE; + } + if (size_in_mb > MAX_DISK_IMAGE_SIZE_MB) { + size_in_mb = MAX_DISK_IMAGE_SIZE_MB; + } + assert_with_errno(snprintf(size, BSIZE_B, "%zum", size_in_mb) >= 3); + + // Unmount and remove the sparseimage if it already exists. + disk_image_destroy(true); + if (removefile(DISK_IMAGE_PATH, NULL, REMOVEFILE_RECURSIVE) < 0) { + assert_with_errno(errno == ENOENT); + } + + // Make the disk image. + assert_no_err(systemx(HDIUTIL_PATH, SYSTEMX_QUIET, "create", "-fs", fstype, + "-size", size, "-type", "SPARSE", "-volname", "apfs_sparse", + DISK_IMAGE_PATH, NULL)); + + // Attach the disk image. + assert_no_err(systemx(HDIUTIL_PATH, SYSTEMX_QUIET, "attach", DISK_IMAGE_PATH, NULL)); +} + +void disk_image_destroy(bool allow_failure) { + // If the caller allows, ignore any failures (also silence stderr). + if (allow_failure) { + systemx(HDIUTIL_PATH, "eject", MOUNT_PATH, SYSTEMX_QUIET, SYSTEMX_QUIET_STDERR, NULL); + } else { + assert_no_err(systemx(HDIUTIL_PATH, "eject", MOUNT_PATH, SYSTEMX_QUIET, NULL)); + } +} diff --git a/copyfile_test/test_utils.h b/copyfile_test/test_utils.h new file mode 100644 index 0000000..a8fabf8 --- /dev/null +++ b/copyfile_test/test_utils.h @@ -0,0 +1,119 @@ +// +// test_utils.h +// copyfile_test +// Based on the test routines from the apfs project. +// + +#ifndef test_utils_h +#define test_utils_h + +#include +#include +#include +#include +#include + +#include "../copyfile.h" + +#define BSIZE_B 128 +#define MAX_DISK_IMAGE_SIZE_MB 1024 + +#define DISK_IMAGE_PATH "/tmp/copyfile_sparse.sparseimage" +#define VOLUME_NAME "apfs_sparse" +#define DEFAULT_FSTYPE "JHFS+" +#define APFS_FSTYPE "apfs" + +// We assume that we're mounted on /Volumes. +#define MOUNT_PATH "/Volumes/" VOLUME_NAME + +#define HDIUTIL_PATH "/usr/bin/hdiutil" +#define DIFF_PATH "/usr/bin/diff" + +// Test routine helpers. +bool verify_fd_contents(int orig_fd, off_t orig_pos, int copy_fd, off_t copy_pos, size_t length); +bool verify_copy_contents(const char *orig_name, const char *copy_name); +bool verify_copy_sizes(struct stat *orig_sb, struct stat *copy_sb, copyfile_state_t cpf_state, + bool do_sparse, off_t src_start); +int create_hole_in_fd(int fd, off_t offset, off_t length); +void create_test_file_name(const char *dir, const char *postfix, int id, char *string_out); + +// Our disk image test functions. +void disk_image_create(const char *fstype, size_t size_in_mb); +void disk_image_destroy(bool allow_failure); + +// Assertion functions/macros for tests. +static inline void +__attribute__((format (printf, 3, 4))) +__attribute__((noreturn)) +assert_fail_(const char *file, int line, const char *assertion, ...) +{ + va_list args; + va_start(args, assertion); + char *msg; + vasprintf(&msg, assertion, args); + va_end(args); + printf("\n%s:%u: error: %s\n", file, line, msg); + exit(1); +} + +#define assert_fail(str, ...) \ + assert_fail_(__FILE__, __LINE__, str, ## __VA_ARGS__) + +#undef assert +#define assert(condition) \ + do { \ + if (!(condition)) \ + assert_fail_(__FILE__, __LINE__, \ + "assertion failed: %s", #condition); \ + } while (0) + +#define assert_with_errno_(condition, condition_str) \ + do { \ + if (!(condition)) \ + assert_fail_(__FILE__, __LINE__, "%s failed: %s", \ + condition_str, strerror(errno)); \ + } while (0) + +#define assert_with_errno(condition) \ + assert_with_errno_((condition), #condition) + +#define assert_no_err(condition) \ + assert_with_errno_(!(condition), #condition) + +#define assert_equal(lhs, rhs, fmt) \ + do { \ + typeof (lhs) lhs_ = (lhs); \ + typeof (lhs) rhs_ = (rhs); \ + if (lhs_ != rhs_) \ + assert_fail(#lhs " (" fmt ") != " #rhs " (" fmt ")", \ + lhs_, rhs_); \ + } while (0) + +#define assert_equal_(lhs, rhs, lhs_str, rhs_str, fmt) \ + do { \ + typeof (lhs) lhs_ = (lhs); \ + typeof (lhs) rhs_ = (rhs); \ + if (lhs_ != rhs_) \ + assert_fail(lhs_str " (" fmt ") != " rhs_str " (" fmt ")", \ + lhs_, rhs_); \ + } while (0) + +#define assert_equal_int(lhs, rhs) assert_equal_(lhs, rhs, #lhs, #rhs, "%d") +#define assert_equal_ll(lhs, rhs) assert_equal_(lhs, rhs, #lhs, #rhs, "%lld") +#define assert_equal_str(lhs, rhs) \ + do { \ + const char *lhs_ = (lhs), *rhs_ = (rhs); \ + if (strcmp(lhs_, rhs_)) \ + assert_fail("\"%s\" != \"%s\"", lhs_, rhs_); \ + } while (0) + +#define ignore_eintr(x, error_val) \ + ({ \ + typeof(x) eintr_ret_; \ + do { \ + eintr_ret_ = (x); \ + } while (eintr_ret_ == (error_val) && errno == EINTR); \ + eintr_ret_; \ + }) + +#endif /* test_utils_h */ -- 2.45.2