+/*
+ * ubc_getpager
+ *
+ * Get the pager associated with the ubc_info associated with the vnode.
+ *
+ * Parameters: vp The vnode to obtain the pager from
+ *
+ * Returns: !VNODE_PAGER_NULL The memory_object_t for the pager
+ * VNODE_PAGER_NULL There is no ubc_info for this vnode
+ *
+ * Notes: For each vnode that has a ubc_info associated with it, that
+ * ubc_info SHALL have a pager associated with it, so in the
+ * normal case, it's impossible to return VNODE_PAGER_NULL for
+ * a vnode with an associated ubc_info.
+ */
+__private_extern__ memory_object_t
+ubc_getpager(struct vnode *vp)
+{
+ if (UBCINFOEXISTS(vp)) {
+ return vp->v_ubcinfo->ui_pager;
+ }
+
+ return 0;
+}
+
+
+/*
+ * ubc_getobject
+ *
+ * Get the memory object control associated with the ubc_info associated with
+ * the vnode
+ *
+ * Parameters: vp The vnode to obtain the memory object
+ * from
+ * flags DEPRECATED
+ *
+ * Returns: !MEMORY_OBJECT_CONTROL_NULL
+ * MEMORY_OBJECT_CONTROL_NULL
+ *
+ * Notes: Historically, if the flags were not "do not reactivate", this
+ * function would look up the memory object using the pager if
+ * it did not exist (this could be the case if the vnode had
+ * been previously reactivated). The flags would also permit a
+ * hold to be requested, which would have created an object
+ * reference, if one had not already existed. This usage is
+ * deprecated, as it would permit a race between finding and
+ * taking the reference vs. a single reference being dropped in
+ * another thread.
+ */
+memory_object_control_t
+ubc_getobject(struct vnode *vp, __unused int flags)
+{
+ if (UBCINFOEXISTS(vp)) {
+ return vp->v_ubcinfo->ui_control;
+ }
+
+ return MEMORY_OBJECT_CONTROL_NULL;
+}
+
+/*
+ * ubc_blktooff
+ *
+ * Convert a given block number to a memory backing object (file) offset for a
+ * given vnode
+ *
+ * Parameters: vp The vnode in which the block is located
+ * blkno The block number to convert
+ *
+ * Returns: !-1 The offset into the backing object
+ * -1 There is no ubc_info associated with
+ * the vnode
+ * -1 An error occurred in the underlying VFS
+ * while translating the block to an
+ * offset; the most likely cause is that
+ * the caller specified a block past the
+ * end of the file, but this could also be
+ * any other error from VNOP_BLKTOOFF().
+ *
+ * Note: Representing the error in band loses some information, but does
+ * not occlude a valid offset, since an off_t of -1 is normally
+ * used to represent EOF. If we had a more reliable constant in
+ * our header files for it (i.e. explicitly cast to an off_t), we
+ * would use it here instead.
+ */
+off_t
+ubc_blktooff(vnode_t vp, daddr64_t blkno)
+{
+ off_t file_offset = -1;
+ int error;
+
+ if (UBCINFOEXISTS(vp)) {
+ error = VNOP_BLKTOOFF(vp, blkno, &file_offset);
+ if (error) {
+ file_offset = -1;
+ }
+ }
+
+ return file_offset;
+}
+
+
+/*
+ * ubc_offtoblk
+ *
+ * Convert a given offset in a memory backing object into a block number for a
+ * given vnode
+ *
+ * Parameters: vp The vnode in which the offset is
+ * located
+ * offset The offset into the backing object
+ *
+ * Returns: !-1 The returned block number
+ * -1 There is no ubc_info associated with
+ * the vnode
+ * -1 An error occurred in the underlying VFS
+ * while translating the block to an
+ * offset; the most likely cause is that
+ * the caller specified a block past the
+ * end of the file, but this could also be
+ * any other error from VNOP_OFFTOBLK().
+ *
+ * Note: Representing the error in band loses some information, but does
+ * not occlude a valid block number, since block numbers exceed
+ * the valid range for offsets, due to their relative sizes. If
+ * we had a more reliable constant than -1 in our header files
+ * for it (i.e. explicitly cast to an daddr64_t), we would use it
+ * here instead.
+ */
+daddr64_t
+ubc_offtoblk(vnode_t vp, off_t offset)
+{
+ daddr64_t blkno = -1;
+ int error = 0;
+
+ if (UBCINFOEXISTS(vp)) {
+ error = VNOP_OFFTOBLK(vp, offset, &blkno);
+ if (error) {
+ blkno = -1;
+ }
+ }
+
+ return blkno;
+}
+
+
+/*
+ * ubc_pages_resident
+ *
+ * Determine whether or not a given vnode has pages resident via the memory
+ * object control associated with the ubc_info associated with the vnode
+ *
+ * Parameters: vp The vnode we want to know about
+ *
+ * Returns: 1 Yes
+ * 0 No
+ */
+int
+ubc_pages_resident(vnode_t vp)
+{
+ kern_return_t kret;
+ boolean_t has_pages_resident;
+
+ if (!UBCINFOEXISTS(vp)) {
+ return 0;
+ }
+
+ /*
+ * The following call may fail if an invalid ui_control is specified,
+ * or if there is no VM object associated with the control object. In
+ * either case, reacting to it as if there were no pages resident will
+ * result in correct behavior.
+ */
+ kret = memory_object_pages_resident(vp->v_ubcinfo->ui_control, &has_pages_resident);
+
+ if (kret != KERN_SUCCESS) {
+ return 0;
+ }
+
+ if (has_pages_resident == TRUE) {
+ return 1;
+ }
+
+ return 0;
+}
+
+/*
+ * ubc_msync
+ *
+ * Clean and/or invalidate a range in the memory object that backs this vnode
+ *
+ * Parameters: vp The vnode whose associated ubc_info's
+ * associated memory object is to have a
+ * range invalidated within it
+ * beg_off The start of the range, as an offset
+ * end_off The end of the range, as an offset
+ * resid_off The address of an off_t supplied by the
+ * caller; may be set to NULL to ignore
+ * flags See ubc_msync_internal()
+ *
+ * Returns: 0 Success
+ * !0 Failure; an errno is returned
+ *
+ * Implicit Returns:
+ * *resid_off, modified If non-NULL, the contents are ALWAYS
+ * modified; they are initialized to the
+ * beg_off, and in case of an I/O error,
+ * the difference between beg_off and the
+ * current value will reflect what was
+ * able to be written before the error
+ * occurred. If no error is returned, the
+ * value of the resid_off is undefined; do
+ * NOT use it in place of end_off if you
+ * intend to increment from the end of the
+ * last call and call iteratively.
+ *
+ * Notes: see ubc_msync_internal() for more detailed information.
+ *
+ */
+errno_t
+ubc_msync(vnode_t vp, off_t beg_off, off_t end_off, off_t *resid_off, int flags)
+{
+ int retval;
+ int io_errno = 0;
+
+ if (resid_off) {
+ *resid_off = beg_off;
+ }
+
+ retval = ubc_msync_internal(vp, beg_off, end_off, resid_off, flags, &io_errno);
+
+ if (retval == 0 && io_errno == 0) {
+ return EINVAL;
+ }
+ return io_errno;
+}
+
+
+/*
+ * ubc_msync_internal
+ *
+ * Clean and/or invalidate a range in the memory object that backs this vnode
+ *
+ * Parameters: vp The vnode whose associated ubc_info's
+ * associated memory object is to have a
+ * range invalidated within it
+ * beg_off The start of the range, as an offset
+ * end_off The end of the range, as an offset
+ * resid_off The address of an off_t supplied by the
+ * caller; may be set to NULL to ignore
+ * flags MUST contain at least one of the flags
+ * UBC_INVALIDATE, UBC_PUSHDIRTY, or
+ * UBC_PUSHALL; if UBC_PUSHDIRTY is used,
+ * UBC_SYNC may also be specified to cause
+ * this function to block until the
+ * operation is complete. The behavior
+ * of UBC_SYNC is otherwise undefined.
+ * io_errno The address of an int to contain the
+ * errno from a failed I/O operation, if
+ * one occurs; may be set to NULL to
+ * ignore
+ *
+ * Returns: 1 Success
+ * 0 Failure
+ *
+ * Implicit Returns:
+ * *resid_off, modified The contents of this offset MAY be
+ * modified; in case of an I/O error, the
+ * difference between beg_off and the
+ * current value will reflect what was
+ * able to be written before the error
+ * occurred.
+ * *io_errno, modified The contents of this offset are set to
+ * an errno, if an error occurs; if the
+ * caller supplies an io_errno parameter,
+ * they should be careful to initialize it
+ * to 0 before calling this function to
+ * enable them to distinguish an error
+ * with a valid *resid_off from an invalid
+ * one, and to avoid potentially falsely
+ * reporting an error, depending on use.
+ *
+ * Notes: If there is no ubc_info associated with the vnode supplied,
+ * this function immediately returns success.
+ *
+ * If the value of end_off is less than or equal to beg_off, this
+ * function immediately returns success; that is, end_off is NOT
+ * inclusive.
+ *
+ * IMPORTANT: one of the flags UBC_INVALIDATE, UBC_PUSHDIRTY, or
+ * UBC_PUSHALL MUST be specified; that is, it is NOT possible to
+ * attempt to block on in-progress I/O by calling this function
+ * with UBC_PUSHDIRTY, and then later call it with just UBC_SYNC
+ * in order to block pending on the I/O already in progress.
+ *
+ * The start offset is truncated to the page boundary and the
+ * size is adjusted to include the last page in the range; that
+ * is, end_off on exactly a page boundary will not change if it
+ * is rounded, and the range of bytes written will be from the
+ * truncate beg_off to the rounded (end_off - 1).
+ */
+static int
+ubc_msync_internal(vnode_t vp, off_t beg_off, off_t end_off, off_t *resid_off, int flags, int *io_errno)
+{
+ memory_object_size_t tsize;
+ kern_return_t kret;
+ int request_flags = 0;
+ int flush_flags = MEMORY_OBJECT_RETURN_NONE;
+
+ if (!UBCINFOEXISTS(vp)) {
+ return 0;
+ }
+ if ((flags & (UBC_INVALIDATE | UBC_PUSHDIRTY | UBC_PUSHALL)) == 0) {
+ return 0;
+ }
+ if (end_off <= beg_off) {
+ return 1;
+ }
+
+ if (flags & UBC_INVALIDATE) {
+ /*
+ * discard the resident pages
+ */
+ request_flags = (MEMORY_OBJECT_DATA_FLUSH | MEMORY_OBJECT_DATA_NO_CHANGE);
+ }
+
+ if (flags & UBC_SYNC) {
+ /*
+ * wait for all the I/O to complete before returning
+ */
+ request_flags |= MEMORY_OBJECT_IO_SYNC;
+ }
+
+ if (flags & UBC_PUSHDIRTY) {
+ /*
+ * we only return the dirty pages in the range
+ */
+ flush_flags = MEMORY_OBJECT_RETURN_DIRTY;
+ }
+
+ if (flags & UBC_PUSHALL) {
+ /*
+ * then return all the interesting pages in the range (both
+ * dirty and precious) to the pager
+ */
+ flush_flags = MEMORY_OBJECT_RETURN_ALL;
+ }
+
+ beg_off = trunc_page_64(beg_off);
+ end_off = round_page_64(end_off);
+ tsize = (memory_object_size_t)end_off - beg_off;
+
+ /* flush and/or invalidate pages in the range requested */
+ kret = memory_object_lock_request(vp->v_ubcinfo->ui_control,
+ beg_off, tsize,
+ (memory_object_offset_t *)resid_off,
+ io_errno, flush_flags, request_flags,
+ VM_PROT_NO_CHANGE);
+
+ return (kret == KERN_SUCCESS) ? 1 : 0;
+}
+
+
+/*
+ * ubc_map
+ *
+ * Explicitly map a vnode that has an associate ubc_info, and add a reference
+ * to it for the ubc system, if there isn't one already, so it will not be
+ * recycled while it's in use, and set flags on the ubc_info to indicate that
+ * we have done this
+ *
+ * Parameters: vp The vnode to map
+ * flags The mapping flags for the vnode; this
+ * will be a combination of one or more of
+ * PROT_READ, PROT_WRITE, and PROT_EXEC
+ *
+ * Returns: 0 Success
+ * EPERM Permission was denied
+ *
+ * Notes: An I/O reference on the vnode must already be held on entry
+ *
+ * If there is no ubc_info associated with the vnode, this function
+ * will return success.
+ *
+ * If a permission error occurs, this function will return
+ * failure; all other failures will cause this function to return
+ * success.
+ *
+ * IMPORTANT: This is an internal use function, and its symbols
+ * are not exported, hence its error checking is not very robust.
+ * It is primarily used by:
+ *
+ * o mmap(), when mapping a file
+ * o When mapping a shared file (a shared library in the
+ * shared segment region)
+ * o When loading a program image during the exec process
+ *
+ * ...all of these uses ignore the return code, and any fault that
+ * results later because of a failure is handled in the fix-up path
+ * of the fault handler. The interface exists primarily as a
+ * performance hint.
+ *
+ * Given that third party implementation of the type of interfaces
+ * that would use this function, such as alternative executable
+ * formats, etc., are unsupported, this function is not exported
+ * for general use.
+ *
+ * The extra reference is held until the VM system unmaps the
+ * vnode from its own context to maintain a vnode reference in
+ * cases like open()/mmap()/close(), which leave the backing
+ * object referenced by a mapped memory region in a process
+ * address space.
+ */
+__private_extern__ int
+ubc_map(vnode_t vp, int flags)
+{
+ struct ubc_info *uip;
+ int error = 0;
+ int need_ref = 0;
+ int need_wakeup = 0;
+
+ if (UBCINFOEXISTS(vp)) {
+ vnode_lock(vp);
+ uip = vp->v_ubcinfo;
+
+ while (ISSET(uip->ui_flags, UI_MAPBUSY)) {
+ SET(uip->ui_flags, UI_MAPWAITING);
+ (void) msleep(&uip->ui_flags, &vp->v_lock,
+ PRIBIO, "ubc_map", NULL);
+ }
+ SET(uip->ui_flags, UI_MAPBUSY);
+ vnode_unlock(vp);
+
+ error = VNOP_MMAP(vp, flags, vfs_context_current());
+
+ /*
+ * rdar://problem/22587101 required that we stop propagating
+ * EPERM up the stack. Otherwise, we would have to funnel up
+ * the error at all the call sites for memory_object_map().
+ * The risk is in having to undo the map/object/entry state at
+ * all these call sites. It would also affect more than just mmap()
+ * e.g. vm_remap().
+ *
+ * if (error != EPERM)
+ * error = 0;
+ */
+
+ error = 0;
+
+ vnode_lock_spin(vp);
+
+ if (error == 0) {
+ if (!ISSET(uip->ui_flags, UI_ISMAPPED)) {
+ need_ref = 1;
+ }
+ SET(uip->ui_flags, (UI_WASMAPPED | UI_ISMAPPED));
+ if (flags & PROT_WRITE) {
+ SET(uip->ui_flags, UI_MAPPEDWRITE);
+ }
+ }
+ CLR(uip->ui_flags, UI_MAPBUSY);
+
+ if (ISSET(uip->ui_flags, UI_MAPWAITING)) {
+ CLR(uip->ui_flags, UI_MAPWAITING);
+ need_wakeup = 1;
+ }
+ vnode_unlock(vp);
+
+ if (need_wakeup) {
+ wakeup(&uip->ui_flags);
+ }
+
+ if (need_ref) {
+ /*
+ * Make sure we get a ref as we can't unwind from here
+ */
+ if (vnode_ref_ext(vp, 0, VNODE_REF_FORCE)) {
+ panic("%s : VNODE_REF_FORCE failed\n", __FUNCTION__);
+ }
+ /*
+ * Vnodes that are on "unreliable" media (like disk
+ * images, network filesystems, 3rd-party filesystems,
+ * and possibly external devices) could see their
+ * contents be changed via the backing store without
+ * triggering copy-on-write, so we can't fully rely
+ * on copy-on-write and might have to resort to
+ * copy-on-read to protect "privileged" processes and
+ * prevent privilege escalation.
+ *
+ * The root filesystem is considered "reliable" because
+ * there's not much point in trying to protect
+ * ourselves from such a vulnerability and the extra
+ * cost of copy-on-read (CPU time and memory pressure)
+ * could result in some serious regressions.
+ */
+ if (vp->v_mount != NULL &&
+ ((vp->v_mount->mnt_flag & MNT_ROOTFS) ||
+ vnode_on_reliable_media(vp))) {
+ /*
+ * This vnode is deemed "reliable" so mark
+ * its VM object as "trusted".
+ */
+ memory_object_mark_trusted(uip->ui_control);
+ } else {
+// printf("BUGGYCOW: %s:%d vp %p \"%s\" in mnt %p \"%s\" is untrusted\n", __FUNCTION__, __LINE__, vp, vp->v_name, vp->v_mount, vp->v_mount->mnt_vnodecovered->v_name);
+ }
+ }
+ }
+ return error;
+}
+
+
+/*
+ * ubc_destroy_named
+ *
+ * Destroy the named memory object associated with the ubc_info control object
+ * associated with the designated vnode, if there is a ubc_info associated
+ * with the vnode, and a control object is associated with it
+ *
+ * Parameters: vp The designated vnode
+ *
+ * Returns: (void)
+ *
+ * Notes: This function is called on vnode termination for all vnodes,
+ * and must therefore not assume that there is a ubc_info that is
+ * associated with the vnode, nor that there is a control object
+ * associated with the ubc_info.
+ *
+ * If all the conditions necessary are present, this function
+ * calls memory_object_destory(), which will in turn end up
+ * calling ubc_unmap() to release any vnode references that were
+ * established via ubc_map().
+ *
+ * IMPORTANT: This is an internal use function that is used
+ * exclusively by the internal use function vclean().
+ */
+__private_extern__ void
+ubc_destroy_named(vnode_t vp)
+{
+ memory_object_control_t control;
+ struct ubc_info *uip;
+ kern_return_t kret;
+
+ if (UBCINFOEXISTS(vp)) {
+ uip = vp->v_ubcinfo;
+
+ /* Terminate the memory object */
+ control = ubc_getobject(vp, UBC_HOLDOBJECT);
+ if (control != MEMORY_OBJECT_CONTROL_NULL) {
+ kret = memory_object_destroy(control, 0);
+ if (kret != KERN_SUCCESS) {
+ panic("ubc_destroy_named: memory_object_destroy failed");
+ }
+ }
+ }
+}
+
+
+/*
+ * ubc_isinuse
+ *
+ * Determine whether or not a vnode is currently in use by ubc at a level in
+ * excess of the requested busycount
+ *
+ * Parameters: vp The vnode to check
+ * busycount The threshold busy count, used to bias
+ * the count usually already held by the
+ * caller to avoid races
+ *
+ * Returns: 1 The vnode is in use over the threshold
+ * 0 The vnode is not in use over the
+ * threshold
+ *
+ * Notes: Because the vnode is only held locked while actually asking
+ * the use count, this function only represents a snapshot of the
+ * current state of the vnode. If more accurate information is
+ * required, an additional busycount should be held by the caller
+ * and a non-zero busycount used.
+ *
+ * If there is no ubc_info associated with the vnode, this
+ * function will report that the vnode is not in use by ubc.
+ */
+int
+ubc_isinuse(struct vnode *vp, int busycount)
+{
+ if (!UBCINFOEXISTS(vp)) {
+ return 0;
+ }
+ return ubc_isinuse_locked(vp, busycount, 0);
+}
+
+
+/*
+ * ubc_isinuse_locked
+ *
+ * Determine whether or not a vnode is currently in use by ubc at a level in
+ * excess of the requested busycount
+ *
+ * Parameters: vp The vnode to check
+ * busycount The threshold busy count, used to bias
+ * the count usually already held by the
+ * caller to avoid races
+ * locked True if the vnode is already locked by
+ * the caller
+ *
+ * Returns: 1 The vnode is in use over the threshold
+ * 0 The vnode is not in use over the
+ * threshold
+ *
+ * Notes: If the vnode is not locked on entry, it is locked while
+ * actually asking the use count. If this is the case, this
+ * function only represents a snapshot of the current state of
+ * the vnode. If more accurate information is required, the
+ * vnode lock should be held by the caller, otherwise an
+ * additional busycount should be held by the caller and a
+ * non-zero busycount used.
+ *
+ * If there is no ubc_info associated with the vnode, this
+ * function will report that the vnode is not in use by ubc.
+ */
+int
+ubc_isinuse_locked(struct vnode *vp, int busycount, int locked)
+{
+ int retval = 0;
+
+
+ if (!locked) {
+ vnode_lock_spin(vp);
+ }
+
+ if ((vp->v_usecount - vp->v_kusecount) > busycount) {
+ retval = 1;
+ }
+
+ if (!locked) {
+ vnode_unlock(vp);
+ }
+ return retval;
+}
+
+
+/*
+ * ubc_unmap
+ *
+ * Reverse the effects of a ubc_map() call for a given vnode
+ *
+ * Parameters: vp vnode to unmap from ubc
+ *
+ * Returns: (void)
+ *
+ * Notes: This is an internal use function used by vnode_pager_unmap().
+ * It will attempt to obtain a reference on the supplied vnode,
+ * and if it can do so, and there is an associated ubc_info, and
+ * the flags indicate that it was mapped via ubc_map(), then the
+ * flag is cleared, the mapping removed, and the reference taken
+ * by ubc_map() is released.
+ *
+ * IMPORTANT: This MUST only be called by the VM
+ * to prevent race conditions.
+ */
+__private_extern__ void
+ubc_unmap(struct vnode *vp)
+{
+ struct ubc_info *uip;
+ int need_rele = 0;
+ int need_wakeup = 0;
+
+ if (vnode_getwithref(vp)) {
+ return;
+ }
+
+ if (UBCINFOEXISTS(vp)) {
+ bool want_fsevent = false;
+
+ vnode_lock(vp);
+ uip = vp->v_ubcinfo;
+
+ while (ISSET(uip->ui_flags, UI_MAPBUSY)) {
+ SET(uip->ui_flags, UI_MAPWAITING);
+ (void) msleep(&uip->ui_flags, &vp->v_lock,
+ PRIBIO, "ubc_unmap", NULL);
+ }
+ SET(uip->ui_flags, UI_MAPBUSY);
+
+ if (ISSET(uip->ui_flags, UI_ISMAPPED)) {
+ if (ISSET(uip->ui_flags, UI_MAPPEDWRITE)) {
+ want_fsevent = true;
+ }
+
+ need_rele = 1;
+
+ /*
+ * We want to clear the mapped flags after we've called
+ * VNOP_MNOMAP to avoid certain races and allow
+ * VNOP_MNOMAP to call ubc_is_mapped_writable.
+ */
+ }
+ vnode_unlock(vp);
+
+ if (need_rele) {
+ vfs_context_t ctx = vfs_context_current();
+
+ (void)VNOP_MNOMAP(vp, ctx);
+
+#if CONFIG_FSE
+ /*
+ * Why do we want an fsevent here? Normally the
+ * content modified fsevent is posted when a file is
+ * closed and only if it's written to via conventional
+ * means. It's perfectly legal to close a file and
+ * keep your mappings and we don't currently track
+ * whether it was written to via a mapping.
+ * Therefore, we need to post an fsevent here if the
+ * file was mapped writable. This may result in false
+ * events, i.e. we post a notification when nothing
+ * has really changed.
+ */
+ if (want_fsevent && need_fsevent(FSE_CONTENT_MODIFIED, vp)) {
+ add_fsevent(FSE_CONTENT_MODIFIED, ctx,
+ FSE_ARG_VNODE, vp,
+ FSE_ARG_DONE);
+ }
+#endif
+
+ vnode_rele(vp);
+ }
+
+ vnode_lock_spin(vp);
+
+ if (need_rele) {
+ CLR(uip->ui_flags, UI_ISMAPPED | UI_MAPPEDWRITE);
+ }
+
+ CLR(uip->ui_flags, UI_MAPBUSY);
+
+ if (ISSET(uip->ui_flags, UI_MAPWAITING)) {
+ CLR(uip->ui_flags, UI_MAPWAITING);
+ need_wakeup = 1;
+ }
+ vnode_unlock(vp);
+
+ if (need_wakeup) {
+ wakeup(&uip->ui_flags);
+ }
+ }
+ /*
+ * the drop of the vnode ref will cleanup
+ */
+ vnode_put(vp);
+}
+
+
+/*
+ * ubc_page_op
+ *
+ * Manipulate individual page state for a vnode with an associated ubc_info
+ * with an associated memory object control.
+ *
+ * Parameters: vp The vnode backing the page
+ * f_offset A file offset interior to the page
+ * ops The operations to perform, as a bitmap
+ * (see below for more information)
+ * phys_entryp The address of a ppnum_t; may be NULL
+ * to ignore
+ * flagsp A pointer to an int to contain flags;
+ * may be NULL to ignore
+ *
+ * Returns: KERN_SUCCESS Success
+ * KERN_INVALID_ARGUMENT If the memory object control has no VM
+ * object associated
+ * KERN_INVALID_OBJECT If UPL_POP_PHYSICAL and the object is
+ * not physically contiguous
+ * KERN_INVALID_OBJECT If !UPL_POP_PHYSICAL and the object is
+ * physically contiguous
+ * KERN_FAILURE If the page cannot be looked up
+ *
+ * Implicit Returns:
+ * *phys_entryp (modified) If phys_entryp is non-NULL and
+ * UPL_POP_PHYSICAL
+ * *flagsp (modified) If flagsp is non-NULL and there was
+ * !UPL_POP_PHYSICAL and a KERN_SUCCESS
+ *
+ * Notes: For object boundaries, it is considerably more efficient to
+ * ensure that f_offset is in fact on a page boundary, as this
+ * will avoid internal use of the hash table to identify the
+ * page, and would therefore skip a number of early optimizations.
+ * Since this is a page operation anyway, the caller should try
+ * to pass only a page aligned offset because of this.
+ *
+ * *flagsp may be modified even if this function fails. If it is
+ * modified, it will contain the condition of the page before the
+ * requested operation was attempted; these will only include the
+ * bitmap flags, and not the PL_POP_PHYSICAL, UPL_POP_DUMP,
+ * UPL_POP_SET, or UPL_POP_CLR bits.
+ *
+ * The flags field may contain a specific operation, such as
+ * UPL_POP_PHYSICAL or UPL_POP_DUMP:
+ *
+ * o UPL_POP_PHYSICAL Fail if not contiguous; if
+ * *phys_entryp and successful, set
+ * *phys_entryp
+ * o UPL_POP_DUMP Dump the specified page
+ *
+ * Otherwise, it is treated as a bitmap of one or more page
+ * operations to perform on the final memory object; allowable
+ * bit values are:
+ *
+ * o UPL_POP_DIRTY The page is dirty
+ * o UPL_POP_PAGEOUT The page is paged out
+ * o UPL_POP_PRECIOUS The page is precious
+ * o UPL_POP_ABSENT The page is absent
+ * o UPL_POP_BUSY The page is busy
+ *
+ * If the page status is only being queried and not modified, then
+ * not other bits should be specified. However, if it is being
+ * modified, exactly ONE of the following bits should be set:
+ *
+ * o UPL_POP_SET Set the current bitmap bits
+ * o UPL_POP_CLR Clear the current bitmap bits
+ *
+ * Thus to effect a combination of setting an clearing, it may be
+ * necessary to call this function twice. If this is done, the
+ * set should be used before the clear, since clearing may trigger
+ * a wakeup on the destination page, and if the page is backed by
+ * an encrypted swap file, setting will trigger the decryption
+ * needed before the wakeup occurs.
+ */
+kern_return_t
+ubc_page_op(
+ struct vnode *vp,
+ off_t f_offset,
+ int ops,
+ ppnum_t *phys_entryp,
+ int *flagsp)
+{
+ memory_object_control_t control;
+
+ control = ubc_getobject(vp, UBC_FLAGS_NONE);
+ if (control == MEMORY_OBJECT_CONTROL_NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ return memory_object_page_op(control,
+ (memory_object_offset_t)f_offset,
+ ops,
+ phys_entryp,
+ flagsp);
+}
+
+
+/*
+ * ubc_range_op
+ *
+ * Manipulate page state for a range of memory for a vnode with an associated
+ * ubc_info with an associated memory object control, when page level state is
+ * not required to be returned from the call (i.e. there are no phys_entryp or
+ * flagsp parameters to this call, and it takes a range which may contain
+ * multiple pages, rather than an offset interior to a single page).
+ *
+ * Parameters: vp The vnode backing the page
+ * f_offset_beg A file offset interior to the start page
+ * f_offset_end A file offset interior to the end page
+ * ops The operations to perform, as a bitmap
+ * (see below for more information)
+ * range The address of an int; may be NULL to
+ * ignore
+ *
+ * Returns: KERN_SUCCESS Success
+ * KERN_INVALID_ARGUMENT If the memory object control has no VM
+ * object associated
+ * KERN_INVALID_OBJECT If the object is physically contiguous
+ *
+ * Implicit Returns:
+ * *range (modified) If range is non-NULL, its contents will
+ * be modified to contain the number of
+ * bytes successfully operated upon.
+ *
+ * Notes: IMPORTANT: This function cannot be used on a range that
+ * consists of physically contiguous pages.
+ *
+ * For object boundaries, it is considerably more efficient to
+ * ensure that f_offset_beg and f_offset_end are in fact on page
+ * boundaries, as this will avoid internal use of the hash table
+ * to identify the page, and would therefore skip a number of
+ * early optimizations. Since this is an operation on a set of
+ * pages anyway, the caller should try to pass only a page aligned
+ * offsets because of this.
+ *
+ * *range will be modified only if this function succeeds.
+ *
+ * The flags field MUST contain a specific operation; allowable
+ * values are:
+ *
+ * o UPL_ROP_ABSENT Returns the extent of the range
+ * presented which is absent, starting
+ * with the start address presented
+ *
+ * o UPL_ROP_PRESENT Returns the extent of the range
+ * presented which is present (resident),
+ * starting with the start address
+ * presented
+ * o UPL_ROP_DUMP Dump the pages which are found in the
+ * target object for the target range.
+ *
+ * IMPORTANT: For UPL_ROP_ABSENT and UPL_ROP_PRESENT; if there are
+ * multiple regions in the range, only the first matching region
+ * is returned.
+ */
+kern_return_t
+ubc_range_op(
+ struct vnode *vp,
+ off_t f_offset_beg,
+ off_t f_offset_end,
+ int ops,
+ int *range)
+{
+ memory_object_control_t control;
+
+ control = ubc_getobject(vp, UBC_FLAGS_NONE);
+ if (control == MEMORY_OBJECT_CONTROL_NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ return memory_object_range_op(control,
+ (memory_object_offset_t)f_offset_beg,
+ (memory_object_offset_t)f_offset_end,
+ ops,
+ range);
+}
+
+
+/*
+ * ubc_create_upl
+ *
+ * Given a vnode, cause the population of a portion of the vm_object; based on
+ * the nature of the request, the pages returned may contain valid data, or
+ * they may be uninitialized.
+ *
+ * Parameters: vp The vnode from which to create the upl
+ * f_offset The start offset into the backing store
+ * represented by the vnode
+ * bufsize The size of the upl to create
+ * uplp Pointer to the upl_t to receive the
+ * created upl; MUST NOT be NULL
+ * plp Pointer to receive the internal page
+ * list for the created upl; MAY be NULL
+ * to ignore
+ *
+ * Returns: KERN_SUCCESS The requested upl has been created
+ * KERN_INVALID_ARGUMENT The bufsize argument is not an even
+ * multiple of the page size
+ * KERN_INVALID_ARGUMENT There is no ubc_info associated with
+ * the vnode, or there is no memory object
+ * control associated with the ubc_info
+ * memory_object_upl_request:KERN_INVALID_VALUE
+ * The supplied upl_flags argument is
+ * invalid
+ * Implicit Returns:
+ * *uplp (modified)
+ * *plp (modified) If non-NULL, the value of *plp will be
+ * modified to point to the internal page
+ * list; this modification may occur even
+ * if this function is unsuccessful, in
+ * which case the contents may be invalid
+ *
+ * Note: If successful, the returned *uplp MUST subsequently be freed
+ * via a call to ubc_upl_commit(), ubc_upl_commit_range(),
+ * ubc_upl_abort(), or ubc_upl_abort_range().
+ */
+kern_return_t
+ubc_create_upl_external(
+ struct vnode *vp,
+ off_t f_offset,
+ int bufsize,
+ upl_t *uplp,
+ upl_page_info_t **plp,
+ int uplflags)
+{
+ return ubc_create_upl_kernel(vp, f_offset, bufsize, uplp, plp, uplflags, vm_tag_bt());
+}
+
+kern_return_t
+ubc_create_upl_kernel(
+ struct vnode *vp,
+ off_t f_offset,
+ int bufsize,
+ upl_t *uplp,
+ upl_page_info_t **plp,
+ int uplflags,
+ vm_tag_t tag)
+{
+ memory_object_control_t control;
+ kern_return_t kr;
+
+ if (plp != NULL) {
+ *plp = NULL;
+ }
+ *uplp = NULL;
+
+ if (bufsize & 0xfff) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ if (bufsize > MAX_UPL_SIZE_BYTES) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ if (uplflags & (UPL_UBC_MSYNC | UPL_UBC_PAGEOUT | UPL_UBC_PAGEIN)) {
+ if (uplflags & UPL_UBC_MSYNC) {
+ uplflags &= UPL_RET_ONLY_DIRTY;
+
+ uplflags |= UPL_COPYOUT_FROM | UPL_CLEAN_IN_PLACE |
+ UPL_SET_INTERNAL | UPL_SET_LITE;
+ } else if (uplflags & UPL_UBC_PAGEOUT) {
+ uplflags &= UPL_RET_ONLY_DIRTY;
+
+ if (uplflags & UPL_RET_ONLY_DIRTY) {
+ uplflags |= UPL_NOBLOCK;
+ }
+
+ uplflags |= UPL_FOR_PAGEOUT | UPL_CLEAN_IN_PLACE |
+ UPL_COPYOUT_FROM | UPL_SET_INTERNAL | UPL_SET_LITE;
+ } else {
+ uplflags |= UPL_RET_ONLY_ABSENT |
+ UPL_NO_SYNC | UPL_CLEAN_IN_PLACE |
+ UPL_SET_INTERNAL | UPL_SET_LITE;
+
+ /*
+ * if the requested size == PAGE_SIZE, we don't want to set
+ * the UPL_NOBLOCK since we may be trying to recover from a
+ * previous partial pagein I/O that occurred because we were low
+ * on memory and bailed early in order to honor the UPL_NOBLOCK...
+ * since we're only asking for a single page, we can block w/o fear
+ * of tying up pages while waiting for more to become available
+ */
+ if (bufsize > PAGE_SIZE) {
+ uplflags |= UPL_NOBLOCK;
+ }
+ }
+ } else {
+ uplflags &= ~UPL_FOR_PAGEOUT;
+
+ if (uplflags & UPL_WILL_BE_DUMPED) {
+ uplflags &= ~UPL_WILL_BE_DUMPED;
+ uplflags |= (UPL_NO_SYNC | UPL_SET_INTERNAL);
+ } else {
+ uplflags |= (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL);
+ }
+ }
+ control = ubc_getobject(vp, UBC_FLAGS_NONE);
+ if (control == MEMORY_OBJECT_CONTROL_NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ kr = memory_object_upl_request(control, f_offset, bufsize, uplp, NULL, NULL, uplflags, tag);
+ if (kr == KERN_SUCCESS && plp != NULL) {
+ *plp = UPL_GET_INTERNAL_PAGE_LIST(*uplp);
+ }
+ return kr;
+}
+
+
+/*
+ * ubc_upl_maxbufsize
+ *
+ * Return the maximum bufsize ubc_create_upl( ) will take.
+ *
+ * Parameters: none
+ *
+ * Returns: maximum size buffer (in bytes) ubc_create_upl( ) will take.
+ */
+upl_size_t
+ubc_upl_maxbufsize(
+ void)
+{
+ return MAX_UPL_SIZE_BYTES;
+}
+
+/*
+ * ubc_upl_map
+ *
+ * Map the page list assocated with the supplied upl into the kernel virtual
+ * address space at the virtual address indicated by the dst_addr argument;
+ * the entire upl is mapped
+ *
+ * Parameters: upl The upl to map
+ * dst_addr The address at which to map the upl
+ *
+ * Returns: KERN_SUCCESS The upl has been mapped
+ * KERN_INVALID_ARGUMENT The upl is UPL_NULL
+ * KERN_FAILURE The upl is already mapped
+ * vm_map_enter:KERN_INVALID_ARGUMENT
+ * A failure code from vm_map_enter() due
+ * to an invalid argument
+ */
+kern_return_t
+ubc_upl_map(
+ upl_t upl,
+ vm_offset_t *dst_addr)
+{
+ return vm_upl_map(kernel_map, upl, dst_addr);
+}
+
+
+/*
+ * ubc_upl_unmap
+ *
+ * Unmap the page list assocated with the supplied upl from the kernel virtual
+ * address space; the entire upl is unmapped.
+ *
+ * Parameters: upl The upl to unmap
+ *
+ * Returns: KERN_SUCCESS The upl has been unmapped
+ * KERN_FAILURE The upl is not currently mapped
+ * KERN_INVALID_ARGUMENT If the upl is UPL_NULL
+ */
+kern_return_t
+ubc_upl_unmap(
+ upl_t upl)
+{
+ return vm_upl_unmap(kernel_map, upl);
+}
+
+
+/*
+ * ubc_upl_commit
+ *
+ * Commit the contents of the upl to the backing store
+ *
+ * Parameters: upl The upl to commit
+ *
+ * Returns: KERN_SUCCESS The upl has been committed
+ * KERN_INVALID_ARGUMENT The supplied upl was UPL_NULL
+ * KERN_FAILURE The supplied upl does not represent
+ * device memory, and the offset plus the
+ * size would exceed the actual size of
+ * the upl
+ *
+ * Notes: In practice, the only return value for this function should be
+ * KERN_SUCCESS, unless there has been data structure corruption;
+ * since the upl is deallocated regardless of success or failure,
+ * there's really nothing to do about this other than panic.
+ *
+ * IMPORTANT: Use of this function should not be mixed with use of
+ * ubc_upl_commit_range(), due to the unconditional deallocation
+ * by this function.
+ */
+kern_return_t
+ubc_upl_commit(
+ upl_t upl)
+{
+ upl_page_info_t *pl;
+ kern_return_t kr;
+
+ pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
+ kr = upl_commit(upl, pl, MAX_UPL_SIZE_BYTES >> PAGE_SHIFT);
+ upl_deallocate(upl);
+ return kr;
+}
+
+
+/*
+ * ubc_upl_commit
+ *
+ * Commit the contents of the specified range of the upl to the backing store
+ *
+ * Parameters: upl The upl to commit
+ * offset The offset into the upl
+ * size The size of the region to be committed,
+ * starting at the specified offset
+ * flags commit type (see below)
+ *
+ * Returns: KERN_SUCCESS The range has been committed
+ * KERN_INVALID_ARGUMENT The supplied upl was UPL_NULL
+ * KERN_FAILURE The supplied upl does not represent
+ * device memory, and the offset plus the
+ * size would exceed the actual size of
+ * the upl
+ *
+ * Notes: IMPORTANT: If the commit is successful, and the object is now
+ * empty, the upl will be deallocated. Since the caller cannot
+ * check that this is the case, the UPL_COMMIT_FREE_ON_EMPTY flag
+ * should generally only be used when the offset is 0 and the size
+ * is equal to the upl size.
+ *
+ * The flags argument is a bitmap of flags on the rage of pages in
+ * the upl to be committed; allowable flags are:
+ *
+ * o UPL_COMMIT_FREE_ON_EMPTY Free the upl when it is
+ * both empty and has been
+ * successfully committed
+ * o UPL_COMMIT_CLEAR_DIRTY Clear each pages dirty
+ * bit; will prevent a
+ * later pageout
+ * o UPL_COMMIT_SET_DIRTY Set each pages dirty
+ * bit; will cause a later
+ * pageout
+ * o UPL_COMMIT_INACTIVATE Clear each pages
+ * reference bit; the page
+ * will not be accessed
+ * o UPL_COMMIT_ALLOW_ACCESS Unbusy each page; pages
+ * become busy when an
+ * IOMemoryDescriptor is
+ * mapped or redirected,
+ * and we have to wait for
+ * an IOKit driver
+ *
+ * The flag UPL_COMMIT_NOTIFY_EMPTY is used internally, and should
+ * not be specified by the caller.
+ *
+ * The UPL_COMMIT_CLEAR_DIRTY and UPL_COMMIT_SET_DIRTY flags are
+ * mutually exclusive, and should not be combined.
+ */
+kern_return_t
+ubc_upl_commit_range(
+ upl_t upl,
+ upl_offset_t offset,
+ upl_size_t size,
+ int flags)
+{
+ upl_page_info_t *pl;
+ boolean_t empty;
+ kern_return_t kr;
+
+ if (flags & UPL_COMMIT_FREE_ON_EMPTY) {
+ flags |= UPL_COMMIT_NOTIFY_EMPTY;
+ }
+
+ if (flags & UPL_COMMIT_KERNEL_ONLY_FLAGS) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
+
+ kr = upl_commit_range(upl, offset, size, flags,
+ pl, MAX_UPL_SIZE_BYTES >> PAGE_SHIFT, &empty);
+
+ if ((flags & UPL_COMMIT_FREE_ON_EMPTY) && empty) {
+ upl_deallocate(upl);
+ }
+
+ return kr;
+}
+
+
+/*
+ * ubc_upl_abort_range
+ *
+ * Abort the contents of the specified range of the specified upl
+ *
+ * Parameters: upl The upl to abort
+ * offset The offset into the upl
+ * size The size of the region to be aborted,
+ * starting at the specified offset
+ * abort_flags abort type (see below)
+ *
+ * Returns: KERN_SUCCESS The range has been aborted
+ * KERN_INVALID_ARGUMENT The supplied upl was UPL_NULL
+ * KERN_FAILURE The supplied upl does not represent
+ * device memory, and the offset plus the
+ * size would exceed the actual size of
+ * the upl
+ *
+ * Notes: IMPORTANT: If the abort is successful, and the object is now
+ * empty, the upl will be deallocated. Since the caller cannot
+ * check that this is the case, the UPL_ABORT_FREE_ON_EMPTY flag
+ * should generally only be used when the offset is 0 and the size
+ * is equal to the upl size.
+ *
+ * The abort_flags argument is a bitmap of flags on the range of
+ * pages in the upl to be aborted; allowable flags are:
+ *
+ * o UPL_ABORT_FREE_ON_EMPTY Free the upl when it is both
+ * empty and has been successfully
+ * aborted
+ * o UPL_ABORT_RESTART The operation must be restarted
+ * o UPL_ABORT_UNAVAILABLE The pages are unavailable
+ * o UPL_ABORT_ERROR An I/O error occurred
+ * o UPL_ABORT_DUMP_PAGES Just free the pages
+ * o UPL_ABORT_NOTIFY_EMPTY RESERVED
+ * o UPL_ABORT_ALLOW_ACCESS RESERVED
+ *
+ * The UPL_ABORT_NOTIFY_EMPTY is an internal use flag and should
+ * not be specified by the caller. It is intended to fulfill the
+ * same role as UPL_COMMIT_NOTIFY_EMPTY does in the function
+ * ubc_upl_commit_range(), but is never referenced internally.
+ *
+ * The UPL_ABORT_ALLOW_ACCESS is defined, but neither set nor
+ * referenced; do not use it.
+ */
+kern_return_t
+ubc_upl_abort_range(
+ upl_t upl,
+ upl_offset_t offset,
+ upl_size_t size,
+ int abort_flags)
+{
+ kern_return_t kr;
+ boolean_t empty = FALSE;
+
+ if (abort_flags & UPL_ABORT_FREE_ON_EMPTY) {
+ abort_flags |= UPL_ABORT_NOTIFY_EMPTY;
+ }
+
+ kr = upl_abort_range(upl, offset, size, abort_flags, &empty);
+
+ if ((abort_flags & UPL_ABORT_FREE_ON_EMPTY) && empty) {
+ upl_deallocate(upl);
+ }
+
+ return kr;
+}
+
+
+/*
+ * ubc_upl_abort
+ *
+ * Abort the contents of the specified upl
+ *
+ * Parameters: upl The upl to abort
+ * abort_type abort type (see below)
+ *
+ * Returns: KERN_SUCCESS The range has been aborted
+ * KERN_INVALID_ARGUMENT The supplied upl was UPL_NULL
+ * KERN_FAILURE The supplied upl does not represent
+ * device memory, and the offset plus the
+ * size would exceed the actual size of
+ * the upl
+ *
+ * Notes: IMPORTANT: If the abort is successful, and the object is now
+ * empty, the upl will be deallocated. Since the caller cannot
+ * check that this is the case, the UPL_ABORT_FREE_ON_EMPTY flag
+ * should generally only be used when the offset is 0 and the size
+ * is equal to the upl size.
+ *
+ * The abort_type is a bitmap of flags on the range of
+ * pages in the upl to be aborted; allowable flags are:
+ *
+ * o UPL_ABORT_FREE_ON_EMPTY Free the upl when it is both
+ * empty and has been successfully
+ * aborted
+ * o UPL_ABORT_RESTART The operation must be restarted
+ * o UPL_ABORT_UNAVAILABLE The pages are unavailable
+ * o UPL_ABORT_ERROR An I/O error occurred
+ * o UPL_ABORT_DUMP_PAGES Just free the pages
+ * o UPL_ABORT_NOTIFY_EMPTY RESERVED
+ * o UPL_ABORT_ALLOW_ACCESS RESERVED
+ *
+ * The UPL_ABORT_NOTIFY_EMPTY is an internal use flag and should
+ * not be specified by the caller. It is intended to fulfill the
+ * same role as UPL_COMMIT_NOTIFY_EMPTY does in the function
+ * ubc_upl_commit_range(), but is never referenced internally.
+ *
+ * The UPL_ABORT_ALLOW_ACCESS is defined, but neither set nor
+ * referenced; do not use it.
+ */
+kern_return_t
+ubc_upl_abort(
+ upl_t upl,
+ int abort_type)
+{
+ kern_return_t kr;
+
+ kr = upl_abort(upl, abort_type);
+ upl_deallocate(upl);
+ return kr;
+}
+
+
+/*
+ * ubc_upl_pageinfo
+ *
+ * Retrieve the internal page list for the specified upl
+ *
+ * Parameters: upl The upl to obtain the page list from
+ *
+ * Returns: !NULL The (upl_page_info_t *) for the page
+ * list internal to the upl
+ * NULL Error/no page list associated
+ *
+ * Notes: IMPORTANT: The function is only valid on internal objects
+ * where the list request was made with the UPL_INTERNAL flag.
+ *
+ * This function is a utility helper function, since some callers
+ * may not have direct access to the header defining the macro,
+ * due to abstraction layering constraints.
+ */
+upl_page_info_t *
+ubc_upl_pageinfo(
+ upl_t upl)
+{
+ return UPL_GET_INTERNAL_PAGE_LIST(upl);
+}
+
+
+int
+UBCINFOEXISTS(const struct vnode * vp)
+{
+ return (vp) && ((vp)->v_type == VREG) && ((vp)->v_ubcinfo != UBC_INFO_NULL);
+}
+
+
+void
+ubc_upl_range_needed(
+ upl_t upl,
+ int index,
+ int count)
+{
+ upl_range_needed(upl, index, count);
+}
+
+boolean_t
+ubc_is_mapped(const struct vnode *vp, boolean_t *writable)
+{
+ if (!UBCINFOEXISTS(vp) || !ISSET(vp->v_ubcinfo->ui_flags, UI_ISMAPPED)) {
+ return FALSE;
+ }
+ if (writable) {
+ *writable = ISSET(vp->v_ubcinfo->ui_flags, UI_MAPPEDWRITE);
+ }
+ return TRUE;
+}
+
+boolean_t
+ubc_is_mapped_writable(const struct vnode *vp)
+{
+ boolean_t writable;
+ return ubc_is_mapped(vp, &writable) && writable;
+}
+
+
+/*
+ * CODE SIGNING
+ */
+static atomic_size_t cs_blob_size = 0;
+static atomic_uint_fast32_t cs_blob_count = 0;
+static atomic_size_t cs_blob_size_peak = 0;
+static atomic_size_t cs_blob_size_max = 0;
+static atomic_uint_fast32_t cs_blob_count_peak = 0;
+
+SYSCTL_UINT(_vm, OID_AUTO, cs_blob_count, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_blob_count, 0, "Current number of code signature blobs");
+SYSCTL_ULONG(_vm, OID_AUTO, cs_blob_size, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_blob_size, "Current size of all code signature blobs");
+SYSCTL_UINT(_vm, OID_AUTO, cs_blob_count_peak, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_blob_count_peak, 0, "Peak number of code signature blobs");
+SYSCTL_ULONG(_vm, OID_AUTO, cs_blob_size_peak, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_blob_size_peak, "Peak size of code signature blobs");
+SYSCTL_ULONG(_vm, OID_AUTO, cs_blob_size_max, CTLFLAG_RD | CTLFLAG_LOCKED, &cs_blob_size_max, "Size of biggest code signature blob");
+
+/*
+ * Function: csblob_parse_teamid
+ *
+ * Description: This function returns a pointer to the team id
+ * stored within the codedirectory of the csblob.
+ * If the codedirectory predates team-ids, it returns
+ * NULL.
+ * This does not copy the name but returns a pointer to
+ * it within the CD. Subsequently, the CD must be
+ * available when this is used.
+ */
+
+static const char *
+csblob_parse_teamid(struct cs_blob *csblob)
+{
+ const CS_CodeDirectory *cd;
+
+ cd = csblob->csb_cd;
+
+ if (ntohl(cd->version) < CS_SUPPORTSTEAMID) {
+ return NULL;
+ }
+
+ if (cd->teamOffset == 0) {
+ return NULL;
+ }
+
+ const char *name = ((const char *)cd) + ntohl(cd->teamOffset);
+ if (cs_debug > 1) {
+ printf("found team-id %s in cdblob\n", name);
+ }
+
+ return name;
+}
+
+
+kern_return_t
+ubc_cs_blob_allocate(
+ vm_offset_t *blob_addr_p,
+ vm_size_t *blob_size_p)
+{
+ kern_return_t kr = KERN_FAILURE;
+
+ {
+ *blob_addr_p = (vm_offset_t) kalloc_tag(*blob_size_p, VM_KERN_MEMORY_SECURITY);
+
+ if (*blob_addr_p == 0) {
+ kr = KERN_NO_SPACE;
+ } else {
+ kr = KERN_SUCCESS;
+ }
+ }
+
+ return kr;
+}
+
+void
+ubc_cs_blob_deallocate(
+ vm_offset_t blob_addr,
+ vm_size_t blob_size)
+{
+ {
+ kfree(blob_addr, blob_size);
+ }
+}
+
+/*
+ * Some codesigned files use a lowest common denominator page size of
+ * 4KiB, but can be used on systems that have a runtime page size of
+ * 16KiB. Since faults will only occur on 16KiB ranges in
+ * cs_validate_range(), we can convert the original Code Directory to
+ * a multi-level scheme where groups of 4 hashes are combined to form
+ * a new hash, which represents 16KiB in the on-disk file. This can
+ * reduce the wired memory requirement for the Code Directory by
+ * 75%. Care must be taken for binaries that use the "fourk" VM pager
+ * for unaligned access, which may still attempt to validate on
+ * non-16KiB multiples for compatibility with 3rd party binaries.
+ */
+static boolean_t
+ubc_cs_supports_multilevel_hash(struct cs_blob *blob __unused)
+{
+ const CS_CodeDirectory *cd;
+
+
+ /*
+ * Only applies to binaries that ship as part of the OS,
+ * primarily the shared cache.
+ */
+ if (!blob->csb_platform_binary || blob->csb_teamid != NULL) {
+ return FALSE;
+ }
+
+ /*
+ * If the runtime page size matches the code signing page
+ * size, there is no work to do.
+ */
+ if (PAGE_SHIFT <= blob->csb_hash_pageshift) {
+ return FALSE;
+ }
+
+ cd = blob->csb_cd;
+
+ /*
+ * There must be a valid integral multiple of hashes
+ */
+ if (ntohl(cd->nCodeSlots) & (PAGE_MASK >> blob->csb_hash_pageshift)) {
+ return FALSE;
+ }
+
+ /*
+ * Scatter lists must also have ranges that have an integral number of hashes
+ */
+ if ((ntohl(cd->version) >= CS_SUPPORTSSCATTER) && (ntohl(cd->scatterOffset))) {
+ const SC_Scatter *scatter = (const SC_Scatter*)
+ ((const char*)cd + ntohl(cd->scatterOffset));
+ /* iterate all scatter structs to make sure they are all aligned */
+ do {
+ uint32_t sbase = ntohl(scatter->base);
+ uint32_t scount = ntohl(scatter->count);
+
+ /* last scatter? */
+ if (scount == 0) {
+ break;
+ }
+
+ if (sbase & (PAGE_MASK >> blob->csb_hash_pageshift)) {
+ return FALSE;
+ }
+
+ if (scount & (PAGE_MASK >> blob->csb_hash_pageshift)) {
+ return FALSE;
+ }
+
+ scatter++;
+ } while (1);
+ }
+
+ /* Covered range must be a multiple of the new page size */
+ if (ntohl(cd->codeLimit) & PAGE_MASK) {
+ return FALSE;
+ }
+
+ /* All checks pass */
+ return TRUE;
+}
+
+/*
+ * Given a cs_blob with an already chosen best code directory, this
+ * function allocates memory and copies into it only the blobs that
+ * will be needed by the kernel, namely the single chosen code
+ * directory (and not any of its alternatives) and the entitlement
+ * blob.
+ *
+ * This saves significant memory with agile signatures, and additional
+ * memory for 3rd Party Code because we also omit the CMS blob.
+ *
+ * To support multilevel and other potential code directory rewriting,
+ * the size of a new code directory can be specified. Since that code
+ * directory will replace the existing code directory,
+ * ubc_cs_reconstitute_code_signature does not copy the original code
+ * directory when a size is given, and the caller must fill it in.
+ */
+static int
+ubc_cs_reconstitute_code_signature(struct cs_blob const *blob, vm_size_t optional_new_cd_size,
+ vm_address_t *new_blob_addr_p, vm_size_t *new_blob_size_p,
+ CS_CodeDirectory **new_cd_p, CS_GenericBlob const **new_entitlements_p)
+{
+ const CS_CodeDirectory *old_cd, *cd;
+ CS_CodeDirectory *new_cd;
+ const CS_GenericBlob *entitlements;
+ vm_offset_t new_blob_addr;
+ vm_size_t new_blob_size;
+ vm_size_t new_cdsize;
+ kern_return_t kr;
+ int error;
+
+ old_cd = blob->csb_cd;
+
+ new_cdsize = optional_new_cd_size != 0 ? optional_new_cd_size : htonl(old_cd->length);
+
+ new_blob_size = sizeof(CS_SuperBlob);
+ new_blob_size += sizeof(CS_BlobIndex);
+ new_blob_size += new_cdsize;
+
+ if (blob->csb_entitlements_blob) {
+ /* We need to add a slot for the entitlements */
+ ptrauth_utils_auth_blob_generic(blob->csb_entitlements_blob,
+ ntohl(blob->csb_entitlements_blob->length),
+ OS_PTRAUTH_DISCRIMINATOR("cs_blob.csb_entitlements_blob_signature"),
+ PTRAUTH_ADDR_DIVERSIFY,
+ blob->csb_entitlements_blob_signature);
+
+ new_blob_size += sizeof(CS_BlobIndex);
+ new_blob_size += ntohl(blob->csb_entitlements_blob->length);
+ }
+
+ kr = ubc_cs_blob_allocate(&new_blob_addr, &new_blob_size);
+ if (kr != KERN_SUCCESS) {
+ if (cs_debug > 1) {
+ printf("CODE SIGNING: Failed to allocate memory for new Code Signing Blob: %d\n",
+ kr);
+ }
+ return ENOMEM;
+ }
+
+ CS_SuperBlob *new_superblob;
+
+ new_superblob = (CS_SuperBlob *)new_blob_addr;
+ new_superblob->magic = htonl(CSMAGIC_EMBEDDED_SIGNATURE);
+ new_superblob->length = htonl((uint32_t)new_blob_size);
+ if (blob->csb_entitlements_blob) {
+ vm_size_t ent_offset, cd_offset;
+
+ cd_offset = sizeof(CS_SuperBlob) + 2 * sizeof(CS_BlobIndex);
+ ent_offset = cd_offset + new_cdsize;
+
+ new_superblob->count = htonl(2);
+ new_superblob->index[0].type = htonl(CSSLOT_CODEDIRECTORY);
+ new_superblob->index[0].offset = htonl((uint32_t)cd_offset);
+ new_superblob->index[1].type = htonl(CSSLOT_ENTITLEMENTS);
+ new_superblob->index[1].offset = htonl((uint32_t)ent_offset);
+
+ ptrauth_utils_auth_blob_generic(blob->csb_entitlements_blob,
+ ntohl(blob->csb_entitlements_blob->length),
+ OS_PTRAUTH_DISCRIMINATOR("cs_blob.csb_entitlements_blob_signature"),
+ PTRAUTH_ADDR_DIVERSIFY,
+ blob->csb_entitlements_blob_signature);
+
+ memcpy((void *)(new_blob_addr + ent_offset), blob->csb_entitlements_blob, ntohl(blob->csb_entitlements_blob->length));
+
+ new_cd = (CS_CodeDirectory *)(new_blob_addr + cd_offset);
+ } else {
+ // Blob is the code directory, directly.
+ new_cd = (CS_CodeDirectory *)new_blob_addr;
+ }
+
+ if (optional_new_cd_size == 0) {
+ // Copy code directory, and revalidate.
+ memcpy(new_cd, old_cd, new_cdsize);
+
+ vm_size_t length = new_blob_size;
+
+ error = cs_validate_csblob((const uint8_t *)new_blob_addr, length, &cd, &entitlements);
+
+ if (error) {
+ printf("CODE SIGNING: Failed to validate new Code Signing Blob: %d\n",
+ error);
+
+ ubc_cs_blob_deallocate(new_blob_addr, new_blob_size);
+ return error;
+ }
+ *new_entitlements_p = entitlements;
+ } else {
+ // Caller will fill out and validate code directory.
+ memset(new_cd, 0, new_cdsize);
+ *new_entitlements_p = NULL;
+ }
+
+ *new_blob_addr_p = new_blob_addr;
+ *new_blob_size_p = new_blob_size;
+ *new_cd_p = new_cd;
+
+ return 0;
+}
+
+static int
+ubc_cs_convert_to_multilevel_hash(struct cs_blob *blob)
+{
+ const CS_CodeDirectory *old_cd, *cd;
+ CS_CodeDirectory *new_cd;
+ const CS_GenericBlob *entitlements;
+ vm_offset_t new_blob_addr;
+ vm_size_t new_blob_size;
+ vm_size_t new_cdsize;
+ int error;
+
+ uint32_t hashes_per_new_hash_shift = (uint32_t)(PAGE_SHIFT - blob->csb_hash_pageshift);
+
+ if (cs_debug > 1) {
+ printf("CODE SIGNING: Attempting to convert Code Directory for %lu -> %lu page shift\n",
+ (unsigned long)blob->csb_hash_pageshift, (unsigned long)PAGE_SHIFT);
+ }
+
+ old_cd = blob->csb_cd;
+
+ /* Up to the hashes, we can copy all data */
+ new_cdsize = ntohl(old_cd->hashOffset);
+ new_cdsize += (ntohl(old_cd->nCodeSlots) >> hashes_per_new_hash_shift) * old_cd->hashSize;
+
+ error = ubc_cs_reconstitute_code_signature(blob, new_cdsize,
+ &new_blob_addr, &new_blob_size, &new_cd,
+ &entitlements);
+ if (error != 0) {
+ printf("CODE SIGNING: Failed to reconsitute code signature: %d\n", error);
+ return error;
+ }
+
+ memcpy(new_cd, old_cd, ntohl(old_cd->hashOffset));
+
+ /* Update fields in the Code Directory structure */
+ new_cd->length = htonl((uint32_t)new_cdsize);
+
+ uint32_t nCodeSlots = ntohl(new_cd->nCodeSlots);
+ nCodeSlots >>= hashes_per_new_hash_shift;
+ new_cd->nCodeSlots = htonl(nCodeSlots);
+
+ new_cd->pageSize = (uint8_t)PAGE_SHIFT; /* Not byte-swapped */
+
+ if ((ntohl(new_cd->version) >= CS_SUPPORTSSCATTER) && (ntohl(new_cd->scatterOffset))) {
+ SC_Scatter *scatter = (SC_Scatter*)
+ ((char *)new_cd + ntohl(new_cd->scatterOffset));
+ /* iterate all scatter structs to scale their counts */
+ do {
+ uint32_t scount = ntohl(scatter->count);
+ uint32_t sbase = ntohl(scatter->base);
+
+ /* last scatter? */
+ if (scount == 0) {
+ break;
+ }
+
+ scount >>= hashes_per_new_hash_shift;
+ scatter->count = htonl(scount);
+
+ sbase >>= hashes_per_new_hash_shift;
+ scatter->base = htonl(sbase);
+
+ scatter++;
+ } while (1);
+ }
+
+ /* For each group of hashes, hash them together */
+ const unsigned char *src_base = (const unsigned char *)old_cd + ntohl(old_cd->hashOffset);
+ unsigned char *dst_base = (unsigned char *)new_cd + ntohl(new_cd->hashOffset);
+
+ uint32_t hash_index;
+ for (hash_index = 0; hash_index < nCodeSlots; hash_index++) {
+ union cs_hash_union mdctx;
+
+ uint32_t source_hash_len = old_cd->hashSize << hashes_per_new_hash_shift;
+ const unsigned char *src = src_base + hash_index * source_hash_len;
+ unsigned char *dst = dst_base + hash_index * new_cd->hashSize;
+
+ blob->csb_hashtype->cs_init(&mdctx);
+ blob->csb_hashtype->cs_update(&mdctx, src, source_hash_len);
+ blob->csb_hashtype->cs_final(dst, &mdctx);
+ }
+
+ error = cs_validate_csblob((const uint8_t *)new_blob_addr, new_blob_size, &cd, &entitlements);
+ if (error != 0) {
+ printf("CODE SIGNING: Failed to validate new Code Signing Blob: %d\n",
+ error);
+
+ ubc_cs_blob_deallocate(new_blob_addr, new_blob_size);
+ return error;
+ }
+
+ /* New Code Directory is ready for use, swap it out in the blob structure */
+ ubc_cs_blob_deallocate((vm_offset_t)blob->csb_mem_kaddr, blob->csb_mem_size);
+
+ blob->csb_mem_size = new_blob_size;
+ blob->csb_mem_kaddr = (void *)new_blob_addr;
+ blob->csb_cd = cd;
+ blob->csb_entitlements_blob = entitlements;
+ if (blob->csb_entitlements_blob != NULL) {
+ blob->csb_entitlements_blob_signature = ptrauth_utils_sign_blob_generic(blob->csb_entitlements_blob,
+ ntohl(blob->csb_entitlements_blob->length),
+ OS_PTRAUTH_DISCRIMINATOR("cs_blob.csb_entitlements_blob_signature"),
+ PTRAUTH_ADDR_DIVERSIFY);
+ }
+
+ /* The blob has some cached attributes of the Code Directory, so update those */
+
+ blob->csb_hash_firstlevel_pageshift = blob->csb_hash_pageshift; /* Save the original page size */
+
+ blob->csb_hash_pageshift = PAGE_SHIFT;
+ blob->csb_end_offset = ntohl(cd->codeLimit);
+ if ((ntohl(cd->version) >= CS_SUPPORTSSCATTER) && (ntohl(cd->scatterOffset))) {
+ const SC_Scatter *scatter = (const SC_Scatter*)
+ ((const char*)cd + ntohl(cd->scatterOffset));
+ blob->csb_start_offset = ((off_t)ntohl(scatter->base)) * PAGE_SIZE;
+ } else {
+ blob->csb_start_offset = 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Validate the code signature blob, create a struct cs_blob wrapper
+ * and return it together with a pointer to the chosen code directory
+ * and entitlements blob.
+ *
+ * Note that this takes ownership of the memory as addr, mainly because
+ * this function can actually replace the passed in blob with another
+ * one, e.g. when performing multilevel hashing optimization.
+ */
+int
+cs_blob_create_validated(
+ vm_address_t * const addr,
+ vm_size_t size,
+ struct cs_blob ** const ret_blob,
+ CS_CodeDirectory const ** const ret_cd)
+{
+ struct cs_blob *blob;
+ int error = EINVAL;
+ const CS_CodeDirectory *cd;
+ const CS_GenericBlob *entitlements;
+ union cs_hash_union mdctx;
+ size_t length;
+
+ if (ret_blob) {
+ *ret_blob = NULL;
+ }
+
+ blob = (struct cs_blob *) kalloc(sizeof(struct cs_blob));
+ if (blob == NULL) {
+ return ENOMEM;
+ }
+
+ /* fill in the new blob */
+ blob->csb_mem_size = size;
+ blob->csb_mem_offset = 0;
+ blob->csb_mem_kaddr = (void *)*addr;
+ blob->csb_flags = 0;
+ blob->csb_signer_type = CS_SIGNER_TYPE_UNKNOWN;
+ blob->csb_platform_binary = 0;
+ blob->csb_platform_path = 0;
+ blob->csb_teamid = NULL;
+#if CONFIG_SUPPLEMENTAL_SIGNATURES
+ blob->csb_supplement_teamid = NULL;
+#endif
+ blob->csb_entitlements_blob = NULL;
+ blob->csb_entitlements = NULL;
+ blob->csb_reconstituted = false;
+
+ /* Transfer ownership. Even on error, this function will deallocate */
+ *addr = 0;
+
+ /*
+ * Validate the blob's contents
+ */
+ length = (size_t) size;
+ error = cs_validate_csblob((const uint8_t *)blob->csb_mem_kaddr,
+ length, &cd, &entitlements);
+ if (error) {
+ if (cs_debug) {
+ printf("CODESIGNING: csblob invalid: %d\n", error);
+ }
+ /*
+ * The vnode checker can't make the rest of this function
+ * succeed if csblob validation failed, so bail */
+ goto out;
+ } else {
+ const unsigned char *md_base;
+ uint8_t hash[CS_HASH_MAX_SIZE];
+ int md_size;
+ vm_offset_t hash_pagemask;
+
+ blob->csb_cd = cd;
+ blob->csb_entitlements_blob = entitlements; /* may be NULL, not yet validated */
+ if (blob->csb_entitlements_blob != NULL) {
+ blob->csb_entitlements_blob_signature = ptrauth_utils_sign_blob_generic(blob->csb_entitlements_blob,
+ ntohl(blob->csb_entitlements_blob->length),
+ OS_PTRAUTH_DISCRIMINATOR("cs_blob.csb_entitlements_blob_signature"),
+ PTRAUTH_ADDR_DIVERSIFY);
+ }
+ blob->csb_hashtype = cs_find_md(cd->hashType);
+ if (blob->csb_hashtype == NULL || blob->csb_hashtype->cs_digest_size > sizeof(hash)) {
+ panic("validated CodeDirectory but unsupported type");
+ }
+
+ blob->csb_hash_pageshift = cd->pageSize;
+ hash_pagemask = (1U << cd->pageSize) - 1;
+ blob->csb_hash_firstlevel_pageshift = 0;
+ blob->csb_flags = (ntohl(cd->flags) & CS_ALLOWED_MACHO) | CS_VALID;
+ blob->csb_end_offset = (((vm_offset_t)ntohl(cd->codeLimit) + hash_pagemask) & ~hash_pagemask);
+ if ((ntohl(cd->version) >= CS_SUPPORTSSCATTER) && (ntohl(cd->scatterOffset))) {
+ const SC_Scatter *scatter = (const SC_Scatter*)
+ ((const char*)cd + ntohl(cd->scatterOffset));
+ blob->csb_start_offset = ((off_t)ntohl(scatter->base)) * (1U << blob->csb_hash_pageshift);
+ } else {
+ blob->csb_start_offset = 0;
+ }
+ /* compute the blob's cdhash */
+ md_base = (const unsigned char *) cd;
+ md_size = ntohl(cd->length);
+
+ blob->csb_hashtype->cs_init(&mdctx);
+ blob->csb_hashtype->cs_update(&mdctx, md_base, md_size);
+ blob->csb_hashtype->cs_final(hash, &mdctx);
+
+ memcpy(blob->csb_cdhash, hash, CS_CDHASH_LEN);
+ blob->csb_cdhash_signature = ptrauth_utils_sign_blob_generic(blob->csb_cdhash,
+ sizeof(blob->csb_cdhash),
+ OS_PTRAUTH_DISCRIMINATOR("cs_blob.csb_cd_signature"),
+ PTRAUTH_ADDR_DIVERSIFY);
+
+#if CONFIG_SUPPLEMENTAL_SIGNATURES
+ blob->csb_linkage_hashtype = NULL;
+ if (ntohl(cd->version) >= CS_SUPPORTSLINKAGE && cd->linkageHashType != 0 &&
+ ntohl(cd->linkageSize) >= CS_CDHASH_LEN) {
+ blob->csb_linkage_hashtype = cs_find_md(cd->linkageHashType);
+
+ if (blob->csb_linkage_hashtype != NULL) {
+ memcpy(blob->csb_linkage, (uint8_t const*)cd + ntohl(cd->linkageOffset),
+ CS_CDHASH_LEN);
+ }
+ }
+#endif
+ }
+
+ error = 0;
+
+out:
+ if (error != 0) {
+ cs_blob_free(blob);
+ blob = NULL;
+ cd = NULL;
+ }
+
+ if (ret_blob != NULL) {
+ *ret_blob = blob;
+ }
+ if (ret_cd != NULL) {
+ *ret_cd = cd;
+ }
+
+ return error;
+}
+
+/*
+ * Free a cs_blob previously created by cs_blob_create_validated.
+ */
+void
+cs_blob_free(
+ struct cs_blob * const blob)
+{
+ if (blob != NULL) {
+ if (blob->csb_mem_kaddr) {
+ ubc_cs_blob_deallocate((vm_offset_t)blob->csb_mem_kaddr, blob->csb_mem_size);
+ blob->csb_mem_kaddr = NULL;
+ }
+ if (blob->csb_entitlements != NULL) {
+ osobject_release(blob->csb_entitlements);
+ blob->csb_entitlements = NULL;
+ }
+ (kfree)(blob, sizeof(*blob));
+ }
+}
+#if CONFIG_SUPPLEMENTAL_SIGNATURES
+static void
+cs_blob_supplement_free(struct cs_blob * const blob)
+{
+ if (blob != NULL) {
+ if (blob->csb_supplement_teamid != NULL) {
+ vm_size_t teamid_size = strlen(blob->csb_supplement_teamid) + 1;
+ kfree(blob->csb_supplement_teamid, teamid_size);
+ blob->csb_supplement_teamid = NULL;
+ }
+ cs_blob_free(blob);
+ }
+}
+#endif
+
+static void
+ubc_cs_blob_adjust_statistics(struct cs_blob const *blob)
+{
+ /* Note that the atomic ops are not enough to guarantee
+ * correctness: If a blob with an intermediate size is inserted
+ * concurrently, we can lose a peak value assignment. But these
+ * statistics are only advisory anyway, so we're not going to
+ * employ full locking here. (Consequently, we are also okay with
+ * relaxed ordering of those accesses.)
+ */
+
+ unsigned int new_cs_blob_count = os_atomic_add(&cs_blob_count, 1, relaxed);
+ if (new_cs_blob_count > os_atomic_load(&cs_blob_count_peak, relaxed)) {
+ os_atomic_store(&cs_blob_count_peak, new_cs_blob_count, relaxed);
+ }
+
+ size_t new_cs_blob_size = os_atomic_add(&cs_blob_size, blob->csb_mem_size, relaxed);
+
+ if (new_cs_blob_size > os_atomic_load(&cs_blob_size_peak, relaxed)) {
+ os_atomic_store(&cs_blob_size_peak, new_cs_blob_size, relaxed);
+ }
+ if (blob->csb_mem_size > os_atomic_load(&cs_blob_size_max, relaxed)) {
+ os_atomic_store(&cs_blob_size_max, blob->csb_mem_size, relaxed);
+ }
+}
+
+int
+ubc_cs_blob_add(
+ struct vnode *vp,
+ uint32_t platform,
+ cpu_type_t cputype,
+ cpu_subtype_t cpusubtype,
+ off_t base_offset,
+ vm_address_t *addr,
+ vm_size_t size,
+ struct image_params *imgp,
+ __unused int flags,
+ struct cs_blob **ret_blob)
+{
+ kern_return_t kr;
+ struct ubc_info *uip;
+ struct cs_blob *blob = NULL, *oblob = NULL;
+ int error;
+ CS_CodeDirectory const *cd;
+ off_t blob_start_offset, blob_end_offset;
+ boolean_t record_mtime;
+
+ record_mtime = FALSE;
+ if (ret_blob) {
+ *ret_blob = NULL;
+ }
+
+ /* Create the struct cs_blob wrapper that will be attached to the vnode.
+ * Validates the passed in blob in the process. */
+ error = cs_blob_create_validated(addr, size, &blob, &cd);
+
+ if (error != 0) {
+ printf("malform code signature blob: %d\n", error);
+ return error;
+ }
+
+ blob->csb_cpu_type = cputype;
+ blob->csb_cpu_subtype = cpusubtype & ~CPU_SUBTYPE_MASK;
+ blob->csb_base_offset = base_offset;
+
+ /*
+ * Let policy module check whether the blob's signature is accepted.
+ */
+#if CONFIG_MACF
+ unsigned int cs_flags = blob->csb_flags;
+ unsigned int signer_type = blob->csb_signer_type;
+ error = mac_vnode_check_signature(vp, blob, imgp, &cs_flags, &signer_type, flags, platform);
+ blob->csb_flags = cs_flags;
+ blob->csb_signer_type = signer_type;
+
+ if (error) {
+ if (cs_debug) {
+ printf("check_signature[pid: %d], error = %d\n", current_proc()->p_pid, error);
+ }
+ goto out;
+ }
+ if ((flags & MAC_VNODE_CHECK_DYLD_SIM) && !(blob->csb_flags & CS_PLATFORM_BINARY)) {
+ if (cs_debug) {
+ printf("check_signature[pid: %d], is not apple signed\n", current_proc()->p_pid);
+ }
+ error = EPERM;
+ goto out;
+ }
+#endif
+
+#if CONFIG_ENFORCE_SIGNED_CODE
+ /*
+ * Reconstitute code signature
+ */
+ {
+ vm_address_t new_mem_kaddr = 0;
+ vm_size_t new_mem_size = 0;
+
+ CS_CodeDirectory *new_cd = NULL;
+ CS_GenericBlob const *new_entitlements = NULL;
+
+ error = ubc_cs_reconstitute_code_signature(blob, 0,
+ &new_mem_kaddr, &new_mem_size,
+ &new_cd, &new_entitlements);
+
+ if (error != 0) {
+ printf("failed code signature reconstitution: %d\n", error);
+ goto out;
+ }
+
+ ubc_cs_blob_deallocate((vm_offset_t)blob->csb_mem_kaddr, blob->csb_mem_size);
+
+ blob->csb_mem_kaddr = (void *)new_mem_kaddr;
+ blob->csb_mem_size = new_mem_size;
+ blob->csb_cd = new_cd;
+ blob->csb_entitlements_blob = new_entitlements;
+ if (blob->csb_entitlements_blob != NULL) {
+ blob->csb_entitlements_blob_signature = ptrauth_utils_sign_blob_generic(blob->csb_entitlements_blob,
+ ntohl(blob->csb_entitlements_blob->length),
+ OS_PTRAUTH_DISCRIMINATOR("cs_blob.csb_entitlements_blob_signature"),
+ PTRAUTH_ADDR_DIVERSIFY);
+ }
+ blob->csb_reconstituted = true;
+ }
+#endif
+
+
+ if (blob->csb_flags & CS_PLATFORM_BINARY) {
+ if (cs_debug > 1) {
+ printf("check_signature[pid: %d]: platform binary\n", current_proc()->p_pid);
+ }
+ blob->csb_platform_binary = 1;
+ blob->csb_platform_path = !!(blob->csb_flags & CS_PLATFORM_PATH);
+ } else {
+ blob->csb_platform_binary = 0;
+ blob->csb_platform_path = 0;
+ blob->csb_teamid = csblob_parse_teamid(blob);
+ if (cs_debug > 1) {
+ if (blob->csb_teamid) {
+ printf("check_signature[pid: %d]: team-id is %s\n", current_proc()->p_pid, blob->csb_teamid);
+ } else {
+ printf("check_signature[pid: %d]: no team-id\n", current_proc()->p_pid);
+ }
+ }
+ }
+
+ /*
+ * Validate the blob's coverage
+ */
+ blob_start_offset = blob->csb_base_offset + blob->csb_start_offset;
+ blob_end_offset = blob->csb_base_offset + blob->csb_end_offset;
+
+ if (blob_start_offset >= blob_end_offset ||
+ blob_start_offset < 0 ||
+ blob_end_offset <= 0) {
+ /* reject empty or backwards blob */
+ error = EINVAL;
+ goto out;
+ }
+
+ if (ubc_cs_supports_multilevel_hash(blob)) {
+ error = ubc_cs_convert_to_multilevel_hash(blob);
+ if (error != 0) {
+ printf("failed multilevel hash conversion: %d\n", error);
+ goto out;
+ }
+ blob->csb_reconstituted = true;
+ }
+
+ vnode_lock(vp);
+ if (!UBCINFOEXISTS(vp)) {
+ vnode_unlock(vp);
+ error = ENOENT;
+ goto out;
+ }
+ uip = vp->v_ubcinfo;
+
+ /* check if this new blob overlaps with an existing blob */
+ for (oblob = uip->cs_blobs;
+ oblob != NULL;
+ oblob = oblob->csb_next) {
+ off_t oblob_start_offset, oblob_end_offset;
+
+ if (blob->csb_signer_type != oblob->csb_signer_type) { // signer type needs to be the same for slices
+ vnode_unlock(vp);
+ error = EALREADY;
+ goto out;
+ } else if (blob->csb_platform_binary) { //platform binary needs to be the same for app slices
+ if (!oblob->csb_platform_binary) {
+ vnode_unlock(vp);
+ error = EALREADY;
+ goto out;
+ }
+ } else if (blob->csb_teamid) { //teamid binary needs to be the same for app slices
+ if (oblob->csb_platform_binary ||
+ oblob->csb_teamid == NULL ||
+ strcmp(oblob->csb_teamid, blob->csb_teamid) != 0) {
+ vnode_unlock(vp);
+ error = EALREADY;
+ goto out;
+ }
+ } else { // non teamid binary needs to be the same for app slices
+ if (oblob->csb_platform_binary ||
+ oblob->csb_teamid != NULL) {
+ vnode_unlock(vp);
+ error = EALREADY;
+ goto out;
+ }
+ }
+
+ oblob_start_offset = (oblob->csb_base_offset +
+ oblob->csb_start_offset);
+ oblob_end_offset = (oblob->csb_base_offset +
+ oblob->csb_end_offset);
+ if (blob_start_offset >= oblob_end_offset ||
+ blob_end_offset <= oblob_start_offset) {
+ /* no conflict with this existing blob */
+ } else {
+ /* conflict ! */
+ if (blob_start_offset == oblob_start_offset &&
+ blob_end_offset == oblob_end_offset &&
+ blob->csb_mem_size == oblob->csb_mem_size &&
+ blob->csb_flags == oblob->csb_flags &&
+ (blob->csb_cpu_type == CPU_TYPE_ANY ||
+ oblob->csb_cpu_type == CPU_TYPE_ANY ||
+ blob->csb_cpu_type == oblob->csb_cpu_type) &&
+ !bcmp(blob->csb_cdhash,
+ oblob->csb_cdhash,
+ CS_CDHASH_LEN)) {
+ /*
+ * We already have this blob:
+ * we'll return success but
+ * throw away the new blob.
+ */
+ if (oblob->csb_cpu_type == CPU_TYPE_ANY) {
+ /*
+ * The old blob matches this one
+ * but doesn't have any CPU type.
+ * Update it with whatever the caller
+ * provided this time.
+ */
+ oblob->csb_cpu_type = cputype;
+ }
+
+ /* The signature is still accepted, so update the
+ * generation count. */
+ uip->cs_add_gen = cs_blob_generation_count;
+
+ vnode_unlock(vp);
+ if (ret_blob) {
+ *ret_blob = oblob;
+ }
+ error = EAGAIN;
+ goto out;
+ } else {
+ /* different blob: reject the new one */
+ vnode_unlock(vp);
+ error = EALREADY;
+ goto out;
+ }
+ }
+ }
+
+
+ /* mark this vnode's VM object as having "signed pages" */
+ kr = memory_object_signed(uip->ui_control, TRUE);
+ if (kr != KERN_SUCCESS) {
+ vnode_unlock(vp);
+ error = ENOENT;
+ goto out;
+ }
+
+ if (uip->cs_blobs == NULL) {
+ /* loading 1st blob: record the file's current "modify time" */
+ record_mtime = TRUE;
+ }
+
+ /* set the generation count for cs_blobs */
+ uip->cs_add_gen = cs_blob_generation_count;
+
+ /*
+ * Add this blob to the list of blobs for this vnode.
+ * We always add at the front of the list and we never remove a
+ * blob from the list, so ubc_cs_get_blobs() can return whatever
+ * the top of the list was and that list will remain valid
+ * while we validate a page, even after we release the vnode's lock.
+ */
+ blob->csb_next = uip->cs_blobs;
+ uip->cs_blobs = blob;
+
+ ubc_cs_blob_adjust_statistics(blob);
+
+ if (cs_debug > 1) {
+ proc_t p;
+ const char *name = vnode_getname_printable(vp);
+ p = current_proc();
+ printf("CODE SIGNING: proc %d(%s) "
+ "loaded %s signatures for file (%s) "
+ "range 0x%llx:0x%llx flags 0x%x\n",
+ p->p_pid, p->p_comm,
+ blob->csb_cpu_type == -1 ? "detached" : "embedded",
+ name,
+ blob->csb_base_offset + blob->csb_start_offset,
+ blob->csb_base_offset + blob->csb_end_offset,
+ blob->csb_flags);
+ vnode_putname_printable(name);
+ }
+
+ vnode_unlock(vp);
+
+ if (record_mtime) {
+ vnode_mtime(vp, &uip->cs_mtime, vfs_context_current());
+ }
+
+ if (ret_blob) {
+ *ret_blob = blob;
+ }
+
+ error = 0; /* success ! */
+
+out:
+ if (error) {
+ if (cs_debug) {
+ printf("check_signature[pid: %d]: error = %d\n", current_proc()->p_pid, error);
+ }
+
+ cs_blob_free(blob);
+ }
+
+ if (error == EAGAIN) {
+ /*
+ * See above: error is EAGAIN if we were asked
+ * to add an existing blob again. We cleaned the new
+ * blob and we want to return success.
+ */
+ error = 0;
+ }
+
+ return error;
+}
+
+#if CONFIG_SUPPLEMENTAL_SIGNATURES
+int
+ubc_cs_blob_add_supplement(
+ struct vnode *vp,
+ struct vnode *orig_vp,
+ off_t base_offset,
+ vm_address_t *addr,
+ vm_size_t size,
+ struct cs_blob **ret_blob)
+{
+ kern_return_t kr;
+ struct ubc_info *uip, *orig_uip;
+ int error;
+ struct cs_blob *blob, *orig_blob;
+ CS_CodeDirectory const *cd;
+ off_t blob_start_offset, blob_end_offset;
+
+ if (ret_blob) {
+ *ret_blob = NULL;
+ }
+
+ /* Create the struct cs_blob wrapper that will be attached to the vnode.
+ * Validates the passed in blob in the process. */
+ error = cs_blob_create_validated(addr, size, &blob, &cd);
+
+ if (error != 0) {
+ printf("malformed code signature supplement blob: %d\n", error);
+ return error;
+ }
+
+ blob->csb_cpu_type = -1;
+ blob->csb_base_offset = base_offset;
+
+ blob->csb_reconstituted = false;
+
+ vnode_lock(orig_vp);
+ if (!UBCINFOEXISTS(orig_vp)) {
+ vnode_unlock(orig_vp);
+ error = ENOENT;
+ goto out;
+ }
+
+ orig_uip = orig_vp->v_ubcinfo;
+
+ /* check that the supplement's linked cdhash matches a cdhash of
+ * the target image.
+ */
+
+ if (blob->csb_linkage_hashtype == NULL) {
+ proc_t p;
+ const char *iname = vnode_getname_printable(vp);
+ p = current_proc();
+
+ printf("CODE SIGNING: proc %d(%s) supplemental signature for file (%s) "
+ "is not a supplemental.\n",
+ p->p_pid, p->p_comm, iname);
+
+ error = EINVAL;
+
+ vnode_putname_printable(iname);
+ vnode_unlock(orig_vp);
+ goto out;
+ }
+
+ for (orig_blob = orig_uip->cs_blobs; orig_blob != NULL;
+ orig_blob = orig_blob->csb_next) {
+ ptrauth_utils_auth_blob_generic(orig_blob->csb_cdhash,
+ sizeof(orig_blob->csb_cdhash),
+ OS_PTRAUTH_DISCRIMINATOR("cs_blob.csb_cd_signature"),
+ PTRAUTH_ADDR_DIVERSIFY,
+ orig_blob->csb_cdhash_signature);
+ if (orig_blob->csb_hashtype == blob->csb_linkage_hashtype &&
+ memcmp(orig_blob->csb_cdhash, blob->csb_linkage, CS_CDHASH_LEN) == 0) {
+ // Found match!
+ break;
+ }
+ }
+
+ if (orig_blob == NULL) {
+ // Not found.
+
+ proc_t p;
+ const char *iname = vnode_getname_printable(vp);
+ p = current_proc();
+
+ printf("CODE SIGNING: proc %d(%s) supplemental signature for file (%s) "
+ "does not match any attached cdhash.\n",
+ p->p_pid, p->p_comm, iname);
+
+ error = ESRCH;
+
+ vnode_putname_printable(iname);
+ vnode_unlock(orig_vp);
+ goto out;
+ }
+
+ vnode_unlock(orig_vp);
+
+ // validate the signature against policy!
+#if CONFIG_MACF
+ unsigned int signer_type = blob->csb_signer_type;
+ error = mac_vnode_check_supplemental_signature(vp, blob, orig_vp, orig_blob, &signer_type);
+ blob->csb_signer_type = signer_type;
+
+
+ if (error) {
+ if (cs_debug) {
+ printf("check_supplemental_signature[pid: %d], error = %d\n", current_proc()->p_pid, error);
+ }
+ goto out;
+ }
+#endif
+
+ // We allowed the supplemental signature blob so
+ // copy the platform bit or team-id from the linked signature and whether or not the original is developer code
+ blob->csb_platform_binary = 0;
+ blob->csb_platform_path = 0;
+ if (orig_blob->csb_platform_binary == 1) {
+ blob->csb_platform_binary = orig_blob->csb_platform_binary;
+ blob->csb_platform_path = orig_blob->csb_platform_path;
+ } else if (orig_blob->csb_teamid != NULL) {
+ vm_size_t teamid_size = strlen(orig_blob->csb_teamid) + 1;
+ blob->csb_supplement_teamid = kalloc(teamid_size);
+ if (blob->csb_supplement_teamid == NULL) {
+ error = ENOMEM;
+ goto out;
+ }
+ strlcpy(blob->csb_supplement_teamid, orig_blob->csb_teamid, teamid_size);
+ }
+ blob->csb_flags = (orig_blob->csb_flags & CS_DEV_CODE);
+
+ // Validate the blob's coverage
+ blob_start_offset = blob->csb_base_offset + blob->csb_start_offset;
+ blob_end_offset = blob->csb_base_offset + blob->csb_end_offset;
+
+ if (blob_start_offset >= blob_end_offset || blob_start_offset < 0 || blob_end_offset <= 0) {
+ /* reject empty or backwards blob */
+ error = EINVAL;
+ goto out;
+ }
+
+ vnode_lock(vp);
+ if (!UBCINFOEXISTS(vp)) {
+ vnode_unlock(vp);
+ error = ENOENT;
+ goto out;
+ }
+ uip = vp->v_ubcinfo;
+
+ struct cs_blob *existing = uip->cs_blob_supplement;
+ if (existing != NULL) {
+ if (blob->csb_hashtype == existing->csb_hashtype &&
+ memcmp(blob->csb_cdhash, existing->csb_cdhash, CS_CDHASH_LEN) == 0) {
+ error = EAGAIN; // non-fatal
+ } else {
+ error = EALREADY; // fatal
+ }
+
+ vnode_unlock(vp);
+ goto out;
+ }
+
+ /* Unlike regular cs_blobs, we only ever support one supplement. */
+ blob->csb_next = NULL;
+ uip->cs_blob_supplement = blob;
+
+ /* mark this vnode's VM object as having "signed pages" */
+ kr = memory_object_signed(uip->ui_control, TRUE);
+ if (kr != KERN_SUCCESS) {
+ vnode_unlock(vp);
+ error = ENOENT;
+ goto out;
+ }
+
+ vnode_unlock(vp);
+
+ /* We still adjust statistics even for supplemental blobs, as they
+ * consume memory just the same. */
+ ubc_cs_blob_adjust_statistics(blob);
+
+ if (cs_debug > 1) {
+ proc_t p;
+ const char *name = vnode_getname_printable(vp);
+ p = current_proc();
+ printf("CODE SIGNING: proc %d(%s) "
+ "loaded supplemental signature for file (%s) "
+ "range 0x%llx:0x%llx\n",
+ p->p_pid, p->p_comm,
+ name,
+ blob->csb_base_offset + blob->csb_start_offset,
+ blob->csb_base_offset + blob->csb_end_offset);
+ vnode_putname_printable(name);
+ }
+
+ if (ret_blob) {
+ *ret_blob = blob;
+ }
+
+ error = 0; // Success!
+out:
+ if (error) {
+ if (cs_debug) {
+ printf("ubc_cs_blob_add_supplement[pid: %d]: error = %d\n", current_proc()->p_pid, error);
+ }
+
+ cs_blob_supplement_free(blob);
+ }
+
+ if (error == EAGAIN) {
+ /* We were asked to add an existing blob.
+ * We cleaned up and ignore the attempt. */
+ error = 0;
+ }
+
+ return error;
+}
+#endif
+
+
+
+void
+csvnode_print_debug(struct vnode *vp)
+{
+ const char *name = NULL;
+ struct ubc_info *uip;
+ struct cs_blob *blob;
+
+ name = vnode_getname_printable(vp);
+ if (name) {
+ printf("csvnode: name: %s\n", name);
+ vnode_putname_printable(name);
+ }
+
+ vnode_lock_spin(vp);
+
+ if (!UBCINFOEXISTS(vp)) {
+ blob = NULL;
+ goto out;
+ }
+
+ uip = vp->v_ubcinfo;
+ for (blob = uip->cs_blobs; blob != NULL; blob = blob->csb_next) {
+ printf("csvnode: range: %lu -> %lu flags: 0x%08x platform: %s path: %s team: %s\n",
+ (unsigned long)blob->csb_start_offset,
+ (unsigned long)blob->csb_end_offset,
+ blob->csb_flags,
+ blob->csb_platform_binary ? "yes" : "no",
+ blob->csb_platform_path ? "yes" : "no",
+ blob->csb_teamid ? blob->csb_teamid : "<NO-TEAM>");
+ }
+
+out:
+ vnode_unlock(vp);
+}
+
+#if CONFIG_SUPPLEMENTAL_SIGNATURES
+struct cs_blob *
+ubc_cs_blob_get_supplement(
+ struct vnode *vp,
+ off_t offset)
+{
+ struct cs_blob *blob;
+ off_t offset_in_blob;
+
+ vnode_lock_spin(vp);
+
+ if (!UBCINFOEXISTS(vp)) {
+ blob = NULL;
+ goto out;
+ }
+
+ blob = vp->v_ubcinfo->cs_blob_supplement;
+
+ if (blob == NULL) {
+ // no supplemental blob
+ goto out;
+ }
+
+
+ if (offset != -1) {
+ offset_in_blob = offset - blob->csb_base_offset;
+ if (offset_in_blob < blob->csb_start_offset || offset_in_blob >= blob->csb_end_offset) {
+ // not actually covered by this blob
+ blob = NULL;
+ }
+ }
projects / apple / xnu.git / blobdiff