Compile fixes for O_DIRECTIO

Signed-off-by: Jorgen Lundman <[email protected]>

include/os/macos/spl/sys/param.h

@@ -38,6 +38,8 @@
 #define	ptob(pages)			(pages << PAGE_SHIFT)
 #define	btop(bytes)			(bytes >> PAGE_SHIFT)
 
+#define	PAGESHIFT			PAGE_SHIFT
+
 #define	MAXUID				UINT32_MAX
 
 #endif /* SPL_PARAM_H */

include/os/macos/spl/sys/uio.h

@@ -70,6 +70,15 @@ typedef enum uio_rw zfs_uio_rw_t;
 typedef size_t (*zfs_uio_func)(char *addr, uint64_t offset, size_t len,
     zfs_uio_rw_t rw, const void *privptr);
 
+/*
+ * This structure is used when doing Direct I/O.
+ */
+typedef void vm_page_t;
+typedef struct {
+	vm_page_t	*pages;
+	int		npages;
+} zfs_uio_dio_t;
+
 /*
  * Hybrid uio, use OS uio for IO and communicating with XNU
  * and internal uio for ZFS / crypto. The default mode is
@@ -86,13 +95,15 @@ typedef struct zfs_uio {
 	const struct iovec	*uio_iov;
 	int			uio_iovcnt;
 	off_t			uio_loffset;
+	off_t			uio_soffset;
 	zfs_uio_seg_t		uio_segflg;
 	boolean_t		uio_fault_disable;
 	uint16_t		uio_fmode;
 	uint16_t		uio_extflg;
 	ssize_t			uio_resid;
 	size_t			uio_skip;
 	zfs_uio_func		uio_iofunc;
+	zfs_uio_dio_t	uio_dio;
 } zfs_uio_t;
 
 
@@ -147,6 +158,12 @@ zfs_uio_offset(zfs_uio_t *uio)
 	return (uio->uio_loffset);
 }
 
+static inline off_t
+zfs_uio_soffset(zfs_uio_t *uio)
+{
+	return (uio->uio_soffset);
+}
+
 static inline size_t
 zfs_uio_resid(zfs_uio_t *uio)
 {
@@ -165,6 +182,12 @@ zfs_uio_setoffset(zfs_uio_t *uio, off_t off)
 	uio->uio_loffset = off;
 }
 
+static inline void
+zfs_uio_setsoffset(zfs_uio_t *uio, offset_t off)
+{
+	uio->uio_soffset = off;
+}
+
 static inline void
 zfs_uio_advance(zfs_uio_t *uio, size_t size)
 {
@@ -209,6 +232,7 @@ zfs_uio_iovec_init(zfs_uio_t *uio, const struct iovec *iov,
 	uio->uio_iov = iov;
 	uio->uio_iovcnt = nr_segs;
 	uio->uio_loffset = offset;
+	uio->uio_soffset = offset;
 	uio->uio_segflg = seg;
 	uio->uio_fmode = 0;
 	uio->uio_extflg = 0;

include/os/macos/zfs/sys/abd_os.h

@@ -45,6 +45,11 @@ struct abd_linear {
 	void		*abd_buf;
 };
 
+#ifdef _KERNEL
+__attribute__((malloc))
+struct abd *abd_alloc_from_pages(vm_page_t *, unsigned long, uint64_t);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

module/os/macos/Makefile.am

@@ -151,10 +151,12 @@ zfs_common_SRCS = \
 	%D%/../../zfs/dbuf.c \
 	%D%/../../zfs/dbuf_stats.c \
 	%D%/../../zfs/ddt.c \
+	%D%/../../zfs/ddt_log.c \
 	%D%/../../zfs/ddt_stats.c \
 	%D%/../../zfs/ddt_zap.c \
 	%D%/../../zfs/dmu.c \
 	%D%/../../zfs/dmu_diff.c \
+	%D%/../../zfs/dmu_direct.c \
 	%D%/../../zfs/dmu_object.c \
 	%D%/../../zfs/dmu_objset.c \
 	%D%/../../zfs/dmu_recv.c \
@@ -250,6 +252,7 @@ zfs_common_SRCS = \
 	%D%/../../zfs/zfs_rlock.c \
 	%D%/../../zfs/zfs_sa.c \
 	%D%/../../zfs/zfs_vnops.c \
+	%D%/../../zfs/zfs_znode.c \
 	%D%/../../zfs/zil.c \
 	%D%/../../zfs/zio.c \
 	%D%/../../zfs/zio_checksum.c \
@@ -286,7 +289,7 @@ zfs_os_SRCS = \
 	%D%/zfs/zfs_vnops_osx.c \
 	%D%/zfs/zfs_vnops_osx_lib.c \
 	%D%/zfs/zfs_vnops_osx_xattr.c \
-	%D%/zfs/zfs_znode.c \
+	%D%/zfs/zfs_znode_os.c \
 	%D%/zfs/zio_crypt.c \
 	%D%/zfs/zvol_os.c \
 	%D%/zfs/zvolIO.cpp \
@@ -301,7 +304,6 @@ zcommon_SRCS = \
 	%D%/../../nvpair/nvpair_alloc_fixed.c \
 	%D%/../../nvpair/nvpair_alloc_spl.c \
 	%D%/../../unicode/u8_textprep.c \
-	%D%/../../unicode/uconv.c \
 	%D%/../../zcommon/zfs_comutil.c \
 	%D%/../../zcommon/zfs_deleg.c \
 	%D%/../../zcommon/zfs_fletcher.c \
@@ -324,7 +326,6 @@ icp_SRCS = \
 	%D%/../../icp/spi/kcf_spi.c \
 	%D%/../../icp/io/aes.c \
 	%D%/../../icp/io/sha2_mod.c \
-	%D%/../../icp/io/skein_mod.c \
 	%D%/../../icp/algs/aes/aes_impl_aesni.c \
 	%D%/../../icp/algs/aes/aes_impl_aesv8.c \
 	%D%/../../icp/algs/aes/aes_impl_generic.c \
@@ -335,10 +336,7 @@ icp_SRCS = \
 	%D%/../../icp/algs/blake3/blake3_generic.c \
 	%D%/../../icp/algs/blake3/blake3_impl.c \
 	%D%/../../icp/algs/edonr/edonr.c \
-	%D%/../../icp/algs/modes/cbc.c \
 	%D%/../../icp/algs/modes/ccm.c \
-	%D%/../../icp/algs/modes/ctr.c \
-	%D%/../../icp/algs/modes/ecb.c \
 	%D%/../../icp/algs/modes/gcm_generic.c \
 	%D%/../../icp/algs/modes/gcm.c \
 	%D%/../../icp/algs/modes/modes.c \

module/os/macos/spl/spl-uio.c

@@ -146,3 +146,39 @@ zfs_uio_prefaultpages(ssize_t n, zfs_uio_t *uio)
 {
 	return (0);
 }
+
+/*
+ * Check if the uio is page-aligned in memory.
+ */
+boolean_t
+zfs_uio_page_aligned(zfs_uio_t *uio)
+{
+	for (int i = zfs_uio_iovcnt(uio); i > 0; i--) {
+		uintptr_t addr = (uintptr_t)zfs_uio_iovbase(uio, i);
+		size_t size = zfs_uio_iovlen(uio, i);
+		if ((addr & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) {
+				return (B_FALSE);
+		}
+	}
+
+	return (B_TRUE);
+}
+
+void
+zfs_uio_free_dio_pages(zfs_uio_t *uio, zfs_uio_rw_t rw)
+{
+#if 0
+	ASSERT(uio->uio_extflg & UIO_DIRECT);
+	ASSERT3P(uio->uio_dio.pages, !=, NULL);
+	ASSERT(zfs_uio_rw(uio) == rw);
+
+	if (rw == UIO_WRITE)
+		zfs_uio_release_stable_pages(uio);
+
+	vm_page_unhold_pages(&uio->uio_dio.pages[0],
+	    uio->uio_dio.npages);
+
+	kmem_free(uio->uio_dio.pages,
+	    uio->uio_dio.npages * sizeof (vm_page_t));
+#endif
+}

module/os/macos/zfs/abd_os.c

@@ -12,6 +12,8 @@
 /*
  * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
  * Copyright (c) 2016 by Delphix. All rights reserved.
+ * Copyright (c) 2020 by Jorgen Lundman. All rights reserved.
+ * Copyright (c) 2021 by Sean Doran. All rights reserved.
  */
 
 /*
@@ -107,8 +109,6 @@ struct {
 #define	ABD_PGSIZE	PAGE_SIZE
 #endif
 
-#define	PAGE_MASK (ABD_PGSIZE - 1ULL)
-
 const static size_t zfs_abd_chunk_size = ABD_PGSIZE;
 
 kmem_cache_t *abd_chunk_cache;
@@ -459,6 +459,48 @@ abd_get_offset_scatter(abd_t *abd, abd_t *sabd, size_t off,
 	return (abd);
 }
 
+/*
+ * Allocate a scatter ABD structure from user pages.
+ */
+abd_t *
+abd_alloc_from_pages(vm_page_t *pages, unsigned long offset, uint64_t size)
+{
+	VERIFY3U(size, <=, DMU_MAX_ACCESS);
+	ASSERT3U(offset, <, PAGE_SIZE);
+	ASSERT3P(pages, !=, NULL);
+
+	/* Until we do DIRECTIO */
+	VERIFY3U(0, !=, 0);
+	abd_t *abd = NULL;
+#if 0
+	abd_t *abd = abd_alloc_struct(size);
+	abd->abd_flags |= ABD_FLAG_OWNER | ABD_FLAG_FROM_PAGES;
+	abd->abd_size = size;
+
+	if ((offset + size) <= PAGE_SIZE) {
+		/*
+		 * There is only a single page worth of data, so we will just
+		 * use  a linear ABD. We have to make sure to take into account
+		 * the offset though. In all other cases our offset will be 0
+		 * as we are always PAGE_SIZE aligned.
+		 */
+		abd->abd_flags |= ABD_FLAG_LINEAR | ABD_FLAG_LINEAR_PAGE;
+		ABD_LINEAR_BUF(abd) = (char *)zfs_map_page(pages[0],
+		    &abd->abd_u.abd_linear.sf) + offset;
+	} else {
+		ABD_SCATTER(abd).abd_offset = offset;
+		ASSERT0(ABD_SCATTER(abd).abd_offset);
+
+		/*
+		 * Setting the ABD's abd_chunks to point to the user pages.
+		 */
+		for (int i = 0; i < abd_chunkcnt_for_bytes(size); i++)
+			ABD_SCATTER(abd).abd_chunks[i] = pages[i];
+	}
+#endif
+	return (abd);
+}
+
 /*
  * Initialize the abd_iter.
  */
@@ -567,6 +609,103 @@ abd_cache_reap_now(void)
 	 */
 }
 
+/*
+ * Borrow a raw buffer from an ABD without copying the contents of the ABD
+ * into the buffer. If the ABD is scattered, this will alloate a raw buffer
+ * whose contents are undefined. To copy over the existing data in the ABD, use
+ * abd_borrow_buf_copy() instead.
+ */
+void *
+abd_borrow_buf(abd_t *abd, size_t n)
+{
+	void *buf;
+	abd_verify(abd);
+	ASSERT3U(abd->abd_size, >=, 0);
+	if (abd_is_linear(abd)) {
+		buf = abd_to_buf(abd);
+	} else {
+		buf = zio_buf_alloc(n);
+	}
+#ifdef ZFS_DEBUG
+	(void) zfs_refcount_add_many(&abd->abd_children, n, buf);
+#endif
+	return (buf);
+}
+
+void *
+abd_borrow_buf_copy(abd_t *abd, size_t n)
+{
+	void *buf = abd_borrow_buf(abd, n);
+	if (!abd_is_linear(abd)) {
+		abd_copy_to_buf(buf, abd, n);
+	}
+	return (buf);
+}
+
+/*
+ * Return a borrowed raw buffer to an ABD. If the ABD is scattered, this will
+ * not change the contents of the ABD. If you want any changes you made to
+ * buf to be copied back to abd, use abd_return_buf_copy() instead. If the
+ * ABD is not constructed from user pages from Direct I/O then an ASSERT
+ * checks to make sure the contents of the buffer have not changed since it was
+ * borrowed. We can not ASSERT the contents of the buffer have not changed if
+ * it is composed of user pages. While Direct I/O write pages are placed under
+ * write protection and can not be changed, this is not the case for Direct I/O
+ * reads. The pages of a Direct I/O read could be manipulated at any time.
+ * Checksum verifications in the ZIO pipeline check for this issue and handle
+ * it by returning an error on checksum verification failure.
+ */
+void
+abd_return_buf(abd_t *abd, void *buf, size_t n)
+{
+	abd_verify(abd);
+	ASSERT3U(abd->abd_size, >=, n);
+#ifdef ZFS_DEBUG
+	(void) zfs_refcount_remove_many(&abd->abd_children, n, buf);
+#endif
+	if (abd_is_from_pages(abd)) {
+		if (!abd_is_linear_page(abd))
+			zio_buf_free(buf, n);
+	} else if (abd_is_linear(abd)) {
+		ASSERT3P(buf, ==, abd_to_buf(abd));
+	} else if (abd_is_gang(abd)) {
+#ifdef ZFS_DEBUG
+		/*
+		 * We have to be careful with gang ABD's that we do not ASSERT
+		 * for any ABD's that contain user pages from Direct I/O. See
+		 * the comment above about Direct I/O read buffers possibly
+		 * being manipulated. In order to handle this, we jsut iterate
+		 * through the gang ABD and only verify ABD's that are not from
+		 * user pages.
+		 */
+		void *cmp_buf = buf;
+
+		for (abd_t *cabd = list_head(&ABD_GANG(abd).abd_gang_chain);
+		    cabd != NULL;
+		    cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
+			if (!abd_is_from_pages(cabd)) {
+				ASSERT0(abd_cmp_buf(cabd, cmp_buf,
+				    cabd->abd_size));
+			}
+			cmp_buf = (char *)cmp_buf + cabd->abd_size;
+		}
+#endif
+		zio_buf_free(buf, n);
+	} else {
+		ASSERT0(abd_cmp_buf(abd, buf, n));
+		zio_buf_free(buf, n);
+	}
+}
+
+void
+abd_return_buf_copy(abd_t *abd, void *buf, size_t n)
+{
+	if (!abd_is_linear(abd)) {
+		abd_copy_from_buf(abd, buf, n);
+	}
+	abd_return_buf(abd, buf, n);
+}
+
 /* Tunable Parameters */
 module_param(zfs_abd_scatter_enabled, int, 0644);
 MODULE_PARM_DESC(zfs_abd_scatter_enabled,

module/os/macos/zfs/arc_os.c

@@ -169,7 +169,7 @@ arc_memory_throttle(spa_t *spa, uint64_t reserve, uint64_t txg)
  */
 static void arc_kmem_reap_now(void)
 {
-	arc_wait_for_eviction(0, B_FALSE);
+	arc_wait_for_eviction(0, B_FALSE, B_FALSE);
 
 	/* arc.c will do the heavy lifting */
 	arc_kmem_reap_soon();
@@ -223,7 +223,7 @@ arc_reclaim_thread(void *unused)
 			    (arc_c >> arc_shrink_shift)));
 		}
 
-		arc_wait_for_eviction(0, B_FALSE);
+		arc_wait_for_eviction(0, B_FALSE, B_FALSE);
 
 		int64_t free_memory = arc_available_memory();