zfs: support force exporting pools

This is primarily of use when a pool has lost its disk, while the user
doesn't care about any pending (or otherwise) transactions.

Implement various control methods to make this feasible:
- txg_wait can now take a NOSUSPEND flag, in which case the caller will be
  alerted if their txg can't be committed.  This is primarily of interest
  for callers that would normally pass TXG_WAIT, but don't want to wait if
  the pool becomes suspended, which allows unwinding in some cases,
  specifically when one is attempting a non-forced export.  Without this,
  the non-forced export would preclude a forced export by virtue of holding
  the namespace lock indefinitely.
- txg_wait also returns failure for TXG_WAIT users if a pool is actually
  being force exported.  Adjust most callers to tolerate this.
- spa_config_enter_flags now takes a NOSUSPEND flag to the same effect.
- DMU objset "killer" flag which may be set on an objset being forcibly
  exported / unmounted.
- SPA "killer" flag which may be set on a pool being forcibly exported.
- DMU send/recv now use an interruption mechanism which relies on the SPA
  killer being able to enumerate datasets and closing any send/recv streams,
  causing their EINTR paths to be invoked.
- ZIO now has a cancel entry point, which tells all suspended zios to fail,
  and which suppresses the failures for non-CANFAIL users.
- metaslab, etc. cleanup, which consists of simply throwing away any changes
  that were not able to be synced out.
- Linux specific: introduce a new tunable, zfs_forced_export_unmount_enabled,
  which allows the filesystem to remain in a modified 'unmounted' state upon
  exiting zpl_umount_begin, to achieve parity with FreeBSD and illumos,
  which have VFS-level support for yanking filesystems out from under users.
  However, this only helps when the user is actively performing I/O, while
  not sitting on the filesystem.  In particular, this allows test #3 below
  to pass on Linux.
- Add basic logic to zpool to indicate a force-exporting pool, instead of
  crashing due to lack of config, etc.

Add tests which cover the basic use cases:
- Force export while a send is in progress
- Force export while a recv is in progress
- Force export while POSIX I/O is in progress

Signed-off-by:	Will Andrews <[email protected]>
Sponsored-by:	Klara, Inc.
Sponsored-by:	Catalogics, Inc.
Closes openzfs#3461

cmd/zpool/zpool_main.c

@@ -1776,7 +1776,7 @@ zpool_export_one(zpool_handle_t *zhp, void *data)
 {
 	export_cbdata_t *cb = data;
 
-	if (zpool_disable_datasets(zhp, cb->force) != 0)
+	if (zpool_disable_datasets(zhp, cb->force || cb->hardforce) != 0)
 		return (1);
 
 	/* The history must be logged as part of the export */

include/libzfs.h

@@ -391,6 +391,7 @@ typedef enum {
 	ZPOOL_STATUS_REBUILDING,	/* device being rebuilt */
 	ZPOOL_STATUS_REBUILD_SCRUB,	/* recommend scrubbing the pool */
 	ZPOOL_STATUS_NON_NATIVE_ASHIFT,	/* (e.g. 512e dev with ashift of 9) */
+	ZPOOL_STATUS_FORCE_EXPORTING,	/* pool is being force exported */
 
 	/*
 	 * Finally, the following indicates a healthy pool.

include/libzfs_impl.h

@@ -72,6 +72,7 @@ struct libzfs_handle {
 	boolean_t libzfs_prop_debug;
 	regex_t libzfs_urire;
 	uint64_t libzfs_max_nvlist;
+	boolean_t libzfs_force_export;
 };
 
 struct zfs_handle {

include/os/freebsd/spl/sys/thread.h

@@ -31,4 +31,7 @@
 
 #define	getcomm() curthread->td_name
 #define	getpid() curthread->td_tid
+#define thread_signal spl_kthread_signal
+extern int spl_kthread_signal(kthread_t *tsk, int sig);
+
 #endif

include/os/linux/spl/sys/kmem_cache.h

@@ -99,7 +99,7 @@ extern struct rw_semaphore spl_kmem_cache_sem;
 #endif
 
 #define	POINTER_IS_VALID(p)		0	/* Unimplemented */
-#define	POINTER_INVALIDATE(pp)			/* Unimplemented */
+#define	POINTER_INVALIDATE(pp)		(*(pp) = (void *)((uintptr_t)(*(pp)) | 0x1))
 
 typedef int (*spl_kmem_ctor_t)(void *, void *, int);
 typedef void (*spl_kmem_dtor_t)(void *, void *);

include/os/linux/spl/sys/thread.h

@@ -55,6 +55,7 @@ typedef void (*thread_func_t)(void *);
 	#func, arg, len, pp, state, pri)
 /* END CSTYLED */
 
+#define	thread_signal(t, s)		spl_kthread_signal(t, s)
 #define	thread_exit()			__thread_exit()
 #define	thread_join(t)			VERIFY(0)
 #define	curthread			current
@@ -67,6 +68,7 @@ extern kthread_t *__thread_create(caddr_t stk, size_t  stksize,
 extern void __thread_exit(void);
 extern struct task_struct *spl_kthread_create(int (*func)(void *),
     void *data, const char namefmt[], ...);
+extern int spl_kthread_signal(kthread_t *tsk, int sig);
 
 extern proc_t p0;
 

include/os/linux/zfs/sys/zfs_vfsops_os.h

@@ -101,7 +101,7 @@ struct zfsvfs {
 	boolean_t	z_utf8;		/* utf8-only */
 	int		z_norm;		/* normalization flags */
 	boolean_t	z_relatime;	/* enable relatime mount option */
-	boolean_t	z_unmounted;	/* unmounted */
+	uint8_t		z_unmounted;	/* mount status */
 	rrmlock_t	z_teardown_lock;
 	krwlock_t	z_teardown_inactive_lock;
 	list_t		z_all_znodes;	/* all znodes in the fs */
@@ -138,6 +138,11 @@ struct zfsvfs {
 	taskqid_t	z_drain_task;	/* task id for the unlink drain task */
 };
 
+/* z_unmounted values */
+#define	Z_MOUNTED	(0)		/* filesystem is mounted */
+#define	Z_UNMOUNT_BEGIN (1)		/* filesystem has been force unmounted */
+#define	Z_UNMOUNTED	(2)		/* filesystem is unmounted */
+
 #define	ZSB_XATTR	0x0001		/* Enable user xattrs */
 
 /*

include/os/linux/zfs/sys/zfs_znode_impl.h

@@ -80,14 +80,24 @@ extern "C" {
 #define	ZFS_ENTER_ERROR(zfsvfs, error)				\
 do {								\
 	rrm_enter_read(&(zfsvfs)->z_teardown_lock, FTAG);	\
-	if ((zfsvfs)->z_unmounted) {				\
+	if ((zfsvfs)->z_unmounted != Z_MOUNTED) {		\
 		ZFS_EXIT(zfsvfs);				\
 		return (error);					\
 	}							\
 } while (0)
 #define	ZFS_ENTER(zfsvfs)	ZFS_ENTER_ERROR(zfsvfs, EIO)
 #define	ZPL_ENTER(zfsvfs)	ZFS_ENTER_ERROR(zfsvfs, -EIO)
 
+/* ZFS_ENTER but ok with unmount having begun */
+#define	ZFS_ENTER_UNMOUNTOK(zfsvfs)				\
+do {								\
+	rrm_enter_read(&(zfsvfs)->z_teardown_lock, FTAG);	\
+	if ((zfsvfs)->z_unmounted == Z_UNMOUNTED) {		\
+		ZFS_EXIT(zfsvfs);				\
+		return (EIO);					\
+	}							\
+} while (0)
+
 /* Must be called before exiting the operation. */
 #define	ZFS_EXIT(zfsvfs)					\
 do {								\

include/sys/dmu.h

@@ -279,6 +279,7 @@ typedef enum dmu_object_type {
 #define	TXG_NOWAIT	(0ULL)
 #define	TXG_WAIT	(1ULL<<0)
 #define	TXG_NOTHROTTLE	(1ULL<<1)
+#define	TXG_NOSUSPEND	(2ULL<<2)
 
 void byteswap_uint64_array(void *buf, size_t size);
 void byteswap_uint32_array(void *buf, size_t size);

include/sys/dmu_impl.h

@@ -241,6 +241,7 @@ typedef struct dmu_sendstatus {
 	list_node_t dss_link;
 	int dss_outfd;
 	proc_t *dss_proc;
+	kthread_t *dss_thread;
 	offset_t *dss_off;
 	uint64_t dss_blocks; /* blocks visited during the sending process */
 } dmu_sendstatus_t;

include/sys/dmu_objset.h

@@ -172,6 +172,7 @@ struct objset {
 
 	/* Protected by os_lock */
 	kmutex_t os_lock;
+	kthread_t *os_killer;
 	multilist_t *os_dirty_dnodes[TXG_SIZE];
 	list_t os_dnodes;
 	list_t os_downgraded_dbufs;
@@ -263,6 +264,10 @@ int dmu_fsname(const char *snapname, char *buf);
 void dmu_objset_evict_done(objset_t *os);
 void dmu_objset_willuse_space(objset_t *os, int64_t space, dmu_tx_t *tx);
 
+int dmu_objset_shutdown_register(objset_t *os);
+boolean_t dmu_objset_exiting(objset_t *os);
+void dmu_objset_shutdown_unregister(objset_t *os);
+
 void dmu_objset_init(void);
 void dmu_objset_fini(void);
 

include/sys/dmu_recv.h

@@ -40,6 +40,7 @@ extern const char *recv_clone_name;
 
 typedef struct dmu_recv_cookie {
 	struct dsl_dataset *drc_ds;
+	kthread_t *drc_initiator;
 	struct dmu_replay_record *drc_drr_begin;
 	struct drr_begin *drc_drrb;
 	const char *drc_tofs;
@@ -56,6 +57,8 @@ typedef struct dmu_recv_cookie {
 	nvlist_t *drc_keynvl;
 	uint64_t drc_fromsnapobj;
 	uint64_t drc_ivset_guid;
+	unsigned int drc_flags;
+	void *drc_rwa;
 	void *drc_owner;
 	cred_t *drc_cred;
 	proc_t *drc_proc;
@@ -82,6 +85,7 @@ int dmu_recv_begin(char *, char *, dmu_replay_record_t *,
     boolean_t, boolean_t, nvlist_t *, nvlist_t *, char *,
     dmu_recv_cookie_t *, zfs_file_t *, offset_t *);
 int dmu_recv_stream(dmu_recv_cookie_t *, offset_t *);
+int dmu_recv_close(dsl_dataset_t *ds);
 int dmu_recv_end(dmu_recv_cookie_t *, void *);
 boolean_t dmu_objset_is_receiving(objset_t *);
 

include/sys/dmu_send.h

@@ -61,6 +61,7 @@ int dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap,
     boolean_t embedok, boolean_t large_block_ok, boolean_t compressok,
     boolean_t rawok, boolean_t savedok, int outfd, offset_t *off,
     struct dmu_send_outparams *dso);
+int dmu_send_close(struct dsl_dataset *ds);
 
 typedef int (*dmu_send_outfunc_t)(objset_t *os, void *buf, int len, void *arg);
 typedef struct dmu_send_outparams {

include/sys/dsl_dataset.h

@@ -243,6 +243,8 @@ typedef struct dsl_dataset {
 	kmutex_t ds_sendstream_lock;
 	list_t ds_sendstreams;
 
+	void *ds_receiver;	/* really a dmu_recv_cookie_t */
+
 	/*
 	 * When in the middle of a resumable receive, tracks how much
 	 * progress we have made.
@@ -318,6 +320,7 @@ typedef struct dsl_dataset_snapshot_arg {
 typedef enum ds_hold_flags {
 	DS_HOLD_FLAG_NONE	= 0 << 0,
 	DS_HOLD_FLAG_DECRYPT	= 1 << 0 /* needs access to encrypted data */
+	DS_HOLD_FLAG_MUST_BE_OPEN = 1 << 1, /* dataset must already be open */
 } ds_hold_flags_t;
 
 int dsl_dataset_hold(struct dsl_pool *dp, const char *name, void *tag,
@@ -445,6 +448,8 @@ void dsl_dataset_long_hold(dsl_dataset_t *ds, void *tag);
 void dsl_dataset_long_rele(dsl_dataset_t *ds, void *tag);
 boolean_t dsl_dataset_long_held(dsl_dataset_t *ds);
 
+int dsl_dataset_sendrecv_cancel_all(spa_t *spa);
+
 int dsl_dataset_clone_swap_check_impl(dsl_dataset_t *clone,
     dsl_dataset_t *origin_head, boolean_t force, void *owner, dmu_tx_t *tx);
 void dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,

include/sys/dsl_scan.h

@@ -171,7 +171,7 @@ int dsl_scan(struct dsl_pool *, pool_scan_func_t);
 void dsl_scan_assess_vdev(struct dsl_pool *dp, vdev_t *vd);
 boolean_t dsl_scan_scrubbing(const struct dsl_pool *dp);
 int dsl_scrub_set_pause_resume(const struct dsl_pool *dp, pool_scrub_cmd_t cmd);
-void dsl_scan_restart_resilver(struct dsl_pool *, uint64_t txg);
+int dsl_scan_restart_resilver(struct dsl_pool *, uint64_t txg);
 boolean_t dsl_scan_resilvering(struct dsl_pool *dp);
 boolean_t dsl_scan_resilver_scheduled(struct dsl_pool *dp);
 boolean_t dsl_dataset_unstable(struct dsl_dataset *ds);

include/sys/metaslab.h

@@ -111,6 +111,7 @@ boolean_t metaslab_class_throttle_reserve(metaslab_class_t *, int, int,
     zio_t *, int);
 void metaslab_class_throttle_unreserve(metaslab_class_t *, int, int, zio_t *);
 void metaslab_class_evict_old(metaslab_class_t *, uint64_t);
+void metaslab_class_force_discard(metaslab_class_t *);
 uint64_t metaslab_class_get_alloc(metaslab_class_t *);
 uint64_t metaslab_class_get_space(metaslab_class_t *);
 uint64_t metaslab_class_get_dspace(metaslab_class_t *);

include/sys/spa.h

@@ -978,6 +978,7 @@ extern void spa_iostats_trim_add(spa_t *spa, trim_type_t type,
 /* Config lock handling flags */
 typedef enum {
 	SCL_FLAG_TRYENTER	= 1U << 0,
+	SCL_FLAG_NOSUSPEND	= 1U << 1,
 } spa_config_flag_t;
 
 extern void spa_import_progress_add(spa_t *spa);
@@ -1052,6 +1053,7 @@ extern uint64_t spa_last_synced_txg(spa_t *spa);
 extern uint64_t spa_first_txg(spa_t *spa);
 extern uint64_t spa_syncing_txg(spa_t *spa);
 extern uint64_t spa_final_dirty_txg(spa_t *spa);
+extern void spa_verify_dirty_txg(spa_t *spa, uint64_t txg);
 extern uint64_t spa_version(spa_t *spa);
 extern pool_state_t spa_state(spa_t *spa);
 extern spa_load_state_t spa_load_state(spa_t *spa);
@@ -1070,6 +1072,8 @@ extern metaslab_class_t *spa_dedup_class(spa_t *spa);
 extern metaslab_class_t *spa_preferred_class(spa_t *spa, uint64_t size,
     dmu_object_type_t objtype, uint_t level, uint_t special_smallblk);
 
+extern void spa_evicting_os_lock(spa_t *);
+extern void spa_evicting_os_unlock(spa_t *);
 extern void spa_evicting_os_register(spa_t *, objset_t *os);
 extern void spa_evicting_os_deregister(spa_t *, objset_t *os);
 extern void spa_evicting_os_wait(spa_t *spa);
@@ -1159,6 +1163,8 @@ extern void spa_history_log_internal_dd(dsl_dir_t *dd, const char *operation,
 
 extern const char *spa_state_to_name(spa_t *spa);
 
+extern boolean_t spa_exiting_any(spa_t *spa);
+extern boolean_t spa_exiting(spa_t *spa);
 extern int spa_operation_interrupted(spa_t *spa);
 
 /* error handling */

include/sys/spa_impl.h

@@ -238,6 +238,7 @@ struct spa {
 	kmutex_t	spa_evicting_os_lock;	/* Evicting objset list lock */
 	list_t		spa_evicting_os_list;	/* Objsets being evicted. */
 	kcondvar_t	spa_evicting_os_cv;	/* Objset Eviction Completion */
+	kthread_t	*spa_killer;		/* thread killing the pool */
 	txg_list_t	spa_vdev_txg_list;	/* per-txg dirty vdev list */
 	vdev_t		*spa_root_vdev;		/* top-level vdev container */
 	uint64_t	spa_min_ashift;		/* of vdevs in normal class */

include/sys/txg.h

@@ -66,11 +66,12 @@ typedef struct txg_list {
 } txg_list_t;
 
 struct dsl_pool;
+struct dmu_tx;
 
 extern void txg_init(struct dsl_pool *dp, uint64_t txg);
 extern void txg_fini(struct dsl_pool *dp);
 extern void txg_sync_start(struct dsl_pool *dp);
-extern void txg_sync_stop(struct dsl_pool *dp);
+extern int txg_sync_stop(struct dsl_pool *dp, uint64_t txg_how);
 extern uint64_t txg_hold_open(struct dsl_pool *dp, txg_handle_t *txghp);
 extern void txg_rele_to_quiesce(txg_handle_t *txghp);
 extern void txg_rele_to_sync(txg_handle_t *txghp);
@@ -92,6 +93,8 @@ extern void txg_kick(struct dsl_pool *dp);
 typedef enum {
 	/* Reject the call with EINTR upon receiving a signal. */
 	TXG_WAIT_F_SIGNAL	= (1U << 0),
+	/* Reject the call with EAGAIN upon suspension. */
+	TXG_WAIT_F_NOSUSPEND	= (1U << 1),
 } txg_wait_flag_t;
 extern int txg_wait_synced(struct dsl_pool *dp, uint64_t txg);
 extern int txg_wait_synced_tx(struct dsl_pool *dp, uint64_t txg,
@@ -113,6 +116,8 @@ extern int txg_wait_synced_tx(struct dsl_pool *dp, uint64_t txg,
 extern void txg_wait_open(struct dsl_pool *dp, uint64_t txg,
     boolean_t should_quiesce);
 
+void txg_force_export(spa_t *spa);
+
 /*
  * Returns TRUE if we are "backed up" waiting for the syncing
  * transaction to complete; otherwise returns FALSE.
@@ -124,6 +129,8 @@ extern boolean_t txg_sync_waiting(struct dsl_pool *dp);
 
 extern void txg_verify(spa_t *spa, uint64_t txg);
 
+extern void txg_completion_notify(struct dsl_pool *dp);
+
 /*
  * Wait for pending commit callbacks of already-synced transactions to finish
  * processing.

include/sys/zfs_context.h

@@ -231,6 +231,7 @@ typedef pthread_t	kthread_t;
 	zk_thread_create(func, arg, stksize, state)
 #define	thread_create(stk, stksize, func, arg, len, pp, state, pri)	\
 	zk_thread_create(func, arg, stksize, state)
+#define	thread_signal(t, s) pthread_kill((pthread_t)t, s)
 #define	thread_exit()	pthread_exit(NULL)
 #define	thread_join(t)	pthread_join((pthread_t)(t), NULL)
 
@@ -430,8 +431,8 @@ void procfs_list_add(procfs_list_t *procfs_list, void *p);
 #define	kmem_debugging()	0
 #define	kmem_cache_reap_now(_c)	umem_cache_reap_now(_c);
 #define	kmem_cache_set_move(_c, _cb)	/* nothing */
-#define	POINTER_INVALIDATE(_pp)		/* nothing */
-#define	POINTER_IS_VALID(_p)	0
+#define	POINTER_INVALIDATE(_pp)	(*(_pp) = (void *)((uintptr_t)(*(_pp)) | 0x1))
+#define	POINTER_IS_VALID(_p)	(!((uintptr_t)(p) & 0x1))
 
 typedef umem_cache_t kmem_cache_t;
 

include/sys/zio.h

@@ -421,6 +421,7 @@ typedef zio_t *zio_pipe_stage_t(zio_t *zio);
  */
 #define	ZIO_REEXECUTE_NOW	0x01
 #define	ZIO_REEXECUTE_SUSPEND	0x02
+#define	ZIO_REEXECUTE_CANCELLED	0x04
 
 /*
  * The io_trim flags are used to specify the type of TRIM to perform.  They
@@ -646,7 +647,7 @@ extern uint8_t zio_complevel_select(spa_t *spa, enum zio_compress compress,
 
 extern void zio_suspend(spa_t *spa, zio_t *zio, zio_suspend_reason_t);
 extern int zio_resume(spa_t *spa);
-extern void zio_resume_wait(spa_t *spa);
+extern void zio_cancel(spa_t *spa);
 
 extern boolean_t zfs_blkptr_verify(spa_t *spa, const blkptr_t *bp,
     boolean_t config_held, enum blk_verify_flag blk_verify);

lib/libzfs/libzfs_dataset.c

@@ -486,18 +486,28 @@ make_dataset_handle(libzfs_handle_t *hdl, const char *path)
 
 	zhp->zfs_hdl = hdl;
 	(void) strlcpy(zhp->zfs_name, path, sizeof (zhp->zfs_name));
-	if (zcmd_alloc_dst_nvlist(hdl, &zc, 0) != 0) {
-		free(zhp);
-		return (NULL);
-	}
-	if (get_stats_ioctl(zhp, &zc) == -1) {
-		zcmd_free_nvlists(&zc);
-		free(zhp);
-		return (NULL);
-	}
-	if (make_dataset_handle_common(zhp, &zc) == -1) {
-		free(zhp);
-		zhp = NULL;
+
+	if (!hdl->libzfs_force_export) {
+		if (zcmd_alloc_dst_nvlist(hdl, &zc, 0) != 0) {
+			free(zhp);
+			return (NULL);
+		}
+		if (get_stats_ioctl(zhp, &zc) == -1) {
+			zcmd_free_nvlists(&zc);
+			free(zhp);
+			return (NULL);
+		}
+		if (make_dataset_handle_common(zhp, &zc) == -1) {
+			free(zhp);
+			zhp = NULL;
+		}
+	} else {
+		/*
+		 * Called from zpool_disable_datasets(), which uses mount
+		 * entries, so de facto the dataset is a ZFS.
+		 */
+		zhp->zfs_dmustats.dds_type = DMU_OST_ZFS;
+		zhp->zfs_type = ZFS_TYPE_FILESYSTEM;
 	}
 	zcmd_free_nvlists(&zc);
 	return (zhp);

lib/libzfs/libzfs_mount.c

@@ -1493,6 +1493,7 @@ zpool_disable_datasets(zpool_handle_t *zhp, boolean_t force)
 	int ret = -1;
 	int flags = (force ? MS_FORCE : 0);
 
+	hdl->libzfs_force_export = force;
 	namelen = strlen(zhp->zpool_name);
 
 	/* Reopen MNTTAB to prevent reading stale data from open file */
@@ -1595,9 +1596,11 @@ zpool_disable_datasets(zpool_handle_t *zhp, boolean_t force)
 			goto out;
 	}
 
-	for (i = 0; i < used; i++) {
-		if (datasets[i])
-			remove_mountpoint(datasets[i]);
+	if (!hdl->libzfs_force_export) {
+		for (i = 0; i < used; i++) {
+			if (datasets[i])
+				remove_mountpoint(datasets[i]);
+		}
 	}
 
 	ret = 0;