block deleting namespace if it contains a secure variable (#13888)

When we delete a namespace, we check to ensure that there are no non-terminal
jobs or CSI volume, which also covers evals, allocs, etc. Secure variables are
also namespaces, so extend this check to them as well.

Author: tgross

nomad/state/state_store.go

@@ -6352,6 +6352,17 @@ func (s *StateStore) DeleteNamespaces(index uint64, names []string) error {
 				"All CSI volumes in namespace must be deleted before it can be deleted", name, vol.ID)
 		}
 
+		varIter, err := s.getSecureVariablesByNamespaceImpl(txn, nil, name)
+		if err != nil {
+			return err
+		}
+		if varIter.Next() != nil {
+			// unlike job/volume, don't show the path here because the user may
+			// not have List permissions on the secure vars in this namespace
+			return fmt.Errorf("namespace %q contains at least one secure variable. "+
+				"All secure variables in namespace must be deleted before it can be deleted", name)
+		}
+
 		// Delete the namespace
 		if err := txn.Delete(TableNamespaces, existing); err != nil {
 			return fmt.Errorf("namespace deletion failed: %v", err)

nomad/state/state_store_secure_variables.go

@@ -29,7 +29,11 @@ func (s *StateStore) SecureVariables(ws memdb.WatchSet) (memdb.ResultIterator, e
 func (s *StateStore) GetSecureVariablesByNamespace(
 	ws memdb.WatchSet, namespace string) (memdb.ResultIterator, error) {
 	txn := s.db.ReadTxn()
+	return s.getSecureVariablesByNamespaceImpl(txn, ws, namespace)
+}
 
+func (s *StateStore) getSecureVariablesByNamespaceImpl(
+	txn *txn, ws memdb.WatchSet, namespace string) (memdb.ResultIterator, error) {
 	// Walk the entire table.
 	iter, err := txn.Get(TableSecureVariables, indexID+"_prefix", namespace, "")
 	if err != nil {

nomad/state/state_store_test.go

@@ -1044,6 +1044,39 @@ func TestStateStore_DeleteNamespaces_CSIVolumes(t *testing.T) {
 	require.False(t, watchFired(ws))
 }
 
+func TestStateStore_DeleteNamespaces_SecureVariables(t *testing.T) {
+	ci.Parallel(t)
+
+	state := testStateStore(t)
+
+	ns := mock.Namespace()
+	require.NoError(t, state.UpsertNamespaces(1000, []*structs.Namespace{ns}))
+
+	sv := mock.SecureVariableEncrypted()
+	sv.Namespace = ns.Name
+	require.NoError(t, state.UpsertSecureVariables(structs.MsgTypeTestSetup, 1001, []*structs.SecureVariableEncrypted{sv}))
+
+	// Create a watchset so we can test that delete fires the watch
+	ws := memdb.NewWatchSet()
+	_, err := state.NamespaceByName(ws, ns.Name)
+	require.NoError(t, err)
+
+	err = state.DeleteNamespaces(1002, []string{ns.Name})
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "one secure variable")
+	require.False(t, watchFired(ws))
+
+	ws = memdb.NewWatchSet()
+	out, err := state.NamespaceByName(ws, ns.Name)
+	require.NoError(t, err)
+	require.NotNil(t, out)
+
+	index, err := state.Index(TableNamespaces)
+	require.NoError(t, err)
+	require.EqualValues(t, 1000, index)
+	require.False(t, watchFired(ws))
+}
+
 func TestStateStore_Namespaces(t *testing.T) {
 	ci.Parallel(t)