diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c
index 49e82131141f0a..fdbef0d89d3e62 100644
--- a/sound/soc/soc-core.c
+++ b/sound/soc/soc-core.c
@@ -946,9 +946,9 @@ static void soc_remove_dai(struct snd_soc_dai *dai, int order)
 {
 	int err;
 
-	if (dai && dai->probed &&
-			dai->driver->remove_order == order) {
-		if (dai->driver->remove) {
+	if (dai && dai->probed) {
+		if (dai->driver && dai->driver->remove_order == order &&
+		    dai->driver->remove) {
 			err = dai->driver->remove(dai);
 			if (err < 0)
 				dev_err(dai->dev,
diff --git a/sound/soc/soc-topology.c b/sound/soc/soc-topology.c
index 00ab616f1bfc67..fa72c892e461a1 100644
--- a/sound/soc/soc-topology.c
+++ b/sound/soc/soc-topology.c
@@ -508,6 +508,7 @@ static void remove_dai(struct snd_soc_component *comp,
 {
 	struct snd_soc_dai_driver *dai_drv =
 		container_of(dobj, struct snd_soc_dai_driver, dobj);
+	struct snd_soc_dai *dai;
 
 	if (pass != SOC_TPLG_PASS_PCM_DAI)
 		return;
@@ -515,6 +516,10 @@ static void remove_dai(struct snd_soc_component *comp,
 	if (dobj->ops && dobj->ops->dai_unload)
 		dobj->ops->dai_unload(comp, dobj);
 
+	list_for_each_entry(dai, &comp->dai_list, list)
+		if (dai->driver == dai_drv)
+			dai->driver = NULL;
+
 	kfree(dai_drv->name);
 	list_del(&dobj->list);
 	kfree(dai_drv);