diff --git a/esrally/config.py b/esrally/config.py
index 694520b61..7d6df3300 100644
--- a/esrally/config.py
+++ b/esrally/config.py
@@ -106,6 +106,8 @@ def auto_load_local_config(base_config, additional_sections=None, config_file_cl
 
 
 class Config:
+    EARLIEST_SUPPORTED_VERSION = 12
+
     CURRENT_CONFIG_VERSION = 15
 
     """
@@ -586,6 +588,10 @@ def ask_property(self, prompt, mandatory=True, check_path_exists=False, check_pa
 
 
 def migrate(config_file, current_version, target_version, out=print, i=input):
+    if current_version < Config.EARLIEST_SUPPORTED_VERSION:
+        raise ConfigError("The config file in {} is too old. Please delete it and reconfigure Rally from scratch with {} configure."
+                          .format(config_file.location, PROGRAM_NAME))
+
     prompter = Prompter(i=i, o=out, assume_defaults=False)
     logger.info("Upgrading configuration from version [%s] to [%s]." % (current_version, target_version))
     # Something is really fishy. We don't want to downgrade the configuration.
@@ -596,178 +602,6 @@ def migrate(config_file, current_version, target_version, out=print, i=input):
     config_file.backup()
     config = config_file.load(interpolation=None)
 
-    if current_version == 0 and target_version > current_version:
-        logger.info("Migrating config from version [0] to [1]")
-        current_version = 1
-        config["meta"] = {}
-        config["meta"]["config.version"] = str(current_version)
-        # in version 1 we changed some directories from being absolute to being relative
-        config["system"]["log.root.dir"] = "logs"
-        config["provisioning"]["local.install.dir"] = "install"
-        config["reporting"]["report.base.dir"] = "reports"
-    if current_version == 1 and target_version > current_version:
-        logger.info("Migrating config from version [1] to [2]")
-        current_version = 2
-        config["meta"]["config.version"] = str(current_version)
-        # no need to ask the user now if we are about to upgrade to version 4
-        config["reporting"]["datastore.type"] = "in-memory"
-        config["reporting"]["datastore.host"] = ""
-        config["reporting"]["datastore.port"] = ""
-        config["reporting"]["datastore.secure"] = ""
-        config["reporting"]["datastore.user"] = ""
-        config["reporting"]["datastore.password"] = ""
-        config["system"]["env.name"] = "local"
-    if current_version == 2 and target_version > current_version:
-        logger.info("Migrating config from version [2] to [3]")
-        current_version = 3
-        config["meta"]["config.version"] = str(current_version)
-        # Remove obsolete settings
-        config["reporting"].pop("report.base.dir")
-        config["reporting"].pop("output.html.report.filename")
-    if current_version == 3 and target_version > current_version:
-        root_dir = config["system"]["root.dir"]
-        out(
-            """
-            *****************************************************************************************
-
-            You have an old configuration of Rally. Rally has now a much simpler setup
-            routine which will autodetect lots of settings for you and it also does not
-            require you to setup a metrics store anymore.
-
-            Rally will now migrate your configuration but if you don't need advanced features
-            like a metrics store, then you should delete the configuration directory:
-
-              rm -rf {0}
-
-            and then rerun Rally's configuration routine:
-
-              {1} configure
-
-            Please also note you have {2:.1f} GB of data in your current benchmark directory at
-
-              {3}
-
-            You might want to clean up this directory also.
-
-            For more details please see {4}
-
-            *****************************************************************************************
-
-            Pausing for 10 seconds to let you consider this message.
-            """.format(config_file.config_dir,
-                       PROGRAM_NAME,
-                       convert.bytes_to_gb(io.get_size(root_dir)),
-                       root_dir,
-                       console.format.link("https://github.com/elastic/rally/blob/master/CHANGELOG.md#030")))
-        time.sleep(10)
-        logger.info("Migrating config from version [3] to [4]")
-        current_version = 4
-        config["meta"]["config.version"] = str(current_version)
-        if len(config["reporting"]["datastore.host"]) > 0:
-            config["reporting"]["datastore.type"] = "elasticsearch"
-        else:
-            config["reporting"]["datastore.type"] = "in-memory"
-        # Remove obsolete settings
-        config["build"].pop("maven.bin")
-        config["benchmarks"].pop("metrics.stats.disk.device")
-
-    if current_version == 4 and target_version > current_version:
-        config["tracks"] = {}
-        config["tracks"]["default.url"] = "https://github.com/elastic/rally-tracks"
-        current_version = 5
-        config["meta"]["config.version"] = str(current_version)
-
-    if current_version == 5 and target_version > current_version:
-        config["defaults"] = {}
-        config["defaults"]["preserve_benchmark_candidate"] = str(False)
-        current_version = 6
-        config["meta"]["config.version"] = str(current_version)
-
-    if current_version == 6 and target_version > current_version:
-        # Remove obsolete settings
-        config.pop("provisioning")
-        config["system"].pop("log.root.dir")
-        current_version = 7
-        config["meta"]["config.version"] = str(current_version)
-
-    if current_version == 7 and target_version > current_version:
-        # move [system][root.dir] to [node][root.dir]
-        if "node" not in config:
-            config["node"] = {}
-        config["node"]["root.dir"] = config["system"].pop("root.dir")
-        # also move all references!
-        for section in config:
-            for k, v in config[section].items():
-                config[section][k] = v.replace("${system:root.dir}", "${node:root.dir}")
-        current_version = 8
-        config["meta"]["config.version"] = str(current_version)
-    if current_version == 8 and target_version > current_version:
-        config["teams"] = {}
-        config["teams"]["default.url"] = "https://github.com/elastic/rally-teams"
-        current_version = 9
-        config["meta"]["config.version"] = str(current_version)
-    if current_version == 9 and target_version > current_version:
-        config["distributions"] = {}
-        config["distributions"]["release.1.url"] = "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-" \
-                                                   "{{VERSION}}.tar.gz"
-        config["distributions"]["release.2.url"] = "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/" \
-                                                   "distribution/tar/elasticsearch/{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz"
-        config["distributions"]["release.url"] = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz"
-        config["distributions"]["release.cache"] = "true"
-        current_version = 10
-        config["meta"]["config.version"] = str(current_version)
-    if current_version == 10 and target_version > current_version:
-        config["runtime"]["java.home"] = config["runtime"].pop("java8.home")
-        current_version = 11
-        config["meta"]["config.version"] = str(current_version)
-    if current_version == 11 and target_version > current_version:
-        # As this is a rather complex migration, we log more than usual to understand potential migration problems better.
-        if "source" in config:
-            if "local.src.dir" in config["source"]:
-                previous_root = config["source"].pop("local.src.dir")
-                logger.info("Set [source][local.src.dir] to [%s]." % previous_root)
-                # if this directory was Rally's default location, then move it on the file system to allow for checkouts of plugins
-                # in the sibling directory.
-                if previous_root == os.path.join(config["node"]["root.dir"], "src"):
-                    new_root_dir_all_sources = previous_root
-                    new_es_sub_dir = "elasticsearch"
-                    new_root = os.path.join(new_root_dir_all_sources, new_es_sub_dir)
-                    # only attempt to move if the directory exists. It may be possible that users never ran a source benchmark although they
-                    # have configured it. In that case the source directory will not yet exist.
-                    if io.exists(previous_root):
-                        logger.info("Previous source directory was at Rally's default location [%s]. Moving to [%s]."
-                                    % (previous_root, new_root))
-                        try:
-                            # we need to do this in two steps as we need to move the sources to a subdirectory
-                            tmp_path = io.normalize_path(os.path.join(new_root_dir_all_sources, os.pardir, "tmp_src_mig"))
-                            os.rename(previous_root, tmp_path)
-                            io.ensure_dir(new_root)
-                            os.rename(tmp_path, new_root)
-                        except OSError:
-                            logger.exception("Could not move source directory from [%s] to [%s]." % (previous_root, new_root))
-                            # A warning is sufficient as Rally should just do a fresh checkout if moving did not work.
-                            console.warn("Elasticsearch source directory could not be moved from [%s] to [%s]. Please check the logs."
-                                         % (previous_root, new_root))
-                    else:
-                        logger.info("Source directory is configured at Rally's default location [%s] but does not exist yet."
-                                    % previous_root)
-                else:
-                    logger.info("Previous source directory was the custom directory [%s]." % previous_root)
-                    new_root_dir_all_sources = io.normalize_path(os.path.join(previous_root, os.path.pardir))
-                    # name of the elasticsearch project directory.
-                    new_es_sub_dir = io.basename(previous_root)
-
-                logger.info("Setting [node][src.root.dir] to [%s]." % new_root_dir_all_sources)
-                config["node"]["src.root.dir"] = new_root_dir_all_sources
-                logger.info("Setting [source][elasticsearch.src.subdir] to [%s]" % new_es_sub_dir)
-                config["source"]["elasticsearch.src.subdir"] = new_es_sub_dir
-            else:
-                logger.info("Key [local.src.dir] not found. Advancing without changes.")
-        else:
-            logger.info("No section named [source] found in config. Advancing without changes.")
-        current_version = 12
-        config["meta"]["config.version"] = str(current_version)
-
     if current_version == 12 and target_version > current_version:
         # the current configuration allows to benchmark from sources
         if "build" in config and "gradle.bin" in config["build"]:
diff --git a/tests/config_test.py b/tests/config_test.py
index a368b47e3..43ba46d27 100644
--- a/tests/config_test.py
+++ b/tests/config_test.py
@@ -419,11 +419,7 @@ def test_create_advanced_config(self, guess_install_location, guess_java_home, i
 
 
 class ConfigMigrationTests(TestCase):
-    # catch all test, migrations are checked in more detail in the other tests
-    @mock.patch("esrally.utils.io.get_size")
-    @mock.patch("esrally.time.sleep")
-    def test_migrate_from_0_to_latest(self, sleep, get_size):
-        get_size.return_value = 0
+    def test_does_not_migrate_outdated_config(self):
         config_file = InMemoryConfigStore("test")
         sample_config = {
             "system": {
@@ -448,347 +444,45 @@ def test_migrate_from_0_to_latest(self, sleep, get_size):
         }
 
         config_file.store(sample_config)
-        config.migrate(config_file, 0, config.Config.CURRENT_CONFIG_VERSION, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual(str(config.Config.CURRENT_CONFIG_VERSION), config_file.config["meta"]["config.version"])
-
-    def test_migrate_from_2_to_3(self):
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 2
-            },
-            "system": {
-                "root.dir": "in-memory"
-            },
-            "reporting": {
-                "report.base.dir": "/tests/rally/reporting",
-                "output.html.report.filename": "index.html"
-            },
-        }
-
-        config_file.store(sample_config)
-        config.migrate(config_file, 2, 3, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("3", config_file.config["meta"]["config.version"])
-        # Did not delete the section...
-        self.assertTrue("reporting" in config_file.config)
-        # ... but the key
-        self.assertFalse("report.base.dir" in config_file.config["reporting"])
-        self.assertFalse("output.html.report.filename" in config_file.config["reporting"])
+        with self.assertRaisesRegex(config.ConfigError, "The config file.*is too old. Please delete it and reconfigure Rally from scratch"):
+            config.migrate(config_file, config.Config.EARLIEST_SUPPORTED_VERSION - 1, config.Config.CURRENT_CONFIG_VERSION, out=null_output)
 
+    # catch all test, migrations are checked in more detail in the other tests
     @mock.patch("esrally.utils.io.get_size")
     @mock.patch("esrally.time.sleep")
-    def test_migrate_from_3_to_4(self, sleep, get_size):
+    def test_migrate_from_earliest_supported_to_latest(self, sleep, get_size):
         get_size.return_value = 0
         config_file = InMemoryConfigStore("test")
         sample_config = {
             "meta": {
-                "config.version": 3
+                "config.version": config.Config.EARLIEST_SUPPORTED_VERSION
             },
             "system": {
                 "root.dir": "in-memory"
             },
-            "reporting": {
-                "datastore.host": ""
+            "provisioning": {
+
             },
             "build": {
                 "maven.bin": "/usr/local/mvn"
             },
             "benchmarks": {
                 "metrics.stats.disk.device": "/dev/hdd1"
-            }
-        }
-
-        config_file.store(sample_config)
-        config.migrate(config_file, 3, 4, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("4", config_file.config["meta"]["config.version"])
-        # Did not delete the section...
-        self.assertTrue("build" in config_file.config)
-        # ... but the key
-        self.assertFalse("maven.bin" in config_file.config["build"])
-        self.assertTrue("benchmarks" in config_file.config)
-        self.assertFalse("metrics.stats.disk.device" in config_file.config["benchmarks"])
-        self.assertEqual("in-memory", config_file.config["reporting"]["datastore.type"])
-
-    def test_migrate_from_4_to_5(self):
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 4
-            }
-        }
-        config_file.store(sample_config)
-        config.migrate(config_file, 4, 5, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("5", config_file.config["meta"]["config.version"])
-        self.assertTrue("tracks" in config_file.config)
-        self.assertEqual("https://github.com/elastic/rally-tracks", config_file.config["tracks"]["default.url"])
-
-    def test_migrate_from_5_to_6(self):
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 5
-            }
-        }
-        config_file.store(sample_config)
-        config.migrate(config_file, 5, 6, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("6", config_file.config["meta"]["config.version"])
-        self.assertTrue("defaults" in config_file.config)
-        self.assertEqual("False", config_file.config["defaults"]["preserve_benchmark_candidate"])
-
-    def test_migrate_from_6_to_7(self):
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 6
-            },
-            "system": {
-                "log.root.dir": "logs"
-            },
-            "provisioning": {
-                "local.install.dir": "install"
-            },
-        }
-        config_file.store(sample_config)
-        config.migrate(config_file, 6, 7, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("7", config_file.config["meta"]["config.version"])
-        self.assertTrue("provisioning" not in config_file.config)
-        self.assertTrue("log.root.dir" not in config_file.config["system"])
-
-    def test_migrate_from_7_to_8(self):
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 7
-            },
-            "system": {
-                "root.dir": "~/.rally/benchmarks",
-                "environment.name": "local"
-            },
-            "benchmarks": {
-                "local.dataset.cache": "${system:root.dir}/data",
-                "some.other.cache": "/data"
-            }
-        }
-        config_file.store(sample_config)
-        config.migrate(config_file, 7, 8, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("8", config_file.config["meta"]["config.version"])
-        self.assertTrue("root.dir" not in config_file.config["system"])
-        self.assertEqual("~/.rally/benchmarks", config_file.config["node"]["root.dir"])
-        self.assertEqual("local", config_file.config["system"]["environment.name"])
-        self.assertEqual("${node:root.dir}/data", config_file.config["benchmarks"]["local.dataset.cache"])
-        self.assertEqual("/data", config_file.config["benchmarks"]["some.other.cache"])
-
-    def test_migrate_from_8_to_9(self):
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 8
-            },
-            "system": {
-                "root.dir": "~/.rally/benchmarks",
-                "environment.name": "local"
             },
-            "benchmarks": {
-                "local.dataset.cache": "${system:root.dir}/data",
-                "some.other.cache": "/data"
-            }
-        }
-        config_file.store(sample_config)
-        config.migrate(config_file, 8, 9, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("9", config_file.config["meta"]["config.version"])
-        self.assertTrue("teams" in config_file.config)
-        self.assertEqual("https://github.com/elastic/rally-teams", config_file.config["teams"]["default.url"])
-
-    def test_migrate_from_9_to_10(self):
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 9
-            },
-            "system": {
-                "root.dir": "~/.rally/benchmarks",
-                "environment.name": "local"
-            },
-            "benchmarks": {
-                "local.dataset.cache": "${system:root.dir}/data",
-                "some.other.cache": "/data"
-            }
-        }
-        config_file.store(sample_config)
-        config.migrate(config_file, 9, 10, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("10", config_file.config["meta"]["config.version"])
-        self.assertTrue("distributions" in config_file.config)
-        self.assertEqual("https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
-                         config_file.config["distributions"]["release.1.url"])
-        self.assertEqual("https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/"
-                         "{{VERSION}}/elasticsearch-{{VERSION}}.tar.gz",
-                         config_file.config["distributions"]["release.2.url"])
-        self.assertEqual("https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{VERSION}}.tar.gz",
-                         config_file.config["distributions"]["release.url"])
-        self.assertEqual("true",
-                         config_file.config["distributions"]["release.cache"])
-
-    def test_migrate_from_10_to_11(self):
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 10
+            "reporting": {
+                "report.base.dir": "/tests/rally/reporting",
+                "output.html.report.filename": "index.html"
             },
             "runtime": {
                 "java8.home": "/opt/jdk/8",
             }
         }
-        config_file.store(sample_config)
-        config.migrate(config_file, 10, 11, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("11", config_file.config["meta"]["config.version"])
-        self.assertTrue("runtime" in config_file.config)
-        self.assertFalse("java8.home" in config_file.config["runtime"])
-        self.assertEqual("/opt/jdk/8", config_file.config["runtime"]["java.home"])
-
-    @mock.patch("esrally.utils.io.exists")
-    @mock.patch("os.rename")
-    def test_migrate_from_11_to_12_with_default_src_config_repo_checked_out(self, path_rename, path_exists):
-        path_exists.return_value = True
-
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 11
-            },
-            "node": {
-                "root.dir": io.normalize_path("~/.rally/benchmarks")
-            },
-            "source": {
-                "local.src.dir": io.normalize_path("~/.rally/benchmarks/src")
-            }
-        }
-        config_file.store(sample_config)
-        config.migrate(config_file, 11, 12, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("12", config_file.config["meta"]["config.version"])
-        self.assertEqual(io.normalize_path("~/.rally/benchmarks/src"), config_file.config["node"]["src.root.dir"])
-        self.assertEqual("elasticsearch", config_file.config["source"]["elasticsearch.src.subdir"])
-
-        path_rename.assert_has_calls(
-            [
-                mock.call(io.normalize_path("~/.rally/benchmarks/src"), io.normalize_path("~/.rally/benchmarks/tmp_src_mig")),
-                mock.call(io.normalize_path("~/.rally/benchmarks/tmp_src_mig"),
-                          io.normalize_path("~/.rally/benchmarks/src/elasticsearch")),
-             ]
-        )
-
-    @mock.patch("esrally.utils.io.exists")
-    @mock.patch("os.rename")
-    def test_migrate_from_11_to_12_with_default_src_config_repo_not_checked_out(self, path_rename, path_exists):
-        path_exists.return_value = False
-
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 11
-            },
-            "node": {
-                "root.dir": io.normalize_path("~/.rally/benchmarks")
-            },
-            "source": {
-                "local.src.dir": io.normalize_path("~/.rally/benchmarks/src")
-            }
-        }
-        config_file.store(sample_config)
-        config.migrate(config_file, 11, 12, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("12", config_file.config["meta"]["config.version"])
-        self.assertEqual(io.normalize_path("~/.rally/benchmarks/src"), config_file.config["node"]["src.root.dir"])
-        self.assertEqual("elasticsearch", config_file.config["source"]["elasticsearch.src.subdir"])
-        # did all the migrations but nothing moved
-        path_rename.assert_not_called()
-
-    def test_migrate_from_11_to_12_without_src_config(self):
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 11
-            },
-            "node": {
-                "root.dir": "~/.rally/benchmarks"
-            }
-        }
-        config_file.store(sample_config)
-        config.migrate(config_file, 11, 12, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("12", config_file.config["meta"]["config.version"])
-        self.assertFalse("src.root.dir" in config_file.config["node"])
-
-    def test_migrate_from_11_to_12_with_partial_src_config(self):
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 11
-            },
-            "node": {
-                "root.dir": "~/.rally/benchmarks"
-            },
-            "source": {
-                # a source config section without any keys should be treated like a missing source config section
-            }
-        }
-        config_file.store(sample_config)
-        config.migrate(config_file, 11, 12, out=null_output)
-
-        self.assertTrue(config_file.backup_created)
-        self.assertEqual("12", config_file.config["meta"]["config.version"])
-        self.assertFalse("src.root.dir" in config_file.config["node"])
-        self.assertFalse("elasticsearch.src.subdir" in config_file.config["source"])
 
-    @mock.patch("esrally.utils.io.exists")
-    @mock.patch("os.rename")
-    def test_migrate_from_11_to_12_with_custom_src_config(self, path_rename, path_exists):
-        path_exists.return_value = False
-
-        config_file = InMemoryConfigStore("test")
-        sample_config = {
-            "meta": {
-                "config.version": 11
-            },
-            "node": {
-                "root.dir": io.normalize_path("~/.rally/benchmarks")
-            },
-            "source": {
-                "local.src.dir": io.normalize_path("~/Projects/elasticsearch/master/es")
-            }
-        }
         config_file.store(sample_config)
-        config.migrate(config_file, 11, 12, out=null_output)
+        config.migrate(config_file, config.Config.EARLIEST_SUPPORTED_VERSION, config.Config.CURRENT_CONFIG_VERSION, out=null_output)
 
         self.assertTrue(config_file.backup_created)
-        self.assertEqual("12", config_file.config["meta"]["config.version"])
-        self.assertEqual(io.normalize_path("~/Projects/elasticsearch/master"), config_file.config["node"]["src.root.dir"])
-        self.assertEqual("es", config_file.config["source"]["elasticsearch.src.subdir"])
-        # did all the migrations but nothing moved
-        path_rename.assert_not_called()
+        self.assertEqual(str(config.Config.CURRENT_CONFIG_VERSION), config_file.config["meta"]["config.version"])
 
     def test_migrate_from_12_to_13_without_gradle(self):
         config_file = InMemoryConfigStore("test")