diff --git a/DependencyInjection/FOSOAuthServerExtension.php b/DependencyInjection/FOSOAuthServerExtension.php
index f9aeab42..d4230a3d 100644
--- a/DependencyInjection/FOSOAuthServerExtension.php
+++ b/DependencyInjection/FOSOAuthServerExtension.php
@@ -85,10 +85,12 @@ public function load(array $configs, ContainerBuilder $container)
             $ormEntityManagerDefinition->setFactory([new Reference('doctrine'), 'getManager']);
         }
 
-        if (!empty($config['authorize'])) {
-            $this->loadAuthorize($config['authorize'], $container, $loader);
+        if (empty($config['authorize'])) {
+            return;
         }
 
+        $this->loadAuthorize($config['authorize'], $container, $loader);
+
         // Authorize form factory definition
         // TODO: Go back to xml configuration when bumping the requirement to Symfony >=2.6
         $authorizeFormDefinition = $container->getDefinition('fos_oauth_server.authorize.form');