Enabled VSTs on Qt 6 (#7273)

* enabled VST support for Qt 6 builds
* Note : Embedding on QT6 will be buggy on linux as a result of using qt embedding, which unfortunately is a qt bug which hasn't been resolved.

CMakeLists.txt

@@ -175,11 +175,8 @@ check_library_exists(rt shm_open "" LMMS_HAVE_LIBRT)
 LIST(APPEND CMAKE_PREFIX_PATH "${CMAKE_INSTALL_PREFIX}")
 
 if(WANT_QT6)
-	message("Building with Qt6 is experimental")
 	set(QT_VERSION_MAJOR 6)
 	set(LMMS_QT_MIN_VERSION 6.0.0)
-	set(WANT_VST OFF)
-	set(STATUS_VST "VSTs not supported with Qt 6")
 	set(STATUS_QT6 "Enabled")
 else()
 	set(QT_VERSION_MAJOR 5)
@@ -215,8 +212,7 @@ if(WANT_QT6)
     set(QT_LIBRARIES ${QT_LIBRARIES} Qt6::Core5Compat)
 endif()
 
-IF(LMMS_BUILD_LINUX AND WANT_VST)
-	# Todo, enable qt6 for vst support
+IF(LMMS_BUILD_LINUX AND WANT_VST AND NOT WANT_QT6)
     FIND_PACKAGE(Qt5 COMPONENTS X11Extras REQUIRED)
     LIST(APPEND QT_LIBRARIES Qt5::X11Extras)
 ENDIF()

plugins/VstBase/VstPlugin.cpp

@@ -34,7 +34,7 @@
 #include <QLocale>
 #include <QTemporaryFile>
 
-#ifdef LMMS_BUILD_LINUX
+#if defined(LMMS_BUILD_LINUX) && (QT_VERSION < QT_VERSION_CHECK(6,0,0))
 #	include <QX11Info>
 #	include <X11EmbedContainer.h>
 #endif
@@ -416,7 +416,7 @@ bool VstPlugin::processMessage( const message & _m )
 					(LONG_PTR) gui::getGUI()->mainWindow()->winId() );
 #endif
 
-#ifdef LMMS_BUILD_LINUX
+#if defined(LMMS_BUILD_LINUX) && (QT_VERSION < QT_VERSION_CHECK(6,0,0))
 			XSetTransientForHint( QX11Info::display(),
 					m_pluginWindowID,
 					gui::getGUI()->mainWindow()->winId() );
@@ -773,7 +773,7 @@ void VstPlugin::createUI( QWidget * parent )
 	} else
 #endif
 
-#ifdef LMMS_BUILD_LINUX
+#if defined(LMMS_BUILD_LINUX) && (QT_VERSION < QT_VERSION_CHECK(6,0,0))
 	if (m_embedMethod == "xembed" )
 	{
 		if (parent)

plugins/VstBase/vstbase/CMakeLists.txt

@@ -15,6 +15,6 @@ TARGET_INCLUDE_DIRECTORIES(vstbase
 	PUBLIC ../
 )
 
-IF(LMMS_BUILD_LINUX)
+IF(LMMS_BUILD_LINUX AND NOT WANT_QT6)
 	TARGET_LINK_LIBRARIES(vstbase qx11embedcontainer)
 ENDIF()

src/3rdparty/CMakeLists.txt

@@ -1,6 +1,6 @@
 set_directory_properties(PROPERTIES SYSTEM TRUE)
 
-if(LMMS_BUILD_LINUX AND LMMS_HAVE_VST)
+if(LMMS_BUILD_LINUX AND LMMS_HAVE_VST AND NOT WANT_QT6)
 	set(BUILD_SHARED_LIBS OFF)
 	add_subdirectory(qt5-x11embed)
 ENDIF()

src/core/ConfigManager.cpp

@@ -202,7 +202,7 @@ QStringList ConfigManager::availableVstEmbedMethods()
 #ifdef LMMS_BUILD_WIN32
 	methods.append("win32");
 #endif
-#ifdef LMMS_BUILD_LINUX
+#if defined(LMMS_BUILD_LINUX) && (QT_VERSION < QT_VERSION_CHECK(6,0,0))
 	if (static_cast<QGuiApplication*>(QApplication::instance())->
 		platformName() == "xcb")
 	{