diff --git a/include/slang/driver/SourceLoader.h b/include/slang/driver/SourceLoader.h index e18e3c700..ef89aba6f 100644 --- a/include/slang/driver/SourceLoader.h +++ b/include/slang/driver/SourceLoader.h @@ -22,6 +22,7 @@ namespace slang { class Bag; class SourceManager; +struct SourceBuffer; } // namespace slang @@ -31,28 +32,75 @@ class SyntaxTree; namespace slang::driver { -struct SourceOptions { +/// Specifies options used when loading source files. +struct SLANG_EXPORT SourceOptions { + /// The number of threads to use for loading and parsing. std::optional numThreads; + + /// If set to true, all source files will be treated as part of a single + /// compilation unit, meaning all of their text will be merged together. bool singleUnit; + + /// If true, only perform linting of code, don't try to elaborate a full hierarchy. bool onlyLint; + + /// If true, library files will inherit macro definitions from primary source files. bool librariesInheritMacros; }; +/// @brief Handles loading and parsing of groups of source files +/// +/// This class handles high-level descriptions of how to load and parse source files, +/// such as via library mapping files or search directories to look in. The actual +/// loading and parsing are delegated to classes like @a SourceManager and @a SyntaxTree. class SLANG_EXPORT SourceLoader { public: using SyntaxTreeList = std::vector>; using ErrorCallback = std::function; + /// Constructs a new instance of the SourceLoader class. SourceLoader(SourceManager& sourceManager, const Bag& optionBag, ErrorCallback errorCallback); + /// @brief Adds files to be loaded, specified via the given @a pattern. + /// + /// All of the files that match the pattern will be added for loading. + /// If no files match and the pattern is actually just a specific filename + /// an error will be issued. void addFiles(std::string_view pattern); - bool addLibraryMaps(std::string_view pattern); + + /// @brief Adds library files to be loaded, specified via the given @a pattern. + /// + /// All of the files that match the pattern will be added for loading. + /// If no files match and the pattern is actually just a specific filename + /// an error will be issued. + /// + /// Library files differ from regular source files in that they are only + /// considered "used" if referenced in the main source and their modules + /// are not automatically instantiated. void addLibraryFiles(std::string_view libraryName, std::string_view pattern); + + /// @brief Adds directories in which to search for library module files. + /// + /// A search for a library module occurs when there are instantiations found + /// for unknown modules (or interfaces or programs). The given directories + /// will be searched for files with the missing module's name plus any registered + /// search extensions. void addSearchDirectories(std::span directories); + + /// @brief Adds extensions used to search for library module files. + /// + /// A search for a library module occurs when there are instantiations found + /// for unknown modules (or interfaces or programs). The search will be for + /// files with the given @a extensions. + /// + /// Note that the extensions ".v" and ".sv" are always automatically included + /// in the search set. void addSearchExtensions(std::span extensions); + bool addLibraryMaps(std::string_view pattern); const SyntaxTreeList& getLibraryMaps() const { return libraryMaps; } + /// Loads and parses all of the source files that have been added to the loader. SyntaxTreeList loadAndParseSources(); private: @@ -62,6 +110,9 @@ class SLANG_EXPORT SourceLoader { }; void createLibrary(const syntax::LibraryDeclarationSyntax& syntax); + void loadSource(const std::filesystem::path& path, bool isLibrary, + std::vector& singleUnitBuffers, + std::vector& deferredLibBuffers, SyntaxTreeList& syntaxTrees); SourceManager& sourceManager; const Bag& optionBag; @@ -75,6 +126,8 @@ class SLANG_EXPORT SourceLoader { flat_hash_set uniqueExtensions; SyntaxTreeList libraryMaps; ErrorCallback errorCallback; + + static constexpr int MinFilesForThreading = 4; }; } // namespace slang::driver diff --git a/source/driver/SourceLoader.cpp b/source/driver/SourceLoader.cpp index 15998a7ee..96533096b 100644 --- a/source/driver/SourceLoader.cpp +++ b/source/driver/SourceLoader.cpp @@ -7,6 +7,8 @@ //------------------------------------------------------------------------------ #include "slang/driver/SourceLoader.h" +#include + #include "slang/syntax/AllSyntax.h" #include "slang/syntax/SyntaxTree.h" #include "slang/text/SourceManager.h" @@ -35,7 +37,13 @@ SourceLoader::SourceLoader(SourceManager& sourceManager, const Bag& optionBag, void SourceLoader::addFiles(std::string_view pattern) { SmallVector files; - svGlob("", pattern, GlobMode::Files, files); + auto rank = svGlob("", pattern, GlobMode::Files, files); + + if (files.empty()) { + if (rank == GlobRank::ExactName) + errorCallback(fmt::format("no such file: '{}'", pattern)); + return; + } filePaths.reserve(filePaths.size() + files.size()); for (auto&& path : files) @@ -87,7 +95,13 @@ void SourceLoader::addLibraryFiles(std::string_view, std::string_view pattern) { // TODO: lookup library SmallVector files; - svGlob("", pattern, GlobMode::Files, files); + auto rank = svGlob("", pattern, GlobMode::Files, files); + + if (files.empty()) { + if (rank == GlobRank::ExactName) + errorCallback(fmt::format("no such file: '{}'", pattern)); + return; + } filePaths.reserve(filePaths.size() + files.size()); for (auto&& path : files) @@ -108,73 +122,95 @@ void SourceLoader::addSearchExtensions(std::span extensions) } SourceLoader::SyntaxTreeList SourceLoader::loadAndParseSources() { - auto loadSource = [this](const std::filesystem::path& path, bool isLibrary, - std::vector& singleUnitBuffers, - std::vector& deferredLibBuffers, - SyntaxTreeList& results) { - // TODO: Figure out which library this file is in. - const SourceLibrary* library = nullptr; - - // Load into memory. - auto buffer = sourceManager.readSource(path, library); - if (!buffer) { - // TODO: error checking - } + SyntaxTreeList syntaxTrees; + std::span inheritedMacros; - if (!isLibrary && srcOptions.singleUnit) { - // If this file was directly specified (i.e. not via - // a library mapping) and we're in single-unit mode, - // collect it for later parsing. - singleUnitBuffers.push_back(buffer); - } - else if (srcOptions.librariesInheritMacros) { - // If libraries inherit macros then we can't parse here, - // we need to wait for the main compilation unit to be - // parsed. - SLANG_ASSERT(isLibrary); - deferredLibBuffers.push_back(buffer); - } - else { - // Otherwise we can parse right away. - auto tree = SyntaxTree::fromBuffer(buffer, sourceManager, optionBag); - if (isLibrary || srcOptions.onlyLint) + auto parseSingleUnit = [&](std::span buffers) { + // If we waited to parse direct buffers due to wanting a single unit, parse that unit now. + if (!buffers.empty()) { + auto tree = SyntaxTree::fromBuffers(buffers, sourceManager, optionBag); + if (srcOptions.onlyLint) tree->isLibrary = true; - results.emplace_back(std::move(tree)); + syntaxTrees.emplace_back(std::move(tree)); + inheritedMacros = syntaxTrees.back()->getDefinedMacros(); } }; - SyntaxTreeList syntaxTrees; - std::span inheritedMacros; - - if (filePaths.size() > 4 && srcOptions.numThreads != 1u) { + if (filePaths.size() >= MinFilesForThreading && srcOptions.numThreads != 1u) { // If there are enough files to parse and the user hasn't disabled // the use of threads, do the parsing via a thread pool. ThreadPool threadPool(srcOptions.numThreads.value_or(0u)); - // TODO: + std::vector singleUnitBuffers; + std::vector deferredLibBuffers; + std::mutex mut; + + // Load all source files that were specified on the command line + // or via library maps. + threadPool.pushLoop(size_t(0), filePaths.size(), [&](size_t start, size_t end) { + SyntaxTreeList localTrees; + std::vector localSingleUnitBufs; + std::vector localDeferredLibBufs; + + const size_t count = end - start; + localTrees.reserve(count); + localSingleUnitBufs.reserve(count); + localDeferredLibBufs.reserve(count); + + for (size_t i = start; i < end; i++) { + auto& [path, isLibrary] = filePaths[i]; + loadSource(path, isLibrary, localSingleUnitBufs, localDeferredLibBufs, localTrees); + } + + // Merge our local results into the shared lists. + std::unique_lock lock(mut); + syntaxTrees.insert(syntaxTrees.end(), localTrees.begin(), localTrees.end()); + singleUnitBuffers.insert(singleUnitBuffers.end(), localSingleUnitBufs.begin(), + localSingleUnitBufs.end()); + deferredLibBuffers.insert(deferredLibBuffers.end(), localDeferredLibBufs.begin(), + localDeferredLibBufs.end()); + }); + threadPool.waitForAll(); + + parseSingleUnit(singleUnitBuffers); + + // If we deferred libraries due to wanting to inherit macros, parse them now. + threadPool.pushLoop(size_t(0), deferredLibBuffers.size(), [&](size_t start, size_t end) { + SyntaxTreeList localTrees; + localTrees.reserve(end - start); + + for (size_t i = start; i < end; i++) { + auto tree = SyntaxTree::fromBuffer(deferredLibBuffers[i], sourceManager, optionBag, + inheritedMacros); + tree->isLibrary = true; + localTrees.emplace_back(std::move(tree)); + } + + std::unique_lock lock(mut); + syntaxTrees.insert(syntaxTrees.end(), localTrees.begin(), localTrees.end()); + }); + threadPool.waitForAll(); } else { std::vector singleUnitBuffers; std::vector deferredLibBuffers; + const size_t count = filePaths.size(); + syntaxTrees.reserve(count); + singleUnitBuffers.reserve(count); + deferredLibBuffers.reserve(count); + // Load all source files that were specified on the command line // or via library maps. for (auto& [path, isLibrary] : filePaths) loadSource(path, isLibrary, singleUnitBuffers, deferredLibBuffers, syntaxTrees); - // If we waited to parse direct buffers due to wanting a single unit, parse that unit now. - if (!singleUnitBuffers.empty()) { - auto tree = SyntaxTree::fromBuffers(singleUnitBuffers, sourceManager, optionBag); - if (srcOptions.onlyLint) - tree->isLibrary = true; - - syntaxTrees.emplace_back(std::move(tree)); - } + parseSingleUnit(singleUnitBuffers); // If we deferred libraries due to wanting to inherit macros, parse them now. if (!deferredLibBuffers.empty()) { - inheritedMacros = syntaxTrees.back()->getDefinedMacros(); + syntaxTrees.reserve(syntaxTrees.size() + deferredLibBuffers.size()); for (auto& buffer : deferredLibBuffers) { auto tree = SyntaxTree::fromBuffer(buffer, sourceManager, optionBag, inheritedMacros); @@ -311,4 +347,41 @@ void SourceLoader::createLibrary(const LibraryDeclarationSyntax& syntax) { libraries.emplace(libName, Library{libName, std::move(files)}); } +void SourceLoader::loadSource(const std::filesystem::path& path, bool isLibrary, + std::vector& singleUnitBuffers, + std::vector& deferredLibBuffers, + SyntaxTreeList& syntaxTrees) { + // TODO: Figure out which library this file is in. + const SourceLibrary* library = nullptr; + + // Load into memory. + auto buffer = sourceManager.readSource(path, library); + if (!buffer) { + errorCallback(fmt::format("unable to open file: '{}'\n", getU8Str(path))); + return; + } + + if (!isLibrary && srcOptions.singleUnit) { + // If this file was directly specified (i.e. not via + // a library mapping) and we're in single-unit mode, + // collect it for later parsing. + singleUnitBuffers.push_back(buffer); + } + else if (srcOptions.librariesInheritMacros) { + // If libraries inherit macros then we can't parse here, + // we need to wait for the main compilation unit to be + // parsed. + SLANG_ASSERT(isLibrary); + deferredLibBuffers.push_back(buffer); + } + else { + // Otherwise we can parse right away. + auto tree = SyntaxTree::fromBuffer(buffer, sourceManager, optionBag); + if (isLibrary || srcOptions.onlyLint) + tree->isLibrary = true; + + syntaxTrees.emplace_back(std::move(tree)); + } +}; + } // namespace slang::driver