Skip to content

Commit

Permalink
More SourceLoader functionality
Browse files Browse the repository at this point in the history
  • Loading branch information
@MikePopoloski
MikePopoloski committed Jul 9, 2023
1 parent 9c5a19e commit 5820a4a
Show file tree
Hide file tree
Showing 2 changed files with 174 additions and 48 deletions.
57 changes: 55 additions & 2 deletions include/slang/driver/SourceLoader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ namespace slang {

class Bag;
class SourceManager;
struct SourceBuffer;

} // namespace slang

Expand All @@ -31,28 +32,75 @@ class SyntaxTree;

namespace slang::driver {

struct SourceOptions {
/// Specifies options used when loading source files.
struct SLANG_EXPORT SourceOptions {
/// The number of threads to use for loading and parsing.
std::optional<uint32_t> numThreads;

/// If set to true, all source files will be treated as part of a single
/// compilation unit, meaning all of their text will be merged together.
bool singleUnit;

/// If true, only perform linting of code, don't try to elaborate a full hierarchy.
bool onlyLint;

/// If true, library files will inherit macro definitions from primary source files.
bool librariesInheritMacros;
};

/// @brief Handles loading and parsing of groups of source files
///
/// This class handles high-level descriptions of how to load and parse source files,
/// such as via library mapping files or search directories to look in. The actual
/// loading and parsing are delegated to classes like @a SourceManager and @a SyntaxTree.
class SLANG_EXPORT SourceLoader {
public:
using SyntaxTreeList = std::vector<std::shared_ptr<syntax::SyntaxTree>>;
using ErrorCallback = std::function<void(const std::string&)>;

/// Constructs a new instance of the SourceLoader class.
SourceLoader(SourceManager& sourceManager, const Bag& optionBag, ErrorCallback errorCallback);

/// @brief Adds files to be loaded, specified via the given @a pattern.
///
/// All of the files that match the pattern will be added for loading.
/// If no files match and the pattern is actually just a specific filename
/// an error will be issued.
void addFiles(std::string_view pattern);
bool addLibraryMaps(std::string_view pattern);

/// @brief Adds library files to be loaded, specified via the given @a pattern.
///
/// All of the files that match the pattern will be added for loading.
/// If no files match and the pattern is actually just a specific filename
/// an error will be issued.
///
/// Library files differ from regular source files in that they are only
/// considered "used" if referenced in the main source and their modules
/// are not automatically instantiated.
void addLibraryFiles(std::string_view libraryName, std::string_view pattern);

/// @brief Adds directories in which to search for library module files.
///
/// A search for a library module occurs when there are instantiations found
/// for unknown modules (or interfaces or programs). The given directories
/// will be searched for files with the missing module's name plus any registered
/// search extensions.
void addSearchDirectories(std::span<const std::string> directories);

/// @brief Adds extensions used to search for library module files.
///
/// A search for a library module occurs when there are instantiations found
/// for unknown modules (or interfaces or programs). The search will be for
/// files with the given @a extensions.
///
/// Note that the extensions ".v" and ".sv" are always automatically included
/// in the search set.
void addSearchExtensions(std::span<const std::string> extensions);

bool addLibraryMaps(std::string_view pattern);
const SyntaxTreeList& getLibraryMaps() const { return libraryMaps; }

/// Loads and parses all of the source files that have been added to the loader.
SyntaxTreeList loadAndParseSources();

private:
Expand All @@ -62,6 +110,9 @@ class SLANG_EXPORT SourceLoader {
};

void createLibrary(const syntax::LibraryDeclarationSyntax& syntax);
void loadSource(const std::filesystem::path& path, bool isLibrary,
std::vector<SourceBuffer>& singleUnitBuffers,
std::vector<SourceBuffer>& deferredLibBuffers, SyntaxTreeList& syntaxTrees);

SourceManager& sourceManager;
const Bag& optionBag;
Expand All @@ -75,6 +126,8 @@ class SLANG_EXPORT SourceLoader {
flat_hash_set<std::string_view> uniqueExtensions;
SyntaxTreeList libraryMaps;
ErrorCallback errorCallback;

static constexpr int MinFilesForThreading = 4;
};

} // namespace slang::driver
165 changes: 119 additions & 46 deletions source/driver/SourceLoader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
//------------------------------------------------------------------------------
#include "slang/driver/SourceLoader.h"

#include <fmt/core.h>

#include "slang/syntax/AllSyntax.h"
#include "slang/syntax/SyntaxTree.h"
#include "slang/text/SourceManager.h"
Expand Down Expand Up @@ -35,7 +37,13 @@ SourceLoader::SourceLoader(SourceManager& sourceManager, const Bag& optionBag,

void SourceLoader::addFiles(std::string_view pattern) {
SmallVector<fs::path> files;
svGlob("", pattern, GlobMode::Files, files);
auto rank = svGlob("", pattern, GlobMode::Files, files);

if (files.empty()) {
if (rank == GlobRank::ExactName)
errorCallback(fmt::format("no such file: '{}'", pattern));
return;
}

filePaths.reserve(filePaths.size() + files.size());
for (auto&& path : files)
Expand Down Expand Up @@ -87,7 +95,13 @@ void SourceLoader::addLibraryFiles(std::string_view, std::string_view pattern) {
// TODO: lookup library

SmallVector<fs::path> files;
svGlob("", pattern, GlobMode::Files, files);
auto rank = svGlob("", pattern, GlobMode::Files, files);

if (files.empty()) {
if (rank == GlobRank::ExactName)
errorCallback(fmt::format("no such file: '{}'", pattern));
return;
}

filePaths.reserve(filePaths.size() + files.size());
for (auto&& path : files)
Expand All @@ -108,73 +122,95 @@ void SourceLoader::addSearchExtensions(std::span<const std::string> extensions)
}

SourceLoader::SyntaxTreeList SourceLoader::loadAndParseSources() {
auto loadSource = [this](const std::filesystem::path& path, bool isLibrary,
std::vector<SourceBuffer>& singleUnitBuffers,
std::vector<SourceBuffer>& deferredLibBuffers,
SyntaxTreeList& results) {
// TODO: Figure out which library this file is in.
const SourceLibrary* library = nullptr;

// Load into memory.
auto buffer = sourceManager.readSource(path, library);
if (!buffer) {
// TODO: error checking
}
SyntaxTreeList syntaxTrees;
std::span<const DefineDirectiveSyntax* const> inheritedMacros;

if (!isLibrary && srcOptions.singleUnit) {
// If this file was directly specified (i.e. not via
// a library mapping) and we're in single-unit mode,
// collect it for later parsing.
singleUnitBuffers.push_back(buffer);
}
else if (srcOptions.librariesInheritMacros) {
// If libraries inherit macros then we can't parse here,
// we need to wait for the main compilation unit to be
// parsed.
SLANG_ASSERT(isLibrary);
deferredLibBuffers.push_back(buffer);
}
else {
// Otherwise we can parse right away.
auto tree = SyntaxTree::fromBuffer(buffer, sourceManager, optionBag);
if (isLibrary || srcOptions.onlyLint)
auto parseSingleUnit = [&](std::span<const SourceBuffer> buffers) {
// If we waited to parse direct buffers due to wanting a single unit, parse that unit now.
if (!buffers.empty()) {
auto tree = SyntaxTree::fromBuffers(buffers, sourceManager, optionBag);
if (srcOptions.onlyLint)
tree->isLibrary = true;

results.emplace_back(std::move(tree));
syntaxTrees.emplace_back(std::move(tree));
inheritedMacros = syntaxTrees.back()->getDefinedMacros();
}
};

SyntaxTreeList syntaxTrees;
std::span<const DefineDirectiveSyntax* const> inheritedMacros;

if (filePaths.size() > 4 && srcOptions.numThreads != 1u) {
if (filePaths.size() >= MinFilesForThreading && srcOptions.numThreads != 1u) {
// If there are enough files to parse and the user hasn't disabled
// the use of threads, do the parsing via a thread pool.
ThreadPool threadPool(srcOptions.numThreads.value_or(0u));

// TODO:
std::vector<SourceBuffer> singleUnitBuffers;
std::vector<SourceBuffer> deferredLibBuffers;
std::mutex mut;

// Load all source files that were specified on the command line
// or via library maps.
threadPool.pushLoop(size_t(0), filePaths.size(), [&](size_t start, size_t end) {
SyntaxTreeList localTrees;
std::vector<SourceBuffer> localSingleUnitBufs;
std::vector<SourceBuffer> localDeferredLibBufs;

const size_t count = end - start;
localTrees.reserve(count);
localSingleUnitBufs.reserve(count);
localDeferredLibBufs.reserve(count);

for (size_t i = start; i < end; i++) {
auto& [path, isLibrary] = filePaths[i];
loadSource(path, isLibrary, localSingleUnitBufs, localDeferredLibBufs, localTrees);
}

// Merge our local results into the shared lists.
std::unique_lock lock(mut);
syntaxTrees.insert(syntaxTrees.end(), localTrees.begin(), localTrees.end());
singleUnitBuffers.insert(singleUnitBuffers.end(), localSingleUnitBufs.begin(),
localSingleUnitBufs.end());
deferredLibBuffers.insert(deferredLibBuffers.end(), localDeferredLibBufs.begin(),
localDeferredLibBufs.end());
});
threadPool.waitForAll();

parseSingleUnit(singleUnitBuffers);

// If we deferred libraries due to wanting to inherit macros, parse them now.
threadPool.pushLoop(size_t(0), deferredLibBuffers.size(), [&](size_t start, size_t end) {
SyntaxTreeList localTrees;
localTrees.reserve(end - start);

for (size_t i = start; i < end; i++) {
auto tree = SyntaxTree::fromBuffer(deferredLibBuffers[i], sourceManager, optionBag,
inheritedMacros);
tree->isLibrary = true;
localTrees.emplace_back(std::move(tree));
}

std::unique_lock lock(mut);
syntaxTrees.insert(syntaxTrees.end(), localTrees.begin(), localTrees.end());
});
threadPool.waitForAll();
}
else {
std::vector<SourceBuffer> singleUnitBuffers;
std::vector<SourceBuffer> deferredLibBuffers;

const size_t count = filePaths.size();
syntaxTrees.reserve(count);
singleUnitBuffers.reserve(count);
deferredLibBuffers.reserve(count);

// Load all source files that were specified on the command line
// or via library maps.
for (auto& [path, isLibrary] : filePaths)
loadSource(path, isLibrary, singleUnitBuffers, deferredLibBuffers, syntaxTrees);

// If we waited to parse direct buffers due to wanting a single unit, parse that unit now.
if (!singleUnitBuffers.empty()) {
auto tree = SyntaxTree::fromBuffers(singleUnitBuffers, sourceManager, optionBag);
if (srcOptions.onlyLint)
tree->isLibrary = true;

syntaxTrees.emplace_back(std::move(tree));
}
parseSingleUnit(singleUnitBuffers);

// If we deferred libraries due to wanting to inherit macros, parse them now.
if (!deferredLibBuffers.empty()) {
inheritedMacros = syntaxTrees.back()->getDefinedMacros();
syntaxTrees.reserve(syntaxTrees.size() + deferredLibBuffers.size());
for (auto& buffer : deferredLibBuffers) {
auto tree = SyntaxTree::fromBuffer(buffer, sourceManager, optionBag,
inheritedMacros);
Expand Down Expand Up @@ -311,4 +347,41 @@ void SourceLoader::createLibrary(const LibraryDeclarationSyntax& syntax) {
libraries.emplace(libName, Library{libName, std::move(files)});
}

void SourceLoader::loadSource(const std::filesystem::path& path, bool isLibrary,
std::vector<SourceBuffer>& singleUnitBuffers,
std::vector<SourceBuffer>& deferredLibBuffers,
SyntaxTreeList& syntaxTrees) {
// TODO: Figure out which library this file is in.
const SourceLibrary* library = nullptr;

// Load into memory.
auto buffer = sourceManager.readSource(path, library);
if (!buffer) {
errorCallback(fmt::format("unable to open file: '{}'\n", getU8Str(path)));
return;
}

if (!isLibrary && srcOptions.singleUnit) {
// If this file was directly specified (i.e. not via
// a library mapping) and we're in single-unit mode,
// collect it for later parsing.
singleUnitBuffers.push_back(buffer);
}
else if (srcOptions.librariesInheritMacros) {
// If libraries inherit macros then we can't parse here,
// we need to wait for the main compilation unit to be
// parsed.
SLANG_ASSERT(isLibrary);
deferredLibBuffers.push_back(buffer);
}
else {
// Otherwise we can parse right away.
auto tree = SyntaxTree::fromBuffer(buffer, sourceManager, optionBag);
if (isLibrary || srcOptions.onlyLint)
tree->isLibrary = true;

syntaxTrees.emplace_back(std::move(tree));
}
};

} // namespace slang::driver

0 comments on commit 5820a4a

Please sign in to comment.