Skip to content

Commit db555d5

Browse files
wlei-llvmwlei-llvm
committed
[SampleFDO] Refactoring to sample reader to support on-demand read profiles for given functions
1 parent 981191a commit db555d5

File tree

2 files changed

+179
-85
lines changed

2 files changed

+179
-85
lines changed

‎llvm/include/llvm/ProfileData/SampleProfReader.h

+39
Original file line numberDiff line numberDiff line change
@@ -380,6 +380,22 @@ class SampleProfileReader {
380380
return sampleprof_error::success;
381381
}
382382

383+
/// Read sample profiles for the given functions. Currently it's only used for
384+
/// extended binary format to load the profiles on-demand.
385+
std::error_code read(const DenseSet<StringRef> &FuncsToUse) {
386+
if (std::error_code EC = read(FuncsToUse, Profiles))
387+
return EC;
388+
return sampleprof_error::success;
389+
};
390+
391+
/// Read sample profiles for the given functions and write them to the given
392+
/// profile map. Currently it's only used for extended binary format to load
393+
/// the profiles on-demand.
394+
virtual std::error_code read(const DenseSet<StringRef> &FuncsToUse,
395+
SampleProfileMap &Profiles) {
396+
return sampleprof_error::not_implemented;
397+
};
398+
383399
/// The implementaion to read sample profiles from the associated file.
384400
virtual std::error_code readImpl() = 0;
385401

@@ -522,6 +538,16 @@ class SampleProfileReader {
522538

523539
std::unique_ptr<SampleProfileReaderItaniumRemapper> Remapper;
524540

541+
// A map from a function's context hash to its meta data section range, used
542+
// for on-demand read function profile metadata.
543+
std::unordered_map<uint64_t, std::pair<const uint8_t *, const uint8_t *>>
544+
FuncMetadataIndex;
545+
546+
std::pair<const uint8_t *, const uint8_t *> LBRProfileSecRange;
547+
548+
/// Whether the profile has attribute metadata.
549+
bool ProfileHasAttribute = false;
550+
525551
/// \brief Whether samples are collected based on pseudo probes.
526552
bool ProfileIsProbeBased = false;
527553

@@ -621,6 +647,8 @@ class SampleProfileReaderBinary : public SampleProfileReader {
621647

622648
/// Read the next function profile instance.
623649
std::error_code readFuncProfile(const uint8_t *Start);
650+
std::error_code readFuncProfile(const uint8_t *Start,
651+
SampleProfileMap &Profiles);
624652

625653
/// Read the contents of the given profile instance.
626654
std::error_code readProfile(FunctionSamples &FProfile);
@@ -720,11 +748,15 @@ class SampleProfileReaderExtBinaryBase : public SampleProfileReaderBinary {
720748
std::error_code readSecHdrTableEntry(uint64_t Idx);
721749
std::error_code readSecHdrTable();
722750

751+
std::error_code readFuncMetadata(bool ProfileHasAttribute,
752+
SampleProfileMap &Profiles);
723753
std::error_code readFuncMetadata(bool ProfileHasAttribute);
724754
std::error_code readFuncMetadata(bool ProfileHasAttribute,
725755
FunctionSamples *FProfile);
726756
std::error_code readFuncOffsetTable();
727757
std::error_code readFuncProfiles();
758+
std::error_code readFuncProfiles(const DenseSet<StringRef> &FuncsToUse,
759+
SampleProfileMap &Profiles);
728760
std::error_code readNameTableSec(bool IsMD5, bool FixedLengthMD5);
729761
std::error_code readCSNameTableSec();
730762
std::error_code readProfileSymbolList();
@@ -776,6 +808,13 @@ class SampleProfileReaderExtBinaryBase : public SampleProfileReaderBinary {
776808
/// the reader has been given a module.
777809
bool collectFuncsFromModule() override;
778810

811+
/// Read the profiles on-demand for the given functions. This is used after
812+
/// stale call graph matching finds new functions whose profiles aren't loaded
813+
/// at the beginning and we need to loaded the profiles explicitly for
814+
/// potential matching.
815+
std::error_code read(const DenseSet<StringRef> &FuncsToUse,
816+
SampleProfileMap &Profiles) override;
817+
779818
std::unique_ptr<ProfileSymbolList> getProfileSymbolList() override {
780819
return std::move(ProfSymList);
781820
};

‎llvm/lib/ProfileData/SampleProfReader.cpp

+140-85
Original file line numberDiff line numberDiff line change
@@ -653,7 +653,8 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
653653
}
654654

655655
std::error_code
656-
SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
656+
SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start,
657+
SampleProfileMap &Profiles) {
657658
Data = Start;
658659
auto NumHeadSamples = readNumber<uint64_t>();
659660
if (std::error_code EC = NumHeadSamples.getError())
@@ -678,6 +679,11 @@ SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
678679
return sampleprof_error::success;
679680
}
680681

682+
std::error_code
683+
SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
684+
return readFuncProfile(Start, Profiles);
685+
}
686+
681687
std::error_code SampleProfileReaderBinary::readImpl() {
682688
ProfileIsFS = ProfileIsFSDisciminator;
683689
FunctionSamples::ProfileIsFS = ProfileIsFS;
@@ -725,6 +731,7 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
725731
break;
726732
}
727733
case SecLBRProfile:
734+
LBRProfileSecRange = std::make_pair(Data, End);
728735
if (std::error_code EC = readFuncProfiles())
729736
return EC;
730737
break;
@@ -745,9 +752,9 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
745752
ProfileIsProbeBased =
746753
hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased);
747754
FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
748-
bool HasAttribute =
755+
ProfileHasAttribute =
749756
hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute);
750-
if (std::error_code EC = readFuncMetadata(HasAttribute))
757+
if (std::error_code EC = readFuncMetadata(ProfileHasAttribute))
751758
return EC;
752759
break;
753760
}
@@ -791,6 +798,19 @@ bool SampleProfileReaderExtBinaryBase::useFuncOffsetList() const {
791798
return false;
792799
}
793800

801+
std::error_code
802+
SampleProfileReaderExtBinaryBase::read(const DenseSet<StringRef> &FuncsToUse,
803+
SampleProfileMap &Profiles) {
804+
Data = LBRProfileSecRange.first;
805+
End = LBRProfileSecRange.second;
806+
if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
807+
return EC;
808+
End = Data;
809+
810+
if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, Profiles))
811+
return EC;
812+
return sampleprof_error::success;
813+
}
794814

795815
bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() {
796816
if (!M)
@@ -838,6 +858,97 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() {
838858
return sampleprof_error::success;
839859
}
840860

861+
std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles(
862+
const DenseSet<StringRef> &FuncsToUse, SampleProfileMap &Profiles) {
863+
const uint8_t *Start = Data;
864+
865+
if (Remapper) {
866+
for (auto Name : FuncsToUse) {
867+
Remapper->insert(Name);
868+
}
869+
}
870+
871+
if (ProfileIsCS) {
872+
assert(useFuncOffsetList());
873+
DenseSet<uint64_t> FuncGuidsToUse;
874+
if (useMD5()) {
875+
for (auto Name : FuncsToUse)
876+
FuncGuidsToUse.insert(Function::getGUID(Name));
877+
}
878+
879+
// For each function in current module, load all context profiles for
880+
// the function as well as their callee contexts which can help profile
881+
// guided importing for ThinLTO. This can be achieved by walking
882+
// through an ordered context container, where contexts are laid out
883+
// as if they were walked in preorder of a context trie. While
884+
// traversing the trie, a link to the highest common ancestor node is
885+
// kept so that all of its decendants will be loaded.
886+
const SampleContext *CommonContext = nullptr;
887+
for (const auto &NameOffset : FuncOffsetList) {
888+
const auto &FContext = NameOffset.first;
889+
FunctionId FName = FContext.getFunction();
890+
StringRef FNameString;
891+
if (!useMD5())
892+
FNameString = FName.stringRef();
893+
894+
// For function in the current module, keep its farthest ancestor
895+
// context. This can be used to load itself and its child and
896+
// sibling contexts.
897+
if ((useMD5() && FuncGuidsToUse.count(FName.getHashCode())) ||
898+
(!useMD5() && (FuncsToUse.count(FNameString) ||
899+
(Remapper && Remapper->exist(FNameString))))) {
900+
if (!CommonContext || !CommonContext->isPrefixOf(FContext))
901+
CommonContext = &FContext;
902+
}
903+
904+
if (CommonContext == &FContext ||
905+
(CommonContext && CommonContext->isPrefixOf(FContext))) {
906+
// Load profile for the current context which originated from
907+
// the common ancestor.
908+
const uint8_t *FuncProfileAddr = Start + NameOffset.second;
909+
if (std::error_code EC = readFuncProfile(FuncProfileAddr))
910+
return EC;
911+
}
912+
}
913+
} else if (useMD5()) {
914+
assert(!useFuncOffsetList());
915+
for (auto Name : FuncsToUse) {
916+
auto GUID = MD5Hash(Name);
917+
auto iter = FuncOffsetTable.find(GUID);
918+
if (iter == FuncOffsetTable.end())
919+
continue;
920+
const uint8_t *FuncProfileAddr = Start + iter->second;
921+
if (std::error_code EC = readFuncProfile(FuncProfileAddr, Profiles))
922+
return EC;
923+
}
924+
} else if (Remapper) {
925+
assert(useFuncOffsetList());
926+
for (auto NameOffset : FuncOffsetList) {
927+
SampleContext FContext(NameOffset.first);
928+
auto FuncName = FContext.getFunction();
929+
StringRef FuncNameStr = FuncName.stringRef();
930+
if (!FuncsToUse.count(FuncNameStr) && !Remapper->exist(FuncNameStr))
931+
continue;
932+
const uint8_t *FuncProfileAddr = Start + NameOffset.second;
933+
if (std::error_code EC = readFuncProfile(FuncProfileAddr, Profiles))
934+
return EC;
935+
}
936+
} else {
937+
assert(!useFuncOffsetList());
938+
for (auto Name : FuncsToUse) {
939+
940+
auto iter = FuncOffsetTable.find(MD5Hash(Name));
941+
if (iter == FuncOffsetTable.end())
942+
continue;
943+
const uint8_t *FuncProfileAddr = Start + iter->second;
944+
if (std::error_code EC = readFuncProfile(FuncProfileAddr, Profiles))
945+
return EC;
946+
}
947+
}
948+
949+
return sampleprof_error::success;
950+
}
951+
841952
std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
842953
// Collect functions used by current module if the Reader has been
843954
// given a module.
@@ -858,88 +969,8 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
858969
assert(Data == End && "More data is read than expected");
859970
} else {
860971
// Load function profiles on demand.
861-
if (Remapper) {
862-
for (auto Name : FuncsToUse) {
863-
Remapper->insert(Name);
864-
}
865-
}
866-
867-
if (ProfileIsCS) {
868-
assert(useFuncOffsetList());
869-
DenseSet<uint64_t> FuncGuidsToUse;
870-
if (useMD5()) {
871-
for (auto Name : FuncsToUse)
872-
FuncGuidsToUse.insert(Function::getGUID(Name));
873-
}
874-
875-
// For each function in current module, load all context profiles for
876-
// the function as well as their callee contexts which can help profile
877-
// guided importing for ThinLTO. This can be achieved by walking
878-
// through an ordered context container, where contexts are laid out
879-
// as if they were walked in preorder of a context trie. While
880-
// traversing the trie, a link to the highest common ancestor node is
881-
// kept so that all of its decendants will be loaded.
882-
const SampleContext *CommonContext = nullptr;
883-
for (const auto &NameOffset : FuncOffsetList) {
884-
const auto &FContext = NameOffset.first;
885-
FunctionId FName = FContext.getFunction();
886-
StringRef FNameString;
887-
if (!useMD5())
888-
FNameString = FName.stringRef();
889-
890-
// For function in the current module, keep its farthest ancestor
891-
// context. This can be used to load itself and its child and
892-
// sibling contexts.
893-
if ((useMD5() && FuncGuidsToUse.count(FName.getHashCode())) ||
894-
(!useMD5() && (FuncsToUse.count(FNameString) ||
895-
(Remapper && Remapper->exist(FNameString))))) {
896-
if (!CommonContext || !CommonContext->isPrefixOf(FContext))
897-
CommonContext = &FContext;
898-
}
899-
900-
if (CommonContext == &FContext ||
901-
(CommonContext && CommonContext->isPrefixOf(FContext))) {
902-
// Load profile for the current context which originated from
903-
// the common ancestor.
904-
const uint8_t *FuncProfileAddr = Start + NameOffset.second;
905-
if (std::error_code EC = readFuncProfile(FuncProfileAddr))
906-
return EC;
907-
}
908-
}
909-
} else if (useMD5()) {
910-
assert(!useFuncOffsetList());
911-
for (auto Name : FuncsToUse) {
912-
auto GUID = MD5Hash(Name);
913-
auto iter = FuncOffsetTable.find(GUID);
914-
if (iter == FuncOffsetTable.end())
915-
continue;
916-
const uint8_t *FuncProfileAddr = Start + iter->second;
917-
if (std::error_code EC = readFuncProfile(FuncProfileAddr))
918-
return EC;
919-
}
920-
} else if (Remapper) {
921-
assert(useFuncOffsetList());
922-
for (auto NameOffset : FuncOffsetList) {
923-
SampleContext FContext(NameOffset.first);
924-
auto FuncName = FContext.getFunction();
925-
StringRef FuncNameStr = FuncName.stringRef();
926-
if (!FuncsToUse.count(FuncNameStr) && !Remapper->exist(FuncNameStr))
927-
continue;
928-
const uint8_t *FuncProfileAddr = Start + NameOffset.second;
929-
if (std::error_code EC = readFuncProfile(FuncProfileAddr))
930-
return EC;
931-
}
932-
} else {
933-
assert(!useFuncOffsetList());
934-
for (auto Name : FuncsToUse) {
935-
auto iter = FuncOffsetTable.find(MD5Hash(Name));
936-
if (iter == FuncOffsetTable.end())
937-
continue;
938-
const uint8_t *FuncProfileAddr = Start + iter->second;
939-
if (std::error_code EC = readFuncProfile(FuncProfileAddr))
940-
return EC;
941-
}
942-
}
972+
if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))
973+
return EC;
943974
Data = End;
944975
}
945976
assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
@@ -1245,6 +1276,27 @@ SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
12451276
return sampleprof_error::success;
12461277
}
12471278

1279+
std::error_code
1280+
SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
1281+
SampleProfileMap &Profiles) {
1282+
if (FuncMetadataIndex.empty())
1283+
return sampleprof_error::success;
1284+
1285+
for (auto &I : Profiles) {
1286+
FunctionSamples *FProfile = &I.second;
1287+
auto R = FuncMetadataIndex.find(FProfile->getContext().getHashCode());
1288+
if (R == FuncMetadataIndex.end())
1289+
continue;
1290+
1291+
Data = R->second.first;
1292+
End = R->second.second;
1293+
if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))
1294+
return EC;
1295+
assert(Data == End && "More data is read than expected");
1296+
}
1297+
return sampleprof_error::success;
1298+
}
1299+
12481300
std::error_code
12491301
SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {
12501302
while (Data < End) {
@@ -1257,8 +1309,11 @@ SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {
12571309
if (It != Profiles.end())
12581310
FProfile = &It->second;
12591311

1312+
const uint8_t *Start = Data;
12601313
if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))
12611314
return EC;
1315+
1316+
FuncMetadataIndex[FContext.getHashCode()] = {Start, Data};
12621317
}
12631318

12641319
assert(Data == End && "More data is read than expected");

0 commit comments

Comments
 (0)