Skip to content

Commit

Permalink
Merge branch 'main' into gyani/persistent-directory-name
Browse files Browse the repository at this point in the history
  • Loading branch information
@h4ck3rk3y
h4ck3rk3y authored Jan 2, 2024
2 parents d77db4d + d032ff5 commit 5d1abd0
Show file tree
Hide file tree
Showing 4 changed files with 42 additions and 24 deletions.
12 changes: 7 additions & 5 deletions ...alized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,25 +206,27 @@ func (manager *LogFileManager) createSymlinkLogFile(targetLogFilePath, symlinkLo
return nil
}

// TODO: Implement a FilePath Builder to remove exploding constructors
// TODO: Implement a FilePath Builder to centralize log file path creation across the entire module
// creates a filepath of format /<filepath_base>/year/week/<enclave>/serviceIdentifier.<filetype>
func getFilepathStr(year, week int, enclaveUuid, serviceIdentifier string) string {
return fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(week), enclaveUuid, serviceIdentifier, volume_consts.Filetype)
formattedWeekNum := fmt.Sprintf("%02d", week)
return fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), formattedWeekNum, enclaveUuid, serviceIdentifier, volume_consts.Filetype)
}

// creates a directory path of format /<filepath_base>/year/week/<enclave>/
func getEnclaveLogsDirPath(year, week int, enclaveUuid string) string {
logsDirPathForYearAndWeek := getLogsDirPathForWeek(year, week)
return fmt.Sprintf("%s/%s/", logsDirPathForYearAndWeek, enclaveUuid)
return fmt.Sprintf("%s%s/", logsDirPathForYearAndWeek, enclaveUuid)
}

// creates a directory path of format /<filepath_base>/year/week/
func getLogsDirPathForWeek(year, week int) string {
logsDirPathForYear := getLogsDirPathForYear(year)
return fmt.Sprintf("%s/%s/", logsDirPathForYear, strconv.Itoa(week))
formattedWeekNum := fmt.Sprintf("%02d", week)
return fmt.Sprintf("%s%s/", logsDirPathForYear, formattedWeekNum)
}

// creates a directory path of format /<filepath_base>/year/
func getLogsDirPathForYear(year int) string {
return fmt.Sprintf("%s/%s/", volume_consts.LogsStorageDirpath, strconv.Itoa(year))
return fmt.Sprintf("%s%s/", volume_consts.LogsStorageDirpath, strconv.Itoa(year))
}
39 changes: 25 additions & 14 deletions ...s/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,9 @@ func TestStreamUserServiceLogsPerWeek_ThousandsOfLogLinesSuccessfulExecution(t *
}

underlyingFs := volume_filesystem.NewMockedVolumeFilesystem()
file1PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(startingWeek), string(enclaveUuid), testUserService1Uuid, volume_consts.Filetype)
// %02d to format week num with leading zeros so 1-9 are converted to 01-09 for %V format
formattedWeekNum := fmt.Sprintf("%02d", startingWeek)
file1PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, string(enclaveUuid), testUserService1Uuid, volume_consts.Filetype)
file1, err := underlyingFs.Create(file1PathStr)
require.NoError(t, err)
_, err = file1.WriteString(logLinesStr)
Expand Down Expand Up @@ -393,7 +395,8 @@ func TestStreamUserServiceLogsPerWeek_EmptyLogLines(t *testing.T) {
logLinesStr := ""

underlyingFs := volume_filesystem.NewMockedVolumeFilesystem()
file1PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(startingWeek), string(enclaveUuid), testUserService1Uuid, volume_consts.Filetype)
formattedWeekNum := fmt.Sprintf("%02d", startingWeek)
file1PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, string(enclaveUuid), testUserService1Uuid, volume_consts.Filetype)
file1, err := underlyingFs.Create(file1PathStr)
require.NoError(t, err)
_, err = file1.WriteString(logLinesStr)
Expand Down Expand Up @@ -452,13 +455,15 @@ func TestStreamUserServiceLogsPerWeek_WithLogsAcrossWeeks(t *testing.T) {
week3logLinesStr := strings.Join(week3logLines, "\n") + "\n"
week4logLinesStr := strings.Join(week4logLines, "\n")

week4filepath := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(4), testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
formattedWeekFour := fmt.Sprintf("%02d", 4)
week4filepath := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekFour, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
week4, err := underlyingFs.Create(week4filepath)
require.NoError(t, err)
_, err = week4.WriteString(week4logLinesStr)
require.NoError(t, err)

week3filepath := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(3), testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
formattedWeekThree := fmt.Sprintf("%02d", 3)
week3filepath := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekThree, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
week3, err := underlyingFs.Create(week3filepath)
require.NoError(t, err)
_, err = week3.WriteString(week3logLinesStr)
Expand Down Expand Up @@ -519,13 +524,15 @@ func TestStreamUserServiceLogsPerWeek_WithLogLineAcrossWeeks(t *testing.T) {
week3logLinesStr := strings.Join(week3logLines, "\n") + "\n"
week4logLinesStr := strings.Join(week4logLines, "\n") + "\n"

week4filepath := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(4), testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
formattedWeekFour := fmt.Sprintf("%02d", 4)
week4filepath := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekFour, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
week4, err := underlyingFs.Create(week4filepath)
require.NoError(t, err)
_, err = week4.WriteString(week4logLinesStr)
require.NoError(t, err)

week3filepath := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(3), testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
formattedWeekThree := fmt.Sprintf("%02d", 3)
week3filepath := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekThree, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
week3, err := underlyingFs.Create(week3filepath)
require.NoError(t, err)
_, err = week3.WriteString(week3logLinesStr)
Expand Down Expand Up @@ -575,7 +582,8 @@ func TestStreamUserServiceLogsPerWeekReturnsTimestampedLogLines(t *testing.T) {

underlyingFs := volume_filesystem.NewMockedVolumeFilesystem()

filepath := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(startingWeek), testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
formattedWeekNum := fmt.Sprintf("%02d", startingWeek)
filepath := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
file, err := underlyingFs.Create(filepath)
require.NoError(t, err)
_, err = file.WriteString(timestampedLogLinesStr)
Expand Down Expand Up @@ -766,10 +774,11 @@ func createFilledPerWeekFilesystem(week int) volume_filesystem.VolumeFilesystem
logLines := []string{logLine1, logLine2, logLine3a, logLine3b, logLine4, logLine5, logLine6, logLine7, logLine8}

logLinesStr := strings.Join(logLines, "\n")

file1PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(week), testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
file2PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(week), testEnclaveUuid, testUserService2Uuid, volume_consts.Filetype)
file3PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(week), testEnclaveUuid, testUserService3Uuid, volume_consts.Filetype)
// %02d to format week num with leading zeros so 1-9 are converted to 01-09 for %V format
formattedWeekNum := fmt.Sprintf("%02d", week)
file1PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
file2PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService2Uuid, volume_consts.Filetype)
file3PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService3Uuid, volume_consts.Filetype)

mapFs := volume_filesystem.NewMockedVolumeFilesystem()

Expand Down Expand Up @@ -800,9 +809,11 @@ func createEmptyPerFileFilesystem() volume_filesystem.VolumeFilesystem {
}

func createEmptyPerWeekFilesystem(week int) volume_filesystem.VolumeFilesystem {
file1PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(week), testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
file2PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(week), testEnclaveUuid, testUserService2Uuid, volume_consts.Filetype)
file3PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), strconv.Itoa(week), testEnclaveUuid, testUserService3Uuid, volume_consts.Filetype)
// %02d to format week num with leading zeros so 1-9 are converted to 01-09 for %V format
formattedWeekNum := fmt.Sprintf("%02d", week)
file1PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
file2PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService2Uuid, volume_consts.Filetype)
file3PathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(defaultYear), formattedWeekNum, testEnclaveUuid, testUserService3Uuid, volume_consts.Filetype)

mapFs := volume_filesystem.NewMockedVolumeFilesystem()

Expand Down
11 changes: 7 additions & 4 deletions ...t_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ func (strategy *PerWeekStreamLogsStrategy) StreamLogs(
// [getLogFilePaths] returns a list of log file paths containing logs for [serviceUuid] in [enclaveUuid]
// going [retentionPeriodInWeeks] back from the [currentWeek].
// Notes:
// - File paths are of the format '/week/enclave uuid/service uuid.json' where 'week' is %W strftime specifier
// - The list of file paths is returned in order of oldest logs to most recent logs e.g. [ 3/80124/1234.json, /4/801234/1234.json, ...]
// - File paths are of the format '/week/enclave uuid/service uuid.json' where 'week' is %V strftime specifier
// - The list of file paths is returned in order of oldest logs to most recent logs e.g. [ 03/80124/1234.json, /04/801234/1234.json, ...]
// - If a file path does not exist, the function with exits and returns whatever file paths were found
func (strategy *PerWeekStreamLogsStrategy) getLogFilePaths(filesystem volume_filesystem.VolumeFilesystem, retentionPeriodInWeeks int, enclaveUuid, serviceUuid string) ([]string, error) {
var paths []string
Expand All @@ -121,7 +121,9 @@ func (strategy *PerWeekStreamLogsStrategy) getLogFilePaths(filesystem volume_fil
firstWeekWithLogs := 0
for i := 0; i < retentionPeriodInWeeks; i++ {
year, week := currentTime.Add(time.Duration(-i) * oneWeek).ISOWeek()
filePathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(week), enclaveUuid, serviceUuid, volume_consts.Filetype)
// %02d to format week num with leading zeros so 1-9 are converted to 01-09 for %V format
formattedWeekNum := fmt.Sprintf("%02d", week)
filePathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), formattedWeekNum, enclaveUuid, serviceUuid, volume_consts.Filetype)
if _, err := filesystem.Stat(filePathStr); err == nil {
paths = append(paths, filePathStr)
firstWeekWithLogs = i
Expand All @@ -137,7 +139,8 @@ func (strategy *PerWeekStreamLogsStrategy) getLogFilePaths(filesystem volume_fil
// scan for remaining files as far back as they exist
for i := firstWeekWithLogs + 1; i < retentionPeriodInWeeks; i++ {
year, week := currentTime.Add(time.Duration(-i) * oneWeek).ISOWeek()
filePathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(week), enclaveUuid, serviceUuid, volume_consts.Filetype)
formattedWeekNum := fmt.Sprintf("%02d", week)
filePathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), formattedWeekNum, enclaveUuid, serviceUuid, volume_consts.Filetype)
if _, err := filesystem.Stat(filePathStr); err != nil {
break
}
Expand Down
4 changes: 3 additions & 1 deletion ...lementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,9 @@ func TestIsWithinRetentionPeriod(t *testing.T) {
}

func getWeekFilepathStr(year, week int) string {
return fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(week), testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
// %02d to format week num with leading zeros so 1-9 are converted to 01-09 for %V format
formattedWeekNum := fmt.Sprintf("%02d", week)
return fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), formattedWeekNum, testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype)
}

func TestGetCompleteJsonLogString(t *testing.T) {
Expand Down

0 comments on commit 5d1abd0

Please sign in to comment.