Merge branch 'master' into beacons-to-tsfull

parser/fsimporter.go

@@ -747,6 +747,14 @@ func (fs *FSImporter) updateTimestampRange() (int64, int64) {
 		return 0, 0
 	}
 
+	// since zeek records connections when they close, some connections that started before the ingested
+	// observation period can skew the ts range. We need to cap observation period to the last 24 hours
+	// for accurate beaconing analysis
+	tsMinCapped := resultMax.Timestamp - 24*60*60
+	if tsMinCapped > resultMin.Timestamp {
+		resultMin.Timestamp = tsMinCapped
+	}
+
 	// set range in metadatabase
 	err = fs.metaDB.AddTSRange(fs.database.GetSelectedDB(), resultMin.Timestamp, resultMax.Timestamp)
 	if err != nil {

pkg/beacon/analyzer.go

@@ -103,6 +103,16 @@ func (a *analyzer) start() {
 				}
 				sort.Sort(util.SortableInt64(diff))
 
+				//find the delta times between full list of timestamps
+				//(this will be used for the intervals list. Bowleys skew
+				//must use a unique timestamp list with no duplicates)
+				tsLengthFull := len(res.TsListFull) - 1
+				//find the delta times between the timestamps
+				diffFull := make([]int64, tsLengthFull)
+				for i := 0; i < tsLengthFull; i++ {
+					diffFull[i] = res.TsListFull[i+1] - res.TsListFull[i]
+				}
+
 				//perfect beacons should have symmetric delta time and size distributions
 				//Bowley's measure of skew is used to check symmetry
 				tsSkew := float64(0)
@@ -159,7 +169,9 @@ func (a *analyzer) start() {
 				//get a list of the intervals found in the data,
 				//the number of times the interval was found,
 				//and the most occurring interval
-				intervals, intervalCounts, tsMode, tsModeCount := createCountMap(diff)
+				//sort intervals list (origbytes already sorted)
+				sort.Sort(util.SortableInt64(diffFull))
+				intervals, intervalCounts, tsMode, tsModeCount := createCountMap(diffFull)
 				dsSizes, dsCounts, dsMode, dsModeCount := createCountMap(res.OrigBytesList)
 
 				//more skewed distributions receive a lower score

pkg/beacon/sorter.go

@@ -57,6 +57,7 @@ func (s *sorter) start() {
 			if (data.TsList) != nil {
 				//sort the size and timestamps to compute quantiles in the analyzer
 				sort.Sort(util.SortableInt64(data.TsList))
+				sort.Sort(util.SortableInt64(data.TsListFull))
 				sort.Sort(util.SortableInt64(data.OrigBytesList))
 			}
 			s.sortedCallback(data)