From 1cfafd0b20565cbf509dbcd6c3f3bd53c1f2fc82 Mon Sep 17 00:00:00 2001 From: Jernej Kos Date: Fri, 17 Feb 2023 11:17:00 +0100 Subject: [PATCH] go/runtime: Also re-attest based on MaxAttestationAge --- .changelog/5187.bugfix.md | 1 + go/common/node/node.go | 3 - go/common/node/sgx.go | 4 +- go/runtime/host/host.go | 3 + go/runtime/host/mock/mock.go | 5 ++ go/runtime/host/multi/multi.go | 11 ++++ go/runtime/host/sandbox/sandbox.go | 65 +++++++++++++----- go/runtime/host/sgx/sgx.go | 58 ++++++++-------- go/runtime/registry/host.go | 102 +++++++++++++++++++++++++++-- 9 files changed, 196 insertions(+), 56 deletions(-) create mode 100644 .changelog/5187.bugfix.md diff --git a/.changelog/5187.bugfix.md b/.changelog/5187.bugfix.md new file mode 100644 index 00000000000..3266a25ee45 --- /dev/null +++ b/.changelog/5187.bugfix.md @@ -0,0 +1 @@ +go/runtime: Also re-attest based on MaxAttestationAge diff --git a/go/common/node/node.go b/go/common/node/node.go index 8941f5071bf..d43a99be0b6 100644 --- a/go/common/node/node.go +++ b/go/common/node/node.go @@ -41,9 +41,6 @@ var ( // signature fails verification. ErrInvalidAttestationSignature = errors.New("node: invalid TEE attestation signature") - // ErrAttestationNotFresh is the error returned when the TEE attestation is - // not fresh enough. - ErrAttestationNotFresh = errors.New("node: TEE attestation not fresh enough") // ErrAttestationFromFuture is the error returned when the TEE attestation appears // to be from the future. ErrAttestationFromFuture = errors.New("node: TEE attestation from the future") diff --git a/go/common/node/sgx.go b/go/common/node/sgx.go index b2fa27b7fda..71f5df9e32b 100644 --- a/go/common/node/sgx.go +++ b/go/common/node/sgx.go @@ -269,8 +269,8 @@ func (sa *SGXAttestation) verifyAttestationSignature( if sa.Height > height { return ErrAttestationFromFuture } - if height-sa.Height > sc.MaxAttestationAge { - return ErrAttestationNotFresh + if age := height - sa.Height; age > sc.MaxAttestationAge { + return fmt.Errorf("node: TEE attestation not fresh enough (age: %d max: %d)", age, sc.MaxAttestationAge) } return nil diff --git a/go/runtime/host/host.go b/go/runtime/host/host.go index bca51a7a4bd..4642f010bdd 100644 --- a/go/runtime/host/host.go +++ b/go/runtime/host/host.go @@ -56,6 +56,9 @@ type Runtime interface { // response (which may be a failure). Call(ctx context.Context, body *protocol.Body) (*protocol.Body, error) + // UpdateCapabilityTEE asks the runtime to update its CapabilityTEE with latest data. + UpdateCapabilityTEE(ctx context.Context) error + // WatchEvents subscribes to runtime status events. WatchEvents(ctx context.Context) (<-chan *Event, pubsub.ClosableSubscription, error) diff --git a/go/runtime/host/mock/mock.go b/go/runtime/host/mock/mock.go index b408d24ba5f..34621f44b53 100644 --- a/go/runtime/host/mock/mock.go +++ b/go/runtime/host/mock/mock.go @@ -173,6 +173,11 @@ func (r *runtime) Call(ctx context.Context, body *protocol.Body) (*protocol.Body } } +// Implements host.Runtime. +func (r *runtime) UpdateCapabilityTEE(ctx context.Context) error { + return nil +} + // Implements host.Runtime. func (r *runtime) WatchEvents(ctx context.Context) (<-chan *host.Event, pubsub.ClosableSubscription, error) { typedCh := make(chan *host.Event) diff --git a/go/runtime/host/multi/multi.go b/go/runtime/host/multi/multi.go index 945f7965668..e77b240a329 100644 --- a/go/runtime/host/multi/multi.go +++ b/go/runtime/host/multi/multi.go @@ -116,6 +116,17 @@ func (agg *Aggregate) Call(ctx context.Context, body *protocol.Body) (rsp *proto return } +// UpdateCapabilityTEE implements host.Runtime. +func (agg *Aggregate) UpdateCapabilityTEE(ctx context.Context) error { + agg.l.RLock() + defer agg.l.RUnlock() + + if agg.active == nil { + return ErrNoActiveVersion + } + return agg.active.host.UpdateCapabilityTEE(ctx) +} + // WatchEvents implements host.Runtime. func (agg *Aggregate) WatchEvents(ctx context.Context) (<-chan *host.Event, pubsub.ClosableSubscription, error) { typedCh := make(chan *host.Event) diff --git a/go/runtime/host/sandbox/sandbox.go b/go/runtime/host/sandbox/sandbox.go index 3e6540babdf..9c5e93cf6e7 100644 --- a/go/runtime/host/sandbox/sandbox.go +++ b/go/runtime/host/sandbox/sandbox.go @@ -11,6 +11,7 @@ import ( "time" "github.com/cenkalti/backoff/v4" + "github.com/eapache/channels" "github.com/oasisprotocol/oasis-core/go/common" cmnBackoff "github.com/oasisprotocol/oasis-core/go/common/backoff" @@ -45,7 +46,7 @@ type Config struct { // HostInitializer is a function that additionally initializes the runtime host. In case it is // not specified a default function is used. - HostInitializer func(context.Context, host.Runtime, version.Version, process.Process, protocol.Connection) (*host.StartedEvent, error) + HostInitializer func(context.Context, *HostInitializerParams) (*host.StartedEvent, error) // Logger is an optional logger to use with this provisioner. In case it is not specified a // default logger will be created. @@ -58,6 +59,16 @@ type Config struct { InsecureNoSandbox bool } +// HostInitializerParams contains parameters for the HostInitializer function. +type HostInitializerParams struct { + Runtime host.Runtime + Version version.Version + Process process.Process + Connection protocol.Connection + + NotifyUpdateCapabilityTEE <-chan struct{} +} + type provisioner struct { cfg Config } @@ -67,14 +78,15 @@ func (p *provisioner) NewRuntime(ctx context.Context, cfg host.Config) (host.Run id := cfg.Bundle.Manifest.ID r := &sandboxedRuntime{ - cfg: p.cfg, - rtCfg: cfg, - id: id, - stopCh: make(chan struct{}), - quitCh: make(chan struct{}), - ctrlCh: make(chan interface{}, ctrlChannelBufferSize), - notifier: pubsub.NewBroker(false), - logger: p.cfg.Logger.With("runtime_id", id), + cfg: p.cfg, + rtCfg: cfg, + id: id, + stopCh: make(chan struct{}), + quitCh: make(chan struct{}), + ctrlCh: make(chan interface{}, ctrlChannelBufferSize), + notifier: pubsub.NewBroker(false), + notifyUpdateCapabilityTEE: channels.NewRingChannel(1), + logger: p.cfg.Logger.With("runtime_id", id), } return r, nil @@ -103,6 +115,8 @@ type sandboxedRuntime struct { conn protocol.Connection notifier *pubsub.Broker + notifyUpdateCapabilityTEE *channels.RingChannel + logger *logging.Logger } @@ -151,6 +165,16 @@ func (r *sandboxedRuntime) Call(ctx context.Context, body *protocol.Body) (rsp * return } +// Implements host.Runtime. +func (r *sandboxedRuntime) UpdateCapabilityTEE(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case r.notifyUpdateCapabilityTEE.In() <- struct{}{}: + } + return nil +} + // Implements host.Runtime. func (r *sandboxedRuntime) WatchEvents(ctx context.Context) (<-chan *host.Event, pubsub.ClosableSubscription, error) { typedCh := make(chan *host.Event) @@ -356,10 +380,21 @@ func (r *sandboxedRuntime) startProcess() (err error) { return fmt.Errorf("version mismatch (runtime reported: %s bundle: %s)", *rtVersion, bndVersion) } + notifyUpdateCapabilityTEECh := make(chan struct{}) + channels.Unwrap(r.notifyUpdateCapabilityTEE, notifyUpdateCapabilityTEECh) + + hp := &HostInitializerParams{ + Runtime: r, + Version: *rtVersion, + Process: p, + Connection: pc, + NotifyUpdateCapabilityTEE: notifyUpdateCapabilityTEECh, + } + // Perform configuration-specific host initialization. exInitCtx, cancelExInit := context.WithTimeout(ctx, runtimeExtendedInitTimeout) defer cancelExInit() - ev, err := r.cfg.HostInitializer(exInitCtx, r, *rtVersion, p, pc) + ev, err := r.cfg.HostInitializer(exInitCtx, hp) if err != nil { return fmt.Errorf("failed to initialize connection: %w", err) } @@ -559,15 +594,9 @@ func New(cfg Config) (host.Provisioner, error) { } // Use a default HostInitializer if none was provided. if cfg.HostInitializer == nil { - cfg.HostInitializer = func( - ctx context.Context, - rt host.Runtime, - version version.Version, - p process.Process, - conn protocol.Connection, - ) (*host.StartedEvent, error) { + cfg.HostInitializer = func(ctx context.Context, hp *HostInitializerParams) (*host.StartedEvent, error) { return &host.StartedEvent{ - Version: version, + Version: hp.Version, }, nil } } diff --git a/go/runtime/host/sgx/sgx.go b/go/runtime/host/sgx/sgx.go index 5384f77a497..f1ddecbce80 100644 --- a/go/runtime/host/sgx/sgx.go +++ b/go/runtime/host/sgx/sgx.go @@ -259,28 +259,22 @@ func (s *sgxProvisioner) getSandboxConfig(rtCfg host.Config, socketPath, runtime }, nil } -func (s *sgxProvisioner) hostInitializer( - ctx context.Context, - rt host.Runtime, - version version.Version, - p process.Process, - conn protocol.Connection, -) (*host.StartedEvent, error) { +func (s *sgxProvisioner) hostInitializer(ctx context.Context, hp *sandbox.HostInitializerParams) (*host.StartedEvent, error) { // Initialize TEE. var err error var ts *teeState - if ts, err = s.initCapabilityTEE(ctx, rt, conn, version); err != nil { + if ts, err = s.initCapabilityTEE(ctx, hp.Runtime, hp.Connection, hp.Version); err != nil { return nil, fmt.Errorf("failed to initialize TEE: %w", err) } var capabilityTEE *node.CapabilityTEE - if capabilityTEE, err = s.updateCapabilityTEE(ctx, s.logger, ts, conn); err != nil { + if capabilityTEE, err = s.updateCapabilityTEE(ctx, s.logger, ts, hp.Connection); err != nil { return nil, fmt.Errorf("failed to initialize TEE: %w", err) } - go s.attestationWorker(ts, p, conn, version) + go s.attestationWorker(ts, hp) return &host.StartedEvent{ - Version: version, + Version: hp.Version, CapabilityTEE: capabilityTEE, }, nil } @@ -349,7 +343,7 @@ func (s *sgxProvisioner) updateCapabilityTEE(ctx context.Context, logger *loggin return capabilityTEE, nil } -func (s *sgxProvisioner) attestationWorker(ts *teeState, p process.Process, conn protocol.Connection, version version.Version) { +func (s *sgxProvisioner) attestationWorker(ts *teeState, hp *sandbox.HostInitializerParams) { t := time.NewTicker(s.cfg.RuntimeAttestInterval) defer t.Stop() @@ -357,27 +351,33 @@ func (s *sgxProvisioner) attestationWorker(ts *teeState, p process.Process, conn for { select { - case <-p.Wait(): + case <-hp.Process.Wait(): // Process has terminated. return case <-t.C: - // Update CapabilityTEE. - logger.Info("regenerating CapabilityTEE") - - capabilityTEE, err := s.updateCapabilityTEE(context.Background(), logger, ts, conn) - if err != nil { - logger.Error("failed to regenerate CapabilityTEE", - "err", err, - ) - continue - } - - // Emit event about the updated CapabilityTEE. - ts.eventEmitter.EmitEvent(&host.Event{Updated: &host.UpdatedEvent{ - Version: version, - CapabilityTEE: capabilityTEE, - }}) + // Re-attest based on the configured interval. + case <-hp.NotifyUpdateCapabilityTEE: + // Re-attest when explicitly requested. Also reset the periodic ticker to make sure we + // don't needlessly re-attest too often. + t.Reset(s.cfg.RuntimeAttestInterval) } + + // Update CapabilityTEE. + logger.Info("regenerating CapabilityTEE") + + capabilityTEE, err := s.updateCapabilityTEE(context.Background(), logger, ts, hp.Connection) + if err != nil { + logger.Error("failed to regenerate CapabilityTEE", + "err", err, + ) + continue + } + + // Emit event about the updated CapabilityTEE. + ts.eventEmitter.EmitEvent(&host.Event{Updated: &host.UpdatedEvent{ + Version: hp.Version, + CapabilityTEE: capabilityTEE, + }}) } } diff --git a/go/runtime/registry/host.go b/go/runtime/registry/host.go index 0cf002a8d8a..7736b764279 100644 --- a/go/runtime/registry/host.go +++ b/go/runtime/registry/host.go @@ -4,11 +4,13 @@ import ( "context" "errors" "fmt" + "math/rand" "sync" "time" "github.com/eapache/channels" + beacon "github.com/oasisprotocol/oasis-core/go/beacon/api" "github.com/oasisprotocol/oasis-core/go/common" "github.com/oasisprotocol/oasis-core/go/common/cbor" "github.com/oasisprotocol/oasis-core/go/common/identity" @@ -35,6 +37,9 @@ const ( // retryInterval is the time interval used between failed key manager updates. retryInterval = time.Second + + // minAttestationInterval is the minimum attestation interval. + minAttestationInterval = 5 * time.Minute ) // RuntimeHostNode provides methods for nodes that need to host runtimes. @@ -743,14 +748,30 @@ func (n *runtimeHostNotifier) watchConsensusLightBlocks() { // Create a ring channel with a capacity of one as we only care about the latest block. blkCh := channels.NewRingChannel(channels.BufferCap(1)) go func() { + defer blkCh.Close() + for blk := range rawCh { blkCh.In() <- blk } - blkCh.Close() }() + // Subscribe to runtime descriptor updates. + dscCh, dscSub, err := n.runtime.WatchRegistryDescriptor() + if err != nil { + n.logger.Error("failed to subscribe to registry descriptor updates", + "err", err, + ) + return + } + defer dscSub.Close() + n.logger.Debug("watching consensus layer blocks") + var ( + maxAttestationAge uint64 + lastAttestationUpdateHeight uint64 + lastAttestationUpdate time.Time + ) for { select { case <-n.ctx.Done(): @@ -759,26 +780,99 @@ func (n *runtimeHostNotifier) watchConsensusLightBlocks() { case <-n.stopCh: n.logger.Debug("termination requested") return + case dsc := <-dscCh: + // We only care about TEE-enabled runtimes. + if dsc.TEEHardware != node.TEEHardwareIntelSGX { + continue + } + + var epoch beacon.EpochTime + epoch, err = n.consensus.Beacon().GetEpoch(n.ctx, consensus.HeightLatest) + if err != nil { + n.logger.Error("failed to query current epoch", + "err", err, + ) + continue + } + + // Fetch the active deployment. + vi := dsc.ActiveDeployment(epoch) + if vi == nil { + continue + } + + // Parse SGX constraints. + var sc node.SGXConstraints + if err = cbor.Unmarshal(vi.TEE, &sc); err != nil { + n.logger.Error("malformed SGX constraints", + "err", err, + ) + continue + } + + // Apply defaults. + var params *registry.ConsensusParameters + params, err = n.consensus.Registry().ConsensusParameters(n.ctx, consensus.HeightLatest) + if err != nil { + n.logger.Error("failed to query registry parameters", + "err", err, + ) + continue + } + if params.TEEFeatures != nil { + params.TEEFeatures.SGX.ApplyDefaultConstraints(&sc) + } + + // Pick a random interval between 50% and 90% of the MaxAttestationAge. + if sc.MaxAttestationAge > 2 { // Ensure a is non-zero. + a := (sc.MaxAttestationAge * 4) / 10 // 40% + b := sc.MaxAttestationAge / 2 // 50% + maxAttestationAge = b + uint64(rand.Int63n(int64(a))) + } else { + maxAttestationAge = 0 // Disarm height-based re-attestation. + } case rawBlk, ok := <-blkCh.Out(): + // New consensus layer block. if !ok { return } blk := rawBlk.(*consensus.Block) + height := uint64(blk.Height) // Notify the runtime that a new consensus layer block is available. ctx, cancel := context.WithTimeout(n.ctx, notifyTimeout) - err = n.host.ConsensusSync(ctx, uint64(blk.Height)) + err = n.host.ConsensusSync(ctx, height) cancel() if err != nil { n.logger.Error("failed to notify runtime of a new consensus layer block", "err", err, - "height", blk.Height, + "height", height, ) continue } n.logger.Debug("runtime notified of new consensus layer block", - "height", blk.Height, + "height", height, ) + + // Assume runtime has already done the initial attestation. + if lastAttestationUpdate.IsZero() { + lastAttestationUpdateHeight = height + lastAttestationUpdate = time.Now() + } + // Periodically trigger re-attestation. + if maxAttestationAge > 0 && height-lastAttestationUpdateHeight > maxAttestationAge && + time.Since(lastAttestationUpdate) > minAttestationInterval { + + n.logger.Debug("requesting the runtime to update CapabilityTEE") + + if err = n.host.UpdateCapabilityTEE(n.ctx); err != nil { + n.logger.Error("failed to update runtime CapabilityTEE", + "err", err, + ) + } + lastAttestationUpdateHeight = height + lastAttestationUpdate = time.Now() + } } } }