From 39e379f0f214e89a9c86d541f1bd580e837da9fa Mon Sep 17 00:00:00 2001 From: Tim Voronov Date: Fri, 30 Nov 2018 19:30:55 -0500 Subject: [PATCH] Decoupled runtime and HTML driver initialization (#198) * Decoupled runtime and HTML driver initialization * Updates --- README.md | 33 ++++++++++++++----- cli/exec.go | 5 +-- cli/options.go | 24 ++++++++++++++ cli/repl.go | 5 +-- e2e/runner/runner.go | 12 +++++-- examples/embedded.go | 15 +++++++-- examples/extensible.go | 13 +++++--- pkg/html/common/ua.go | 5 +-- pkg/html/driver.go | 65 ++++--------------------------------- pkg/html/dynamic/driver.go | 52 ++++++++++++++++++++--------- pkg/html/dynamic/options.go | 18 ++++++++++ pkg/html/static/driver.go | 41 ++++++++++++++++------- pkg/html/static/options.go | 13 ++++++++ pkg/runtime/env/env.go | 31 ------------------ pkg/runtime/options.go | 61 ++++++++-------------------------- pkg/runtime/program.go | 5 +-- 16 files changed, 204 insertions(+), 194 deletions(-) delete mode 100644 pkg/runtime/env/env.go diff --git a/README.md b/README.md index 217d08e3..a3db77b3 100644 --- a/README.md +++ b/README.md @@ -219,14 +219,16 @@ import ( "context" "encoding/json" "fmt" - "github.com/MontFerret/ferret/pkg/compiler" "os" + + "github.com/MontFerret/ferret/pkg/compiler" + "github.com/MontFerret/ferret/pkg/html" ) type Topic struct { Name string `json:"name"` Description string `json:"description"` - Url string `json:"url"` + URL string `json:"url"` } func main() { @@ -238,7 +240,7 @@ func main() { } for _, topic := range topics { - fmt.Println(fmt.Sprintf("%s: %s %s", topic.Name, topic.Description, topic.Url)) + fmt.Println(fmt.Sprintf("%s: %s %s", topic.Name, topic.Description, topic.URL)) } } @@ -267,7 +269,16 @@ func getTopTenTrendingTopics() ([]*Topic, error) { return nil, err } - out, err := program.Run(context.Background()) + // create a root context + ctx := context.Background() + + // enable HTML drivers + // by default, Ferret Runtime knows nothing about HTML drivers + // all HTML manipulations are done via functions from standard library + ctx = html.WithDynamicDriver(ctx) + ctx = html.WithStaticDriver(ctx) + + out, err := program.Run(ctx) if err != nil { return nil, err @@ -283,6 +294,7 @@ func getTopTenTrendingTopics() ([]*Topic, error) { return res, nil } + ``` ## Extensibility @@ -296,10 +308,12 @@ import ( "context" "encoding/json" "fmt" + "os" + "strings" + "github.com/MontFerret/ferret/pkg/compiler" "github.com/MontFerret/ferret/pkg/runtime/core" "github.com/MontFerret/ferret/pkg/runtime/values" - "os" ) func main() { @@ -319,7 +333,7 @@ func getStrings() ([]string, error) { // function implements is a type of a function that ferret supports as a runtime function transform := func(ctx context.Context, args ...core.Value) (core.Value, error) { // it's just a helper function which helps to validate a number of passed args - err := core.ValidateArgs(args, 1) + err := core.ValidateArgs(args, 1, 1) if err != nil { // it's recommended to return built-in None type, instead of nil @@ -336,7 +350,7 @@ func getStrings() ([]string, error) { // cast to built-in string type str := args[0].(values.String) - return str.Concat(values.NewString("_ferret")).ToUpper(), nil + return values.NewString(strings.ToUpper(str.String() + "_ferret")), nil } query := ` @@ -346,7 +360,10 @@ func getStrings() ([]string, error) { ` comp := compiler.New() - comp.RegisterFunction("transform", transform) + + if err := comp.RegisterFunction("transform", transform); err != nil { + return nil, err + } program, err := comp.Compile(query) diff --git a/cli/exec.go b/cli/exec.go index d3619383..c799caf0 100644 --- a/cli/exec.go +++ b/cli/exec.go @@ -38,7 +38,7 @@ func Exec(query string, opts Options) { l := NewLogger() - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(opts.WithContext(context.Background())) c := make(chan os.Signal, 1) signal.Notify(c, syscall.SIGHUP) @@ -59,12 +59,9 @@ func Exec(query string, opts Options) { out, err := prog.Run( ctx, - runtime.WithBrowser(opts.Cdp), runtime.WithLog(l), runtime.WithLogLevel(logging.DebugLevel), runtime.WithParams(opts.Params), - runtime.WithProxy(opts.Proxy), - runtime.WithUserAgent(opts.UserAgent), ) if opts.ShowTime { diff --git a/cli/options.go b/cli/options.go index 03667559..f2756bee 100644 --- a/cli/options.go +++ b/cli/options.go @@ -1,5 +1,12 @@ package cli +import ( + "context" + "github.com/MontFerret/ferret/pkg/html" + "github.com/MontFerret/ferret/pkg/html/dynamic" + "github.com/MontFerret/ferret/pkg/html/static" +) + type Options struct { Cdp string Params map[string]interface{} @@ -7,3 +14,20 @@ type Options struct { UserAgent string ShowTime bool } + +func (opts Options) WithContext(ctx context.Context) context.Context { + ctx = html.WithDynamicDriver( + ctx, + dynamic.WithCDP(opts.Cdp), + dynamic.WithProxy(opts.Proxy), + dynamic.WithUserAgent(opts.UserAgent), + ) + + ctx = html.WithStaticDriver( + ctx, + static.WithProxy(opts.Proxy), + static.WithUserAgent(opts.UserAgent), + ) + + return ctx +} diff --git a/cli/repl.go b/cli/repl.go index e58b0efd..15c90c23 100644 --- a/cli/repl.go +++ b/cli/repl.go @@ -42,7 +42,7 @@ func Repl(version string, opts Options) { l := NewLogger() - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(opts.WithContext(context.Background())) c := make(chan os.Signal, 1) signal.Notify(c, syscall.SIGHUP) @@ -110,12 +110,9 @@ func Repl(version string, opts Options) { out, err := program.Run( ctx, - runtime.WithBrowser(opts.Cdp), runtime.WithLog(l), runtime.WithLogLevel(logging.DebugLevel), runtime.WithParams(opts.Params), - runtime.WithProxy(opts.Proxy), - runtime.WithUserAgent(opts.UserAgent), ) if err != nil { diff --git a/e2e/runner/runner.go b/e2e/runner/runner.go index 8d2f287e..913f103c 100644 --- a/e2e/runner/runner.go +++ b/e2e/runner/runner.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "github.com/MontFerret/ferret/pkg/compiler" + "github.com/MontFerret/ferret/pkg/html" + "github.com/MontFerret/ferret/pkg/html/dynamic" "github.com/MontFerret/ferret/pkg/runtime" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -131,10 +133,16 @@ func (r *Runner) runQuery(c *compiler.FqlCompiler, name, script string) Result { } } + ctx := context.Background() + ctx = html.WithDynamicDriver( + ctx, + dynamic.WithCDP(r.settings.CDPAddress), + ) + ctx = html.WithStaticDriver(ctx) + out, err := p.Run( - context.Background(), + ctx, runtime.WithLog(os.Stdout), - runtime.WithBrowser(r.settings.CDPAddress), runtime.WithParam("static", r.settings.StaticServerAddress), runtime.WithParam("dynamic", r.settings.DynamicServerAddress), ) diff --git a/examples/embedded.go b/examples/embedded.go index 8419a52d..e4128eae 100644 --- a/examples/embedded.go +++ b/examples/embedded.go @@ -4,8 +4,10 @@ import ( "context" "encoding/json" "fmt" - "github.com/MontFerret/ferret/pkg/compiler" "os" + + "github.com/MontFerret/ferret/pkg/compiler" + "github.com/MontFerret/ferret/pkg/html" ) type Topic struct { @@ -52,7 +54,16 @@ func getTopTenTrendingTopics() ([]*Topic, error) { return nil, err } - out, err := program.Run(context.Background()) + // create a root context + ctx := context.Background() + + // enable HTML drivers + // by default, Ferret Runtime knows nothing about HTML drivers + // all HTML manipulations are done via functions from standard library + ctx = html.WithDynamicDriver(ctx) + ctx = html.WithStaticDriver(ctx) + + out, err := program.Run(ctx) if err != nil { return nil, err diff --git a/examples/extensible.go b/examples/extensible.go index c5b5b186..ea27b066 100644 --- a/examples/extensible.go +++ b/examples/extensible.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "fmt" + "os" + "strings" + "github.com/MontFerret/ferret/pkg/compiler" "github.com/MontFerret/ferret/pkg/runtime/core" "github.com/MontFerret/ferret/pkg/runtime/values" - "os" ) func main() { @@ -27,7 +29,7 @@ func getStrings() ([]string, error) { // function implements is a type of a function that ferret supports as a runtime function transform := func(ctx context.Context, args ...core.Value) (core.Value, error) { // it's just a helper function which helps to validate a number of passed args - err := core.ValidateArgs(args, 1) + err := core.ValidateArgs(args, 1, 1) if err != nil { // it's recommended to return built-in None type, instead of nil @@ -44,7 +46,7 @@ func getStrings() ([]string, error) { // cast to built-in string type str := args[0].(values.String) - return str.Concat(values.NewString("_ferret")).ToUpper(), nil + return values.NewString(strings.ToUpper(str.String() + "_ferret")), nil } query := ` @@ -54,7 +56,10 @@ func getStrings() ([]string, error) { ` comp := compiler.New() - comp.RegisterFunction("transform", transform) + + if err := comp.RegisterFunction("transform", transform); err != nil { + return nil, err + } program, err := comp.Compile(query) diff --git a/pkg/html/common/ua.go b/pkg/html/common/ua.go index 6e0bd82c..1489a18c 100644 --- a/pkg/html/common/ua.go +++ b/pkg/html/common/ua.go @@ -1,16 +1,17 @@ package common import ( - "github.com/MontFerret/ferret/pkg/runtime/env" "github.com/corpix/uarand" ) +const RandomUserAgent = "*" + func GetUserAgent(val string) string { if val == "" { return val } - if val != env.RandomUserAgent { + if val != RandomUserAgent { return val } diff --git a/pkg/html/driver.go b/pkg/html/driver.go index b6d1c055..044d10c5 100644 --- a/pkg/html/driver.go +++ b/pkg/html/driver.go @@ -6,15 +6,10 @@ import ( "github.com/MontFerret/ferret/pkg/html/dynamic" "github.com/MontFerret/ferret/pkg/html/static" "github.com/MontFerret/ferret/pkg/runtime/core" - "github.com/MontFerret/ferret/pkg/runtime/env" "github.com/MontFerret/ferret/pkg/runtime/values" ) -type ( - DriverName string - dynamicCtxKey struct{} - staticCtxKey struct{} -) +type DriverName string const ( Dynamic DriverName = "dynamic" @@ -26,67 +21,21 @@ type Driver interface { Close() error } -func ToContext(ctx context.Context, name DriverName, drv Driver) context.Context { - var key interface{} - - switch name { - case Dynamic: - key = dynamicCtxKey{} - case Static: - key = staticCtxKey{} - default: - return ctx - } - - return context.WithValue(ctx, key, drv) -} - func FromContext(ctx context.Context, name DriverName) (Driver, error) { - var key interface{} - switch name { case Dynamic: - key = dynamicCtxKey{} + return dynamic.FromContext(ctx) case Static: - key = staticCtxKey{} + return static.FromContext(ctx) default: return nil, core.Error(core.ErrInvalidArgument, fmt.Sprintf("%s driver", name)) } - - val := ctx.Value(key) - - drv, ok := val.(Driver) - - if ok { - return drv, nil - } - - return nil, core.Error(core.ErrNotFound, fmt.Sprintf("%s driver", name)) } -func WithDynamicDriver(ctx context.Context) context.Context { - e := env.FromContext(ctx) - - return context.WithValue( - ctx, - dynamicCtxKey{}, - dynamic.NewDriver( - e.CDPAddress, - dynamic.WithProxy(e.ProxyAddress), - dynamic.WithUserAgent(e.UserAgent), - ), - ) +func WithDynamicDriver(ctx context.Context, opts ...dynamic.Option) context.Context { + return dynamic.WithContext(ctx, dynamic.NewDriver(opts...)) } -func WithStaticDriver(ctx context.Context) context.Context { - e := env.FromContext(ctx) - - return context.WithValue( - ctx, - staticCtxKey{}, - static.NewDriver( - static.WithProxy(e.ProxyAddress), - static.WithUserAgent(e.UserAgent), - ), - ) +func WithStaticDriver(ctx context.Context, opts ...static.Option) context.Context { + return static.WithContext(ctx, static.NewDriver(opts...)) } diff --git a/pkg/html/dynamic/driver.go b/pkg/html/dynamic/driver.go index a93cb7fd..0ac5e18f 100644 --- a/pkg/html/dynamic/driver.go +++ b/pkg/html/dynamic/driver.go @@ -2,6 +2,9 @@ package dynamic import ( "context" + "github.com/MontFerret/ferret/pkg/runtime/core" + "sync" + "github.com/MontFerret/ferret/pkg/html/common" "github.com/MontFerret/ferret/pkg/runtime/logging" "github.com/MontFerret/ferret/pkg/runtime/values" @@ -13,28 +16,47 @@ import ( "github.com/mafredri/cdp/rpcc" "github.com/mafredri/cdp/session" "github.com/pkg/errors" - "sync" ) -type Driver struct { - sync.Mutex - dev *devtool.DevTools - conn *rpcc.Conn - client *cdp.Client - session *session.Manager - contextID target.BrowserContextID - options *Options +type ( + ctxKey struct{} + + Driver struct { + sync.Mutex + dev *devtool.DevTools + conn *rpcc.Conn + client *cdp.Client + session *session.Manager + contextID target.BrowserContextID + options *Options + } +) + +func WithContext(ctx context.Context, drv *Driver) context.Context { + return context.WithValue( + ctx, + ctxKey{}, + drv, + ) } -func NewDriver(address string, opts ...Option) *Driver { - drv := new(Driver) - drv.dev = devtool.New(address) - drv.options = new(Options) +func FromContext(ctx context.Context) (*Driver, error) { + val := ctx.Value(ctxKey{}) + + drv, ok := val.(*Driver) - for _, opt := range opts { - opt(drv.options) + if !ok { + return nil, core.Error(core.ErrNotFound, "dynamic HTML Driver") } + return drv, nil +} + +func NewDriver(opts ...Option) *Driver { + drv := new(Driver) + drv.options = newOptions(opts) + drv.dev = devtool.New(drv.options.cdp) + return drv } diff --git a/pkg/html/dynamic/options.go b/pkg/html/dynamic/options.go index b0f3b5a1..bb93e6a9 100644 --- a/pkg/html/dynamic/options.go +++ b/pkg/html/dynamic/options.go @@ -4,11 +4,29 @@ type ( Options struct { proxy string userAgent string + cdp string } Option func(opts *Options) ) +func newOptions(setters []Option) *Options { + opts := new(Options) + opts.cdp = "http://127.0.0.1:9222" + + for _, setter := range setters { + setter(opts) + } + + return opts +} + +func WithCDP(address string) Option { + return func(opts *Options) { + opts.cdp = address + } +} + func WithProxy(address string) Option { return func(opts *Options) { opts.proxy = address diff --git a/pkg/html/static/driver.go b/pkg/html/static/driver.go index c6baf536..86fa669f 100644 --- a/pkg/html/static/driver.go +++ b/pkg/html/static/driver.go @@ -3,6 +3,7 @@ package static import ( "bytes" "context" + "github.com/MontFerret/ferret/pkg/runtime/core" "net/http" "net/url" @@ -14,23 +15,39 @@ import ( "github.com/sethgrid/pester" ) -type Driver struct { - client *pester.Client - options *Options -} +type ( + ctxKey struct{} -func NewDriver(opts ...Option) *Driver { - drv := new(Driver) - drv.options = &Options{ - concurrency: 3, - maxRetries: 5, - backoff: pester.ExponentialBackoff, + Driver struct { + client *pester.Client + options *Options } +) + +func WithContext(ctx context.Context, drv *Driver) context.Context { + return context.WithValue( + ctx, + ctxKey{}, + drv, + ) +} + +func FromContext(ctx context.Context) (*Driver, error) { + val := ctx.Value(ctxKey{}) - for _, opt := range opts { - opt(drv.options) + drv, ok := val.(*Driver) + + if !ok { + return nil, core.Error(core.ErrNotFound, "static HTML Driver") } + return drv, nil +} + +func NewDriver(opts ...Option) *Driver { + drv := new(Driver) + drv.options = newOptions(opts) + if drv.options.proxy == "" { drv.client = pester.New() } else { diff --git a/pkg/html/static/options.go b/pkg/html/static/options.go index 7559437b..3f2fa563 100644 --- a/pkg/html/static/options.go +++ b/pkg/html/static/options.go @@ -15,6 +15,19 @@ type ( } ) +func newOptions(setters []Option) *Options { + opts := new(Options) + opts.backoff = pester.ExponentialBackoff + opts.concurrency = 3 + opts.maxRetries = 5 + + for _, setter := range setters { + setter(opts) + } + + return opts +} + func WithDefaultBackoff() Option { return func(opts *Options) { opts.backoff = pester.DefaultBackoff diff --git a/pkg/runtime/env/env.go b/pkg/runtime/env/env.go deleted file mode 100644 index 37a22886..00000000 --- a/pkg/runtime/env/env.go +++ /dev/null @@ -1,31 +0,0 @@ -package env - -import "context" - -type ( - ctxKey struct{} - - Environment struct { - CDPAddress string - ProxyAddress string - UserAgent string - } -) - -const RandomUserAgent = "*" - -func WithContext(ctx context.Context, e Environment) context.Context { - return context.WithValue(ctx, ctxKey{}, e) -} - -func FromContext(ctx context.Context) Environment { - res := ctx.Value(ctxKey{}) - - val, ok := res.(Environment) - - if !ok { - return Environment{} - } - - return val -} diff --git a/pkg/runtime/options.go b/pkg/runtime/options.go index 65d8491d..c1d10174 100644 --- a/pkg/runtime/options.go +++ b/pkg/runtime/options.go @@ -2,35 +2,37 @@ package runtime import ( "context" + "io" + "os" + "github.com/MontFerret/ferret/pkg/runtime/core" - "github.com/MontFerret/ferret/pkg/runtime/env" "github.com/MontFerret/ferret/pkg/runtime/logging" "github.com/MontFerret/ferret/pkg/runtime/values" - "io" - "os" ) type ( Options struct { - proxy string - cdp string - params map[string]core.Value - logging *logging.Options - userAgent string + params map[string]core.Value + logging *logging.Options } Option func(*Options) ) -func NewOptions() *Options { - return &Options{ - cdp: "http://127.0.0.1:9222", +func NewOptions(setters []Option) *Options { + opts := &Options{ params: make(map[string]core.Value), logging: &logging.Options{ Writer: os.Stdout, Level: logging.ErrorLevel, }, } + + for _, setter := range setters { + setter(opts) + } + + return opts } func WithParam(name string, value interface{}) Option { @@ -47,30 +49,6 @@ func WithParams(params map[string]interface{}) Option { } } -func WithBrowser(address string) Option { - return func(options *Options) { - options.cdp = address - } -} - -func WithProxy(address string) Option { - return func(options *Options) { - options.proxy = address - } -} - -func WithUserAgent(value string) Option { - return func(options *Options) { - options.userAgent = value - } -} - -func WithRandomUserAgent() Option { - return func(options *Options) { - options.userAgent = env.RandomUserAgent - } -} - func WithLog(writer io.Writer) Option { return func(options *Options) { options.logging.Writer = writer @@ -83,22 +61,9 @@ func WithLogLevel(lvl logging.Level) Option { } } -func (opts *Options) Apply(setters ...Option) *Options { - for _, setter := range setters { - setter(opts) - } - - return opts -} - func (opts *Options) WithContext(parent context.Context) context.Context { ctx := core.ParamsWith(parent, opts.params) ctx = logging.WithContext(ctx, opts.logging) - ctx = env.WithContext(ctx, env.Environment{ - CDPAddress: opts.cdp, - ProxyAddress: opts.proxy, - UserAgent: opts.userAgent, - }) return ctx } diff --git a/pkg/runtime/program.go b/pkg/runtime/program.go index ae871661..7934a486 100644 --- a/pkg/runtime/program.go +++ b/pkg/runtime/program.go @@ -2,7 +2,6 @@ package runtime import ( "context" - "github.com/MontFerret/ferret/pkg/html" "github.com/MontFerret/ferret/pkg/runtime/core" "github.com/MontFerret/ferret/pkg/runtime/logging" "github.com/MontFerret/ferret/pkg/runtime/values" @@ -32,9 +31,7 @@ func (p *Program) Source() string { } func (p *Program) Run(ctx context.Context, setters ...Option) (result []byte, err error) { - ctx = NewOptions().Apply(setters...).WithContext(ctx) - ctx = html.WithDynamicDriver(ctx) - ctx = html.WithStaticDriver(ctx) + ctx = NewOptions(setters).WithContext(ctx) logger := logging.FromContext(ctx)