Decoupled runtime and HTML driver initialization (#198)

* Decoupled runtime and HTML driver initialization

* Updates

README.md

@@ -219,14 +219,16 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/MontFerret/ferret/pkg/compiler"
 	"os"
+
+    "github.com/MontFerret/ferret/pkg/compiler"
+    "github.com/MontFerret/ferret/pkg/html"
 )
 
 type Topic struct {
 	Name        string `json:"name"`
 	Description string `json:"description"`
-	Url         string `json:"url"`
+	URL         string `json:"url"`
 }
 
 func main() {
@@ -238,7 +240,7 @@ func main() {
 	}
 
 	for _, topic := range topics {
-		fmt.Println(fmt.Sprintf("%s: %s %s", topic.Name, topic.Description, topic.Url))
+		fmt.Println(fmt.Sprintf("%s: %s %s", topic.Name, topic.Description, topic.URL))
 	}
 }
 
@@ -267,7 +269,16 @@ func getTopTenTrendingTopics() ([]*Topic, error) {
 		return nil, err
 	}
 
-	out, err := program.Run(context.Background())
+	// create a root context
+	ctx := context.Background()
+
+	// enable HTML drivers
+	// by default, Ferret Runtime knows nothing about HTML drivers
+	// all HTML manipulations are done via functions from standard library
+	ctx = html.WithDynamicDriver(ctx)
+	ctx = html.WithStaticDriver(ctx)
+
+	out, err := program.Run(ctx)
 
 	if err != nil {
 		return nil, err
@@ -283,6 +294,7 @@ func getTopTenTrendingTopics() ([]*Topic, error) {
 
 	return res, nil
 }
+
 ```
 
 ## Extensibility
@@ -296,10 +308,12 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
+	"strings"
+
 	"github.com/MontFerret/ferret/pkg/compiler"
 	"github.com/MontFerret/ferret/pkg/runtime/core"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
-	"os"
 )
 
 func main() {
@@ -319,7 +333,7 @@ func getStrings() ([]string, error) {
 	// function implements is a type of a function that ferret supports as a runtime function
 	transform := func(ctx context.Context, args ...core.Value) (core.Value, error) {
 		// it's just a helper function which helps to validate a number of passed args
-		err := core.ValidateArgs(args, 1)
+		err := core.ValidateArgs(args, 1, 1)
 
 		if err != nil {
 			// it's recommended to return built-in None type, instead of nil
@@ -336,7 +350,7 @@ func getStrings() ([]string, error) {
 		// cast to built-in string type
 		str := args[0].(values.String)
 
-		return str.Concat(values.NewString("_ferret")).ToUpper(), nil
+		return values.NewString(strings.ToUpper(str.String() + "_ferret")), nil
 	}
 
 	query := `
@@ -346,7 +360,10 @@ func getStrings() ([]string, error) {
 	`
 
 	comp := compiler.New()
-	comp.RegisterFunction("transform", transform)
+
+	if err := comp.RegisterFunction("transform", transform); err != nil {
+		return nil, err
+	}
 
 	program, err := comp.Compile(query)
 

cli/exec.go

@@ -38,7 +38,7 @@ func Exec(query string, opts Options) {
 
 	l := NewLogger()
 
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(opts.WithContext(context.Background()))
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, syscall.SIGHUP)
 
@@ -59,12 +59,9 @@ func Exec(query string, opts Options) {
 
 	out, err := prog.Run(
 		ctx,
-		runtime.WithBrowser(opts.Cdp),
 		runtime.WithLog(l),
 		runtime.WithLogLevel(logging.DebugLevel),
 		runtime.WithParams(opts.Params),
-		runtime.WithProxy(opts.Proxy),
-		runtime.WithUserAgent(opts.UserAgent),
 	)
 
 	if opts.ShowTime {

cli/options.go

@@ -1,9 +1,33 @@
 package cli
 
+import (
+	"context"
+	"github.com/MontFerret/ferret/pkg/html"
+	"github.com/MontFerret/ferret/pkg/html/dynamic"
+	"github.com/MontFerret/ferret/pkg/html/static"
+)
+
 type Options struct {
 	Cdp       string
 	Params    map[string]interface{}
 	Proxy     string
 	UserAgent string
 	ShowTime  bool
 }
+
+func (opts Options) WithContext(ctx context.Context) context.Context {
+	ctx = html.WithDynamicDriver(
+		ctx,
+		dynamic.WithCDP(opts.Cdp),
+		dynamic.WithProxy(opts.Proxy),
+		dynamic.WithUserAgent(opts.UserAgent),
+	)
+
+	ctx = html.WithStaticDriver(
+		ctx,
+		static.WithProxy(opts.Proxy),
+		static.WithUserAgent(opts.UserAgent),
+	)
+
+	return ctx
+}

cli/repl.go

@@ -42,7 +42,7 @@ func Repl(version string, opts Options) {
 
 	l := NewLogger()
 
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(opts.WithContext(context.Background()))
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, syscall.SIGHUP)
 
@@ -110,12 +110,9 @@ func Repl(version string, opts Options) {
 
 		out, err := program.Run(
 			ctx,
-			runtime.WithBrowser(opts.Cdp),
 			runtime.WithLog(l),
 			runtime.WithLogLevel(logging.DebugLevel),
 			runtime.WithParams(opts.Params),
-			runtime.WithProxy(opts.Proxy),
-			runtime.WithUserAgent(opts.UserAgent),
 		)
 
 		if err != nil {

e2e/runner/runner.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"encoding/json"
 	"github.com/MontFerret/ferret/pkg/compiler"
+	"github.com/MontFerret/ferret/pkg/html"
+	"github.com/MontFerret/ferret/pkg/html/dynamic"
 	"github.com/MontFerret/ferret/pkg/runtime"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -131,10 +133,16 @@ func (r *Runner) runQuery(c *compiler.FqlCompiler, name, script string) Result {
 		}
 	}
 
+	ctx := context.Background()
+	ctx = html.WithDynamicDriver(
+		ctx,
+		dynamic.WithCDP(r.settings.CDPAddress),
+	)
+	ctx = html.WithStaticDriver(ctx)
+
 	out, err := p.Run(
-		context.Background(),
+		ctx,
 		runtime.WithLog(os.Stdout),
-		runtime.WithBrowser(r.settings.CDPAddress),
 		runtime.WithParam("static", r.settings.StaticServerAddress),
 		runtime.WithParam("dynamic", r.settings.DynamicServerAddress),
 	)

examples/embedded.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/MontFerret/ferret/pkg/compiler"
 	"os"
+
+	"github.com/MontFerret/ferret/pkg/compiler"
+	"github.com/MontFerret/ferret/pkg/html"
 )
 
 type Topic struct {
@@ -52,7 +54,16 @@ func getTopTenTrendingTopics() ([]*Topic, error) {
 		return nil, err
 	}
 
-	out, err := program.Run(context.Background())
+	// create a root context
+	ctx := context.Background()
+
+	// enable HTML drivers
+	// by default, Ferret Runtime knows nothing about HTML drivers
+	// all HTML manipulations are done via functions from standard library
+	ctx = html.WithDynamicDriver(ctx)
+	ctx = html.WithStaticDriver(ctx)
+
+	out, err := program.Run(ctx)
 
 	if err != nil {
 		return nil, err

examples/extensible.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
+	"strings"
+
 	"github.com/MontFerret/ferret/pkg/compiler"
 	"github.com/MontFerret/ferret/pkg/runtime/core"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
-	"os"
 )
 
 func main() {
@@ -27,7 +29,7 @@ func getStrings() ([]string, error) {
 	// function implements is a type of a function that ferret supports as a runtime function
 	transform := func(ctx context.Context, args ...core.Value) (core.Value, error) {
 		// it's just a helper function which helps to validate a number of passed args
-		err := core.ValidateArgs(args, 1)
+		err := core.ValidateArgs(args, 1, 1)
 
 		if err != nil {
 			// it's recommended to return built-in None type, instead of nil
@@ -44,7 +46,7 @@ func getStrings() ([]string, error) {
 		// cast to built-in string type
 		str := args[0].(values.String)
 
-		return str.Concat(values.NewString("_ferret")).ToUpper(), nil
+		return values.NewString(strings.ToUpper(str.String() + "_ferret")), nil
 	}
 
 	query := `
@@ -54,7 +56,10 @@ func getStrings() ([]string, error) {
 	`
 
 	comp := compiler.New()
-	comp.RegisterFunction("transform", transform)
+
+	if err := comp.RegisterFunction("transform", transform); err != nil {
+		return nil, err
+	}
 
 	program, err := comp.Compile(query)
 

pkg/html/common/ua.go

@@ -1,16 +1,17 @@
 package common
 
 import (
-	"github.com/MontFerret/ferret/pkg/runtime/env"
 	"github.com/corpix/uarand"
 )
 
+const RandomUserAgent = "*"
+
 func GetUserAgent(val string) string {
 	if val == "" {
 		return val
 	}
 
-	if val != env.RandomUserAgent {
+	if val != RandomUserAgent {
 		return val
 	}
 

pkg/html/driver.go

@@ -6,15 +6,10 @@ import (
 	"github.com/MontFerret/ferret/pkg/html/dynamic"
 	"github.com/MontFerret/ferret/pkg/html/static"
 	"github.com/MontFerret/ferret/pkg/runtime/core"
-	"github.com/MontFerret/ferret/pkg/runtime/env"
 	"github.com/MontFerret/ferret/pkg/runtime/values"
 )
 
-type (
-	DriverName    string
-	dynamicCtxKey struct{}
-	staticCtxKey  struct{}
-)
+type DriverName string
 
 const (
 	Dynamic DriverName = "dynamic"
@@ -26,67 +21,21 @@ type Driver interface {
 	Close() error
 }
 
-func ToContext(ctx context.Context, name DriverName, drv Driver) context.Context {
-	var key interface{}
-
-	switch name {
-	case Dynamic:
-		key = dynamicCtxKey{}
-	case Static:
-		key = staticCtxKey{}
-	default:
-		return ctx
-	}
-
-	return context.WithValue(ctx, key, drv)
-}
-
 func FromContext(ctx context.Context, name DriverName) (Driver, error) {
-	var key interface{}
-
 	switch name {
 	case Dynamic:
-		key = dynamicCtxKey{}
+		return dynamic.FromContext(ctx)
 	case Static:
-		key = staticCtxKey{}
+		return static.FromContext(ctx)
 	default:
 		return nil, core.Error(core.ErrInvalidArgument, fmt.Sprintf("%s driver", name))
 	}
-
-	val := ctx.Value(key)
-
-	drv, ok := val.(Driver)
-
-	if ok {
-		return drv, nil
-	}
-
-	return nil, core.Error(core.ErrNotFound, fmt.Sprintf("%s driver", name))
 }
 
-func WithDynamicDriver(ctx context.Context) context.Context {
-	e := env.FromContext(ctx)
-
-	return context.WithValue(
-		ctx,
-		dynamicCtxKey{},
-		dynamic.NewDriver(
-			e.CDPAddress,
-			dynamic.WithProxy(e.ProxyAddress),
-			dynamic.WithUserAgent(e.UserAgent),
-		),
-	)
+func WithDynamicDriver(ctx context.Context, opts ...dynamic.Option) context.Context {
+	return dynamic.WithContext(ctx, dynamic.NewDriver(opts...))
 }
 
-func WithStaticDriver(ctx context.Context) context.Context {
-	e := env.FromContext(ctx)
-
-	return context.WithValue(
-		ctx,
-		staticCtxKey{},
-		static.NewDriver(
-			static.WithProxy(e.ProxyAddress),
-			static.WithUserAgent(e.UserAgent),
-		),
-	)
+func WithStaticDriver(ctx context.Context, opts ...static.Option) context.Context {
+	return static.WithContext(ctx, static.NewDriver(opts...))
 }