Merge pull request #20 from bytedance/dev/spport_foreach

Dev/spport foreach

arishem/arishem_config.go

@@ -37,17 +37,39 @@ type TreeCache interface {
 	Get(expr string) (tc *RuleTree, ok bool)
 }
 
+type SubConditionManage interface {
+	// WhenConditionParsed will be called when a condition expression successfully parsed
+	WhenConditionParsed(condName, expr string, tree antlr.ParseTree)
+	// GetConditionTree defines arishem how to get the condition parse tree by the condition name/key
+	GetConditionTree(condName string) (antlr.ParseTree, error)
+	// RuleIdentityMapAsCondName if this function return true,
+	// then every rule identity will be the condition name stored into cache when parse rule or condition
+	RuleIdentityMapAsCondName() bool
+}
+
 type Configuration struct {
 	Granularity func(int, ExecuteMode) int
+
 	// prefetch feature
 	Prefetch bool
 	TCache   TreeCache
+
 	// max parallel computation
 	MaxParallel        int
 	RuleComputePool    typedef.ConcurrentPool
 	FeatureFetchPool   typedef.ConcurrentPool
 	FeatFetcherFactory func() typedef.FeatureFetcher
 	FeatVisitCache     typedef.SharedVisitCache
+
+	// sub condition configuration
+	SubCond SubConditionManage
+}
+
+func (c *Configuration) getConditionFinder() func(string) (antlr.ParseTree, error) {
+	if c.SubCond == nil {
+		return nil
+	}
+	return c.SubCond.GetConditionTree
 }
 
 const (

arishem/arishem_config_init_test.go

@@ -31,6 +31,7 @@ func init() {
 		WithCustomNoParamFuncs(NoParamFnPair{Name: "NPHello", Fn: MyCustomNoFuncHelloArishem}),
 		WithCustomMapParamFuncs(MapParamFnPair{Name: "MPHello", Fn: MyCustomMapFuncHelloArishem}),
 		WithCustomListParamFuncs(ListParamFnPair{Name: "LPHello", Fn: MyCustomListFuncHelloArishem}),
+		WithEnableSubCondition("OnlyPriceMatch", `{"OpLogic":"&&","Conditions":[{"Operator":">","Lhs":{"VarExpr":"price"},"Rhs":{"Const":{"NumConst":10}}}]}`),
 	)
 }
 

arishem/arishem_def_opts.go

@@ -18,13 +18,16 @@ package arishem
 
 import (
 	"context"
+	"errors"
+	"github.com/antlr/antlr4/runtime/Go/antlr/v4"
 	"github.com/bytedance/arishem/internal/pool"
 	"github.com/bytedance/arishem/tools"
 	"github.com/bytedance/arishem/typedef"
 	"github.com/dgraph-io/ristretto"
 	"math"
 	"runtime"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -40,7 +43,7 @@ var (
 
 func WithDefMaxParallels() Option {
 	return func(cfg *Configuration) {
-		cfg.MaxParallel = 10 * (1 << 12)
+		cfg.MaxParallel = 1 << 12
 	}
 }
 
@@ -96,6 +99,57 @@ func WithDefFeatVisitCache() Option {
 	}
 }
 
+// WithEnableSubCondition will enable arishem using sub condition to judge a parent-condition which right hand type is SubCondExpr,
+// pairs allow users to add some default conditions when enable this feature
+// PAY ATTENTION: WithEnableSubCondition will panic if initialize invalid condition pair.
+func WithEnableSubCondition(pairs ...string) Option {
+	return func(cfg *Configuration) {
+		cfg.SubCond = &defaultSubConditionManage{
+			ExprDict: make(map[string]string, 128),
+		}
+		err := AddSubCondition(pairs...)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+func WithCustomSubConditionConfig(sc SubConditionManage) Option {
+	return func(cfg *Configuration) {
+		cfg.SubCond = sc
+	}
+}
+
+type defaultSubConditionManage struct {
+	lock     sync.RWMutex
+	ExprDict map[string]string
+}
+
+func (d *defaultSubConditionManage) WhenConditionParsed(condName, expr string, _ antlr.ParseTree) {
+	d.lock.Lock()
+	d.ExprDict[condName] = expr
+	d.lock.Unlock()
+}
+
+func (d *defaultSubConditionManage) GetConditionTree(condName string) (antlr.ParseTree, error) {
+	d.lock.RLock()
+	expr, exist := d.ExprDict[condName]
+	d.lock.RUnlock()
+
+	if !exist {
+		return nil, errors.New("expression not find by condition name: " + condName)
+	}
+	tree, err := ParseCondition(expr)
+	if err != nil {
+		return nil, err
+	}
+	return tree.Tree, nil
+}
+
+func (d *defaultSubConditionManage) RuleIdentityMapAsCondName() bool {
+	return true
+}
+
 type defaultFeatureFeature struct{}
 
 func (d *defaultFeatureFeature) AddFetchObserver(v ...typedef.FeatureFetchObserver) {}

arishem/arishem_lw_api.go

@@ -34,17 +34,47 @@ type dummyVisitTarget struct{ name string }
 
 func (d *dummyVisitTarget) Identifier() string { return d.name }
 
-// DataContext will create a new DataContext by fact meta json.
+// DataContext will create a new typedef.DataCtx by fact meta json.
 func DataContext(ctx context.Context, json string) (dc typedef.DataCtx, err error) {
 	dc, err = core.NewArishemDataCtx(ctx, json, arishemConfiguration.FeatFetcherFactory())
 	return
 }
 
+// DataContextFromMeta will create a new typedef.DataCtx by fact meta map
+func DataContextFromMeta(ctx context.Context, meta typedef.MetaType) (dc typedef.DataCtx, err error) {
+	dc, err = core.NewArishemDataCtxFromMeta(ctx, meta, arishemConfiguration.FeatFetcherFactory())
+	return
+}
+
 // ParseCondition will parse a condition string and return the rule tree with feature parameter pre-parsed.
 func ParseCondition(condition string) (tree *RuleTree, err error) {
 	return ParseRuleTree(condition, ExprTypeCondition)
 }
 
+// AddSubCondition adds a sub condition to arishem, it will be used at the moment of judging a condition which right hand type is SubCondExpr,
+// pairs allow users to add some conditions when enable arishem's sub-condition feature
+func AddSubCondition(pairs ...string) error {
+	size := len(pairs)
+	if size%2 != 0 {
+		return errors.New("pair number not valid")
+	}
+	if arishemConfiguration.SubCond == nil {
+		return errors.New("sub condition not enabled")
+	}
+	for i := 0; i < size; i += 2 {
+		condName := pairs[i]
+		condition := pairs[i+1]
+		tree, err := ParseRuleTree(condition, ExprTypeCondition)
+		if err != nil {
+			return err
+		}
+		if tree != nil && tree.Tree != nil {
+			arishemConfiguration.SubCond.WhenConditionParsed(condName, condition, tree.Tree)
+		}
+	}
+	return nil
+}
+
 // ParseAim will parse an aim string and return the rule tree with feature parameter pre-parsed.
 func ParseAim(aim string) (tree *RuleTree, err error) {
 	return ParseRuleTree(aim, ExprTypeAim)
@@ -86,7 +116,7 @@ func WalkAim(aimExpr string, dc typedef.DataCtx, opts ...ExecuteOption) (aim typ
 	if err != nil {
 		return
 	}
-	rv := core.NewArishemRuleVisitor()
+	rv := core.NewArishemRuleVisitor(arishemConfiguration.getConditionFinder())
 	ApplyExecuteOptions(rv, dc, opts...)
 	if len(tree.FeatParams) > 0 {
 		dc.PrefetchFeatures(tree.FeatParams)
@@ -97,7 +127,7 @@ func WalkAim(aimExpr string, dc typedef.DataCtx, opts ...ExecuteOption) (aim typ
 
 // WalkAimTree will visit the aimTree to get the result
 func WalkAimTree(aimTree antlr.ParseTree, dc typedef.DataCtx, opts ...ExecuteOption) (aim typedef.Aim) {
-	rv := core.NewArishemRuleVisitor()
+	rv := core.NewArishemRuleVisitor(arishemConfiguration.getConditionFinder())
 	ApplyExecuteOptions(rv, dc, opts...)
 	aim = rv.VisitAim(aimTree, dc, &dummyVisitTarget{name: aimTree.GetText()})
 	return
@@ -127,7 +157,7 @@ func JudgeConditionWithFactMeta(ctx context.Context, condition, factMeta string,
 	if err != nil {
 		return
 	}
-	rv := core.NewArishemRuleVisitor()
+	rv := core.NewArishemRuleVisitor(arishemConfiguration.getConditionFinder())
 	ApplyExecuteOptions(rv, dc, opts...)
 	if len(tree.FeatParams) > 0 {
 		dc.PrefetchFeatures(tree.FeatParams)

arishem/arishem_rule.go

@@ -106,7 +106,7 @@ func NewNoPriorityRule(name string, cdtExpr, aimExpr string) (npr RuleTarget, er
 		err = errors.New("aim expression is empty")
 		return
 	}
-	cdtTree, aimTree, err := tryParse(cdtExpr, aimExpr)
+	cdtTree, aimTree, err := tryParse(cdtExpr, aimExpr, name)
 	if err != nil {
 		return
 	}
@@ -161,7 +161,7 @@ func (p *PriorityRule) Compare(other typedef.Comparable) int {
 	return cmp
 }
 
-func tryParse(cdtExpr, aimExpr string) (cdtTree, aimTree *RuleTree, err error) {
+func tryParse(cdtExpr, aimExpr, name string) (cdtTree, aimTree *RuleTree, err error) {
 	// compat two expressions
 	cdtExpr, err = tools.Compat(cdtExpr)
 	if err != nil {
@@ -205,6 +205,9 @@ func tryParse(cdtExpr, aimExpr string) (cdtTree, aimTree *RuleTree, err error) {
 
 	if cdtTree != nil {
 		arishemConfiguration.TCache.Put(cdtExpr, cdtTree)
+		if arishemConfiguration.SubCond != nil && arishemConfiguration.SubCond.RuleIdentityMapAsCondName() {
+			arishemConfiguration.SubCond.WhenConditionParsed(name, cdtExpr, cdtTree.Tree)
+		}
 	}
 	if aimTree != nil {
 		arishemConfiguration.TCache.Put(aimExpr, aimTree)

arishem/arishem_rule_execute.go

@@ -42,7 +42,7 @@ func ExecuteSingleRule(rule RuleTarget, dc typedef.DataCtx, opts ...ExecuteOptio
 		passed: false,
 		aim:    nil,
 	}
-	rv := core.NewArishemRuleVisitor()
+	rv := core.NewArishemRuleVisitor(arishemConfiguration.getConditionFinder())
 	if len(opts) > 0 {
 		// consume visit context option
 		ApplyExecuteOptions(rv, nil, opts...)
@@ -106,7 +106,7 @@ func executePriorityRules(priRules []RuleTarget, dc typedef.DataCtx, opts ...Exe
 	// batch rules
 	batchedRules := batchRules(priRules, arishemConfiguration.Granularity(len(priRules), ExecuteModePriority))
 	// create visitor
-	rv := core.NewArishemRuleVisitor()
+	rv := core.NewArishemRuleVisitor(arishemConfiguration.getConditionFinder())
 	ApplyExecuteOptions(rv, dc, opts...)
 outer:
 	for idx, rules := range batchedRules {
@@ -152,7 +152,7 @@ func executeNoPriorityRules(noPriRules []RuleTarget, dc typedef.DataCtx, opts ..
 				defer wg.Done()
 				r := rI.(RuleTarget)
 
-				rv := core.NewArishemRuleVisitor()
+				rv := core.NewArishemRuleVisitor(arishemConfiguration.getConditionFinder())
 				ApplyExecuteOptions(rv, nil, opts...)
 				pass := rv.VisitCondition(r.ConditionPTree(), dc, r)
 				if pass {

arishem/arishem_rule_execute_func_test.go

@@ -35,7 +35,7 @@ func TestFuncExprNoParam(t *testing.T) {
             },
             "Rhs": {
                 "Const": {
-                    "NumConst": 2023
+                    "NumConst": 2024
                 }
             }
         }

arishem/arishem_rule_execute_test.go

@@ -357,3 +357,71 @@ func TestExecuteRules(t *testing.T) {
 		}
 	}
 }
+
+func TestForeachWithSubCondition(t *testing.T) {
+	// case1-1 test OnlyPriceMatch, OnlyPriceMatch added when initialize the arishem
+	pass, err := JudgeConditionWithFactMeta(
+		context.Background(),
+		`{"OpLogic":"&&","Conditions":[{"Operator":"FOREACH SUB_COND and","Lhs":{"VarExpr":"item_info.item_list"},"Rhs":{"SubCondExpr":{"CondName":"OnlyPriceMatch"}}}]}`,
+		`{"item_info":{"item_list":[{"name":"name@1","price":100},{"name":"name@2","price":102.13},{"name":"name@3","price":200},{"name":"name@4","price":100},{"name":"name@5","price":101},{"name":"name@6","price":303.1234}]}}`,
+	)
+	assert.Nil(t, err)
+	assert.True(t, pass)
+	// case1-2 use rule execution while dependent on the sub condition
+	rule, err := NewNoPriorityRule(
+		"depend-OnlyPriceMatch",
+		`{"OpLogic":"&&","Conditions":[{"Operator":"FOREACH SUB_COND and","Lhs":{"VarExpr":"item_info.item_list"},"Rhs":{"SubCondExpr":{"CondName":"OnlyPriceMatch"}}}]}`,
+		`{
+    "Const": {
+        "StrConst": "depend-OnlyPriceMatch"
+    }
+}`)
+	assert.Nil(t, err)
+	dc, err := DataContext(context.Background(), `{"item_info":{"item_list":[{"name":"name@1","price":100},{"name":"name@2","price":102.13},{"name":"name@3","price":200},{"name":"name@4","price":100},{"name":"name@5","price":101},{"name":"name@6","price":303.1234}]}}`)
+	assert.Nil(t, err)
+	//rr := ExecuteSingleRule(rule, dc, WithVisitObserver(NewMyObserver("obs")))
+	rr := ExecuteSingleRule(rule, dc)
+	assert.NotNil(t, rr)
+	assert.True(t, rr.Passed())
+
+	// case2 test OnlyNameMatch, OnlyNameMatch added by create a rule
+	rule, err = NewNoPriorityRule("OnlyNameMatch", `{"OpLogic":"&&","Conditions":[{"Operator":"STRING_CONTAINS","Lhs":{"VarExpr":"name"},"Rhs":{"Const":{"StrConst":"name@"}}}]}`, `{
+		"Const": {
+			"StrConst": "OnlyNameMatch"
+		}
+	}`)
+	assert.Nil(t, err)
+	pass, err = JudgeConditionWithFactMeta(
+		context.Background(),
+		`{"OpLogic":"&&","Conditions":[{"Operator":"FOREACH SUB_COND and","Lhs":{"VarExpr":"item_info.item_list"},"Rhs":{"SubCondExpr":{"CondName":"OnlyNameMatch"}}}]}`,
+		`{"item_info":{"item_list":[{"name":"name@1","price":100},{"name":"name@2","price":102.13},{"name":"name@3","price":200},{"name":"name@4","price":100},{"name":"name@5","price":101},{"name":"name@6","price":303.1234}]}}`,
+	)
+	assert.Nil(t, err)
+	assert.True(t, pass)
+	// case3 test NameAndPriceMatch, NameAndPriceMatch added by AddSubCondition
+	err = AddSubCondition("NameAndPriceMatch", `{"OpLogic":"&&","Conditions":[{"Operator":">","Lhs":{"VarExpr":"price"},"Rhs":{"Const":{"NumConst":10}}},{"Operator":"STRING_CONTAINS","Lhs":{"VarExpr":"name"},"Rhs":{"Const":{"StrConst":"name@"}}}]}`)
+	assert.Nil(t, err)
+	pass, err = JudgeConditionWithFactMeta(
+		context.Background(),
+		`{"OpLogic":"&&","Conditions":[{"Operator":"FOREACH SUB_COND and","Lhs":{"VarExpr":"item_info.item_list"},"Rhs":{"SubCondExpr":{"CondName":"NameAndPriceMatch"}}}]}`,
+		`{"item_info":{"item_list":[{"name":"name@1","price":100},{"name":"name@2","price":102.13},{"name":"name@3","price":200},{"name":"name@4","price":100},{"name":"name@5","price":101},{"name":"name@6","price":303.1234}]}}`,
+	)
+	assert.Nil(t, err)
+	assert.True(t, pass)
+	// case4 normal foreach test
+	pass, err = JudgeConditionWithFactMeta(
+		context.Background(),
+		`{"OpLogic":"&&","Conditions":[{"Operator":"FOREACH < or","Lhs":{"VarExpr":"item_info.price_list"},"Rhs":{"Const":{"NumConst":-1}}}]}`,
+		`{"item_info":{"price_list":[100,102.13,200,100,101,303.1234]}}`,
+	)
+	assert.Nil(t, err)
+	assert.False(t, pass)
+
+	pass, err = JudgeConditionWithFactMeta(
+		context.Background(),
+		`{"OpLogic":"&&","Conditions":[{"Operator":"FOREACH > or","Lhs":{"VarExpr":"item_info.price_list"},"Rhs":{"Const":{"NumConst":-1}}}]}`,
+		`{"item_info":{"price_list":[100,102.13,200,100,101,303.1234]}}`,
+	)
+	assert.Nil(t, err)
+	assert.True(t, pass)
+}

arishem/arishem_rule_test.go

@@ -71,7 +71,7 @@ func TestTryParse(t *testing.T) {
             }
         }
     }
-}`)
+}`, "NAME")
 	assert.NotNil(t, err)
 	t.Log(err.Error())
 	assert.Nil(t, cdtTree)
@@ -90,7 +90,7 @@ func TestTryParse2(t *testing.T) {
             }
         }
     }
-}`)
+}`, "NAME")
 	assert.NotNil(t, err)
 	assert.Nil(t, cdtTree)
 	assert.Nil(t, aimTree)