Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add optimization with cost and fix unit test #1233

Merged
merged 2 commits into from
Jun 10, 2019
Merged

Add optimization with cost and fix unit test #1233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
40e9717
Add optimization with cost and fix unit testush
chenhao7253886 Jun 4, 2019
fb3b920
Fix bug that MultExpression list in on group generates circle references
chenhao7253886 Jun 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions fe/src/main/java/org/apache/doris/optimizer/ExpressionToStmtConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,14 @@ public PlanNode convertPhysical(OptExpression expr) {
OptOperator op = expr.getOp();
switch (op.getType()) {
case OP_PHYSICAL_MYSQL_SCAN:
return convertMysqlScan(expr)
return convertMysqlScan(expr);
}
return null;
}

public MysqlScanNode convertMysqlScan(OptExpression expr) {
OptPhysicalMysqlScan scanOp = (OptPhysicalMysqlScan) expr.getOp();
MysqlScanNode node = new MysqlScanNode()
// OptPhysicalMysqlScan scanOp = (OptPhysicalMysqlScan) expr.getOp();
// MysqlScanNode node = new MysqlScanNode();
return null;
}

Expand Down Expand Up @@ -106,6 +106,7 @@ public Expr convertItem(OptExpression expr) throws UserException {
case OP_ITEM_IS_NULL_PREDICATE:
return convertIsNullPred(expr);
}
return null;
}

private Expr convertConst(OptExpression expr) {
Expand Down
32 changes: 24 additions & 8 deletions fe/src/main/java/org/apache/doris/optimizer/MultiExpression.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.doris.optimizer.base.OptCostContext;
import org.apache.doris.optimizer.base.OptimizationContext;
import org.apache.doris.optimizer.base.RequiredLogicalProperty;
import org.apache.doris.optimizer.cost.CostModel;
import org.apache.doris.optimizer.operator.OptExpressionHandle;
import org.apache.doris.optimizer.operator.OptLogical;
import org.apache.doris.optimizer.operator.OptOperator;
Expand Down Expand Up @@ -81,7 +84,7 @@ public MultiExpression(OptOperator op, List<OptGroup> inputs) {
public OptRuleType getRuleTypeDerivedFrom() { return ruleTypeDerivedFrom; }
public void setStatus(MEState status) { this.status = status; }
public MEState getStatus() { return status; }
public boolean isImplemented() { return status == MEState.Implemented; }
public boolean isImplemented() { return status == MEState.Implemented || op.isPhysical(); }
public boolean isOptimized() { return status == MEState.Optimized; }
public void setNext(MultiExpression next) { this.next = next; }
public void setInvalid() { this.group = null; }
Expand All @@ -96,9 +99,12 @@ public void addImplementedMExpr(MultiExpression mExpr) {
this.implementedMExprs.add(mExpr);
}

public Statistics deriveStat(OptExpressionHandle exprHandle, RequiredLogicalProperty property) {
final OptLogical logical = (OptLogical)op;
return logical.deriveStat(exprHandle, property);
public OptCostContext computeCost(OptimizationContext optContext, CostModel costModel) {
final OptCostContext costContext = new OptCostContext(this, optContext);
optContext.getChildrenOptContext().stream().forEach(
childOptContext -> costContext.addChildrenOptContext(childOptContext));
costContext.compute(costModel);
return costContext;
}

public String debugString() {
Expand All @@ -111,25 +117,35 @@ public String debugString() {
}

public final String getExplainString() {
return getExplainString("", "");
return getExplainString("", "", OptGroup.ExplainType.ALL);
}

public String getExplainString(String headlinePrefix, String detailPrefix) {
public String getLogicalExplainString(String headlinePrefix, String detailPrefix) {
return getExplainString("", "", OptGroup.ExplainType.LOGICAL);
}

public String getPhysicalExplainString(String headlinePrefix, String detailPrefix) {
return getExplainString("", "", OptGroup.ExplainType.PHYSICAL);
}

public String getExplainString(String headlinePrefix, String detailPrefix, OptGroup.ExplainType type) {
StringBuilder sb = new StringBuilder();
sb.append(headlinePrefix).append("MultiExpression ").append(id);
if (ruleTypeDerivedFrom != OptRuleType.RULE_NONE) {
sb.append(" (from MultiExpression ")
sb.append(" (from MultiExpression: ")
.append(sourceMExprId)
.append(" rule:")
.append(ruleTypeDerivedFrom)
.append(" Status:")
.append(status.toString())
.append(")");
}
sb.append(' ')
.append(op.getExplainString(detailPrefix)).append('\n');
String childHeadlinePrefix = detailPrefix + OptUtils.HEADLINE_PREFIX;
String childDetailPrefix = detailPrefix + OptUtils.DETAIL_PREFIX;
for (OptGroup input : inputs) {
sb.append(input.getExplain(childHeadlinePrefix, childDetailPrefix));
sb.append(input.getExplain(childHeadlinePrefix, childDetailPrefix, type));
}
return sb.toString();
}
Expand Down
20 changes: 16 additions & 4 deletions fe/src/main/java/org/apache/doris/optimizer/OptBinding.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,14 @@ public static OptExpression bind(OptExpression pattern, MultiExpression mExpr, O
}
// If MultiExpression's arity isn't equal with pattern's, return null
int arity = mExpr.arity();
if (arity != pattern.arity()) {
if (arity < pattern.arity()) {
return null;
}
// we should binding children
List<OptExpression> boundInputs = Lists.newArrayList();
if (lastExpr == null) {
for (int i = 0; i < arity; ++i) {
OptExpression inputPattern = pattern.getInput(i);
OptExpression inputPattern = extractPattern(pattern, i);
OptGroup inputGroup = mExpr.getInput(i);

OptExpression boundInput = bind(inputPattern, inputGroup, null);
Expand Down Expand Up @@ -105,8 +105,10 @@ private static OptExpression bind(OptExpression pattern, OptGroup group, OptExpr
MultiExpression mExpr;
if (lastExpr != null) {
mExpr = lastExpr.getMExpr();
} else {
} else if (group.isItemGroup()) {
mExpr = group.getFirstMultiExpression();
} else {
mExpr = group.getFirstLogicalMultiExpression();
}

// Fast path. When pattern is PatternLeaf and lastExpr is not null it means that
Expand All @@ -115,14 +117,24 @@ private static OptExpression bind(OptExpression pattern, OptGroup group, OptExpr
if (pattern.getOp().isPatternAndLeaf() && lastExpr != null) {
return null;
}
//
// if (pattern.getOp().isPatternAndTree() || pattern.getOp().isPatternAndMultiTree()) {
//
// }

do {
OptExpression expr = bind(pattern, mExpr, lastExpr);
if (expr != null) {
return expr;
}
lastExpr = null;
mExpr = mExpr.next();
if (group.isItemGroup()) {
group.getItemExpression();
mExpr = mExpr.next();
} else {
mExpr = group.nextLogicalExpr(mExpr);
}

} while (mExpr != null);
return null;
}
Expand Down
39 changes: 28 additions & 11 deletions fe/src/main/java/org/apache/doris/optimizer/OptExpression.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,11 @@

package org.apache.doris.optimizer;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.doris.optimizer.base.OptCost;
import org.apache.doris.optimizer.base.OptItemProperty;
import org.apache.doris.optimizer.base.OptLogicalProperty;
import org.apache.doris.optimizer.base.OptPhysicalProperty;
import org.apache.doris.optimizer.base.OptProperty;
import org.apache.doris.optimizer.base.*;
import org.apache.doris.optimizer.cost.OptCost;
import org.apache.doris.optimizer.operator.OptExpressionHandle;
import org.apache.doris.optimizer.operator.OptLogicalProject;
import org.apache.doris.optimizer.operator.OptOperator;
import org.apache.doris.optimizer.stat.Statistics;
import org.apache.logging.log4j.LogManager;
Expand Down Expand Up @@ -124,9 +121,17 @@ public static OptExpression createBindingInternalExpression(
public OptExpression getInput(int idx) { return inputs.get(idx); }
public MultiExpression getMExpr() { return mExpr; }
public void setProperty(OptProperty property) { this.property = property; };
public OptProperty getProperty() { return deriveProperty(); }
public OptLogicalProperty getLogicalProperty() { return (OptLogicalProperty) deriveProperty(); }
public OptItemProperty getItemProperty() { return (OptItemProperty) deriveProperty(); }
public OptProperty getProperty() { return property; }
public OptLogicalProperty getLogicalProperty() {
final OptProperty property = deriveProperty();
Preconditions.checkArgument(property instanceof OptLogicalProperty);
return (OptLogicalProperty)property;
}
public OptItemProperty getItemProperty() {
final OptProperty property = deriveProperty();
Preconditions.checkArgument(property instanceof OptItemProperty);
return (OptItemProperty) deriveProperty();
}
public OptPhysicalProperty getPhysicalProperty() { return (OptPhysicalProperty) deriveProperty(); }
public Statistics getStatistics() { return statistics; }
public void setStatistics(Statistics statistics) { this.statistics = statistics; }
Expand Down Expand Up @@ -174,12 +179,24 @@ public OptProperty deriveProperty() {
for (OptExpression input : inputs) {
input.deriveProperty();
}
property = op.createProperty();
OptExpressionHandle handle = new OptExpressionHandle(this);
property.derive(handle);
handle.deriveExpressionLogicalOrItemProperty();
property = handle.getProperty();
return property;
}

public Statistics deriveStat(RequiredLogicalProperty requiredProperty) {
if (statistics != null
&& !requiredProperty.isDifferent(statistics.getProperty())) {
return statistics;
}

final OptExpressionHandle expressionHandle = new OptExpressionHandle(this);
expressionHandle.deriveExpressionStats(requiredProperty);
statistics = expressionHandle.getStatistics();
return statistics;
}

@Override
public int hashCode() {
int hash = op.hashCode();
Expand Down
Loading