diff --git a/config/settings.properties b/config/settings.properties index 5ed7e35..dbfbffe 100644 --- a/config/settings.properties +++ b/config/settings.properties @@ -20,9 +20,9 @@ tabby.build.thread.timeout = 2 tabby.build.isNeedToCreateIgnoreList = false # targets to analyse -tabby.build.target = cases/emcsqlproxy.war +tabby.build.target = cases/commons-collections-3.2.1.jar tabby.build.libraries = libs -tabby.build.mode = web +tabby.build.mode = gadget # db settings tabby.cache.path = ./cache/dev diff --git a/src/main/java/tabby/config/GlobalConfiguration.java b/src/main/java/tabby/config/GlobalConfiguration.java index b51f0d1..d6570fd 100644 --- a/src/main/java/tabby/config/GlobalConfiguration.java +++ b/src/main/java/tabby/config/GlobalConfiguration.java @@ -67,6 +67,7 @@ public class GlobalConfiguration { public static boolean IS_NEED_TO_CREATE_IGNORE_LIST = true; public static boolean isInitialed = false; + public static boolean isNeedStop = false; static { if(!FileUtils.fileExists(RULES_PATH)){ diff --git a/src/main/java/tabby/core/Analyser.java b/src/main/java/tabby/core/Analyser.java index 5e7a860..3a62836 100644 --- a/src/main/java/tabby/core/Analyser.java +++ b/src/main/java/tabby/core/Analyser.java @@ -12,7 +12,6 @@ import tabby.core.container.RulesContainer; import tabby.core.scanner.CallGraphScanner; import tabby.core.scanner.ClassInfoScanner; -import tabby.core.scanner.FullCallGraphScanner; import tabby.util.FileUtils; import java.io.File; @@ -39,8 +38,6 @@ public class Analyser { @Autowired private CallGraphScanner callGraphScanner; - @Autowired - private FullCallGraphScanner fullCallGraphScanner; @Autowired private RulesContainer rulesContainer; @Autowired @@ -71,12 +68,7 @@ public void run() throws IOException { // 收集目标 GlobalConfiguration.rulesContainer = rulesContainer; if(!GlobalConfiguration.IS_JDK_ONLY){ -// Map files = FileUtils.getTargetDirectoryJarFiles(GlobalConfiguration.TARGET); - long start = System.nanoTime(); Map files = fileCollector.collect(GlobalConfiguration.TARGET); - long time = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); - log.info("cost {} min {} seconds." - , time/60, time%60); cps.putAll(files); targets.putAll(files); } @@ -137,11 +129,8 @@ public void runSootAnalysis(Map targets, List classpaths // 类信息抽取 classInfoScanner.run(realTargets); // 全量函数调用图构建 - if(GlobalConfiguration.IS_FULL_CALL_GRAPH_CONSTRUCT){ - fullCallGraphScanner.run(); - }else{ - callGraphScanner.run(); - } + callGraphScanner.run(); + rulesContainer.saveStatus(); long time = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start); log.info("Total cost {} min {} seconds." diff --git a/src/main/java/tabby/core/collector/CallEdgeCollector.java b/src/main/java/tabby/core/collector/CallEdgeCollector.java new file mode 100644 index 0000000..21b1672 --- /dev/null +++ b/src/main/java/tabby/core/collector/CallEdgeCollector.java @@ -0,0 +1,88 @@ +package tabby.core.collector; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; +import soot.Modifier; +import soot.SootMethod; +import soot.Unit; +import soot.jimple.InvokeExpr; +import soot.jimple.JimpleBody; +import soot.jimple.Stmt; +import tabby.core.container.DataContainer; +import tabby.core.model.DefaultInvokeModel; +import tabby.core.switcher.Switcher; +import tabby.dal.caching.bean.ref.MethodReference; +import tabby.util.TickTock; + +/** + * @author wh1t3p1g + * @since 2023/1/12 + */ +@Slf4j +@Service +public class CallEdgeCollector { + + @Async("tabby-collector") + public void collect(MethodReference methodRef, DataContainer dataContainer, TickTock tickTock){ + try{ + SootMethod method = methodRef.getMethod(); + if(method == null) { + tickTock.countDown(); + return; // 提取不出内容，不分析 + } + + if(methodRef.isIgnore() || methodRef.isSink()){ + tickTock.countDown(); + return; // 消除后续的调用边 + } + + if(method.isStatic() && method.getParameterCount() == 0){ + // 静态函数 且 函数入参数量为0 此类函数不影响分析 + methodRef.setInitialed(true); + tickTock.countDown(); + return; + } + + if(method.isAbstract() + || Modifier.isNative(method.getModifiers()) + || method.isPhantom()){ + methodRef.setInitialed(true); + methodRef.setActionInitialed(true); + tickTock.countDown(); + return; + } + + JimpleBody body = (JimpleBody) Switcher.retrieveBody(method, method.getSignature()); + if(body == null) { + tickTock.countDown(); + return; + } + + DefaultInvokeModel model = new DefaultInvokeModel(); + for(Unit unit:body.getUnits()){ + Stmt stmt = (Stmt) unit; + if(stmt.containsInvokeExpr()){ + InvokeExpr ie = stmt.getInvokeExpr(); + SootMethod targetMethod = ie.getMethod(); + MethodReference targetMethodRef + = dataContainer.getOrAddMethodRef(ie.getMethodRef(), targetMethod); + model.apply(stmt, false, methodRef, targetMethodRef, dataContainer); + } + } + }catch (RuntimeException e){ +// log.error(e.getMessage()); + log.error("Something error on call graph. "+methodRef.getSignature()); + log.error(e.getMessage()); +// e.printStackTrace(); + }catch (Exception e){ + if(e instanceof InterruptedException) { + log.error("Thread interrupted. " + methodRef.getSignature()); + } else { + log.error("Something error on call graph. "+methodRef.getSignature()); + e.printStackTrace(); + } + } + tickTock.countDown(); + } +} diff --git a/src/main/java/tabby/core/collector/CallGraphCollector.java b/src/main/java/tabby/core/collector/CallGraphCollector.java index c50321d..4370cc2 100644 --- a/src/main/java/tabby/core/collector/CallGraphCollector.java +++ b/src/main/java/tabby/core/collector/CallGraphCollector.java @@ -10,6 +10,7 @@ import tabby.core.switcher.Switcher; import tabby.core.toolkit.PollutedVarsPointsToAnalysis; import tabby.dal.caching.bean.ref.MethodReference; +import tabby.util.TickTock; /** * @author wh1t3P1g @@ -20,16 +21,20 @@ @Setter public class CallGraphCollector { -// @Async("multiCallGraphCollector") - public void collect(MethodReference methodRef, DataContainer dataContainer){ +// @Async("tabby-collector") + public void collect(MethodReference methodRef, DataContainer dataContainer, TickTock tickTock){ try{ SootMethod method = methodRef.getMethod(); - if(method == null) return; // 提取不出内容，不分析 + if(method == null) { + tickTock.countDown(); + return; // 提取不出内容，不分析 + } if(method.isPhantom() || methodRef.isSink() || methodRef.isIgnore() || method.isAbstract() || Modifier.isNative(method.getModifiers())){ methodRef.setInitialed(true); + tickTock.countDown(); return; // sink点为不动点，无需分析该函数内的调用情况 native/抽象函数没有具体的body } @@ -37,6 +42,7 @@ public void collect(MethodReference methodRef, DataContainer dataContainer){ // 静态函数 且 函数入参数量为0 此类函数 // 对于反序列化来说 均不可控 不进行分析 methodRef.setInitialed(true); + tickTock.countDown(); return; } @@ -49,10 +55,17 @@ public void collect(MethodReference methodRef, DataContainer dataContainer){ context, dataContainer, method, methodRef); context.clear(); - }catch (RuntimeException e){ e.printStackTrace(); + }catch (Exception e){ + if(e instanceof InterruptedException) { + log.error("Thread interrupted. " + methodRef.getSignature()); + } else { + log.error("Something error on call graph. "+methodRef.getSignature()); + e.printStackTrace(); + } } + tickTock.countDown(); } } diff --git a/src/main/java/tabby/core/scanner/CallGraphScanner.java b/src/main/java/tabby/core/scanner/CallGraphScanner.java index 0e180d3..bb458fb 100644 --- a/src/main/java/tabby/core/scanner/CallGraphScanner.java +++ b/src/main/java/tabby/core/scanner/CallGraphScanner.java @@ -4,10 +4,13 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import tabby.config.GlobalConfiguration; +import tabby.core.collector.CallEdgeCollector; import tabby.core.collector.CallGraphCollector; import tabby.core.container.DataContainer; import tabby.dal.caching.bean.ref.MethodReference; import tabby.dal.caching.service.MethodRefService; +import tabby.util.TickTock; import java.util.ArrayList; import java.util.Collection; @@ -26,13 +29,10 @@ public class CallGraphScanner { public MethodRefService methodRefService; @Autowired public DataContainer dataContainer; - @Autowired - public CallGraphCollector collector; - - public static int total; - public static int split; - public static int current; + public CallGraphCollector callGraphCollector; + @Autowired + private CallEdgeCollector callEdgeCollector; public void run() { collect(); @@ -42,21 +42,16 @@ public void run() { public void collect() { Collection targets = new ArrayList<>(dataContainer.getSavedMethodRefs().values()); -// log.info("Load necessary method refs."); -// dataContainer.loadNecessaryMethodRefs(); log.info("Build call graph. START!"); - total = targets.size(); - split = total / 10; - split = split==0?1:split; - int count = 0; + TickTock tickTock = new TickTock(targets.size(), true); for (MethodReference target : targets) { - if(count%split == 0){ - log.info("Status: {}%, Remain: {}", String.format("%.1f",count*0.1/total*1000), (total-count)); + if(GlobalConfiguration.IS_FULL_CALL_GRAPH_CONSTRUCT){ + callEdgeCollector.collect(target, dataContainer, tickTock); + }else{ + callGraphCollector.collect(target, dataContainer, tickTock); } - collector.collect(target, dataContainer); - count++; } - log.info("Status: 100%, Remain: 0"); + tickTock.await(); log.info("Build call graph. DONE!"); } diff --git a/src/main/java/tabby/core/scanner/ClassInfoScanner.java b/src/main/java/tabby/core/scanner/ClassInfoScanner.java index cf4025c..968daee 100644 --- a/src/main/java/tabby/core/scanner/ClassInfoScanner.java +++ b/src/main/java/tabby/core/scanner/ClassInfoScanner.java @@ -59,14 +59,15 @@ public Map> loadAndExtract(List classes = getTargetClasses(path, moduleClasses); if(classes == null) continue; for (String cl : classes) { try{ - SootClass theClass = SemanticHelper.loadClass(cl); +// SootClass theClass = SemanticHelper.loadClass(cl); + SootClass theClass = Scene.v().loadClassAndSupport(cl); if (!theClass.isPhantom()) { // 这里存在类数量不一致的情况，是因为存在重复的对象 results.put(cl, collector.collect(theClass)); diff --git a/src/main/java/tabby/core/scanner/FullCallGraphScanner.java b/src/main/java/tabby/core/scanner/FullCallGraphScanner.java deleted file mode 100644 index 0070779..0000000 --- a/src/main/java/tabby/core/scanner/FullCallGraphScanner.java +++ /dev/null @@ -1,118 +0,0 @@ -package tabby.core.scanner; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import soot.Body; -import soot.Modifier; -import soot.SootMethod; -import soot.Unit; -import soot.jimple.InvokeExpr; -import soot.jimple.JimpleBody; -import soot.jimple.Stmt; -import tabby.core.model.DefaultInvokeModel; -import tabby.core.switcher.Switcher; -import tabby.dal.caching.bean.ref.MethodReference; -import tabby.config.GlobalConfiguration; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.concurrent.*; - -/** - * @author wh1t3p1g - * @since 2021/12/2 - */ -@Slf4j -@Component -public class FullCallGraphScanner extends CallGraphScanner{ - - @Override - public void collect() { - Collection targets = - new ArrayList<>(dataContainer.getSavedMethodRefs().values()); - log.info("Build call graph. START!"); - total = targets.size(); - split = total / 10; - split = split==0?1:split; - int count = 0; - for (MethodReference target : targets) { - if(count%split == 0){ - log.info("Status: {}%, Remain: {}", String.format("%.1f",count*0.1/total*1000), (total-count)); - } - buildCallEdge(target); - count++; - } - log.info("Status: 100%, Remain: 0"); - log.info("Build call graph. DONE!"); - } - - public void buildCallEdge(MethodReference methodRef){ - try{ - SootMethod method = methodRef.getMethod(); - if(method == null) { - return; // 提取不出内容，不分析 - } - - if(methodRef.isIgnore() || methodRef.isSink()){ - return; // 消除后续的调用边 - } - - if(method.isStatic() && method.getParameterCount() == 0){ - // 静态函数 且 函数入参数量为0 此类函数不影响分析 - methodRef.setInitialed(true); - return; - } - - if(method.isAbstract() - || Modifier.isNative(method.getModifiers()) - || method.isPhantom()){ - methodRef.setInitialed(true); - methodRef.setActionInitialed(true); - return; - } - - JimpleBody body = (JimpleBody) Switcher.retrieveBody(method, method.getSignature()); - if(body == null) return; - - DefaultInvokeModel model = new DefaultInvokeModel(); - for(Unit unit:body.getUnits()){ - Stmt stmt = (Stmt) unit; - if(stmt.containsInvokeExpr()){ - InvokeExpr ie = stmt.getInvokeExpr(); - SootMethod targetMethod = ie.getMethod(); - MethodReference targetMethodRef - = dataContainer.getOrAddMethodRef(ie.getMethodRef(), targetMethod); - model.apply(stmt, false, methodRef, targetMethodRef, dataContainer); - } - } - }catch (RuntimeException e){ -// log.error(e.getMessage()); - log.error("Something error on call graph. "+methodRef.getSignature()); - log.error(e.getMessage()); -// e.printStackTrace(); - } - } - - public static Body retrieveBody(SootMethod method, String signature){ - ExecutorService executor = Executors.newSingleThreadExecutor(); - Future future = executor.submit(() -> (JimpleBody) method.retrieveActiveBody()); - - JimpleBody body = null; - try{ - // 为了解决soot获取body不停止的问题，添加线程且最多执行2分钟 - // 超过2分钟可以获取到的body，也可以间接认为是非常大的body，暂不分析 - // 这里两分钟改成配置文件timeout-1，最短1分钟 - body = future.get( Integer.max(GlobalConfiguration.TIMEOUT-1, 1) * 60L, TimeUnit.SECONDS); - }catch (TimeoutException e){ - throw new RuntimeException("Method Fetch Timeout "+signature); - } catch (ExecutionException e) { - throw new RuntimeException(e); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } finally { - executor.shutdown(); - } - - return body; - } -} diff --git a/src/main/java/tabby/core/toolkit/PollutedVarsPointsToAnalysis.java b/src/main/java/tabby/core/toolkit/PollutedVarsPointsToAnalysis.java index b96ac59..8286b9b 100644 --- a/src/main/java/tabby/core/toolkit/PollutedVarsPointsToAnalysis.java +++ b/src/main/java/tabby/core/toolkit/PollutedVarsPointsToAnalysis.java @@ -7,6 +7,7 @@ import soot.jimple.InstanceFieldRef; import soot.toolkits.graph.DirectedGraph; import soot.toolkits.scalar.ForwardFlowAnalysis; +import tabby.config.GlobalConfiguration; import tabby.dal.caching.bean.ref.MethodReference; import tabby.core.data.Context; import tabby.core.container.DataContainer; @@ -107,6 +108,9 @@ public void doAnalysis(){ @Override protected void flowThrough(Map in, Unit d, Map out) { + if(GlobalConfiguration.isNeedStop){ + return; + } Map newIn = new HashMap<>(); copy(in, newIn); context.setLocalMap(newIn); diff --git a/src/main/java/tabby/dal/caching/bean/ref/MethodReference.java b/src/main/java/tabby/dal/caching/bean/ref/MethodReference.java index dbe8648..f6b2fe3 100644 --- a/src/main/java/tabby/dal/caching/bean/ref/MethodReference.java +++ b/src/main/java/tabby/dal/caching/bean/ref/MethodReference.java @@ -110,6 +110,8 @@ public class MethodReference { @org.springframework.data.annotation.Transient private transient Set childAliasEdges = new HashSet<>(); + private transient SootMethod sootMethod = null; + public static MethodReference newInstance(String name, String signature){ MethodReference methodRef = new MethodReference(); String id = null; @@ -154,17 +156,23 @@ public static MethodReference newInstance(String classname, SootMethod method){ } public SootMethod getMethod(){ - try{ - SootClass sc = SemanticHelper.getSootClass(classname); - if(!sc.isPhantom()){ - return SemanticHelper.getMethod(sc, subSignature); - } - }catch (Exception e){ - System.out.println("getMethod error: "+classname+", signature: "+signature); + if(sootMethod != null) return sootMethod; + + SootClass sc = SemanticHelper.getSootClass(classname); + if(!sc.isPhantom()){ + sootMethod = SemanticHelper.getMethod(sc, subSignature); + return sootMethod; } + return null; } + public void setMethod(SootMethod method){ + if(sootMethod == null){ + sootMethod = method; + } + } + public void addAction(String key, String value){ actions.put(key, value); } diff --git a/src/main/java/tabby/util/FileUtils.java b/src/main/java/tabby/util/FileUtils.java index 9c02d1a..4f63fed 100644 --- a/src/main/java/tabby/util/FileUtils.java +++ b/src/main/java/tabby/util/FileUtils.java @@ -224,7 +224,7 @@ public static void putRawContent(String path, Collection data){ */ public static Path registerTempDirectory(String directory) throws IOException { - final Path tmpDir = Files.createTempDirectory(directory); + final Path tmpDir = Files.createTempDirectory("tabby_"+directory); // Delete the temp directory at shutdown Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { diff --git a/src/main/java/tabby/util/SemanticHelper.java b/src/main/java/tabby/util/SemanticHelper.java index 3f51fb2..26e1957 100644 --- a/src/main/java/tabby/util/SemanticHelper.java +++ b/src/main/java/tabby/util/SemanticHelper.java @@ -1,6 +1,7 @@ package tabby.util; import com.google.common.hash.Hashing; +import lombok.extern.slf4j.Slf4j; import soot.*; import soot.jimple.*; import soot.jimple.internal.JimpleLocalBox; @@ -15,6 +16,7 @@ * @author wh1t3p1g * @since 2022/1/7 */ +@Slf4j public class SemanticHelper { private static List ARRAY_TYPES @@ -312,10 +314,20 @@ public static SootField getField(SootClass cls, String fieldName){ return null; } - public static SootClass getSootClass(String cls){ +// public static SootClass getSootClass(String cls){ +// try{ +// return loadClass(cls); +// }catch (Exception ignore){ +// } +// return null; +// } + + public static SootClass getSootClass(String cls) { + if(cls == null) return null; try{ - return loadClass(cls); - }catch (Exception ignore){ + return Scene.v().getSootClass(cls); + }catch (Exception e){ + log.warn("Load class {} error", cls); } return null; } diff --git a/src/main/java/tabby/util/TickTock.java b/src/main/java/tabby/util/TickTock.java new file mode 100644 index 0000000..9d0c68b --- /dev/null +++ b/src/main/java/tabby/util/TickTock.java @@ -0,0 +1,90 @@ +package tabby.util; + +import lombok.extern.slf4j.Slf4j; +import tabby.config.GlobalConfiguration; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * @author wh1t3p1g + * @since 2022/3/25 + */ +@Slf4j +public class TickTock { + + private int total; + private int split; + private boolean show = false; + private CountDownLatch latch; + + + public TickTock(int total, boolean show) { + this.total = total; + this.show = show; + this.latch = new CountDownLatch(total); + this.split = (int) (total * 0.05); // 5% 输出一次 + if(this.split == 0){ + this.split = 1; + } + } + + public void await() { + long timeout = GlobalConfiguration.TIMEOUT * 60L; // 预计每个都处理1s + 额外的最大时延10分钟 + info("Wait for all tasks to complete. Timeout: {}s", timeout); + try { + if(!latch.await(timeout, TimeUnit.SECONDS)){ + error("Still have {} methods to analysis, but it reached the max timeout.", latch.getCount()); + GlobalConfiguration.isNeedStop = true; + info("Try to force stopping running task. Timeout: {}s", timeout); + latch.await(timeout, TimeUnit.SECONDS); + error("Remain {} running task", latch.getCount()); + }else{ + info("All tasks completed."); + } + } catch (InterruptedException e) { + e.printStackTrace(); + error("Still have {} methods to analysis, but it reached the max timeout.", latch.getCount()); + } + GlobalConfiguration.isNeedStop = false; + } + + public void info(String msg, Object... objs){ + if(show){ + log.info(msg, objs); + } + } + + public void error(String msg, Object... objs){ + if(show){ + log.error(msg, objs); + } + } + + public void awaitWithoutTimeout(){ + try { + info("Waiting for {} classes to be collected...", latch.getCount()); + latch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + error("Still have {} classes to collected.", latch.getCount()); + } + } + + public void countDown(){ + latch.countDown(); + ticktock(); + } + + public long getCount(){ + return latch.getCount(); + } + + public void ticktock(){ + long remain = latch.getCount(); + long finished = total - remain; + if(finished % split == 0 || finished == total){ + info("Status: {}%, Remain: {}", String.format("%.1f",finished*0.1/total*1000), remain); + } + } +}