diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 117f62e287..f1a80eccd9 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -1122,6 +1122,7 @@ public static URLTemplate createUrlTemplate(String url, Method method) { private int lastMergeAsyncOutsideTs = 0; public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { + loggerMaker.infoAndAddToDb("Started building from dB", LogDb.RUNTIME); if (mergeAsyncOutside) { if (Context.now() - lastMergeAsyncOutsideTs > 600) { this.lastMergeAsyncOutsideTs = Context.now(); @@ -1138,7 +1139,10 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { try { List allCollections = ApiCollectionsDao.instance.getMetaAll(); for(ApiCollection apiCollection: allCollections) { + int start = Context.now(); + loggerMaker.infoAndAddToDb("Started merging API collection " + apiCollection.getId(), LogDb.RUNTIME); mergeUrlsAndSave(apiCollection.getId()); + loggerMaker.infoAndAddToDb("Finished merging API collection " + apiCollection.getId() + " in " + (Context.now() - start) + " seconds", LogDb.RUNTIME); } } catch (Exception e) { System.out.println("mergeUrlsAndSave: " + e.getMessage()); diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/Main.java b/apps/api-runtime/src/main/java/com/akto/runtime/Main.java index 36aabc16a4..8356489892 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/Main.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/Main.java @@ -36,8 +36,8 @@ public class Main { public static final String GROUP_NAME = "group_name"; public static final String VXLAN_ID = "vxlanId"; public static final String VPC_CIDR = "vpc_cidr"; - private static final Logger logger = LoggerFactory.getLogger(HttpCallParser.class); - private static final LoggerMaker loggerMaker = new LoggerMaker(HttpCallParser.class); + private static final Logger logger = LoggerFactory.getLogger(Main.class); + private static final LoggerMaker loggerMaker = new LoggerMaker(Main.class); // this sync threshold time is used for deleting sample data public static final int sync_threshold_time = 120; @@ -260,6 +260,7 @@ public void run() { if ((Context.now() - accountInfo.lastEstimatedCountTime) > 60*60) { accountInfo.lastEstimatedCountTime = Context.now(); accountInfo.estimatedCount = SingleTypeInfoDao.instance.getMCollection().estimatedDocumentCount(); + loggerMaker.infoAndAddToDb("STI Estimated count: " + accountInfo.estimatedCount, LogDb.RUNTIME); } if (!isDashboardInstance && accountInfo.estimatedCount> 20_000_000) { diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicy.java b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicy.java index 6ca2ad4e31..5838273b1f 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicy.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicy.java @@ -321,7 +321,6 @@ public static void main(String[] args) { } public void process(HttpResponseParams httpResponseParams) throws Exception { - logger.info("processing...."); List customAuthTypes = SingleTypeInfo.activeCustomAuthTypes; ApiInfo.ApiInfoKey apiInfoKey = ApiInfo.ApiInfoKey.generateFromHttpResponseParams(httpResponseParams); PolicyCatalog policyCatalog = getApiInfoFromMap(apiInfoKey); diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java index af20956e98..660397971e 100644 --- a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java @@ -136,6 +136,13 @@ public String execute() { return ERROR.toUpperCase(); } + try { + customDataType.validateRaw("some_key", "some_value"); + } catch (Exception e) { + addActionError("There is something wrong in the data type conditions"); + return ERROR.toUpperCase(); + } + if (this.createNew) { CustomDataType customDataTypeFromDb = CustomDataTypeDao.instance.findOne(Filters.eq(CustomDataType.NAME, name)); if (customDataTypeFromDb != null) { diff --git a/libs/dao/src/main/java/com/akto/dto/CustomDataType.java b/libs/dao/src/main/java/com/akto/dto/CustomDataType.java index 078cddc242..e15c78fe27 100644 --- a/libs/dao/src/main/java/com/akto/dto/CustomDataType.java +++ b/libs/dao/src/main/java/com/akto/dto/CustomDataType.java @@ -55,6 +55,14 @@ public SingleTypeInfo.SubType toSubType() { } public boolean validate(Object value, Object key) { + try { + return this.validateRaw(value, key); + } catch (Exception e) { + return false; + } + } + + public boolean validateRaw(Object value, Object key) throws Exception { if (this.keyConditions == null && this.valueConditions==null) return false; boolean keyResult = true; if (this.keyConditions != null) { diff --git a/pii-types/fintech.json b/pii-types/fintech.json new file mode 100644 index 0000000000..1b4f88b8cc --- /dev/null +++ b/pii-types/fintech.json @@ -0,0 +1,5 @@ +{ + "types": [ + {"name": "PAN CARD", "regexPattern" : "[A-Z]{5}[0-9]{4}[A-Z]{1}", "sensitive":false, "onKey": false, "active" : false} + ] +}