From a86fbfe24314662ba4a17b7563cca579bba0afb4 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 25 Feb 2025 14:19:16 +0100 Subject: [PATCH 1/4] Change api fetch to also include allowedIPAddresses --- .../agent_api/background/cloud/api/ReportingApi.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java index bf9f2d0c..4074e2b9 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java @@ -25,11 +25,16 @@ public abstract class ReportingApi { */ public abstract Optional report(String token, APIEvent event, int timeoutInSec); - public record APIListsResponse(List blockedIPAddresses, String blockedUserAgents) {} + public record APIListsResponse( + List blockedIPAddresses, + List allowedIPAddresses, + String blockedUserAgents + ) {} public record ListsResponseEntry(String source, String description, List ips) {} /** * Fetch blocked lists using a separate API call, these can include : * -> blocked IP Addresses (e.g. geo restrictions) + * -> allowed IP Addresses (e.g. geo restrictions) * -> blocked User-Agents (e.g. bot blocking) * @param token the authentication token */ From bbe7d0ae0d7fb04594db6c60f754612a20dde56c Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 25 Feb 2025 14:19:49 +0100 Subject: [PATCH 2/4] Update the thread cache object to check allowed ips --- .../thread_cache/ThreadCacheObject.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/agent_api/src/main/java/dev/aikido/agent_api/thread_cache/ThreadCacheObject.java b/agent_api/src/main/java/dev/aikido/agent_api/thread_cache/ThreadCacheObject.java index 6a26b533..8f1412b5 100644 --- a/agent_api/src/main/java/dev/aikido/agent_api/thread_cache/ThreadCacheObject.java +++ b/agent_api/src/main/java/dev/aikido/agent_api/thread_cache/ThreadCacheObject.java @@ -23,9 +23,10 @@ public class ThreadCacheObject { private final Hostnames hostnames; private final Routes routes; - // IP Blocking (e.g. Geo-IP Restrictions) : - public record BlockedIpEntry(IPList blocklist, String description) {} - private List blockedIps = new ArrayList<>(); + // IP restrictions (e.g. Geo-IP Restrictions) : + public record IPListEntry(IPList ipList, String description) {} + private List blockedIps = new ArrayList<>(); + private List allowedIps = new ArrayList<>(); // User-Agent Blocking (e.g. bot blocking) : private Pattern blockedUserAgentRegex; @@ -72,8 +73,13 @@ public boolean isBypassedIP(String ip) { * Check if the IP is blocked (e.g. Geo IP Restrictions) */ public BlockedResult isIpBlocked(String ip) { - for (BlockedIpEntry entry: blockedIps) { - if (entry.blocklist.matches(ip)) { + for (IPListEntry entry: allowedIps) { + if (!entry.ipList.matches(ip)) { + return new BlockedResult(true, entry.description); + } + } + for (IPListEntry entry: blockedIps) { + if (entry.ipList.matches(ip)) { return new BlockedResult(true, entry.description); } } @@ -87,7 +93,14 @@ public void updateBlockedLists(Optional blockedLi if (res.blockedIPAddresses() != null) { for (ReportingApi.ListsResponseEntry entry : res.blockedIPAddresses()) { IPList ipList = createIPList(entry.ips()); - blockedIps.add(new BlockedIpEntry(ipList, entry.description())); + blockedIps.add(new IPListEntry(ipList, entry.description())); + } + } + // Update allowed IP addresses (e.g. for geo restrictions) : + if (res.allowedIPAddresses() != null) { + for (ReportingApi.ListsResponseEntry entry: res.allowedIPAddresses()) { + IPList ipList = createIPList(entry.ips()); + this.allowedIps.add(new IPListEntry(ipList, entry.description())); } } // Update Blocked User-Agents regex From 072f4ca0d172c45416bad027893d3310dca2b274 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 25 Feb 2025 14:23:41 +0100 Subject: [PATCH 3/4] Update broken tests --- .../java/background/ServiceConfigurationTest.java | 2 +- .../java/collectors/WebRequestCollectorTest.java | 12 ++++++------ .../java/thread_cache/ThreadCacheObjectTest.java | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/agent_api/src/test/java/background/ServiceConfigurationTest.java b/agent_api/src/test/java/background/ServiceConfigurationTest.java index e3b19bec..f60c9b62 100644 --- a/agent_api/src/test/java/background/ServiceConfigurationTest.java +++ b/agent_api/src/test/java/background/ServiceConfigurationTest.java @@ -104,7 +104,7 @@ void updateConfig_ShouldHandleAllUpdates_WhenApiResponseIsComplete() { @Test void testSetForBlockedIpRes() { assertNull(serviceConfiguration.blockedListsRes); - serviceConfiguration.storeBlockedListsRes(Optional.of(new ReportingApi.APIListsResponse(null, null))); + serviceConfiguration.storeBlockedListsRes(Optional.of(new ReportingApi.APIListsResponse(null, null, null))); assertNotNull(serviceConfiguration.blockedListsRes); serviceConfiguration.storeBlockedListsRes(Optional.empty()); diff --git a/agent_api/src/test/java/collectors/WebRequestCollectorTest.java b/agent_api/src/test/java/collectors/WebRequestCollectorTest.java index 169bd569..140b132c 100644 --- a/agent_api/src/test/java/collectors/WebRequestCollectorTest.java +++ b/agent_api/src/test/java/collectors/WebRequestCollectorTest.java @@ -105,7 +105,7 @@ void testReport_noThreadCacheObject() { void testReport_ipBlockedTwice() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("192.168.1.1")) - ), ""); + ), null, ""); // Mock ThreadCache threadCacheObject = new ThreadCacheObject(List.of(new Endpoint( "GET", "/api/resource", 100, 100, @@ -126,7 +126,7 @@ void testReport_ipBlockedTwice() { void testReport_ipBlockedUsingLists() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("bullshit.ip", "192.168.1.1")) - ), ""); + ), null, ""); // Mock ThreadCache threadCacheObject = new ThreadCacheObject(List.of(), Set.of(), Set.of(), new Routes(), Optional.of(blockedListsRes)); ThreadCache.set(threadCacheObject); @@ -144,7 +144,7 @@ void testReport_ipBlockedUsingLists() { void testReport_ipNotBlockedUsingListsNorUserAgent() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) - ), "Unrelated|random"); + ), null, "Unrelated|random"); // Mock ThreadCache threadCacheObject = new ThreadCacheObject(List.of(), Set.of(), Set.of(), new Routes(), Optional.of(blockedListsRes)); ThreadCache.set(threadCacheObject); @@ -160,7 +160,7 @@ void testReport_ipNotBlockedUsingListsNorUserAgent() { void testReport_userAgentBlocked() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) - ), "AI2Bot|hacker"); + ), null, "AI2Bot|hacker"); // Mock ThreadCache threadCacheObject = new ThreadCacheObject(List.of(), Set.of(), Set.of(), new Routes(), Optional.of(blockedListsRes)); ThreadCache.set(threadCacheObject); @@ -178,7 +178,7 @@ void testReport_userAgentBlocked() { void testReport_userAgentBlocked_Ip_Bypassed() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("192.168.1.2", "192.168.1.3")) - ), "AI2Bot|hacker"); + ), null, "AI2Bot|hacker"); // Mock ThreadCache threadCacheObject = new ThreadCacheObject(List.of(), Set.of(), /* bypassedIps : */ Set.of("192.168.1.1"), new Routes(), Optional.of(blockedListsRes)); @@ -196,7 +196,7 @@ void testReport_userAgentBlocked_Ip_Bypassed() { void testReport_ipBlockedUsingLists_Ip_Bypassed() { ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(List.of( new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("bullshit.ip", "192.168.1.1")) - ), ""); + ), null, ""); // Mock ThreadCache threadCacheObject = new ThreadCacheObject(List.of(), Set.of(), /* bypassedIps : */ Set.of("192.168.1.1"), new Routes(), Optional.of(blockedListsRes)); diff --git a/agent_api/src/test/java/thread_cache/ThreadCacheObjectTest.java b/agent_api/src/test/java/thread_cache/ThreadCacheObjectTest.java index 545f5288..95ba4f5b 100644 --- a/agent_api/src/test/java/thread_cache/ThreadCacheObjectTest.java +++ b/agent_api/src/test/java/thread_cache/ThreadCacheObjectTest.java @@ -24,7 +24,7 @@ public void update() { "fd00:3234:5678:9abc::1/64", "5.6.7.8/32" )) - ), "Test|One"))); + ), null, "Test|One"))); assertEquals(new ThreadCacheObject.BlockedResult(true, "description"), tCache.isIpBlocked("1.2.3.4")); assertEquals(new ThreadCacheObject.BlockedResult(false, null), tCache.isIpBlocked("2.3.4.5")); @@ -59,9 +59,9 @@ public void updateEmpty() { "fd00:3234:5678:9abc::1/64", "5.6.7.8/32" )) - ), "Test|One"))); + ), null, "Test|One"))); - tCache.updateBlockedLists(Optional.of(new ReportingApi.APIListsResponse(null, null))); + tCache.updateBlockedLists(Optional.of(new ReportingApi.APIListsResponse(null, null,null))); assertEquals(new ThreadCacheObject.BlockedResult(true, "description"), tCache.isIpBlocked("1.2.3.4")); assertEquals(new ThreadCacheObject.BlockedResult(false, null), tCache.isIpBlocked("2.3.4.5")); @@ -96,16 +96,16 @@ public void updateRegexes() { "fd00:3234:5678:9abc::1/64", "5.6.7.8/32" )) - ), "Test|One"))); + ), null, "Test|One"))); - tCache.updateBlockedLists(Optional.of(new ReportingApi.APIListsResponse(null, ""))); + tCache.updateBlockedLists(Optional.of(new ReportingApi.APIListsResponse(null, null, ""))); assertTrue(tCache.isBlockedUserAgent("This is my TEST user agent")); assertTrue(tCache.isBlockedUserAgent("Test")); assertTrue(tCache.isBlockedUserAgent("TEst and ONE")); assertFalse(tCache.isBlockedUserAgent("Est|On")); assertFalse(tCache.isBlockedUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")); - tCache.updateBlockedLists(Optional.of(new ReportingApi.APIListsResponse(null, "Mozilla"))); + tCache.updateBlockedLists(Optional.of(new ReportingApi.APIListsResponse(null, null, "Mozilla"))); assertFalse(tCache.isBlockedUserAgent("This is my TEST user agent")); assertFalse(tCache.isBlockedUserAgent("Test")); From 94cc5723e706a6e2dc43c234539cf65db1e88d67 Mon Sep 17 00:00:00 2001 From: Wout Feys Date: Tue, 25 Feb 2025 14:28:46 +0100 Subject: [PATCH 4/4] Add some unit tests --- .../collectors/WebRequestCollectorTest.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/agent_api/src/test/java/collectors/WebRequestCollectorTest.java b/agent_api/src/test/java/collectors/WebRequestCollectorTest.java index 140b132c..f54d3334 100644 --- a/agent_api/src/test/java/collectors/WebRequestCollectorTest.java +++ b/agent_api/src/test/java/collectors/WebRequestCollectorTest.java @@ -138,6 +138,38 @@ void testReport_ipBlockedUsingLists() { assertEquals("Your IP address is not allowed to access this resource. (Your IP: 192.168.1.1)", response.msg()); assertEquals(403, response.status()); } + @SetEnvironmentVariable(key = "AIKIDO_TOKEN", value = "test-token") + @Test + void testReport_ipNotAllowedUsingLists() { + ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(null, List.of( + new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("192.168.2.1")) + ), ""); + // Mock ThreadCache + threadCacheObject = new ThreadCacheObject(List.of(), Set.of(), Set.of(), new Routes(), Optional.of(blockedListsRes)); + ThreadCache.set(threadCacheObject); + + + WebRequestCollector.Res response = WebRequestCollector.report(contextObject); + + assertNotNull(response); + assertEquals("Your IP address is not allowed to access this resource. (Your IP: 192.168.1.1)", response.msg()); + assertEquals(403, response.status()); + } + @SetEnvironmentVariable(key = "AIKIDO_TOKEN", value = "test-token") + @Test + void testReport_ipInAllowlist() { + ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse(null, List.of( + new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("192.168.1.1", "10.0.0.0/24")) + ), ""); + // Mock ThreadCache + threadCacheObject = new ThreadCacheObject(List.of(), Set.of(), Set.of(), new Routes(), Optional.of(blockedListsRes)); + ThreadCache.set(threadCacheObject); + + + WebRequestCollector.Res response = WebRequestCollector.report(contextObject); + + assertNull(response); + } @SetEnvironmentVariable(key = "AIKIDO_TOKEN", value = "test-token") @Test @@ -203,6 +235,26 @@ void testReport_ipBlockedUsingLists_Ip_Bypassed() { ThreadCache.set(threadCacheObject); + WebRequestCollector.Res response = WebRequestCollector.report(contextObject); + + assertNull(response); + assertNull(Context.get()); + } + + @SetEnvironmentVariable(key = "AIKIDO_TOKEN", value = "test-token") + @Test + void testReport_ipNotAllowedUsingLists_Ip_Bypassed() { + ReportingApi.APIListsResponse blockedListsRes = new ReportingApi.APIListsResponse( + null, + List.of(new ReportingApi.ListsResponseEntry("geoip", "geoip restrictions", List.of("1.2.3.4"))), + "" + ); + // Mock ThreadCache + threadCacheObject = new ThreadCacheObject(List.of(), Set.of(), + /* bypassedIps : */ Set.of("192.168.1.1"), new Routes(), Optional.of(blockedListsRes)); + ThreadCache.set(threadCacheObject); + + WebRequestCollector.Res response = WebRequestCollector.report(contextObject); assertNull(response);