Merge branch 'development' into new/http-ph7

Signed-off-by: DL6ER <[email protected]>

.circleci/config.yml

@@ -3,52 +3,76 @@ version: 2
 .job_steps: &job_steps
   steps:
     - checkout
-    - run:
-        name: "Setup"
-        command: |
-          if [[ $CIRCLE_JOB == *"qemu"* ]] ; then sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset ; fi
     - run:
         name: "Build"
         no_output_timeout: 30m
         command: |
           BRANCH=$([ -z "$CIRCLE_TAG" ] && echo "$CIRCLE_BRANCH" || echo "master")
-          [[ $CIRCLE_JOB == *"qemu"* ]] && DOCKERIFNEEDED="docker run --rm -v $(pwd):/workspace -w /workspace pihole/ftl-build:arm-qemu "
-          $DOCKERIFNEEDED bash .circleci/build-CI.sh "${STATIC}" "${BRANCH}" "${CIRCLE_TAG}" "${CIRCLE_JOB}"
+          bash .circleci/build-CI.sh "${STATIC}" "${BRANCH}" "${CIRCLE_TAG}" "${CIRCLE_JOB}"
     - run:
         name: "Binary checks"
         command: bash test/arch_test.sh
     - run:
-        name: "Upload"
+        name: "Compute checksum"
         command: |
-          [ -z "${CIRCLE_PR_USERNAME}" ] || exit 0
-          DIR="${CIRCLE_TAG:-${CIRCLE_BRANCH}}"
           mv pihole-FTL "${BIN_NAME}"
           sha1sum pihole-FTL-* > ${BIN_NAME}.sha1
+    - run:
+        name: "Upload binary to binary bucket"
+        command: |
+          [ -z "${CIRCLE_PR_USERNAME}" ] || exit 0
+          DIR="${CIRCLE_TAG:-${CIRCLE_BRANCH}}"
           mkdir -p ~/.ssh/
           ssh-keyscan -H $SSH_HOST >> ~/.ssh/known_hosts
-          sftp -b - $SSH_USER@$SSH_HOST <<< "-mkdir ${DIR}
-          put ${BIN_NAME}* ${DIR}"
-          mv "${BIN_NAME}" pihole-FTL
+          sftp -b - $SSH_USER@$SSH_HOST <<< "-mkdir html/${DIR}
+          put ${BIN_NAME}* html/${DIR}"
+    - run:
+        name: "Verify uploaded binary"
+        command: |
+          [ -z "${CIRCLE_PR_USERNAME}" ] || exit 0
+          DIR="${CIRCLE_TAG:-${CIRCLE_BRANCH}}"
+          mkdir download
+          cd download
+          wget "https://ftl.pi-hole.net/${DIR}/${BIN_NAME}"
+          wget "https://ftl.pi-hole.net/${DIR}/${BIN_NAME}.sha1"
+          sha1sum -c "${BIN_NAME}.sha1"
+          cd ..
     - run:
         name: "Test"
         command: |
+          mv "${BIN_NAME}" pihole-FTL
           test/run.sh
 
 .docker_template: &docker_template
   docker:
-    - image: pihole/ftl-build:v1.3-$CIRCLE_JOB
+    - image: pihole/ftl-build:v1.7-$CIRCLE_JOB
   <<: *job_steps
 
 jobs:
-  arm:
+  armv4t:
     <<: *docker_template
     environment:
-      BIN_NAME: "pihole-FTL-arm-linux-gnueabi"
+      BIN_NAME: "pihole-FTL-armv4-linux-gnueabi"
 
-  armhf:
+  armv5te:
     <<: *docker_template
     environment:
-      BIN_NAME: "pihole-FTL-arm-linux-gnueabihf"
+      BIN_NAME: "pihole-FTL-armv5-linux-gnueabi"
+
+  armv6hf:
+    <<: *docker_template
+    environment:
+      BIN_NAME: "pihole-FTL-armv6-linux-gnueabihf"
+
+  armv7hf:
+    <<: *docker_template
+    environment:
+      BIN_NAME: "pihole-FTL-armv7-linux-gnueabihf"
+
+  armv8a:
+    <<: *docker_template
+    environment:
+      BIN_NAME: "pihole-FTL-armv8-linux-gnueabihf"
 
   aarch64:
     <<: *docker_template
@@ -70,26 +94,27 @@ jobs:
     environment:
       BIN_NAME: "pihole-FTL-linux-x86_32"
 
-  arm-qemu:
-    machine:
-      enabled: true
-    environment:
-      BIN_NAME: "pihole-FTL-armel-native"
-    <<: *job_steps
-
 workflows:
   version: 2
   build:
     jobs:
-      - arm:
+      - armv4t:
+          filters:
+            tags:
+              only: /^v.*/
+      - armv5te:
+          filters:
+            tags:
+              only: /^v.*/
+      - armv6hf:
           filters:
             tags:
               only: /^v.*/
-      - arm-qemu:
+      - armv7hf:
           filters:
             tags:
               only: /^v.*/
-      - armhf:
+      - armv8a:
           filters:
             tags:
               only: /^v.*/

.gitignore

@@ -8,9 +8,11 @@ pihole-FTL
 version.h
 version~
 
-# CMake files
+# CMake files generated during compilation
+/cmake/
 /cmake-build-debug/
 /cmake-build-release/
+/cmake/
 
 # IDE files
 .idea/

src/CMakeLists.txt

@@ -112,10 +112,17 @@ set(sources
         daemon.h
         datastructure.c
         datastructure.h
+        dhcp-discover.c
+        dhcp-discover.h
         dnsmasq_interface.c
         dnsmasq_interface.h
         fifo.c
         fifo.h
+        edns0.c
+        edns0.h
+        enums.h
+        events.c
+        events.h
         files.c
         files.h
         FTL.h
@@ -155,8 +162,14 @@ add_custom_target(
         COMMAND ${CMAKE_COMMAND} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -P ${CMAKE_CURRENT_SOURCE_DIR}/gen_version.cmake
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
+add_library(FTL OBJECT ${sources})
+target_compile_options(FTL PRIVATE ${EXTRAWARN})
+target_include_directories(FTL PRIVATE ${PROJECT_SOURCE_DIR}/src)
+add_dependencies(FTL gen_version)
+
+
 add_executable(pihole-FTL
-        ${sources}
+        $<TARGET_OBJECTS:FTL>
         $<TARGET_OBJECTS:api>
         $<TARGET_OBJECTS:webserver>
         $<TARGET_OBJECTS:ph7_ext>
@@ -166,16 +179,13 @@ add_executable(pihole-FTL
         $<TARGET_OBJECTS:database>
         $<TARGET_OBJECTS:dnsmasq>
         $<TARGET_OBJECTS:sqlite3>
+        $<TARGET_OBJECTS:tre-regex>
         )
 if(STATIC STREQUAL "true")
     set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_START_STATIC ON)
     set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_END_STATIC ON)
     target_link_libraries(pihole-FTL -static-libgcc -static -static-pie)
 endif()
-target_compile_options(pihole-FTL PRIVATE ${EXTRAWARN})
-target_include_directories(pihole-FTL PRIVATE ${PROJECT_SOURCE_DIR}/src)
-add_dependencies(pihole-FTL gen_version)
-
 set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
 set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_package(Threads REQUIRED)
@@ -198,7 +208,7 @@ find_program(SETCAP setcap)
 install(TARGETS pihole-FTL
         RUNTIME DESTINATION bin
         PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
-install(CODE "execute_process(COMMAND ${SETCAP} CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN+eip ${CMAKE_INSTALL_PREFIX}/bin/pihole-FTL)")
+install(CODE "execute_process(COMMAND ${SETCAP} CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN+eip \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/bin/pihole-FTL)")
 
 add_subdirectory(api)
 add_subdirectory(webserver)
@@ -207,3 +217,4 @@ add_subdirectory(cJSON)
 add_subdirectory(ph7)
 add_subdirectory(database)
 add_subdirectory(dnsmasq)
+add_subdirectory(tre-regex)

src/FTL.h

@@ -108,17 +108,16 @@
 // How many authenticated API clients are allowed simultaneously? [.]
 #define API_MAX_CLIENTS 16
 
-// FTLDNS enums
-enum { QUERIES, UPSTREAMS, CLIENTS, DOMAINS, OVERTIME, WILDCARD, DNS_CACHE };
-enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED };
-enum { QUERY_UNKNOWN, QUERY_GRAVITY, QUERY_FORWARDED, QUERY_CACHE, QUERY_REGEX, QUERY_BLACKLIST, \
-       QUERY_EXTERNAL_BLOCKED_IP, QUERY_EXTERNAL_BLOCKED_NULL, QUERY_EXTERNAL_BLOCKED_NXRA, \
-       QUERY_GRAVITY_CNAME, QUERY_REGEX_CNAME, QUERY_BLACKLIST_CNAME, QUERY_STATUS_MAX };
-enum { TYPE_A = 0, TYPE_AAAA, TYPE_ANY, TYPE_SRV, TYPE_SOA, TYPE_PTR, TYPE_TXT, TYPE_NAPTR, TYPE_UNKN, TYPE_MAX };
-enum { REPLY_UNKNOWN, REPLY_NODATA, REPLY_NXDOMAIN, REPLY_CNAME, REPLY_IP, REPLY_DOMAIN, REPLY_RRNAME, REPLY_SERVFAIL, REPLY_REFUSED, REPLY_NOTIMP, REPLY_OTHER };
-enum { PRIVACY_SHOW_ALL = 0, PRIVACY_HIDE_DOMAINS, PRIVACY_HIDE_DOMAINS_CLIENTS, PRIVACY_MAXIMUM, PRIVACY_NOSTATS };
-enum { MODE_IP, MODE_NX, MODE_NULL, MODE_IP_NODATA_AAAA, MODE_NODATA };
-enum { REGEX_BLACKLIST, REGEX_WHITELIST };
+// After how many seconds do we check again if a client can be identified by other means?
+// (e.g., interface, MAC address, hostname)
+// Default: 60 (after one minutee)
+#define RECHECK_DELAY 60
+
+// How often should we check again if a client can be identified by other means?
+// (e.g., interface, MAC address, hostname)
+// Default: 3 (once after RECHECK_DELAY seconds, then again after 2*RECHECK_DELAY and 3*RECHECK_DELAY)
+// Important: This number has to be smaller than 256 for this mechanism to work
+#define NUM_RECHECKS 3
 
 // Use out own memory handling functions that will detect possible errors
 // and report accordingly in the log. This will make debugging FTL crashs

src/api/stats.c

@@ -543,12 +543,10 @@ int api_stats_upstreams(struct mg_connection *conn)
 	JSON_SEND_OBJECT(json);
 }
 
-static const char *querytypes[TYPE_MAX] = {"A","AAAA","ANY","SRV","SOA","PTR","TXT","NAPTR","UNKN"};
-
 int api_stats_query_types(struct mg_connection *conn)
 {
 	cJSON *json = JSON_NEW_ARRAY();
-	for(int i=0; i < TYPE_MAX; i++)
+	for(int i = TYPE_A; i < TYPE_MAX; i++)
 	{
 		cJSON *item = JSON_NEW_OBJ();
 		JSON_OBJ_REF_STR(item, "name", querytypes[i]);
@@ -849,6 +847,29 @@ int api_stats_history(struct mg_connection *conn)
 		// Skip if domain is not identical with what the user wants to see
 		if(filterdomainname && query->domainID != domainid)
 			continue;
+		if(filterdomainname)
+		{
+			// Check direct match
+			if(query->domainID == domainid)
+			{
+				// Get this query
+			}
+			// If the domain of this query did not match, the CNAME
+			// domain may still match - we have to check it in
+			// addition if this query is of CNAME blocked type
+			else if((query->status == QUERY_GRAVITY_CNAME ||
+			         query->status == QUERY_BLACKLIST_CNAME ||
+			         query->status == QUERY_REGEX_CNAME) &&
+			         query->CNAME_domainID == domainid)
+			{
+				// Get this query
+			}
+			else
+			{
+				// Skip this query
+				continue;
+			}
+		}
 
 		// Skip if client name and IP are not identical with what the user wants to see
 		if(filterclientname && query->clientID != clientid)
@@ -897,6 +918,23 @@ int api_stats_history(struct mg_connection *conn)
 		if(delay > 1.8e7)
 			delay = 0;
 
+		// Get domain blocked during deep CNAME inspection, if applicable
+		const char *CNAME_domain = "N/A";
+		if(query->CNAME_domainID > -1)
+		{
+			CNAME_domain = getCNAMEDomainString(query);
+		}
+
+		// Get ID of blocking regex, if applicable
+		int regex_idx = -1;
+		if (query->status == QUERY_REGEX || query->status == QUERY_REGEX_CNAME)
+		{
+			unsigned int cacheID = findCacheID(query->domainID, query->clientID, query->type);
+			DNSCacheData *dns_cache = getDNSCache(cacheID, true);
+			if(dns_cache != NULL)
+				regex_idx = dns_cache->black_regex_idx;
+		}
+
 		cJSON *item = JSON_NEW_OBJ();
 		JSON_OBJ_ADD_NUMBER(item, "timestamp", query->timestamp);
 		JSON_OBJ_ADD_NUMBER(item, "type", query->type);
@@ -906,6 +944,8 @@ int api_stats_history(struct mg_connection *conn)
 		JSON_OBJ_ADD_NUMBER(item, "dnssec", query->dnssec);
 		JSON_OBJ_ADD_NUMBER(item, "reply", query->reply);
 		JSON_OBJ_ADD_NUMBER(item, "response_time", delay);
+		JSON_OBJ_COPY_STR(item, "CNAME_domain", CNAME_domain);
+		JSON_OBJ_ADD_NUMBER(item, "regex_idx", regex_idx);
 		if(config.debug & DEBUG_API)
 			JSON_OBJ_ADD_NUMBER(item, "queryID", queryID);
 		JSON_ARRAY_ADD_ITEM(history, item);

src/api/stats_database.c

@@ -8,12 +8,13 @@
 *  This file is copyright under the latest version of the EUPL.
 *  Please see LICENSE file for your rights under this license. */
 
-#include "FTL.h"
+#include "../FTL.h"
 #include "../webserver/http-common.h"
 #include "../webserver/json_macros.h"
 #include "routes.h"
-#include "shmem.h"
-#include "datastructure.h"
+#include "../shmem.h"
+// querytypes[]
+#include "../datastructure.h"
 // logg()
 #include "log.h"
 // FTL_db
@@ -721,7 +722,6 @@ int api_stats_database_overTime_clients(struct mg_connection *conn)
 	JSON_SEND_OBJECT(json);
 }
 
-static const char *querytypes[8] = {"A","AAAA","ANY","SRV","SOA","PTR","TXT","UNKN"};
 int api_stats_database_query_types(struct mg_connection *conn)
 {
 	unsigned int from = 0, until = 0;
@@ -752,7 +752,7 @@ int api_stats_database_query_types(struct mg_connection *conn)
 
 	// Perform SQL queries
 	cJSON *json = JSON_NEW_ARRAY();
-	for(int i=0; i < TYPE_MAX; i++)
+	for(int i = TYPE_A; i < TYPE_MAX; i++)
 	{
 		const char *querystr = "SELECT COUNT(*) FROM queries "
 		                       "WHERE timestamp >= :from AND timestamp <= :until "