Merge: + qlog: anonymize_client_ip setting

Close #916

* commit 'a0be7f5566a7d5986eb04ed7ffb79c5230171e2a':
  + client: handle hide_client_ip
  + qlog: hide_client_ip setting

AGHTechDoc.md

@@ -1287,12 +1287,22 @@ Request:
 	{
 		"enabled": true | false
 		"interval": 1 | 7 | 30 | 90
+		"anonymize_client_ip": true | false // anonymize clients' IP addresses
 	}
 
 Response:
 
 	200 OK
 
+`anonymize_client_ip`:
+1. New log entries written to a log file will contain modified client IP addresses.  Note that there's no way to obtain the full IP address later for these entries.
+2. `GET /control/querylog` response data will contain modified client IP addresses (masked /24 or /112).
+3. Searching by client IP won't work for the previously stored entries.
+
+How `anonymize_client_ip` affects Stats:
+1. After AGH restart, new stats entries will contain modified client IP addresses.
+2. Existing entries are not affected.
+
 
 ### API: Get querylog parameters
 
@@ -1307,6 +1317,7 @@ Response:
 	{
 		"enabled": true | false
 		"interval": 1 | 7 | 30 | 90
+		"anonymize_client_ip": true | false
 	}
 
 

client/src/__locales/en.json

@@ -199,6 +199,8 @@
     "query_log_disabled": "The query log is disabled and can be configured in the <0>settings</0>",
     "query_log_strict_search": "Use double quotes for strict search",
     "query_log_retention_confirm": "Are you sure you want to change query log retention? If you decrease the interval value, some data will be lost",
+    "anonymize_client_ip": "Anonymize client IP",
+    "anonymize_client_ip_desc": "Don't save the full IP address of the client in logs and statistics",
     "dns_config": "DNS server configuration",
     "blocking_mode": "Blocking mode",
     "default": "Default",

client/src/components/Settings/LogsConfig/Form.js

@@ -42,6 +42,16 @@ const Form = (props) => {
                     disabled={processing}
                 />
             </div>
+            <div className="form__group form__group--settings">
+                <Field
+                    name="anonymize_client_ip"
+                    type="checkbox"
+                    component={renderSelectField}
+                    placeholder={t('anonymize_client_ip')}
+                    subtitle={t('anonymize_client_ip_desc')}
+                    disabled={processing}
+                />
+            </div>
             <label className="form__label">
                 <Trans>query_log_retention</Trans>
             </label>

client/src/components/Settings/LogsConfig/index.js

@@ -30,7 +30,7 @@ class LogsConfig extends Component {
 
     render() {
         const {
-            t, enabled, interval, processing, processingClear,
+            t, enabled, interval, processing, processingClear, anonymize_client_ip,
         } = this.props;
 
         return (
@@ -44,6 +44,7 @@ class LogsConfig extends Component {
                         initialValues={{
                             enabled,
                             interval,
+                            anonymize_client_ip,
                         }}
                         onSubmit={this.handleFormSubmit}
                         processing={processing}
@@ -59,6 +60,7 @@ class LogsConfig extends Component {
 LogsConfig.propTypes = {
     interval: PropTypes.number.isRequired,
     enabled: PropTypes.bool.isRequired,
+    anonymize_client_ip: PropTypes.bool.isRequired,
     processing: PropTypes.bool.isRequired,
     processingClear: PropTypes.bool.isRequired,
     setLogsConfig: PropTypes.func.isRequired,

client/src/components/Settings/index.js

@@ -106,6 +106,7 @@ class Settings extends Component {
                                 <LogsConfig
                                     enabled={queryLogs.enabled}
                                     interval={queryLogs.interval}
+                                    anonymize_client_ip={queryLogs.anonymize_client_ip}
                                     processing={queryLogs.processingSetConfig}
                                     processingClear={queryLogs.processingClear}
                                     setLogsConfig={setLogsConfig}

client/src/reducers/queryLogs.js

@@ -134,6 +134,7 @@ const queryLogs = handleActions(
         oldest: '',
         filter: DEFAULT_LOGS_FILTER,
         isFiltered: false,
+        anonymize_client_ip: false,
     },
 );
 

home/config.go

@@ -77,9 +77,10 @@ type dnsConfig struct {
 	// time interval for statistics (in days)
 	StatsInterval uint32 `yaml:"statistics_interval"`
 
-	QueryLogEnabled  bool   `yaml:"querylog_enabled"`     // if true, query log is enabled
-	QueryLogInterval uint32 `yaml:"querylog_interval"`    // time interval for query log (in days)
-	QueryLogMemSize  uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk
+	QueryLogEnabled   bool   `yaml:"querylog_enabled"`     // if true, query log is enabled
+	QueryLogInterval  uint32 `yaml:"querylog_interval"`    // time interval for query log (in days)
+	QueryLogMemSize   uint32 `yaml:"querylog_size_memory"` // number of entries kept in memory before they are flushed to disk
+	AnonymizeClientIP bool   `yaml:"anonymize_client_ip"`  // anonymize clients' IP addresses in logs and stats
 
 	dnsforward.FilteringConfig `yaml:",inline"`
 
@@ -242,6 +243,7 @@ func (c *configuration) write() error {
 		config.DNS.QueryLogEnabled = dc.Enabled
 		config.DNS.QueryLogInterval = dc.Interval
 		config.DNS.QueryLogMemSize = dc.MemSize
+		config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP
 	}
 
 	if Context.dnsFilter != nil {

home/dns.go

@@ -29,22 +29,24 @@ func initDNSServer() error {
 	baseDir := Context.getDataDir()
 
 	statsConf := stats.Config{
-		Filename:       filepath.Join(baseDir, "stats.db"),
-		LimitDays:      config.DNS.StatsInterval,
-		ConfigModified: onConfigModified,
-		HTTPRegister:   httpRegister,
+		Filename:          filepath.Join(baseDir, "stats.db"),
+		LimitDays:         config.DNS.StatsInterval,
+		AnonymizeClientIP: config.DNS.AnonymizeClientIP,
+		ConfigModified:    onConfigModified,
+		HTTPRegister:      httpRegister,
 	}
 	Context.stats, err = stats.New(statsConf)
 	if err != nil {
 		return fmt.Errorf("Couldn't initialize statistics module")
 	}
 	conf := querylog.Config{
-		Enabled:        config.DNS.QueryLogEnabled,
-		BaseDir:        baseDir,
-		Interval:       config.DNS.QueryLogInterval,
-		MemSize:        config.DNS.QueryLogMemSize,
-		ConfigModified: onConfigModified,
-		HTTPRegister:   httpRegister,
+		Enabled:           config.DNS.QueryLogEnabled,
+		BaseDir:           baseDir,
+		Interval:          config.DNS.QueryLogInterval,
+		MemSize:           config.DNS.QueryLogMemSize,
+		AnonymizeClientIP: config.DNS.AnonymizeClientIP,
+		ConfigModified:    onConfigModified,
+		HTTPRegister:      httpRegister,
 	}
 	Context.queryLog = querylog.New(conf)
 

openapi/openapi.yaml

@@ -1585,6 +1585,9 @@ definitions:
             interval:
                 type: "integer"
                 description: "Time period to keep data (1 | 7 | 30 | 90)"
+            anonymize_client_ip:
+                type: "boolean"
+                description: "Anonymize clients' IP addresses"
 
     TlsConfig:
         type: "object"

querylog/qlog.go

@@ -2,6 +2,7 @@ package querylog
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -66,6 +67,7 @@ func (l *queryLog) WriteDiskConfig(dc *DiskConfig) {
 	dc.Enabled = l.conf.Enabled
 	dc.Interval = l.conf.Interval
 	dc.MemSize = l.conf.MemSize
+	dc.AnonymizeClientIP = l.conf.AnonymizeClientIP
 }
 
 // Clear memory buffer and remove log files
@@ -123,7 +125,7 @@ func (l *queryLog) Add(params AddParams) {
 
 	now := time.Now()
 	entry := logEntry{
-		IP:   params.ClientIP.String(),
+		IP:   l.getClientIP(params.ClientIP.String()),
 		Time: now,
 
 		Result:   *params.Result,
@@ -196,6 +198,10 @@ const (
 func (l *queryLog) getData(params getDataParams) map[string]interface{} {
 	now := time.Now()
 
+	if len(params.Client) != 0 && l.conf.AnonymizeClientIP {
+		params.Client = l.getClientIP(params.Client)
+	}
+
 	// add from file
 	fileEntries, oldest, total := l.searchFiles(params)
 
@@ -246,7 +252,7 @@ func (l *queryLog) getData(params getDataParams) map[string]interface{} {
 	// the elements order is already reversed (from newer to older)
 	for i := 0; i < len(entries); i++ {
 		entry := entries[i]
-		jsonEntry := logEntryToJSONEntry(entry)
+		jsonEntry := l.logEntryToJSONEntry(entry)
 		data = append(data, jsonEntry)
 	}
 
@@ -262,7 +268,26 @@ func (l *queryLog) getData(params getDataParams) map[string]interface{} {
 	return result
 }
 
-func logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
+// Get Client IP address
+func (l *queryLog) getClientIP(clientIP string) string {
+	if l.conf.AnonymizeClientIP {
+		ip := net.ParseIP(clientIP)
+		if ip != nil {
+			ip4 := ip.To4()
+			const AnonymizeClientIP4Mask = 24
+			const AnonymizeClientIP6Mask = 112
+			if ip4 != nil {
+				clientIP = ip4.Mask(net.CIDRMask(AnonymizeClientIP4Mask, 32)).String()
+			} else {
+				clientIP = ip.Mask(net.CIDRMask(AnonymizeClientIP6Mask, 128)).String()
+			}
+		}
+	}
+
+	return clientIP
+}
+
+func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
 	var msg *dns.Msg
 
 	if len(entry.Answer) > 0 {
@@ -277,7 +302,7 @@ func logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
 		"reason":    entry.Result.Reason.String(),
 		"elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
 		"time":      entry.Time.Format(time.RFC3339Nano),
-		"client":    entry.IP,
+		"client":    l.getClientIP(entry.IP),
 	}
 	jsonEntry["question"] = map[string]interface{}{
 		"host":  entry.QHost,

querylog/qlog_http.go

@@ -106,15 +106,17 @@ func (l *queryLog) handleQueryLogClear(w http.ResponseWriter, r *http.Request) {
 }
 
 type qlogConfig struct {
-	Enabled  bool   `json:"enabled"`
-	Interval uint32 `json:"interval"`
+	Enabled           bool   `json:"enabled"`
+	Interval          uint32 `json:"interval"`
+	AnonymizeClientIP bool   `json:"anonymize_client_ip"`
 }
 
 // Get configuration
 func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
 	resp := qlogConfig{}
 	resp.Enabled = l.conf.Enabled
 	resp.Interval = l.conf.Interval
+	resp.AnonymizeClientIP = l.conf.AnonymizeClientIP
 
 	jsonVal, err := json.Marshal(resp)
 	if err != nil {
@@ -151,6 +153,9 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
 	if req.Exists("interval") {
 		conf.Interval = d.Interval
 	}
+	if req.Exists("anonymize_client_ip") {
+		conf.AnonymizeClientIP = d.AnonymizeClientIP
+	}
 	l.conf = &conf
 	l.lock.Unlock()
 

querylog/querylog.go

@@ -11,9 +11,10 @@ import (
 
 // DiskConfig - configuration settings that are stored on disk
 type DiskConfig struct {
-	Enabled  bool
-	Interval uint32
-	MemSize  uint32
+	Enabled           bool
+	Interval          uint32
+	MemSize           uint32
+	AnonymizeClientIP bool
 }
 
 // QueryLog - main interface
@@ -32,10 +33,11 @@ type QueryLog interface {
 
 // Config - configuration object
 type Config struct {
-	Enabled  bool
-	BaseDir  string // directory where log file is stored
-	Interval uint32 // interval to rotate logs (in days)
-	MemSize  uint32 // number of entries kept in memory before they are flushed to disk
+	Enabled           bool
+	BaseDir           string // directory where log file is stored
+	Interval          uint32 // interval to rotate logs (in days)
+	MemSize           uint32 // number of entries kept in memory before they are flushed to disk
+	AnonymizeClientIP bool   // anonymize clients' IP addresses
 
 	// Called when the configuration is changed by HTTP request
 	ConfigModified func()

stats/stats.go

@@ -16,9 +16,10 @@ type DiskConfig struct {
 
 // Config - module configuration
 type Config struct {
-	Filename  string         // database file name
-	LimitDays uint32         // time limit (in days)
-	UnitID    unitIDCallback // user function to get the current unit ID.  If nil, the current time hour is used.
+	Filename          string         // database file name
+	LimitDays         uint32         // time limit (in days)
+	UnitID            unitIDCallback // user function to get the current unit ID.  If nil, the current time hour is used.
+	AnonymizeClientIP bool           // anonymize clients' IP addresses
 
 	// Called when the configuration is changed by HTTP request
 	ConfigModified func()

stats/stats_unit.go

@@ -5,6 +5,7 @@ import (
 	"encoding/binary"
 	"encoding/gob"
 	"fmt"
+	"net"
 	"os"
 	"sort"
 	"sync"
@@ -442,14 +443,33 @@ func (s *statsCtx) clear() {
 	log.Debug("Stats: cleared")
 }
 
+// Get Client IP address
+func (s *statsCtx) getClientIP(clientIP string) string {
+	if s.conf.AnonymizeClientIP {
+		ip := net.ParseIP(clientIP)
+		if ip != nil {
+			ip4 := ip.To4()
+			const AnonymizeClientIP4Mask = 24
+			const AnonymizeClientIP6Mask = 112
+			if ip4 != nil {
+				clientIP = ip4.Mask(net.CIDRMask(AnonymizeClientIP4Mask, 32)).String()
+			} else {
+				clientIP = ip.Mask(net.CIDRMask(AnonymizeClientIP6Mask, 128)).String()
+			}
+		}
+	}
+
+	return clientIP
+}
+
 func (s *statsCtx) Update(e Entry) {
 	if e.Result == 0 ||
 		e.Result >= rLast ||
 		len(e.Domain) == 0 ||
 		!(len(e.Client) == 4 || len(e.Client) == 16) {
 		return
 	}
-	client := e.Client.String()
+	client := s.getClientIP(e.Client.String())
 
 	s.unitLock.Lock()
 	u := s.unit