Use temp unprotected list and user list for all features (#993)

* Use temp unprotected list and user list for all features * Uncomment config lines * Use updated APIs from remote config * Fix https test * lint * Test updates * Respect lists for GPC * More explicit naming of https feature * Test fix * More test fix
duckduckgo · Nov 4, 2021 · b5b9bbd · b5b9bbd
1 parent 27c7a17
commit b5b9bbd
Show file tree

Hide file tree

Showing 12 changed files with 259 additions and 170 deletions.
diff --git a/Core/DoNotSellUserScript.swift b/Core/DoNotSellUserScript.swift
@@ -23,9 +23,18 @@ import BrowserServicesKit
 
 public class DoNotSellUserScript: NSObject, UserScript {
     public var source: String {
+
+        let privacyConfiguration = PrivacyConfigurationManager.shared.privacyConfig
+
+        let remoteUnprotectedDomains = (privacyConfiguration.tempUnprotectedDomains.joined(separator: "\n"))
+            + "\n"
+            + (privacyConfiguration.exceptionsList(forFeature: .gpc).joined(separator: "\n"))
+
         return Self.loadJS("donotsell", from: Bundle.core, withReplacements: [
             "$GPC_ENABLED$": PrivacyConfigurationManager.shared.privacyConfig
-                .isEnabled(featureKey: .gpc) ? "true" : "false"
+                .isEnabled(featureKey: .gpc) ? "true" : "false",
+            "$TEMP_UNPROTECTED_DOMAINS$": remoteUnprotectedDomains,
+            "$USER_UNPROTECTED_DOMAINS$": privacyConfiguration.userUnprotectedDomains.joined(separator: "\n")
         ])
     }
 

diff --git a/Core/FingerprintUserScript.swift b/Core/FingerprintUserScript.swift
@@ -42,11 +42,17 @@ public class FingerprintUserScript: NSObject, UserScript {
         let screenSizeExceptions = PrivacyConfigurationManager.shared.privacyConfig
             .exceptionsList(forFeature: .fingerprintingScreenSize).joined(separator: "\n")
 
+        let tempExceptions = PrivacyConfigurationManager.shared.privacyConfig
+            .tempUnprotectedDomains.joined(separator: "\n")
+        let userExceptions = PrivacyConfigurationManager.shared.privacyConfig.userUnprotectedDomains.joined(separator: "\n")
+
         return Self.loadJS("fingerprint", from: Bundle.core, withReplacements: [
             "$FEATURE_SETTINGS$": featureSettings,
             "$TEMP_STORAGE_EXCEPTIONS$": tempStorageExceptions,
             "$BATTERY_EXCEPTIONS$": batteryExceptions,
-            "$SCREEN_SIZE_EXCEPTIONS$": screenSizeExceptions
+            "$SCREEN_SIZE_EXCEPTIONS$": screenSizeExceptions,
+            "$TEMP_UNPROTECTED_DOMAINS$": tempExceptions,
+            "$USER_UNPROTECTED_DOMAINS$": userExceptions
         ])
     }
 

diff --git a/Core/HTTPSUpgrade.swift b/Core/HTTPSUpgrade.swift
@@ -28,11 +28,14 @@ public class HTTPSUpgrade {
     private let dataReloadLock = NSLock()
     private let store: HTTPSUpgradeStore
     private let appUrls: AppUrls
+    private let privacyConfig: PrivacyConfiguration
     private var bloomFilter: BloomFilterWrapper?
 
-    init(store: HTTPSUpgradeStore = HTTPSUpgradePersistence(), appUrls: AppUrls = AppUrls()) {
+    init(store: HTTPSUpgradeStore = HTTPSUpgradePersistence(), appUrls: AppUrls = AppUrls(),
+         privacyConfig: PrivacyConfiguration = PrivacyConfigurationManager.shared.privacyConfig) {
         self.store = store
         self.appUrls = appUrls
+        self.privacyConfig = privacyConfig
     }
 
     public func isUgradeable(url: URL, completion: @escaping UpgradeCheckCompletion) {
@@ -52,6 +55,25 @@ public class HTTPSUpgrade {
             return
         }
 
+        if privacyConfig.isEnabled(featureKey: .httpsUpgrade) {
+            // Check exception lists before upgrading
+            if privacyConfig.isTempUnprotected(domain: host) {
+                completion(false)
+                return
+            }
+            if privacyConfig.isUserUnprotected(domain: host) {
+                completion(false)
+                return
+            }
+            if privacyConfig.isInExceptionList(domain: host, forFeature: .httpsUpgrade) {
+                completion(false)
+                return
+            }
+        } else {
+            completion(false)
+            return
+        }
+
         waitForAnyReloadsToComplete()
         let isUpgradable = isInUpgradeList(host: host)
         os_log("%s %s upgradable", log: generalLog, type: .debug, host, isUpgradable ? "is" : "is not")

diff --git a/Core/PrivacyConfiguration.swift b/Core/PrivacyConfiguration.swift
@@ -76,6 +76,7 @@ public enum PrivacyFeature: String {
     case fingerprintingBattery
     case fingerprintingScreenSize
     case gpc
+    case httpsUpgrade = "https"
 }
 
 public struct AppPrivacyConfiguration: PrivacyConfiguration {

diff --git a/Core/PrivacyConfigurationManager.swift b/Core/PrivacyConfigurationManager.swift
@@ -21,8 +21,8 @@ import Foundation
 
 public class PrivacyConfigurationManager {
     public struct Constants {
-        public static let embeddedConfigETag = "\"a4bae5e53ca1ac5e1ad7ebd5a2bc3f5a\""
-        public static let embeddedConfigurationSHA = "S2/XfJs7hKiPAX1h1j8w06g/3N5vOVLi4BuDWcEQCus="
+        public static let embeddedConfigETag = "\"5041da4ee875accf79e3b5e50716d509\""
+        public static let embeddedConfigurationSHA = "psu3Kq1cFxtopqDFOIk14NDBh/7oueu1Pz5xlMsnarw="
     }
 
     public enum ReloadResult {

diff --git a/Core/donotsell.js b/Core/donotsell.js
@@ -18,22 +18,58 @@
 //
 
 (function () {
-    const gpcEnabled = $GPC_ENABLED$
-    if (!gpcEnabled) {
-        return
+    function getTopLevelURL () {
+        try {
+            // FROM: https://stackoverflow.com/a/7739035/73479
+            // FIX: Better capturing of top level URL so that trackers in embedded documents are not considered first party
+            return new URL(window.location !== window.parent.location ? document.referrer : document.location.href)
+        } catch (error) {
+            return new URL(location.href)
+        }
     }
 
-    if (navigator.globalPrivacyControl === undefined) {
-        Object.defineProperty(Navigator.prototype, 'globalPrivacyControl', {
-            get: () => true,
-            configurable: true,
-            enumerable: true
-        })
-    } else {
-        try {
-            navigator.globalPrivacyControl = true
-        } catch (e) {
-            console.error('globalPrivacyControl is not writable: ', e)
+    let gpcEnabled = $GPC_ENABLED$
+
+    const topLevelUrl = getTopLevelURL()
+    const domainParts = topLevelUrl && topLevelUrl.host ? topLevelUrl.host.split('.') : []
+
+    const userExcluded = `$USER_UNPROTECTED_DOMAINS$`.split("\n").filter(domain => domain.trim() == topLevelUrl.host).length > 0
+    if (userExcluded) {
+        return;
+    }
+
+    while (domainParts.length > 1 && gpcEnabled) {
+        const partialDomain = domainParts.join('.')
+        const gpcExcluded = `$TEMP_UNPROTECTED_DOMAINS$`.split('\n').filter(domain => domain.trim() === partialDomain).length > 0
+        console.log(partialDomain, gpcExcluded)
+        if (gpcExcluded) {
+            gpcEnabled = false
+            break
         }
+        domainParts.shift()
+    }
+    if (!gpcEnabled) {
+        return
     }
+
+    const scriptContent = `
+        if (navigator.globalPrivacyControl === undefined) {
+            Object.defineProperty(Navigator.prototype, 'globalPrivacyControl', {
+                get: () => true,
+                configurable: true,
+                enumerable: true
+            });
+        } else {
+            try {
+                navigator.globalPrivacyControl = true;
+            } catch (e) {
+                console.error('globalPrivacyControl is not writable: ', e);
+            }
+        }
+    `
+
+    const e = document.createElement('script')
+    e.textContent = scriptContent;
+    (document.head || document.documentElement).appendChild(e)
+    e.remove()
 })()
diff --git a/Core/fingerprint.js b/Core/fingerprint.js
@@ -179,11 +179,26 @@
     let excludeTempStorage = false
     let excludeBattery = false
     let excludeScreenSize = false
+    var excludeAll = false;
     const domainParts = topLevelUrl && topLevelUrl.host ? topLevelUrl.host.split('.') : []
 
+    const userExcluded = `
+                    $USER_UNPROTECTED_DOMAINS$
+                    `.split("\n").filter(domain => domain.trim() == topLevelUrl.host).length > 0;
+    if (userExcluded) {
+        return;
+    }
+
     // walk up the domain to see if it's unprotected
     while (domainParts.length > 1) {
         const partialDomain = domainParts.join('.')
+
+        excludeAll = `
+                $TEMP_UNPROTECTED_DOMAINS$
+                `.split("\n").filter(domain => domain.trim() == partialDomain).length > 0;
+        if (excludeAll) {
+            break;
+        }
 
         if (!excludeTempStorage) {
             excludeTempStorage = `
@@ -203,6 +218,11 @@
 
         domainParts.shift()
     }
+
+    // Check if domain on temp exceptions
+    if (excludeAll) {
+        return;
+    }
 
     function getTopLevelURL () {
         try {