diff --git a/chrome/browser/about_flags.cc b/chrome/browser/about_flags.cc index 3bb61c5ecd896..27f4a1cfe5c7b 100644 --- a/chrome/browser/about_flags.cc +++ b/chrome/browser/about_flags.cc @@ -3200,6 +3200,11 @@ const FeatureEntry kFeatureEntries[] = { flag_descriptions::kCaptureThumbnailOnLoadFinishedDescription, kOsDesktop, FEATURE_VALUE_TYPE(features::kCaptureThumbnailOnLoadFinished)}, + {"use-new-accept-language-header", + flag_descriptions::kUseNewAcceptLanguageHeaderName, + flag_descriptions::kUseNewAcceptLanguageHeaderDescription, kOsAll, + FEATURE_VALUE_TYPE(features::kUseNewAcceptLanguageHeader)}, + #if defined(OS_WIN) {"enable-d3d-vsync", flag_descriptions::kEnableD3DVsync, flag_descriptions::kEnableD3DVsyncDescription, kOsWin, diff --git a/chrome/browser/flag_descriptions.cc b/chrome/browser/flag_descriptions.cc index da560e197f555..ce057061c7ebc 100644 --- a/chrome/browser/flag_descriptions.cc +++ b/chrome/browser/flag_descriptions.cc @@ -942,6 +942,12 @@ const char kOverlayScrollbarsFlashWhenMouseEnterDescription[] = "Flash Overlay Scrollbars When Mouse Enter a scrollable area. You must also" " enable Overlay Scrollbars."; +const char kUseNewAcceptLanguageHeaderName[] = "Use new Accept-Language header"; +const char kUseNewAcceptLanguageHeaderDescription[] = + "Adds the base language code after other corresponding language+region " + "codes. This ensures that users receive content in their preferred " + "language."; + const char kOverscrollHistoryNavigationName[] = "Overscroll history navigation"; const char kOverscrollHistoryNavigationDescription[] = "Experimental history navigation in response to horizontal overscroll."; diff --git a/chrome/browser/flag_descriptions.h b/chrome/browser/flag_descriptions.h index ecb001da02765..862f2943c1933 100644 --- a/chrome/browser/flag_descriptions.h +++ b/chrome/browser/flag_descriptions.h @@ -583,6 +583,9 @@ extern const char kOverlayScrollbarsFlashAfterAnyScrollUpdateDescription[]; extern const char kOverlayScrollbarsFlashWhenMouseEnterName[]; extern const char kOverlayScrollbarsFlashWhenMouseEnterDescription[]; +extern const char kUseNewAcceptLanguageHeaderName[]; +extern const char kUseNewAcceptLanguageHeaderDescription[]; + extern const char kOverscrollHistoryNavigationName[]; extern const char kOverscrollHistoryNavigationDescription[]; extern const char kOverscrollHistoryNavigationSimpleUi[]; diff --git a/chrome/browser/net/chrome_http_user_agent_settings.cc b/chrome/browser/net/chrome_http_user_agent_settings.cc index 25e3c7da4321e..5d2f98f9a6cf4 100644 --- a/chrome/browser/net/chrome_http_user_agent_settings.cc +++ b/chrome/browser/net/chrome_http_user_agent_settings.cc @@ -4,18 +4,71 @@ #include "chrome/browser/net/chrome_http_user_agent_settings.h" +#include "base/feature_list.h" +#include "base/strings/string_piece.h" +#include "base/strings/string_split.h" +#include "base/strings/string_util.h" +#include "base/strings/stringprintf.h" #include "chrome/common/chrome_content_client.h" +#include "chrome/common/chrome_features.h" #include "chrome/common/pref_names.h" #include "components/prefs/pref_service.h" #include "content/public/browser/browser_thread.h" #include "net/http/http_util.h" +namespace { + +// Helper class that builds the list of languages for the Accept-Language +// headers. +// The output is a comma-separated list of languages as string. +// Duplicates are removed. +class AcceptLanguageBuilder { + public: + // Adds a language to the string. + // Duplicates are ignored. + void AddLanguageCode(const std::string& language) { + if (seen_.find(language) == seen_.end()) { + if (str_.empty()) { + base::StringAppendF(&str_, "%s", language.c_str()); + } else { + base::StringAppendF(&str_, ",%s", language.c_str()); + } + seen_.insert(language); + } + } + + // Returns the string constructed up to this point. + std::string GetString() const { return str_; } + + private: + // The string that contains the list of languages, comma-separated. + std::string str_; + // Set the remove duplicates. + std::unordered_set seen_; +}; + +// Extract the base language code from a language code. +// If there is no '-' in the code, the original code is returned. +std::string GetBaseLanguageCode(const std::string& language_code) { + const std::vector tokens = base::SplitString( + language_code, "-", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL); + return tokens.empty() ? "" : tokens[0]; +} + +} // namespace + ChromeHttpUserAgentSettings::ChromeHttpUserAgentSettings(PrefService* prefs) { DCHECK_CURRENTLY_ON(content::BrowserThread::UI); pref_accept_language_.Init(prefs::kAcceptLanguages, prefs); last_pref_accept_language_ = *pref_accept_language_; + + const std::string accept_languages_str = + base::FeatureList::IsEnabled(features::kUseNewAcceptLanguageHeader) + ? ExpandLanguageList(last_pref_accept_language_) + : last_pref_accept_language_; last_http_accept_language_ = - net::HttpUtil::GenerateAcceptLanguageHeader(last_pref_accept_language_); + net::HttpUtil::GenerateAcceptLanguageHeader(accept_languages_str); + pref_accept_language_.MoveToThread( content::BrowserThread::GetTaskRunnerForThread( content::BrowserThread::IO)); @@ -25,6 +78,35 @@ ChromeHttpUserAgentSettings::~ChromeHttpUserAgentSettings() { DCHECK_CURRENTLY_ON(content::BrowserThread::IO); } +std::string ChromeHttpUserAgentSettings::ExpandLanguageList( + const std::string& language_prefs) { + const std::vector languages = base::SplitString( + language_prefs, ",", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL); + + if (languages.empty()) + return ""; + + AcceptLanguageBuilder builder; + + const int size = languages.size(); + for (int i = 0; i < size; ++i) { + const std::string& language = languages[i]; + builder.AddLanguageCode(language); + + // Extract the base language + const std::string& base_language = GetBaseLanguageCode(language); + + // Look ahead and add the base language if the next language is not part + // of the same family. + const int j = i + 1; + if (j >= size || GetBaseLanguageCode(languages[j]) != base_language) { + builder.AddLanguageCode(base_language); + } + } + + return builder.GetString(); +} + void ChromeHttpUserAgentSettings::CleanupOnUIThread() { DCHECK_CURRENTLY_ON(content::BrowserThread::UI); pref_accept_language_.Destroy(); @@ -33,11 +115,17 @@ void ChromeHttpUserAgentSettings::CleanupOnUIThread() { std::string ChromeHttpUserAgentSettings::GetAcceptLanguage() const { DCHECK_CURRENTLY_ON(content::BrowserThread::IO); std::string new_pref_accept_language = *pref_accept_language_; + if (new_pref_accept_language != last_pref_accept_language_) { + const std::string accept_languages_str = + base::FeatureList::IsEnabled(features::kUseNewAcceptLanguageHeader) + ? ExpandLanguageList(new_pref_accept_language) + : new_pref_accept_language; last_http_accept_language_ = - net::HttpUtil::GenerateAcceptLanguageHeader(new_pref_accept_language); + net::HttpUtil::GenerateAcceptLanguageHeader(accept_languages_str); last_pref_accept_language_ = new_pref_accept_language; } + return last_http_accept_language_; } diff --git a/chrome/browser/net/chrome_http_user_agent_settings.h b/chrome/browser/net/chrome_http_user_agent_settings.h index 4bef1cd675856..031bb2f7f562b 100644 --- a/chrome/browser/net/chrome_http_user_agent_settings.h +++ b/chrome/browser/net/chrome_http_user_agent_settings.h @@ -23,6 +23,9 @@ class ChromeHttpUserAgentSettings : public net::HttpUserAgentSettings { // Must be called on the IO thread. ~ChromeHttpUserAgentSettings() override; + // Adds the base language if a corresponding language+region code is present. + static std::string ExpandLanguageList(const std::string& language_prefs); + void CleanupOnUIThread(); // net::HttpUserAgentSettings implementation diff --git a/chrome/browser/net/chrome_http_user_agent_settings_unittest.cc b/chrome/browser/net/chrome_http_user_agent_settings_unittest.cc new file mode 100644 index 0000000000000..5f4f65f019e18 --- /dev/null +++ b/chrome/browser/net/chrome_http_user_agent_settings_unittest.cc @@ -0,0 +1,39 @@ +// Copyright (c) 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/browser/net/chrome_http_user_agent_settings.h" + +#include "testing/gtest/include/gtest/gtest.h" + +// Test the expansion of the Language List. +TEST(ChromeHttpUserAgentSettings, ExpandLanguageList) { + std::string output = ChromeHttpUserAgentSettings::ExpandLanguageList(""); + EXPECT_EQ("", output); + + output = ChromeHttpUserAgentSettings::ExpandLanguageList("en-US"); + EXPECT_EQ("en-US,en", output); + + output = ChromeHttpUserAgentSettings::ExpandLanguageList("fr"); + EXPECT_EQ("fr", output); + + // The base language is added after all regional codes... + output = ChromeHttpUserAgentSettings::ExpandLanguageList("en-US,en-CA"); + EXPECT_EQ("en-US,en-CA,en", output); + + // ... but before other language families. + output = ChromeHttpUserAgentSettings::ExpandLanguageList("en-US,en-CA,fr"); + EXPECT_EQ("en-US,en-CA,en,fr", output); + + output = + ChromeHttpUserAgentSettings::ExpandLanguageList("en-US,en-CA,fr,en-AU"); + EXPECT_EQ("en-US,en-CA,en,fr,en-AU", output); + + output = ChromeHttpUserAgentSettings::ExpandLanguageList("en-US,en-CA,fr-CA"); + EXPECT_EQ("en-US,en-CA,en,fr-CA,fr", output); + + // Add a base language even if it's already in the list. + output = ChromeHttpUserAgentSettings::ExpandLanguageList( + "en-US,fr-CA,it,fr,es-AR,it-IT"); + EXPECT_EQ("en-US,en,fr-CA,fr,it,es-AR,es,it-IT", output); +} diff --git a/chrome/common/chrome_features.cc b/chrome/common/chrome_features.cc index c271394f722c3..a1fe637aa2fb1 100644 --- a/chrome/common/chrome_features.cc +++ b/chrome/common/chrome_features.cc @@ -326,6 +326,12 @@ const base::Feature kOneGoogleBarOnLocalNtp{"OneGoogleBarOnLocalNtp", base::FEATURE_DISABLED_BY_DEFAULT}; #endif +// Adds the base language code to the Language-Accept headers if at least one +// corresponding language+region code is present in the user preferences. +// For example: "en-US, fr-FR" --> "en-US, en, fr-FR, fr". +const base::Feature kUseNewAcceptLanguageHeader{ + "UseNewAcceptLanguageHeader", base::FEATURE_DISABLED_BY_DEFAULT}; + // Enables Permissions Blacklisting via Safe Browsing. const base::Feature kPermissionsBlacklist{ "PermissionsBlacklist", base::FEATURE_DISABLED_BY_DEFAULT}; diff --git a/chrome/common/chrome_features.h b/chrome/common/chrome_features.h index 7612bcb98bcfe..38c69fec3513e 100644 --- a/chrome/common/chrome_features.h +++ b/chrome/common/chrome_features.h @@ -169,6 +169,8 @@ extern const base::Feature kOfflinePageDownloadSuggestionsFeature; extern const base::Feature kOneGoogleBarOnLocalNtp; #endif +extern const base::Feature kUseNewAcceptLanguageHeader; + extern const base::Feature kPermissionsBlacklist; #if defined(OS_WIN) diff --git a/chrome/test/BUILD.gn b/chrome/test/BUILD.gn index 345d0bc51239e..b5e154b5ebffb 100644 --- a/chrome/test/BUILD.gn +++ b/chrome/test/BUILD.gn @@ -3202,6 +3202,7 @@ test("unit_tests") { "../browser/metrics/thread_watcher_android_unittest.cc", "../browser/metrics/thread_watcher_unittest.cc", "../browser/mod_pagespeed/mod_pagespeed_metrics_unittest.cc", + "../browser/net/chrome_http_user_agent_settings_unittest.cc", "../browser/net/chrome_network_delegate_unittest.cc", "../browser/net/dns_probe_runner_unittest.cc", "../browser/net/dns_probe_service_unittest.cc", diff --git a/net/http/http_util.cc b/net/http/http_util.cc index 95ba606a7e0fa..98abf9631d60a 100644 --- a/net/http/http_util.cc +++ b/net/http/http_util.cc @@ -738,12 +738,8 @@ std::string HttpUtil::ConvertHeadersBackToHTTPResponse(const std::string& str) { return disassembled_headers; } -// TODO(jungshik): 1. If the list is 'fr-CA,fr-FR,en,de', we have to add -// 'fr' after 'fr-CA' with the same q-value as 'fr-CA' because -// web servers, in general, do not fall back to 'fr' and may end up picking -// 'en' which has a lower preference than 'fr-CA' and 'fr-FR'. -// 2. This function assumes that the input is a comma separated list -// without any whitespace. As long as it comes from the preference and +// TODO(jungshik): This function assumes that the input is a comma separated +// list without any whitespace. As long as it comes from the preference and // a user does not manually edit the preference file, it's the case. Still, // we may have to make it more robust. std::string HttpUtil::GenerateAcceptLanguageHeader( @@ -751,7 +747,7 @@ std::string HttpUtil::GenerateAcceptLanguageHeader( // We use integers for qvalue and qvalue decrement that are 10 times // larger than actual values to avoid a problem with comparing // two floating point numbers. - const unsigned int kQvalueDecrement10 = 2; + const unsigned int kQvalueDecrement10 = 1; unsigned int qvalue10 = 10; base::StringTokenizer t(raw_language_list, ","); std::string lang_list_with_q; diff --git a/net/http/http_util_unittest.cc b/net/http/http_util_unittest.cc index f56f48ebdae0a..d0b7802333240 100644 --- a/net/http/http_util_unittest.cc +++ b/net/http/http_util_unittest.cc @@ -709,11 +709,19 @@ TEST(HttpUtilTest, SpecForRequestForUrlWithFtpScheme) { } TEST(HttpUtilTest, GenerateAcceptLanguageHeader) { - EXPECT_EQ(std::string("en-US,fr;q=0.8,de;q=0.6"), - HttpUtil::GenerateAcceptLanguageHeader("en-US,fr,de")); - EXPECT_EQ(std::string("en-US,fr;q=0.8,de;q=0.6,ko;q=0.4,zh-CN;q=0.2," - "ja;q=0.2"), - HttpUtil::GenerateAcceptLanguageHeader("en-US,fr,de,ko,zh-CN,ja")); + std::string header = HttpUtil::GenerateAcceptLanguageHeader(""); + EXPECT_TRUE(header.empty()); + + header = HttpUtil::GenerateAcceptLanguageHeader("es"); + EXPECT_EQ(std::string("es"), header); + + header = HttpUtil::GenerateAcceptLanguageHeader("en-US,fr,de"); + EXPECT_EQ(std::string("en-US,fr;q=0.9,de;q=0.8"), header); + + header = HttpUtil::GenerateAcceptLanguageHeader("en-US,fr,de,ko,zh-CN,ja"); + EXPECT_EQ( + std::string("en-US,fr;q=0.9,de;q=0.8,ko;q=0.7,zh-CN;q=0.6,ja;q=0.5"), + header); } // HttpResponseHeadersTest.GetMimeType also tests ParseContentType. diff --git a/tools/metrics/histograms/enums.xml b/tools/metrics/histograms/enums.xml index 8296dacc0e28f..9c0e4f7c9b0b3 100644 --- a/tools/metrics/histograms/enums.xml +++ b/tools/metrics/histograms/enums.xml @@ -23157,6 +23157,7 @@ uploading your change for review. These are checked by presubmit scripts. + @@ -23557,6 +23558,7 @@ uploading your change for review. These are checked by presubmit scripts. +