From aac86200c79f5360cc05835d31a0684f948a5d7b Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 10 Nov 2021 17:48:29 +0000 Subject: [PATCH 001/148] Add has_imported_stats boolean to Site --- lib/plausible/site/schema.ex | 1 + .../20211110174617_add_site_imported_boolean.exs | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 priv/repo/migrations/20211110174617_add_site_imported_boolean.exs diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index f15f9ef76ff8..6cd0ccca0d78 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -11,6 +11,7 @@ defmodule Plausible.Site do field :public, :boolean field :locked, :boolean field :has_stats, :boolean + field :has_imported_stats, :boolean many_to_many :members, User, join_through: Plausible.Site.Membership has_many :memberships, Plausible.Site.Membership diff --git a/priv/repo/migrations/20211110174617_add_site_imported_boolean.exs b/priv/repo/migrations/20211110174617_add_site_imported_boolean.exs new file mode 100644 index 000000000000..a64377bea5c4 --- /dev/null +++ b/priv/repo/migrations/20211110174617_add_site_imported_boolean.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.GoogleAuthImportedBoolean do + use Ecto.Migration + + def change do + alter table(:sites) do + add :has_imported_stats, :boolean + end + end +end From 9bae8c7bae58e53737dfff9b04a1ff1f16a2a126 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 10 Nov 2021 17:49:16 +0000 Subject: [PATCH 002/148] Add Google Analytics import panel to general settings --- .../controllers/site_controller.ex | 2 +- .../templates/site/settings_general.html.eex | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 068cd3b040b9..4f93d827b5d0 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -170,7 +170,7 @@ defmodule PlausibleWeb.SiteController do def settings_general(conn, _params) do site = conn.assigns[:site] - |> Repo.preload(:custom_domain) + |> Repo.preload([:custom_domain, :google_auth]) conn |> assign(:skip_plausible_tracking, true) diff --git a/lib/plausible_web/templates/site/settings_general.html.eex b/lib/plausible_web/templates/site/settings_general.html.eex index fd5dc21a1e01..b70269c0b262 100644 --- a/lib/plausible_web/templates/site/settings_general.html.eex +++ b/lib/plausible_web/templates/site/settings_general.html.eex @@ -46,3 +46,31 @@ <% end %> + +
+
+

Data Import from Google Analytics

+

Import existing data from your Google Analytics account.

+ <%= link(to: "https://docs.plausible.io/import-data/", target: "_blank", rel: "noreferrer") do %> + + <% end %> +
+ + <%= if Keyword.get(Application.get_env(:plausible, :google), :client_id) do %> + <%= if @site.google_auth do %> +
+ Linked Google account: <%= @site.google_auth.email %> + + <%= link("Unlink Google account", to: "/#{URI.encode_www_form(@site.domain)}/settings/google", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete") %> + + <% else %> + <%= button("Continue with Google", to: Plausible.Google.Api.authorize_url(@site.id), class: "button mt-8") %> + <% end %> + <% else %> +
+ +

An extra step is needed to set up your Plausible Analytics Self Hosted for the Google Search Console integration. + Find instructions <%= link("here", to: "https://plausible.io/docs/self-hosting-configuration#google-search-integration", class: "text-indigo-500") %>

+
+ <% end %> +
From 167f31af20b25d0c1b6328a39fce3846c0c786c9 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 10 Nov 2021 19:29:10 +0000 Subject: [PATCH 003/148] Get GA profiles to display in import settings panel --- lib/plausible/google/api.ex | 41 +++++++++++++++++-- .../controllers/auth_controller.ex | 6 ++- .../controllers/site_controller.ex | 6 +++ .../templates/site/settings_general.html.eex | 22 +++++++++- .../site/settings_search_console.html.eex | 2 +- 5 files changed, 69 insertions(+), 8 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index db9a37bd1455..4386c6846b21 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -1,12 +1,14 @@ defmodule Plausible.Google.Api do - @scope URI.encode_www_form("https://www.googleapis.com/auth/webmasters.readonly email") + @scope URI.encode_www_form( + "https://www.googleapis.com/auth/webmasters.readonly email https://www.googleapis.com/auth/analytics.readonly" + ) @verified_permission_levels ["siteOwner", "siteFullUser", "siteRestrictedUser"] - def authorize_url(site_id) do + def authorize_url(site_id, redirect_to) do if Application.get_env(:plausible, :environment) == "test" do "" else - "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@scope}&state=#{site_id}" + "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@scope}&state=" <> Jason.encode!([site_id, redirect_to]) end end @@ -113,6 +115,39 @@ defmodule Plausible.Google.Api do end end + def get_analytics_view_ids(site) do + with {:ok, auth} <- refresh_if_needed(site.google_auth) do + do_get_analytics_view_ids(auth) + end + end + + def do_get_analytics_view_ids(auth) do + res = + HTTPoison.get!( + "https://www.googleapis.com/analytics/v3/management/accounts/~all/webproperties/~all/profiles", + Authorization: "Bearer #{auth.access_token}" + ) + + case res.status_code do + 200 -> + profiles = + Jason.decode!(res.body) + |> Map.get("items") + |> Enum.map(fn item -> + uri = URI.parse(Map.get(item, "websiteUrl")) + name = Map.get(item, "name") + {"#{uri.host} - #{name}", Map.get(item, "id")} + end) + |> Map.new() + + {:ok, profiles} + + _ -> + Sentry.capture_message("Error fetching Google view ID", extra: Jason.decode!(res.body)) + {:error, res.body} + end + end + defp refresh_if_needed(auth) do if Timex.before?(auth.expires, Timex.now() |> Timex.shift(seconds: 30)) do refresh_token(auth) diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex index d60ca17dbbda..917e0201d40a 100644 --- a/lib/plausible_web/controllers/auth_controller.ex +++ b/lib/plausible_web/controllers/auth_controller.ex @@ -536,12 +536,14 @@ defmodule PlausibleWeb.AuthController do |> redirect(to: redirect_to) end - def google_auth_callback(conn, %{"code" => code, "state" => site_id}) do + def google_auth_callback(conn, %{"code" => code, "state" => state}) do res = Plausible.Google.Api.fetch_access_token(code) id_token = res["id_token"] [_, body, _] = String.split(id_token, ".") id = body |> Base.decode64!(padding: false) |> Jason.decode!() + [site_id, redirect_to] = Jason.decode!(state) + Plausible.Site.GoogleAuth.changeset(%Plausible.Site.GoogleAuth{}, %{ email: id["email"], refresh_token: res["refresh_token"], @@ -554,6 +556,6 @@ defmodule PlausibleWeb.AuthController do site = Repo.get(Plausible.Site, site_id) - redirect(conn, to: "/#{URI.encode_www_form(site.domain)}/settings/search-console") + redirect(conn, to: "/#{URI.encode_www_form(site.domain)}/settings/#{redirect_to}") end end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 4f93d827b5d0..84ff07f2dde7 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -172,10 +172,16 @@ defmodule PlausibleWeb.SiteController do conn.assigns[:site] |> Repo.preload([:custom_domain, :google_auth]) + google_profiles = + if site.google_auth do + Plausible.Google.Api.get_analytics_view_ids(site) + end + conn |> assign(:skip_plausible_tracking, true) |> render("settings_general.html", site: site, + google_profiles: google_profiles, changeset: Plausible.Site.changeset(site, %{}), layout: {PlausibleWeb.LayoutView, "site_settings.html"} ) diff --git a/lib/plausible_web/templates/site/settings_general.html.eex b/lib/plausible_web/templates/site/settings_general.html.eex index b70269c0b262..484ba784d2cd 100644 --- a/lib/plausible_web/templates/site/settings_general.html.eex +++ b/lib/plausible_web/templates/site/settings_general.html.eex @@ -61,10 +61,28 @@
Linked Google account: <%= @site.google_auth.email %> - <%= link("Unlink Google account", to: "/#{URI.encode_www_form(@site.domain)}/settings/google", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete") %> + <%= case @google_profiles do %> + <% {:ok, profiles} -> %> +

+ Select the Google Analytics profile you would like to use to fetch data from. +

+ + <%= form_for @conn, "/#{URI.encode_www_form(@site.domain)}/settings/google-import", [class: "max-w-xs"], fn f -> %> +
+
+ <%= select f, :profile, profiles, prompt: "(Choose profile)", class: "dark:bg-gray-800 mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 dark:border-gray-500 outline-none focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md dark:text-gray-100" %> +
+
+ <%= submit "Save", class: "button" %> + <% end %> + + <% {:error, error} -> %> +

The following error occurred when fetching your Google Analytics profiles.

+

<%= error %>

+ <% end %> <% else %> - <%= button("Continue with Google", to: Plausible.Google.Api.authorize_url(@site.id), class: "button mt-8") %> + <%= button("Continue with Google", to: Plausible.Google.Api.authorize_url(@site.id, "general"), class: "button mt-8") %> <% end %> <% else %>
diff --git a/lib/plausible_web/templates/site/settings_search_console.html.eex b/lib/plausible_web/templates/site/settings_search_console.html.eex index 0177ec2894a8..7f34dc131a5f 100644 --- a/lib/plausible_web/templates/site/settings_search_console.html.eex +++ b/lib/plausible_web/templates/site/settings_search_console.html.eex @@ -40,7 +40,7 @@

<%= error %>

<% end %> <% else %> - <%= button("Continue with Google", to: Plausible.Google.Api.authorize_url(@site.id), class: "button mt-8") %> + <%= button("Continue with Google", to: Plausible.Google.Api.authorize_url(@site.id, "search-console"), class: "button mt-8") %>
NB: You also need to set up your site on <%= link("Google Search Console", to: "https://search.google.com/search-console/about") %> for the integration to work. <%= link("Read the docs", to: "https://plausible.io/docs/google-search-console-integration", class: "text-indigo-500", rel: "noreferrer") %> From 18559421bd861566dcc626d22cc240cd841c9e34 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 10 Nov 2021 19:47:43 +0000 Subject: [PATCH 004/148] Add import_from_google method as entrypoint to import data --- .../controllers/site_controller.ex | 27 +++++++++++++++++++ lib/plausible_web/router.ex | 2 ++ .../templates/site/settings_general.html.eex | 4 +-- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 84ff07f2dde7..88da550cad94 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -623,4 +623,31 @@ defmodule PlausibleWeb.SiteController do |> put_flash(:success, "Custom domain deleted successfully") |> redirect(to: "/#{URI.encode_www_form(site.domain)}/settings/general") end + + def import_from_google(conn, %{"profile" => profile}) do + site = + conn.assigns[:site] + |> Repo.preload(:google_auth) + + cond do + site.has_imported_stats -> + conn + |> put_flash(:error, "Third party data already imported") + |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) + + profile == "" -> + conn + |> put_flash(:error, "A Google Analytics profile must be selected") + |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) + + true -> + site.google_auth + |> Plausible.Site.changeset(%{has_imported_stats: true}) + |> Repo.update!() + + conn + |> put_flash(:success, "Google Analytics data successfully imported") + |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) + end + end end diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 64080a668883..59ae8304b01a 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -239,5 +239,7 @@ defmodule PlausibleWeb.Router do get "/:domain/export", StatsController, :csv_export get "/:domain/*path", StatsController, :stats + + post "/:website/settings/google-import", SiteController, :import_from_google end end diff --git a/lib/plausible_web/templates/site/settings_general.html.eex b/lib/plausible_web/templates/site/settings_general.html.eex index 484ba784d2cd..d0e8041cf251 100644 --- a/lib/plausible_web/templates/site/settings_general.html.eex +++ b/lib/plausible_web/templates/site/settings_general.html.eex @@ -64,7 +64,7 @@ <%= case @google_profiles do %> <% {:ok, profiles} -> %>

- Select the Google Analytics profile you would like to use to fetch data from. + Select the Google Analytics profile you would like to import data from.

<%= form_for @conn, "/#{URI.encode_www_form(@site.domain)}/settings/google-import", [class: "max-w-xs"], fn f -> %> @@ -73,7 +73,7 @@ <%= select f, :profile, profiles, prompt: "(Choose profile)", class: "dark:bg-gray-800 mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 dark:border-gray-500 outline-none focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md dark:text-gray-100" %>
- <%= submit "Save", class: "button" %> + <%= submit "Import", class: "button" %> <% end %> <% {:error, error} -> %> From 9abe8d3a403bd4a7969803679502543310dabd03 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 12 Nov 2021 13:30:00 +0000 Subject: [PATCH 005/148] Add imported_visitors table --- lib/plausible/imported/visitors.ex | 38 +++++++++++++++++++ ...0211112130238_create_imported_visitors.exs | 13 +++++++ 2 files changed, 51 insertions(+) create mode 100644 lib/plausible/imported/visitors.ex create mode 100644 priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs diff --git a/lib/plausible/imported/visitors.ex b/lib/plausible/imported/visitors.ex new file mode 100644 index 000000000000..da7f63a52ca3 --- /dev/null +++ b/lib/plausible/imported/visitors.ex @@ -0,0 +1,38 @@ +defmodule Plausible.Imported.Visitors do + use Ecto.Schema + import Ecto.Changeset + + @primary_key false + schema "imported_visitors" do + field :domain, :string + field :date, :naive_datetime + field :visitors, :integer + field :pageviews, :integer + field :bounce_rate, :integer + field :avg_visit_duration, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :date, + :visitors, + :pageviews, + :bounce_rate, + :avg_visit_duration + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :date, + :visitors, + :pageviews, + :bounce_rate, + :avg_visit_duration + ]) + end +end diff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs new file mode 100644 index 000000000000..0e251d00519c --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs @@ -0,0 +1,13 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_visitors, engine: "MergeTree() ORDER BY (date) SETTINGS index_granularity = 1") do + add :date, :naive_datetime + add :visitors, :UInt64 + add :pageviews, :UInt64 + add :bounce_rate, :UInt32 + add :avg_visit_duration, :UInt32 + end + end +end From 25061f2b83168fb2d98055f1e8862244cae0bae4 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 12 Nov 2021 13:30:41 +0000 Subject: [PATCH 006/148] Remove conflicting code from migration --- .../20191010031425_add_property_to_google_auth.exs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/priv/repo/migrations/20191010031425_add_property_to_google_auth.exs b/priv/repo/migrations/20191010031425_add_property_to_google_auth.exs index 9c16c8bd2028..c9e3ab112676 100644 --- a/priv/repo/migrations/20191010031425_add_property_to_google_auth.exs +++ b/priv/repo/migrations/20191010031425_add_property_to_google_auth.exs @@ -6,15 +6,5 @@ defmodule Plausible.Repo.Migrations.AddPropertyToGoogleAuth do alter table(:google_auth) do add :property, :text end - - flush() - - for auth <- Repo.all(Plausible.Site.GoogleAuth) do - auth = Repo.preload(auth, :site) - property = "https://#{auth.site.domain}" - - Plausible.Site.GoogleAuth.set_property(auth, %{property: property}) - |> Repo.update!() - end end end From 5a292ca929b8ea5b75e5a66cdb3cdf0a6261c3de Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 12 Nov 2021 16:46:36 +0000 Subject: [PATCH 007/148] Import visitors data into clickhouse database --- lib/plausible/google/api.ex | 95 ++++++++++++++++++- lib/plausible/imported/visitors.ex | 28 ++++++ lib/plausible/stats/clickhouse.ex | 8 ++ ...0211112130238_create_imported_visitors.exs | 1 + 4 files changed, 131 insertions(+), 1 deletion(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 4386c6846b21..e34fd447cdae 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -1,4 +1,6 @@ defmodule Plausible.Google.Api do + alias Plausible.Imported + @scope URI.encode_www_form( "https://www.googleapis.com/auth/webmasters.readonly email https://www.googleapis.com/auth/analytics.readonly" ) @@ -8,7 +10,8 @@ defmodule Plausible.Google.Api do if Application.get_env(:plausible, :environment) == "test" do "" else - "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@scope}&state=" <> Jason.encode!([site_id, redirect_to]) + "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@scope}&state=" <> + Jason.encode!([site_id, redirect_to]) end end @@ -148,6 +151,96 @@ defmodule Plausible.Google.Api do end end + def import_analytics(site, profile) do + with {:ok, auth} <- refresh_if_needed(site.google_auth) do + do_import_analytics(site, auth, profile) + end + end + + @doc """ + API reference: + https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#ReportRequest + + Dimensions reference: https://ga-dev-tools.web.app/dimensions-metrics-explorer + """ + def do_import_analytics(site, auth, profile) do + end_date = + Plausible.Stats.Clickhouse.pageviews_begin(site) + |> NaiveDateTime.to_date() + + start_date = Date.add(end_date, -365) + + request = %{ + auth: auth, + profile: profile, + start_date: Date.to_iso8601(start_date), + end_date: Date.to_iso8601(end_date) + } + + visitors = + fetch_analytic_report(request, ["ga:date"], [ + "ga:users", + "ga:pageviews", + "ga:bounceRate", + "ga:avgSessionDuration" + ]) + + case visitors do + {:ok, data} -> + maybe_error = + List.first(data["reports"])["data"]["rows"] + |> Enum.map(&Imported.Visitors.from_google_analytics(site.domain, &1)) + |> Enum.map(&Plausible.ClickhouseRepo.insert(&1, on_conflict: :replace_all)) + |> Keyword.get(:error) + + case maybe_error do + nil -> + {:ok, nil} + + error -> + {:error, error} + end + + {:error, error} -> + Sentry.capture_message("Error fetching Google analytics data", extra: error) + {:error, error} + end + end + + defp fetch_analytic_report(request, dimensions, metrics) do + res = + HTTPoison.post!( + "https://analyticsreporting.googleapis.com/v4/reports:batchGet", + Jason.encode!(%{ + reportRequests: [ + %{ + viewId: request.profile, + dateRanges: [ + %{ + startDate: request.start_date, + endDate: request.end_date + } + ], + dimensions: Enum.map(dimensions, &%{name: &1, histogramBuckets: []}), + metrics: Enum.map(metrics, &%{expression: &1}), + hideTotals: true, + hideValueRanges: true + } + ] + }), + Authorization: "Bearer #{request.auth.access_token}" + ) + + status = + if res.status_code == 200 do + :ok + else + :error + end + + {status, Jason.decode!(res.body)} + end + defp refresh_if_needed(auth) do if Timex.before?(auth.expires, Timex.now() |> Timex.shift(seconds: 30)) do refresh_token(auth) diff --git a/lib/plausible/imported/visitors.ex b/lib/plausible/imported/visitors.ex index da7f63a52ca3..8e91e15875a7 100644 --- a/lib/plausible/imported/visitors.ex +++ b/lib/plausible/imported/visitors.ex @@ -35,4 +35,32 @@ defmodule Plausible.Imported.Visitors do :avg_visit_duration ]) end + + def from_google_analytics(domain, %{ + "dimensions" => [date], + "metrics" => [%{"values" => values}] + }) do + [visitors, pageviews, bounce_rate, avg_session_duration] = + values + |> Enum.map(&Integer.parse/1) + |> Enum.map(&elem(&1, 0)) + + {year, monthday} = String.split_at(date, 4) + {month, day} = String.split_at(monthday, 2) + + datetime = + [year, month, day] + |> Enum.map(&Kernel.elem(Integer.parse(&1), 0)) + |> List.to_tuple() + |> (&NaiveDateTime.from_erl!({&1, {12, 0, 0}})).() + + new(%{ + domain: domain, + date: datetime, + visitors: visitors, + pageviews: pageviews, + bounce_rate: bounce_rate, + avg_visit_duration: avg_session_duration + }) + end end diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index 5fbb89255b8a..69e2a332e6a9 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -560,4 +560,12 @@ defmodule Plausible.Stats.Clickhouse do db_query end end + + def pageviews_begin(site) do + ClickhouseRepo.one( + from e in "events", + where: e.domain == ^site.domain and e.name == "pageview", + select: min(e.timestamp) + ) + end end diff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs index 0e251d00519c..f433a8f9fed9 100644 --- a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs +++ b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs @@ -3,6 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do def change do create_if_not_exists table(:imported_visitors, engine: "MergeTree() ORDER BY (date) SETTINGS index_granularity = 1") do + add :domain, :string add :date, :naive_datetime add :visitors, :UInt64 add :pageviews, :UInt64 From 56a5b5e3f0be98fea4ebc0c017a2ba2176592acc Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 22 Oct 2021 00:33:06 +0100 Subject: [PATCH 008/148] Pass another dataset to main graph for rendering in red This adds another entry to the JSON data returned via the main graph API called `imported_plot`, which is similar to `plot` in form but will be completed with previously imported data. Currently it simply returns the values from `plot` / 2. The data is rendered in the main graph in red without fill, and without an indicator for the present. Rationale: imported data will not continue to grow so there is no projection forward, only backwards. --- assets/js/dashboard/stats/visitor-graph.js | 31 ++++++++++++++----- .../controllers/api/stats_controller.ex | 8 ++++- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js index fda5453592b6..85d9b1403cc3 100644 --- a/assets/js/dashboard/stats/visitor-graph.js +++ b/assets/js/dashboard/stats/visitor-graph.js @@ -6,11 +6,16 @@ import numberFormatter, {durationFormatter} from '../util/number-formatter' import * as api from '../api' import LazyLoader from '../components/lazy-loader' -function buildDataSet(plot, present_index, ctx, label) { +function buildDataSet(graphData, ctx, label) { var gradient = ctx.createLinearGradient(0, 0, 0, 300); gradient.addColorStop(0, 'rgba(101,116,205, 0.2)'); gradient.addColorStop(1, 'rgba(101,116,205, 0)'); + const present_index = graphData.present_index; + var plot = graphData.plot; + var imported_plot = graphData.imported_plot; + var data = []; + if (present_index) { var dashedPart = plot.slice(present_index - 1, present_index + 1); var dashedPlot = (new Array(present_index - 1)).concat(dashedPart) @@ -18,7 +23,7 @@ function buildDataSet(plot, present_index, ctx, label) { plot[i] = undefined } - return [{ + data.push(...[{ label: label, data: plot, borderWidth: 3, @@ -36,9 +41,9 @@ function buildDataSet(plot, present_index, ctx, label) { pointBackgroundColor: 'rgba(101,116,205)', backgroundColor: gradient, fill: true - }] + }]) } else { - return [{ + data.push({ label: label, data: plot, borderWidth: 3, @@ -46,8 +51,20 @@ function buildDataSet(plot, present_index, ctx, label) { pointBackgroundColor: 'rgba(101,116,205)', backgroundColor: gradient, fill: true - }] + }) + }; + + if (imported_plot) { + data.push({ + label: label, + data: imported_plot, + borderWidth: 3, + borderColor: 'rgba(205,116,101)', + fill: false + }) } + + return data; } const MONTHS = [ @@ -109,7 +126,7 @@ class LineGraph extends React.Component { const {graphData} = this.props this.ctx = document.getElementById("main-graph-canvas").getContext('2d'); const label = this.props.query.filters.goal ? 'Converted visitors' : graphData.interval === 'minute' ? 'Pageviews' : 'Visitors' - const dataSet = buildDataSet(graphData.plot, graphData.present_index, this.ctx, label) + const dataSet = buildDataSet(graphData, this.ctx, label) return new Chart(this.ctx, { type: 'line', @@ -196,7 +213,7 @@ class LineGraph extends React.Component { componentDidUpdate(prevProps) { if (this.props.graphData !== prevProps.graphData) { const label = this.props.query.filters.goal ? 'Converted visitors' : this.props.graphData.interval === 'minute' ? 'Pageviews' : 'Visitors' - const newDataset = buildDataSet(this.props.graphData.plot, this.props.graphData.present_index, this.ctx, label) + const newDataset = buildDataSet(this.props.graphData, this.ctx, label) for (let i = 0; i < newDataset[0].data.length; i++) { this.chart.data.datasets[0].data[i] = newDataset[0].data[i] diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 85940990db9c..f3eef5980eef 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -24,13 +24,19 @@ defmodule PlausibleWeb.Api.StatsController do labels = Enum.map(timeseries_result, fn row -> row["date"] end) present_index = present_index_for(site, query, labels) + imported_plot = + if site.has_imported_stats do + Enum.map(plot, &(&1 / 2)) + end + json(conn, %{ plot: plot, labels: labels, present_index: present_index, top_stats: top_stats, interval: query.interval, - sample_percent: sample_percent + sample_percent: sample_percent, + imported_plot: imported_plot }) end From e0ca9bb263131d6864e17645e6be61f755e4a223 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 17 Nov 2021 15:23:40 +0000 Subject: [PATCH 009/148] Hook imported GA data to dashboard timeseries plot --- lib/plausible/imported/visitors.ex | 33 +++++++++++++++---- lib/plausible/site/schema.ex | 2 +- lib/plausible/stats/base.ex | 6 ++-- lib/plausible/stats/timeseries.ex | 16 ++++----- .../controllers/api/stats_controller.ex | 3 +- .../controllers/site_controller.ex | 22 +++++++++---- ...0211112130238_create_imported_visitors.exs | 4 +-- 7 files changed, 58 insertions(+), 28 deletions(-) diff --git a/lib/plausible/imported/visitors.ex b/lib/plausible/imported/visitors.ex index 8e91e15875a7..e4533d028001 100644 --- a/lib/plausible/imported/visitors.ex +++ b/lib/plausible/imported/visitors.ex @@ -1,11 +1,13 @@ defmodule Plausible.Imported.Visitors do use Ecto.Schema + use Plausible.ClickhouseRepo import Ecto.Changeset + alias Plausible.Stats @primary_key false schema "imported_visitors" do field :domain, :string - field :date, :naive_datetime + field :timestamp, :naive_datetime field :visitors, :integer field :pageviews, :integer field :bounce_rate, :integer @@ -18,7 +20,7 @@ defmodule Plausible.Imported.Visitors do attrs, [ :domain, - :date, + :timestamp, :visitors, :pageviews, :bounce_rate, @@ -28,7 +30,7 @@ defmodule Plausible.Imported.Visitors do ) |> validate_required([ :domain, - :date, + :timestamp, :visitors, :pageviews, :bounce_rate, @@ -37,7 +39,7 @@ defmodule Plausible.Imported.Visitors do end def from_google_analytics(domain, %{ - "dimensions" => [date], + "dimensions" => [timestamp], "metrics" => [%{"values" => values}] }) do [visitors, pageviews, bounce_rate, avg_session_duration] = @@ -45,7 +47,7 @@ defmodule Plausible.Imported.Visitors do |> Enum.map(&Integer.parse/1) |> Enum.map(&elem(&1, 0)) - {year, monthday} = String.split_at(date, 4) + {year, monthday} = String.split_at(timestamp, 4) {month, day} = String.split_at(monthday, 2) datetime = @@ -56,11 +58,30 @@ defmodule Plausible.Imported.Visitors do new(%{ domain: domain, - date: datetime, + timestamp: datetime, visitors: visitors, pageviews: pageviews, bounce_rate: bounce_rate, avg_visit_duration: avg_session_duration }) end + + def timeseries(site, query) do + {first_datetime, last_datetime} = Stats.Base.utc_boundaries(query, site.timezone) + + result = + from(v in "imported_visitors", + group_by: fragment("date"), + where: v.domain == ^site.domain, + where: v.timestamp >= ^first_datetime and v.timestamp < ^last_datetime, + select: %{"visitors" => sum(v.visitors)} + ) + |> Stats.Timeseries.select_bucket(site, query) + |> ClickhouseRepo.all() + |> Enum.map(fn row -> {row["date"], row["visitors"]} end) + |> Map.new() + + Stats.Timeseries.buckets(query) + |> Enum.map(fn step -> Map.get(result, step, 0) end) + end end diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index 6cd0ccca0d78..ece54e9ac7e4 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -27,7 +27,7 @@ defmodule Plausible.Site do def changeset(site, attrs \\ %{}) do site - |> cast(attrs, [:domain, :timezone]) + |> cast(attrs, [:domain, :timezone, :has_imported_stats]) |> validate_required([:domain, :timezone]) |> validate_format(:domain, ~r/^[a-zA-Z0-9\-\.\/\:]*$/, message: "only letters, numbers, slashes and period allowed" diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 9d356f449cc2..2702a638580f 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -319,21 +319,21 @@ defmodule Plausible.Stats.Base do defp db_prop_val(:utm_term, @no_ref), do: "" defp db_prop_val(_, val), do: val - defp utc_boundaries(%Query{period: "realtime"}, _timezone) do + def utc_boundaries(%Query{period: "realtime"}, _timezone) do last_datetime = NaiveDateTime.utc_now() |> Timex.shift(seconds: 5) first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -5) {first_datetime, last_datetime} end - defp utc_boundaries(%Query{period: "30m"}, _timezone) do + def utc_boundaries(%Query{period: "30m"}, _timezone) do last_datetime = NaiveDateTime.utc_now() |> Timex.shift(seconds: 5) first_datetime = NaiveDateTime.utc_now() |> Timex.shift(minutes: -30) {first_datetime, last_datetime} end - defp utc_boundaries(%Query{date_range: date_range}, timezone) do + def utc_boundaries(%Query{date_range: date_range}, timezone) do {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00]) first_datetime = diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 253598b03d57..91a73762aed1 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -52,7 +52,7 @@ defmodule Plausible.Stats.Timeseries do |> ClickhouseRepo.all() end - defp buckets(%Query{interval: "month"} = query) do + def buckets(%Query{interval: "month"} = query) do n_buckets = Timex.diff(query.date_range.last, query.date_range.first, :months) Enum.map(n_buckets..0, fn shift -> @@ -62,11 +62,11 @@ defmodule Plausible.Stats.Timeseries do end) end - defp buckets(%Query{interval: "date"} = query) do + def buckets(%Query{interval: "date"} = query) do Enum.into(query.date_range, []) end - defp buckets(%Query{interval: "hour"} = query) do + def buckets(%Query{interval: "hour"} = query) do Enum.map(0..23, fn step -> Timex.to_datetime(query.date_range.first) |> Timex.shift(hours: step) @@ -74,11 +74,11 @@ defmodule Plausible.Stats.Timeseries do end) end - defp buckets(%Query{period: "30m", interval: "minute"}) do + def buckets(%Query{period: "30m", interval: "minute"}) do Enum.into(-30..-1, []) end - defp select_bucket(q, site, %Query{interval: "month"}) do + def select_bucket(q, site, %Query{interval: "month"}) do from( e in q, select_merge: %{ @@ -88,7 +88,7 @@ defmodule Plausible.Stats.Timeseries do ) end - defp select_bucket(q, site, %Query{interval: "date"}) do + def select_bucket(q, site, %Query{interval: "date"}) do from( e in q, select_merge: %{ @@ -97,7 +97,7 @@ defmodule Plausible.Stats.Timeseries do ) end - defp select_bucket(q, site, %Query{interval: "hour"}) do + def select_bucket(q, site, %Query{interval: "hour"}) do from( e in q, select_merge: %{ @@ -106,7 +106,7 @@ defmodule Plausible.Stats.Timeseries do ) end - defp select_bucket(q, _site, %Query{interval: "minute"}) do + def select_bucket(q, _site, %Query{interval: "minute"}) do from( e in q, select_merge: %{ diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index f3eef5980eef..b3a3b5df5197 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -4,6 +4,7 @@ defmodule PlausibleWeb.Api.StatsController do use Plug.ErrorHandler alias Plausible.Stats alias Plausible.Stats.{Query, Filters} + alias Plausible.Imported def main_graph(conn, params) do site = conn.assigns[:site] @@ -26,7 +27,7 @@ defmodule PlausibleWeb.Api.StatsController do imported_plot = if site.has_imported_stats do - Enum.map(plot, &(&1 / 2)) + Imported.Visitors.timeseries(site, query) end json(conn, %{ diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 88da550cad94..a76bb34225f1 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -641,13 +641,21 @@ defmodule PlausibleWeb.SiteController do |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) true -> - site.google_auth - |> Plausible.Site.changeset(%{has_imported_stats: true}) - |> Repo.update!() - - conn - |> put_flash(:success, "Google Analytics data successfully imported") - |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) + case Plausible.Google.Api.import_analytics(site, profile) do + {:ok, _} -> + site + |> Plausible.Site.changeset(%{has_imported_stats: true}) + |> Repo.update!() + + conn + |> put_flash(:success, "Google Analytics data successfully imported") + |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) + + {:error, error} -> + conn + |> put_flash(:error, "Error while fetching: #{error}") + |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) + end end end end diff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs index f433a8f9fed9..dbc12d35027f 100644 --- a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs +++ b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs @@ -2,9 +2,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do use Ecto.Migration def change do - create_if_not_exists table(:imported_visitors, engine: "MergeTree() ORDER BY (date) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_visitors, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :domain, :string - add :date, :naive_datetime + add :timestamp, :naive_datetime add :visitors, :UInt64 add :pageviews, :UInt64 add :bounce_rate, :UInt32 From bffb113192b800a3a87240871c3579e095b09e5d Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 17 Nov 2021 16:59:18 +0000 Subject: [PATCH 010/148] Add settings option to forget imported data --- lib/plausible/clickhouse_repo.ex | 5 ++ lib/plausible/imported/site.ex | 5 ++ lib/plausible/site/schema.ex | 2 +- .../controllers/site_controller.ex | 31 +++++++++-- lib/plausible_web/router.ex | 1 + .../templates/site/settings_general.html.eex | 55 ++++++++++++------- ...211110174617_add_site_imported_boolean.exs | 2 +- 7 files changed, 74 insertions(+), 27 deletions(-) create mode 100644 lib/plausible/imported/site.ex diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex index 6304c49501f4..be2bddf518c1 100644 --- a/lib/plausible/clickhouse_repo.ex +++ b/lib/plausible/clickhouse_repo.ex @@ -17,4 +17,9 @@ defmodule Plausible.ClickhouseRepo do Ecto.Adapters.SQL.query!(__MODULE__, events_sql, [domain]) Ecto.Adapters.SQL.query!(__MODULE__, sessions_sql, [domain]) end + + def clear_imported_stats_for(domain) do + visitors_sql = "ALTER TABLE imported_visitors DELETE WHERE domain = ?" + Ecto.Adapters.SQL.query!(__MODULE__, visitors_sql, [domain]) + end end diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex new file mode 100644 index 000000000000..7aed0a799cd1 --- /dev/null +++ b/lib/plausible/imported/site.ex @@ -0,0 +1,5 @@ +defmodule Plausible.Imported do + def forget(site) do + Plausible.ClickhouseRepo.clear_imported_stats_for(site.domain) + end +end diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index ece54e9ac7e4..fc6437577d14 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -11,7 +11,7 @@ defmodule Plausible.Site do field :public, :boolean field :locked, :boolean field :has_stats, :boolean - field :has_imported_stats, :boolean + field :has_imported_stats, :string many_to_many :members, User, join_through: Plausible.Site.Membership has_many :memberships, Plausible.Site.Membership diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index a76bb34225f1..18c8901d81c1 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -173,8 +173,8 @@ defmodule PlausibleWeb.SiteController do |> Repo.preload([:custom_domain, :google_auth]) google_profiles = - if site.google_auth do - Plausible.Google.Api.get_analytics_view_ids(site) + if is_nil(site.has_imported_stats) and site.google_auth do + Plausible.Google.Api.get_analytics_view_ids(site) end conn @@ -182,6 +182,7 @@ defmodule PlausibleWeb.SiteController do |> render("settings_general.html", site: site, google_profiles: google_profiles, + imported_from: site.has_imported_stats, changeset: Plausible.Site.changeset(site, %{}), layout: {PlausibleWeb.LayoutView, "site_settings.html"} ) @@ -632,7 +633,7 @@ defmodule PlausibleWeb.SiteController do cond do site.has_imported_stats -> conn - |> put_flash(:error, "Third party data already imported") + |> put_flash(:error, "Data already imported from: #{site.has_imported_stats}") |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) profile == "" -> @@ -644,7 +645,7 @@ defmodule PlausibleWeb.SiteController do case Plausible.Google.Api.import_analytics(site, profile) do {:ok, _} -> site - |> Plausible.Site.changeset(%{has_imported_stats: true}) + |> Plausible.Site.changeset(%{has_imported_stats: "Google Analytics"}) |> Repo.update!() conn @@ -658,4 +659,26 @@ defmodule PlausibleWeb.SiteController do end end end + + def forget_imported(conn, _params) do + site = conn.assigns[:site] + + cond do + site.has_imported_stats -> + Plausible.Imported.forget(site) + + site + |> Plausible.Site.changeset(%{has_imported_stats: nil}) + |> Repo.update!() + + conn + |> put_flash(:success, "Imported data has been forgotten") + |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) + + true -> + conn + |> put_flash(:error, "No data has been imported") + |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) + end + end end diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 59ae8304b01a..607331bb29aa 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -241,5 +241,6 @@ defmodule PlausibleWeb.Router do get "/:domain/*path", StatsController, :stats post "/:website/settings/google-import", SiteController, :import_from_google + delete "/:website/settings/forget-imported", SiteController, :forget_imported end end diff --git a/lib/plausible_web/templates/site/settings_general.html.eex b/lib/plausible_web/templates/site/settings_general.html.eex index d0e8041cf251..606cbabf801a 100644 --- a/lib/plausible_web/templates/site/settings_general.html.eex +++ b/lib/plausible_web/templates/site/settings_general.html.eex @@ -57,32 +57,45 @@ <%= if Keyword.get(Application.get_env(:plausible, :google), :client_id) do %> - <%= if @site.google_auth do %> -
- Linked Google account: <%= @site.google_auth.email %> + <%= cond do %> + <% @imported_from -> %> +
  • +
    +

    + Forget Imported Data +

    +

    + Removes all data imported from <%= @imported_from %> +

    +
    + <%= link("Forget imported stats", to: "/#{URI.encode_www_form(@site.domain)}/settings/forget-imported", method: :delete, class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150") %> +
    • + <% @site.google_auth -> %> +
    + Linked Google account: <%= @site.google_auth.email %> - <%= case @google_profiles do %> - <% {:ok, profiles} -> %> -

    - Select the Google Analytics profile you would like to import data from. -

    + <%= case @google_profiles do %> + <% {:ok, profiles} -> %> +

    + Select the Google Analytics profile you would like to import data from. +

    - <%= form_for @conn, "/#{URI.encode_www_form(@site.domain)}/settings/google-import", [class: "max-w-xs"], fn f -> %> -
    -
    - <%= select f, :profile, profiles, prompt: "(Choose profile)", class: "dark:bg-gray-800 mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 dark:border-gray-500 outline-none focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md dark:text-gray-100" %> + <%= form_for @conn, "/#{URI.encode_www_form(@site.domain)}/settings/google-import", [class: "max-w-xs"], fn f -> %> +
    +
    + <%= select f, :profile, profiles, prompt: "(Choose profile)", class: "dark:bg-gray-800 mt-1 block w-full pl-3 pr-10 py-2 text-base border-gray-300 dark:border-gray-500 outline-none focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm rounded-md dark:text-gray-100" %> +
    -
    - <%= submit "Import", class: "button" %> - <% end %> + <%= submit "Import", class: "button" %> + <% end %> - <% {:error, error} -> %> -

    The following error occurred when fetching your Google Analytics profiles.

    -

    <%= error %>

    - <% end %> + <% {:error, error} -> %> +

    The following error occurred when fetching your Google Analytics profiles.

    +

    <%= error %>

    + <% end %> - <% else %> - <%= button("Continue with Google", to: Plausible.Google.Api.authorize_url(@site.id, "general"), class: "button mt-8") %> + <% true -> %> + <%= button("Continue with Google", to: Plausible.Google.Api.authorize_url(@site.id, "general"), class: "button mt-8") %> <% end %> <% else %>
    diff --git a/priv/repo/migrations/20211110174617_add_site_imported_boolean.exs b/priv/repo/migrations/20211110174617_add_site_imported_boolean.exs index a64377bea5c4..e6461b4cab59 100644 --- a/priv/repo/migrations/20211110174617_add_site_imported_boolean.exs +++ b/priv/repo/migrations/20211110174617_add_site_imported_boolean.exs @@ -3,7 +3,7 @@ defmodule Plausible.Repo.Migrations.GoogleAuthImportedBoolean do def change do alter table(:sites) do - add :has_imported_stats, :boolean + add :has_imported_stats, :string, null: true, default: nil end end end From 54703d2c1f0e601f4439b71b04732273fb352db7 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 18 Nov 2021 16:22:50 +0000 Subject: [PATCH 011/148] Import sources from google analytics --- lib/plausible/clickhouse_repo.ex | 2 + lib/plausible/google/api.ex | 99 ++++++++++++------- lib/plausible/imported/site.ex | 61 ++++++++++++ lib/plausible/imported/sources.ex | 33 +++++++ lib/plausible/imported/visitors.ex | 28 ------ .../controllers/site_controller.ex | 9 +- ...20211118160420_create_imported_sources.exs | 12 +++ 7 files changed, 176 insertions(+), 68 deletions(-) create mode 100644 lib/plausible/imported/sources.ex create mode 100644 priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex index be2bddf518c1..58ab212edc6d 100644 --- a/lib/plausible/clickhouse_repo.ex +++ b/lib/plausible/clickhouse_repo.ex @@ -20,6 +20,8 @@ defmodule Plausible.ClickhouseRepo do def clear_imported_stats_for(domain) do visitors_sql = "ALTER TABLE imported_visitors DELETE WHERE domain = ?" + sources_sql = "ALTER TABLE imported_sources DELETE WHERE domain = ?" Ecto.Adapters.SQL.query!(__MODULE__, visitors_sql, [domain]) + Ecto.Adapters.SQL.query!(__MODULE__, sources_sql, [domain]) end end diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index e34fd447cdae..35b5e806d9ee 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -177,27 +177,48 @@ defmodule Plausible.Google.Api do end_date: Date.to_iso8601(end_date) } - visitors = - fetch_analytic_report(request, ["ga:date"], [ - "ga:users", - "ga:pageviews", - "ga:bounceRate", - "ga:avgSessionDuration" - ]) - - case visitors do + request_data = [ + # Visitors + { + ["ga:date"], + [ + "ga:users", + "ga:pageviews", + "ga:bounceRate", + "ga:avgSessionDuration" + ] + }, + # Sources + { + ["ga:date", "ga:source"], + ["ga:users"] + } + ] + + response = fetch_analytic_reports(request, request_data) + + case response do {:ok, data} -> maybe_error = - List.first(data["reports"])["data"]["rows"] - |> Enum.map(&Imported.Visitors.from_google_analytics(site.domain, &1)) - |> Enum.map(&Plausible.ClickhouseRepo.insert(&1, on_conflict: :replace_all)) + ["visitors", "sources"] + |> Enum.with_index() + |> Enum.map(fn {metric, index} -> + Task.async(fn -> + Enum.fetch!(data, index) + |> Imported.from_google_analytics(site.domain, metric) + end) + end) + |> Enum.map(&Task.await/1) |> Keyword.get(:error) case maybe_error do nil -> {:ok, nil} - error -> + {:error, error} -> + Plausible.ClickhouseRepo.clear_imported_stats_for(site.domain) + + Sentry.capture_message("Error saving Google analytics data", extra: error) {:error, error} end @@ -207,38 +228,40 @@ defmodule Plausible.Google.Api do end end - defp fetch_analytic_report(request, dimensions, metrics) do + defp fetch_analytic_reports(request, request_data) do + reports = + Enum.map(request_data, fn {dimensions, metrics} -> + %{ + viewId: request.profile, + dateRanges: [ + %{ + startDate: request.start_date, + endDate: request.end_date + } + ], + dimensions: Enum.map(dimensions, &%{name: &1, histogramBuckets: []}), + metrics: Enum.map(metrics, &%{expression: &1}), + hideTotals: true, + hideValueRanges: true + } + end) + res = HTTPoison.post!( "https://analyticsreporting.googleapis.com/v4/reports:batchGet", - Jason.encode!(%{ - reportRequests: [ - %{ - viewId: request.profile, - dateRanges: [ - %{ - startDate: request.start_date, - endDate: request.end_date - } - ], - dimensions: Enum.map(dimensions, &%{name: &1, histogramBuckets: []}), - metrics: Enum.map(metrics, &%{expression: &1}), - hideTotals: true, - hideValueRanges: true - } - ] - }), + Jason.encode!(%{reportRequests: reports}), Authorization: "Bearer #{request.auth.access_token}" ) - status = - if res.status_code == 200 do - :ok - else - :error - end + if res.status_code == 200 do + data = + Jason.decode!(res.body)["reports"] + |> Enum.map(& &1["data"]["rows"]) - {status, Jason.decode!(res.body)} + {:ok, data} + else + {:error, Jason.decode!(res.body)} + end end defp refresh_if_needed(auth) do diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 7aed0a799cd1..5ab06d717cec 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -1,5 +1,66 @@ defmodule Plausible.Imported do + alias Plausible.Imported.{Visitors, Sources} + def forget(site) do Plausible.ClickhouseRepo.clear_imported_stats_for(site.domain) end + + def from_google_analytics(data, domain, metric) do + maybe_error = + data + |> Enum.map(&new_from_google_analytics(domain, metric, &1)) + |> Enum.map(&Plausible.ClickhouseRepo.insert(&1, on_conflict: :replace_all)) + |> Keyword.get(:error) + + case maybe_error do + nil -> + {:ok, nil} + + error -> + {:error, error} + end + end + + defp new_from_google_analytics(domain, "visitors", %{ + "dimensions" => [timestamp], + "metrics" => [%{"values" => values}] + }) do + [visitors, pageviews, bounce_rate, avg_session_duration] = + values + |> Enum.map(&Integer.parse/1) + |> Enum.map(&elem(&1, 0)) + + Visitors.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + visitors: visitors, + pageviews: pageviews, + bounce_rate: bounce_rate, + avg_visit_duration: avg_session_duration + }) + end + + defp new_from_google_analytics(domain, "sources", %{ + "dimensions" => [timestamp, source], + "metrics" => [%{"values" => [value]}] + }) do + {visitors, ""} = Integer.parse(value) + + Sources.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + source: source, + visitors: visitors + }) + end + + defp format_timestamp(timestamp) do + {year, monthday} = String.split_at(timestamp, 4) + {month, day} = String.split_at(monthday, 2) + + [year, month, day] + |> Enum.map(&Kernel.elem(Integer.parse(&1), 0)) + |> List.to_tuple() + |> (&NaiveDateTime.from_erl!({&1, {12, 0, 0}})).() + end end diff --git a/lib/plausible/imported/sources.ex b/lib/plausible/imported/sources.ex new file mode 100644 index 000000000000..24edafda6e08 --- /dev/null +++ b/lib/plausible/imported/sources.ex @@ -0,0 +1,33 @@ +defmodule Plausible.Imported.Sources do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_sources" do + field :domain, :string + field :timestamp, :naive_datetime + field :source, :string + field :visitors, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :source, + :visitors + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :source, + :visitors + ]) + end +end diff --git a/lib/plausible/imported/visitors.ex b/lib/plausible/imported/visitors.ex index e4533d028001..f12da891f58a 100644 --- a/lib/plausible/imported/visitors.ex +++ b/lib/plausible/imported/visitors.ex @@ -38,34 +38,6 @@ defmodule Plausible.Imported.Visitors do ]) end - def from_google_analytics(domain, %{ - "dimensions" => [timestamp], - "metrics" => [%{"values" => values}] - }) do - [visitors, pageviews, bounce_rate, avg_session_duration] = - values - |> Enum.map(&Integer.parse/1) - |> Enum.map(&elem(&1, 0)) - - {year, monthday} = String.split_at(timestamp, 4) - {month, day} = String.split_at(monthday, 2) - - datetime = - [year, month, day] - |> Enum.map(&Kernel.elem(Integer.parse(&1), 0)) - |> List.to_tuple() - |> (&NaiveDateTime.from_erl!({&1, {12, 0, 0}})).() - - new(%{ - domain: domain, - timestamp: datetime, - visitors: visitors, - pageviews: pageviews, - bounce_rate: bounce_rate, - avg_visit_duration: avg_session_duration - }) - end - def timeseries(site, query) do {first_datetime, last_datetime} = Stats.Base.utc_boundaries(query, site.timezone) diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 18c8901d81c1..2dcbecf42f85 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -174,7 +174,7 @@ defmodule PlausibleWeb.SiteController do google_profiles = if is_nil(site.has_imported_stats) and site.google_auth do - Plausible.Google.Api.get_analytics_view_ids(site) + Plausible.Google.Api.get_analytics_view_ids(site) end conn @@ -653,8 +653,13 @@ defmodule PlausibleWeb.SiteController do |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) {:error, error} -> + message = + error + |> Map.get("error") + |> Map.get("message") + conn - |> put_flash(:error, "Error while fetching: #{error}") + |> put_flash(:error, "Error while fetching: #{message}") |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) end end diff --git a/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs b/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs new file mode 100644 index 000000000000..26da7ee3ca2a --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs @@ -0,0 +1,12 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedSources do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_sources, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :source, :string + add :visitors, :UInt64 + end + end +end From b5b5ceccb7181aad80ca2fe124d31a4dc36dc506 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 19 Nov 2021 16:05:45 +0000 Subject: [PATCH 012/148] Merge imported sources when queried --- lib/plausible/stats/base.ex | 26 ++++++++++++++++++++++++++ lib/plausible/stats/breakdown.ex | 1 + 2 files changed, 27 insertions(+) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 2702a638580f..d19ad695521f 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -355,6 +355,32 @@ defmodule Plausible.Stats.Base do |> String.replace(~r/(?= ^first_datetime and i.timestamp < ^last_datetime, + select: %{source: i.source, visitors: sum(i.visitors)} + ) + + from(s in q, + join: i in subquery(imported_q), + on: s.source == i.source + ) + else + q + end + end + + def merge_imported(q, _, _, _), do: q + defp add_sample_hint(db_q, query) do case query.sample_threshold do "infinite" -> diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 72eb1e651f42..84403a993cdb 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -189,6 +189,7 @@ defmodule Plausible.Stats.Breakdown do |> filter_converted_sessions(site, query) |> do_group_by(property) |> select_session_metrics(metrics) + |> merge_imported(site, query, property) |> ClickhouseRepo.all() end From e6ab2703e384342d0c5131a826f26002c71e1c30 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 23 Nov 2021 23:05:22 +0000 Subject: [PATCH 013/148] Merge imported source data native data when querying sources --- lib/plausible/stats/base.ex | 48 ++++++++++--------- lib/plausible/stats/breakdown.ex | 9 ++-- lib/plausible/stats/timeseries.ex | 2 +- .../controllers/api/stats_controller.ex | 8 ++-- 4 files changed, 38 insertions(+), 29 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index d19ad695521f..f6c85f924676 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -216,7 +216,7 @@ defmodule Plausible.Stats.Base do def select_event_metrics(q, ["visitors" | rest]) do from(e in q, select_merge: %{ - "visitors" => fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id) + visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id) } ) |> select_event_metrics(rest) @@ -268,7 +268,7 @@ defmodule Plausible.Stats.Base do def select_session_metrics(q, ["visitors" | rest]) do from(s in q, select_merge: %{ - "visitors" => fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id) + visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id) } ) |> select_session_metrics(rest) @@ -355,31 +355,35 @@ defmodule Plausible.Stats.Base do |> String.replace(~r/(?= ^first_datetime and i.timestamp < ^last_datetime, - select: %{source: i.source, visitors: sum(i.visitors)} - ) + def merge_imported(q, site, query, "visit:source", {limit, page}) do + offset = (page - 1) * limit + {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) - from(s in q, - join: i in subquery(imported_q), - on: s.source == i.source + imported_q = + from( + i in "imported_sources", + group_by: i.source, + where: i.domain == ^site.domain, + where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, + select: %{source: i.source, visitors: sum(i.visitors)} ) - else - q - end + + from(s in Ecto.Query.subquery(q), + full_join: i in subquery(imported_q), + on: s.source == i.source, + select: %{ + source: fragment("if(empty(?), ?, ?)", s.source, i.source, s.source), + visitors: fragment("toUInt64(?) + toUInt64(?)", s.visitors, i.visitors) + }, + order_by: [desc: fragment("toUInt64(?) + toUInt64(?)", s.visitors, i.visitors)], + limit: ^limit, + offset: ^offset + ) end - def merge_imported(q, _, _, _), do: q + def merge_imported(q, _, _, _, _), do: q defp add_sample_hint(db_q, query) do case query.sample_threshold do diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 84403a993cdb..04549707e6d8 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -92,7 +92,9 @@ defmodule Plausible.Stats.Breakdown do event_metrics = Enum.filter(metrics, &(&1 in @event_metrics)) session_metrics = Enum.filter(metrics, &(&1 in @session_metrics)) - event_result = breakdown_events(site, query, "event:page", event_metrics, pagination) + event_result = + breakdown_events(site, query, "event:page", event_metrics, pagination) + |> transform_keys(%{visitors: "visitors"}) event_result = if "time_on_page" in metrics do @@ -148,6 +150,7 @@ defmodule Plausible.Stats.Breakdown do def breakdown(site, query, property, metrics, pagination) do breakdown_sessions(site, query, property, metrics, pagination) + |> transform_keys(%{visitors: "visitors"}) end defp zip_results(event_result, session_result, property, metrics) do @@ -189,7 +192,7 @@ defmodule Plausible.Stats.Breakdown do |> filter_converted_sessions(site, query) |> do_group_by(property) |> select_session_metrics(metrics) - |> merge_imported(site, query, property) + |> merge_imported(site, query, property, {limit, page}) |> ClickhouseRepo.all() end @@ -327,7 +330,7 @@ defmodule Plausible.Stats.Breakdown do s in q, group_by: s.referrer_source, select_merge: %{ - "source" => fragment("if(empty(?), ?, ?)", s.referrer_source, @no_ref, s.referrer_source) + source: fragment("if(empty(?), ?, ?)", s.referrer_source, @no_ref, s.referrer_source) } ) end diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 91a73762aed1..9ae23d3f7ad9 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -119,7 +119,7 @@ defmodule Plausible.Stats.Timeseries do Enum.reduce(metrics, %{"date" => date}, fn metric, row -> case metric do "pageviews" -> Map.merge(row, %{"pageviews" => 0}) - "visitors" -> Map.merge(row, %{"visitors" => 0}) + "visitors" -> Map.merge(row, %{visitors: 0}) "visits" -> Map.merge(row, %{"visits" => 0}) "bounce_rate" -> Map.merge(row, %{"bounce_rate" => nil}) "visit_duration" -> Map.merge(row, %{"visit_duration" => nil}) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index b3a3b5df5197..e36e345e39bc 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -21,7 +21,7 @@ defmodule PlausibleWeb.Api.StatsController do {top_stats, sample_percent} = fetch_top_stats(site, query) timeseries_result = Task.await(timeseries) - plot = Enum.map(timeseries_result, fn row -> row["visitors"] end) + plot = Enum.map(timeseries_result, fn row -> row[:visitors] end) labels = Enum.map(timeseries_result, fn row -> row["date"] end) present_index = present_index_for(site, query, labels) @@ -220,8 +220,9 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:source", metrics, pagination) + |> IO.inspect() |> maybe_add_cr(site, query, pagination, "source", "visit:source") - |> transform_keys(%{"source" => "name"}) + |> transform_keys(%{source: "name"}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do @@ -446,6 +447,7 @@ defmodule PlausibleWeb.Api.StatsController do pages = Stats.breakdown(site, query, "event:page", metrics, pagination) + |> transform_keys(%{visitors: "visitors"}) |> maybe_add_cr(site, query, pagination, "page", "event:page") |> transform_keys(%{"page" => "name"}) @@ -824,7 +826,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{ params["prop_name"] => "name", "events" => "total_conversions", - "visitors" => "unique_conversions" + :visitors => "unique_conversions" }) |> Enum.map(fn prop -> Map.put( From 0c5c4b7c86ad869a7859bad0401de09c7f0e26d3 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 24 Nov 2021 11:08:57 +0000 Subject: [PATCH 014/148] Start converting metrics to atoms so they can be subqueried This changes "visitors" and in some places "sources" to atoms. This does not change the behaviour of the functions - the tests all pass unchanged following this commit. This is necessary as joining subqueries requires that the keys in `select` statements be atoms and not strings. --- lib/plausible/stats/aggregate.ex | 2 +- lib/plausible/stats/base.ex | 4 +- lib/plausible/stats/breakdown.ex | 9 +- lib/plausible/stats/timeseries.ex | 4 +- .../api/external_stats_controller.ex | 6 + .../controllers/api/stats_controller.ex | 124 +++++++++--------- .../controllers/stats_controller.ex | 2 +- 7 files changed, 76 insertions(+), 75 deletions(-) diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 1880c6278243..2b1c6deebf22 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -3,7 +3,7 @@ defmodule Plausible.Stats.Aggregate do use Plausible.ClickhouseRepo import Plausible.Stats.Base - @event_metrics ["visitors", "pageviews", "events", "sample_percent"] + @event_metrics [:visitors, "pageviews", "events", "sample_percent"] @session_metrics ["visits", "bounce_rate", "visit_duration", "sample_percent"] def aggregate(site, query, metrics) do diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index f6c85f924676..ad9fc81d5642 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -213,7 +213,7 @@ defmodule Plausible.Stats.Base do |> select_event_metrics(rest) end - def select_event_metrics(q, ["visitors" | rest]) do + def select_event_metrics(q, [:visitors | rest]) do from(e in q, select_merge: %{ visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id) @@ -265,7 +265,7 @@ defmodule Plausible.Stats.Base do |> select_session_metrics(rest) end - def select_session_metrics(q, ["visitors" | rest]) do + def select_session_metrics(q, [:visitors | rest]) do from(s in q, select_merge: %{ visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id) diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 04549707e6d8..b6b52e97638f 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -4,7 +4,7 @@ defmodule Plausible.Stats.Breakdown do alias Plausible.Stats.Query @no_ref "Direct / None" - @event_metrics ["visitors", "pageviews", "events"] + @event_metrics [:visitors, "pageviews", "events"] @session_metrics ["visits", "bounce_rate", "visit_duration"] @event_props ["event:page", "event:page_match", "event:name"] @@ -92,9 +92,7 @@ defmodule Plausible.Stats.Breakdown do event_metrics = Enum.filter(metrics, &(&1 in @event_metrics)) session_metrics = Enum.filter(metrics, &(&1 in @session_metrics)) - event_result = - breakdown_events(site, query, "event:page", event_metrics, pagination) - |> transform_keys(%{visitors: "visitors"}) + event_result = breakdown_events(site, query, "event:page", event_metrics, pagination) event_result = if "time_on_page" in metrics do @@ -150,11 +148,10 @@ defmodule Plausible.Stats.Breakdown do def breakdown(site, query, property, metrics, pagination) do breakdown_sessions(site, query, property, metrics, pagination) - |> transform_keys(%{visitors: "visitors"}) end defp zip_results(event_result, session_result, property, metrics) do - sort_by = if Enum.member?(metrics, "visitors"), do: "visitors", else: List.first(metrics) + sort_by = if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics) property = property diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 9ae23d3f7ad9..5760ca47d480 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -4,7 +4,7 @@ defmodule Plausible.Stats.Timeseries do import Plausible.Stats.Base use Plausible.Stats.Fragments - @event_metrics ["visitors", "pageviews"] + @event_metrics [:visitors, "pageviews"] @session_metrics ["visits", "bounce_rate", "visit_duration"] def timeseries(site, query, metrics) do steps = buckets(query) @@ -119,7 +119,7 @@ defmodule Plausible.Stats.Timeseries do Enum.reduce(metrics, %{"date" => date}, fn metric, row -> case metric do "pageviews" -> Map.merge(row, %{"pageviews" => 0}) - "visitors" -> Map.merge(row, %{visitors: 0}) + :visitors -> Map.merge(row, %{visitors: 0}) "visits" -> Map.merge(row, %{"visits" => 0}) "bounce_rate" -> Map.merge(row, %{"bounce_rate" => nil}) "visit_duration" -> Map.merge(row, %{"visit_duration" => nil}) diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index 3631785ddefe..b9127e169556 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -133,6 +133,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController do "The metric `#{invalid_metric}` is not recognized. Find valid metrics from the documentation: https://plausible.io/docs/stats-api#get-apiv1statsbreakdown"} end else + metrics = metrics |> Enum.map(fn item -> + case item do + "visitors" -> :visitors + metric -> metric + end + end) {:ok, metrics} end end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index e36e345e39bc..f89082df6c01 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -17,7 +17,7 @@ defmodule PlausibleWeb.Api.StatsController do query end - timeseries = Task.async(fn -> Stats.timeseries(site, timeseries_query, ["visitors"]) end) + timeseries = Task.async(fn -> Stats.timeseries(site, timeseries_query, [:visitors]) end) {top_stats, sample_percent} = fetch_top_stats(site, query) timeseries_result = Task.await(timeseries) @@ -74,9 +74,9 @@ defmodule PlausibleWeb.Api.StatsController do query_30m = %Query{query | period: "30m"} %{ - "visitors" => %{"value" => visitors}, + :visitors => %{"value" => visitors}, "pageviews" => %{"value" => pageviews} - } = Stats.aggregate(site, query_30m, ["visitors", "pageviews"]) + } = Stats.aggregate(site, query_30m, [:visitors, "pageviews"]) stats = [ %{ @@ -102,22 +102,22 @@ defmodule PlausibleWeb.Api.StatsController do prev_total_query = Query.shift_back(total_q, site) %{ - "visitors" => %{"value" => unique_visitors} - } = Stats.aggregate(site, total_q, ["visitors"]) + :visitors => %{"value" => unique_visitors} + } = Stats.aggregate(site, total_q, [:visitors]) %{ - "visitors" => %{"value" => prev_unique_visitors} - } = Stats.aggregate(site, prev_total_query, ["visitors"]) + :visitors => %{"value" => prev_unique_visitors} + } = Stats.aggregate(site, prev_total_query, [:visitors]) %{ - "visitors" => %{"value" => converted_visitors}, + :visitors => %{"value" => converted_visitors}, "events" => %{"value" => completions} - } = Stats.aggregate(site, query, ["visitors", "events"]) + } = Stats.aggregate(site, query, [:visitors, "events"]) %{ - "visitors" => %{"value" => prev_converted_visitors}, + :visitors => %{"value" => prev_converted_visitors}, "events" => %{"value" => prev_completions} - } = Stats.aggregate(site, prev_query, ["visitors", "events"]) + } = Stats.aggregate(site, prev_query, [:visitors, "events"]) conversion_rate = calculate_cr(unique_visitors, converted_visitors) prev_conversion_rate = calculate_cr(prev_unique_visitors, prev_converted_visitors) @@ -153,9 +153,9 @@ defmodule PlausibleWeb.Api.StatsController do metrics = if query.filters["event:page"] do - ["visitors", "pageviews", "bounce_rate", "time_on_page", "sample_percent"] + [:visitors, "pageviews", "bounce_rate", "time_on_page", "sample_percent"] else - ["visitors", "pageviews", "bounce_rate", "visit_duration", "sample_percent"] + [:visitors, "pageviews", "bounce_rate", "visit_duration", "sample_percent"] end current_results = Stats.aggregate(site, query, metrics) @@ -163,7 +163,7 @@ defmodule PlausibleWeb.Api.StatsController do stats = [ - top_stats_entry(current_results, prev_results, "Unique visitors", "visitors"), + top_stats_entry(current_results, prev_results, "Unique visitors", :visitors), top_stats_entry(current_results, prev_results, "Total pageviews", "pageviews"), top_stats_entry(current_results, prev_results, "Bounce rate", "bounce_rate"), top_stats_entry(current_results, prev_results, "Visit duration", "visit_duration"), @@ -216,21 +216,20 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) metrics = - if params["detailed"], do: ["visitors", "bounce_rate", "visit_duration"], else: ["visitors"] + if params["detailed"], do: [:visitors, "bounce_rate", "visit_duration"], else: [:visitors] res = Stats.breakdown(site, query, "visit:source", metrics, pagination) - |> IO.inspect() - |> maybe_add_cr(site, query, pagination, "source", "visit:source") + |> maybe_add_cr(site, query, pagination, :source, "visit:source") |> transform_keys(%{source: "name"}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do res - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", "visitors", "bounce_rate", "visit_duration"]) + res |> to_csv(["name", :visitors, "bounce_rate", "visit_duration"]) end else json(conn, res) @@ -246,7 +245,7 @@ defmodule PlausibleWeb.Api.StatsController do |> maybe_hide_noref("visit:utm_medium", params) pagination = parse_pagination(params) - metrics = ["visitors", "bounce_rate", "visit_duration"] + metrics = [:visitors, "bounce_rate", "visit_duration"] res = Stats.breakdown(site, query, "visit:utm_medium", metrics, pagination) @@ -256,10 +255,10 @@ defmodule PlausibleWeb.Api.StatsController do if params["csv"] do if Map.has_key?(query.filters, "event:goal") do res - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", "visitors", "bounce_rate", "visit_duration"]) + res |> to_csv(["name", :visitors, "bounce_rate", "visit_duration"]) end else json(conn, res) @@ -275,7 +274,7 @@ defmodule PlausibleWeb.Api.StatsController do |> maybe_hide_noref("visit:utm_campaign", params) pagination = parse_pagination(params) - metrics = ["visitors", "bounce_rate", "visit_duration"] + metrics = [:visitors, "bounce_rate", "visit_duration"] res = Stats.breakdown(site, query, "visit:utm_campaign", metrics, pagination) @@ -285,10 +284,10 @@ defmodule PlausibleWeb.Api.StatsController do if params["csv"] do if Map.has_key?(query.filters, "event:goal") do res - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", "visitors", "bounce_rate", "visit_duration"]) + res |> to_csv(["name", :visitors, "bounce_rate", "visit_duration"]) end else json(conn, res) @@ -362,7 +361,7 @@ defmodule PlausibleWeb.Api.StatsController do |> maybe_hide_noref("visit:utm_source", params) pagination = parse_pagination(params) - metrics = ["visitors", "bounce_rate", "visit_duration"] + metrics = [:visitors, "bounce_rate", "visit_duration"] res = Stats.breakdown(site, query, "visit:utm_source", metrics, pagination) @@ -372,10 +371,10 @@ defmodule PlausibleWeb.Api.StatsController do if params["csv"] do if Map.has_key?(query.filters, "event:goal") do res - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", "visitors", "bounce_rate", "visit_duration"]) + res |> to_csv(["name", :visitors, "bounce_rate", "visit_duration"]) end else json(conn, res) @@ -395,7 +394,7 @@ defmodule PlausibleWeb.Api.StatsController do google_api().fetch_stats(site, query, params["limit"] || 9) end - %{"visitors" => %{"value" => total_visitors}} = Stats.aggregate(site, query, ["visitors"]) + %{:visitors => %{"value" => total_visitors}} = Stats.aggregate(site, query, [:visitors]) case search_terms do nil -> @@ -423,14 +422,14 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) metrics = - if params["detailed"], do: ["visitors", "bounce_rate", "visit_duration"], else: ["visitors"] + if params["detailed"], do: [:visitors, "bounce_rate", "visit_duration"], else: [:visitors] referrers = Stats.breakdown(site, query, "visit:referrer", metrics, pagination) |> maybe_add_cr(site, query, pagination, "referrer", "visit:referrer") |> transform_keys(%{"referrer" => "name"}) - %{"visitors" => %{"value" => total_visitors}} = Stats.aggregate(site, query, ["visitors"]) + %{:visitors => %{"value" => total_visitors}} = Stats.aggregate(site, query, [:visitors]) json(conn, %{referrers: referrers, total_visitors: total_visitors}) end @@ -440,16 +439,15 @@ defmodule PlausibleWeb.Api.StatsController do metrics = if params["detailed"], - do: ["visitors", "pageviews", "bounce_rate", "time_on_page"], - else: ["visitors"] + do: [:visitors, "pageviews", "bounce_rate", "time_on_page"], + else: [:visitors] pagination = parse_pagination(params) pages = Stats.breakdown(site, query, "event:page", metrics, pagination) - |> transform_keys(%{visitors: "visitors"}) |> maybe_add_cr(site, query, pagination, "page", "event:page") - |> transform_keys(%{"page" => "name"}) + |> transform_keys(%{"page" => "name", visitors: "visitors"}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do @@ -468,14 +466,14 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = Query.from(site.timezone, params) |> Filters.add_prefix() pagination = parse_pagination(params) - metrics = ["visitors", "visits", "visit_duration"] + metrics = [:visitors, "visits", "visit_duration"] entry_pages = Stats.breakdown(site, query, "visit:entry_page", metrics, pagination) |> maybe_add_cr(site, query, pagination, "entry_page", "visit:entry_page") |> transform_keys(%{ "entry_page" => "name", - "visitors" => "unique_entrances", + :visitors => "unique_entrances", "visits" => "total_entrances" }) @@ -496,14 +494,14 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = Query.from(site.timezone, params) |> Filters.add_prefix() {limit, page} = parse_pagination(params) - metrics = ["visitors", "visits"] + metrics = [:visitors, "visits"] exit_pages = Stats.breakdown(site, query, "visit:exit_page", metrics, {limit, page}) |> maybe_add_cr(site, query, {limit, page}, "exit_page", "visit:exit_page") |> transform_keys(%{ "exit_page" => "name", - "visitors" => "unique_exits", + :visitors => "unique_exits", "visits" => "total_exits" }) @@ -552,7 +550,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) countries = - Stats.breakdown(site, query, "visit:country", ["visitors"], pagination) + Stats.breakdown(site, query, "visit:country", [:visitors], pagination) |> maybe_add_cr(site, query, {300, 1}, "country", "visit:country") |> transform_keys(%{"country" => "code"}) |> maybe_add_percentages(query) @@ -567,10 +565,10 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do countries - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - countries |> to_csv(["name", "visitors"]) + countries |> to_csv(["name", :visitors]) end else countries = @@ -666,7 +664,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) browsers = - Stats.breakdown(site, query, "visit:browser", ["visitors"], pagination) + Stats.breakdown(site, query, "visit:browser", [:visitors], pagination) |> maybe_add_cr(site, query, pagination, "browser", "visit:browser") |> transform_keys(%{"browser" => "name"}) |> maybe_add_percentages(query) @@ -674,10 +672,10 @@ defmodule PlausibleWeb.Api.StatsController do if params["csv"] do if Map.has_key?(query.filters, "event:goal") do browsers - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - browsers |> to_csv(["name", "visitors"]) + browsers |> to_csv(["name", :visitors]) end else json(conn, browsers) @@ -690,7 +688,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) versions = - Stats.breakdown(site, query, "visit:browser_version", ["visitors"], pagination) + Stats.breakdown(site, query, "visit:browser_version", [:visitors], pagination) |> maybe_add_cr(site, query, pagination, "browser_version", "visit:browser_version") |> transform_keys(%{"browser_version" => "name"}) |> maybe_add_percentages(query) @@ -704,7 +702,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) systems = - Stats.breakdown(site, query, "visit:os", ["visitors"], pagination) + Stats.breakdown(site, query, "visit:os", [:visitors], pagination) |> maybe_add_cr(site, query, pagination, "os", "visit:os") |> transform_keys(%{"os" => "name"}) |> maybe_add_percentages(query) @@ -712,10 +710,10 @@ defmodule PlausibleWeb.Api.StatsController do if params["csv"] do if Map.has_key?(query.filters, "event:goal") do systems - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - systems |> to_csv(["name", "visitors"]) + systems |> to_csv(["name", :visitors]) end else json(conn, systems) @@ -728,7 +726,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) versions = - Stats.breakdown(site, query, "visit:os_version", ["visitors"], pagination) + Stats.breakdown(site, query, "visit:os_version", [:visitors], pagination) |> maybe_add_cr(site, query, pagination, "os_version", "visit:os_version") |> transform_keys(%{"os_version" => "name"}) |> maybe_add_percentages(query) @@ -742,7 +740,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) sizes = - Stats.breakdown(site, query, "visit:device", ["visitors"], pagination) + Stats.breakdown(site, query, "visit:device", [:visitors], pagination) |> maybe_add_cr(site, query, pagination, "device", "visit:device") |> transform_keys(%{"device" => "name"}) |> maybe_add_percentages(query) @@ -750,10 +748,10 @@ defmodule PlausibleWeb.Api.StatsController do if params["csv"] do if Map.has_key?(query.filters, "event:goal") do sizes - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - sizes |> to_csv(["name", "visitors"]) + sizes |> to_csv(["name", :visitors]) end else json(conn, sizes) @@ -781,7 +779,7 @@ defmodule PlausibleWeb.Api.StatsController do total_q = Query.remove_goal(query) - %{"visitors" => %{"value" => total_visitors}} = Stats.aggregate(site, total_q, ["visitors"]) + %{visitors: %{"value" => total_visitors}} = Stats.aggregate(site, total_q, [:visitors]) prop_names = if query.filters["event:goal"] do @@ -791,10 +789,10 @@ defmodule PlausibleWeb.Api.StatsController do end conversions = - Stats.breakdown(site, query, "event:goal", ["visitors", "events"], {100, 1}) + Stats.breakdown(site, query, "event:goal", [:visitors, "events"], {100, 1}) |> transform_keys(%{ "goal" => "name", - "visitors" => "unique_conversions", + :visitors => "unique_conversions", "events" => "total_conversions" }) |> Enum.map(fn goal -> @@ -817,12 +815,12 @@ defmodule PlausibleWeb.Api.StatsController do total_q = Query.remove_goal(query) - %{"visitors" => %{"value" => unique_visitors}} = Stats.aggregate(site, total_q, ["visitors"]) + %{:visitors => %{"value" => unique_visitors}} = Stats.aggregate(site, total_q, [:visitors]) prop_name = "event:props:" <> params["prop_name"] props = - Stats.breakdown(site, query, prop_name, ["visitors", "events"], pagination) + Stats.breakdown(site, query, prop_name, [:visitors, "events"], pagination) |> transform_keys(%{ params["prop_name"] => "name", "events" => "total_conversions", @@ -907,10 +905,10 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do stat_list else - total = Enum.reduce(stat_list, 0, fn %{"visitors" => count}, total -> total + count end) + total = Enum.reduce(stat_list, 0, fn %{visitors: count}, total -> total + count end) Enum.map(stat_list, fn stat -> - Map.put(stat, "percentage", round(stat["visitors"] / total * 100)) + Map.put(stat, "percentage", round(stat[:visitors] / total * 100)) end) end end @@ -931,8 +929,8 @@ defmodule PlausibleWeb.Api.StatsController do without_goal = Enum.find(list_without_goals, fn s -> s[key_name] === item[key_name] end) item - |> Map.put(:total_visitors, without_goal["visitors"]) - |> Map.put("conversion_rate", calculate_cr(without_goal["visitors"], item["visitors"])) + |> Map.put(:total_visitors, without_goal[:visitors]) + |> Map.put(:conversion_rate, calculate_cr(without_goal[:visitors], item[:visitors])) end) end @@ -948,7 +946,7 @@ defmodule PlausibleWeb.Api.StatsController do |> Query.remove_goal() res_without_goal = - Stats.breakdown(site, query_without_goal, filter_name, ["visitors"], pagination) + Stats.breakdown(site, query_without_goal, filter_name, [:visitors], pagination) list |> add_cr(res_without_goal, key_name) diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index 9352a9e1b6a1..da1d92a4dbdc 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -50,7 +50,7 @@ defmodule PlausibleWeb.StatsController do site = conn.assigns[:site] query = Query.from(site.timezone, params) |> Filters.add_prefix() - metrics = ["visitors", "pageviews", "bounce_rate", "visit_duration"] + metrics = [:visitors, "pageviews", "bounce_rate", "visit_duration"] graph = Plausible.Stats.timeseries(site, query, metrics) headers = ["date" | metrics] From 9e1b46d9c1f89fa77531560980a69495b889a139 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 24 Nov 2021 13:28:19 +0000 Subject: [PATCH 015/148] Convery GA (direct) source to empty string --- lib/plausible/imported/site.ex | 6 ++++++ lib/plausible/imported/sources.ex | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 5ab06d717cec..a81d51793094 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -46,6 +46,12 @@ defmodule Plausible.Imported do }) do {visitors, ""} = Integer.parse(value) + source = + case source do + "(direct)" -> "" + src -> src + end + Sources.new(%{ domain: domain, timestamp: format_timestamp(timestamp), diff --git a/lib/plausible/imported/sources.ex b/lib/plausible/imported/sources.ex index 24edafda6e08..8b3a2b6ddf54 100644 --- a/lib/plausible/imported/sources.ex +++ b/lib/plausible/imported/sources.ex @@ -7,7 +7,7 @@ defmodule Plausible.Imported.Sources do schema "imported_sources" do field :domain, :string field :timestamp, :naive_datetime - field :source, :string + field :source, :string, default: "" field :visitors, :integer end @@ -26,7 +26,6 @@ defmodule Plausible.Imported.Sources do |> validate_required([ :domain, :timestamp, - :source, :visitors ]) end From a2c37fbe37e90cf2681e354468be8c1fc69ad467 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 24 Nov 2021 14:14:13 +0000 Subject: [PATCH 016/148] Import utm campaign and utm medium from GA --- lib/plausible/clickhouse_repo.ex | 4 ++ lib/plausible/google/api.ex | 14 +++++- lib/plausible/imported/site.ex | 44 +++++++++++++++---- lib/plausible/imported/utm_campaigns.ex | 32 ++++++++++++++ lib/plausible/imported/utm_mediums.ex | 32 ++++++++++++++ ...1124135248_create_imported_utm_mediums.exs | 12 +++++ ...24135252_create_imported_utm_campaigns.exs | 12 +++++ 7 files changed, 140 insertions(+), 10 deletions(-) create mode 100644 lib/plausible/imported/utm_campaigns.ex create mode 100644 lib/plausible/imported/utm_mediums.ex create mode 100644 priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs create mode 100644 priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex index 58ab212edc6d..db598a6dcaf1 100644 --- a/lib/plausible/clickhouse_repo.ex +++ b/lib/plausible/clickhouse_repo.ex @@ -21,7 +21,11 @@ defmodule Plausible.ClickhouseRepo do def clear_imported_stats_for(domain) do visitors_sql = "ALTER TABLE imported_visitors DELETE WHERE domain = ?" sources_sql = "ALTER TABLE imported_sources DELETE WHERE domain = ?" + utm_mediums_sql = "ALTER TABLE imported_utm_mediums DELETE WHERE domain = ?" + utm_campaigns_sql = "ALTER TABLE imported_utm_campaigns DELETE WHERE domain = ?" Ecto.Adapters.SQL.query!(__MODULE__, visitors_sql, [domain]) Ecto.Adapters.SQL.query!(__MODULE__, sources_sql, [domain]) + Ecto.Adapters.SQL.query!(__MODULE__, utm_mediums_sql, [domain]) + Ecto.Adapters.SQL.query!(__MODULE__, utm_campaigns_sql, [domain]) end end diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 35b5e806d9ee..3819c02c9815 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -192,7 +192,17 @@ defmodule Plausible.Google.Api do { ["ga:date", "ga:source"], ["ga:users"] - } + }, + # UTM Mediums + { + ["ga:date", "ga:medium"], + ["ga:users"] + }, + # UTM Campaigns + { + ["ga:date", "ga:campaign"], + ["ga:users"] + }, ] response = fetch_analytic_reports(request, request_data) @@ -200,7 +210,7 @@ defmodule Plausible.Google.Api do case response do {:ok, data} -> maybe_error = - ["visitors", "sources"] + ["visitors", "sources", "utm_mediums", "utm_campaigns"] |> Enum.with_index() |> Enum.map(fn {metric, index} -> Task.async(fn -> diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index a81d51793094..b80e97598392 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -1,5 +1,5 @@ defmodule Plausible.Imported do - alias Plausible.Imported.{Visitors, Sources} + alias Plausible.Imported def forget(site) do Plausible.ClickhouseRepo.clear_imported_stats_for(site.domain) @@ -30,7 +30,7 @@ defmodule Plausible.Imported do |> Enum.map(&Integer.parse/1) |> Enum.map(&elem(&1, 0)) - Visitors.new(%{ + Imported.Visitors.new(%{ domain: domain, timestamp: format_timestamp(timestamp), visitors: visitors, @@ -46,13 +46,9 @@ defmodule Plausible.Imported do }) do {visitors, ""} = Integer.parse(value) - source = - case source do - "(direct)" -> "" - src -> src - end + source = if source == "(direct)", do: "", else: source - Sources.new(%{ + Imported.Sources.new(%{ domain: domain, timestamp: format_timestamp(timestamp), source: source, @@ -60,6 +56,38 @@ defmodule Plausible.Imported do }) end + defp new_from_google_analytics(domain, "utm_mediums", %{ + "dimensions" => [timestamp, medium], + "metrics" => [%{"values" => [value]}] + }) do + {visitors, ""} = Integer.parse(value) + + medium = if medium == "(none)", do: "", else: medium + + Imported.UtmMediums.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + medium: medium, + visitors: visitors + }) + end + + defp new_from_google_analytics(domain, "utm_campaigns", %{ + "dimensions" => [timestamp, campaign], + "metrics" => [%{"values" => [value]}] + }) do + {visitors, ""} = Integer.parse(value) + + campaign = if campaign == "(not set)", do: "", else: campaign + + Imported.UtmCampaigns.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + campaign: campaign, + visitors: visitors + }) + end + defp format_timestamp(timestamp) do {year, monthday} = String.split_at(timestamp, 4) {month, day} = String.split_at(monthday, 2) diff --git a/lib/plausible/imported/utm_campaigns.ex b/lib/plausible/imported/utm_campaigns.ex new file mode 100644 index 000000000000..1efbf57583af --- /dev/null +++ b/lib/plausible/imported/utm_campaigns.ex @@ -0,0 +1,32 @@ +defmodule Plausible.Imported.UtmCampaigns do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_utm_campaigns" do + field :domain, :string + field :timestamp, :naive_datetime + field :utm_campaign, :string, default: "" + field :visitors, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :utm_campaign, + :visitors + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :visitors + ]) + end +end diff --git a/lib/plausible/imported/utm_mediums.ex b/lib/plausible/imported/utm_mediums.ex new file mode 100644 index 000000000000..be2cedf600aa --- /dev/null +++ b/lib/plausible/imported/utm_mediums.ex @@ -0,0 +1,32 @@ +defmodule Plausible.Imported.UtmMediums do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_utm_mediums" do + field :domain, :string + field :timestamp, :naive_datetime + field :utm_medium, :string, default: "" + field :visitors, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :utm_medium, + :visitors + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :visitors + ]) + end +end diff --git a/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs b/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs new file mode 100644 index 000000000000..ac77bd54078d --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs @@ -0,0 +1,12 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmMediums do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_utm_mediums, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :utm_medium, :string + add :visitors, :UInt64 + end + end +end diff --git a/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs b/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs new file mode 100644 index 000000000000..e5dd1c7aa651 --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs @@ -0,0 +1,12 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmCampaigns do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_utm_campaigns, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :utm_campaign, :string + add :visitors, :UInt64 + end + end +end From be412a385a814bcf16c0cfdc93da1e8c23193dbb Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 24 Nov 2021 14:14:47 +0000 Subject: [PATCH 017/148] format --- lib/plausible/google/api.ex | 2 +- .../controllers/api/external_stats_controller.ex | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 3819c02c9815..d926f341ac49 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -202,7 +202,7 @@ defmodule Plausible.Google.Api do { ["ga:date", "ga:campaign"], ["ga:users"] - }, + } ] response = fetch_analytic_reports(request, request_data) diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index b9127e169556..ecd77830b247 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -133,12 +133,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController do "The metric `#{invalid_metric}` is not recognized. Find valid metrics from the documentation: https://plausible.io/docs/stats-api#get-apiv1statsbreakdown"} end else - metrics = metrics |> Enum.map(fn item -> - case item do - "visitors" -> :visitors - metric -> metric - end - end) + metrics = + metrics + |> Enum.map(fn item -> + case item do + "visitors" -> :visitors + metric -> metric + end + end) + {:ok, metrics} end end From afe3535ce46eefc6a2a5d250b03cc662f3173891 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 11:20:30 +0000 Subject: [PATCH 018/148] Import all data types from GA into new tables --- lib/plausible/google/api.ex | 49 ++++++++- lib/plausible/imported/browsers.ex | 34 ++++++ lib/plausible/imported/countries.ex | 32 ++++++ lib/plausible/imported/devices.ex | 32 ++++++ lib/plausible/imported/entry_pages.ex | 32 ++++++ lib/plausible/imported/exit_pages.ex | 32 ++++++ lib/plausible/imported/operation_systems.ex | 34 ++++++ lib/plausible/imported/pages.ex | 32 ++++++ lib/plausible/imported/site.ex | 100 ++++++++++++++++++ .../20211129111618_create_imported_pages.exs | 12 +++ ...1129111622_create_imported_entry_pages.exs | 12 +++ ...11129111630_create_imported_exit_pages.exs | 12 +++ ...211129111639_create_imported_countries.exs | 12 +++ ...20211129111644_create_imported_devices.exs | 12 +++ ...0211129111648_create_imported_browsers.exs | 13 +++ ...1653_create_imported_operating_systems.exs | 13 +++ 16 files changed, 462 insertions(+), 1 deletion(-) create mode 100644 lib/plausible/imported/browsers.ex create mode 100644 lib/plausible/imported/countries.ex create mode 100644 lib/plausible/imported/devices.ex create mode 100644 lib/plausible/imported/entry_pages.ex create mode 100644 lib/plausible/imported/exit_pages.ex create mode 100644 lib/plausible/imported/operation_systems.ex create mode 100644 lib/plausible/imported/pages.ex create mode 100644 priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs create mode 100644 priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs create mode 100644 priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs create mode 100644 priv/clickhouse_repo/migrations/20211129111639_create_imported_countries.exs create mode 100644 priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs create mode 100644 priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs create mode 100644 priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index d926f341ac49..342da1eb903b 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -202,6 +202,41 @@ defmodule Plausible.Google.Api do { ["ga:date", "ga:campaign"], ["ga:users"] + }, + # Pages + { + ["ga:date", "ga:pagePath"], + ["ga:users"] + }, + # Entry pages + { + ["ga:date", "ga:landingPagePath"], + ["ga:users"] + }, + # Exit pages + { + ["ga:date", "ga:exitPagePath"], + ["ga:users"] + }, + # Country + { + ["ga:date", "ga:countryIsoCode"], + ["ga:users"] + }, + # Device + { + ["ga:date", "ga:deviceCategory"], + ["ga:users"] + }, + # Browser + { + ["ga:date", "ga:browser", "ga:browserVersion"], + ["ga:users"] + }, + # OS + { + ["ga:date", "ga:operatingSystem", "ga:operatingSystemVersion"], + ["ga:users"] } ] @@ -210,7 +245,19 @@ defmodule Plausible.Google.Api do case response do {:ok, data} -> maybe_error = - ["visitors", "sources", "utm_mediums", "utm_campaigns"] + [ + "visitors", + "sources", + "utm_mediums", + "utm_campaigns", + "pages", + "entry_pages", + "exit_pages", + "countries", + "devices", + "browsers", + "operating_systems" + ] |> Enum.with_index() |> Enum.map(fn {metric, index} -> Task.async(fn -> diff --git a/lib/plausible/imported/browsers.ex b/lib/plausible/imported/browsers.ex new file mode 100644 index 000000000000..17ba925d2b0f --- /dev/null +++ b/lib/plausible/imported/browsers.ex @@ -0,0 +1,34 @@ +defmodule Plausible.Imported.Browsers do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_utm_mediums" do + field :domain, :string + field :timestamp, :naive_datetime + field :browser, :string + field :version, :string + field :visitors, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :browser, + :versions, + :visitors + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :visitors + ]) + end +end diff --git a/lib/plausible/imported/countries.ex b/lib/plausible/imported/countries.ex new file mode 100644 index 000000000000..dc9a4e9f28a7 --- /dev/null +++ b/lib/plausible/imported/countries.ex @@ -0,0 +1,32 @@ +defmodule Plausible.Imported.Countries do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_countries" do + field :domain, :string + field :timestamp, :naive_datetime + field :country, :string + field :visitors, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :country, + :visitors + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :visitors + ]) + end +end diff --git a/lib/plausible/imported/devices.ex b/lib/plausible/imported/devices.ex new file mode 100644 index 000000000000..d3e971818d02 --- /dev/null +++ b/lib/plausible/imported/devices.ex @@ -0,0 +1,32 @@ +defmodule Plausible.Imported.Devices do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_devices" do + field :domain, :string + field :timestamp, :naive_datetime + field :device, :string + field :visitors, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :device, + :visitors + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :visitors + ]) + end +end diff --git a/lib/plausible/imported/entry_pages.ex b/lib/plausible/imported/entry_pages.ex new file mode 100644 index 000000000000..6be7489195f4 --- /dev/null +++ b/lib/plausible/imported/entry_pages.ex @@ -0,0 +1,32 @@ +defmodule Plausible.Imported.EntryPages do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_entry_pages" do + field :domain, :string + field :timestamp, :naive_datetime + field :entry_page, :string + field :visitors, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :entry_page, + :visitors + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :visitors + ]) + end +end diff --git a/lib/plausible/imported/exit_pages.ex b/lib/plausible/imported/exit_pages.ex new file mode 100644 index 000000000000..0351f698ace2 --- /dev/null +++ b/lib/plausible/imported/exit_pages.ex @@ -0,0 +1,32 @@ +defmodule Plausible.Imported.ExitPages do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_exit_pages" do + field :domain, :string + field :timestamp, :naive_datetime + field :exit_page, :string + field :visitors, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :exit_page, + :visitors + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :visitors + ]) + end +end diff --git a/lib/plausible/imported/operation_systems.ex b/lib/plausible/imported/operation_systems.ex new file mode 100644 index 000000000000..7ef5a7a4d789 --- /dev/null +++ b/lib/plausible/imported/operation_systems.ex @@ -0,0 +1,34 @@ +defmodule Plausible.Imported.OperatingSystems do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_operating_systems" do + field :domain, :string + field :timestamp, :naive_datetime + field :os, :string + field :version, :string + field :visitors, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :os, + :version, + :visitors + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :visitors + ]) + end +end diff --git a/lib/plausible/imported/pages.ex b/lib/plausible/imported/pages.ex new file mode 100644 index 000000000000..078e40d80007 --- /dev/null +++ b/lib/plausible/imported/pages.ex @@ -0,0 +1,32 @@ +defmodule Plausible.Imported.Pages do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_pages" do + field :domain, :string + field :timestamp, :naive_datetime + field :page, :string + field :visitors, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :page, + :visitors + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :visitors + ]) + end +end diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index b80e97598392..3159819f4c67 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -88,6 +88,106 @@ defmodule Plausible.Imported do }) end + defp new_from_google_analytics(domain, "pages", %{ + "dimensions" => [timestamp, page], + "metrics" => [%{"values" => [value]}] + }) do + {visitors, ""} = Integer.parse(value) + + Imported.Pages.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + page: page, + visitors: visitors + }) + end + + defp new_from_google_analytics(domain, "entry_pages", %{ + "dimensions" => [timestamp, entry_page], + "metrics" => [%{"values" => [value]}] + }) do + {visitors, ""} = Integer.parse(value) + + Imported.EntryPages.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + entry_page: entry_page, + visitors: visitors + }) + end + + defp new_from_google_analytics(domain, "exit_pages", %{ + "dimensions" => [timestamp, exit_page], + "metrics" => [%{"values" => [value]}] + }) do + {visitors, ""} = Integer.parse(value) + + Imported.ExitPages.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + exit_page: exit_page, + visitors: visitors + }) + end + + defp new_from_google_analytics(domain, "countries", %{ + "dimensions" => [timestamp, country], + "metrics" => [%{"values" => [value]}] + }) do + {visitors, ""} = Integer.parse(value) + + Imported.Countries.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + country: country, + visitors: visitors + }) + end + + defp new_from_google_analytics(domain, "devices", %{ + "dimensions" => [timestamp, device], + "metrics" => [%{"values" => [value]}] + }) do + {visitors, ""} = Integer.parse(value) + + Imported.Devices.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + device: device, + visitors: visitors + }) + end + + defp new_from_google_analytics(domain, "browsers", %{ + "dimensions" => [timestamp, browser, version], + "metrics" => [%{"values" => [value]}] + }) do + {visitors, ""} = Integer.parse(value) + + Imported.Browsers.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + browser: browser, + version: version, + visitors: visitors + }) + end + + defp new_from_google_analytics(domain, "operating_systems", %{ + "dimensions" => [timestamp, os, version], + "metrics" => [%{"values" => [value]}] + }) do + {visitors, ""} = Integer.parse(value) + + Imported.OperatingSystems.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + os: os, + version: version, + visitors: visitors + }) + end + defp format_timestamp(timestamp) do {year, monthday} = String.split_at(timestamp, 4) {month, day} = String.split_at(monthday, 2) diff --git a/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs b/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs new file mode 100644 index 000000000000..e39a970e6fdf --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs @@ -0,0 +1,12 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedPages do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_pages, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :page, :string + add :visitors, :UInt64 + end + end +end diff --git a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs new file mode 100644 index 000000000000..e7d81edb8f14 --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs @@ -0,0 +1,12 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedEntryPages do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_entry_pages, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :entry_page, :string + add :visitors, :UInt64 + end + end +end diff --git a/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs b/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs new file mode 100644 index 000000000000..42226cdeb06c --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs @@ -0,0 +1,12 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedExitPages do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_exit_pages, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :exit_page, :string + add :visitors, :UInt64 + end + end +end diff --git a/priv/clickhouse_repo/migrations/20211129111639_create_imported_countries.exs b/priv/clickhouse_repo/migrations/20211129111639_create_imported_countries.exs new file mode 100644 index 000000000000..17b2e3d59178 --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211129111639_create_imported_countries.exs @@ -0,0 +1,12 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedCountries do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_countries, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :country, :string + add :visitors, :UInt64 + end + end +end diff --git a/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs b/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs new file mode 100644 index 000000000000..55457f396534 --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs @@ -0,0 +1,12 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedDevices do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_devices, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :device, :string + add :visitors, :UInt64 + end + end +end diff --git a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs new file mode 100644 index 000000000000..e012f67d2d13 --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs @@ -0,0 +1,13 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedBrowsers do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_browsers, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :browser, :string + add :version, :string + add :visitors, :UInt64 + end + end +end diff --git a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs new file mode 100644 index 000000000000..1aa73ee1b51d --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs @@ -0,0 +1,13 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedOperatingSystems do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_operating_systems, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :operating_system, :string + add :version, :string + add :visitors, :UInt64 + end + end +end From 9ea2cd374efdb78633d73a90ce04739cc2fa7a61 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 11:28:43 +0000 Subject: [PATCH 019/148] Handle large amounts of more data more safely --- lib/plausible/google/api.ex | 11 +++++++++-- lib/plausible/imported/site.ex | 6 ++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 342da1eb903b..ac1be9b18457 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -265,7 +265,7 @@ defmodule Plausible.Google.Api do |> Imported.from_google_analytics(site.domain, metric) end) end) - |> Enum.map(&Task.await/1) + |> Enum.map(&Task.await(&1, 120_000)) |> Keyword.get(:error) case maybe_error do @@ -299,7 +299,14 @@ defmodule Plausible.Google.Api do dimensions: Enum.map(dimensions, &%{name: &1, histogramBuckets: []}), metrics: Enum.map(metrics, &%{expression: &1}), hideTotals: true, - hideValueRanges: true + hideValueRanges: true, + orderBys: [ + %{ + fieldName: "ga:date", + sortOrder: "DESCENDING" + } + ], + pageSize: 100_000 } end) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 3159819f4c67..6cf45aea79f5 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -8,8 +8,10 @@ defmodule Plausible.Imported do def from_google_analytics(data, domain, metric) do maybe_error = data - |> Enum.map(&new_from_google_analytics(domain, metric, &1)) - |> Enum.map(&Plausible.ClickhouseRepo.insert(&1, on_conflict: :replace_all)) + |> Enum.map(fn row -> + new_from_google_analytics(domain, metric, row) + |> Plausible.ClickhouseRepo.insert(on_conflict: :replace_all) + end) |> Keyword.get(:error) case maybe_error do From f5b8f958bff9e112b115e396ce0fdbdd02d8a51e Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 11:51:45 +0000 Subject: [PATCH 020/148] Fix some mistakes in tables --- lib/plausible/imported/browsers.ex | 4 ++-- lib/plausible/imported/operation_systems.ex | 4 ++-- lib/plausible/imported/site.ex | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/plausible/imported/browsers.ex b/lib/plausible/imported/browsers.ex index 17ba925d2b0f..b91750050409 100644 --- a/lib/plausible/imported/browsers.ex +++ b/lib/plausible/imported/browsers.ex @@ -4,7 +4,7 @@ defmodule Plausible.Imported.Browsers do import Ecto.Changeset @primary_key false - schema "imported_utm_mediums" do + schema "imported_browsers" do field :domain, :string field :timestamp, :naive_datetime field :browser, :string @@ -20,7 +20,7 @@ defmodule Plausible.Imported.Browsers do :domain, :timestamp, :browser, - :versions, + :version, :visitors ], empty_values: [nil, ""] diff --git a/lib/plausible/imported/operation_systems.ex b/lib/plausible/imported/operation_systems.ex index 7ef5a7a4d789..4c61d68344a4 100644 --- a/lib/plausible/imported/operation_systems.ex +++ b/lib/plausible/imported/operation_systems.ex @@ -7,7 +7,7 @@ defmodule Plausible.Imported.OperatingSystems do schema "imported_operating_systems" do field :domain, :string field :timestamp, :naive_datetime - field :os, :string + field :operating_system, :string field :version, :string field :visitors, :integer end @@ -19,7 +19,7 @@ defmodule Plausible.Imported.OperatingSystems do [ :domain, :timestamp, - :os, + :operating_system, :version, :visitors ], diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 6cf45aea79f5..77410931c3db 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -184,7 +184,7 @@ defmodule Plausible.Imported do Imported.OperatingSystems.new(%{ domain: domain, timestamp: format_timestamp(timestamp), - os: os, + operating_system: os, version: version, visitors: visitors }) From dd0462e5c1780614845ef1bf38e36d04e8915fa0 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 11:52:54 +0000 Subject: [PATCH 021/148] Make GA requests in chunks of 5 queries --- lib/plausible/google/api.ex | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index ac1be9b18457..fed2b95be435 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -240,10 +240,19 @@ defmodule Plausible.Google.Api do } ] - response = fetch_analytic_reports(request, request_data) + # batchGet can receive a maximum of 5 requests. + responses = + request_data + |> Enum.chunk_every(5) + |> Enum.map(&fetch_analytic_reports(&1, request)) + + case Keyword.get(responses, :error) do + nil -> + data = + responses + |> Enum.map(fn {:ok, resp} -> resp end) + |> Enum.concat() - case response do - {:ok, data} -> maybe_error = [ "visitors", @@ -279,13 +288,13 @@ defmodule Plausible.Google.Api do {:error, error} end - {:error, error} -> + error -> Sentry.capture_message("Error fetching Google analytics data", extra: error) {:error, error} end end - defp fetch_analytic_reports(request, request_data) do + defp fetch_analytic_reports(request_data, request) do reports = Enum.map(request_data, fn {dimensions, metrics} -> %{ From 2b86326635d35fd8b6f73d1134cc3e7be3c8a7b1 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 12:09:27 +0000 Subject: [PATCH 022/148] Only display imported timeseries when there is no filter --- lib/plausible_web/controllers/api/stats_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index f89082df6c01..4c4485f91136 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -26,7 +26,7 @@ defmodule PlausibleWeb.Api.StatsController do present_index = present_index_for(site, query, labels) imported_plot = - if site.has_imported_stats do + if params["filters"] == "{}" && site.has_imported_stats do Imported.Visitors.timeseries(site, query) end From 8a969ea0f70d77134cc02db8f0f5c9484b73d4ea Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 12:10:10 +0000 Subject: [PATCH 023/148] Correctly show last 30 minutes timeseries when 'realtime' --- lib/plausible_web/controllers/api/stats_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 4c4485f91136..733484dd149b 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -27,7 +27,7 @@ defmodule PlausibleWeb.Api.StatsController do imported_plot = if params["filters"] == "{}" && site.has_imported_stats do - Imported.Visitors.timeseries(site, query) + Imported.Visitors.timeseries(site, timeseries_query) end json(conn, %{ From 5b09c8cb071998fa01e630f473f44b19036dc0e8 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 13:01:40 +0000 Subject: [PATCH 024/148] Add with_imported key to Query struct --- lib/plausible/stats/base.ex | 1 + lib/plausible/stats/query.ex | 22 ++++++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index ad9fc81d5642..d3727f3dc923 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -356,6 +356,7 @@ defmodule Plausible.Stats.Base do end def merge_imported(q, %Plausible.Site{has_imported_stats: nil}, _, _, _), do: q + def merge_imported(q, _, %Query{with_imported: false}, _, _), do: q def merge_imported(q, site, query, "visit:source", {limit, page}) do offset = (page - 1) * limit diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 1342fd3970dd..f363c6588043 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -4,6 +4,7 @@ defmodule Plausible.Stats.Query do period: nil, filters: %{}, sample_threshold: 20_000_000 + with_imported: false @default_sample_threshold 20_000_000 @@ -57,7 +58,8 @@ defmodule Plausible.Stats.Query do date_range: Date.range(date, date), interval: "hour", filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), + with_imported: determine_also_imported(params) } end @@ -70,7 +72,8 @@ defmodule Plausible.Stats.Query do date_range: Date.range(start_date, end_date), interval: "date", filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), + with_imported: determine_also_imported(params) } end @@ -83,7 +86,8 @@ defmodule Plausible.Stats.Query do date_range: Date.range(start_date, end_date), interval: "date", filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), + with_imported: determine_also_imported(params) } end @@ -98,7 +102,8 @@ defmodule Plausible.Stats.Query do date_range: Date.range(start_date, end_date), interval: "date", filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), + with_imported: determine_also_imported(params) } end @@ -116,7 +121,8 @@ defmodule Plausible.Stats.Query do date_range: Date.range(start_date, end_date), interval: Map.get(params, "interval", "month"), filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), + with_imported: determine_also_imported(params) } end @@ -134,7 +140,8 @@ defmodule Plausible.Stats.Query do date_range: Date.range(start_date, end_date), interval: Map.get(params, "interval", "month"), filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), + with_imported: determine_also_imported(params) } end @@ -218,6 +225,9 @@ defmodule Plausible.Stats.Query do defp parse_filters(%{"filters" => filters}) when is_map(filters), do: filters defp parse_filters(_), do: %{} + defp determine_also_imported(%{"filters" => "{}"}), do: true + defp determine_also_imported(_), do: false + defp parse_filter_expression(str) do filters = String.split(str, ";") From e2c185a752a5a885703025d02f7957299a9ba38d Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 13:02:06 +0000 Subject: [PATCH 025/148] Account for injected :is_not filter on sources from dashboard --- lib/plausible/stats/base.ex | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index d3727f3dc923..5ceba6136833 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -363,13 +363,29 @@ defmodule Plausible.Stats.Base do {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) imported_q = - from( - i in "imported_sources", - group_by: i.source, - where: i.domain == ^site.domain, - where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, - select: %{source: i.source, visitors: sum(i.visitors)} - ) + case query.filters["visitor:source"] do + {:is_not, is_not} -> + # TODO: make this no_ref is actually being excluded. + # This is needed as the Top Sources panel expects these filtered. + is_not = if is_not == @no_ref, do: "", else: is_not + + from( + i in "imported_sources", + group_by: i.source, + where: i.domain == ^site.domain and i.source != ^is_not, + where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, + select: %{source: i.source, visitors: sum(i.visitors)} + ) + + _ -> + from( + i in "imported_sources", + group_by: i.source, + where: i.domain == ^site.domain, + where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, + select: %{source: i.source, visitors: sum(i.visitors)} + ) + end from(s in Ecto.Query.subquery(q), full_join: i in subquery(imported_q), From 0a62a4d6ea60f3988e43ec6750248bed9d58a57b Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 17:19:27 +0000 Subject: [PATCH 026/148] Also add tentative imported_utm_sources table This needs a bit more work on the google import side, as GA do not report sources and utm sources as distinct things. --- lib/plausible/imported/site.ex | 2 ++ lib/plausible/imported/utm_sources.ex | 32 +++++++++++++++++++ ...1129164103_create_imported_utm_sources.exs | 12 +++++++ 3 files changed, 46 insertions(+) create mode 100644 lib/plausible/imported/utm_sources.ex create mode 100644 priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 77410931c3db..29f916cd38da 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -58,6 +58,8 @@ defmodule Plausible.Imported do }) end + # TODO: utm_sources. Google reports sources and utm_sources unified. + defp new_from_google_analytics(domain, "utm_mediums", %{ "dimensions" => [timestamp, medium], "metrics" => [%{"values" => [value]}] diff --git a/lib/plausible/imported/utm_sources.ex b/lib/plausible/imported/utm_sources.ex new file mode 100644 index 000000000000..226aad2322d4 --- /dev/null +++ b/lib/plausible/imported/utm_sources.ex @@ -0,0 +1,32 @@ +defmodule Plausible.Imported.UtmSources do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_utm_sources" do + field :domain, :string + field :timestamp, :naive_datetime + field :utm_source, :string, default: "" + field :visitors, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :utm_source, + :visitors + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :visitors + ]) + end +end diff --git a/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs b/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs new file mode 100644 index 000000000000..b029bf0c07a6 --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs @@ -0,0 +1,12 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmSources do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_utm_sources, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :utm_source, :string + add :visitors, :UInt64 + end + end +end From 5a0a81dc4504f0681eaf4dcb61cb2b442ac07581 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 17:20:41 +0000 Subject: [PATCH 027/148] Return imported data to dashboard for rest of Sources panel This extends the merge_imported function definition for sources to utm_sources, utm_mediums and utm_campaigns too. This appears to be working on the DB side but something is incomplete on the client side. --- lib/plausible/stats/base.ex | 111 +++++++++++++----- lib/plausible/stats/breakdown.ex | 6 +- .../controllers/api/stats_controller.ex | 12 +- 3 files changed, 92 insertions(+), 37 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 5ceba6136833..0238f43a49be 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -1,6 +1,7 @@ defmodule Plausible.Stats.Base do use Plausible.ClickhouseRepo alias Plausible.Stats.{Query, Filters} + import Ecto.Query @no_ref "Direct / None" @@ -358,46 +359,94 @@ defmodule Plausible.Stats.Base do def merge_imported(q, %Plausible.Site{has_imported_stats: nil}, _, _, _), do: q def merge_imported(q, _, %Query{with_imported: false}, _, _), do: q - def merge_imported(q, site, query, "visit:source", {limit, page}) do + def merge_imported(q, site, query, property, {limit, page}) + when property in [ + "visit:source", + "visit:utm_medium", + "visit:utm_source", + "visit:utm_campaign" + ] do offset = (page - 1) * limit {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) + dim = String.trim_leading(property, "visit:") + table = "imported_#{dim}s" + dim = String.to_atom(dim) + imported_q = - case query.filters["visitor:source"] do - {:is_not, is_not} -> - # TODO: make this no_ref is actually being excluded. - # This is needed as the Top Sources panel expects these filtered. - is_not = if is_not == @no_ref, do: "", else: is_not + from( + i in table, + group_by: field(i, ^dim), + where: i.domain == ^site.domain, + where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, + select: %{visitors: sum(i.visitors)} + ) - from( - i in "imported_sources", - group_by: i.source, - where: i.domain == ^site.domain and i.source != ^is_not, - where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, - select: %{source: i.source, visitors: sum(i.visitors)} - ) + imported_q = + case query.filters[property] do + {:is_not, value} -> + # TODO: make this no_ref is actually being excluded. + # This is needed as the sources panel expects these filtered. + value = if value == @no_ref, do: "", else: value + where(imported_q, [i], field(i, ^dim) != ^value) _ -> - from( - i in "imported_sources", - group_by: i.source, - where: i.domain == ^site.domain, - where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, - select: %{source: i.source, visitors: sum(i.visitors)} - ) + imported_q end - from(s in Ecto.Query.subquery(q), - full_join: i in subquery(imported_q), - on: s.source == i.source, - select: %{ - source: fragment("if(empty(?), ?, ?)", s.source, i.source, s.source), - visitors: fragment("toUInt64(?) + toUInt64(?)", s.visitors, i.visitors) - }, - order_by: [desc: fragment("toUInt64(?) + toUInt64(?)", s.visitors, i.visitors)], - limit: ^limit, - offset: ^offset - ) + imported_q = + case dim do + :source -> + imported_q |> select_merge([i], %{source: i.source}) + + :utm_medium -> + imported_q |> select_merge([i], %{utm_medium: i.utm_medium}) + + :utm_source -> + imported_q |> select_merge([i], %{utm_source: i.utm_source}) + + :utm_campaign -> + imported_q |> select_merge([i], %{utm_campaign: i.utm_campaign}) + end + + q = + from(s in Ecto.Query.subquery(q), + full_join: i in subquery(imported_q), + on: field(s, ^dim) == field(i, ^dim), + select: %{ + :visitors => fragment("toUInt64(?) + toUInt64(?)", s.visitors, i.visitors) + }, + order_by: [desc: fragment("toUInt64(?) + toUInt64(?)", s.visitors, i.visitors)], + limit: ^limit, + offset: ^offset + ) + + case dim do + :source -> + q + |> select_merge([i, s], %{ + source: fragment("if(empty(?), ?, ?)", s.source, i.source, s.source) + }) + + :utm_medium -> + q + |> select_merge([i, s], %{ + utm_medium: fragment("if(empty(?), ?, ?)", s.utm_medium, i.utm_medium, s.utm_medium) + }) + + :utm_source -> + q + |> select_merge([i, s], %{ + utm_source: fragment("if(empty(?), ?, ?)", s.utm_source, i.utm_source, s.utm_source) + }) + + :utm_campaign -> + q + |> select_merge([i, s], %{ + utm_campaign: + fragment("if(empty(?), ?, ?)", s.utm_campaign, i.utm_campaign, s.utm_campaign) + }) + end end def merge_imported(q, _, _, _, _), do: q diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index b6b52e97638f..220e293dd547 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -390,7 +390,7 @@ defmodule Plausible.Stats.Breakdown do s in q, group_by: s.utm_medium, select_merge: %{ - "utm_medium" => fragment("if(empty(?), ?, ?)", s.utm_medium, @no_ref, s.utm_medium) + utm_medium: fragment("if(empty(?), ?, ?)", s.utm_medium, @no_ref, s.utm_medium) } ) end @@ -400,7 +400,7 @@ defmodule Plausible.Stats.Breakdown do s in q, group_by: s.utm_source, select_merge: %{ - "utm_source" => fragment("if(empty(?), ?, ?)", s.utm_source, @no_ref, s.utm_source) + utm_source: fragment("if(empty(?), ?, ?)", s.utm_source, @no_ref, s.utm_source) } ) end @@ -410,7 +410,7 @@ defmodule Plausible.Stats.Breakdown do s in q, group_by: s.utm_campaign, select_merge: %{ - "utm_campaign" => fragment("if(empty(?), ?, ?)", s.utm_campaign, @no_ref, s.utm_campaign) + utm_campaign: fragment("if(empty(?), ?, ?)", s.utm_campaign, @no_ref, s.utm_campaign) } ) end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 733484dd149b..fda7aa69ba1e 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -245,7 +245,9 @@ defmodule PlausibleWeb.Api.StatsController do |> maybe_hide_noref("visit:utm_medium", params) pagination = parse_pagination(params) - metrics = [:visitors, "bounce_rate", "visit_duration"] + + metrics = + if params["detailed"], do: [:visitors, "bounce_rate", "visit_duration"], else: [:visitors] res = Stats.breakdown(site, query, "visit:utm_medium", metrics, pagination) @@ -274,7 +276,9 @@ defmodule PlausibleWeb.Api.StatsController do |> maybe_hide_noref("visit:utm_campaign", params) pagination = parse_pagination(params) - metrics = [:visitors, "bounce_rate", "visit_duration"] + + metrics = + if params["detailed"], do: [:visitors, "bounce_rate", "visit_duration"], else: [:visitors] res = Stats.breakdown(site, query, "visit:utm_campaign", metrics, pagination) @@ -361,7 +365,9 @@ defmodule PlausibleWeb.Api.StatsController do |> maybe_hide_noref("visit:utm_source", params) pagination = parse_pagination(params) - metrics = [:visitors, "bounce_rate", "visit_duration"] + + metrics = + if params["detailed"], do: [:visitors, "bounce_rate", "visit_duration"], else: [:visitors] res = Stats.breakdown(site, query, "visit:utm_source", metrics, pagination) From fa65a50c0f8531c7dd63816fecfc4a1e12cb4a73 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 18:13:11 +0000 Subject: [PATCH 028/148] Clear imported stats from all tables when requested --- lib/plausible/clickhouse_repo.ex | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex index db598a6dcaf1..ca42906cfcfd 100644 --- a/lib/plausible/clickhouse_repo.ex +++ b/lib/plausible/clickhouse_repo.ex @@ -19,13 +19,23 @@ defmodule Plausible.ClickhouseRepo do end def clear_imported_stats_for(domain) do - visitors_sql = "ALTER TABLE imported_visitors DELETE WHERE domain = ?" - sources_sql = "ALTER TABLE imported_sources DELETE WHERE domain = ?" - utm_mediums_sql = "ALTER TABLE imported_utm_mediums DELETE WHERE domain = ?" - utm_campaigns_sql = "ALTER TABLE imported_utm_campaigns DELETE WHERE domain = ?" - Ecto.Adapters.SQL.query!(__MODULE__, visitors_sql, [domain]) - Ecto.Adapters.SQL.query!(__MODULE__, sources_sql, [domain]) - Ecto.Adapters.SQL.query!(__MODULE__, utm_mediums_sql, [domain]) - Ecto.Adapters.SQL.query!(__MODULE__, utm_campaigns_sql, [domain]) + [ + "imported_visitors", + "imported_sources", + "imported_utm_mediums", + "imported_utm_sources", + "imported_utm_campaigns", + "imported_pages", + "imported_entry_pages", + "imported_exit_pages", + "imported_countries", + "imported_devices", + "imported_browsers", + "imported_operating_systems" + ] + |> Enum.map(fn table -> + sql = "ALTER TABLE #{table} DELETE WHERE domain = ?" + Ecto.Adapters.SQL.query!(__MODULE__, sql, [domain]) + end) end end From a8b0b100ac98b29bece8710fafbe259fd9d13d79 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 18:34:27 +0000 Subject: [PATCH 029/148] Merge entry pages and exit pages from imported data into unfiltered dashboard view This requires converting the `"visits"` and `"visit_duration"` metrics to atoms so that they can be used in ecto subqueries. --- lib/plausible/stats/aggregate.ex | 2 +- lib/plausible/stats/base.ex | 30 +++++++++++++++---- lib/plausible/stats/breakdown.ex | 8 ++--- lib/plausible/stats/timeseries.ex | 6 ++-- .../api/external_stats_controller.ex | 2 +- .../controllers/api/stats_controller.ex | 30 +++++++++---------- .../controllers/stats_controller.ex | 2 +- 7 files changed, 50 insertions(+), 30 deletions(-) diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 2b1c6deebf22..b695388afd61 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -4,7 +4,7 @@ defmodule Plausible.Stats.Aggregate do import Plausible.Stats.Base @event_metrics [:visitors, "pageviews", "events", "sample_percent"] - @session_metrics ["visits", "bounce_rate", "visit_duration", "sample_percent"] + @session_metrics [:visits, "bounce_rate", :visit_duration, "sample_percent"] def aggregate(site, query, metrics) do event_metrics = Enum.filter(metrics, &(&1 in @event_metrics)) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 0238f43a49be..8dff71438225 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -247,10 +247,10 @@ defmodule Plausible.Stats.Base do |> select_session_metrics(rest) end - def select_session_metrics(q, ["visits" | rest]) do + def select_session_metrics(q, [:visits | rest]) do from(s in q, select_merge: %{ - "visits" => fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.session_id) + visits: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.session_id) } ) |> select_session_metrics(rest) @@ -275,10 +275,10 @@ defmodule Plausible.Stats.Base do |> select_session_metrics(rest) end - def select_session_metrics(q, ["visit_duration" | rest]) do + def select_session_metrics(q, [:visit_duration | rest]) do from(s in q, select_merge: %{ - "visit_duration" => + :visit_duration => fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))") } ) @@ -364,7 +364,9 @@ defmodule Plausible.Stats.Base do "visit:source", "visit:utm_medium", "visit:utm_source", - "visit:utm_campaign" + "visit:utm_campaign", + "visit:entry_page", + "visit:exit_page" ] do offset = (page - 1) * limit {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) @@ -407,6 +409,12 @@ defmodule Plausible.Stats.Base do :utm_campaign -> imported_q |> select_merge([i], %{utm_campaign: i.utm_campaign}) + + :entry_page -> + imported_q |> select_merge([i], %{entry_page: i.entry_page}) + + :exit_page -> + imported_q |> select_merge([i], %{exit_page: i.exit_page}) end q = @@ -446,6 +454,18 @@ defmodule Plausible.Stats.Base do utm_campaign: fragment("if(empty(?), ?, ?)", s.utm_campaign, i.utm_campaign, s.utm_campaign) }) + + :entry_page -> + q + |> select_merge([i, s], %{ + entry_page: fragment("if(empty(?), ?, ?)", s.entry_page, i.entry_page, s.entry_page) + }) + + :exit_page -> + q + |> select_merge([i, s], %{ + exit_page: fragment("if(empty(?), ?, ?)", s.exit_page, i.exit_page, s.exit_page) + }) end end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 220e293dd547..02a0dda18186 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -5,7 +5,7 @@ defmodule Plausible.Stats.Breakdown do @no_ref "Direct / None" @event_metrics [:visitors, "pageviews", "events"] - @session_metrics ["visits", "bounce_rate", "visit_duration"] + @session_metrics [:visits, "bounce_rate", :visit_duration] @event_props ["event:page", "event:page_match", "event:name"] def breakdown(site, query, "event:goal", metrics, pagination) do @@ -119,7 +119,7 @@ defmodule Plausible.Stats.Breakdown do session_result = breakdown_sessions(site, new_query, "visit:entry_page", session_metrics, {limit, 1}) - |> transform_keys(%{"entry_page" => "page"}) + |> transform_keys(%{entry_page: "page"}) zip_results( event_result, @@ -363,7 +363,7 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.entry_page, - select_merge: %{"entry_page" => s.entry_page} + select_merge: %{entry_page: s.entry_page} ) end @@ -371,7 +371,7 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.exit_page, - select_merge: %{"exit_page" => s.exit_page} + select_merge: %{exit_page: s.exit_page} ) end diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 5760ca47d480..c160a2ee899d 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -5,7 +5,7 @@ defmodule Plausible.Stats.Timeseries do use Plausible.Stats.Fragments @event_metrics [:visitors, "pageviews"] - @session_metrics ["visits", "bounce_rate", "visit_duration"] + @session_metrics [:visits, "bounce_rate", :visit_duration] def timeseries(site, query, metrics) do steps = buckets(query) @@ -120,9 +120,9 @@ defmodule Plausible.Stats.Timeseries do case metric do "pageviews" -> Map.merge(row, %{"pageviews" => 0}) :visitors -> Map.merge(row, %{visitors: 0}) - "visits" -> Map.merge(row, %{"visits" => 0}) + :visits -> Map.merge(row, %{visits: 0}) "bounce_rate" -> Map.merge(row, %{"bounce_rate" => nil}) - "visit_duration" -> Map.merge(row, %{"visit_duration" => nil}) + :visit_duration -> Map.merge(row, %{:visit_duration => nil}) end end) end diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index ecd77830b247..0be2afb70d61 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -101,7 +101,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do defp event_only_property?(_), do: false @event_metrics ["visitors", "pageviews", "events"] - @session_metrics ["visits", "bounce_rate", "visit_duration"] + @session_metrics [:visits, "bounce_rate", :visit_duration] defp parse_metrics(params, property, query) do metrics = Map.get(params, "metrics", "visitors") diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index fda7aa69ba1e..5f62bec496d1 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -155,7 +155,7 @@ defmodule PlausibleWeb.Api.StatsController do if query.filters["event:page"] do [:visitors, "pageviews", "bounce_rate", "time_on_page", "sample_percent"] else - [:visitors, "pageviews", "bounce_rate", "visit_duration", "sample_percent"] + [:visitors, "pageviews", "bounce_rate", :visit_duration, "sample_percent"] end current_results = Stats.aggregate(site, query, metrics) @@ -166,7 +166,7 @@ defmodule PlausibleWeb.Api.StatsController do top_stats_entry(current_results, prev_results, "Unique visitors", :visitors), top_stats_entry(current_results, prev_results, "Total pageviews", "pageviews"), top_stats_entry(current_results, prev_results, "Bounce rate", "bounce_rate"), - top_stats_entry(current_results, prev_results, "Visit duration", "visit_duration"), + top_stats_entry(current_results, prev_results, "Visit duration", :visit_duration), top_stats_entry(current_results, prev_results, "Time on page", "time_on_page") ] |> Enum.filter(& &1) @@ -216,7 +216,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) metrics = - if params["detailed"], do: [:visitors, "bounce_rate", "visit_duration"], else: [:visitors] + if params["detailed"], do: [:visitors, "bounce_rate", :visit_duration], else: [:visitors] res = Stats.breakdown(site, query, "visit:source", metrics, pagination) @@ -247,7 +247,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) metrics = - if params["detailed"], do: [:visitors, "bounce_rate", "visit_duration"], else: [:visitors] + if params["detailed"], do: [:visitors, "bounce_rate", :visit_duration], else: [:visitors] res = Stats.breakdown(site, query, "visit:utm_medium", metrics, pagination) @@ -278,7 +278,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) metrics = - if params["detailed"], do: [:visitors, "bounce_rate", "visit_duration"], else: [:visitors] + if params["detailed"], do: [:visitors, "bounce_rate", :visit_duration], else: [:visitors] res = Stats.breakdown(site, query, "visit:utm_campaign", metrics, pagination) @@ -367,7 +367,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) metrics = - if params["detailed"], do: [:visitors, "bounce_rate", "visit_duration"], else: [:visitors] + if params["detailed"], do: [:visitors, "bounce_rate", :visit_duration], else: [:visitors] res = Stats.breakdown(site, query, "visit:utm_source", metrics, pagination) @@ -428,7 +428,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) metrics = - if params["detailed"], do: [:visitors, "bounce_rate", "visit_duration"], else: [:visitors] + if params["detailed"], do: [:visitors, "bounce_rate", :visit_duration], else: [:visitors] referrers = Stats.breakdown(site, query, "visit:referrer", metrics, pagination) @@ -472,15 +472,15 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = Query.from(site.timezone, params) |> Filters.add_prefix() pagination = parse_pagination(params) - metrics = [:visitors, "visits", "visit_duration"] + metrics = [:visitors, :visits, :visit_duration] entry_pages = Stats.breakdown(site, query, "visit:entry_page", metrics, pagination) - |> maybe_add_cr(site, query, pagination, "entry_page", "visit:entry_page") + |> maybe_add_cr(site, query, pagination, :entry_page, "visit:entry_page") |> transform_keys(%{ - "entry_page" => "name", + :entry_page => "name", :visitors => "unique_entrances", - "visits" => "total_entrances" + :visits => "total_entrances" }) if params["csv"] do @@ -500,15 +500,15 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = Query.from(site.timezone, params) |> Filters.add_prefix() {limit, page} = parse_pagination(params) - metrics = [:visitors, "visits"] + metrics = [:visitors, :visits] exit_pages = Stats.breakdown(site, query, "visit:exit_page", metrics, {limit, page}) - |> maybe_add_cr(site, query, {limit, page}, "exit_page", "visit:exit_page") + |> maybe_add_cr(site, query, {limit, page}, :exit_page, "visit:exit_page") |> transform_keys(%{ - "exit_page" => "name", + :exit_page => "name", :visitors => "unique_exits", - "visits" => "total_exits" + :visits => "total_exits" }) pages = Enum.map(exit_pages, & &1["name"]) diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index da1d92a4dbdc..ef0d85b621df 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -50,7 +50,7 @@ defmodule PlausibleWeb.StatsController do site = conn.assigns[:site] query = Query.from(site.timezone, params) |> Filters.add_prefix() - metrics = [:visitors, "pageviews", "bounce_rate", "visit_duration"] + metrics = [:visitors, "pageviews", "bounce_rate", :visit_duration] graph = Plausible.Stats.timeseries(site, query, metrics) headers = ["date" | metrics] From b8ee7df3e13011b7297a5880bf1a9089db38d6d3 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 18:50:35 +0000 Subject: [PATCH 030/148] Display imported devices, browsers and OSs on dashboard --- lib/plausible/stats/base.ex | 50 +++++++++++++++++-- lib/plausible/stats/breakdown.ex | 6 +-- .../controllers/api/stats_controller.ex | 12 ++--- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 8dff71438225..2a3c9e168703 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -366,14 +366,23 @@ defmodule Plausible.Stats.Base do "visit:utm_source", "visit:utm_campaign", "visit:entry_page", - "visit:exit_page" + "visit:exit_page", + "visit:device", + "visit:browser", + "visit:os" ] do offset = (page - 1) * limit {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) - dim = String.trim_leading(property, "visit:") - table = "imported_#{dim}s" - dim = String.to_atom(dim) + {table, dim} = + case property do + "visit:os" -> + {"imported_operating_systems", :operating_system} + + _ -> + dim = String.trim_leading(property, "visit:") + {"imported_#{dim}s", String.to_atom(dim)} + end imported_q = from( @@ -415,6 +424,15 @@ defmodule Plausible.Stats.Base do :exit_page -> imported_q |> select_merge([i], %{exit_page: i.exit_page}) + + :device -> + imported_q |> select_merge([i], %{device: i.device}) + + :browser -> + imported_q |> select_merge([i], %{browser: i.browser}) + + :operating_system -> + imported_q |> select_merge([i], %{operating_system: i.operating_system}) end q = @@ -466,6 +484,30 @@ defmodule Plausible.Stats.Base do |> select_merge([i, s], %{ exit_page: fragment("if(empty(?), ?, ?)", s.exit_page, i.exit_page, s.exit_page) }) + + :device -> + q + |> select_merge([i, s], %{ + device: fragment("if(empty(?), ?, ?)", s.device, i.device, s.device) + }) + + :browser -> + q + |> select_merge([i, s], %{ + browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser) + }) + + :operating_system -> + q + |> select_merge([i, s], %{ + operating_system: + fragment( + "if(empty(?), ?, ?)", + s.operating_system, + i.operating_system, + s.operating_system + ) + }) end end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 02a0dda18186..0e5e8e90c3cc 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -439,7 +439,7 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.screen_size, - select_merge: %{"device" => s.screen_size} + select_merge: %{device: s.screen_size} ) end @@ -447,7 +447,7 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.operating_system, - select_merge: %{"os" => s.operating_system} + select_merge: %{operating_system: s.operating_system} ) end @@ -463,7 +463,7 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.browser, - select_merge: %{"browser" => s.browser} + select_merge: %{browser: s.browser} ) end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 5f62bec496d1..e87b0eae975a 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -671,8 +671,8 @@ defmodule PlausibleWeb.Api.StatsController do browsers = Stats.breakdown(site, query, "visit:browser", [:visitors], pagination) - |> maybe_add_cr(site, query, pagination, "browser", "visit:browser") - |> transform_keys(%{"browser" => "name"}) + |> maybe_add_cr(site, query, pagination, :browser, "visit:browser") + |> transform_keys(%{browser: "name"}) |> maybe_add_percentages(query) if params["csv"] do @@ -709,8 +709,8 @@ defmodule PlausibleWeb.Api.StatsController do systems = Stats.breakdown(site, query, "visit:os", [:visitors], pagination) - |> maybe_add_cr(site, query, pagination, "os", "visit:os") - |> transform_keys(%{"os" => "name"}) + |> maybe_add_cr(site, query, pagination, :operating_system, "visit:os") + |> transform_keys(%{operating_system: "name"}) |> maybe_add_percentages(query) if params["csv"] do @@ -747,8 +747,8 @@ defmodule PlausibleWeb.Api.StatsController do sizes = Stats.breakdown(site, query, "visit:device", [:visitors], pagination) - |> maybe_add_cr(site, query, pagination, "device", "visit:device") - |> transform_keys(%{"device" => "name"}) + |> maybe_add_cr(site, query, pagination, :device, "visit:device") + |> transform_keys(%{device: "name"}) |> maybe_add_percentages(query) if params["csv"] do From 0e57d58be06f03fb41f249c29a59b5b67e336c69 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 18:53:53 +0000 Subject: [PATCH 031/148] Display imported country data on dashboard --- lib/plausible/stats/base.ex | 15 +++++++++++++++ lib/plausible/stats/breakdown.ex | 2 +- .../controllers/api/stats_controller.ex | 4 ++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 2a3c9e168703..3fbd8c4a26c5 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -367,6 +367,7 @@ defmodule Plausible.Stats.Base do "visit:utm_campaign", "visit:entry_page", "visit:exit_page", + "visit:country", "visit:device", "visit:browser", "visit:os" @@ -376,6 +377,9 @@ defmodule Plausible.Stats.Base do {table, dim} = case property do + "visit:country" -> + {"imported_countries", :country} + "visit:os" -> {"imported_operating_systems", :operating_system} @@ -405,6 +409,7 @@ defmodule Plausible.Stats.Base do imported_q end + # TODO: DRY imported_q = case dim do :source -> @@ -425,6 +430,9 @@ defmodule Plausible.Stats.Base do :exit_page -> imported_q |> select_merge([i], %{exit_page: i.exit_page}) + :country -> + imported_q |> select_merge([i], %{country: i.country}) + :device -> imported_q |> select_merge([i], %{device: i.device}) @@ -447,6 +455,7 @@ defmodule Plausible.Stats.Base do offset: ^offset ) + # TODO: DRY case dim do :source -> q @@ -485,6 +494,12 @@ defmodule Plausible.Stats.Base do exit_page: fragment("if(empty(?), ?, ?)", s.exit_page, i.exit_page, s.exit_page) }) + :country -> + q + |> select_merge([i, s], %{ + country: fragment("if(empty(?), ?, ?)", s.country, i.country, s.country) + }) + :device -> q |> select_merge([i, s], %{ diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 0e5e8e90c3cc..c38ab846b17b 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -337,7 +337,7 @@ defmodule Plausible.Stats.Breakdown do s in q, group_by: s.country_code, where: s.country_code != "\0\0", - select_merge: %{"country" => s.country_code} + select_merge: %{country: s.country_code} ) end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index e87b0eae975a..2b14f5c1ad81 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -557,8 +557,8 @@ defmodule PlausibleWeb.Api.StatsController do countries = Stats.breakdown(site, query, "visit:country", [:visitors], pagination) - |> maybe_add_cr(site, query, {300, 1}, "country", "visit:country") - |> transform_keys(%{"country" => "code"}) + |> maybe_add_cr(site, query, {300, 1}, :country, "visit:country") + |> transform_keys(%{country: "code"}) |> maybe_add_percentages(query) if params["csv"] do From aac954d5b6c2a3e81997c3ded642e9a8c7652fa4 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 29 Nov 2021 19:24:49 +0000 Subject: [PATCH 032/148] Add more metrics to entries/exits for modals --- lib/plausible/google/api.ex | 4 ++-- lib/plausible/imported/entry_pages.ex | 12 +++++++++-- lib/plausible/imported/exit_pages.ex | 8 ++++++-- lib/plausible/imported/site.ex | 17 +++++++++++----- lib/plausible/stats/base.ex | 20 +++++++++++++++---- ...1129111622_create_imported_entry_pages.exs | 2 ++ ...11129111630_create_imported_exit_pages.exs | 1 + 7 files changed, 49 insertions(+), 15 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index fed2b95be435..8f6cb04afae2 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -211,12 +211,12 @@ defmodule Plausible.Google.Api do # Entry pages { ["ga:date", "ga:landingPagePath"], - ["ga:users"] + ["ga:users", "ga:entrances", "ga:sessionDuration"] }, # Exit pages { ["ga:date", "ga:exitPagePath"], - ["ga:users"] + ["ga:users", "ga:exits"] }, # Country { diff --git a/lib/plausible/imported/entry_pages.ex b/lib/plausible/imported/entry_pages.ex index 6be7489195f4..4bc4a4aa3d2d 100644 --- a/lib/plausible/imported/entry_pages.ex +++ b/lib/plausible/imported/entry_pages.ex @@ -9,6 +9,9 @@ defmodule Plausible.Imported.EntryPages do field :timestamp, :naive_datetime field :entry_page, :string field :visitors, :integer + field :entrances, :integer + # Sum total + field :visit_duration, :integer end def new(attrs) do @@ -19,14 +22,19 @@ defmodule Plausible.Imported.EntryPages do :domain, :timestamp, :entry_page, - :visitors + :visitors, + :entrances, + :visit_duration ], empty_values: [nil, ""] ) |> validate_required([ :domain, :timestamp, - :visitors + :entry_page, + :visitors, + :entrances, + :visit_duration ]) end end diff --git a/lib/plausible/imported/exit_pages.ex b/lib/plausible/imported/exit_pages.ex index 0351f698ace2..4b40d205789c 100644 --- a/lib/plausible/imported/exit_pages.ex +++ b/lib/plausible/imported/exit_pages.ex @@ -9,6 +9,7 @@ defmodule Plausible.Imported.ExitPages do field :timestamp, :naive_datetime field :exit_page, :string field :visitors, :integer + field :exits, :integer end def new(attrs) do @@ -19,14 +20,17 @@ defmodule Plausible.Imported.ExitPages do :domain, :timestamp, :exit_page, - :visitors + :visitors, + :exits ], empty_values: [nil, ""] ) |> validate_required([ :domain, :timestamp, - :visitors + :exit_page, + :visitors, + :exits ]) end end diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 29f916cd38da..a9f885028285 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -108,29 +108,36 @@ defmodule Plausible.Imported do defp new_from_google_analytics(domain, "entry_pages", %{ "dimensions" => [timestamp, entry_page], - "metrics" => [%{"values" => [value]}] + "metrics" => [%{"values" => [visitors, entrances, visit_duration]}] }) do - {visitors, ""} = Integer.parse(value) + {visitors, ""} = Integer.parse(visitors) + {entrances, ""} = Integer.parse(entrances) + + {visit_duration, _} = Integer.parse(visit_duration) Imported.EntryPages.new(%{ domain: domain, timestamp: format_timestamp(timestamp), entry_page: entry_page, - visitors: visitors + visitors: visitors, + entrances: entrances, + visit_duration: visit_duration }) end defp new_from_google_analytics(domain, "exit_pages", %{ "dimensions" => [timestamp, exit_page], - "metrics" => [%{"values" => [value]}] + "metrics" => [%{"values" => [value, exits]}] }) do {visitors, ""} = Integer.parse(value) + {exits, ""} = Integer.parse(exits) Imported.ExitPages.new(%{ domain: domain, timestamp: format_timestamp(timestamp), exit_page: exit_page, - visitors: visitors + visitors: visitors, + exits: exits }) end diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 3fbd8c4a26c5..3ca47fa63d60 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -425,10 +425,15 @@ defmodule Plausible.Stats.Base do imported_q |> select_merge([i], %{utm_campaign: i.utm_campaign}) :entry_page -> - imported_q |> select_merge([i], %{entry_page: i.entry_page}) + imported_q + |> select_merge([i], %{ + entry_page: i.entry_page, + visits: sum(i.entrances), + visit_duration: sum(i.visit_duration), + }) :exit_page -> - imported_q |> select_merge([i], %{exit_page: i.exit_page}) + imported_q |> select_merge([i], %{exit_page: i.exit_page, visits: sum(i.exits)}) :country -> imported_q |> select_merge([i], %{country: i.country}) @@ -485,13 +490,20 @@ defmodule Plausible.Stats.Base do :entry_page -> q |> select_merge([i, s], %{ - entry_page: fragment("if(empty(?), ?, ?)", s.entry_page, i.entry_page, s.entry_page) + entry_page: fragment("if(empty(?), ?, ?)", s.entry_page, i.entry_page, s.entry_page), + visits: fragment("toUInt64(?) + toUInt64(coalesce(?, 0))", s.visits, i.visits), + # TODO: This fragment is not correct. + visit_duration: fragment( + "(? + ? * ?) / (? + ?)", + i.visit_duration, s.visit_duration, s.visits, s.visits, i.visits + ) }) :exit_page -> q |> select_merge([i, s], %{ - exit_page: fragment("if(empty(?), ?, ?)", s.exit_page, i.exit_page, s.exit_page) + exit_page: fragment("if(empty(?), ?, ?)", s.exit_page, i.exit_page, s.exit_page), + visits: fragment("toUInt64(?) + toUInt64(coalesce(?, 0))", s.visits, i.visits) }) :country -> diff --git a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs index e7d81edb8f14..35b203199247 100644 --- a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs @@ -7,6 +7,8 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedEntryPages do add :timestamp, :naive_datetime add :entry_page, :string add :visitors, :UInt64 + add :entrances, :UInt64 + add :visit_duration, :UInt64 end end end diff --git a/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs b/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs index 42226cdeb06c..b21117ad119c 100644 --- a/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs @@ -7,6 +7,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedExitPages do add :timestamp, :naive_datetime add :exit_page, :string add :visitors, :UInt64 + add :exits, :UInt64 end end end From f2d7eabb2ed68b5d13c6d037dccfd2c17a900a13 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 1 Dec 2021 10:41:10 +0000 Subject: [PATCH 033/148] make sure data is returned via API with correct keys --- lib/plausible_web/controllers/api/stats_controller.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 2b14f5c1ad81..8349db75124e 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -252,7 +252,7 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:utm_medium", metrics, pagination) |> maybe_add_cr(site, query, pagination, "utm_medium", "visit:utm_medium") - |> transform_keys(%{"utm_medium" => "name"}) + |> transform_keys(%{utm_medium: "name"}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do @@ -283,7 +283,7 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:utm_campaign", metrics, pagination) |> maybe_add_cr(site, query, pagination, "utm_campaign", "visit:utm_campaign") - |> transform_keys(%{"utm_campaign" => "name"}) + |> transform_keys(%{utm_campaign: "name"}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do @@ -372,7 +372,7 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:utm_source", metrics, pagination) |> maybe_add_cr(site, query, pagination, "utm_source", "visit:utm_source") - |> transform_keys(%{"utm_source" => "name"}) + |> transform_keys(%{utm_source: "name"}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do From 3ab7d57905393b2043b1fdee1050e2641e437160 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 1 Dec 2021 10:53:57 +0000 Subject: [PATCH 034/148] Import regions and cities from GA --- lib/plausible/google/api.ex | 2 +- lib/plausible/imported/countries.ex | 8 ++++++-- lib/plausible/imported/site.ex | 6 ++++-- ...s.exs => 20211129111639_create_imported_locations.exs} | 6 ++++-- 4 files changed, 15 insertions(+), 7 deletions(-) rename priv/clickhouse_repo/migrations/{20211129111639_create_imported_countries.exs => 20211129111639_create_imported_locations.exs} (56%) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 8f6cb04afae2..3a343123b6e1 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -220,7 +220,7 @@ defmodule Plausible.Google.Api do }, # Country { - ["ga:date", "ga:countryIsoCode"], + ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode", "ga:cityId"], ["ga:users"] }, # Device diff --git a/lib/plausible/imported/countries.ex b/lib/plausible/imported/countries.ex index dc9a4e9f28a7..587f0617ee70 100644 --- a/lib/plausible/imported/countries.ex +++ b/lib/plausible/imported/countries.ex @@ -1,13 +1,15 @@ -defmodule Plausible.Imported.Countries do +defmodule Plausible.Imported.Locations do use Ecto.Schema use Plausible.ClickhouseRepo import Ecto.Changeset @primary_key false - schema "imported_countries" do + schema "imported_locations" do field :domain, :string field :timestamp, :naive_datetime field :country, :string + field :region, :string + field :city, :string field :visitors, :integer end @@ -19,6 +21,8 @@ defmodule Plausible.Imported.Countries do :domain, :timestamp, :country, + :region, + :city, :visitors ], empty_values: [nil, ""] diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index a9f885028285..128f7301d570 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -142,15 +142,17 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(domain, "countries", %{ - "dimensions" => [timestamp, country], + "dimensions" => [timestamp, country, region, city], "metrics" => [%{"values" => [value]}] }) do {visitors, ""} = Integer.parse(value) - Imported.Countries.new(%{ + Imported.Locations.new(%{ domain: domain, timestamp: format_timestamp(timestamp), country: country, + region: region, + city: city, visitors: visitors }) end diff --git a/priv/clickhouse_repo/migrations/20211129111639_create_imported_countries.exs b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs similarity index 56% rename from priv/clickhouse_repo/migrations/20211129111639_create_imported_countries.exs rename to priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs index 17b2e3d59178..024ae8940c6c 100644 --- a/priv/clickhouse_repo/migrations/20211129111639_create_imported_countries.exs +++ b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs @@ -1,11 +1,13 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedCountries do +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedLocations do use Ecto.Migration def change do - create_if_not_exists table(:imported_countries, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_locations, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :domain, :string add :timestamp, :naive_datetime add :country, :string + add :region, :string + add :city, :string add :visitors, :UInt64 end end From 87ba204c9bf95d95ce009ef75e1a293fcc2fb78d Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 1 Dec 2021 11:19:16 +0000 Subject: [PATCH 035/148] Capitalize device upon import to match native data --- lib/plausible/imported/site.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 128f7301d570..754218ff4f6d 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -166,7 +166,7 @@ defmodule Plausible.Imported do Imported.Devices.new(%{ domain: domain, timestamp: format_timestamp(timestamp), - device: device, + device: String.capitalize(device), visitors: visitors }) end From c5bc70a142d9d76061de0378d50ecfa5d04cc762 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 1 Dec 2021 16:41:26 +0000 Subject: [PATCH 036/148] Leave query limits/offsets until after possibly joining with imported data --- lib/plausible/stats/base.ex | 15 ++++++--------- lib/plausible/stats/breakdown.ex | 25 ++++++++++++++----------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 3ca47fa63d60..76286dba2bac 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -356,10 +356,10 @@ defmodule Plausible.Stats.Base do |> String.replace(~r/(? select_merge([i], %{ entry_page: i.entry_page, visits: sum(i.entrances), - visit_duration: sum(i.visit_duration), + visit_duration: sum(i.visit_duration) }) :exit_page -> @@ -455,9 +454,7 @@ defmodule Plausible.Stats.Base do select: %{ :visitors => fragment("toUInt64(?) + toUInt64(?)", s.visitors, i.visitors) }, - order_by: [desc: fragment("toUInt64(?) + toUInt64(?)", s.visitors, i.visitors)], - limit: ^limit, - offset: ^offset + order_by: [desc: fragment("toUInt64(?) + toUInt64(?)", s.visitors, i.visitors)] ) # TODO: DRY @@ -538,7 +535,7 @@ defmodule Plausible.Stats.Base do end end - def merge_imported(q, _, _, _, _), do: q + def merge_imported(q, _, _, _), do: q defp add_sample_hint(db_q, query) do case query.sample_threshold do diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index c38ab846b17b..50fa09b777a3 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -177,35 +177,30 @@ defmodule Plausible.Stats.Breakdown do defp breakdown_sessions(_, _, _, [], _), do: [] - defp breakdown_sessions(site, query, property, metrics, {limit, page}) do - offset = (page - 1) * limit - + defp breakdown_sessions(site, query, property, metrics, pagination) do from(s in query_sessions(site, query), order_by: [desc: fragment("uniq(?)", s.user_id), asc: fragment("min(?)", s.start)], - limit: ^limit, - offset: ^offset, select: %{} ) |> filter_converted_sessions(site, query) |> do_group_by(property) |> select_session_metrics(metrics) - |> merge_imported(site, query, property, {limit, page}) + |> merge_imported(site, query, property) + |> apply_pagination(pagination) |> ClickhouseRepo.all() end defp breakdown_events(_, _, _, [], _), do: [] - defp breakdown_events(site, query, property, metrics, {limit, page}) do - offset = (page - 1) * limit - + defp breakdown_events(site, query, property, metrics, pagination) do from(e in base_event_query(site, query), order_by: [desc: fragment("uniq(?)", e.user_id)], - limit: ^limit, - offset: ^offset, select: %{} ) |> do_group_by(property) |> select_event_metrics(metrics) + # |> merge_imported(site, query, property) + |> apply_pagination(pagination) |> ClickhouseRepo.all() end @@ -483,4 +478,12 @@ defmodule Plausible.Stats.Breakdown do |> Enum.into(%{}) end) end + + defp apply_pagination(q, {limit, page}) do + offset = (page - 1) * limit + + q + |> Ecto.Query.limit(^limit) + |> Ecto.Query.offset(^offset) + end end From 7936e6f2d546b2ec6f12f14498fc6164b67ac126 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 1 Dec 2021 18:15:47 +0000 Subject: [PATCH 037/148] Also import timeOnPage and pageviews for pages from GA --- lib/plausible/google/api.ex | 2 +- lib/plausible/imported/pages.ex | 6 +++++- lib/plausible/imported/site.ex | 6 +++++- .../migrations/20211129111618_create_imported_pages.exs | 2 ++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 3a343123b6e1..9ef1a50c4e0e 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -205,7 +205,7 @@ defmodule Plausible.Google.Api do }, # Pages { - ["ga:date", "ga:pagePath"], + ["ga:date", "ga:pagePath", "ga:pageviews", "ga:timeOnPage"], ["ga:users"] }, # Entry pages diff --git a/lib/plausible/imported/pages.ex b/lib/plausible/imported/pages.ex index 078e40d80007..3851940acf47 100644 --- a/lib/plausible/imported/pages.ex +++ b/lib/plausible/imported/pages.ex @@ -9,6 +9,8 @@ defmodule Plausible.Imported.Pages do field :timestamp, :naive_datetime field :page, :string field :visitors, :integer + field :pageviews, :integer + field :time_on_page, :integer end def new(attrs) do @@ -19,7 +21,9 @@ defmodule Plausible.Imported.Pages do :domain, :timestamp, :page, - :visitors + :visitors, + :pageviews, + :time_on_page ], empty_values: [nil, ""] ) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 754218ff4f6d..3c388f787654 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -93,16 +93,20 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(domain, "pages", %{ - "dimensions" => [timestamp, page], + "dimensions" => [timestamp, page, pageviews, time_on_page], "metrics" => [%{"values" => [value]}] }) do {visitors, ""} = Integer.parse(value) + {pageviews, ""} = Integer.parse(pageviews) + {time_on_page, ""} = Integer.parse(time_on_page) Imported.Pages.new(%{ domain: domain, timestamp: format_timestamp(timestamp), page: page, visitors: visitors + pageviews: pageviews, + time_on_page: time_on_page }) end diff --git a/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs b/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs index e39a970e6fdf..6aaaad392d10 100644 --- a/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs @@ -7,6 +7,8 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedPages do add :timestamp, :naive_datetime add :page, :string add :visitors, :UInt64 + add :pageviews, :UInt64 + add :time_on_page, :UInt64 end end end From bca7e246b82e3440a18a8a78923cce7bcd030396 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 1 Dec 2021 19:41:48 +0000 Subject: [PATCH 038/148] imported_countries -> imported_locations --- lib/plausible/clickhouse_repo.ex | 2 +- lib/plausible/stats/base.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex index ca42906cfcfd..1d8c719192b1 100644 --- a/lib/plausible/clickhouse_repo.ex +++ b/lib/plausible/clickhouse_repo.ex @@ -28,7 +28,7 @@ defmodule Plausible.ClickhouseRepo do "imported_pages", "imported_entry_pages", "imported_exit_pages", - "imported_countries", + "imported_locations", "imported_devices", "imported_browsers", "imported_operating_systems" diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 76286dba2bac..cd900e9a6105 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -377,7 +377,7 @@ defmodule Plausible.Stats.Base do {table, dim} = case property do "visit:country" -> - {"imported_countries", :country} + {"imported_locations", :country} "visit:os" -> {"imported_operating_systems", :operating_system} From 3f38a3eb8e98f3f679876e75cdc60b31cd8aba9b Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 1 Dec 2021 19:43:38 +0000 Subject: [PATCH 039/148] Get timeOnPage and pageviews for pages from GA These are needed for the pages modal, and for calculating exit rates for exit pages. --- lib/plausible/google/api.ex | 4 ++-- lib/plausible/imported/site.ex | 8 +++---- lib/plausible/stats/base.ex | 40 +++++++++++++++++++++++++++++----- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 9ef1a50c4e0e..a56b7cee5637 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -205,8 +205,8 @@ defmodule Plausible.Google.Api do }, # Pages { - ["ga:date", "ga:pagePath", "ga:pageviews", "ga:timeOnPage"], - ["ga:users"] + ["ga:date", "ga:pagePath"], + ["ga:users", "ga:pageviews", "ga:timeOnPage"] }, # Entry pages { diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 3c388f787654..2f8153551e1d 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -93,18 +93,18 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(domain, "pages", %{ - "dimensions" => [timestamp, page, pageviews, time_on_page], - "metrics" => [%{"values" => [value]}] + "dimensions" => [timestamp, page], + "metrics" => [%{"values" => [value, pageviews, time_on_page]}] }) do {visitors, ""} = Integer.parse(value) {pageviews, ""} = Integer.parse(pageviews) - {time_on_page, ""} = Integer.parse(time_on_page) + {time_on_page, _} = Integer.parse(time_on_page) Imported.Pages.new(%{ domain: domain, timestamp: format_timestamp(timestamp), page: page, - visitors: visitors + visitors: visitors, pageviews: pageviews, time_on_page: time_on_page }) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index cd900e9a6105..d82e84b90e2c 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -370,7 +370,8 @@ defmodule Plausible.Stats.Base do "visit:country", "visit:device", "visit:browser", - "visit:os" + "visit:os", + "event:page" ] do {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) @@ -382,6 +383,9 @@ defmodule Plausible.Stats.Base do "visit:os" -> {"imported_operating_systems", :operating_system} + "event:page" -> + {"imported_pages", :page} + _ -> dim = String.trim_leading(property, "visit:") {"imported_#{dim}s", String.to_atom(dim)} @@ -404,6 +408,9 @@ defmodule Plausible.Stats.Base do value = if value == @no_ref, do: "", else: value where(imported_q, [i], field(i, ^dim) != ^value) + {:member, list} -> + where(imported_q, [i], field(i, ^dim) in ^list) + _ -> imported_q end @@ -423,6 +430,14 @@ defmodule Plausible.Stats.Base do :utm_campaign -> imported_q |> select_merge([i], %{utm_campaign: i.utm_campaign}) + :page -> + imported_q + |> select_merge([i], %{ + page: i.page, + pageviews: sum(i.pageviews), + time_on_page: sum(i.time_on_page) + }) + :entry_page -> imported_q |> select_merge([i], %{ @@ -457,7 +472,6 @@ defmodule Plausible.Stats.Base do order_by: [desc: fragment("toUInt64(?) + toUInt64(?)", s.visitors, i.visitors)] ) - # TODO: DRY case dim do :source -> q @@ -484,16 +498,30 @@ defmodule Plausible.Stats.Base do fragment("if(empty(?), ?, ?)", s.utm_campaign, i.utm_campaign, s.utm_campaign) }) + :page -> + q + |> select_merge([i, s], %{ + page: fragment("if(empty(?), ?, ?)", s.entry_page, i.entry_page, s.entry_page), + pageviews: fragment("? + coalesce(?, 0)", s.pageviews, i.pageviews), + # TODO: This fragment is not correct. + #time_on_page: fragment() + }) + :entry_page -> q |> select_merge([i, s], %{ entry_page: fragment("if(empty(?), ?, ?)", s.entry_page, i.entry_page, s.entry_page), visits: fragment("toUInt64(?) + toUInt64(coalesce(?, 0))", s.visits, i.visits), # TODO: This fragment is not correct. - visit_duration: fragment( - "(? + ? * ?) / (? + ?)", - i.visit_duration, s.visit_duration, s.visits, s.visits, i.visits - ) + visit_duration: + fragment( + "(? + ? * ?) / (? + ?)", + i.visit_duration, + s.visit_duration, + s.visits, + s.visits, + i.visits + ) }) :exit_page -> From a03759e6b3803d5bb475e9fba4db3e1955a91cc4 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 2 Dec 2021 14:11:03 +0000 Subject: [PATCH 040/148] Add indicator to dashboard when imported data is being used --- assets/js/dashboard/stats/visitor-graph.js | 13 +++++++++++++ .../controllers/api/stats_controller.ex | 1 + 2 files changed, 14 insertions(+) diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js index 85d9b1403cc3..c0c99b35b68a 100644 --- a/assets/js/dashboard/stats/visitor-graph.js +++ b/assets/js/dashboard/stats/visitor-graph.js @@ -364,6 +364,18 @@ class LineGraph extends React.Component { } } + importedNotice() { + const hasImported = this.props.graphData.has_imported + + if (hasImported) { + return ( +
    + { hasImported[0].toUpperCase() } +
    + ) + } + } + render() { const extraClass = this.props.graphData.interval === 'hour' ? '' : 'cursor-pointer' @@ -375,6 +387,7 @@ class LineGraph extends React.Component {
    { this.downloadLink() } { this.samplingNotice() } + { this.importedNotice() }
    diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 8349db75124e..6b250c8812aa 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -37,6 +37,7 @@ defmodule PlausibleWeb.Api.StatsController do top_stats: top_stats, interval: query.interval, sample_percent: sample_percent, + has_imported: imported_plot && site.has_imported_stats, imported_plot: imported_plot }) end From b8b8ccfb8b8862f20dc825cc9ba1fedccbb1293a Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 2 Dec 2021 16:19:23 +0000 Subject: [PATCH 041/148] Don't show imported data as separately line on main graph --- assets/js/dashboard/stats/visitor-graph.js | 31 +++++-------------- .../controllers/api/stats_controller.ex | 11 ++++--- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js index c0c99b35b68a..5c25045412ef 100644 --- a/assets/js/dashboard/stats/visitor-graph.js +++ b/assets/js/dashboard/stats/visitor-graph.js @@ -6,16 +6,11 @@ import numberFormatter, {durationFormatter} from '../util/number-formatter' import * as api from '../api' import LazyLoader from '../components/lazy-loader' -function buildDataSet(graphData, ctx, label) { +function buildDataSet(plot, present_index, ctx, label) { var gradient = ctx.createLinearGradient(0, 0, 0, 300); gradient.addColorStop(0, 'rgba(101,116,205, 0.2)'); gradient.addColorStop(1, 'rgba(101,116,205, 0)'); - const present_index = graphData.present_index; - var plot = graphData.plot; - var imported_plot = graphData.imported_plot; - var data = []; - if (present_index) { var dashedPart = plot.slice(present_index - 1, present_index + 1); var dashedPlot = (new Array(present_index - 1)).concat(dashedPart) @@ -23,7 +18,7 @@ function buildDataSet(graphData, ctx, label) { plot[i] = undefined } - data.push(...[{ + return [{ label: label, data: plot, borderWidth: 3, @@ -41,9 +36,9 @@ function buildDataSet(graphData, ctx, label) { pointBackgroundColor: 'rgba(101,116,205)', backgroundColor: gradient, fill: true - }]) + }] } else { - data.push({ + return [{ label: label, data: plot, borderWidth: 3, @@ -51,20 +46,8 @@ function buildDataSet(graphData, ctx, label) { pointBackgroundColor: 'rgba(101,116,205)', backgroundColor: gradient, fill: true - }) - }; - - if (imported_plot) { - data.push({ - label: label, - data: imported_plot, - borderWidth: 3, - borderColor: 'rgba(205,116,101)', - fill: false - }) + }] } - - return data; } const MONTHS = [ @@ -126,7 +109,7 @@ class LineGraph extends React.Component { const {graphData} = this.props this.ctx = document.getElementById("main-graph-canvas").getContext('2d'); const label = this.props.query.filters.goal ? 'Converted visitors' : graphData.interval === 'minute' ? 'Pageviews' : 'Visitors' - const dataSet = buildDataSet(graphData, this.ctx, label) + const dataSet = buildDataSet(graphData.plot, graphData.present_index, this.ctx, label) return new Chart(this.ctx, { type: 'line', @@ -213,7 +196,7 @@ class LineGraph extends React.Component { componentDidUpdate(prevProps) { if (this.props.graphData !== prevProps.graphData) { const label = this.props.query.filters.goal ? 'Converted visitors' : this.props.graphData.interval === 'minute' ? 'Pageviews' : 'Visitors' - const newDataset = buildDataSet(this.props.graphData, this.ctx, label) + const newDataset = buildDataSet(this.props.graphData.plot, this.props.graphData.present_index, this.ctx, label) for (let i = 0; i < newDataset[0].data.length; i++) { this.chart.data.datasets[0].data[i] = newDataset[0].data[i] diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 6b250c8812aa..661e046382d4 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -25,9 +25,13 @@ defmodule PlausibleWeb.Api.StatsController do labels = Enum.map(timeseries_result, fn row -> row["date"] end) present_index = present_index_for(site, query, labels) - imported_plot = + {plot, has_imported} = if params["filters"] == "{}" && site.has_imported_stats do - Imported.Visitors.timeseries(site, timeseries_query) + plot = Imported.Visitors.timeseries(site, timeseries_query) + |> Enum.zip_with(plot, & &1 + &2) + {plot, site.has_imported_stats} + else + {plot, nil} end json(conn, %{ @@ -37,8 +41,7 @@ defmodule PlausibleWeb.Api.StatsController do top_stats: top_stats, interval: query.interval, sample_percent: sample_percent, - has_imported: imported_plot && site.has_imported_stats, - imported_plot: imported_plot + has_imported: has_imported }) end From 991cf88d26a32789ee2bd5d9c3db20c8beb03e58 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 2 Dec 2021 16:29:12 +0000 Subject: [PATCH 042/148] "bounce_rate" -> :bounce_rate, so it works in subqueries --- lib/plausible/stats/aggregate.ex | 2 +- lib/plausible/stats/base.ex | 4 +-- lib/plausible/stats/breakdown.ex | 2 +- lib/plausible/stats/timeseries.ex | 4 +-- .../api/external_stats_controller.ex | 2 +- .../controllers/api/stats_controller.ex | 30 +++++++++---------- .../controllers/stats_controller.ex | 2 +- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index b695388afd61..b68277bc7bb8 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -4,7 +4,7 @@ defmodule Plausible.Stats.Aggregate do import Plausible.Stats.Base @event_metrics [:visitors, "pageviews", "events", "sample_percent"] - @session_metrics [:visits, "bounce_rate", :visit_duration, "sample_percent"] + @session_metrics [:visits, :bounce_rate, :visit_duration, "sample_percent"] def aggregate(site, query, metrics) do event_metrics = Enum.filter(metrics, &(&1 in @event_metrics)) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index d82e84b90e2c..e125d99aca4b 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -237,10 +237,10 @@ defmodule Plausible.Stats.Base do def select_session_metrics(q, []), do: q - def select_session_metrics(q, ["bounce_rate" | rest]) do + def select_session_metrics(q, [:bounce_rate | rest]) do from(s in q, select_merge: %{ - "bounce_rate" => + bounce_rate: fragment("toUInt32(ifNotFinite(round(sum(is_bounce * sign) / sum(sign) * 100), 0))") } ) diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 50fa09b777a3..1b2fc29075f5 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -5,7 +5,7 @@ defmodule Plausible.Stats.Breakdown do @no_ref "Direct / None" @event_metrics [:visitors, "pageviews", "events"] - @session_metrics [:visits, "bounce_rate", :visit_duration] + @session_metrics [:visits, :bounce_rate, :visit_duration] @event_props ["event:page", "event:page_match", "event:name"] def breakdown(site, query, "event:goal", metrics, pagination) do diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index c160a2ee899d..cef1072e0b5e 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -5,7 +5,7 @@ defmodule Plausible.Stats.Timeseries do use Plausible.Stats.Fragments @event_metrics [:visitors, "pageviews"] - @session_metrics [:visits, "bounce_rate", :visit_duration] + @session_metrics [:visits, :bounce_rate, :visit_duration] def timeseries(site, query, metrics) do steps = buckets(query) @@ -121,7 +121,7 @@ defmodule Plausible.Stats.Timeseries do "pageviews" -> Map.merge(row, %{"pageviews" => 0}) :visitors -> Map.merge(row, %{visitors: 0}) :visits -> Map.merge(row, %{visits: 0}) - "bounce_rate" -> Map.merge(row, %{"bounce_rate" => nil}) + :bounce_rate -> Map.merge(row, %{bounce_rate: nil}) :visit_duration -> Map.merge(row, %{:visit_duration => nil}) end end) diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index 0be2afb70d61..b96c33fe1b06 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -101,7 +101,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do defp event_only_property?(_), do: false @event_metrics ["visitors", "pageviews", "events"] - @session_metrics [:visits, "bounce_rate", :visit_duration] + @session_metrics [:visits, :bounce_rate, :visit_duration] defp parse_metrics(params, property, query) do metrics = Map.get(params, "metrics", "visitors") diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 661e046382d4..6ed0ebb2cfa1 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -157,9 +157,9 @@ defmodule PlausibleWeb.Api.StatsController do metrics = if query.filters["event:page"] do - [:visitors, "pageviews", "bounce_rate", "time_on_page", "sample_percent"] + [:visitors, "pageviews", :bounce_rate, "time_on_page", "sample_percent"] else - [:visitors, "pageviews", "bounce_rate", :visit_duration, "sample_percent"] + [:visitors, "pageviews", :bounce_rate, :visit_duration, "sample_percent"] end current_results = Stats.aggregate(site, query, metrics) @@ -169,7 +169,7 @@ defmodule PlausibleWeb.Api.StatsController do [ top_stats_entry(current_results, prev_results, "Unique visitors", :visitors), top_stats_entry(current_results, prev_results, "Total pageviews", "pageviews"), - top_stats_entry(current_results, prev_results, "Bounce rate", "bounce_rate"), + top_stats_entry(current_results, prev_results, "Bounce rate", :bounce_rate), top_stats_entry(current_results, prev_results, "Visit duration", :visit_duration), top_stats_entry(current_results, prev_results, "Time on page", "time_on_page") ] @@ -188,7 +188,7 @@ defmodule PlausibleWeb.Api.StatsController do end end - defp calculate_change("bounce_rate", old_count, new_count) do + defp calculate_change(:bounce_rate, old_count, new_count) do if old_count > 0, do: new_count - old_count end @@ -220,7 +220,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) metrics = - if params["detailed"], do: [:visitors, "bounce_rate", :visit_duration], else: [:visitors] + if params["detailed"], do: [:visitors, :bounce_rate, :visit_duration], else: [:visitors] res = Stats.breakdown(site, query, "visit:source", metrics, pagination) @@ -233,7 +233,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", :visitors, "bounce_rate", "visit_duration"]) + res |> to_csv(["name", :visitors, :bounce_rate, "visit_duration"]) end else json(conn, res) @@ -251,7 +251,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) metrics = - if params["detailed"], do: [:visitors, "bounce_rate", :visit_duration], else: [:visitors] + if params["detailed"], do: [:visitors, :bounce_rate, :visit_duration], else: [:visitors] res = Stats.breakdown(site, query, "visit:utm_medium", metrics, pagination) @@ -264,7 +264,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", :visitors, "bounce_rate", "visit_duration"]) + res |> to_csv(["name", :visitors, :bounce_rate, "visit_duration"]) end else json(conn, res) @@ -282,7 +282,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) metrics = - if params["detailed"], do: [:visitors, "bounce_rate", :visit_duration], else: [:visitors] + if params["detailed"], do: [:visitors, :bounce_rate, :visit_duration], else: [:visitors] res = Stats.breakdown(site, query, "visit:utm_campaign", metrics, pagination) @@ -295,7 +295,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", :visitors, "bounce_rate", "visit_duration"]) + res |> to_csv(["name", :visitors, :bounce_rate, "visit_duration"]) end else json(conn, res) @@ -371,7 +371,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) metrics = - if params["detailed"], do: [:visitors, "bounce_rate", :visit_duration], else: [:visitors] + if params["detailed"], do: [:visitors, :bounce_rate, :visit_duration], else: [:visitors] res = Stats.breakdown(site, query, "visit:utm_source", metrics, pagination) @@ -384,7 +384,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", :visitors, "bounce_rate", "visit_duration"]) + res |> to_csv(["name", :visitors, :bounce_rate, "visit_duration"]) end else json(conn, res) @@ -432,7 +432,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) metrics = - if params["detailed"], do: [:visitors, "bounce_rate", :visit_duration], else: [:visitors] + if params["detailed"], do: [:visitors, :bounce_rate, :visit_duration], else: [:visitors] referrers = Stats.breakdown(site, query, "visit:referrer", metrics, pagination) @@ -449,7 +449,7 @@ defmodule PlausibleWeb.Api.StatsController do metrics = if params["detailed"], - do: [:visitors, "pageviews", "bounce_rate", "time_on_page"], + do: [:visitors, "pageviews", :bounce_rate, "time_on_page"], else: [:visitors] pagination = parse_pagination(params) @@ -465,7 +465,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{"visitors" => "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - pages |> to_csv(["name", "visitors", "bounce_rate", "time_on_page"]) + pages |> to_csv(["name", "visitors", :bounce_rate, "time_on_page"]) end else json(conn, pages) diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index ef0d85b621df..581044e8596b 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -50,7 +50,7 @@ defmodule PlausibleWeb.StatsController do site = conn.assigns[:site] query = Query.from(site.timezone, params) |> Filters.add_prefix() - metrics = [:visitors, "pageviews", "bounce_rate", :visit_duration] + metrics = [:visitors, "pageviews", :bounce_rate, :visit_duration] graph = Plausible.Stats.timeseries(site, query, metrics) headers = ["date" | metrics] From e03e12815e43ec829dd3e2e03a9a8eba17cce5d7 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 3 Dec 2021 21:37:48 +0300 Subject: [PATCH 043/148] Drop imported browser and OS versions These are not needed. --- lib/plausible/google/api.ex | 4 ++-- lib/plausible/imported/browsers.ex | 2 -- lib/plausible/imported/operation_systems.ex | 2 -- lib/plausible/imported/site.ex | 6 ++---- .../migrations/20211129111648_create_imported_browsers.exs | 1 - .../20211129111653_create_imported_operating_systems.exs | 1 - 6 files changed, 4 insertions(+), 12 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index a56b7cee5637..7d71b1161d21 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -230,12 +230,12 @@ defmodule Plausible.Google.Api do }, # Browser { - ["ga:date", "ga:browser", "ga:browserVersion"], + ["ga:date", "ga:browser"], ["ga:users"] }, # OS { - ["ga:date", "ga:operatingSystem", "ga:operatingSystemVersion"], + ["ga:date", "ga:operatingSystem"], ["ga:users"] } ] diff --git a/lib/plausible/imported/browsers.ex b/lib/plausible/imported/browsers.ex index b91750050409..fa8b52715d70 100644 --- a/lib/plausible/imported/browsers.ex +++ b/lib/plausible/imported/browsers.ex @@ -8,7 +8,6 @@ defmodule Plausible.Imported.Browsers do field :domain, :string field :timestamp, :naive_datetime field :browser, :string - field :version, :string field :visitors, :integer end @@ -20,7 +19,6 @@ defmodule Plausible.Imported.Browsers do :domain, :timestamp, :browser, - :version, :visitors ], empty_values: [nil, ""] diff --git a/lib/plausible/imported/operation_systems.ex b/lib/plausible/imported/operation_systems.ex index 4c61d68344a4..af027ce597d4 100644 --- a/lib/plausible/imported/operation_systems.ex +++ b/lib/plausible/imported/operation_systems.ex @@ -8,7 +8,6 @@ defmodule Plausible.Imported.OperatingSystems do field :domain, :string field :timestamp, :naive_datetime field :operating_system, :string - field :version, :string field :visitors, :integer end @@ -20,7 +19,6 @@ defmodule Plausible.Imported.OperatingSystems do :domain, :timestamp, :operating_system, - :version, :visitors ], empty_values: [nil, ""] diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 2f8153551e1d..696e260c9129 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -176,7 +176,7 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(domain, "browsers", %{ - "dimensions" => [timestamp, browser, version], + "dimensions" => [timestamp, browser], "metrics" => [%{"values" => [value]}] }) do {visitors, ""} = Integer.parse(value) @@ -185,13 +185,12 @@ defmodule Plausible.Imported do domain: domain, timestamp: format_timestamp(timestamp), browser: browser, - version: version, visitors: visitors }) end defp new_from_google_analytics(domain, "operating_systems", %{ - "dimensions" => [timestamp, os, version], + "dimensions" => [timestamp, os], "metrics" => [%{"values" => [value]}] }) do {visitors, ""} = Integer.parse(value) @@ -200,7 +199,6 @@ defmodule Plausible.Imported do domain: domain, timestamp: format_timestamp(timestamp), operating_system: os, - version: version, visitors: visitors }) end diff --git a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs index e012f67d2d13..b58f337f5fa0 100644 --- a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs +++ b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs @@ -6,7 +6,6 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedBrowsers do add :domain, :string add :timestamp, :naive_datetime add :browser, :string - add :version, :string add :visitors, :UInt64 end end diff --git a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs index 1aa73ee1b51d..2e8171bb7644 100644 --- a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs +++ b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs @@ -6,7 +6,6 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedOperatingSystems do add :domain, :string add :timestamp, :naive_datetime add :operating_system, :string - add :version, :string add :visitors, :UInt64 end end From 5dbde9edb4bf5888d86550d896eb38b6dfcd0b76 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 6 Dec 2021 17:36:02 +0300 Subject: [PATCH 044/148] Toggle displaying imported data by clicking indicator --- assets/js/dashboard/api.js | 1 + assets/js/dashboard/query.js | 1 + assets/js/dashboard/stats/visitor-graph.js | 15 +++++++--- lib/plausible/stats/query.ex | 29 +++++++++++-------- .../controllers/api/stats_controller.ex | 2 +- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/assets/js/dashboard/api.js b/assets/js/dashboard/api.js index ddbb90f79888..6722c52d9049 100644 --- a/assets/js/dashboard/api.js +++ b/assets/js/dashboard/api.js @@ -42,6 +42,7 @@ export function serializeQuery(query, extraQuery=[]) { if (query.from) { queryObj.from = formatISO(query.from) } if (query.to) { queryObj.to = formatISO(query.to) } if (query.filters) { queryObj.filters = serializeFilters(query.filters) } + if (query.with_imported) { queryObj.with_imported = query.with_imported } if (SHARED_LINK_AUTH) { queryObj.auth = SHARED_LINK_AUTH } Object.assign(queryObj, ...extraQuery) diff --git a/assets/js/dashboard/query.js b/assets/js/dashboard/query.js index 366d00f263bc..6d68bd2dfbd3 100644 --- a/assets/js/dashboard/query.js +++ b/assets/js/dashboard/query.js @@ -23,6 +23,7 @@ export function parseQuery(querystring, site) { date: q.get('date') ? parseUTCDate(q.get('date')) : nowForSite(site), from: q.get('from') ? parseUTCDate(q.get('from')) : undefined, to: q.get('to') ? parseUTCDate(q.get('to')) : undefined, + with_imported: q.get('with_imported') ? q.get('with_imported') === 'true' : true, filters: { 'goal': q.get('goal'), 'props': JSON.parse(q.get('props')), diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js index 5c25045412ef..64dd6c103488 100644 --- a/assets/js/dashboard/stats/visitor-graph.js +++ b/assets/js/dashboard/stats/visitor-graph.js @@ -1,10 +1,11 @@ import React from 'react'; -import { withRouter } from 'react-router-dom' +import { withRouter, Link } from 'react-router-dom' import Chart from 'chart.js/auto'; import { navigateToQuery } from '../query' import numberFormatter, {durationFormatter} from '../util/number-formatter' import * as api from '../api' import LazyLoader from '../components/lazy-loader' +import * as url from '../url' function buildDataSet(plot, present_index, ctx, label) { var gradient = ctx.createLinearGradient(0, 0, 0, 300); @@ -351,10 +352,16 @@ class LineGraph extends React.Component { const hasImported = this.props.graphData.has_imported if (hasImported) { + const target = this.props.query.with_imported || false ? + url.setQuery('with_imported', false) : + window.location.pathname; + return ( -
    - { hasImported[0].toUpperCase() } -
    + +
    + { hasImported[0].toUpperCase() } +
    + ) } } diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index f363c6588043..72d10b3a26c7 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -4,7 +4,7 @@ defmodule Plausible.Stats.Query do period: nil, filters: %{}, sample_threshold: 20_000_000 - with_imported: false + with_imported: true @default_sample_threshold 20_000_000 @@ -46,7 +46,8 @@ defmodule Plausible.Stats.Query do interval: "minute", date_range: Date.range(date, date), filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), + with_imported: include_imported(false, params) } end @@ -59,7 +60,7 @@ defmodule Plausible.Stats.Query do interval: "hour", filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: determine_also_imported(params) + with_imported: include_imported(true, params) } end @@ -73,7 +74,7 @@ defmodule Plausible.Stats.Query do interval: "date", filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: determine_also_imported(params) + with_imported: include_imported(true, params) } end @@ -87,7 +88,7 @@ defmodule Plausible.Stats.Query do interval: "date", filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: determine_also_imported(params) + with_imported: include_imported(true, params) } end @@ -103,7 +104,7 @@ defmodule Plausible.Stats.Query do interval: "date", filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: determine_also_imported(params) + with_imported: include_imported(true, params) } end @@ -122,7 +123,7 @@ defmodule Plausible.Stats.Query do interval: Map.get(params, "interval", "month"), filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: determine_also_imported(params) + with_imported: include_imported(true, params) } end @@ -141,7 +142,7 @@ defmodule Plausible.Stats.Query do interval: Map.get(params, "interval", "month"), filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: determine_also_imported(params) + with_imported: include_imported(true, params) } end @@ -165,7 +166,8 @@ defmodule Plausible.Stats.Query do date_range: Date.range(from_date, to_date), interval: Map.get(params, "interval", "date"), filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), + with_imported: include_imported(true, params) } end @@ -225,9 +227,6 @@ defmodule Plausible.Stats.Query do defp parse_filters(%{"filters" => filters}) when is_map(filters), do: filters defp parse_filters(_), do: %{} - defp determine_also_imported(%{"filters" => "{}"}), do: true - defp determine_also_imported(_), do: false - defp parse_filter_expression(str) do filters = String.split(str, ";") @@ -257,4 +256,10 @@ defmodule Plausible.Stats.Query do defp parse_goal_filter("Visit " <> page), do: {:is, :page, page} defp parse_goal_filter(event), do: {:is, :event, event} + + defp include_imported(default, params) do + (default && + params["filters"] == "{}" && + params["with_imported"] == "true") + end end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 6ed0ebb2cfa1..cfbbe52c0b05 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -26,7 +26,7 @@ defmodule PlausibleWeb.Api.StatsController do present_index = present_index_for(site, query, labels) {plot, has_imported} = - if params["filters"] == "{}" && site.has_imported_stats do + if query.with_imported && site.has_imported_stats do plot = Imported.Visitors.timeseries(site, timeseries_query) |> Enum.zip_with(plot, & &1 + &2) {plot, site.has_imported_stats} From 71abc96ab3622f602f9713127dbbabdb0cedddda Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 7 Dec 2021 18:29:52 +0300 Subject: [PATCH 045/148] Parse referrers with RefInspector - Use 'ga:fullReferrer' instead of 'ga:source'. This provides the actual referrer host + path, whereas 'ga:source' includes utm_mediums and other values when relevant. - 'ga:fullReferror' does however include search engine names directly, so they are manually checked for as RefInspector won't pick up on these. --- lib/plausible/google/api.ex | 2 +- lib/plausible/imported/site.ex | 4 ++-- lib/plausible/imported/sources.ex | 38 +++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 7d71b1161d21..bf2f12d7d545 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -190,7 +190,7 @@ defmodule Plausible.Google.Api do }, # Sources { - ["ga:date", "ga:source"], + ["ga:date", "ga:fullReferrer"], ["ga:users"] }, # UTM Mediums diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 696e260c9129..c1d0d61be72f 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -48,12 +48,12 @@ defmodule Plausible.Imported do }) do {visitors, ""} = Integer.parse(value) - source = if source == "(direct)", do: "", else: source + source = if source == "(direct)", do: nil, else: source Imported.Sources.new(%{ domain: domain, timestamp: format_timestamp(timestamp), - source: source, + source: Imported.Sources.parse(source), visitors: visitors }) end diff --git a/lib/plausible/imported/sources.ex b/lib/plausible/imported/sources.ex index 8b3a2b6ddf54..f3459ff612c8 100644 --- a/lib/plausible/imported/sources.ex +++ b/lib/plausible/imported/sources.ex @@ -29,4 +29,42 @@ defmodule Plausible.Imported.Sources do :visitors ]) end + + @search_engines %{ + "google" => "Google", + "bing" => "Bing", + "duckduckgo" => "DuckDuckGo", + } + + def parse(nil), do: nil + + def parse(ref) do + se = @search_engines[ref] + + if se do + se + else + ref = "https://" <> ref + + case RefInspector.parse(ref).source do + :unknown -> + uri = URI.parse(String.trim(ref)) + + if right_uri?(uri) do + String.replace_leading(uri.host, "www.", "") + end + + source -> + source + end + end + end + + defp right_uri?(%URI{host: nil}), do: false + + defp right_uri?(%URI{host: host, scheme: scheme}) + when scheme in ["http", "https"] and byte_size(host) > 0, + do: true + + defp right_uri?(_), do: false end From 4bfee86df5c4bad67363c81352ebae2230270981 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 7 Dec 2021 18:40:19 +0300 Subject: [PATCH 046/148] Keep imported data indicator on dashboard and strikethrough when hidden --- assets/js/dashboard/stats/visitor-graph.js | 6 ++++-- lib/plausible_web/controllers/api/stats_controller.ex | 11 +++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js index 64dd6c103488..057146048590 100644 --- a/assets/js/dashboard/stats/visitor-graph.js +++ b/assets/js/dashboard/stats/visitor-graph.js @@ -352,13 +352,15 @@ class LineGraph extends React.Component { const hasImported = this.props.graphData.has_imported if (hasImported) { - const target = this.props.query.with_imported || false ? + const withImported = this.props.query.with_imported + const strike = withImported ? "" : " line-through" + const target = withImported || false ? url.setQuery('with_imported', false) : window.location.pathname; return ( -
    +
    { hasImported[0].toUpperCase() }
    diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index cfbbe52c0b05..1c415f1550c1 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -25,13 +25,12 @@ defmodule PlausibleWeb.Api.StatsController do labels = Enum.map(timeseries_result, fn row -> row["date"] end) present_index = present_index_for(site, query, labels) - {plot, has_imported} = + plot = if query.with_imported && site.has_imported_stats do - plot = Imported.Visitors.timeseries(site, timeseries_query) - |> Enum.zip_with(plot, & &1 + &2) - {plot, site.has_imported_stats} + Imported.Visitors.timeseries(site, timeseries_query) + |> Enum.zip_with(plot, & &1 + &2) else - {plot, nil} + plot end json(conn, %{ @@ -41,7 +40,7 @@ defmodule PlausibleWeb.Api.StatsController do top_stats: top_stats, interval: query.interval, sample_percent: sample_percent, - has_imported: has_imported + has_imported: site.has_imported_stats }) end From 5ff04123488443f32407ef93888a94b34cc05635 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 8 Dec 2021 15:24:27 +0300 Subject: [PATCH 047/148] Add unlink google button to import panel --- .../controllers/site_controller.ex | 17 ++++++++++++++--- lib/plausible_web/router.ex | 3 ++- .../templates/site/settings_general.html.eex | 7 +++++++ .../site/settings_search_console.html.eex | 2 +- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 2dcbecf42f85..3d40d83c633b 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -302,9 +302,20 @@ defmodule PlausibleWeb.SiteController do Repo.delete!(site.google_auth) - conn - |> put_flash(:success, "Google account unlinked from Plausible") - |> redirect(to: Routes.site_path(conn, :settings_search_console, site.domain)) + conn = put_flash(conn, :success, "Google account unlinked from Plausible") + + panel = + conn.path_info + |> List.last() + |> String.split("-") + |> List.last() + + case panel do + "search" -> + redirect(conn, to: Routes.site_path(conn, :settings_search_console, site.domain)) + "import" -> + redirect(conn, to: Routes.site_path(conn, :settings_general, site.domain)) + end end def update_settings(conn, %{"site" => site_params}) do diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 607331bb29aa..c118d053daa2 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -233,7 +233,8 @@ defmodule PlausibleWeb.Router do delete "/:website/goals/:id", SiteController, :delete_goal put "/:website/settings", SiteController, :update_settings put "/:website/settings/google", SiteController, :update_google_auth - delete "/:website/settings/google", SiteController, :delete_google_auth + delete "/:website/settings/google-search", SiteController, :delete_google_auth + delete "/:website/settings/google-import", SiteController, :delete_google_auth delete "/:website", SiteController, :delete_site delete "/:website/stats", SiteController, :reset_stats diff --git a/lib/plausible_web/templates/site/settings_general.html.eex b/lib/plausible_web/templates/site/settings_general.html.eex index 606cbabf801a..184c656978f4 100644 --- a/lib/plausible_web/templates/site/settings_general.html.eex +++ b/lib/plausible_web/templates/site/settings_general.html.eex @@ -70,6 +70,11 @@
    <%= link("Forget imported stats", to: "/#{URI.encode_www_form(@site.domain)}/settings/forget-imported", method: :delete, class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150") %> + + <%= if @site.google_auth do %> + <%= link("Unlink Google account", to: "/#{URI.encode_www_form(@site.domain)}/settings/google-import", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete") %> + <% end %> + <% @site.google_auth -> %>
    Linked Google account: <%= @site.google_auth.email %> @@ -94,6 +99,8 @@

    <%= error %>

    <% end %> + <%= link("Unlink Google account", to: "/#{URI.encode_www_form(@site.domain)}/settings/google-import", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete") %> + <% true -> %> <%= button("Continue with Google", to: Plausible.Google.Api.authorize_url(@site.id, "general"), class: "button mt-8") %> <% end %> diff --git a/lib/plausible_web/templates/site/settings_search_console.html.eex b/lib/plausible_web/templates/site/settings_search_console.html.eex index 7f34dc131a5f..5181e906e01f 100644 --- a/lib/plausible_web/templates/site/settings_search_console.html.eex +++ b/lib/plausible_web/templates/site/settings_search_console.html.eex @@ -12,7 +12,7 @@
    Linked Google account: <%= @site.google_auth.email %> - <%= link("Unlink Google account", to: "/#{URI.encode_www_form(@site.domain)}/settings/google", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete") %> + <%= link("Unlink Google account", to: "/#{URI.encode_www_form(@site.domain)}/settings/google-search", class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150", method: "delete") %> <%= case @search_console_domains do %> <% {:ok, domains} -> %> From 2c6255e2fd6a047f5f555a443b12c721833c849b Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 9 Dec 2021 11:21:02 +0300 Subject: [PATCH 048/148] Rename some GA browsers and OSes to plausible versions --- lib/plausible/imported/site.ex | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index c1d0d61be72f..5d4ae93f5b9e 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -175,6 +175,16 @@ defmodule Plausible.Imported do }) end + @browser_google_to_plausible %{ + "User-Agent:Opera" => "Opera", + "Mozilla Compatible Agent" => "Mobile App", + "Android Webview" => "Mobile App", + "Android Browser" => "Mobile App", + "Safari (in-app)" => "Mobile App", + "User-Agent: Mozilla" => "Firefox", + "(not set)" => "", + } + defp new_from_google_analytics(domain, "browsers", %{ "dimensions" => [timestamp, browser], "metrics" => [%{"values" => [value]}] @@ -184,11 +194,17 @@ defmodule Plausible.Imported do Imported.Browsers.new(%{ domain: domain, timestamp: format_timestamp(timestamp), - browser: browser, + browser: Map.get(@browser_google_to_plausible, browser, browser) visitors: visitors }) end + @os_google_to_plausible %{ + "Macintosh" => "Mac", + "Linux" => "GNU/Linux", + "(not set)" => "", + } + defp new_from_google_analytics(domain, "operating_systems", %{ "dimensions" => [timestamp, os], "metrics" => [%{"values" => [value]}] @@ -198,7 +214,7 @@ defmodule Plausible.Imported do Imported.OperatingSystems.new(%{ domain: domain, timestamp: format_timestamp(timestamp), - operating_system: os, + operating_system: Map.get(@os_google_to_plausible, os, os), visitors: visitors }) end From 543f42b5eadf671d4f3bb4d97ace501260ec843e Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 9 Dec 2021 15:58:54 +0300 Subject: [PATCH 049/148] Get main top pages and exit pages panels working correctly with imported data --- lib/plausible/stats/aggregate.ex | 2 +- lib/plausible/stats/base.ex | 66 ++++++++++++------- lib/plausible/stats/breakdown.ex | 30 +++++---- lib/plausible/stats/clickhouse.ex | 6 +- lib/plausible/stats/filter_suggestions.ex | 4 +- lib/plausible/stats/filters.ex | 2 +- lib/plausible/stats/query.ex | 6 +- lib/plausible/stats/timeseries.ex | 4 +- .../controllers/api/stats_controller.ex | 26 ++++---- .../controllers/stats_controller.ex | 2 +- 10 files changed, 85 insertions(+), 63 deletions(-) diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index b68277bc7bb8..7f699f0e616a 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -3,7 +3,7 @@ defmodule Plausible.Stats.Aggregate do use Plausible.ClickhouseRepo import Plausible.Stats.Base - @event_metrics [:visitors, "pageviews", "events", "sample_percent"] + @event_metrics [:visitors, :pageviews, "events", "sample_percent"] @session_metrics [:visits, :bounce_rate, :visit_duration, "sample_percent"] def aggregate(site, query, metrics) do diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index e125d99aca4b..cd3472c809f9 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -197,10 +197,10 @@ defmodule Plausible.Stats.Base do def select_event_metrics(q, []), do: q - def select_event_metrics(q, ["pageviews" | rest]) do + def select_event_metrics(q, [:pageviews | rest]) do from(e in q, select_merge: %{ - "pageviews" => + pageviews: fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name) } ) @@ -256,10 +256,10 @@ defmodule Plausible.Stats.Base do |> select_session_metrics(rest) end - def select_session_metrics(q, ["pageviews" | rest]) do + def select_session_metrics(q, [:pageviews | rest]) do from(s in q, select_merge: %{ - "pageviews" => + pageviews: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews) } ) @@ -356,10 +356,10 @@ defmodule Plausible.Stats.Base do |> String.replace(~r/(? @@ -447,7 +446,8 @@ defmodule Plausible.Stats.Base do }) :exit_page -> - imported_q |> select_merge([i], %{exit_page: i.exit_page, visits: sum(i.exits)}) + imported_q + |> select_merge([i], %{exit_page: i.exit_page, visits: sum(i.exits)}) :country -> imported_q |> select_merge([i], %{country: i.country}) @@ -465,13 +465,22 @@ defmodule Plausible.Stats.Base do q = from(s in Ecto.Query.subquery(q), full_join: i in subquery(imported_q), - on: field(s, ^dim) == field(i, ^dim), - select: %{ - :visitors => fragment("toUInt64(?) + toUInt64(?)", s.visitors, i.visitors) - }, - order_by: [desc: fragment("toUInt64(?) + toUInt64(?)", s.visitors, i.visitors)] + on: field(s, ^dim) == field(i, ^dim) ) + q = + if :visitors in metrics do + q + |> select_merge([i, s], %{ + :visitors => fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) + }) + |> order_by([i, s], + desc: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) + ) + else + q + end + case dim do :source -> q @@ -501,18 +510,27 @@ defmodule Plausible.Stats.Base do :page -> q |> select_merge([i, s], %{ - page: fragment("if(empty(?), ?, ?)", s.entry_page, i.entry_page, s.entry_page), - pageviews: fragment("? + coalesce(?, 0)", s.pageviews, i.pageviews), - # TODO: This fragment is not correct. - #time_on_page: fragment() + page: fragment("if(empty(?), ?, ?)", i.page, s.page, i.page) + # time_on_page: fragment( + # "coalesce(?, 0) + coalesce(?, 0) * coalesce(?, 0)", + # i.time_on_page, s.pageviews, s.time_on_page + # ), }) + if :pageviews in metrics do + q + |> select_merge([i, s], %{ + pageviews: fragment("coalesce(?, 0) + coalesce(?, 0)", s.pageviews, i.pageviews) + }) + else + q + end + :entry_page -> q |> select_merge([i, s], %{ - entry_page: fragment("if(empty(?), ?, ?)", s.entry_page, i.entry_page, s.entry_page), - visits: fragment("toUInt64(?) + toUInt64(coalesce(?, 0))", s.visits, i.visits), - # TODO: This fragment is not correct. + entry_page: fragment("if(empty(?), ?, ?)", i.entry_page, s.entry_page, i.entry_page), + visits: fragment("? + ?", s.visits, i.visits), visit_duration: fragment( "(? + ? * ?) / (? + ?)", @@ -527,8 +545,8 @@ defmodule Plausible.Stats.Base do :exit_page -> q |> select_merge([i, s], %{ - exit_page: fragment("if(empty(?), ?, ?)", s.exit_page, i.exit_page, s.exit_page), - visits: fragment("toUInt64(?) + toUInt64(coalesce(?, 0))", s.visits, i.visits) + exit_page: fragment("if(empty(?), ?, ?)", i.exit_page, s.exit_page, i.exit_page), + visits: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visits, i.visits) }) :country -> @@ -563,7 +581,7 @@ defmodule Plausible.Stats.Base do end end - def merge_imported(q, _, _, _), do: q + def merge_imported(q, _, _, _, _), do: q defp add_sample_hint(db_q, query) do case query.sample_threshold do diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 1b2fc29075f5..cceee094fabc 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -4,7 +4,7 @@ defmodule Plausible.Stats.Breakdown do alias Plausible.Stats.Query @no_ref "Direct / None" - @event_metrics [:visitors, "pageviews", "events"] + @event_metrics [:visitors, :pageviews, "events"] @session_metrics [:visits, :bounce_rate, :visit_duration] @event_props ["event:page", "event:page_match", "event:name"] @@ -96,11 +96,11 @@ defmodule Plausible.Stats.Breakdown do event_result = if "time_on_page" in metrics do - pages = Enum.map(event_result, & &1["page"]) + pages = Enum.map(event_result, & &1[:page]) time_on_page_result = breakdown_time_on_page(site, query, pages) Enum.map(event_result, fn row -> - Map.put(row, "time_on_page", time_on_page_result[row["page"]]) + Map.put(row, "time_on_page", time_on_page_result[row[:page]]) end) else event_result @@ -112,19 +112,19 @@ defmodule Plausible.Stats.Breakdown do query pages -> - Query.put_filter(query, "visit:entry_page", {:member, Enum.map(pages, & &1["page"])}) + Query.put_filter(query, "visit:entry_page", {:member, Enum.map(pages, & &1[:page])}) end {limit, _page} = pagination session_result = breakdown_sessions(site, new_query, "visit:entry_page", session_metrics, {limit, 1}) - |> transform_keys(%{entry_page: "page"}) + |> transform_keys(%{entry_page: :page}) zip_results( event_result, session_result, - "event:page", + :page, metrics ) end @@ -154,10 +154,14 @@ defmodule Plausible.Stats.Breakdown do sort_by = if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics) property = - property - |> String.trim_leading("event:") - |> String.trim_leading("visit:") - |> String.trim_leading("props:") + if is_binary(property) do + property + |> String.trim_leading("event:") + |> String.trim_leading("visit:") + |> String.trim_leading("props:") + else + property + end null_row = Enum.map(metrics, fn metric -> {metric, nil} end) |> Enum.into(%{}) @@ -185,7 +189,7 @@ defmodule Plausible.Stats.Breakdown do |> filter_converted_sessions(site, query) |> do_group_by(property) |> select_session_metrics(metrics) - |> merge_imported(site, query, property) + |> merge_imported(site, query, property, metrics) |> apply_pagination(pagination) |> ClickhouseRepo.all() end @@ -199,7 +203,7 @@ defmodule Plausible.Stats.Breakdown do ) |> do_group_by(property) |> select_event_metrics(metrics) - # |> merge_imported(site, query, property) + |> merge_imported(site, query, property, metrics) |> apply_pagination(pagination) |> ClickhouseRepo.all() end @@ -296,7 +300,7 @@ defmodule Plausible.Stats.Breakdown do from( e in q, group_by: e.pathname, - select_merge: %{"page" => e.pathname} + select_merge: %{page: e.pathname} ) end diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index 69e2a332e6a9..76dd9eb700e3 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -98,7 +98,7 @@ defmodule Plausible.Stats.Clickhouse do defp filter_converted_sessions(db_query, site, query) do goal = query.filters["goal"] - page = query.filters["page"] + page = query.filters[:page] if is_binary(goal) || is_binary(page) do converted_sessions = @@ -116,7 +116,7 @@ defmodule Plausible.Stats.Clickhouse do end defp apply_page_as_entry_page(db_query, _site, query) do - include_path_filter_entry(db_query, query.filters["page"]) + include_path_filter_entry(db_query, query.filters[:page]) end def current_visitors(site, query) do @@ -382,7 +382,7 @@ defmodule Plausible.Stats.Clickhouse do q end - q = include_path_filter(q, query.filters["page"]) + q = include_path_filter(q, query.filters[:page]) if query.filters["props"] do [{key, val}] = query.filters["props"] |> Enum.into([]) diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex index 569a3ceeee5f..8d97012cd59a 100644 --- a/lib/plausible/stats/filter_suggestions.ex +++ b/lib/plausible/stats/filter_suggestions.ex @@ -131,13 +131,13 @@ defmodule Plausible.Stats.FilterSuggestions do filter_search = if filter_search == nil, do: "", else: filter_search filter_query = - if Enum.member?(["entry_page", "page", "exit_page"], filter_name), + if Enum.member?(["entry_page", :page, "exit_page"], filter_name), do: "%#{String.replace(filter_search, "*", "")}%", else: "%#{filter_search}%" filter_name = case filter_name do - "page" -> "pathname" + :page -> "pathname" "source" -> "referrer_source" "os" -> "operating_system" "os_version" -> "operating_system_version" diff --git a/lib/plausible/stats/filters.ex b/lib/plausible/stats/filters.ex index 2091fe88d934..8678913d5e9f 100644 --- a/lib/plausible/stats/filters.ex +++ b/lib/plausible/stats/filters.ex @@ -22,7 +22,7 @@ defmodule Plausible.Stats.Filters do @event_props [ "name", - "page" + :page ] def visit_props() do diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 72d10b3a26c7..c7d1e2d39872 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -258,8 +258,8 @@ defmodule Plausible.Stats.Query do defp parse_goal_filter(event), do: {:is, :event, event} defp include_imported(default, params) do - (default && - params["filters"] == "{}" && - params["with_imported"] == "true") + default && + params["filters"] == "{}" && + params["with_imported"] == "true" end end diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index cef1072e0b5e..44d5bdf36eee 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -4,7 +4,7 @@ defmodule Plausible.Stats.Timeseries do import Plausible.Stats.Base use Plausible.Stats.Fragments - @event_metrics [:visitors, "pageviews"] + @event_metrics [:visitors, :pageviews] @session_metrics [:visits, :bounce_rate, :visit_duration] def timeseries(site, query, metrics) do steps = buckets(query) @@ -118,7 +118,7 @@ defmodule Plausible.Stats.Timeseries do defp empty_row(date, metrics) do Enum.reduce(metrics, %{"date" => date}, fn metric, row -> case metric do - "pageviews" -> Map.merge(row, %{"pageviews" => 0}) + :pageviews -> Map.merge(row, %{pageviews: 0}) :visitors -> Map.merge(row, %{visitors: 0}) :visits -> Map.merge(row, %{visits: 0}) :bounce_rate -> Map.merge(row, %{bounce_rate: nil}) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 1c415f1550c1..b818490bfa48 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -28,7 +28,7 @@ defmodule PlausibleWeb.Api.StatsController do plot = if query.with_imported && site.has_imported_stats do Imported.Visitors.timeseries(site, timeseries_query) - |> Enum.zip_with(plot, & &1 + &2) + |> Enum.zip_with(plot, &(&1 + &2)) else plot end @@ -77,9 +77,9 @@ defmodule PlausibleWeb.Api.StatsController do query_30m = %Query{query | period: "30m"} %{ - :visitors => %{"value" => visitors}, - "pageviews" => %{"value" => pageviews} - } = Stats.aggregate(site, query_30m, [:visitors, "pageviews"]) + visitors: %{"value" => visitors}, + pageviews: %{"value" => pageviews} + } = Stats.aggregate(site, query_30m, [:visitors, :pageviews]) stats = [ %{ @@ -156,9 +156,9 @@ defmodule PlausibleWeb.Api.StatsController do metrics = if query.filters["event:page"] do - [:visitors, "pageviews", :bounce_rate, "time_on_page", "sample_percent"] + [:visitors, :pageviews, :bounce_rate, "time_on_page", "sample_percent"] else - [:visitors, "pageviews", :bounce_rate, :visit_duration, "sample_percent"] + [:visitors, :pageviews, :bounce_rate, :visit_duration, "sample_percent"] end current_results = Stats.aggregate(site, query, metrics) @@ -167,7 +167,7 @@ defmodule PlausibleWeb.Api.StatsController do stats = [ top_stats_entry(current_results, prev_results, "Unique visitors", :visitors), - top_stats_entry(current_results, prev_results, "Total pageviews", "pageviews"), + top_stats_entry(current_results, prev_results, "Total pageviews", :pageviews), top_stats_entry(current_results, prev_results, "Bounce rate", :bounce_rate), top_stats_entry(current_results, prev_results, "Visit duration", :visit_duration), top_stats_entry(current_results, prev_results, "Time on page", "time_on_page") @@ -448,15 +448,15 @@ defmodule PlausibleWeb.Api.StatsController do metrics = if params["detailed"], - do: [:visitors, "pageviews", :bounce_rate, "time_on_page"], + do: [:visitors, :pageviews, :bounce_rate, "time_on_page"], else: [:visitors] pagination = parse_pagination(params) pages = Stats.breakdown(site, query, "event:page", metrics, pagination) - |> maybe_add_cr(site, query, pagination, "page", "event:page") - |> transform_keys(%{"page" => "name", visitors: "visitors"}) + |> maybe_add_cr(site, query, pagination, :page, "event:page") + |> transform_keys(%{page: "name", visitors: "visitors"}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do @@ -524,13 +524,13 @@ defmodule PlausibleWeb.Api.StatsController do |> Query.put_filter("visit:page", query.filters["event:page"]) total_pageviews = - Stats.breakdown(site, total_visits_query, "event:page", ["pageviews"], {limit, 1}) + Stats.breakdown(site, total_visits_query, "event:page", [:pageviews], {limit, 1}) exit_pages = Enum.map(exit_pages, fn exit_page -> exit_rate = - case Enum.find(total_pageviews, &(&1["page"] == exit_page["name"])) do - %{"pageviews" => pageviews} -> + case Enum.find(total_pageviews, &(&1[:page] == exit_page["name"])) do + %{pageviews: pageviews} -> Float.floor(exit_page["total_exits"] / pageviews * 100) nil -> diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index 581044e8596b..b3f9e386fd45 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -50,7 +50,7 @@ defmodule PlausibleWeb.StatsController do site = conn.assigns[:site] query = Query.from(site.timezone, params) |> Filters.add_prefix() - metrics = [:visitors, "pageviews", :bounce_rate, :visit_duration] + metrics = [:visitors, :pageviews, :bounce_rate, :visit_duration] graph = Plausible.Stats.timeseries(site, query, metrics) headers = ["date" | metrics] From 21fcadaf6b527d43d9d35eb8ff319e2fa14c307c Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 9 Dec 2021 16:39:53 +0300 Subject: [PATCH 050/148] mix format --- lib/plausible/imported/site.ex | 6 +++--- lib/plausible/imported/sources.ex | 2 +- lib/plausible_web/controllers/site_controller.ex | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 5d4ae93f5b9e..d89f5389dac8 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -182,7 +182,7 @@ defmodule Plausible.Imported do "Android Browser" => "Mobile App", "Safari (in-app)" => "Mobile App", "User-Agent: Mozilla" => "Firefox", - "(not set)" => "", + "(not set)" => "" } defp new_from_google_analytics(domain, "browsers", %{ @@ -194,7 +194,7 @@ defmodule Plausible.Imported do Imported.Browsers.new(%{ domain: domain, timestamp: format_timestamp(timestamp), - browser: Map.get(@browser_google_to_plausible, browser, browser) + browser: Map.get(@browser_google_to_plausible, browser, browser), visitors: visitors }) end @@ -202,7 +202,7 @@ defmodule Plausible.Imported do @os_google_to_plausible %{ "Macintosh" => "Mac", "Linux" => "GNU/Linux", - "(not set)" => "", + "(not set)" => "" } defp new_from_google_analytics(domain, "operating_systems", %{ diff --git a/lib/plausible/imported/sources.ex b/lib/plausible/imported/sources.ex index f3459ff612c8..95b33166eadd 100644 --- a/lib/plausible/imported/sources.ex +++ b/lib/plausible/imported/sources.ex @@ -33,7 +33,7 @@ defmodule Plausible.Imported.Sources do @search_engines %{ "google" => "Google", "bing" => "Bing", - "duckduckgo" => "DuckDuckGo", + "duckduckgo" => "DuckDuckGo" } def parse(nil), do: nil diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 3d40d83c633b..f7a1046007ff 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -313,6 +313,7 @@ defmodule PlausibleWeb.SiteController do case panel do "search" -> redirect(conn, to: Routes.site_path(conn, :settings_search_console, site.domain)) + "import" -> redirect(conn, to: Routes.site_path(conn, :settings_general, site.domain)) end From a7da3466df37a5c3ffb6a69f1ef9288ee1df3c97 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 10 Dec 2021 11:49:10 +0300 Subject: [PATCH 051/148] Fetch time_on_pages for imported data when needed --- lib/plausible/stats/base.ex | 4 --- lib/plausible/stats/breakdown.ex | 45 ++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index cd3472c809f9..4a49b6d27903 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -511,10 +511,6 @@ defmodule Plausible.Stats.Base do q |> select_merge([i, s], %{ page: fragment("if(empty(?), ?, ?)", i.page, s.page, i.page) - # time_on_page: fragment( - # "coalesce(?, 0) + coalesce(?, 0) * coalesce(?, 0)", - # i.time_on_page, s.pageviews, s.time_on_page - # ), }) if :pageviews in metrics do diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index cceee094fabc..4fb5e1a98fb6 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -229,10 +229,18 @@ defmodule Plausible.Stats.Breakdown do {base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q) + with_imported = query.with_imported && site.has_imported_stats + select = + if with_imported do + "sum(td), count(case when p2 != p then 1 end)" + else + "round(sum(td)/count(case when p2 != p then 1 end))" + end + time_query = " SELECT p, - round(sum(td)/count(case when p2 != p then 1 end)) as avgTime + #{select} FROM (SELECT p, @@ -250,7 +258,40 @@ defmodule Plausible.Stats.Breakdown do GROUP BY p" {:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [pages]) - res.rows |> Enum.map(fn [page, time] -> {page, time} end) |> Enum.into(%{}) + + if with_imported do + # Imported page views have pre-calculated values + {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) + + res = + res.rows + |> Enum.map(fn [page, time, visits] -> {page, {time, visits}} end) + |> Enum.into(%{}) + + from( + i in "imported_pages", + group_by: i.page, + where: i.domain == ^site.domain, + where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, + where: i.page in ^pages, + select: %{ + page: i.page, + pageviews: sum(i.pageviews), + time_on_page: sum(i.time_on_page), + } + ) + |> ClickhouseRepo.all() + |> Enum.reduce(res, fn %{page: page, pageviews: pageviews, time_on_page: time}, res -> + {restime, resviews} = Map.get(res, page, {0, 0}) + Map.put(res, page, {restime + time, resviews + pageviews}) + end) + |> Enum.map(fn {page, {time, pageviews}} -> + {page, time / pageviews} + end) + |> Enum.into(%{}) + else + res.rows |> Enum.map(fn [page, time] -> {page, time} end) |> Enum.into(%{}) + end end defp do_group_by( From 6ca72a220430726438795c9027d30c792c542d78 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 14 Dec 2021 12:06:08 +0300 Subject: [PATCH 052/148] entry pages need to fetch bounces from GA --- lib/plausible/google/api.ex | 2 +- lib/plausible/imported/entry_pages.ex | 1 + lib/plausible/imported/site.ex | 6 ++++-- .../20211129111622_create_imported_entry_pages.exs | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index bf2f12d7d545..76054269e503 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -211,7 +211,7 @@ defmodule Plausible.Google.Api do # Entry pages { ["ga:date", "ga:landingPagePath"], - ["ga:users", "ga:entrances", "ga:sessionDuration"] + ["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"] }, # Exit pages { diff --git a/lib/plausible/imported/entry_pages.ex b/lib/plausible/imported/entry_pages.ex index 4bc4a4aa3d2d..55ed02183b0e 100644 --- a/lib/plausible/imported/entry_pages.ex +++ b/lib/plausible/imported/entry_pages.ex @@ -10,6 +10,7 @@ defmodule Plausible.Imported.EntryPages do field :entry_page, :string field :visitors, :integer field :entrances, :integer + field :bounces, :integer # Sum total field :visit_duration, :integer end diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index d89f5389dac8..4130864ac7f6 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -112,10 +112,11 @@ defmodule Plausible.Imported do defp new_from_google_analytics(domain, "entry_pages", %{ "dimensions" => [timestamp, entry_page], - "metrics" => [%{"values" => [visitors, entrances, visit_duration]}] + "metrics" => [%{"values" => [visitors, entrances, visit_duration, bounces]}] }) do {visitors, ""} = Integer.parse(visitors) {entrances, ""} = Integer.parse(entrances) + {bounces, ""} = Integer.parse(bounces) {visit_duration, _} = Integer.parse(visit_duration) @@ -125,7 +126,8 @@ defmodule Plausible.Imported do entry_page: entry_page, visitors: visitors, entrances: entrances, - visit_duration: visit_duration + visit_duration: visit_duration, + bounces: bounces }) end diff --git a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs index 35b203199247..0643b2f11c8c 100644 --- a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs @@ -9,6 +9,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedEntryPages do add :visitors, :UInt64 add :entrances, :UInt64 add :visit_duration, :UInt64 + add :bounces, :UInt32 end end end From 93ddbabebb1f18eb4d770fc2f767ae7825b442d9 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 14 Dec 2021 12:08:40 +0300 Subject: [PATCH 053/148] "sample_percent" -> :sample_percent as only atoms can be used in subqueries --- lib/plausible/stats/aggregate.ex | 4 ++-- lib/plausible/stats/base.ex | 8 ++++---- lib/plausible_web/controllers/api/stats_controller.ex | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 7f699f0e616a..e8a751d289d0 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -3,8 +3,8 @@ defmodule Plausible.Stats.Aggregate do use Plausible.ClickhouseRepo import Plausible.Stats.Base - @event_metrics [:visitors, :pageviews, "events", "sample_percent"] - @session_metrics [:visits, :bounce_rate, :visit_duration, "sample_percent"] + @event_metrics [:visitors, :pageviews, "events", :sample_percent] + @session_metrics [:visits, :bounce_rate, :visit_duration, :sample_percent] def aggregate(site, query, metrics) do event_metrics = Enum.filter(metrics, &(&1 in @event_metrics)) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 4a49b6d27903..e5e6678718ba 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -223,10 +223,10 @@ defmodule Plausible.Stats.Base do |> select_event_metrics(rest) end - def select_event_metrics(q, ["sample_percent" | rest]) do + def select_event_metrics(q, [:sample_percent | rest]) do from(e in q, select_merge: %{ - "sample_percent" => + sample_percent: fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") } ) @@ -285,10 +285,10 @@ defmodule Plausible.Stats.Base do |> select_session_metrics(rest) end - def select_session_metrics(q, ["sample_percent" | rest]) do + def select_session_metrics(q, [:sample_percent | rest]) do from(e in q, select_merge: %{ - "sample_percent" => + sample_percent: fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") } ) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index b818490bfa48..b5c971d0ac36 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -156,9 +156,9 @@ defmodule PlausibleWeb.Api.StatsController do metrics = if query.filters["event:page"] do - [:visitors, :pageviews, :bounce_rate, "time_on_page", "sample_percent"] + [:visitors, :pageviews, :bounce_rate, "time_on_page", :sample_percent] else - [:visitors, :pageviews, :bounce_rate, :visit_duration, "sample_percent"] + [:visitors, :pageviews, :bounce_rate, :visit_duration, :sample_percent] end current_results = Stats.aggregate(site, query, metrics) @@ -174,7 +174,7 @@ defmodule PlausibleWeb.Api.StatsController do ] |> Enum.filter(& &1) - {stats, current_results["sample_percent"]["value"]} + {stats, current_results[:sample_percent]["value"]} end defp top_stats_entry(current_results, prev_results, name, key) do From 67fd47a4eb2cc53ab37813b825d7b51f5912e455 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 14 Dec 2021 12:27:23 +0300 Subject: [PATCH 054/148] Calculate bounce_rate for joined native and imported data for top pages modal --- lib/plausible/stats/base.ex | 54 +++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index e5e6678718ba..8331006c1778 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -241,7 +241,8 @@ defmodule Plausible.Stats.Base do from(s in q, select_merge: %{ bounce_rate: - fragment("toUInt32(ifNotFinite(round(sum(is_bounce * sign) / sum(sign) * 100), 0))") + fragment("toUInt32(ifNotFinite(round(sum(is_bounce * sign) / sum(sign) * 100), 0))"), + visits: fragment("toUInt32(sum(sign))") } ) |> select_session_metrics(rest) @@ -438,12 +439,19 @@ defmodule Plausible.Stats.Base do }) :entry_page -> - imported_q - |> select_merge([i], %{ - entry_page: i.entry_page, - visits: sum(i.entrances), - visit_duration: sum(i.visit_duration) - }) + imported_q = + imported_q + |> select_merge([i], %{ + entry_page: i.entry_page, + visits: sum(i.entrances), + visit_duration: sum(i.visit_duration) + }) + + if :bounce_rate in metrics do + select_merge(imported_q, [i], %{bounces: sum(i.bounces)}) + else + imported_q + end :exit_page -> imported_q @@ -523,20 +531,26 @@ defmodule Plausible.Stats.Base do end :entry_page -> - q - |> select_merge([i, s], %{ - entry_page: fragment("if(empty(?), ?, ?)", i.entry_page, s.entry_page, i.entry_page), - visits: fragment("? + ?", s.visits, i.visits), - visit_duration: - fragment( - "(? + ? * ?) / (? + ?)", - i.visit_duration, - s.visit_duration, - s.visits, - s.visits, - i.visits + # TODO: Reverse-engineering the native data bounces to combine with imported data is inefficient. + # Instead both queries should fetch bounces and visits and be used as subqueries to a main query + # that calculates bounce_rate. + q = + q + |> select_merge([s, i], %{ + entry_page: fragment("if(empty(?), ?, ?)", i.entry_page, s.entry_page, i.entry_page), + visits: fragment("? + ?", s.visits, i.visits) + }) + + if :bounce_rate in metrics do + select_merge(q, [s, i], %{ + bounce_rate: fragment( + "round(100 * (coalesce(?, 0) + coalesce((? * ? / 100), 0)) / (coalesce(?, 0) + coalesce(?, 0)))", + i.bounces, s.bounce_rate, s.visits, i.visits, s.visits ) - }) + }) + else + q + end :exit_page -> q From 2158098f06d77366bcc72d0138c7007d68271164 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 14 Dec 2021 12:27:46 +0300 Subject: [PATCH 055/148] Flip some query bindings around to be less misleading --- lib/plausible/stats/base.ex | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 8331006c1778..b1b080949a02 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -479,10 +479,10 @@ defmodule Plausible.Stats.Base do q = if :visitors in metrics do q - |> select_merge([i, s], %{ + |> select_merge([s, i], %{ :visitors => fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) }) - |> order_by([i, s], + |> order_by([s, i], desc: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) ) else @@ -492,38 +492,38 @@ defmodule Plausible.Stats.Base do case dim do :source -> q - |> select_merge([i, s], %{ + |> select_merge([s, i], %{ source: fragment("if(empty(?), ?, ?)", s.source, i.source, s.source) }) :utm_medium -> q - |> select_merge([i, s], %{ + |> select_merge([s, i], %{ utm_medium: fragment("if(empty(?), ?, ?)", s.utm_medium, i.utm_medium, s.utm_medium) }) :utm_source -> q - |> select_merge([i, s], %{ + |> select_merge([s, i], %{ utm_source: fragment("if(empty(?), ?, ?)", s.utm_source, i.utm_source, s.utm_source) }) :utm_campaign -> q - |> select_merge([i, s], %{ + |> select_merge([s, i], %{ utm_campaign: fragment("if(empty(?), ?, ?)", s.utm_campaign, i.utm_campaign, s.utm_campaign) }) :page -> q - |> select_merge([i, s], %{ + |> select_merge([s, i], %{ page: fragment("if(empty(?), ?, ?)", i.page, s.page, i.page) }) if :pageviews in metrics do q - |> select_merge([i, s], %{ + |> select_merge([s, i], %{ pageviews: fragment("coalesce(?, 0) + coalesce(?, 0)", s.pageviews, i.pageviews) }) else @@ -554,7 +554,7 @@ defmodule Plausible.Stats.Base do :exit_page -> q - |> select_merge([i, s], %{ + |> select_merge([s, i], %{ exit_page: fragment("if(empty(?), ?, ?)", i.exit_page, s.exit_page, i.exit_page), visits: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visits, i.visits) }) From 808326118a502bacfb43485ec04fa73827439978 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 14 Dec 2021 13:12:35 +0300 Subject: [PATCH 056/148] Fixup entry page modal visit durations --- lib/plausible/stats/base.ex | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index b1b080949a02..63912404c8a8 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -443,10 +443,16 @@ defmodule Plausible.Stats.Base do imported_q |> select_merge([i], %{ entry_page: i.entry_page, - visits: sum(i.entrances), - visit_duration: sum(i.visit_duration) + visits: sum(i.entrances) }) + imported_q = + if :visit_duration in metrics do + select_merge(imported_q, [i], %{visit_duration: sum(i.visit_duration)}) + else + imported_q + end + if :bounce_rate in metrics do select_merge(imported_q, [i], %{bounces: sum(i.bounces)}) else @@ -531,9 +537,11 @@ defmodule Plausible.Stats.Base do end :entry_page -> - # TODO: Reverse-engineering the native data bounces to combine with imported data is inefficient. - # Instead both queries should fetch bounces and visits and be used as subqueries to a main query - # that calculates bounce_rate. + # TODO: Reverse-engineering the native data bounces and total visit + # durations to combine with imported data is inefficient. Instead both + # queries should fetch bounces/total_visit_duration and visits and be + # used as subqueries to a main query that then find the bounce rate/avg + # visit_duration. q = q |> select_merge([s, i], %{ @@ -541,6 +549,22 @@ defmodule Plausible.Stats.Base do visits: fragment("? + ?", s.visits, i.visits) }) + q = + if :visit_duration in metrics do + select_merge(q, [s, i], %{ + visit_duration: fragment( + "(? + ? * ?) / (? + ?)", + i.visit_duration, + s.visit_duration, + s.visits, + s.visits, + i.visits + ) + }) + else + q + end + if :bounce_rate in metrics do select_merge(q, [s, i], %{ bounce_rate: fragment( From 2881946ccb4548ee6a299591ea1d0f888d0699c5 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 14 Dec 2021 13:13:04 +0300 Subject: [PATCH 057/148] mix format --- lib/plausible/stats/base.ex | 30 ++++++++++++++++++------------ lib/plausible/stats/breakdown.ex | 3 ++- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 63912404c8a8..f4af20226852 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -552,14 +552,15 @@ defmodule Plausible.Stats.Base do q = if :visit_duration in metrics do select_merge(q, [s, i], %{ - visit_duration: fragment( - "(? + ? * ?) / (? + ?)", - i.visit_duration, - s.visit_duration, - s.visits, - s.visits, - i.visits - ) + visit_duration: + fragment( + "(? + ? * ?) / (? + ?)", + i.visit_duration, + s.visit_duration, + s.visits, + s.visits, + i.visits + ) }) else q @@ -567,10 +568,15 @@ defmodule Plausible.Stats.Base do if :bounce_rate in metrics do select_merge(q, [s, i], %{ - bounce_rate: fragment( - "round(100 * (coalesce(?, 0) + coalesce((? * ? / 100), 0)) / (coalesce(?, 0) + coalesce(?, 0)))", - i.bounces, s.bounce_rate, s.visits, i.visits, s.visits - ) + bounce_rate: + fragment( + "round(100 * (coalesce(?, 0) + coalesce((? * ? / 100), 0)) / (coalesce(?, 0) + coalesce(?, 0)))", + i.bounces, + s.bounce_rate, + s.visits, + i.visits, + s.visits + ) }) else q diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 4fb5e1a98fb6..b3f569bd0a85 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -230,6 +230,7 @@ defmodule Plausible.Stats.Breakdown do {base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q) with_imported = query.with_imported && site.has_imported_stats + select = if with_imported do "sum(td), count(case when p2 != p then 1 end)" @@ -277,7 +278,7 @@ defmodule Plausible.Stats.Breakdown do select: %{ page: i.page, pageviews: sum(i.pageviews), - time_on_page: sum(i.time_on_page), + time_on_page: sum(i.time_on_page) } ) |> ClickhouseRepo.all() From d3d9a5c31f39ac22fc53bf52e3fd70ce590110c8 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 15 Dec 2021 13:10:51 +0300 Subject: [PATCH 058/148] Fetch bounces and visit_duration for sources from GA --- lib/plausible/google/api.ex | 6 ++--- lib/plausible/imported/site.ex | 30 +++++++++++++++++-------- lib/plausible/imported/sources.ex | 3 +++ lib/plausible/imported/utm_campaigns.ex | 3 +++ lib/plausible/imported/utm_mediums.ex | 3 +++ lib/plausible/imported/utm_sources.ex | 3 +++ 6 files changed, 36 insertions(+), 12 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 76054269e503..b2ff7f844c31 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -191,17 +191,17 @@ defmodule Plausible.Google.Api do # Sources { ["ga:date", "ga:fullReferrer"], - ["ga:users"] + ["ga:users", "ga:bounces", "ga:sessionDuration"] }, # UTM Mediums { ["ga:date", "ga:medium"], - ["ga:users"] + ["ga:users", "ga:bounces", "ga:sessionDuration"] }, # UTM Campaigns { ["ga:date", "ga:campaign"], - ["ga:users"] + ["ga:users", "ga:bounces", "ga:sessionDuration"] }, # Pages { diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 4130864ac7f6..30732bb51ab9 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -44,9 +44,11 @@ defmodule Plausible.Imported do defp new_from_google_analytics(domain, "sources", %{ "dimensions" => [timestamp, source], - "metrics" => [%{"values" => [value]}] + "metrics" => [%{"values" => [visitors, bounces, visit_duration]}] }) do - {visitors, ""} = Integer.parse(value) + {visitors, ""} = Integer.parse(visitors) + {bounces, ""} = Integer.parse(bounces) + {visit_duration, _} = Integer.parse(visit_duration) source = if source == "(direct)", do: nil, else: source @@ -54,7 +56,9 @@ defmodule Plausible.Imported do domain: domain, timestamp: format_timestamp(timestamp), source: Imported.Sources.parse(source), - visitors: visitors + visitors: visitors, + bounces: bounces, + visit_duration: visit_duration }) end @@ -62,9 +66,11 @@ defmodule Plausible.Imported do defp new_from_google_analytics(domain, "utm_mediums", %{ "dimensions" => [timestamp, medium], - "metrics" => [%{"values" => [value]}] + "metrics" => [%{"values" => [visitors, bounces, visit_duration]}] }) do - {visitors, ""} = Integer.parse(value) + {visitors, ""} = Integer.parse(visitors) + {bounces, ""} = Integer.parse(bounces) + {visit_duration, _} = Integer.parse(visit_duration) medium = if medium == "(none)", do: "", else: medium @@ -72,15 +78,19 @@ defmodule Plausible.Imported do domain: domain, timestamp: format_timestamp(timestamp), medium: medium, - visitors: visitors + visitors: visitors, + bounces: bounces, + visit_duration: visit_duration }) end defp new_from_google_analytics(domain, "utm_campaigns", %{ "dimensions" => [timestamp, campaign], - "metrics" => [%{"values" => [value]}] + "metrics" => [%{"values" => [visitors, bounces, visit_duration]}] }) do - {visitors, ""} = Integer.parse(value) + {visitors, ""} = Integer.parse(visitors) + {bounces, ""} = Integer.parse(bounces) + {visit_duration, _} = Integer.parse(visit_duration) campaign = if campaign == "(not set)", do: "", else: campaign @@ -88,7 +98,9 @@ defmodule Plausible.Imported do domain: domain, timestamp: format_timestamp(timestamp), campaign: campaign, - visitors: visitors + visitors: visitors, + bounces: bounces, + visit_duration: visit_duration }) end diff --git a/lib/plausible/imported/sources.ex b/lib/plausible/imported/sources.ex index 95b33166eadd..1f0064556306 100644 --- a/lib/plausible/imported/sources.ex +++ b/lib/plausible/imported/sources.ex @@ -9,6 +9,9 @@ defmodule Plausible.Imported.Sources do field :timestamp, :naive_datetime field :source, :string, default: "" field :visitors, :integer + field :bounces, :integer + # Sum total + field :visit_duration, :integer end def new(attrs) do diff --git a/lib/plausible/imported/utm_campaigns.ex b/lib/plausible/imported/utm_campaigns.ex index 1efbf57583af..4e0415c818d2 100644 --- a/lib/plausible/imported/utm_campaigns.ex +++ b/lib/plausible/imported/utm_campaigns.ex @@ -9,6 +9,9 @@ defmodule Plausible.Imported.UtmCampaigns do field :timestamp, :naive_datetime field :utm_campaign, :string, default: "" field :visitors, :integer + field :bounces, :integer + # Sum total + field :visit_duration, :integer end def new(attrs) do diff --git a/lib/plausible/imported/utm_mediums.ex b/lib/plausible/imported/utm_mediums.ex index be2cedf600aa..94823477404c 100644 --- a/lib/plausible/imported/utm_mediums.ex +++ b/lib/plausible/imported/utm_mediums.ex @@ -9,6 +9,9 @@ defmodule Plausible.Imported.UtmMediums do field :timestamp, :naive_datetime field :utm_medium, :string, default: "" field :visitors, :integer + field :bounces, :integer + # Sum total + field :visit_duration, :integer end def new(attrs) do diff --git a/lib/plausible/imported/utm_sources.ex b/lib/plausible/imported/utm_sources.ex index 226aad2322d4..b6b2226c0a6e 100644 --- a/lib/plausible/imported/utm_sources.ex +++ b/lib/plausible/imported/utm_sources.ex @@ -9,6 +9,9 @@ defmodule Plausible.Imported.UtmSources do field :timestamp, :naive_datetime field :utm_source, :string, default: "" field :visitors, :integer + field :bounces, :integer + # Sum total + field :visit_duration, :integer end def new(attrs) do From 46fb1877a77ce5dbffb107a609660f4ec361eaa5 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 16 Dec 2021 14:05:06 +0300 Subject: [PATCH 059/148] add more source metrics used for data in modals --- lib/plausible/google/api.ex | 6 +++--- lib/plausible/imported/entry_pages.ex | 2 ++ lib/plausible/imported/site.ex | 12 +++++++++--- lib/plausible/imported/sources.ex | 11 +++++++++-- lib/plausible/imported/utm_campaigns.ex | 11 +++++++++-- lib/plausible/imported/utm_mediums.ex | 11 +++++++++-- lib/plausible/imported/utm_sources.ex | 11 +++++++++-- .../20211118160420_create_imported_sources.exs | 3 +++ .../20211124135248_create_imported_utm_mediums.exs | 3 +++ .../20211124135252_create_imported_utm_campaigns.exs | 3 +++ .../20211129164103_create_imported_utm_sources.exs | 3 +++ 11 files changed, 62 insertions(+), 14 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index b2ff7f844c31..0e1506fc8139 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -191,17 +191,17 @@ defmodule Plausible.Google.Api do # Sources { ["ga:date", "ga:fullReferrer"], - ["ga:users", "ga:bounces", "ga:sessionDuration"] + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, # UTM Mediums { ["ga:date", "ga:medium"], - ["ga:users", "ga:bounces", "ga:sessionDuration"] + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, # UTM Campaigns { ["ga:date", "ga:campaign"], - ["ga:users", "ga:bounces", "ga:sessionDuration"] + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, # Pages { diff --git a/lib/plausible/imported/entry_pages.ex b/lib/plausible/imported/entry_pages.ex index 55ed02183b0e..dae4ab02f321 100644 --- a/lib/plausible/imported/entry_pages.ex +++ b/lib/plausible/imported/entry_pages.ex @@ -25,6 +25,7 @@ defmodule Plausible.Imported.EntryPages do :entry_page, :visitors, :entrances, + :bounces, :visit_duration ], empty_values: [nil, ""] @@ -35,6 +36,7 @@ defmodule Plausible.Imported.EntryPages do :entry_page, :visitors, :entrances, + :bounces, :visit_duration ]) end diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 30732bb51ab9..26c7152b6fbe 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -44,9 +44,10 @@ defmodule Plausible.Imported do defp new_from_google_analytics(domain, "sources", %{ "dimensions" => [timestamp, source], - "metrics" => [%{"values" => [visitors, bounces, visit_duration]}] + "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do {visitors, ""} = Integer.parse(visitors) + {visits, ""} = Integer.parse(visits) {bounces, ""} = Integer.parse(bounces) {visit_duration, _} = Integer.parse(visit_duration) @@ -57,6 +58,7 @@ defmodule Plausible.Imported do timestamp: format_timestamp(timestamp), source: Imported.Sources.parse(source), visitors: visitors, + visits: visits, bounces: bounces, visit_duration: visit_duration }) @@ -66,9 +68,10 @@ defmodule Plausible.Imported do defp new_from_google_analytics(domain, "utm_mediums", %{ "dimensions" => [timestamp, medium], - "metrics" => [%{"values" => [visitors, bounces, visit_duration]}] + "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do {visitors, ""} = Integer.parse(visitors) + {visits, ""} = Integer.parse(visits) {bounces, ""} = Integer.parse(bounces) {visit_duration, _} = Integer.parse(visit_duration) @@ -79,6 +82,7 @@ defmodule Plausible.Imported do timestamp: format_timestamp(timestamp), medium: medium, visitors: visitors, + visits: visits, bounces: bounces, visit_duration: visit_duration }) @@ -86,9 +90,10 @@ defmodule Plausible.Imported do defp new_from_google_analytics(domain, "utm_campaigns", %{ "dimensions" => [timestamp, campaign], - "metrics" => [%{"values" => [visitors, bounces, visit_duration]}] + "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do {visitors, ""} = Integer.parse(visitors) + {visits, ""} = Integer.parse(visits) {bounces, ""} = Integer.parse(bounces) {visit_duration, _} = Integer.parse(visit_duration) @@ -99,6 +104,7 @@ defmodule Plausible.Imported do timestamp: format_timestamp(timestamp), campaign: campaign, visitors: visitors, + visits: visits, bounces: bounces, visit_duration: visit_duration }) diff --git a/lib/plausible/imported/sources.ex b/lib/plausible/imported/sources.ex index 1f0064556306..dbf0103b4de2 100644 --- a/lib/plausible/imported/sources.ex +++ b/lib/plausible/imported/sources.ex @@ -9,6 +9,7 @@ defmodule Plausible.Imported.Sources do field :timestamp, :naive_datetime field :source, :string, default: "" field :visitors, :integer + field :visits, :integer field :bounces, :integer # Sum total field :visit_duration, :integer @@ -22,14 +23,20 @@ defmodule Plausible.Imported.Sources do :domain, :timestamp, :source, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ], empty_values: [nil, ""] ) |> validate_required([ :domain, :timestamp, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ]) end diff --git a/lib/plausible/imported/utm_campaigns.ex b/lib/plausible/imported/utm_campaigns.ex index 4e0415c818d2..b158e575c17b 100644 --- a/lib/plausible/imported/utm_campaigns.ex +++ b/lib/plausible/imported/utm_campaigns.ex @@ -9,6 +9,7 @@ defmodule Plausible.Imported.UtmCampaigns do field :timestamp, :naive_datetime field :utm_campaign, :string, default: "" field :visitors, :integer + field :visits, :integer field :bounces, :integer # Sum total field :visit_duration, :integer @@ -22,14 +23,20 @@ defmodule Plausible.Imported.UtmCampaigns do :domain, :timestamp, :utm_campaign, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ], empty_values: [nil, ""] ) |> validate_required([ :domain, :timestamp, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ]) end end diff --git a/lib/plausible/imported/utm_mediums.ex b/lib/plausible/imported/utm_mediums.ex index 94823477404c..efd119fb17ec 100644 --- a/lib/plausible/imported/utm_mediums.ex +++ b/lib/plausible/imported/utm_mediums.ex @@ -9,6 +9,7 @@ defmodule Plausible.Imported.UtmMediums do field :timestamp, :naive_datetime field :utm_medium, :string, default: "" field :visitors, :integer + field :visits, :integer field :bounces, :integer # Sum total field :visit_duration, :integer @@ -22,14 +23,20 @@ defmodule Plausible.Imported.UtmMediums do :domain, :timestamp, :utm_medium, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ], empty_values: [nil, ""] ) |> validate_required([ :domain, :timestamp, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ]) end end diff --git a/lib/plausible/imported/utm_sources.ex b/lib/plausible/imported/utm_sources.ex index b6b2226c0a6e..4c634944f2b1 100644 --- a/lib/plausible/imported/utm_sources.ex +++ b/lib/plausible/imported/utm_sources.ex @@ -9,6 +9,7 @@ defmodule Plausible.Imported.UtmSources do field :timestamp, :naive_datetime field :utm_source, :string, default: "" field :visitors, :integer + field :visits, :integer field :bounces, :integer # Sum total field :visit_duration, :integer @@ -22,14 +23,20 @@ defmodule Plausible.Imported.UtmSources do :domain, :timestamp, :utm_source, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ], empty_values: [nil, ""] ) |> validate_required([ :domain, :timestamp, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ]) end end diff --git a/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs b/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs index 26da7ee3ca2a..bbb752ff6116 100644 --- a/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs +++ b/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs @@ -7,6 +7,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedSources do add :timestamp, :naive_datetime add :source, :string add :visitors, :UInt64 + add :visits, :UInt64 + add :visit_duration, :UInt64 + add :bounces, :UInt32 end end end diff --git a/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs b/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs index ac77bd54078d..3589a5a2ba58 100644 --- a/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs +++ b/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs @@ -7,6 +7,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmMediums do add :timestamp, :naive_datetime add :utm_medium, :string add :visitors, :UInt64 + add :visits, :UInt64 + add :visit_duration, :UInt64 + add :bounces, :UInt32 end end end diff --git a/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs b/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs index e5dd1c7aa651..10c9122ee68c 100644 --- a/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs +++ b/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs @@ -7,6 +7,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmCampaigns do add :timestamp, :naive_datetime add :utm_campaign, :string add :visitors, :UInt64 + add :visits, :UInt64 + add :visit_duration, :UInt64 + add :bounces, :UInt32 end end end diff --git a/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs b/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs index b029bf0c07a6..e05bac6ebda4 100644 --- a/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs +++ b/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs @@ -7,6 +7,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmSources do add :timestamp, :naive_datetime add :utm_source, :string add :visitors, :UInt64 + add :visits, :UInt64 + add :visit_duration, :UInt64 + add :bounces, :UInt32 end end end From e95e92b914df7cd1d5c788c5b22362595f3eb651 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 16 Dec 2021 14:08:14 +0300 Subject: [PATCH 060/148] Make sources modals display correct values --- lib/plausible/stats/base.ex | 190 ++++++++++++++++++++---------------- 1 file changed, 108 insertions(+), 82 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index f4af20226852..84b3c3baf5f7 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -404,8 +404,6 @@ defmodule Plausible.Stats.Base do imported_q = case query.filters[property] do {:is_not, value} -> - # TODO: make this no_ref is actually being excluded. - # This is needed as the sources panel expects these filtered. value = if value == @no_ref, do: "", else: value where(imported_q, [i], field(i, ^dim) != ^value) @@ -419,16 +417,28 @@ defmodule Plausible.Stats.Base do imported_q = case dim do :source -> - imported_q |> select_merge([i], %{source: i.source}) + imported_q + |> select_merge([i], %{ + source: fragment("if(empty(?), ?, ?)", i.source, @no_ref, i.source) + }) :utm_medium -> - imported_q |> select_merge([i], %{utm_medium: i.utm_medium}) + imported_q + |> select_merge([i], %{ + utm_medium: fragment("if(empty(?), ?, ?)", i.utm_medium, @no_ref, i.utm_medium) + }) :utm_source -> - imported_q |> select_merge([i], %{utm_source: i.utm_source}) + imported_q + |> select_merge([i], %{ + utm_source: fragment("if(empty(?), ?, ?)", i.utm_source, @no_ref, i.utm_source) + }) :utm_campaign -> - imported_q |> select_merge([i], %{utm_campaign: i.utm_campaign}) + imported_q + |> select_merge([i], %{ + utm_campaign: fragment("if(empty(?), ?, ?)", i.utm_campaign, @no_ref, i.utm_campaign) + }) :page -> imported_q @@ -439,25 +449,11 @@ defmodule Plausible.Stats.Base do }) :entry_page -> - imported_q = - imported_q - |> select_merge([i], %{ - entry_page: i.entry_page, - visits: sum(i.entrances) - }) - - imported_q = - if :visit_duration in metrics do - select_merge(imported_q, [i], %{visit_duration: sum(i.visit_duration)}) - else - imported_q - end - - if :bounce_rate in metrics do - select_merge(imported_q, [i], %{bounces: sum(i.bounces)}) - else - imported_q - end + imported_q + |> select_merge([i], %{ + entry_page: i.entry_page, + visits: sum(i.entrances) + }) :exit_page -> imported_q @@ -476,24 +472,16 @@ defmodule Plausible.Stats.Base do imported_q |> select_merge([i], %{operating_system: i.operating_system}) end + imported_q = + imported_q + |> select_imported_metrics(metrics) + q = from(s in Ecto.Query.subquery(q), full_join: i in subquery(imported_q), on: field(s, ^dim) == field(i, ^dim) ) - - q = - if :visitors in metrics do - q - |> select_merge([s, i], %{ - :visitors => fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) - }) - |> order_by([s, i], - desc: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) - ) - else - q - end + |> select_joined_metrics(metrics) case dim do :source -> @@ -537,50 +525,11 @@ defmodule Plausible.Stats.Base do end :entry_page -> - # TODO: Reverse-engineering the native data bounces and total visit - # durations to combine with imported data is inefficient. Instead both - # queries should fetch bounces/total_visit_duration and visits and be - # used as subqueries to a main query that then find the bounce rate/avg - # visit_duration. - q = - q - |> select_merge([s, i], %{ - entry_page: fragment("if(empty(?), ?, ?)", i.entry_page, s.entry_page, i.entry_page), - visits: fragment("? + ?", s.visits, i.visits) - }) - - q = - if :visit_duration in metrics do - select_merge(q, [s, i], %{ - visit_duration: - fragment( - "(? + ? * ?) / (? + ?)", - i.visit_duration, - s.visit_duration, - s.visits, - s.visits, - i.visits - ) - }) - else - q - end - - if :bounce_rate in metrics do - select_merge(q, [s, i], %{ - bounce_rate: - fragment( - "round(100 * (coalesce(?, 0) + coalesce((? * ? / 100), 0)) / (coalesce(?, 0) + coalesce(?, 0)))", - i.bounces, - s.bounce_rate, - s.visits, - i.visits, - s.visits - ) - }) - else - q - end + q + |> select_merge([s, i], %{ + entry_page: fragment("if(empty(?), ?, ?)", i.entry_page, s.entry_page, i.entry_page), + visits: fragment("? + ?", s.visits, i.visits) + }) :exit_page -> q @@ -623,6 +572,83 @@ defmodule Plausible.Stats.Base do def merge_imported(q, _, _, _, _), do: q + def select_imported_metrics(q, []), do: q + + def select_imported_metrics(q, [:bounce_rate | rest]) do + q + |> select_merge([i], %{ + bounces: sum(i.bounces), + visits: sum(i.visits) + }) + |> select_imported_metrics(rest) + end + + def select_imported_metrics(q, [:visit_duration | rest]) do + q + |> select_merge([i], %{visit_duration: sum(i.visit_duration)}) + |> select_imported_metrics(rest) + end + + def select_imported_metrics(q, [_ | rest]) do + q + |> select_imported_metrics(rest) + end + + def select_joined_metrics(q, []), do: q + # TODO: Reverse-engineering the native data bounces and total visit + # durations to combine with imported data is inefficient. Instead both + # queries should fetch bounces/total_visit_duration and visits and be + # used as subqueries to a main query that then find the bounce rate/avg + # visit_duration. + + def select_joined_metrics(q, [:visitors | rest]) do + q + |> select_merge([s, i], %{ + :visitors => fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) + }) + |> order_by([s, i], + desc: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) + ) + |> select_joined_metrics(rest) + end + + def select_joined_metrics(q, [:bounce_rate | rest]) do + q + |> select_merge([s, i], %{ + bounce_rate: + fragment( + "round(100 * (coalesce(?, 0) + coalesce((? * ? / 100), 0)) / (coalesce(?, 0) + coalesce(?, 0)))", + i.bounces, + s.bounce_rate, + s.visits, + i.visits, + s.visits + ) + }) + |> select_joined_metrics(rest) + end + + def select_joined_metrics(q, [:visit_duration | rest]) do + q + |> select_merge([s, i], %{ + visit_duration: + fragment( + "(? + ? * ?) / (? + ?)", + i.visit_duration, + s.visit_duration, + s.visits, + s.visits, + i.visits + ) + }) + |> select_joined_metrics(rest) + end + + def select_joined_metrics(q, [_ | rest]) do + q + |> select_joined_metrics(rest) + end + defp add_sample_hint(db_q, query) do case query.sample_threshold do "infinite" -> From e78c2e52d56c85c058b3809567fa2a9f9910ccd4 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 16 Dec 2021 14:32:43 +0300 Subject: [PATCH 061/148] imported_visitors: bounce_rate -> bounces, avg_visit_duration -> visit_duration --- lib/plausible/google/api.ex | 4 ++-- lib/plausible/imported/site.ex | 6 +++--- lib/plausible/imported/visitors.ex | 13 +++++++------ .../20211112130238_create_imported_visitors.exs | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 0e1506fc8139..6fb436ff4f57 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -184,8 +184,8 @@ defmodule Plausible.Google.Api do [ "ga:users", "ga:pageviews", - "ga:bounceRate", - "ga:avgSessionDuration" + "ga:bounces", + "ga:sessionDuration" ] }, # Sources diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 26c7152b6fbe..a9b883ff8603 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -27,7 +27,7 @@ defmodule Plausible.Imported do "dimensions" => [timestamp], "metrics" => [%{"values" => values}] }) do - [visitors, pageviews, bounce_rate, avg_session_duration] = + [visitors, pageviews, bounces, visit_duration] = values |> Enum.map(&Integer.parse/1) |> Enum.map(&elem(&1, 0)) @@ -37,8 +37,8 @@ defmodule Plausible.Imported do timestamp: format_timestamp(timestamp), visitors: visitors, pageviews: pageviews, - bounce_rate: bounce_rate, - avg_visit_duration: avg_session_duration + bounces: bounces, + visit_duration: visit_duration }) end diff --git a/lib/plausible/imported/visitors.ex b/lib/plausible/imported/visitors.ex index f12da891f58a..54d1cfa3f786 100644 --- a/lib/plausible/imported/visitors.ex +++ b/lib/plausible/imported/visitors.ex @@ -10,8 +10,9 @@ defmodule Plausible.Imported.Visitors do field :timestamp, :naive_datetime field :visitors, :integer field :pageviews, :integer - field :bounce_rate, :integer - field :avg_visit_duration, :integer + field :bounces, :integer + # Sum total + field :visit_duration, :integer end def new(attrs) do @@ -23,8 +24,8 @@ defmodule Plausible.Imported.Visitors do :timestamp, :visitors, :pageviews, - :bounce_rate, - :avg_visit_duration + :bounces, + :visit_duration ], empty_values: [nil, ""] ) @@ -33,8 +34,8 @@ defmodule Plausible.Imported.Visitors do :timestamp, :visitors, :pageviews, - :bounce_rate, - :avg_visit_duration + :bounces, + :visit_duration ]) end diff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs index dbc12d35027f..fbffd343fe7a 100644 --- a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs +++ b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs @@ -7,8 +7,8 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do add :timestamp, :naive_datetime add :visitors, :UInt64 add :pageviews, :UInt64 - add :bounce_rate, :UInt32 - add :avg_visit_duration, :UInt32 + add :bounces, :UInt32 + add :visit_duration, :UInt64 end end end From 9e3963ad90c535b383275631acecad62da24af02 Mon Sep 17 00:00:00 2001 From: mcol Date: Sun, 19 Dec 2021 01:12:08 +0300 Subject: [PATCH 062/148] Merge imported data into aggregate stats --- lib/plausible/google/api.ex | 1 + lib/plausible/imported/site.ex | 3 +- lib/plausible/imported/visitors.ex | 3 + lib/plausible/stats/aggregate.ex | 2 + lib/plausible/stats/base.ex | 57 ++++++++++++++----- ...0211112130238_create_imported_visitors.exs | 3 +- 6 files changed, 52 insertions(+), 17 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 6fb436ff4f57..8eefe9336037 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -185,6 +185,7 @@ defmodule Plausible.Google.Api do "ga:users", "ga:pageviews", "ga:bounces", + "ga:sessions", "ga:sessionDuration" ] }, diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index a9b883ff8603..1c69215a6264 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -27,7 +27,7 @@ defmodule Plausible.Imported do "dimensions" => [timestamp], "metrics" => [%{"values" => values}] }) do - [visitors, pageviews, bounces, visit_duration] = + [visitors, pageviews, bounces, visits, visit_duration] = values |> Enum.map(&Integer.parse/1) |> Enum.map(&elem(&1, 0)) @@ -38,6 +38,7 @@ defmodule Plausible.Imported do visitors: visitors, pageviews: pageviews, bounces: bounces, + visits: visits, visit_duration: visit_duration }) end diff --git a/lib/plausible/imported/visitors.ex b/lib/plausible/imported/visitors.ex index 54d1cfa3f786..1636b5e2d275 100644 --- a/lib/plausible/imported/visitors.ex +++ b/lib/plausible/imported/visitors.ex @@ -11,6 +11,7 @@ defmodule Plausible.Imported.Visitors do field :visitors, :integer field :pageviews, :integer field :bounces, :integer + field :visits, :integer # Sum total field :visit_duration, :integer end @@ -25,6 +26,7 @@ defmodule Plausible.Imported.Visitors do :visitors, :pageviews, :bounces, + :visits, :visit_duration ], empty_values: [nil, ""] @@ -35,6 +37,7 @@ defmodule Plausible.Imported.Visitors do :visitors, :pageviews, :bounces, + :visits, :visit_duration ]) end diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index e8a751d289d0..8aca952fe16e 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -33,6 +33,7 @@ defmodule Plausible.Stats.Aggregate do defp aggregate_events(site, query, metrics) do from(e in base_event_query(site, query), select: %{}) |> select_event_metrics(metrics) + |> merge_imported(site, query, :aggregate, metrics) |> ClickhouseRepo.one() end @@ -44,6 +45,7 @@ defmodule Plausible.Stats.Aggregate do from(e in query_sessions(site, query), select: %{}) |> filter_converted_sessions(site, query) |> select_session_metrics(metrics) + |> merge_imported(site, query, :aggregate, metrics) |> ClickhouseRepo.one() end diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 84b3c3baf5f7..778ac8bef213 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -398,8 +398,9 @@ defmodule Plausible.Stats.Base do group_by: field(i, ^dim), where: i.domain == ^site.domain, where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, - select: %{visitors: sum(i.visitors)} + select: %{} ) + |> select_imported_metrics(metrics) imported_q = case query.filters[property] do @@ -444,7 +445,6 @@ defmodule Plausible.Stats.Base do imported_q |> select_merge([i], %{ page: i.page, - pageviews: sum(i.pageviews), time_on_page: sum(i.time_on_page) }) @@ -472,10 +472,6 @@ defmodule Plausible.Stats.Base do imported_q |> select_merge([i], %{operating_system: i.operating_system}) end - imported_q = - imported_q - |> select_imported_metrics(metrics) - q = from(s in Ecto.Query.subquery(q), full_join: i in subquery(imported_q), @@ -515,15 +511,6 @@ defmodule Plausible.Stats.Base do page: fragment("if(empty(?), ?, ?)", i.page, s.page, i.page) }) - if :pageviews in metrics do - q - |> select_merge([s, i], %{ - pageviews: fragment("coalesce(?, 0) + coalesce(?, 0)", s.pageviews, i.pageviews) - }) - else - q - end - :entry_page -> q |> select_merge([s, i], %{ @@ -570,10 +557,42 @@ defmodule Plausible.Stats.Base do end end + def merge_imported(q, site, query, :aggregate, metrics) do + {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) + + imported_q = + from( + i in "imported_visitors", + where: i.domain == ^site.domain, + where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, + select: %{} + ) + |> select_imported_metrics(metrics) + + from( + s in subquery(q), + cross_join: i in subquery(imported_q), + select: %{} + ) + |> select_joined_metrics(metrics) + end + def merge_imported(q, _, _, _, _), do: q def select_imported_metrics(q, []), do: q + def select_imported_metrics(q, [:visitors | rest]) do + q + |> select_merge([i], %{visitors: sum(i.visitors)}) + |> select_imported_metrics(rest) + end + + def select_imported_metrics(q, [:pageviews | rest]) do + q + |> select_merge([i], %{pageviews: sum(i.pageviews)}) + |> select_imported_metrics(rest) + end + def select_imported_metrics(q, [:bounce_rate | rest]) do q |> select_merge([i], %{ @@ -612,6 +631,14 @@ defmodule Plausible.Stats.Base do |> select_joined_metrics(rest) end + def select_joined_metrics(q, [:pageviews | rest]) do + q + |> select_merge([s, i], %{ + pageviews: fragment("coalesce(?, 0) + coalesce(?, 0)", s.pageviews, i.pageviews) + }) + |> select_joined_metrics(rest) + end + def select_joined_metrics(q, [:bounce_rate | rest]) do q |> select_merge([s, i], %{ diff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs index fbffd343fe7a..78a3a0c05969 100644 --- a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs +++ b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs @@ -7,7 +7,8 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do add :timestamp, :naive_datetime add :visitors, :UInt64 add :pageviews, :UInt64 - add :bounces, :UInt32 + add :bounces, :UInt64 + add :visits, :UInt64 add :visit_duration, :UInt64 end end From 429ec25fb1071f4db344a476648b8df9af2fd77e Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 21 Dec 2021 14:13:17 +0000 Subject: [PATCH 063/148] Reformat top graph side icons --- assets/js/dashboard/stats/visitor-graph.js | 35 +++++++++++++--------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js index 057146048590..d640675d741c 100644 --- a/assets/js/dashboard/stats/visitor-graph.js +++ b/assets/js/dashboard/stats/visitor-graph.js @@ -317,17 +317,19 @@ class LineGraph extends React.Component { if (this.state.exported) { return ( - - - - +
    + + + + +
    ) } else { const endpoint = `/${encodeURIComponent(this.props.site.domain)}/export${api.serializeQuery(this.props.query)}` return ( - - + + ) } @@ -339,8 +341,8 @@ class LineGraph extends React.Component { if (samplePercent < 100) { return ( -
    - +
    +
    @@ -358,10 +360,13 @@ class LineGraph extends React.Component { url.setQuery('with_imported', false) : window.location.pathname; + return ( - -
    - { hasImported[0].toUpperCase() } + +
    + + { hasImported[0].toUpperCase() } +
    ) @@ -377,9 +382,11 @@ class LineGraph extends React.Component { { this.renderTopStats() }
    - { this.downloadLink() } - { this.samplingNotice() } - { this.importedNotice() } +
    + { this.downloadLink() } + { this.samplingNotice() } + { this.importedNotice() } +
    From 393ad2773c36d487ffff65821a75fe8ccacade3e Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 21 Dec 2021 14:24:31 +0000 Subject: [PATCH 064/148] Ensure sample_percent is yielded from aggregate data --- lib/plausible/stats/base.ex | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 778ac8bef213..9c1d40b6b18d 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -671,6 +671,12 @@ defmodule Plausible.Stats.Base do |> select_joined_metrics(rest) end + def select_joined_metrics(q, [:sample_percent | rest]) do + q + |> select_merge([s, i], %{sample_percent: s.sample_percent}) + |> select_joined_metrics(rest) + end + def select_joined_metrics(q, [_ | rest]) do q |> select_joined_metrics(rest) From d4464867fe9c50869405bf6600bd32d3f2f74fe2 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 21 Dec 2021 14:27:27 +0000 Subject: [PATCH 065/148] filter event_props should be strings --- lib/plausible/stats/filters.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plausible/stats/filters.ex b/lib/plausible/stats/filters.ex index 8678913d5e9f..2091fe88d934 100644 --- a/lib/plausible/stats/filters.ex +++ b/lib/plausible/stats/filters.ex @@ -22,7 +22,7 @@ defmodule Plausible.Stats.Filters do @event_props [ "name", - :page + "page" ] def visit_props() do From adb3f228e0a7f9f388f976bf881736cb30923829 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 21 Dec 2021 14:33:53 +0000 Subject: [PATCH 066/148] Hide imported data from frontend when using filter --- .../controllers/api/stats_controller.ex | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index b5c971d0ac36..25d9e6563b91 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -25,12 +25,14 @@ defmodule PlausibleWeb.Api.StatsController do labels = Enum.map(timeseries_result, fn row -> row["date"] end) present_index = present_index_for(site, query, labels) - plot = + {plot, has_imported} = if query.with_imported && site.has_imported_stats do - Imported.Visitors.timeseries(site, timeseries_query) - |> Enum.zip_with(plot, &(&1 + &2)) + plot = + Imported.Visitors.timeseries(site, timeseries_query) + |> Enum.zip_with(plot, &(&1 + &2)) + {plot, site.has_imported_stats} else - plot + {plot, false} end json(conn, %{ @@ -40,7 +42,7 @@ defmodule PlausibleWeb.Api.StatsController do top_stats: top_stats, interval: query.interval, sample_percent: sample_percent, - has_imported: site.has_imported_stats + has_imported: has_imported }) end From f737aa7a5d32f8c99255d7857d707ecca89a92b8 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 23 Dec 2021 13:22:54 +0000 Subject: [PATCH 067/148] Fix existing tests --- .../controllers/api/external_stats_controller.ex | 2 +- lib/plausible_web/controllers/api/stats_controller.ex | 10 +++++----- .../plausible_web/controllers/site_controller_test.exs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index b96c33fe1b06..ecd77830b247 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -101,7 +101,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do defp event_only_property?(_), do: false @event_metrics ["visitors", "pageviews", "events"] - @session_metrics [:visits, :bounce_rate, :visit_duration] + @session_metrics ["visits", "bounce_rate", "visit_duration"] defp parse_metrics(params, property, query) do metrics = Map.get(params, "metrics", "visitors") diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 25d9e6563b91..296b7e00893c 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -234,7 +234,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", :visitors, :bounce_rate, "visit_duration"]) + res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) end else json(conn, res) @@ -265,7 +265,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", :visitors, :bounce_rate, "visit_duration"]) + res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) end else json(conn, res) @@ -296,7 +296,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", :visitors, :bounce_rate, "visit_duration"]) + res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) end else json(conn, res) @@ -385,7 +385,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", :visitors, :bounce_rate, "visit_duration"]) + res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) end else json(conn, res) @@ -494,7 +494,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{"unique_entrances" => "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - entry_pages |> to_csv(["name", "unique_entrances", "total_entrances", "visit_duration"]) + entry_pages |> to_csv(["name", "unique_entrances", "total_entrances", :visit_duration]) end else json(conn, entry_pages) diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs index 6ccf7d62f107..37361bd56c63 100644 --- a/test/plausible_web/controllers/site_controller_test.exs +++ b/test/plausible_web/controllers/site_controller_test.exs @@ -374,7 +374,7 @@ defmodule PlausibleWeb.SiteControllerTest do test "deletes associated google auth", %{conn: conn, user: user, site: site} do insert(:google_auth, user: user, site: site) - conn = delete(conn, "/#{site.domain}/settings/google") + conn = delete(conn, "/#{site.domain}/settings/google-search") refute Repo.exists?(Plausible.Site.GoogleAuth) assert redirected_to(conn, 302) == "/#{site.domain}/settings/search-console" From c6b314f4fb2f725db223caf75b92d033f399db6b Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 29 Dec 2021 19:14:56 +0000 Subject: [PATCH 068/148] fix tests --- lib/plausible/stats/base.ex | 16 ++++++++-------- lib/plausible/stats/breakdown.ex | 17 +++++++++++++++-- lib/plausible/stats/filter_suggestions.ex | 4 ++-- .../api/external_stats_controller.ex | 16 ++++------------ .../controllers/api/stats_controller.ex | 15 +++++++-------- 5 files changed, 36 insertions(+), 32 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 9c1d40b6b18d..57f1eb497b35 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -382,7 +382,7 @@ defmodule Plausible.Stats.Base do {"imported_locations", :country} "visit:os" -> - {"imported_operating_systems", :operating_system} + {"imported_operating_systems", :os} "event:page" -> {"imported_pages", :page} @@ -468,8 +468,8 @@ defmodule Plausible.Stats.Base do :browser -> imported_q |> select_merge([i], %{browser: i.browser}) - :operating_system -> - imported_q |> select_merge([i], %{operating_system: i.operating_system}) + :os -> + imported_q |> select_merge([i], %{os: i.operating_system}) end q = @@ -543,15 +543,15 @@ defmodule Plausible.Stats.Base do browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser) }) - :operating_system -> + :os -> q |> select_merge([i, s], %{ - operating_system: + os: fragment( "if(empty(?), ?, ?)", - s.operating_system, - i.operating_system, - s.operating_system + s.os, + i.os, + s.os ) }) end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index b3f569bd0a85..6666d1166e10 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -121,12 +121,15 @@ defmodule Plausible.Stats.Breakdown do breakdown_sessions(site, new_query, "visit:entry_page", session_metrics, {limit, 1}) |> transform_keys(%{entry_page: :page}) + metrics = metrics ++ [:page] + zip_results( event_result, session_result, :page, metrics ) + |> Enum.map(&Map.take(&1, metrics)) end def breakdown(site, query, property, metrics, pagination) when property in @event_props do @@ -143,7 +146,17 @@ defmodule Plausible.Stats.Breakdown do "visit:utm_term" ] do query = Query.treat_page_filter_as_entry_page(query) - breakdown_sessions(site, query, property, metrics, pagination) + + results = breakdown_sessions(site, query, property, metrics, pagination) + + prop_result = + property + |> String.split(":") + |> Enum.at(1) + |> String.to_existing_atom() + + metrics = metrics ++ [prop_result] + Enum.map(results, &Map.take(&1, metrics)) end def breakdown(site, query, property, metrics, pagination) do @@ -488,7 +501,7 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.operating_system, - select_merge: %{operating_system: s.operating_system} + select_merge: %{os: s.operating_system} ) end diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex index 8d97012cd59a..569a3ceeee5f 100644 --- a/lib/plausible/stats/filter_suggestions.ex +++ b/lib/plausible/stats/filter_suggestions.ex @@ -131,13 +131,13 @@ defmodule Plausible.Stats.FilterSuggestions do filter_search = if filter_search == nil, do: "", else: filter_search filter_query = - if Enum.member?(["entry_page", :page, "exit_page"], filter_name), + if Enum.member?(["entry_page", "page", "exit_page"], filter_name), do: "%#{String.replace(filter_search, "*", "")}%", else: "%#{filter_search}%" filter_name = case filter_name do - :page -> "pathname" + "page" -> "pathname" "source" -> "referrer_source" "os" -> "operating_system" "os_version" -> "operating_system_version" diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index ecd77830b247..e85129731103 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -45,7 +45,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do Plausible.Stats.aggregate(site, query, metrics) end - json(conn, %{"results" => results}) + json(conn, %{"results" => Map.take(results, metrics)}) else {:error, msg} -> conn @@ -133,16 +133,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do "The metric `#{invalid_metric}` is not recognized. Find valid metrics from the documentation: https://plausible.io/docs/stats-api#get-apiv1statsbreakdown"} end else - metrics = - metrics - |> Enum.map(fn item -> - case item do - "visitors" -> :visitors - metric -> metric - end - end) - - {:ok, metrics} + {:ok, Enum.map(metrics, &String.to_atom/1)} end end @@ -156,7 +147,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController do query <- Query.from(site.timezone, params), {:ok, metrics} <- parse_metrics(params, nil, query) do graph = Plausible.Stats.timeseries(site, query, metrics) - json(conn, %{"results" => graph}) + metrics = metrics ++ ["date"] + json(conn, %{"results" => Enum.map(graph, &Map.take(&1, metrics))}) else {:error, msg} -> conn diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 296b7e00893c..6e212be2d80e 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -30,6 +30,7 @@ defmodule PlausibleWeb.Api.StatsController do plot = Imported.Visitors.timeseries(site, timeseries_query) |> Enum.zip_with(plot, &(&1 + &2)) + {plot, site.has_imported_stats} else {plot, false} @@ -251,8 +252,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) - metrics = - if params["detailed"], do: [:visitors, :bounce_rate, :visit_duration], else: [:visitors] + metrics = [:visitors, :bounce_rate, :visit_duration] res = Stats.breakdown(site, query, "visit:utm_medium", metrics, pagination) @@ -282,8 +282,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) - metrics = - if params["detailed"], do: [:visitors, :bounce_rate, :visit_duration], else: [:visitors] + metrics = [:visitors, :bounce_rate, :visit_duration] res = Stats.breakdown(site, query, "visit:utm_campaign", metrics, pagination) @@ -371,8 +370,7 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) - metrics = - if params["detailed"], do: [:visitors, :bounce_rate, :visit_duration], else: [:visitors] + metrics = [:visitors, :bounce_rate, :visit_duration] res = Stats.breakdown(site, query, "visit:utm_source", metrics, pagination) @@ -439,6 +437,7 @@ defmodule PlausibleWeb.Api.StatsController do Stats.breakdown(site, query, "visit:referrer", metrics, pagination) |> maybe_add_cr(site, query, pagination, "referrer", "visit:referrer") |> transform_keys(%{"referrer" => "name"}) + |> Enum.map(&Map.drop(&1, [:visits])) %{:visitors => %{"value" => total_visitors}} = Stats.aggregate(site, query, [:visitors]) json(conn, %{referrers: referrers, total_visitors: total_visitors}) @@ -714,8 +713,8 @@ defmodule PlausibleWeb.Api.StatsController do systems = Stats.breakdown(site, query, "visit:os", [:visitors], pagination) - |> maybe_add_cr(site, query, pagination, :operating_system, "visit:os") - |> transform_keys(%{operating_system: "name"}) + |> maybe_add_cr(site, query, pagination, :os, "visit:os") + |> transform_keys(%{os: "name"}) |> maybe_add_percentages(query) if params["csv"] do From ae84c7e8040a689bca5cb781ab8e2fc4c733342f Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 30 Dec 2021 12:32:46 +0000 Subject: [PATCH 069/148] Fix imported indicator appearing when filtering --- assets/js/dashboard/stats/visitor-graph.js | 14 ++++++------ lib/plausible/stats/query.ex | 22 +++++++++---------- .../controllers/api/stats_controller.ex | 17 ++++++++++---- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js index d640675d741c..159c133c3dee 100644 --- a/assets/js/dashboard/stats/visitor-graph.js +++ b/assets/js/dashboard/stats/visitor-graph.js @@ -5,7 +5,7 @@ import { navigateToQuery } from '../query' import numberFormatter, {durationFormatter} from '../util/number-formatter' import * as api from '../api' import LazyLoader from '../components/lazy-loader' -import * as url from '../url' +import * as url from '../util/url' function buildDataSet(plot, present_index, ctx, label) { var gradient = ctx.createLinearGradient(0, 0, 0, 300); @@ -351,21 +351,21 @@ class LineGraph extends React.Component { } importedNotice() { - const hasImported = this.props.graphData.has_imported + const source = this.props.graphData.imported_source; - if (hasImported) { - const withImported = this.props.query.with_imported + if (source) { + const withImported = this.props.graphData.with_imported; const strike = withImported ? "" : " line-through" const target = withImported || false ? url.setQuery('with_imported', false) : window.location.pathname; - + const tip = withImported ? "" : "do not "; return ( -
    +
    - { hasImported[0].toUpperCase() } + { source[0].toUpperCase() }
    diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index c7d1e2d39872..17e278517b11 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -47,7 +47,7 @@ defmodule Plausible.Stats.Query do date_range: Date.range(date, date), filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(false, params) + with_imported: false } end @@ -60,7 +60,7 @@ defmodule Plausible.Stats.Query do interval: "hour", filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(true, params) + with_imported: include_imported(params) } end @@ -74,7 +74,7 @@ defmodule Plausible.Stats.Query do interval: "date", filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(true, params) + with_imported: include_imported(params) } end @@ -88,7 +88,7 @@ defmodule Plausible.Stats.Query do interval: "date", filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(true, params) + with_imported: include_imported(params) } end @@ -104,7 +104,7 @@ defmodule Plausible.Stats.Query do interval: "date", filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(true, params) + with_imported: include_imported(params) } end @@ -123,7 +123,7 @@ defmodule Plausible.Stats.Query do interval: Map.get(params, "interval", "month"), filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(true, params) + with_imported: include_imported(params) } end @@ -142,7 +142,7 @@ defmodule Plausible.Stats.Query do interval: Map.get(params, "interval", "month"), filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(true, params) + with_imported: include_imported(params) } end @@ -167,7 +167,7 @@ defmodule Plausible.Stats.Query do interval: Map.get(params, "interval", "date"), filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(true, params) + with_imported: include_imported(params) } end @@ -257,9 +257,7 @@ defmodule Plausible.Stats.Query do defp parse_goal_filter("Visit " <> page), do: {:is, :page, page} defp parse_goal_filter(event), do: {:is, :event, event} - defp include_imported(default, params) do - default && - params["filters"] == "{}" && - params["with_imported"] == "true" + defp include_imported(params) do + params["filters"] == "{}" && params["with_imported"] == "true" end end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 6e212be2d80e..df42628d6293 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -25,15 +25,23 @@ defmodule PlausibleWeb.Api.StatsController do labels = Enum.map(timeseries_result, fn row -> row["date"] end) present_index = present_index_for(site, query, labels) - {plot, has_imported} = + {plot, with_imported, source} = if query.with_imported && site.has_imported_stats do + # Showing imported data. plot = Imported.Visitors.timeseries(site, timeseries_query) |> Enum.zip_with(plot, &(&1 + &2)) - {plot, site.has_imported_stats} + {plot, true, site.has_imported_stats} else - {plot, false} + if Map.get(params, "filters", "{}") != "{}" do + # Hiding imported data due to filtering. + # Setting source to "" hides imported indicator from main graph. + {plot, false, ""} + else + # Hiding imported data either by request or because there is none. + {plot, false, site.has_imported_stats || ""} + end end json(conn, %{ @@ -43,7 +51,8 @@ defmodule PlausibleWeb.Api.StatsController do top_stats: top_stats, interval: query.interval, sample_percent: sample_percent, - has_imported: has_imported + with_imported: with_imported, + imported_source: source, }) end From ecfccd4a2401bfcf0c265fc69af8dceaf59572da Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 31 Dec 2021 11:35:33 +0000 Subject: [PATCH 070/148] comma needed, lost when rebasing --- lib/plausible/stats/query.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 17e278517b11..1d994049c537 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -3,7 +3,7 @@ defmodule Plausible.Stats.Query do interval: nil, period: nil, filters: %{}, - sample_threshold: 20_000_000 + sample_threshold: 20_000_000, with_imported: true @default_sample_threshold 20_000_000 From c48b3b5688ff023914299fd32ac3d49b2597cec0 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 31 Dec 2021 12:18:25 +0000 Subject: [PATCH 071/148] Import utm_terms and utm_content from GA --- lib/plausible/google/api.ex | 12 +++++ lib/plausible/imported/site.ex | 44 +++++++++++++++++++ lib/plausible/imported/utm_content.ex | 42 ++++++++++++++++++ lib/plausible/imported/utm_terms.ex | 42 ++++++++++++++++++ ...211231120503_create_imported_utm_terms.exs | 15 +++++++ ...1231120603_create_imported_utm_content.exs | 15 +++++++ 6 files changed, 170 insertions(+) create mode 100644 lib/plausible/imported/utm_content.ex create mode 100644 lib/plausible/imported/utm_terms.ex create mode 100644 priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs create mode 100644 priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_content.exs diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 8eefe9336037..c6015ea7b051 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -204,6 +204,16 @@ defmodule Plausible.Google.Api do ["ga:date", "ga:campaign"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, + # UTM Terms + { + ["ga:date", "ga:keyword"], + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] + }, + # UTM Content + { + ["ga:date", "ga:adContent"], + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] + }, # Pages { ["ga:date", "ga:pagePath"], @@ -260,6 +270,8 @@ defmodule Plausible.Google.Api do "sources", "utm_mediums", "utm_campaigns", + "utm_terms", + "utm_content", "pages", "entry_pages", "exit_pages", diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 1c69215a6264..6aa96c8f1c06 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -111,6 +111,50 @@ defmodule Plausible.Imported do }) end + defp new_from_google_analytics(domain, "utm_terms", %{ + "dimensions" => [timestamp, term], + "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] + }) do + {visitors, ""} = Integer.parse(visitors) + {visits, ""} = Integer.parse(visits) + {bounces, ""} = Integer.parse(bounces) + {visit_duration, _} = Integer.parse(visit_duration) + + term = if term == "(not set)" or term == "(not provided)", do: "", else: term + + Imported.UtmTerms.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + utm_term: term, + visitors: visitors, + visits: visits, + bounces: bounces, + visit_duration: visit_duration + }) + end + + defp new_from_google_analytics(domain, "utm_content", %{ + "dimensions" => [timestamp, content], + "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] + }) do + {visitors, ""} = Integer.parse(visitors) + {visits, ""} = Integer.parse(visits) + {bounces, ""} = Integer.parse(bounces) + {visit_duration, _} = Integer.parse(visit_duration) + + content = if content == "(not set)", do: "", else: content + + Imported.UtmContent.new(%{ + domain: domain, + timestamp: format_timestamp(timestamp), + utm_content: content, + visitors: visitors, + visits: visits, + bounces: bounces, + visit_duration: visit_duration + }) + end + defp new_from_google_analytics(domain, "pages", %{ "dimensions" => [timestamp, page], "metrics" => [%{"values" => [value, pageviews, time_on_page]}] diff --git a/lib/plausible/imported/utm_content.ex b/lib/plausible/imported/utm_content.ex new file mode 100644 index 000000000000..f7e14a69161f --- /dev/null +++ b/lib/plausible/imported/utm_content.ex @@ -0,0 +1,42 @@ +defmodule Plausible.Imported.UtmContent do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_utm_content" do + field :domain, :string + field :timestamp, :naive_datetime + field :utm_content, :string, default: "" + field :visitors, :integer + field :visits, :integer + field :bounces, :integer + # Sum total + field :visit_duration, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :utm_content, + :visitors, + :visits, + :bounces, + :visit_duration + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :visitors, + :visits, + :bounces, + :visit_duration + ]) + end +end diff --git a/lib/plausible/imported/utm_terms.ex b/lib/plausible/imported/utm_terms.ex new file mode 100644 index 000000000000..911b2d85fda9 --- /dev/null +++ b/lib/plausible/imported/utm_terms.ex @@ -0,0 +1,42 @@ +defmodule Plausible.Imported.UtmTerms do + use Ecto.Schema + use Plausible.ClickhouseRepo + import Ecto.Changeset + + @primary_key false + schema "imported_utm_terms" do + field :domain, :string + field :timestamp, :naive_datetime + field :utm_term, :string, default: "" + field :visitors, :integer + field :visits, :integer + field :bounces, :integer + # Sum total + field :visit_duration, :integer + end + + def new(attrs) do + %__MODULE__{} + |> cast( + attrs, + [ + :domain, + :timestamp, + :utm_term, + :visitors, + :visits, + :bounces, + :visit_duration + ], + empty_values: [nil, ""] + ) + |> validate_required([ + :domain, + :timestamp, + :visitors, + :visits, + :bounces, + :visit_duration + ]) + end +end diff --git a/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs b/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs new file mode 100644 index 000000000000..65283e0811da --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs @@ -0,0 +1,15 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmTerms do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_utm_terms, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :utm_term, :string + add :visitors, :UInt64 + add :visits, :UInt64 + add :visit_duration, :UInt64 + add :bounces, :UInt32 + end + end +end diff --git a/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_content.exs b/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_content.exs new file mode 100644 index 000000000000..293647717e13 --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_content.exs @@ -0,0 +1,15 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmContent do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_utm_content, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + add :domain, :string + add :timestamp, :naive_datetime + add :utm_content, :string + add :visitors, :UInt64 + add :visits, :UInt64 + add :visit_duration, :UInt64 + add :bounces, :UInt32 + end + end +end From bb6b085528ad418cc154c6d0b681d77af6e77005 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 31 Dec 2021 12:42:08 +0000 Subject: [PATCH 072/148] Merge imported utm_term and utm_content --- lib/plausible/stats/base.ex | 29 +++++++++++++++++++ lib/plausible/stats/breakdown.ex | 4 +-- .../controllers/api/stats_controller.ex | 22 +++++++------- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 57f1eb497b35..490914b16a2a 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -366,6 +366,8 @@ defmodule Plausible.Stats.Base do "visit:utm_medium", "visit:utm_source", "visit:utm_campaign", + "visit:utm_term", + "visit:utm_content", "visit:entry_page", "visit:exit_page", "visit:country", @@ -378,6 +380,9 @@ defmodule Plausible.Stats.Base do {table, dim} = case property do + "visit:utm_content" -> + {"imported_utm_content", :utm_content} + "visit:country" -> {"imported_locations", :country} @@ -441,6 +446,18 @@ defmodule Plausible.Stats.Base do utm_campaign: fragment("if(empty(?), ?, ?)", i.utm_campaign, @no_ref, i.utm_campaign) }) + :utm_term -> + imported_q + |> select_merge([i], %{ + utm_term: fragment("if(empty(?), ?, ?)", i.utm_term, @no_ref, i.utm_term) + }) + + :utm_content -> + imported_q + |> select_merge([i], %{ + utm_content: fragment("if(empty(?), ?, ?)", i.utm_content, @no_ref, i.utm_content) + }) + :page -> imported_q |> select_merge([i], %{ @@ -505,6 +522,18 @@ defmodule Plausible.Stats.Base do fragment("if(empty(?), ?, ?)", s.utm_campaign, i.utm_campaign, s.utm_campaign) }) + :utm_term -> + q + |> select_merge([s, i], %{ + utm_term: fragment("if(empty(?), ?, ?)", s.utm_term, i.utm_term, s.utm_term) + }) + + :utm_content -> + q + |> select_merge([s, i], %{ + utm_content: fragment("if(empty(?), ?, ?)", s.utm_content, i.utm_content, s.utm_content) + }) + :page -> q |> select_merge([s, i], %{ diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 6666d1166e10..7ab4543e33eb 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -474,7 +474,7 @@ defmodule Plausible.Stats.Breakdown do s in q, group_by: s.utm_content, select_merge: %{ - "utm_content" => fragment("if(empty(?), ?, ?)", s.utm_content, @no_ref, s.utm_content) + utm_content: fragment("if(empty(?), ?, ?)", s.utm_content, @no_ref, s.utm_content) } ) end @@ -484,7 +484,7 @@ defmodule Plausible.Stats.Breakdown do s in q, group_by: s.utm_term, select_merge: %{ - "utm_term" => fragment("if(empty(?), ?, ?)", s.utm_term, @no_ref, s.utm_term) + utm_term: fragment("if(empty(?), ?, ?)", s.utm_term, @no_ref, s.utm_term) } ) end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index df42628d6293..2b82a275d371 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -52,7 +52,7 @@ defmodule PlausibleWeb.Api.StatsController do interval: query.interval, sample_percent: sample_percent, with_imported: with_imported, - imported_source: source, + imported_source: source }) end @@ -320,20 +320,20 @@ defmodule PlausibleWeb.Api.StatsController do |> maybe_hide_noref("visit:utm_content", params) pagination = parse_pagination(params) - metrics = ["visitors", "bounce_rate", "visit_duration"] + metrics = [:visitors, :bounce_rate, :visit_duration] res = Stats.breakdown(site, query, "visit:utm_content", metrics, pagination) - |> maybe_add_cr(site, query, pagination, "utm_content", "visit:utm_content") - |> transform_keys(%{"utm_content" => "name"}) + |> maybe_add_cr(site, query, pagination, :utm_content, "visit:utm_content") + |> transform_keys(%{utm_content: "name"}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do res - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", "visitors", "bounce_rate", "visit_duration"]) + res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) end else json(conn, res) @@ -349,20 +349,20 @@ defmodule PlausibleWeb.Api.StatsController do |> maybe_hide_noref("visit:utm_term", params) pagination = parse_pagination(params) - metrics = ["visitors", "bounce_rate", "visit_duration"] + metrics = [:visitors, :bounce_rate, :visit_duration] res = Stats.breakdown(site, query, "visit:utm_term", metrics, pagination) - |> maybe_add_cr(site, query, pagination, "utm_term", "visit:utm_term") - |> transform_keys(%{"utm_term" => "name"}) + |> maybe_add_cr(site, query, pagination, :utm_term, "visit:utm_term") + |> transform_keys(%{utm_term: "name"}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do res - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - res |> to_csv(["name", "visitors", "bounce_rate", "visit_duration"]) + res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) end else json(conn, res) From 866097ead5077d5afedb405693c0c722b1eb9480 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 31 Dec 2021 12:44:37 +0000 Subject: [PATCH 073/148] Rename imported Countries data as Locations --- lib/plausible/google/api.ex | 4 ++-- lib/plausible/imported/{countries.ex => locations.ex} | 0 lib/plausible/imported/site.ex | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename lib/plausible/imported/{countries.ex => locations.ex} (100%) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index c6015ea7b051..7090080f44d0 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -229,7 +229,7 @@ defmodule Plausible.Google.Api do ["ga:date", "ga:exitPagePath"], ["ga:users", "ga:exits"] }, - # Country + # Locations { ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode", "ga:cityId"], ["ga:users"] @@ -275,7 +275,7 @@ defmodule Plausible.Google.Api do "pages", "entry_pages", "exit_pages", - "countries", + "locations", "devices", "browsers", "operating_systems" diff --git a/lib/plausible/imported/countries.ex b/lib/plausible/imported/locations.ex similarity index 100% rename from lib/plausible/imported/countries.ex rename to lib/plausible/imported/locations.ex diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 6aa96c8f1c06..c8377d1ca61f 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -210,7 +210,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, "countries", %{ + defp new_from_google_analytics(domain, "locations", %{ "dimensions" => [timestamp, country, region, city], "metrics" => [%{"values" => [value]}] }) do From 98f27f004ea0b788d70b3f15751ead9ff204a7ba Mon Sep 17 00:00:00 2001 From: mcol Date: Sun, 2 Jan 2022 13:51:08 +0000 Subject: [PATCH 074/148] Set imported city schema field to int --- lib/plausible/imported/locations.ex | 6 +++--- lib/plausible_web/controllers/api/stats_controller.ex | 8 ++++---- .../20211129111639_create_imported_locations.exs | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/plausible/imported/locations.ex b/lib/plausible/imported/locations.ex index 587f0617ee70..1493c32fb648 100644 --- a/lib/plausible/imported/locations.ex +++ b/lib/plausible/imported/locations.ex @@ -7,9 +7,9 @@ defmodule Plausible.Imported.Locations do schema "imported_locations" do field :domain, :string field :timestamp, :naive_datetime - field :country, :string - field :region, :string - field :city, :string + field :country, :string, default: "" + field :region, :string, default: "" + field :city, :integer, default: 0 field :visitors, :integer end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 2b82a275d371..a1cd60617a0a 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -612,8 +612,8 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) regions = - Stats.breakdown(site, query, "visit:region", ["visitors"], pagination) - |> transform_keys(%{"region" => "code"}) + Stats.breakdown(site, query, "visit:region", [:visitors], pagination) + |> transform_keys(%{region: "code"}) |> Enum.map(fn region -> region_entry = Location.get_subdivision(region["code"]) @@ -645,8 +645,8 @@ defmodule PlausibleWeb.Api.StatsController do pagination = parse_pagination(params) cities = - Stats.breakdown(site, query, "visit:city", ["visitors"], pagination) - |> transform_keys(%{"city" => "code"}) + Stats.breakdown(site, query, "visit:city", [:visitors], pagination) + |> transform_keys(%{city: "code"}) |> Enum.map(fn city -> city_info = Location.get_city(city["code"]) diff --git a/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs index 024ae8940c6c..cf520c6d0e6b 100644 --- a/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs +++ b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs @@ -7,7 +7,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedLocations do add :timestamp, :naive_datetime add :country, :string add :region, :string - add :city, :string + add :city, :UInt64 add :visitors, :UInt64 end end From ca5f1e5686ca9c65d975219d350e3a791a27994b Mon Sep 17 00:00:00 2001 From: mcol Date: Sun, 2 Jan 2022 14:23:15 +0000 Subject: [PATCH 075/148] Remove utm_terms and utm_content when clearing imported --- lib/plausible/clickhouse_repo.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex index 1d8c719192b1..9a896ce87283 100644 --- a/lib/plausible/clickhouse_repo.ex +++ b/lib/plausible/clickhouse_repo.ex @@ -25,6 +25,8 @@ defmodule Plausible.ClickhouseRepo do "imported_utm_mediums", "imported_utm_sources", "imported_utm_campaigns", + "imported_utm_content", + "imported_utm_terms", "imported_pages", "imported_entry_pages", "imported_exit_pages", From 944d07afe49c25c435de2d25d7a0ffa27aa3427a Mon Sep 17 00:00:00 2001 From: mcol Date: Sun, 2 Jan 2022 14:24:42 +0000 Subject: [PATCH 076/148] Clean locations import from Google Analytics - Country and region should be set to "" when GA provides "(not set)" - City should be set to 0 for "unknown", as we cannot reliably import city data from GA. --- lib/plausible/google/api.ex | 2 +- lib/plausible/imported/site.ex | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 7090080f44d0..da9f36030dcf 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -231,7 +231,7 @@ defmodule Plausible.Google.Api do }, # Locations { - ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode", "ga:cityId"], + ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode"], ["ga:users"] }, # Device diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index c8377d1ca61f..1a783ee5ccf3 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -211,17 +211,19 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(domain, "locations", %{ - "dimensions" => [timestamp, country, region, city], + "dimensions" => [timestamp, country, region], "metrics" => [%{"values" => [value]}] }) do {visitors, ""} = Integer.parse(value) + country = if country == "(not set)", do: "", else: country + region = if region == "(not set)", do: "", else: region Imported.Locations.new(%{ domain: domain, timestamp: format_timestamp(timestamp), country: country, region: region, - city: city, + city: 0, visitors: visitors }) end From c14ded5e1d7205e71dadca994adaa591e367b68d Mon Sep 17 00:00:00 2001 From: mcol Date: Sun, 2 Jan 2022 14:26:34 +0000 Subject: [PATCH 077/148] Display imported region and city in dashboard --- lib/plausible/stats/base.ex | 30 ++++++++++++++++++++++++++++++ lib/plausible/stats/breakdown.ex | 4 ++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 490914b16a2a..136ba8ede500 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -371,6 +371,8 @@ defmodule Plausible.Stats.Base do "visit:entry_page", "visit:exit_page", "visit:country", + "visit:region", + "visit:city", "visit:device", "visit:browser", "visit:os", @@ -386,6 +388,12 @@ defmodule Plausible.Stats.Base do "visit:country" -> {"imported_locations", :country} + "visit:region" -> + {"imported_locations", :region} + + "visit:city" -> + {"imported_locations", :city} + "visit:os" -> {"imported_operating_systems", :os} @@ -479,6 +487,16 @@ defmodule Plausible.Stats.Base do :country -> imported_q |> select_merge([i], %{country: i.country}) + :region -> + imported_q + |> select_merge([i], %{region: i.region}) + |> where([i], i.region != "") + + :city -> + imported_q + |> select_merge([i], %{city: i.city}) + |> where([i], i.city != 0) + :device -> imported_q |> select_merge([i], %{device: i.device}) @@ -560,6 +578,18 @@ defmodule Plausible.Stats.Base do country: fragment("if(empty(?), ?, ?)", s.country, i.country, s.country) }) + :region -> + q + |> select_merge([i, s], %{ + region: fragment("if(empty(?), ?, ?)", s.region, i.region, s.region) + }) + + :city -> + q + |> select_merge([i, s], %{ + city: fragment("coalesce(?, ?)", s.city, i.city) + }) + :device -> q |> select_merge([i, s], %{ diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 7ab4543e33eb..0b5effcb7b35 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -400,7 +400,7 @@ defmodule Plausible.Stats.Breakdown do s in q, group_by: s.subdivision1_code, where: s.subdivision1_code != "", - select_merge: %{"region" => s.subdivision1_code} + select_merge: %{region: s.subdivision1_code} ) end @@ -409,7 +409,7 @@ defmodule Plausible.Stats.Breakdown do s in q, group_by: s.city_geoname_id, where: s.city_geoname_id != 0, - select_merge: %{"city" => s.city_geoname_id} + select_merge: %{city: s.city_geoname_id} ) end From 787ee44ee9229a116ea79898f032abcf4128cf7f Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 3 Jan 2022 11:56:58 +0000 Subject: [PATCH 078/148] os -> operating_system in some parts of code The inconsistency of using os in some places and operating_system in others causes trouble with subqueries and joins for the native and imported data, which would require additional logic to account for. The simplest solution is the just use a consistent word for all uses. This doesn't make any user-facing or database changes. --- lib/plausible/imported/site.ex | 4 ++-- lib/plausible/stats/base.ex | 16 ++++++++-------- lib/plausible/stats/breakdown.ex | 6 +++++- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 1a783ee5ccf3..d49f894b0860 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -273,7 +273,7 @@ defmodule Plausible.Imported do } defp new_from_google_analytics(domain, "operating_systems", %{ - "dimensions" => [timestamp, os], + "dimensions" => [timestamp, operating_system], "metrics" => [%{"values" => [value]}] }) do {visitors, ""} = Integer.parse(value) @@ -281,7 +281,7 @@ defmodule Plausible.Imported do Imported.OperatingSystems.new(%{ domain: domain, timestamp: format_timestamp(timestamp), - operating_system: Map.get(@os_google_to_plausible, os, os), + operating_system: Map.get(@os_google_to_plausible, operating_system, operating_system), visitors: visitors }) end diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 136ba8ede500..445bf98ae1b7 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -395,7 +395,7 @@ defmodule Plausible.Stats.Base do {"imported_locations", :city} "visit:os" -> - {"imported_operating_systems", :os} + {"imported_operating_systems", :operating_system} "event:page" -> {"imported_pages", :page} @@ -503,8 +503,8 @@ defmodule Plausible.Stats.Base do :browser -> imported_q |> select_merge([i], %{browser: i.browser}) - :os -> - imported_q |> select_merge([i], %{os: i.operating_system}) + :operating_system -> + imported_q |> select_merge([i], %{operating_system: i.operating_system}) end q = @@ -602,15 +602,15 @@ defmodule Plausible.Stats.Base do browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser) }) - :os -> + :operating_system -> q |> select_merge([i, s], %{ - os: + operating_system: fragment( "if(empty(?), ?, ?)", - s.os, - i.os, - s.os + s.operating_system, + i.operating_system, + s.operating_system ) }) end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 0b5effcb7b35..8e27634ad824 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -205,6 +205,8 @@ defmodule Plausible.Stats.Breakdown do |> merge_imported(site, query, property, metrics) |> apply_pagination(pagination) |> ClickhouseRepo.all() + # TODO: migrate schema field to 'os' + |> transform_keys(%{operating_system: :os}) end defp breakdown_events(_, _, _, [], _), do: [] @@ -219,6 +221,8 @@ defmodule Plausible.Stats.Breakdown do |> merge_imported(site, query, property, metrics) |> apply_pagination(pagination) |> ClickhouseRepo.all() + # TODO: migrate schema field to 'os' + |> transform_keys(%{operating_system: :os}) end defp breakdown_time_on_page(_site, _query, []) do @@ -501,7 +505,7 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.operating_system, - select_merge: %{os: s.operating_system} + select_merge: %{operating_system: s.operating_system} ) end From e0a06f0e0f124dcfd50d9866160c0db2c2d79374 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 3 Jan 2022 11:59:45 +0000 Subject: [PATCH 079/148] to_atom -> to_existing_atom --- lib/plausible/stats/base.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 445bf98ae1b7..35851b470dd8 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -402,7 +402,7 @@ defmodule Plausible.Stats.Base do _ -> dim = String.trim_leading(property, "visit:") - {"imported_#{dim}s", String.to_atom(dim)} + {"imported_#{dim}s", String.to_existing_atom(dim)} end imported_q = From decc32fbf0ef6c42a8f6bc1773fe204f37079a91 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 3 Jan 2022 12:02:15 +0000 Subject: [PATCH 080/148] format --- lib/plausible/stats/base.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 35851b470dd8..2a58aea47028 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -489,13 +489,13 @@ defmodule Plausible.Stats.Base do :region -> imported_q - |> select_merge([i], %{region: i.region}) - |> where([i], i.region != "") + |> select_merge([i], %{region: i.region}) + |> where([i], i.region != "") :city -> imported_q - |> select_merge([i], %{city: i.city}) - |> where([i], i.city != 0) + |> select_merge([i], %{city: i.city}) + |> where([i], i.city != 0) :device -> imported_q |> select_merge([i], %{device: i.device}) From 7a1695fac210ce461b39f0092d9a14f1f0c2fe8d Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 4 Jan 2022 12:28:02 +0000 Subject: [PATCH 081/148] "events" metric -> :events --- lib/plausible/stats/aggregate.ex | 2 +- lib/plausible/stats/base.ex | 4 ++-- lib/plausible/stats/breakdown.ex | 2 +- .../controllers/api/stats_controller.ex | 24 +++++++++---------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 8aca952fe16e..830c0bd9b86d 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -3,7 +3,7 @@ defmodule Plausible.Stats.Aggregate do use Plausible.ClickhouseRepo import Plausible.Stats.Base - @event_metrics [:visitors, :pageviews, "events", :sample_percent] + @event_metrics [:visitors, :pageviews, :events, :sample_percent] @session_metrics [:visits, :bounce_rate, :visit_duration, :sample_percent] def aggregate(site, query, metrics) do diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 2a58aea47028..49e98dbcefa1 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -207,9 +207,9 @@ defmodule Plausible.Stats.Base do |> select_event_metrics(rest) end - def select_event_metrics(q, ["events" | rest]) do + def select_event_metrics(q, [:events | rest]) do from(e in q, - select_merge: %{"events" => fragment("toUInt64(round(count(*) * any(_sample_factor)))")} + select_merge: %{events: fragment("toUInt64(round(count(*) * any(_sample_factor)))")} ) |> select_event_metrics(rest) end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 8e27634ad824..e3e8a7460d3b 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -4,7 +4,7 @@ defmodule Plausible.Stats.Breakdown do alias Plausible.Stats.Query @no_ref "Direct / None" - @event_metrics [:visitors, :pageviews, "events"] + @event_metrics [:visitors, :pageviews, :events] @session_metrics [:visits, :bounce_rate, :visit_duration] @event_props ["event:page", "event:page_match", "event:name"] diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index a1cd60617a0a..e0b2d6f36917 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -117,22 +117,22 @@ defmodule PlausibleWeb.Api.StatsController do prev_total_query = Query.shift_back(total_q, site) %{ - :visitors => %{"value" => unique_visitors} + visitors: %{"value" => unique_visitors} } = Stats.aggregate(site, total_q, [:visitors]) %{ - :visitors => %{"value" => prev_unique_visitors} + visitors: %{"value" => prev_unique_visitors} } = Stats.aggregate(site, prev_total_query, [:visitors]) %{ - :visitors => %{"value" => converted_visitors}, - "events" => %{"value" => completions} - } = Stats.aggregate(site, query, [:visitors, "events"]) + visitors: %{"value" => converted_visitors}, + events: %{"value" => completions} + } = Stats.aggregate(site, query, [:visitors, :events]) %{ - :visitors => %{"value" => prev_converted_visitors}, - "events" => %{"value" => prev_completions} - } = Stats.aggregate(site, prev_query, [:visitors, "events"]) + visitors: %{"value" => prev_converted_visitors}, + events: %{"value" => prev_completions} + } = Stats.aggregate(site, prev_query, [:visitors, :events]) conversion_rate = calculate_cr(unique_visitors, converted_visitors) prev_conversion_rate = calculate_cr(prev_unique_visitors, prev_converted_visitors) @@ -808,11 +808,11 @@ defmodule PlausibleWeb.Api.StatsController do end conversions = - Stats.breakdown(site, query, "event:goal", [:visitors, "events"], {100, 1}) + Stats.breakdown(site, query, "event:goal", [:visitors, :events], {100, 1}) |> transform_keys(%{ "goal" => "name", :visitors => "unique_conversions", - "events" => "total_conversions" + :events => "total_conversions" }) |> Enum.map(fn goal -> goal @@ -839,10 +839,10 @@ defmodule PlausibleWeb.Api.StatsController do prop_name = "event:props:" <> params["prop_name"] props = - Stats.breakdown(site, query, prop_name, [:visitors, "events"], pagination) + Stats.breakdown(site, query, prop_name, [:visitors, :events], pagination) |> transform_keys(%{ params["prop_name"] => "name", - "events" => "total_conversions", + :events => "total_conversions", :visitors => "unique_conversions" }) |> Enum.map(fn prop -> From 8f3567142098225794d34ce147baa0d2cd6ec1e1 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 4 Jan 2022 12:41:36 +0000 Subject: [PATCH 082/148] ignore imported data when "events" in metrics --- lib/plausible/stats/base.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 49e98dbcefa1..f6a5c2533606 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -359,6 +359,7 @@ defmodule Plausible.Stats.Base do def merge_imported(q, %Plausible.Site{has_imported_stats: nil}, _, _, _), do: q def merge_imported(q, _, %Query{with_imported: false}, _, _), do: q + def merge_imported(q, _, _, _, [:events | _]), do: q def merge_imported(q, site, query, property, metrics) when property in [ From 1d852714691f5be69eb539b9936859c4d87bc74c Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 4 Jan 2022 15:07:48 +0000 Subject: [PATCH 083/148] update "bounce_rate" --- lib/plausible/stats/compare.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/plausible/stats/compare.ex b/lib/plausible/stats/compare.ex index 2e98d908c586..6c98638c61cb 100644 --- a/lib/plausible/stats/compare.ex +++ b/lib/plausible/stats/compare.ex @@ -1,7 +1,7 @@ defmodule Plausible.Stats.Compare do - def calculate_change("bounce_rate", old_stats, new_stats) do - old_count = old_stats["bounce_rate"]["value"] - new_count = new_stats["bounce_rate"]["value"] + def calculate_change(:bounce_rate, old_stats, new_stats) do + old_count = old_stats[:bounce_rate]["value"] + new_count = new_stats[:bounce_rate]["value"] if old_count > 0, do: new_count - old_count end From 0c50991d5de6828ffab31d256cf2d7592765cc08 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 4 Jan 2022 15:31:18 +0000 Subject: [PATCH 084/148] atomise some more metrics from new city and region api --- .../controllers/api/stats_controller.ex | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index e0b2d6f36917..34f6c13e0c05 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -466,15 +466,15 @@ defmodule PlausibleWeb.Api.StatsController do pages = Stats.breakdown(site, query, "event:page", metrics, pagination) |> maybe_add_cr(site, query, pagination, :page, "event:page") - |> transform_keys(%{page: "name", visitors: "visitors"}) + |> transform_keys(%{page: "name"}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do pages - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - pages |> to_csv(["name", "visitors", :bounce_rate, "time_on_page"]) + pages |> to_csv(["name", :visitors, :bounce_rate, "time_on_page"]) end else json(conn, pages) @@ -629,10 +629,10 @@ defmodule PlausibleWeb.Api.StatsController do if params["csv"] do if Map.has_key?(query.filters, "event:goal") do regions - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - regions |> to_csv(["name", "visitors"]) + regions |> to_csv(["name", :visitors]) end else json(conn, regions) @@ -667,10 +667,10 @@ defmodule PlausibleWeb.Api.StatsController do if params["csv"] do if Map.has_key?(query.filters, "event:goal") do cities - |> transform_keys(%{"visitors" => "conversions"}) + |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", "conversion_rate"]) else - cities |> to_csv(["name", "visitors"]) + cities |> to_csv(["name", :visitors]) end else json(conn, cities) From afdd5620413dd3fb96489f008ed2e5e2a01c1718 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 4 Jan 2022 15:31:41 +0000 Subject: [PATCH 085/148] atomise some more metrics for email handlers --- .../templates/email/weekly_report.html.eex | 8 ++++---- lib/workers/send_email_report.ex | 20 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/plausible_web/templates/email/weekly_report.html.eex b/lib/plausible_web/templates/email/weekly_report.html.eex index 6756d75669c2..3e61ec0a781a 100644 --- a/lib/plausible_web/templates/email/weekly_report.html.eex +++ b/lib/plausible_web/templates/email/weekly_report.html.eex @@ -434,7 +434,7 @@ body {
    -

    <%= source["source"] %>

    +

    <%= source[:source] %>

    @@ -453,7 +453,7 @@ body {
    -

    <%= PlausibleWeb.StatsView.large_number_format(source["visitors"]) %>

    +

    <%= PlausibleWeb.StatsView.large_number_format(source[:visitors]) %>

    @@ -563,7 +563,7 @@ body {
    -

    <%= page["page"] %>

    +

    <%= page[:page] %>

    @@ -582,7 +582,7 @@ body {
    -

    <%= PlausibleWeb.StatsView.large_number_format(page["visitors"]) %>

    +

    <%= PlausibleWeb.StatsView.large_number_format(page[:visitors]) %>

    diff --git a/lib/workers/send_email_report.ex b/lib/workers/send_email_report.ex index 5438a10ae5f4..d83520995615 100644 --- a/lib/workers/send_email_report.ex +++ b/lib/workers/send_email_report.ex @@ -50,26 +50,26 @@ defmodule Plausible.Workers.SendEmailReport do defp send_report(email, site, name, unsubscribe_link, query) do prev_query = Query.shift_back(query, site) - curr_period = Stats.aggregate(site, query, ["pageviews", "visitors", "bounce_rate"]) - prev_period = Stats.aggregate(site, prev_query, ["pageviews", "visitors", "bounce_rate"]) + curr_period = Stats.aggregate(site, query, [:pageviews, :visitors, :bounce_rate]) + prev_period = Stats.aggregate(site, prev_query, [:pageviews, :visitors, :bounce_rate]) - change_pageviews = Stats.Compare.calculate_change("pageviews", prev_period, curr_period) - change_visitors = Stats.Compare.calculate_change("visitors", prev_period, curr_period) - change_bounce_rate = Stats.Compare.calculate_change("bounce_rate", prev_period, curr_period) + change_pageviews = Stats.Compare.calculate_change(:pageviews, prev_period, curr_period) + change_visitors = Stats.Compare.calculate_change(:visitors, prev_period, curr_period) + change_bounce_rate = Stats.Compare.calculate_change(:bounce_rate, prev_period, curr_period) source_query = Query.put_filter(query, "visit:source", {:is_not, "Direct / None"}) - sources = Stats.breakdown(site, source_query, "visit:source", ["visitors"], {5, 1}) - pages = Stats.breakdown(site, query, "event:page", ["visitors"], {5, 1}) + sources = Stats.breakdown(site, source_query, "visit:source", [:visitors], {5, 1}) + pages = Stats.breakdown(site, query, "event:page", [:visitors], {5, 1}) user = Plausible.Auth.find_user_by(email: email) login_link = user && Plausible.Sites.is_member?(user.id, site) template = PlausibleWeb.Email.weekly_report(email, site, - unique_visitors: curr_period["visitors"]["value"], + unique_visitors: curr_period[:visitors]["value"], change_visitors: change_visitors, - pageviews: curr_period["pageviews"]["value"], + pageviews: curr_period[:pageviews]["value"], change_pageviews: change_pageviews, - bounce_rate: curr_period["bounce_rate"]["value"], + bounce_rate: curr_period[:bounce_rate]["value"], change_bounce_rate: change_bounce_rate, sources: sources, unsubscribe_link: unsubscribe_link, From 9aeca6f7cae042eb0313f20b9d8568ef5b6094ba Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 4 Jan 2022 16:34:32 +0000 Subject: [PATCH 086/148] "conversion_rate" -> :conversion_rate during csv export --- .../controllers/api/stats_controller.ex | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 34f6c13e0c05..fd7e846cdd52 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -242,7 +242,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do res |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) end @@ -272,7 +272,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do res |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) end @@ -302,7 +302,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do res |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) end @@ -331,7 +331,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do res |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) end @@ -360,7 +360,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do res |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) end @@ -390,7 +390,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do res |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) end @@ -472,7 +472,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do pages |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else pages |> to_csv(["name", :visitors, :bounce_rate, "time_on_page"]) end @@ -500,7 +500,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do entry_pages |> transform_keys(%{"unique_entrances" => "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else entry_pages |> to_csv(["name", "unique_entrances", "total_entrances", :visit_duration]) end @@ -554,7 +554,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do exit_pages |> transform_keys(%{"unique_exits" => "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else exit_pages |> to_csv(["name", "unique_exits", "total_exits", "exit_rate"]) end @@ -585,7 +585,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do countries |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else countries |> to_csv(["name", :visitors]) end @@ -594,12 +594,21 @@ defmodule PlausibleWeb.Api.StatsController do Enum.map(countries, fn row -> country = get_country(row["code"]) - Map.merge(row, %{ - "name" => country.name, - "flag" => country.flag, - "alpha_3" => country.alpha_3, - "code" => country.alpha_2 - }) + if country do + Map.merge(row, %{ + "name" => country.name, + "flag" => country.flag, + "alpha_3" => country.alpha_3, + "code" => country.alpha_2 + }) + else + Map.merge(row, %{ + "name" => row["code"], + "flag" => "", + "alpha_3" => "", + "code" => "" + }) + end end) json(conn, countries) @@ -630,7 +639,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do regions |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else regions |> to_csv(["name", :visitors]) end @@ -668,7 +677,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do cities |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else cities |> to_csv(["name", :visitors]) end @@ -692,7 +701,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do browsers |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else browsers |> to_csv(["name", :visitors]) end @@ -730,7 +739,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do systems |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else systems |> to_csv(["name", :visitors]) end @@ -768,7 +777,7 @@ defmodule PlausibleWeb.Api.StatsController do if Map.has_key?(query.filters, "event:goal") do sizes |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", "conversion_rate"]) + |> to_csv(["name", "conversions", :conversion_rate]) else sizes |> to_csv(["name", :visitors]) end From 173df311bd8a1e5682ef8da06dd4f11d22a158f7 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 5 Jan 2022 11:04:17 +0000 Subject: [PATCH 087/148] Move imported data stats code to own module --- lib/plausible/stats/aggregate.ex | 2 +- lib/plausible/stats/base.ex | 385 ------------------------------ lib/plausible/stats/breakdown.ex | 2 +- lib/plausible/stats/imported.ex | 393 +++++++++++++++++++++++++++++++ 4 files changed, 395 insertions(+), 387 deletions(-) create mode 100644 lib/plausible/stats/imported.ex diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 830c0bd9b86d..725e55498cbb 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -1,7 +1,7 @@ defmodule Plausible.Stats.Aggregate do alias Plausible.Stats.Query use Plausible.ClickhouseRepo - import Plausible.Stats.Base + import Plausible.Stats.{Base, Imported} @event_metrics [:visitors, :pageviews, :events, :sample_percent] @session_metrics [:visits, :bounce_rate, :visit_duration, :sample_percent] diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index f6a5c2533606..5452ada74848 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -357,391 +357,6 @@ defmodule Plausible.Stats.Base do |> String.replace(~r/(? - {"imported_utm_content", :utm_content} - - "visit:country" -> - {"imported_locations", :country} - - "visit:region" -> - {"imported_locations", :region} - - "visit:city" -> - {"imported_locations", :city} - - "visit:os" -> - {"imported_operating_systems", :operating_system} - - "event:page" -> - {"imported_pages", :page} - - _ -> - dim = String.trim_leading(property, "visit:") - {"imported_#{dim}s", String.to_existing_atom(dim)} - end - - imported_q = - from( - i in table, - group_by: field(i, ^dim), - where: i.domain == ^site.domain, - where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, - select: %{} - ) - |> select_imported_metrics(metrics) - - imported_q = - case query.filters[property] do - {:is_not, value} -> - value = if value == @no_ref, do: "", else: value - where(imported_q, [i], field(i, ^dim) != ^value) - - {:member, list} -> - where(imported_q, [i], field(i, ^dim) in ^list) - - _ -> - imported_q - end - - imported_q = - case dim do - :source -> - imported_q - |> select_merge([i], %{ - source: fragment("if(empty(?), ?, ?)", i.source, @no_ref, i.source) - }) - - :utm_medium -> - imported_q - |> select_merge([i], %{ - utm_medium: fragment("if(empty(?), ?, ?)", i.utm_medium, @no_ref, i.utm_medium) - }) - - :utm_source -> - imported_q - |> select_merge([i], %{ - utm_source: fragment("if(empty(?), ?, ?)", i.utm_source, @no_ref, i.utm_source) - }) - - :utm_campaign -> - imported_q - |> select_merge([i], %{ - utm_campaign: fragment("if(empty(?), ?, ?)", i.utm_campaign, @no_ref, i.utm_campaign) - }) - - :utm_term -> - imported_q - |> select_merge([i], %{ - utm_term: fragment("if(empty(?), ?, ?)", i.utm_term, @no_ref, i.utm_term) - }) - - :utm_content -> - imported_q - |> select_merge([i], %{ - utm_content: fragment("if(empty(?), ?, ?)", i.utm_content, @no_ref, i.utm_content) - }) - - :page -> - imported_q - |> select_merge([i], %{ - page: i.page, - time_on_page: sum(i.time_on_page) - }) - - :entry_page -> - imported_q - |> select_merge([i], %{ - entry_page: i.entry_page, - visits: sum(i.entrances) - }) - - :exit_page -> - imported_q - |> select_merge([i], %{exit_page: i.exit_page, visits: sum(i.exits)}) - - :country -> - imported_q |> select_merge([i], %{country: i.country}) - - :region -> - imported_q - |> select_merge([i], %{region: i.region}) - |> where([i], i.region != "") - - :city -> - imported_q - |> select_merge([i], %{city: i.city}) - |> where([i], i.city != 0) - - :device -> - imported_q |> select_merge([i], %{device: i.device}) - - :browser -> - imported_q |> select_merge([i], %{browser: i.browser}) - - :operating_system -> - imported_q |> select_merge([i], %{operating_system: i.operating_system}) - end - - q = - from(s in Ecto.Query.subquery(q), - full_join: i in subquery(imported_q), - on: field(s, ^dim) == field(i, ^dim) - ) - |> select_joined_metrics(metrics) - - case dim do - :source -> - q - |> select_merge([s, i], %{ - source: fragment("if(empty(?), ?, ?)", s.source, i.source, s.source) - }) - - :utm_medium -> - q - |> select_merge([s, i], %{ - utm_medium: fragment("if(empty(?), ?, ?)", s.utm_medium, i.utm_medium, s.utm_medium) - }) - - :utm_source -> - q - |> select_merge([s, i], %{ - utm_source: fragment("if(empty(?), ?, ?)", s.utm_source, i.utm_source, s.utm_source) - }) - - :utm_campaign -> - q - |> select_merge([s, i], %{ - utm_campaign: - fragment("if(empty(?), ?, ?)", s.utm_campaign, i.utm_campaign, s.utm_campaign) - }) - - :utm_term -> - q - |> select_merge([s, i], %{ - utm_term: fragment("if(empty(?), ?, ?)", s.utm_term, i.utm_term, s.utm_term) - }) - - :utm_content -> - q - |> select_merge([s, i], %{ - utm_content: fragment("if(empty(?), ?, ?)", s.utm_content, i.utm_content, s.utm_content) - }) - - :page -> - q - |> select_merge([s, i], %{ - page: fragment("if(empty(?), ?, ?)", i.page, s.page, i.page) - }) - - :entry_page -> - q - |> select_merge([s, i], %{ - entry_page: fragment("if(empty(?), ?, ?)", i.entry_page, s.entry_page, i.entry_page), - visits: fragment("? + ?", s.visits, i.visits) - }) - - :exit_page -> - q - |> select_merge([s, i], %{ - exit_page: fragment("if(empty(?), ?, ?)", i.exit_page, s.exit_page, i.exit_page), - visits: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visits, i.visits) - }) - - :country -> - q - |> select_merge([i, s], %{ - country: fragment("if(empty(?), ?, ?)", s.country, i.country, s.country) - }) - - :region -> - q - |> select_merge([i, s], %{ - region: fragment("if(empty(?), ?, ?)", s.region, i.region, s.region) - }) - - :city -> - q - |> select_merge([i, s], %{ - city: fragment("coalesce(?, ?)", s.city, i.city) - }) - - :device -> - q - |> select_merge([i, s], %{ - device: fragment("if(empty(?), ?, ?)", s.device, i.device, s.device) - }) - - :browser -> - q - |> select_merge([i, s], %{ - browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser) - }) - - :operating_system -> - q - |> select_merge([i, s], %{ - operating_system: - fragment( - "if(empty(?), ?, ?)", - s.operating_system, - i.operating_system, - s.operating_system - ) - }) - end - end - - def merge_imported(q, site, query, :aggregate, metrics) do - {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) - - imported_q = - from( - i in "imported_visitors", - where: i.domain == ^site.domain, - where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, - select: %{} - ) - |> select_imported_metrics(metrics) - - from( - s in subquery(q), - cross_join: i in subquery(imported_q), - select: %{} - ) - |> select_joined_metrics(metrics) - end - - def merge_imported(q, _, _, _, _), do: q - - def select_imported_metrics(q, []), do: q - - def select_imported_metrics(q, [:visitors | rest]) do - q - |> select_merge([i], %{visitors: sum(i.visitors)}) - |> select_imported_metrics(rest) - end - - def select_imported_metrics(q, [:pageviews | rest]) do - q - |> select_merge([i], %{pageviews: sum(i.pageviews)}) - |> select_imported_metrics(rest) - end - - def select_imported_metrics(q, [:bounce_rate | rest]) do - q - |> select_merge([i], %{ - bounces: sum(i.bounces), - visits: sum(i.visits) - }) - |> select_imported_metrics(rest) - end - - def select_imported_metrics(q, [:visit_duration | rest]) do - q - |> select_merge([i], %{visit_duration: sum(i.visit_duration)}) - |> select_imported_metrics(rest) - end - - def select_imported_metrics(q, [_ | rest]) do - q - |> select_imported_metrics(rest) - end - - def select_joined_metrics(q, []), do: q - # TODO: Reverse-engineering the native data bounces and total visit - # durations to combine with imported data is inefficient. Instead both - # queries should fetch bounces/total_visit_duration and visits and be - # used as subqueries to a main query that then find the bounce rate/avg - # visit_duration. - - def select_joined_metrics(q, [:visitors | rest]) do - q - |> select_merge([s, i], %{ - :visitors => fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) - }) - |> order_by([s, i], - desc: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) - ) - |> select_joined_metrics(rest) - end - - def select_joined_metrics(q, [:pageviews | rest]) do - q - |> select_merge([s, i], %{ - pageviews: fragment("coalesce(?, 0) + coalesce(?, 0)", s.pageviews, i.pageviews) - }) - |> select_joined_metrics(rest) - end - - def select_joined_metrics(q, [:bounce_rate | rest]) do - q - |> select_merge([s, i], %{ - bounce_rate: - fragment( - "round(100 * (coalesce(?, 0) + coalesce((? * ? / 100), 0)) / (coalesce(?, 0) + coalesce(?, 0)))", - i.bounces, - s.bounce_rate, - s.visits, - i.visits, - s.visits - ) - }) - |> select_joined_metrics(rest) - end - - def select_joined_metrics(q, [:visit_duration | rest]) do - q - |> select_merge([s, i], %{ - visit_duration: - fragment( - "(? + ? * ?) / (? + ?)", - i.visit_duration, - s.visit_duration, - s.visits, - s.visits, - i.visits - ) - }) - |> select_joined_metrics(rest) - end - - def select_joined_metrics(q, [:sample_percent | rest]) do - q - |> select_merge([s, i], %{sample_percent: s.sample_percent}) - |> select_joined_metrics(rest) - end - - def select_joined_metrics(q, [_ | rest]) do - q - |> select_joined_metrics(rest) - end - defp add_sample_hint(db_q, query) do case query.sample_threshold do "infinite" -> diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index e3e8a7460d3b..fe313a7d007c 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -1,6 +1,6 @@ defmodule Plausible.Stats.Breakdown do use Plausible.ClickhouseRepo - import Plausible.Stats.Base + import Plausible.Stats.{Base, Imported} alias Plausible.Stats.Query @no_ref "Direct / None" diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex new file mode 100644 index 000000000000..9e74e3442ead --- /dev/null +++ b/lib/plausible/stats/imported.ex @@ -0,0 +1,393 @@ +defmodule Plausible.Stats.Imported do + use Plausible.ClickhouseRepo + alias Plausible.Stats.Query + import Plausible.Stats.Base + import Ecto.Query + + @no_ref "Direct / None" + + def merge_imported(q, %Plausible.Site{has_imported_stats: nil}, _, _, _), do: q + def merge_imported(q, _, %Query{with_imported: false}, _, _), do: q + def merge_imported(q, _, _, _, [:events | _]), do: q + + def merge_imported(q, site, query, property, metrics) + when property in [ + "visit:source", + "visit:utm_medium", + "visit:utm_source", + "visit:utm_campaign", + "visit:utm_term", + "visit:utm_content", + "visit:entry_page", + "visit:exit_page", + "visit:country", + "visit:region", + "visit:city", + "visit:device", + "visit:browser", + "visit:os", + "event:page" + ] do + {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) + + {table, dim} = + case property do + "visit:utm_content" -> + {"imported_utm_content", :utm_content} + + "visit:country" -> + {"imported_locations", :country} + + "visit:region" -> + {"imported_locations", :region} + + "visit:city" -> + {"imported_locations", :city} + + "visit:os" -> + {"imported_operating_systems", :operating_system} + + "event:page" -> + {"imported_pages", :page} + + _ -> + dim = String.trim_leading(property, "visit:") + {"imported_#{dim}s", String.to_existing_atom(dim)} + end + + imported_q = + from( + i in table, + group_by: field(i, ^dim), + where: i.domain == ^site.domain, + where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, + select: %{} + ) + |> select_imported_metrics(metrics) + + imported_q = + case query.filters[property] do + {:is_not, value} -> + value = if value == @no_ref, do: "", else: value + where(imported_q, [i], field(i, ^dim) != ^value) + + {:member, list} -> + where(imported_q, [i], field(i, ^dim) in ^list) + + _ -> + imported_q + end + + imported_q = + case dim do + :source -> + imported_q + |> select_merge([i], %{ + source: fragment("if(empty(?), ?, ?)", i.source, @no_ref, i.source) + }) + + :utm_medium -> + imported_q + |> select_merge([i], %{ + utm_medium: fragment("if(empty(?), ?, ?)", i.utm_medium, @no_ref, i.utm_medium) + }) + + :utm_source -> + imported_q + |> select_merge([i], %{ + utm_source: fragment("if(empty(?), ?, ?)", i.utm_source, @no_ref, i.utm_source) + }) + + :utm_campaign -> + imported_q + |> select_merge([i], %{ + utm_campaign: fragment("if(empty(?), ?, ?)", i.utm_campaign, @no_ref, i.utm_campaign) + }) + + :utm_term -> + imported_q + |> select_merge([i], %{ + utm_term: fragment("if(empty(?), ?, ?)", i.utm_term, @no_ref, i.utm_term) + }) + + :utm_content -> + imported_q + |> select_merge([i], %{ + utm_content: fragment("if(empty(?), ?, ?)", i.utm_content, @no_ref, i.utm_content) + }) + + :page -> + imported_q + |> select_merge([i], %{ + page: i.page, + time_on_page: sum(i.time_on_page) + }) + + :entry_page -> + imported_q + |> select_merge([i], %{ + entry_page: i.entry_page, + visits: sum(i.entrances) + }) + + :exit_page -> + imported_q + |> select_merge([i], %{exit_page: i.exit_page, visits: sum(i.exits)}) + + :country -> + imported_q |> select_merge([i], %{country: i.country}) + + :region -> + imported_q + |> select_merge([i], %{region: i.region}) + |> where([i], i.region != "") + + :city -> + imported_q + |> select_merge([i], %{city: i.city}) + |> where([i], i.city != 0) + + :device -> + imported_q |> select_merge([i], %{device: i.device}) + + :browser -> + imported_q |> select_merge([i], %{browser: i.browser}) + + :operating_system -> + imported_q |> select_merge([i], %{operating_system: i.operating_system}) + end + + q = + from(s in Ecto.Query.subquery(q), + full_join: i in subquery(imported_q), + on: field(s, ^dim) == field(i, ^dim) + ) + |> select_joined_metrics(metrics) + + case dim do + :source -> + q + |> select_merge([s, i], %{ + source: fragment("if(empty(?), ?, ?)", s.source, i.source, s.source) + }) + + :utm_medium -> + q + |> select_merge([s, i], %{ + utm_medium: fragment("if(empty(?), ?, ?)", s.utm_medium, i.utm_medium, s.utm_medium) + }) + + :utm_source -> + q + |> select_merge([s, i], %{ + utm_source: fragment("if(empty(?), ?, ?)", s.utm_source, i.utm_source, s.utm_source) + }) + + :utm_campaign -> + q + |> select_merge([s, i], %{ + utm_campaign: + fragment("if(empty(?), ?, ?)", s.utm_campaign, i.utm_campaign, s.utm_campaign) + }) + + :utm_term -> + q + |> select_merge([s, i], %{ + utm_term: fragment("if(empty(?), ?, ?)", s.utm_term, i.utm_term, s.utm_term) + }) + + :utm_content -> + q + |> select_merge([s, i], %{ + utm_content: fragment("if(empty(?), ?, ?)", s.utm_content, i.utm_content, s.utm_content) + }) + + :page -> + q + |> select_merge([s, i], %{ + page: fragment("if(empty(?), ?, ?)", i.page, s.page, i.page) + }) + + :entry_page -> + q + |> select_merge([s, i], %{ + entry_page: fragment("if(empty(?), ?, ?)", i.entry_page, s.entry_page, i.entry_page), + visits: fragment("? + ?", s.visits, i.visits) + }) + + :exit_page -> + q + |> select_merge([s, i], %{ + exit_page: fragment("if(empty(?), ?, ?)", i.exit_page, s.exit_page, i.exit_page), + visits: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visits, i.visits) + }) + + :country -> + q + |> select_merge([i, s], %{ + country: fragment("if(empty(?), ?, ?)", s.country, i.country, s.country) + }) + + :region -> + q + |> select_merge([i, s], %{ + region: fragment("if(empty(?), ?, ?)", s.region, i.region, s.region) + }) + + :city -> + q + |> select_merge([i, s], %{ + city: fragment("coalesce(?, ?)", s.city, i.city) + }) + + :device -> + q + |> select_merge([i, s], %{ + device: fragment("if(empty(?), ?, ?)", s.device, i.device, s.device) + }) + + :browser -> + q + |> select_merge([i, s], %{ + browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser) + }) + + :operating_system -> + q + |> select_merge([i, s], %{ + operating_system: + fragment( + "if(empty(?), ?, ?)", + s.operating_system, + i.operating_system, + s.operating_system + ) + }) + end + end + + def merge_imported(q, site, query, :aggregate, metrics) do + {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) + + imported_q = + from( + i in "imported_visitors", + where: i.domain == ^site.domain, + where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, + select: %{} + ) + |> select_imported_metrics(metrics) + + from( + s in subquery(q), + cross_join: i in subquery(imported_q), + select: %{} + ) + |> select_joined_metrics(metrics) + end + + def merge_imported(q, _, _, _, _), do: q + + defp select_imported_metrics(q, []), do: q + + defp select_imported_metrics(q, [:visitors | rest]) do + q + |> select_merge([i], %{visitors: sum(i.visitors)}) + |> select_imported_metrics(rest) + end + + defp select_imported_metrics(q, [:pageviews | rest]) do + q + |> select_merge([i], %{pageviews: sum(i.pageviews)}) + |> select_imported_metrics(rest) + end + + defp select_imported_metrics(q, [:bounce_rate | rest]) do + q + |> select_merge([i], %{ + bounces: sum(i.bounces), + visits: sum(i.visits) + }) + |> select_imported_metrics(rest) + end + + defp select_imported_metrics(q, [:visit_duration | rest]) do + q + |> select_merge([i], %{visit_duration: sum(i.visit_duration)}) + |> select_imported_metrics(rest) + end + + defp select_imported_metrics(q, [_ | rest]) do + q + |> select_imported_metrics(rest) + end + + defp select_joined_metrics(q, []), do: q + # TODO: Reverse-engineering the native data bounces and total visit + # durations to combine with imported data is inefficient. Instead both + # queries should fetch bounces/total_visit_duration and visits and be + # used as subqueries to a main query that then find the bounce rate/avg + # visit_duration. + + defp select_joined_metrics(q, [:visitors | rest]) do + q + |> select_merge([s, i], %{ + :visitors => fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) + }) + |> order_by([s, i], + desc: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) + ) + |> select_joined_metrics(rest) + end + + defp select_joined_metrics(q, [:pageviews | rest]) do + q + |> select_merge([s, i], %{ + pageviews: fragment("coalesce(?, 0) + coalesce(?, 0)", s.pageviews, i.pageviews) + }) + |> select_joined_metrics(rest) + end + + defp select_joined_metrics(q, [:bounce_rate | rest]) do + q + |> select_merge([s, i], %{ + bounce_rate: + fragment( + "round(100 * (coalesce(?, 0) + coalesce((? * ? / 100), 0)) / (coalesce(?, 0) + coalesce(?, 0)))", + i.bounces, + s.bounce_rate, + s.visits, + i.visits, + s.visits + ) + }) + |> select_joined_metrics(rest) + end + + defp select_joined_metrics(q, [:visit_duration | rest]) do + q + |> select_merge([s, i], %{ + visit_duration: + fragment( + "(? + ? * ?) / (? + ?)", + i.visit_duration, + s.visit_duration, + s.visits, + s.visits, + i.visits + ) + }) + |> select_joined_metrics(rest) + end + + defp select_joined_metrics(q, [:sample_percent | rest]) do + q + |> select_merge([s, i], %{sample_percent: s.sample_percent}) + |> select_joined_metrics(rest) + end + + defp select_joined_metrics(q, [_ | rest]) do + q + |> select_joined_metrics(rest) + end +end From f09a99d0dc4103a48ffe766642078afddd22aaef Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 5 Jan 2022 11:13:13 +0000 Subject: [PATCH 088/148] Move imported timeseries function to Stats.Imported --- lib/plausible/imported/visitors.ex | 20 ------------------ lib/plausible/stats/imported.ex | 21 ++++++++++++++++++- .../controllers/api/stats_controller.ex | 3 +-- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/lib/plausible/imported/visitors.ex b/lib/plausible/imported/visitors.ex index 1636b5e2d275..ca087b27510f 100644 --- a/lib/plausible/imported/visitors.ex +++ b/lib/plausible/imported/visitors.ex @@ -2,7 +2,6 @@ defmodule Plausible.Imported.Visitors do use Ecto.Schema use Plausible.ClickhouseRepo import Ecto.Changeset - alias Plausible.Stats @primary_key false schema "imported_visitors" do @@ -41,23 +40,4 @@ defmodule Plausible.Imported.Visitors do :visit_duration ]) end - - def timeseries(site, query) do - {first_datetime, last_datetime} = Stats.Base.utc_boundaries(query, site.timezone) - - result = - from(v in "imported_visitors", - group_by: fragment("date"), - where: v.domain == ^site.domain, - where: v.timestamp >= ^first_datetime and v.timestamp < ^last_datetime, - select: %{"visitors" => sum(v.visitors)} - ) - |> Stats.Timeseries.select_bucket(site, query) - |> ClickhouseRepo.all() - |> Enum.map(fn row -> {row["date"], row["visitors"]} end) - |> Map.new() - - Stats.Timeseries.buckets(query) - |> Enum.map(fn step -> Map.get(result, step, 0) end) - end end diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index 9e74e3442ead..2dae75763dad 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -1,11 +1,30 @@ defmodule Plausible.Stats.Imported do use Plausible.ClickhouseRepo - alias Plausible.Stats.Query + alias Plausible.Stats.{Query, Timeseries} import Plausible.Stats.Base import Ecto.Query @no_ref "Direct / None" + def timeseries(site, query) do + {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) + + result = + from(v in "imported_visitors", + group_by: fragment("date"), + where: v.domain == ^site.domain, + where: v.timestamp >= ^first_datetime and v.timestamp < ^last_datetime, + select: %{"visitors" => sum(v.visitors)} + ) + |> Timeseries.select_bucket(site, query) + |> ClickhouseRepo.all() + |> Enum.map(fn row -> {row["date"], row["visitors"]} end) + |> Map.new() + + Timeseries.buckets(query) + |> Enum.map(fn step -> Map.get(result, step, 0) end) + end + def merge_imported(q, %Plausible.Site{has_imported_stats: nil}, _, _, _), do: q def merge_imported(q, _, %Query{with_imported: false}, _, _), do: q def merge_imported(q, _, _, _, [:events | _]), do: q diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index fd7e846cdd52..9b2beff221c8 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -4,7 +4,6 @@ defmodule PlausibleWeb.Api.StatsController do use Plug.ErrorHandler alias Plausible.Stats alias Plausible.Stats.{Query, Filters} - alias Plausible.Imported def main_graph(conn, params) do site = conn.assigns[:site] @@ -29,7 +28,7 @@ defmodule PlausibleWeb.Api.StatsController do if query.with_imported && site.has_imported_stats do # Showing imported data. plot = - Imported.Visitors.timeseries(site, timeseries_query) + Stats.Imported.timeseries(site, timeseries_query) |> Enum.zip_with(plot, &(&1 + &2)) {plot, true, site.has_imported_stats} From 8acc0857464445577512f1d07a0b67279533d46f Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 5 Jan 2022 15:38:20 +0000 Subject: [PATCH 089/148] Use Timex.parse to import dates from GA --- lib/plausible/imported/site.ex | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index d49f894b0860..09acb306a11d 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -287,12 +287,6 @@ defmodule Plausible.Imported do end defp format_timestamp(timestamp) do - {year, monthday} = String.split_at(timestamp, 4) - {month, day} = String.split_at(monthday, 2) - - [year, month, day] - |> Enum.map(&Kernel.elem(Integer.parse(&1), 0)) - |> List.to_tuple() - |> (&NaiveDateTime.from_erl!({&1, {12, 0, 0}})).() + Timex.Parse.DateTime.Parser.parse!(timestamp, "{YYYY}{M}{D}") end end From 95e38a4828cee8f727b5eab46008e9295773bebe Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 5 Jan 2022 15:44:32 +0000 Subject: [PATCH 090/148] has_imported_stats -> imported_source --- lib/plausible/site/schema.ex | 4 ++-- lib/plausible/stats/breakdown.ex | 2 +- lib/plausible/stats/imported.ex | 2 +- .../controllers/api/stats_controller.ex | 6 +++--- lib/plausible_web/controllers/site_controller.ex | 14 +++++++------- .../20211110174617_add_site_imported_boolean.exs | 9 --------- .../20211110174617_add_site_imported_source.exs | 9 +++++++++ 7 files changed, 23 insertions(+), 23 deletions(-) delete mode 100644 priv/repo/migrations/20211110174617_add_site_imported_boolean.exs create mode 100644 priv/repo/migrations/20211110174617_add_site_imported_source.exs diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index fc6437577d14..47ef81999891 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -11,7 +11,7 @@ defmodule Plausible.Site do field :public, :boolean field :locked, :boolean field :has_stats, :boolean - field :has_imported_stats, :string + field :imported_source, :string many_to_many :members, User, join_through: Plausible.Site.Membership has_many :memberships, Plausible.Site.Membership @@ -27,7 +27,7 @@ defmodule Plausible.Site do def changeset(site, attrs \\ %{}) do site - |> cast(attrs, [:domain, :timezone, :has_imported_stats]) + |> cast(attrs, [:domain, :timezone, :imported_source]) |> validate_required([:domain, :timezone]) |> validate_format(:domain, ~r/^[a-zA-Z0-9\-\.\/\:]*$/, message: "only letters, numbers, slashes and period allowed" diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index fe313a7d007c..df5248e4e670 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -246,7 +246,7 @@ defmodule Plausible.Stats.Breakdown do {base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q) - with_imported = query.with_imported && site.has_imported_stats + with_imported = query.with_imported && site.imported_source select = if with_imported do diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index 2dae75763dad..dc31f3c802bd 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -25,7 +25,7 @@ defmodule Plausible.Stats.Imported do |> Enum.map(fn step -> Map.get(result, step, 0) end) end - def merge_imported(q, %Plausible.Site{has_imported_stats: nil}, _, _, _), do: q + def merge_imported(q, %Plausible.Site{imported_source: nil}, _, _, _), do: q def merge_imported(q, _, %Query{with_imported: false}, _, _), do: q def merge_imported(q, _, _, _, [:events | _]), do: q diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 9b2beff221c8..27cef7b7f508 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -25,13 +25,13 @@ defmodule PlausibleWeb.Api.StatsController do present_index = present_index_for(site, query, labels) {plot, with_imported, source} = - if query.with_imported && site.has_imported_stats do + if query.with_imported && site.imported_source do # Showing imported data. plot = Stats.Imported.timeseries(site, timeseries_query) |> Enum.zip_with(plot, &(&1 + &2)) - {plot, true, site.has_imported_stats} + {plot, true, site.imported_source} else if Map.get(params, "filters", "{}") != "{}" do # Hiding imported data due to filtering. @@ -39,7 +39,7 @@ defmodule PlausibleWeb.Api.StatsController do {plot, false, ""} else # Hiding imported data either by request or because there is none. - {plot, false, site.has_imported_stats || ""} + {plot, false, site.imported_source || ""} end end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index f7a1046007ff..489bb54d36a3 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -173,7 +173,7 @@ defmodule PlausibleWeb.SiteController do |> Repo.preload([:custom_domain, :google_auth]) google_profiles = - if is_nil(site.has_imported_stats) and site.google_auth do + if is_nil(site.imported_source) and site.google_auth do Plausible.Google.Api.get_analytics_view_ids(site) end @@ -182,7 +182,7 @@ defmodule PlausibleWeb.SiteController do |> render("settings_general.html", site: site, google_profiles: google_profiles, - imported_from: site.has_imported_stats, + imported_from: site.imported_source, changeset: Plausible.Site.changeset(site, %{}), layout: {PlausibleWeb.LayoutView, "site_settings.html"} ) @@ -643,9 +643,9 @@ defmodule PlausibleWeb.SiteController do |> Repo.preload(:google_auth) cond do - site.has_imported_stats -> + site.imported_source -> conn - |> put_flash(:error, "Data already imported from: #{site.has_imported_stats}") + |> put_flash(:error, "Data already imported from: #{site.imported_source}") |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) profile == "" -> @@ -657,7 +657,7 @@ defmodule PlausibleWeb.SiteController do case Plausible.Google.Api.import_analytics(site, profile) do {:ok, _} -> site - |> Plausible.Site.changeset(%{has_imported_stats: "Google Analytics"}) + |> Plausible.Site.changeset(%{imported_source: "Google Analytics"}) |> Repo.update!() conn @@ -681,11 +681,11 @@ defmodule PlausibleWeb.SiteController do site = conn.assigns[:site] cond do - site.has_imported_stats -> + site.imported_source -> Plausible.Imported.forget(site) site - |> Plausible.Site.changeset(%{has_imported_stats: nil}) + |> Plausible.Site.changeset(%{imported_source: nil}) |> Repo.update!() conn diff --git a/priv/repo/migrations/20211110174617_add_site_imported_boolean.exs b/priv/repo/migrations/20211110174617_add_site_imported_boolean.exs deleted file mode 100644 index e6461b4cab59..000000000000 --- a/priv/repo/migrations/20211110174617_add_site_imported_boolean.exs +++ /dev/null @@ -1,9 +0,0 @@ -defmodule Plausible.Repo.Migrations.GoogleAuthImportedBoolean do - use Ecto.Migration - - def change do - alter table(:sites) do - add :has_imported_stats, :string, null: true, default: nil - end - end -end diff --git a/priv/repo/migrations/20211110174617_add_site_imported_source.exs b/priv/repo/migrations/20211110174617_add_site_imported_source.exs new file mode 100644 index 000000000000..f04c2899a46a --- /dev/null +++ b/priv/repo/migrations/20211110174617_add_site_imported_source.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.GoogleAuthImportedSource do + use Ecto.Migration + + def change do + alter table(:sites) do + add :imported_source, :string, null: true, default: nil + end + end +end From 88d925d1b5c21c099e7afa51b4318804bdc41fbd Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 5 Jan 2022 16:05:52 +0000 Subject: [PATCH 091/148] "time_on_page" -> :time_on_page --- lib/plausible/stats/aggregate.ex | 4 ++-- lib/plausible/stats/breakdown.ex | 4 ++-- lib/plausible_web/controllers/api/stats_controller.ex | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index 725e55498cbb..f833a5003cf9 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -13,7 +13,7 @@ defmodule Plausible.Stats.Aggregate do session_task = Task.async(fn -> aggregate_sessions(site, query, session_metrics) end) time_on_page_task = - if "time_on_page" in metrics do + if :time_on_page in metrics do Task.async(fn -> aggregate_time_on_page(site, query) end) else Task.async(fn -> %{} end) @@ -108,6 +108,6 @@ defmodule Plausible.Stats.Aggregate do {:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [where_arg]) [[time_on_page]] = res.rows - %{"time_on_page" => time_on_page} + %{time_on_page: time_on_page} end end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index df5248e4e670..6cdb416ab7ee 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -95,12 +95,12 @@ defmodule Plausible.Stats.Breakdown do event_result = breakdown_events(site, query, "event:page", event_metrics, pagination) event_result = - if "time_on_page" in metrics do + if :time_on_page in metrics do pages = Enum.map(event_result, & &1[:page]) time_on_page_result = breakdown_time_on_page(site, query, pages) Enum.map(event_result, fn row -> - Map.put(row, "time_on_page", time_on_page_result[row[:page]]) + Map.put(row, :time_on_page, time_on_page_result[row[:page]]) end) else event_result diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 27cef7b7f508..5fc85b5b29b1 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -167,7 +167,7 @@ defmodule PlausibleWeb.Api.StatsController do metrics = if query.filters["event:page"] do - [:visitors, :pageviews, :bounce_rate, "time_on_page", :sample_percent] + [:visitors, :pageviews, :bounce_rate, :time_on_page, :sample_percent] else [:visitors, :pageviews, :bounce_rate, :visit_duration, :sample_percent] end @@ -181,7 +181,7 @@ defmodule PlausibleWeb.Api.StatsController do top_stats_entry(current_results, prev_results, "Total pageviews", :pageviews), top_stats_entry(current_results, prev_results, "Bounce rate", :bounce_rate), top_stats_entry(current_results, prev_results, "Visit duration", :visit_duration), - top_stats_entry(current_results, prev_results, "Time on page", "time_on_page") + top_stats_entry(current_results, prev_results, "Time on page", :time_on_page) ] |> Enum.filter(& &1) @@ -457,7 +457,7 @@ defmodule PlausibleWeb.Api.StatsController do metrics = if params["detailed"], - do: [:visitors, :pageviews, :bounce_rate, "time_on_page"], + do: [:visitors, :pageviews, :bounce_rate, :time_on_page], else: [:visitors] pagination = parse_pagination(params) @@ -473,7 +473,7 @@ defmodule PlausibleWeb.Api.StatsController do |> transform_keys(%{visitors: "conversions"}) |> to_csv(["name", "conversions", :conversion_rate]) else - pages |> to_csv(["name", :visitors, :bounce_rate, "time_on_page"]) + pages |> to_csv(["name", :visitors, :bounce_rate, :time_on_page]) end else json(conn, pages) From b337e72591485017cfba57c08caaee1e9e8e7539 Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 5 Jan 2022 18:05:11 +0000 Subject: [PATCH 092/148] Convert imported GA data to UTC --- lib/plausible/google/api.ex | 61 ++++++++++++++++++++++++-------- lib/plausible/imported/site.ex | 64 ++++++++++++++++++---------------- 2 files changed, 80 insertions(+), 45 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index da9f36030dcf..abe737d251b7 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -1,5 +1,6 @@ defmodule Plausible.Google.Api do alias Plausible.Imported + use Timex @scope URI.encode_www_form( "https://www.googleapis.com/auth/webmasters.readonly email https://www.googleapis.com/auth/analytics.readonly" @@ -180,7 +181,7 @@ defmodule Plausible.Google.Api do request_data = [ # Visitors { - ["ga:date"], + ["ga:dateHour"], [ "ga:users", "ga:pageviews", @@ -191,62 +192,62 @@ defmodule Plausible.Google.Api do }, # Sources { - ["ga:date", "ga:fullReferrer"], + ["ga:dateHour", "ga:fullReferrer"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, # UTM Mediums { - ["ga:date", "ga:medium"], + ["ga:dateHour", "ga:medium"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, # UTM Campaigns { - ["ga:date", "ga:campaign"], + ["ga:dateHour", "ga:campaign"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, # UTM Terms { - ["ga:date", "ga:keyword"], + ["ga:dateHour", "ga:keyword"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, # UTM Content { - ["ga:date", "ga:adContent"], + ["ga:dateHour", "ga:adContent"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, # Pages { - ["ga:date", "ga:pagePath"], + ["ga:dateHour", "ga:pagePath"], ["ga:users", "ga:pageviews", "ga:timeOnPage"] }, # Entry pages { - ["ga:date", "ga:landingPagePath"], + ["ga:dateHour", "ga:landingPagePath"], ["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"] }, # Exit pages { - ["ga:date", "ga:exitPagePath"], + ["ga:dateHour", "ga:exitPagePath"], ["ga:users", "ga:exits"] }, # Locations { - ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode"], + ["ga:dateHour", "ga:countryIsoCode", "ga:regionIsoCode"], ["ga:users"] }, # Device { - ["ga:date", "ga:deviceCategory"], + ["ga:dateHour", "ga:deviceCategory"], ["ga:users"] }, # Browser { - ["ga:date", "ga:browser"], + ["ga:dateHour", "ga:browser"], ["ga:users"] }, # OS { - ["ga:date", "ga:operatingSystem"], + ["ga:dateHour", "ga:operatingSystem"], ["ga:users"] } ] @@ -264,6 +265,8 @@ defmodule Plausible.Google.Api do |> Enum.map(fn {:ok, resp} -> resp end) |> Enum.concat() + {:ok, timezone} = get_profile_timezone(auth, profile) + maybe_error = [ "visitors", @@ -284,7 +287,7 @@ defmodule Plausible.Google.Api do |> Enum.map(fn {metric, index} -> Task.async(fn -> Enum.fetch!(data, index) - |> Imported.from_google_analytics(site.domain, metric) + |> Imported.from_google_analytics(site.domain, metric, timezone) end) end) |> Enum.map(&Task.await(&1, 120_000)) @@ -324,7 +327,7 @@ defmodule Plausible.Google.Api do hideValueRanges: true, orderBys: [ %{ - fieldName: "ga:date", + fieldName: "ga:dateHour", sortOrder: "DESCENDING" } ], @@ -350,6 +353,34 @@ defmodule Plausible.Google.Api do end end + defp get_profile_timezone(auth, profile) do + res = + HTTPoison.get!( + "https://www.googleapis.com/analytics/v3/management/accounts/~all/webproperties/~all/profiles", + Authorization: "Bearer #{auth.access_token}" + ) + + case res.status_code do + 200 -> + timezone_info = + Jason.decode!(res.body) + |> Map.get("items") + |> Enum.map(fn item -> {Map.get(item, "id"), Map.get(item, "timezone")} end) + |> Map.new() + |> Map.get(profile) + |> Timezone.get() + + {:ok, timezone_info} + + _ -> + Sentry.capture_message("Error fetching Google view ID during import", + extra: Jason.decode!(res.body) + ) + + {:error, res.body} + end + end + defp refresh_if_needed(auth) do if Timex.before?(auth.expires, Timex.now() |> Timex.shift(seconds: 30)) do refresh_token(auth) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 09acb306a11d..7e0eb7881530 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -1,15 +1,16 @@ defmodule Plausible.Imported do alias Plausible.Imported + use Timex def forget(site) do Plausible.ClickhouseRepo.clear_imported_stats_for(site.domain) end - def from_google_analytics(data, domain, metric) do + def from_google_analytics(data, domain, metric, timezone) do maybe_error = data |> Enum.map(fn row -> - new_from_google_analytics(domain, metric, row) + new_from_google_analytics(domain, timezone, metric, row) |> Plausible.ClickhouseRepo.insert(on_conflict: :replace_all) end) |> Keyword.get(:error) @@ -23,7 +24,7 @@ defmodule Plausible.Imported do end end - defp new_from_google_analytics(domain, "visitors", %{ + defp new_from_google_analytics(domain, timezone, "visitors", %{ "dimensions" => [timestamp], "metrics" => [%{"values" => values}] }) do @@ -34,7 +35,7 @@ defmodule Plausible.Imported do Imported.Visitors.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), visitors: visitors, pageviews: pageviews, bounces: bounces, @@ -43,7 +44,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, "sources", %{ + defp new_from_google_analytics(domain, timezone, "sources", %{ "dimensions" => [timestamp, source], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -56,7 +57,7 @@ defmodule Plausible.Imported do Imported.Sources.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), source: Imported.Sources.parse(source), visitors: visitors, visits: visits, @@ -67,7 +68,7 @@ defmodule Plausible.Imported do # TODO: utm_sources. Google reports sources and utm_sources unified. - defp new_from_google_analytics(domain, "utm_mediums", %{ + defp new_from_google_analytics(domain, timezone, "utm_mediums", %{ "dimensions" => [timestamp, medium], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -80,7 +81,7 @@ defmodule Plausible.Imported do Imported.UtmMediums.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), medium: medium, visitors: visitors, visits: visits, @@ -89,7 +90,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, "utm_campaigns", %{ + defp new_from_google_analytics(domain, timezone, "utm_campaigns", %{ "dimensions" => [timestamp, campaign], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -102,7 +103,7 @@ defmodule Plausible.Imported do Imported.UtmCampaigns.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), campaign: campaign, visitors: visitors, visits: visits, @@ -111,7 +112,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, "utm_terms", %{ + defp new_from_google_analytics(domain, timezone, "utm_terms", %{ "dimensions" => [timestamp, term], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -124,7 +125,7 @@ defmodule Plausible.Imported do Imported.UtmTerms.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), utm_term: term, visitors: visitors, visits: visits, @@ -133,7 +134,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, "utm_content", %{ + defp new_from_google_analytics(domain, timezone, "utm_content", %{ "dimensions" => [timestamp, content], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -146,7 +147,7 @@ defmodule Plausible.Imported do Imported.UtmContent.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), utm_content: content, visitors: visitors, visits: visits, @@ -155,7 +156,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, "pages", %{ + defp new_from_google_analytics(domain, timezone, "pages", %{ "dimensions" => [timestamp, page], "metrics" => [%{"values" => [value, pageviews, time_on_page]}] }) do @@ -165,7 +166,7 @@ defmodule Plausible.Imported do Imported.Pages.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), page: page, visitors: visitors, pageviews: pageviews, @@ -173,7 +174,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, "entry_pages", %{ + defp new_from_google_analytics(domain, timezone, "entry_pages", %{ "dimensions" => [timestamp, entry_page], "metrics" => [%{"values" => [visitors, entrances, visit_duration, bounces]}] }) do @@ -185,7 +186,7 @@ defmodule Plausible.Imported do Imported.EntryPages.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), entry_page: entry_page, visitors: visitors, entrances: entrances, @@ -194,7 +195,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, "exit_pages", %{ + defp new_from_google_analytics(domain, timezone, "exit_pages", %{ "dimensions" => [timestamp, exit_page], "metrics" => [%{"values" => [value, exits]}] }) do @@ -203,14 +204,14 @@ defmodule Plausible.Imported do Imported.ExitPages.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), exit_page: exit_page, visitors: visitors, exits: exits }) end - defp new_from_google_analytics(domain, "locations", %{ + defp new_from_google_analytics(domain, timezone, "locations", %{ "dimensions" => [timestamp, country, region], "metrics" => [%{"values" => [value]}] }) do @@ -220,7 +221,7 @@ defmodule Plausible.Imported do Imported.Locations.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), country: country, region: region, city: 0, @@ -228,7 +229,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, "devices", %{ + defp new_from_google_analytics(domain, timezone, "devices", %{ "dimensions" => [timestamp, device], "metrics" => [%{"values" => [value]}] }) do @@ -236,7 +237,7 @@ defmodule Plausible.Imported do Imported.Devices.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), device: String.capitalize(device), visitors: visitors }) @@ -252,7 +253,7 @@ defmodule Plausible.Imported do "(not set)" => "" } - defp new_from_google_analytics(domain, "browsers", %{ + defp new_from_google_analytics(domain, timezone, "browsers", %{ "dimensions" => [timestamp, browser], "metrics" => [%{"values" => [value]}] }) do @@ -260,7 +261,7 @@ defmodule Plausible.Imported do Imported.Browsers.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), browser: Map.get(@browser_google_to_plausible, browser, browser), visitors: visitors }) @@ -272,7 +273,7 @@ defmodule Plausible.Imported do "(not set)" => "" } - defp new_from_google_analytics(domain, "operating_systems", %{ + defp new_from_google_analytics(domain, timezone, "operating_systems", %{ "dimensions" => [timestamp, operating_system], "metrics" => [%{"values" => [value]}] }) do @@ -280,13 +281,16 @@ defmodule Plausible.Imported do Imported.OperatingSystems.new(%{ domain: domain, - timestamp: format_timestamp(timestamp), + timestamp: format_timestamp(timestamp, timezone), operating_system: Map.get(@os_google_to_plausible, operating_system, operating_system), visitors: visitors }) end - defp format_timestamp(timestamp) do - Timex.Parse.DateTime.Parser.parse!(timestamp, "{YYYY}{M}{D}") + defp format_timestamp(timestamp, timezone) do + Timex.parse!("#{timestamp}", "%Y%m%d%H", :strftime) + |> Timezone.convert(timezone) + |> Timezone.convert("UTC") + |> DateTime.to_naive() end end From 0e48b25786293e10c609cd999b27846eb0839b6a Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 5 Jan 2022 18:36:06 +0000 Subject: [PATCH 093/148] Clean up GA request code a bit There was some weird logic here with two separate lists that really ought to be together, so this merges those. --- lib/plausible/google/api.ex | 65 +++++++++++++++---------------------- 1 file changed, 26 insertions(+), 39 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index abe737d251b7..9ffc38611577 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -178,9 +178,10 @@ defmodule Plausible.Google.Api do end_date: Date.to_iso8601(end_date) } + # Each element is: {dataset, dimensions, metrics} request_data = [ - # Visitors { + "visitors", ["ga:dateHour"], [ "ga:users", @@ -190,63 +191,63 @@ defmodule Plausible.Google.Api do "ga:sessionDuration" ] }, - # Sources { + "sources", ["ga:dateHour", "ga:fullReferrer"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, - # UTM Mediums { + "utm_mediums", ["ga:dateHour", "ga:medium"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, - # UTM Campaigns { + "utm_campaigns", ["ga:dateHour", "ga:campaign"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, - # UTM Terms { + "utm_terms", ["ga:dateHour", "ga:keyword"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, - # UTM Content { + "utm_content", ["ga:dateHour", "ga:adContent"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, - # Pages { + "pages", ["ga:dateHour", "ga:pagePath"], ["ga:users", "ga:pageviews", "ga:timeOnPage"] }, - # Entry pages { + "entry_pages", ["ga:dateHour", "ga:landingPagePath"], ["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"] }, - # Exit pages { + "exit_pages", ["ga:dateHour", "ga:exitPagePath"], ["ga:users", "ga:exits"] }, - # Locations { + "locations", ["ga:dateHour", "ga:countryIsoCode", "ga:regionIsoCode"], ["ga:users"] }, - # Device { + "devices", ["ga:dateHour", "ga:deviceCategory"], ["ga:users"] }, - # Browser { + "browsers", ["ga:dateHour", "ga:browser"], ["ga:users"] }, - # OS { + "operating_systems", ["ga:dateHour", "ga:operatingSystem"], ["ga:users"] } @@ -260,34 +261,15 @@ defmodule Plausible.Google.Api do case Keyword.get(responses, :error) do nil -> - data = - responses - |> Enum.map(fn {:ok, resp} -> resp end) - |> Enum.concat() - {:ok, timezone} = get_profile_timezone(auth, profile) maybe_error = - [ - "visitors", - "sources", - "utm_mediums", - "utm_campaigns", - "utm_terms", - "utm_content", - "pages", - "entry_pages", - "exit_pages", - "locations", - "devices", - "browsers", - "operating_systems" - ] - |> Enum.with_index() - |> Enum.map(fn {metric, index} -> + responses + |> Enum.map(fn {:ok, resp} -> resp end) + |> Enum.concat() + |> Enum.map(fn {dataset, data} -> Task.async(fn -> - Enum.fetch!(data, index) - |> Imported.from_google_analytics(site.domain, metric, timezone) + Imported.from_google_analytics(data, site.domain, dataset, timezone) end) end) |> Enum.map(&Task.await(&1, 120_000)) @@ -312,7 +294,7 @@ defmodule Plausible.Google.Api do defp fetch_analytic_reports(request_data, request) do reports = - Enum.map(request_data, fn {dimensions, metrics} -> + Enum.map(request_data, fn {_, dimensions, metrics} -> %{ viewId: request.profile, dateRanges: [ @@ -345,7 +327,12 @@ defmodule Plausible.Google.Api do if res.status_code == 200 do data = Jason.decode!(res.body)["reports"] - |> Enum.map(& &1["data"]["rows"]) + |> Enum.with_index() + |> Enum.map(fn {report, index} -> + {dataset, _, _} = Enum.at(request_data, index) + {dataset, report["data"]["rows"]} + end) + |> Map.new() {:ok, data} else From 11821fdd2e094ff85be54f18318bd1cbcc5554ce Mon Sep 17 00:00:00 2001 From: mcol Date: Wed, 5 Jan 2022 18:39:24 +0000 Subject: [PATCH 094/148] Fail sooner if GA timezone can't be identified --- lib/plausible/google/api.ex | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 9ffc38611577..cb57bfcd6a17 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -154,7 +154,8 @@ defmodule Plausible.Google.Api do def import_analytics(site, profile) do with {:ok, auth} <- refresh_if_needed(site.google_auth) do - do_import_analytics(site, auth, profile) + {:ok, timezone} = get_profile_timezone(auth, profile) + do_import_analytics(site, auth, profile, timezone) end end @@ -164,7 +165,7 @@ defmodule Plausible.Google.Api do Dimensions reference: https://ga-dev-tools.web.app/dimensions-metrics-explorer """ - def do_import_analytics(site, auth, profile) do + def do_import_analytics(site, auth, profile, timezone) do end_date = Plausible.Stats.Clickhouse.pageviews_begin(site) |> NaiveDateTime.to_date() @@ -261,8 +262,6 @@ defmodule Plausible.Google.Api do case Keyword.get(responses, :error) do nil -> - {:ok, timezone} = get_profile_timezone(auth, profile) - maybe_error = responses |> Enum.map(fn {:ok, resp} -> resp end) From ca8531b8b19626115bc5fe7efe849f4d7e1d294e Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 6 Jan 2022 15:33:13 +0000 Subject: [PATCH 095/148] Link imported tables to site by id --- lib/plausible/clickhouse_repo.ex | 6 +- lib/plausible/google/api.ex | 2 +- lib/plausible/imported/browsers.ex | 6 +- lib/plausible/imported/devices.ex | 6 +- lib/plausible/imported/entry_pages.ex | 6 +- lib/plausible/imported/exit_pages.ex | 6 +- lib/plausible/imported/locations.ex | 6 +- lib/plausible/imported/operation_systems.ex | 6 +- lib/plausible/imported/pages.ex | 6 +- lib/plausible/imported/site.ex | 58 +++++++++---------- lib/plausible/imported/sources.ex | 6 +- lib/plausible/imported/utm_campaigns.ex | 6 +- lib/plausible/imported/utm_content.ex | 6 +- lib/plausible/imported/utm_mediums.ex | 6 +- lib/plausible/imported/utm_sources.ex | 6 +- lib/plausible/imported/utm_terms.ex | 6 +- lib/plausible/imported/visitors.ex | 6 +- lib/plausible/stats/breakdown.ex | 2 +- lib/plausible/stats/imported.ex | 6 +- ...0211112130238_create_imported_visitors.exs | 2 +- ...20211118160420_create_imported_sources.exs | 2 +- ...1124135248_create_imported_utm_mediums.exs | 2 +- ...24135252_create_imported_utm_campaigns.exs | 2 +- .../20211129111618_create_imported_pages.exs | 2 +- ...1129111622_create_imported_entry_pages.exs | 2 +- ...11129111630_create_imported_exit_pages.exs | 2 +- ...211129111639_create_imported_locations.exs | 2 +- ...20211129111644_create_imported_devices.exs | 2 +- ...0211129111648_create_imported_browsers.exs | 2 +- ...1653_create_imported_operating_systems.exs | 2 +- ...1129164103_create_imported_utm_sources.exs | 2 +- ...211231120503_create_imported_utm_terms.exs | 2 +- ...1231120603_create_imported_utm_content.exs | 2 +- 33 files changed, 93 insertions(+), 93 deletions(-) diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex index 9a896ce87283..ca239c9431c3 100644 --- a/lib/plausible/clickhouse_repo.ex +++ b/lib/plausible/clickhouse_repo.ex @@ -18,7 +18,7 @@ defmodule Plausible.ClickhouseRepo do Ecto.Adapters.SQL.query!(__MODULE__, sessions_sql, [domain]) end - def clear_imported_stats_for(domain) do + def clear_imported_stats_for(site_id) do [ "imported_visitors", "imported_sources", @@ -36,8 +36,8 @@ defmodule Plausible.ClickhouseRepo do "imported_operating_systems" ] |> Enum.map(fn table -> - sql = "ALTER TABLE #{table} DELETE WHERE domain = ?" - Ecto.Adapters.SQL.query!(__MODULE__, sql, [domain]) + sql = "ALTER TABLE #{table} DELETE WHERE site_id = ?" + Ecto.Adapters.SQL.query!(__MODULE__, sql, [site_id]) end) end end diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index cb57bfcd6a17..c47f1af365cc 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -268,7 +268,7 @@ defmodule Plausible.Google.Api do |> Enum.concat() |> Enum.map(fn {dataset, data} -> Task.async(fn -> - Imported.from_google_analytics(data, site.domain, dataset, timezone) + Imported.from_google_analytics(data, site.id, dataset, timezone) end) end) |> Enum.map(&Task.await(&1, 120_000)) diff --git a/lib/plausible/imported/browsers.ex b/lib/plausible/imported/browsers.ex index fa8b52715d70..91964d01a755 100644 --- a/lib/plausible/imported/browsers.ex +++ b/lib/plausible/imported/browsers.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.Browsers do @primary_key false schema "imported_browsers" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :browser, :string field :visitors, :integer @@ -16,7 +16,7 @@ defmodule Plausible.Imported.Browsers do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :browser, :visitors @@ -24,7 +24,7 @@ defmodule Plausible.Imported.Browsers do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :visitors ]) diff --git a/lib/plausible/imported/devices.ex b/lib/plausible/imported/devices.ex index d3e971818d02..b83a86bddc41 100644 --- a/lib/plausible/imported/devices.ex +++ b/lib/plausible/imported/devices.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.Devices do @primary_key false schema "imported_devices" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :device, :string field :visitors, :integer @@ -16,7 +16,7 @@ defmodule Plausible.Imported.Devices do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :device, :visitors @@ -24,7 +24,7 @@ defmodule Plausible.Imported.Devices do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :visitors ]) diff --git a/lib/plausible/imported/entry_pages.ex b/lib/plausible/imported/entry_pages.ex index dae4ab02f321..0e5a6363b3c9 100644 --- a/lib/plausible/imported/entry_pages.ex +++ b/lib/plausible/imported/entry_pages.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.EntryPages do @primary_key false schema "imported_entry_pages" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :entry_page, :string field :visitors, :integer @@ -20,7 +20,7 @@ defmodule Plausible.Imported.EntryPages do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :entry_page, :visitors, @@ -31,7 +31,7 @@ defmodule Plausible.Imported.EntryPages do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :entry_page, :visitors, diff --git a/lib/plausible/imported/exit_pages.ex b/lib/plausible/imported/exit_pages.ex index 4b40d205789c..fbb81bc963d9 100644 --- a/lib/plausible/imported/exit_pages.ex +++ b/lib/plausible/imported/exit_pages.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.ExitPages do @primary_key false schema "imported_exit_pages" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :exit_page, :string field :visitors, :integer @@ -17,7 +17,7 @@ defmodule Plausible.Imported.ExitPages do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :exit_page, :visitors, @@ -26,7 +26,7 @@ defmodule Plausible.Imported.ExitPages do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :exit_page, :visitors, diff --git a/lib/plausible/imported/locations.ex b/lib/plausible/imported/locations.ex index 1493c32fb648..2fd32e2c7ad7 100644 --- a/lib/plausible/imported/locations.ex +++ b/lib/plausible/imported/locations.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.Locations do @primary_key false schema "imported_locations" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :country, :string, default: "" field :region, :string, default: "" @@ -18,7 +18,7 @@ defmodule Plausible.Imported.Locations do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :country, :region, @@ -28,7 +28,7 @@ defmodule Plausible.Imported.Locations do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :visitors ]) diff --git a/lib/plausible/imported/operation_systems.ex b/lib/plausible/imported/operation_systems.ex index af027ce597d4..b315458985ad 100644 --- a/lib/plausible/imported/operation_systems.ex +++ b/lib/plausible/imported/operation_systems.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.OperatingSystems do @primary_key false schema "imported_operating_systems" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :operating_system, :string field :visitors, :integer @@ -16,7 +16,7 @@ defmodule Plausible.Imported.OperatingSystems do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :operating_system, :visitors @@ -24,7 +24,7 @@ defmodule Plausible.Imported.OperatingSystems do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :visitors ]) diff --git a/lib/plausible/imported/pages.ex b/lib/plausible/imported/pages.ex index 3851940acf47..c73b66442435 100644 --- a/lib/plausible/imported/pages.ex +++ b/lib/plausible/imported/pages.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.Pages do @primary_key false schema "imported_pages" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :page, :string field :visitors, :integer @@ -18,7 +18,7 @@ defmodule Plausible.Imported.Pages do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :page, :visitors, @@ -28,7 +28,7 @@ defmodule Plausible.Imported.Pages do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :visitors ]) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 7e0eb7881530..dae8776dfa20 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -3,14 +3,14 @@ defmodule Plausible.Imported do use Timex def forget(site) do - Plausible.ClickhouseRepo.clear_imported_stats_for(site.domain) + Plausible.ClickhouseRepo.clear_imported_stats_for(site.site_id) end - def from_google_analytics(data, domain, metric, timezone) do + def from_google_analytics(data, site_id, metric, timezone) do maybe_error = data |> Enum.map(fn row -> - new_from_google_analytics(domain, timezone, metric, row) + new_from_google_analytics(site_id, timezone, metric, row) |> Plausible.ClickhouseRepo.insert(on_conflict: :replace_all) end) |> Keyword.get(:error) @@ -24,7 +24,7 @@ defmodule Plausible.Imported do end end - defp new_from_google_analytics(domain, timezone, "visitors", %{ + defp new_from_google_analytics(site_id, timezone, "visitors", %{ "dimensions" => [timestamp], "metrics" => [%{"values" => values}] }) do @@ -34,7 +34,7 @@ defmodule Plausible.Imported do |> Enum.map(&elem(&1, 0)) Imported.Visitors.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), visitors: visitors, pageviews: pageviews, @@ -44,7 +44,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, timezone, "sources", %{ + defp new_from_google_analytics(site_id, timezone, "sources", %{ "dimensions" => [timestamp, source], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -56,7 +56,7 @@ defmodule Plausible.Imported do source = if source == "(direct)", do: nil, else: source Imported.Sources.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), source: Imported.Sources.parse(source), visitors: visitors, @@ -68,7 +68,7 @@ defmodule Plausible.Imported do # TODO: utm_sources. Google reports sources and utm_sources unified. - defp new_from_google_analytics(domain, timezone, "utm_mediums", %{ + defp new_from_google_analytics(site_id, timezone, "utm_mediums", %{ "dimensions" => [timestamp, medium], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -80,7 +80,7 @@ defmodule Plausible.Imported do medium = if medium == "(none)", do: "", else: medium Imported.UtmMediums.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), medium: medium, visitors: visitors, @@ -90,7 +90,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, timezone, "utm_campaigns", %{ + defp new_from_google_analytics(site_id, timezone, "utm_campaigns", %{ "dimensions" => [timestamp, campaign], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -102,7 +102,7 @@ defmodule Plausible.Imported do campaign = if campaign == "(not set)", do: "", else: campaign Imported.UtmCampaigns.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), campaign: campaign, visitors: visitors, @@ -112,7 +112,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, timezone, "utm_terms", %{ + defp new_from_google_analytics(site_id, timezone, "utm_terms", %{ "dimensions" => [timestamp, term], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -124,7 +124,7 @@ defmodule Plausible.Imported do term = if term == "(not set)" or term == "(not provided)", do: "", else: term Imported.UtmTerms.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), utm_term: term, visitors: visitors, @@ -134,7 +134,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, timezone, "utm_content", %{ + defp new_from_google_analytics(site_id, timezone, "utm_content", %{ "dimensions" => [timestamp, content], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -146,7 +146,7 @@ defmodule Plausible.Imported do content = if content == "(not set)", do: "", else: content Imported.UtmContent.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), utm_content: content, visitors: visitors, @@ -156,7 +156,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, timezone, "pages", %{ + defp new_from_google_analytics(site_id, timezone, "pages", %{ "dimensions" => [timestamp, page], "metrics" => [%{"values" => [value, pageviews, time_on_page]}] }) do @@ -165,7 +165,7 @@ defmodule Plausible.Imported do {time_on_page, _} = Integer.parse(time_on_page) Imported.Pages.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), page: page, visitors: visitors, @@ -174,7 +174,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, timezone, "entry_pages", %{ + defp new_from_google_analytics(site_id, timezone, "entry_pages", %{ "dimensions" => [timestamp, entry_page], "metrics" => [%{"values" => [visitors, entrances, visit_duration, bounces]}] }) do @@ -185,7 +185,7 @@ defmodule Plausible.Imported do {visit_duration, _} = Integer.parse(visit_duration) Imported.EntryPages.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), entry_page: entry_page, visitors: visitors, @@ -195,7 +195,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, timezone, "exit_pages", %{ + defp new_from_google_analytics(site_id, timezone, "exit_pages", %{ "dimensions" => [timestamp, exit_page], "metrics" => [%{"values" => [value, exits]}] }) do @@ -203,7 +203,7 @@ defmodule Plausible.Imported do {exits, ""} = Integer.parse(exits) Imported.ExitPages.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), exit_page: exit_page, visitors: visitors, @@ -211,7 +211,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, timezone, "locations", %{ + defp new_from_google_analytics(site_id, timezone, "locations", %{ "dimensions" => [timestamp, country, region], "metrics" => [%{"values" => [value]}] }) do @@ -220,7 +220,7 @@ defmodule Plausible.Imported do region = if region == "(not set)", do: "", else: region Imported.Locations.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), country: country, region: region, @@ -229,14 +229,14 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(domain, timezone, "devices", %{ + defp new_from_google_analytics(site_id, timezone, "devices", %{ "dimensions" => [timestamp, device], "metrics" => [%{"values" => [value]}] }) do {visitors, ""} = Integer.parse(value) Imported.Devices.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), device: String.capitalize(device), visitors: visitors @@ -253,14 +253,14 @@ defmodule Plausible.Imported do "(not set)" => "" } - defp new_from_google_analytics(domain, timezone, "browsers", %{ + defp new_from_google_analytics(site_id, timezone, "browsers", %{ "dimensions" => [timestamp, browser], "metrics" => [%{"values" => [value]}] }) do {visitors, ""} = Integer.parse(value) Imported.Browsers.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), browser: Map.get(@browser_google_to_plausible, browser, browser), visitors: visitors @@ -273,14 +273,14 @@ defmodule Plausible.Imported do "(not set)" => "" } - defp new_from_google_analytics(domain, timezone, "operating_systems", %{ + defp new_from_google_analytics(site_id, timezone, "operating_systems", %{ "dimensions" => [timestamp, operating_system], "metrics" => [%{"values" => [value]}] }) do {visitors, ""} = Integer.parse(value) Imported.OperatingSystems.new(%{ - domain: domain, + site_id: site_id, timestamp: format_timestamp(timestamp, timezone), operating_system: Map.get(@os_google_to_plausible, operating_system, operating_system), visitors: visitors diff --git a/lib/plausible/imported/sources.ex b/lib/plausible/imported/sources.ex index dbf0103b4de2..25e3d2aa9b2f 100644 --- a/lib/plausible/imported/sources.ex +++ b/lib/plausible/imported/sources.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.Sources do @primary_key false schema "imported_sources" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :source, :string, default: "" field :visitors, :integer @@ -20,7 +20,7 @@ defmodule Plausible.Imported.Sources do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :source, :visitors, @@ -31,7 +31,7 @@ defmodule Plausible.Imported.Sources do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :visitors, :visits, diff --git a/lib/plausible/imported/utm_campaigns.ex b/lib/plausible/imported/utm_campaigns.ex index b158e575c17b..7a534a0cabb0 100644 --- a/lib/plausible/imported/utm_campaigns.ex +++ b/lib/plausible/imported/utm_campaigns.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.UtmCampaigns do @primary_key false schema "imported_utm_campaigns" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :utm_campaign, :string, default: "" field :visitors, :integer @@ -20,7 +20,7 @@ defmodule Plausible.Imported.UtmCampaigns do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :utm_campaign, :visitors, @@ -31,7 +31,7 @@ defmodule Plausible.Imported.UtmCampaigns do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :visitors, :visits, diff --git a/lib/plausible/imported/utm_content.ex b/lib/plausible/imported/utm_content.ex index f7e14a69161f..7fbb3d42ada4 100644 --- a/lib/plausible/imported/utm_content.ex +++ b/lib/plausible/imported/utm_content.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.UtmContent do @primary_key false schema "imported_utm_content" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :utm_content, :string, default: "" field :visitors, :integer @@ -20,7 +20,7 @@ defmodule Plausible.Imported.UtmContent do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :utm_content, :visitors, @@ -31,7 +31,7 @@ defmodule Plausible.Imported.UtmContent do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :visitors, :visits, diff --git a/lib/plausible/imported/utm_mediums.ex b/lib/plausible/imported/utm_mediums.ex index efd119fb17ec..96e30b82fc6f 100644 --- a/lib/plausible/imported/utm_mediums.ex +++ b/lib/plausible/imported/utm_mediums.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.UtmMediums do @primary_key false schema "imported_utm_mediums" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :utm_medium, :string, default: "" field :visitors, :integer @@ -20,7 +20,7 @@ defmodule Plausible.Imported.UtmMediums do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :utm_medium, :visitors, @@ -31,7 +31,7 @@ defmodule Plausible.Imported.UtmMediums do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :visitors, :visits, diff --git a/lib/plausible/imported/utm_sources.ex b/lib/plausible/imported/utm_sources.ex index 4c634944f2b1..038953a4fe81 100644 --- a/lib/plausible/imported/utm_sources.ex +++ b/lib/plausible/imported/utm_sources.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.UtmSources do @primary_key false schema "imported_utm_sources" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :utm_source, :string, default: "" field :visitors, :integer @@ -20,7 +20,7 @@ defmodule Plausible.Imported.UtmSources do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :utm_source, :visitors, @@ -31,7 +31,7 @@ defmodule Plausible.Imported.UtmSources do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :visitors, :visits, diff --git a/lib/plausible/imported/utm_terms.ex b/lib/plausible/imported/utm_terms.ex index 911b2d85fda9..f28db02e34fb 100644 --- a/lib/plausible/imported/utm_terms.ex +++ b/lib/plausible/imported/utm_terms.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.UtmTerms do @primary_key false schema "imported_utm_terms" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :utm_term, :string, default: "" field :visitors, :integer @@ -20,7 +20,7 @@ defmodule Plausible.Imported.UtmTerms do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :utm_term, :visitors, @@ -31,7 +31,7 @@ defmodule Plausible.Imported.UtmTerms do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :visitors, :visits, diff --git a/lib/plausible/imported/visitors.ex b/lib/plausible/imported/visitors.ex index ca087b27510f..b639c0fd1541 100644 --- a/lib/plausible/imported/visitors.ex +++ b/lib/plausible/imported/visitors.ex @@ -5,7 +5,7 @@ defmodule Plausible.Imported.Visitors do @primary_key false schema "imported_visitors" do - field :domain, :string + field :site_id, :integer field :timestamp, :naive_datetime field :visitors, :integer field :pageviews, :integer @@ -20,7 +20,7 @@ defmodule Plausible.Imported.Visitors do |> cast( attrs, [ - :domain, + :site_id, :timestamp, :visitors, :pageviews, @@ -31,7 +31,7 @@ defmodule Plausible.Imported.Visitors do empty_values: [nil, ""] ) |> validate_required([ - :domain, + :site_id, :timestamp, :visitors, :pageviews, diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 6cdb416ab7ee..7c8870e41d39 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -289,7 +289,7 @@ defmodule Plausible.Stats.Breakdown do from( i in "imported_pages", group_by: i.page, - where: i.domain == ^site.domain, + where: i.site_id == ^site.id, where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, where: i.page in ^pages, select: %{ diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index dc31f3c802bd..b9c73eb9308e 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -12,7 +12,7 @@ defmodule Plausible.Stats.Imported do result = from(v in "imported_visitors", group_by: fragment("date"), - where: v.domain == ^site.domain, + where: v.site_id == ^site.id, where: v.timestamp >= ^first_datetime and v.timestamp < ^last_datetime, select: %{"visitors" => sum(v.visitors)} ) @@ -78,7 +78,7 @@ defmodule Plausible.Stats.Imported do from( i in table, group_by: field(i, ^dim), - where: i.domain == ^site.domain, + where: i.site_id == ^site.id, where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, select: %{} ) @@ -291,7 +291,7 @@ defmodule Plausible.Stats.Imported do imported_q = from( i in "imported_visitors", - where: i.domain == ^site.domain, + where: i.site_id == ^site.id, where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, select: %{} ) diff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs index 78a3a0c05969..e0cec007815d 100644 --- a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs +++ b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do def change do create_if_not_exists table(:imported_visitors, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :visitors, :UInt64 add :pageviews, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs b/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs index bbb752ff6116..18dab5895fc4 100644 --- a/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs +++ b/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedSources do def change do create_if_not_exists table(:imported_sources, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :source, :string add :visitors, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs b/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs index 3589a5a2ba58..ac110e113619 100644 --- a/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs +++ b/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmMediums do def change do create_if_not_exists table(:imported_utm_mediums, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :utm_medium, :string add :visitors, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs b/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs index 10c9122ee68c..c9fcfc666b94 100644 --- a/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs +++ b/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmCampaigns do def change do create_if_not_exists table(:imported_utm_campaigns, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :utm_campaign, :string add :visitors, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs b/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs index 6aaaad392d10..1a93f3284693 100644 --- a/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedPages do def change do create_if_not_exists table(:imported_pages, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :page, :string add :visitors, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs index 0643b2f11c8c..291e04aeeadf 100644 --- a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedEntryPages do def change do create_if_not_exists table(:imported_entry_pages, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :entry_page, :string add :visitors, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs b/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs index b21117ad119c..12f73b73bb63 100644 --- a/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedExitPages do def change do create_if_not_exists table(:imported_exit_pages, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :exit_page, :string add :visitors, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs index cf520c6d0e6b..a5fa64c3738d 100644 --- a/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs +++ b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedLocations do def change do create_if_not_exists table(:imported_locations, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :country, :string add :region, :string diff --git a/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs b/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs index 55457f396534..d3f700271126 100644 --- a/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs +++ b/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedDevices do def change do create_if_not_exists table(:imported_devices, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :device, :string add :visitors, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs index b58f337f5fa0..b97be8b6308e 100644 --- a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs +++ b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedBrowsers do def change do create_if_not_exists table(:imported_browsers, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :browser, :string add :visitors, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs index 2e8171bb7644..0d01b30cd8f9 100644 --- a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs +++ b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedOperatingSystems do def change do create_if_not_exists table(:imported_operating_systems, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :operating_system, :string add :visitors, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs b/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs index e05bac6ebda4..7055e276b859 100644 --- a/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs +++ b/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmSources do def change do create_if_not_exists table(:imported_utm_sources, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :utm_source, :string add :visitors, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs b/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs index 65283e0811da..795137a2e410 100644 --- a/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs +++ b/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmTerms do def change do create_if_not_exists table(:imported_utm_terms, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :utm_term, :string add :visitors, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_content.exs b/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_content.exs index 293647717e13..e626ae5b3b2b 100644 --- a/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_content.exs +++ b/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_content.exs @@ -3,7 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmContent do def change do create_if_not_exists table(:imported_utm_content, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do - add :domain, :string + add :site_id, :UInt64 add :timestamp, :naive_datetime add :utm_content, :string add :visitors, :UInt64 From b534f6246f170bac34e9d2c84da86fa7b75f36e3 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 7 Jan 2022 14:33:16 +0000 Subject: [PATCH 096/148] imported_utm_content -> imported_utm_contents --- lib/plausible/clickhouse_repo.ex | 2 +- lib/plausible/google/api.ex | 2 +- lib/plausible/imported/site.ex | 4 ++-- lib/plausible/imported/{utm_content.ex => utm_contents.ex} | 4 ++-- lib/plausible/stats/imported.ex | 3 --- ...nt.exs => 20211231120603_create_imported_utm_contents.exs} | 4 ++-- 6 files changed, 8 insertions(+), 11 deletions(-) rename lib/plausible/imported/{utm_content.ex => utm_contents.ex} (90%) rename priv/clickhouse_repo/migrations/{20211231120603_create_imported_utm_content.exs => 20211231120603_create_imported_utm_contents.exs} (70%) diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex index ca239c9431c3..a1f57fba13fa 100644 --- a/lib/plausible/clickhouse_repo.ex +++ b/lib/plausible/clickhouse_repo.ex @@ -25,7 +25,7 @@ defmodule Plausible.ClickhouseRepo do "imported_utm_mediums", "imported_utm_sources", "imported_utm_campaigns", - "imported_utm_content", + "imported_utm_contents", "imported_utm_terms", "imported_pages", "imported_entry_pages", diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index c47f1af365cc..8256981d6549 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -213,7 +213,7 @@ defmodule Plausible.Google.Api do ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { - "utm_content", + "utm_contents", ["ga:dateHour", "ga:adContent"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index dae8776dfa20..6e0077ecb34d 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -134,7 +134,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(site_id, timezone, "utm_content", %{ + defp new_from_google_analytics(site_id, timezone, "utm_contents", %{ "dimensions" => [timestamp, content], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -145,7 +145,7 @@ defmodule Plausible.Imported do content = if content == "(not set)", do: "", else: content - Imported.UtmContent.new(%{ + Imported.UtmContents.new(%{ site_id: site_id, timestamp: format_timestamp(timestamp, timezone), utm_content: content, diff --git a/lib/plausible/imported/utm_content.ex b/lib/plausible/imported/utm_contents.ex similarity index 90% rename from lib/plausible/imported/utm_content.ex rename to lib/plausible/imported/utm_contents.ex index 7fbb3d42ada4..20ad64d270a1 100644 --- a/lib/plausible/imported/utm_content.ex +++ b/lib/plausible/imported/utm_contents.ex @@ -1,10 +1,10 @@ -defmodule Plausible.Imported.UtmContent do +defmodule Plausible.Imported.UtmContents do use Ecto.Schema use Plausible.ClickhouseRepo import Ecto.Changeset @primary_key false - schema "imported_utm_content" do + schema "imported_utm_contents" do field :site_id, :integer field :timestamp, :naive_datetime field :utm_content, :string, default: "" diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index b9c73eb9308e..f621f56629bb 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -51,9 +51,6 @@ defmodule Plausible.Stats.Imported do {table, dim} = case property do - "visit:utm_content" -> - {"imported_utm_content", :utm_content} - "visit:country" -> {"imported_locations", :country} diff --git a/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_content.exs b/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs similarity index 70% rename from priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_content.exs rename to priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs index e626ae5b3b2b..42c5ee98b1a1 100644 --- a/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_content.exs +++ b/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs @@ -1,8 +1,8 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmContent do +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmContents do use Ecto.Migration def change do - create_if_not_exists table(:imported_utm_content, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_utm_contents, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :naive_datetime add :utm_content, :string From a0e21671e920f09d97f50b2b7f1a8960c2adb658 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 7 Jan 2022 14:40:53 +0000 Subject: [PATCH 097/148] Imported GA from all of time --- lib/plausible/google/api.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 8256981d6549..fe9139803173 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -170,12 +170,9 @@ defmodule Plausible.Google.Api do Plausible.Stats.Clickhouse.pageviews_begin(site) |> NaiveDateTime.to_date() - start_date = Date.add(end_date, -365) - request = %{ auth: auth, profile: profile, - start_date: Date.to_iso8601(start_date), end_date: Date.to_iso8601(end_date) } @@ -298,7 +295,7 @@ defmodule Plausible.Google.Api do viewId: request.profile, dateRanges: [ %{ - startDate: request.start_date, + startDate: "2005-01-01", # The earliest valid date endDate: request.end_date } ], From 1505e7bf70b9c4b1caa97671e41aef68d8d81b6f Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 7 Jan 2022 16:31:09 +0000 Subject: [PATCH 098/148] Reorganise GA data fetch logic - Fetch data from the start of time (2005) - Check whether no data was fetched, and if so, inform user and don't consider data to be imported. --- lib/plausible/google/api.ex | 40 +++++++++++-------- lib/plausible/imported/site.ex | 6 ++- .../controllers/site_controller.ex | 7 +--- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index fe9139803173..790fbcce29e1 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -259,27 +259,34 @@ defmodule Plausible.Google.Api do case Keyword.get(responses, :error) do nil -> - maybe_error = + results = responses |> Enum.map(fn {:ok, resp} -> resp end) |> Enum.concat() - |> Enum.map(fn {dataset, data} -> - Task.async(fn -> - Imported.from_google_analytics(data, site.id, dataset, timezone) + + if Enum.any?(results, fn {_, val} -> val end) do + maybe_error = + results + |> Enum.map(fn {dataset, data} -> + Task.async(fn -> + Imported.from_google_analytics(data, site.id, dataset, timezone) + end) end) - end) - |> Enum.map(&Task.await(&1, 120_000)) - |> Keyword.get(:error) + |> Enum.map(&Task.await(&1, 120_000)) + |> Keyword.get(:error) - case maybe_error do - nil -> - {:ok, nil} + case maybe_error do + nil -> + {:ok, nil} - {:error, error} -> - Plausible.ClickhouseRepo.clear_imported_stats_for(site.domain) + {:error, error} -> + Plausible.ClickhouseRepo.clear_imported_stats_for(site.domain) - Sentry.capture_message("Error saving Google analytics data", extra: error) - {:error, error} + Sentry.capture_message("Error saving Google analytics data", extra: error) + {:error, error["error"]["message"]} + end + else + {:error, "No Google Analytics data found."} end error -> @@ -295,7 +302,8 @@ defmodule Plausible.Google.Api do viewId: request.profile, dateRanges: [ %{ - startDate: "2005-01-01", # The earliest valid date + # The earliest valid date + startDate: "2005-01-01", endDate: request.end_date } ], @@ -332,7 +340,7 @@ defmodule Plausible.Google.Api do {:ok, data} else - {:error, Jason.decode!(res.body)} + {:error, Jason.decode!(res.body)["error"]["message"]} end end diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 6e0077ecb34d..0626310985c0 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -3,9 +3,11 @@ defmodule Plausible.Imported do use Timex def forget(site) do - Plausible.ClickhouseRepo.clear_imported_stats_for(site.site_id) + Plausible.ClickhouseRepo.clear_imported_stats_for(site.id) end + def from_google_analytics(nil, _site_id, _metric, _timezone), do: {:ok, nil} + def from_google_analytics(data, site_id, metric, timezone) do maybe_error = data @@ -20,7 +22,7 @@ defmodule Plausible.Imported do {:ok, nil} error -> - {:error, error} + {:error, error.errors} end end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 489bb54d36a3..90c7765d661b 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -665,13 +665,8 @@ defmodule PlausibleWeb.SiteController do |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) {:error, error} -> - message = - error - |> Map.get("error") - |> Map.get("message") - conn - |> put_flash(:error, "Error while fetching: #{message}") + |> put_flash(:error, "Error while fetching: #{error}") |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) end end From de3220d793c4f38d16da7e96d0122c6db8444c69 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 7 Jan 2022 16:32:32 +0000 Subject: [PATCH 099/148] Clarify removal of "visits" data when it isn't in metrics --- lib/plausible/stats/breakdown.ex | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 7c8870e41d39..c201fc7eb7fb 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -147,16 +147,15 @@ defmodule Plausible.Stats.Breakdown do ] do query = Query.treat_page_filter_as_entry_page(query) - results = breakdown_sessions(site, query, property, metrics, pagination) - - prop_result = - property - |> String.split(":") - |> Enum.at(1) - |> String.to_existing_atom() - - metrics = metrics ++ [prop_result] - Enum.map(results, &Map.take(&1, metrics)) + # "visits" is fetched when querying bounce rate and visit duration, as it + # is needed to calculate these from imported data. Let's remove it from the + # result if it wasn't requested. + if (:bounce_rate in metrics or :visit_duration in metrics) and :visits not in metrics do + breakdown_sessions(site, query, property, metrics, pagination) + |> Enum.map(&Map.delete(&1, :visits)) + else + breakdown_sessions(site, query, property, metrics, pagination) + end end def breakdown(site, query, property, metrics, pagination) do From 5b571432737fb121e1b2a6c9e977500cb70f8880 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 7 Jan 2022 17:36:59 +0000 Subject: [PATCH 100/148] Apply location filters from API This makes it consistent with the sources etc which filter out 'Direct / None' on the API side. These filters are used by both the native and imported data handling code, which would otherwise both duplicate the filters in their `where` clauses. --- lib/plausible/stats/breakdown.ex | 3 --- lib/plausible/stats/imported.ex | 8 ++----- .../controllers/api/stats_controller.ex | 21 ++++++++++++++++--- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index c201fc7eb7fb..ab6cac7abfe9 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -393,7 +393,6 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.country_code, - where: s.country_code != "\0\0", select_merge: %{country: s.country_code} ) end @@ -402,7 +401,6 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.subdivision1_code, - where: s.subdivision1_code != "", select_merge: %{region: s.subdivision1_code} ) end @@ -411,7 +409,6 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.city_geoname_id, - where: s.city_geoname_id != 0, select_merge: %{city: s.city_geoname_id} ) end diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index f621f56629bb..ba1b68d72cf4 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -154,14 +154,10 @@ defmodule Plausible.Stats.Imported do imported_q |> select_merge([i], %{country: i.country}) :region -> - imported_q - |> select_merge([i], %{region: i.region}) - |> where([i], i.region != "") + imported_q |> select_merge([i], %{region: i.region}) :city -> - imported_q - |> select_merge([i], %{city: i.city}) - |> where([i], i.city != 0) + imported_q |> select_merge([i], %{city: i.city}) :device -> imported_q |> select_merge([i], %{device: i.device}) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 5fc85b5b29b1..802a09d789c8 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -564,7 +564,12 @@ defmodule PlausibleWeb.Api.StatsController do def countries(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + + query = + Query.from(site.timezone, params) + |> Filters.add_prefix() + |> Query.put_filter("visit:country", {:is_not, "\0\0"}) + pagination = parse_pagination(params) countries = @@ -616,7 +621,12 @@ defmodule PlausibleWeb.Api.StatsController do def regions(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + + query = + Query.from(site.timezone, params) + |> Filters.add_prefix() + |> Query.put_filter("visit:region", {:is_not, ""}) + pagination = parse_pagination(params) regions = @@ -649,7 +659,12 @@ defmodule PlausibleWeb.Api.StatsController do def cities(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + + query = + Query.from(site.timezone, params) + |> Filters.add_prefix() + |> Query.put_filter("visit:city", {:is_not, 0}) + pagination = parse_pagination(params) cities = From 734a5203b14114d9367822b5d832d13593588bd8 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 7 Jan 2022 17:50:40 +0000 Subject: [PATCH 101/148] Do not use changeset for setting site.imported_source --- lib/plausible/site/schema.ex | 6 +++++- lib/plausible_web/controllers/site_controller.ex | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index 47ef81999891..49bf3b71bb75 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -27,7 +27,7 @@ defmodule Plausible.Site do def changeset(site, attrs \\ %{}) do site - |> cast(attrs, [:domain, :timezone, :imported_source]) + |> cast(attrs, [:domain, :timezone]) |> validate_required([:domain, :timezone]) |> validate_format(:domain, ~r/^[a-zA-Z0-9\-\.\/\:]*$/, message: "only letters, numbers, slashes and period allowed" @@ -48,6 +48,10 @@ defmodule Plausible.Site do change(site, has_stats: has_stats_val) end + def set_imported_source(site, imported_source) do + change(site, imported_source: imported_source) + end + defp clean_domain(changeset) do clean_domain = (get_field(changeset, :domain) || "") diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 90c7765d661b..9d2b6b98099b 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -657,7 +657,7 @@ defmodule PlausibleWeb.SiteController do case Plausible.Google.Api.import_analytics(site, profile) do {:ok, _} -> site - |> Plausible.Site.changeset(%{imported_source: "Google Analytics"}) + |> Plausible.Site.set_imported_source("Google Analytics") |> Repo.update!() conn @@ -680,7 +680,7 @@ defmodule PlausibleWeb.SiteController do Plausible.Imported.forget(site) site - |> Plausible.Site.changeset(%{imported_source: nil}) + |> Plausible.Site.set_imported_source(nil) |> Repo.update!() conn From 1a6ab5b39a96da4517de1bae56e2130d60968765 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 15 Jan 2022 18:56:55 +0000 Subject: [PATCH 102/148] Add all metrics to all dimensions --- lib/plausible/google/api.ex | 8 +-- lib/plausible/imported/browsers.ex | 14 +++++- lib/plausible/imported/devices.ex | 14 +++++- lib/plausible/imported/locations.ex | 14 +++++- lib/plausible/imported/operation_systems.ex | 14 +++++- lib/plausible/imported/site.ex | 49 ++++++++++++++----- ...211129111639_create_imported_locations.exs | 3 ++ ...20211129111644_create_imported_devices.exs | 3 ++ ...0211129111648_create_imported_browsers.exs | 3 ++ ...1653_create_imported_operating_systems.exs | 3 ++ 10 files changed, 100 insertions(+), 25 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 790fbcce29e1..e6768665759e 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -232,22 +232,22 @@ defmodule Plausible.Google.Api do { "locations", ["ga:dateHour", "ga:countryIsoCode", "ga:regionIsoCode"], - ["ga:users"] + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { "devices", ["ga:dateHour", "ga:deviceCategory"], - ["ga:users"] + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { "browsers", ["ga:dateHour", "ga:browser"], - ["ga:users"] + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { "operating_systems", ["ga:dateHour", "ga:operatingSystem"], - ["ga:users"] + ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] } ] diff --git a/lib/plausible/imported/browsers.ex b/lib/plausible/imported/browsers.ex index 91964d01a755..a9cd8d9fdd76 100644 --- a/lib/plausible/imported/browsers.ex +++ b/lib/plausible/imported/browsers.ex @@ -9,6 +9,10 @@ defmodule Plausible.Imported.Browsers do field :timestamp, :naive_datetime field :browser, :string field :visitors, :integer + field :visits, :integer + field :bounces, :integer + # Sum total + field :visit_duration, :integer end def new(attrs) do @@ -19,14 +23,20 @@ defmodule Plausible.Imported.Browsers do :site_id, :timestamp, :browser, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ], empty_values: [nil, ""] ) |> validate_required([ :site_id, :timestamp, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ]) end end diff --git a/lib/plausible/imported/devices.ex b/lib/plausible/imported/devices.ex index b83a86bddc41..576ed79961a3 100644 --- a/lib/plausible/imported/devices.ex +++ b/lib/plausible/imported/devices.ex @@ -9,6 +9,10 @@ defmodule Plausible.Imported.Devices do field :timestamp, :naive_datetime field :device, :string field :visitors, :integer + field :visits, :integer + field :bounces, :integer + # Sum total + field :visit_duration, :integer end def new(attrs) do @@ -19,14 +23,20 @@ defmodule Plausible.Imported.Devices do :site_id, :timestamp, :device, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ], empty_values: [nil, ""] ) |> validate_required([ :site_id, :timestamp, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ]) end end diff --git a/lib/plausible/imported/locations.ex b/lib/plausible/imported/locations.ex index 2fd32e2c7ad7..bbc8f71c4946 100644 --- a/lib/plausible/imported/locations.ex +++ b/lib/plausible/imported/locations.ex @@ -11,6 +11,10 @@ defmodule Plausible.Imported.Locations do field :region, :string, default: "" field :city, :integer, default: 0 field :visitors, :integer + field :visits, :integer + field :bounces, :integer + # Sum total + field :visit_duration, :integer end def new(attrs) do @@ -23,14 +27,20 @@ defmodule Plausible.Imported.Locations do :country, :region, :city, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ], empty_values: [nil, ""] ) |> validate_required([ :site_id, :timestamp, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ]) end end diff --git a/lib/plausible/imported/operation_systems.ex b/lib/plausible/imported/operation_systems.ex index b315458985ad..e8a0c9c332f4 100644 --- a/lib/plausible/imported/operation_systems.ex +++ b/lib/plausible/imported/operation_systems.ex @@ -9,6 +9,10 @@ defmodule Plausible.Imported.OperatingSystems do field :timestamp, :naive_datetime field :operating_system, :string field :visitors, :integer + field :visits, :integer + field :bounces, :integer + # Sum total + field :visit_duration, :integer end def new(attrs) do @@ -19,14 +23,20 @@ defmodule Plausible.Imported.OperatingSystems do :site_id, :timestamp, :operating_system, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ], empty_values: [nil, ""] ) |> validate_required([ :site_id, :timestamp, - :visitors + :visitors, + :visits, + :bounces, + :visit_duration ]) end end diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 0626310985c0..a001ebcd8aa6 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -183,7 +183,6 @@ defmodule Plausible.Imported do {visitors, ""} = Integer.parse(visitors) {entrances, ""} = Integer.parse(entrances) {bounces, ""} = Integer.parse(bounces) - {visit_duration, _} = Integer.parse(visit_duration) Imported.EntryPages.new(%{ @@ -215,11 +214,14 @@ defmodule Plausible.Imported do defp new_from_google_analytics(site_id, timezone, "locations", %{ "dimensions" => [timestamp, country, region], - "metrics" => [%{"values" => [value]}] + "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do - {visitors, ""} = Integer.parse(value) country = if country == "(not set)", do: "", else: country region = if region == "(not set)", do: "", else: region + {visitors, ""} = Integer.parse(visitors) + {visits, ""} = Integer.parse(visits) + {bounces, ""} = Integer.parse(bounces) + {visit_duration, _} = Integer.parse(visit_duration) Imported.Locations.new(%{ site_id: site_id, @@ -227,21 +229,30 @@ defmodule Plausible.Imported do country: country, region: region, city: 0, - visitors: visitors + visitors: visitors, + visits: visits, + bounces: bounces, + visit_duration: visit_duration }) end defp new_from_google_analytics(site_id, timezone, "devices", %{ "dimensions" => [timestamp, device], - "metrics" => [%{"values" => [value]}] + "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do - {visitors, ""} = Integer.parse(value) + {visitors, ""} = Integer.parse(visitors) + {visits, ""} = Integer.parse(visits) + {bounces, ""} = Integer.parse(bounces) + {visit_duration, _} = Integer.parse(visit_duration) Imported.Devices.new(%{ site_id: site_id, timestamp: format_timestamp(timestamp, timezone), device: String.capitalize(device), - visitors: visitors + visitors: visitors, + visits: visits, + bounces: bounces, + visit_duration: visit_duration }) end @@ -257,15 +268,21 @@ defmodule Plausible.Imported do defp new_from_google_analytics(site_id, timezone, "browsers", %{ "dimensions" => [timestamp, browser], - "metrics" => [%{"values" => [value]}] + "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do - {visitors, ""} = Integer.parse(value) + {visitors, ""} = Integer.parse(visitors) + {visits, ""} = Integer.parse(visits) + {bounces, ""} = Integer.parse(bounces) + {visit_duration, _} = Integer.parse(visit_duration) Imported.Browsers.new(%{ site_id: site_id, timestamp: format_timestamp(timestamp, timezone), browser: Map.get(@browser_google_to_plausible, browser, browser), - visitors: visitors + visitors: visitors, + visits: visits, + bounces: bounces, + visit_duration: visit_duration }) end @@ -277,15 +294,21 @@ defmodule Plausible.Imported do defp new_from_google_analytics(site_id, timezone, "operating_systems", %{ "dimensions" => [timestamp, operating_system], - "metrics" => [%{"values" => [value]}] + "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do - {visitors, ""} = Integer.parse(value) + {visitors, ""} = Integer.parse(visitors) + {visits, ""} = Integer.parse(visits) + {bounces, ""} = Integer.parse(bounces) + {visit_duration, _} = Integer.parse(visit_duration) Imported.OperatingSystems.new(%{ site_id: site_id, timestamp: format_timestamp(timestamp, timezone), operating_system: Map.get(@os_google_to_plausible, operating_system, operating_system), - visitors: visitors + visitors: visitors, + visits: visits, + bounces: bounces, + visit_duration: visit_duration }) end diff --git a/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs index a5fa64c3738d..9a6dfeb1d363 100644 --- a/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs +++ b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs @@ -9,6 +9,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedLocations do add :region, :string add :city, :UInt64 add :visitors, :UInt64 + add :visits, :UInt64 + add :visit_duration, :UInt64 + add :bounces, :UInt32 end end end diff --git a/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs b/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs index d3f700271126..112de1a3123d 100644 --- a/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs +++ b/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs @@ -7,6 +7,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedDevices do add :timestamp, :naive_datetime add :device, :string add :visitors, :UInt64 + add :visits, :UInt64 + add :visit_duration, :UInt64 + add :bounces, :UInt32 end end end diff --git a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs index b97be8b6308e..897d21a4c6db 100644 --- a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs +++ b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs @@ -7,6 +7,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedBrowsers do add :timestamp, :naive_datetime add :browser, :string add :visitors, :UInt64 + add :visits, :UInt64 + add :visit_duration, :UInt64 + add :bounces, :UInt32 end end end diff --git a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs index 0d01b30cd8f9..b18d9db0332d 100644 --- a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs +++ b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs @@ -7,6 +7,9 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedOperatingSystems do add :timestamp, :naive_datetime add :operating_system, :string add :visitors, :UInt64 + add :visits, :UInt64 + add :visit_duration, :UInt64 + add :bounces, :UInt32 end end end From bbbc625fc57556531d0779cdfcfdc5e3673c6e5a Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 21 Jan 2022 19:00:06 +0000 Subject: [PATCH 103/148] Run GA import in the background --- config/runtime.exs | 5 ++-- .../controllers/site_controller.ex | 22 +++++--------- lib/workers/import_google_analytics.ex | 29 +++++++++++++++++++ 3 files changed, 39 insertions(+), 17 deletions(-) create mode 100644 lib/workers/import_google_analytics.ex diff --git a/config/runtime.exs b/config/runtime.exs index 91a80a502568..bf9ce9084e8c 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -320,7 +320,8 @@ if config_env() == :prod && !disable_cron do check_stats_emails: 1, site_setup_emails: 1, clean_email_verification_codes: 1, - clean_invitations: 1 + clean_invitations: 1, + google_analytics_imports: 1 ] extra_queues = [ @@ -340,7 +341,7 @@ if config_env() == :prod && !disable_cron do else config :plausible, Oban, repo: Plausible.Repo, - queues: false, + queues: [google_analytics_imports: 1], plugins: false end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 9d2b6b98099b..06b2bb5468fb 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -654,21 +654,13 @@ defmodule PlausibleWeb.SiteController do |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) true -> - case Plausible.Google.Api.import_analytics(site, profile) do - {:ok, _} -> - site - |> Plausible.Site.set_imported_source("Google Analytics") - |> Repo.update!() - - conn - |> put_flash(:success, "Google Analytics data successfully imported") - |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) - - {:error, error} -> - conn - |> put_flash(:error, "Error while fetching: #{error}") - |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) - end + %{"site_id" => site.id, "profile" => profile} + |> Plausible.Workers.ImportGoogleAnalytics.new() + |> Oban.insert() + + conn + |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") + |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) end end diff --git a/lib/workers/import_google_analytics.ex b/lib/workers/import_google_analytics.ex new file mode 100644 index 000000000000..62d9bfa03013 --- /dev/null +++ b/lib/workers/import_google_analytics.ex @@ -0,0 +1,29 @@ +defmodule Plausible.Workers.ImportGoogleAnalytics do + use Plausible.Repo + + use Oban.Worker, + queue: :google_analytics_imports, + max_attempts: 1, + unique: [fields: [:args], period: 60] + + @impl Oban.Worker + def perform(%Oban.Job{args: %{"site_id" => site_id, "profile" => profile}}) do + site = + Repo.get(Plausible.Site, site_id) + |> Repo.preload(:google_auth) + + case Plausible.Google.Api.import_analytics(site, profile) do + {:ok, _} -> + site + |> Plausible.Site.set_imported_source("Google Analytics") + |> Repo.update!() + + # TODO: Send email to update user + :ok + + {:error, error} -> + # TODO: Send failure email + {:error, error} + end + end +end From 2ba78d3fd14c7fa5fa3026bf4de997d36544b29c Mon Sep 17 00:00:00 2001 From: mcol Date: Sun, 23 Jan 2022 17:24:23 +0000 Subject: [PATCH 104/148] Send email when GA import completes --- lib/plausible_web/email.ex | 23 +++++++++++++++++++ .../email/google_analytics_import.html.eex | 13 +++++++++++ lib/workers/import_google_analytics.ex | 18 ++++++++++++--- 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 lib/plausible_web/templates/email/google_analytics_import.html.eex diff --git a/lib/plausible_web/email.ex b/lib/plausible_web/email.ex index ab1c5870ad8b..345dd3ca1b5f 100644 --- a/lib/plausible_web/email.ex +++ b/lib/plausible_web/email.ex @@ -280,6 +280,29 @@ defmodule PlausibleWeb.Email do ) end + def import_success(email, site) do + base_email() + |> to(email) + |> tag("import-success-email") + |> subject("Google Analytics data imported for #{site.domain}") + |> render("google_analytics_import.html", %{ + site: site, + link: PlausibleWeb.Endpoint.url() <> "/" <> URI.encode_www_form(site.domain), + success: true + }) + end + + def import_failure(email, site) do + base_email() + |> to(email) + |> tag("import-failure-email") + |> subject("Google Analytics import failed for #{site.domain}") + |> render("google_analytics_import.html", %{ + site: site, + success: false + }) + end + defp base_email() do mailer_from = Application.get_env(:plausible, :mailer_email) diff --git a/lib/plausible_web/templates/email/google_analytics_import.html.eex b/lib/plausible_web/templates/email/google_analytics_import.html.eex new file mode 100644 index 000000000000..65f4e0949c4d --- /dev/null +++ b/lib/plausible_web/templates/email/google_analytics_import.html.eex @@ -0,0 +1,13 @@ +Hey <%= user_salutation(@user) %>, +

    +<%= if @success do %> + Your Google Analytics import has complete. +

    + View dashboard: @link +<% else %> + Unfortunately, your Google Analytics import failed. +<% end %> +

    +-- +

    +<%= plausible_url() %>
    diff --git a/lib/workers/import_google_analytics.ex b/lib/workers/import_google_analytics.ex index 62d9bfa03013..6900d87b588d 100644 --- a/lib/workers/import_google_analytics.ex +++ b/lib/workers/import_google_analytics.ex @@ -10,7 +10,7 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do def perform(%Oban.Job{args: %{"site_id" => site_id, "profile" => profile}}) do site = Repo.get(Plausible.Site, site_id) - |> Repo.preload(:google_auth) + |> Repo.preload([:google_auth, :members]) case Plausible.Google.Api.import_analytics(site, profile) do {:ok, _} -> @@ -18,11 +18,23 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do |> Plausible.Site.set_imported_source("Google Analytics") |> Repo.update!() - # TODO: Send email to update user + Enum.each(site.members, fn member -> + if Enum.member?(member.role, [:owner, :admin]) do + PlausibleWeb.Email.import_success(member.user.email, site) + |> Plausible.Mailer.send_email_safe() + end + end) + :ok {:error, error} -> - # TODO: Send failure email + Enum.each(site.members, fn member -> + if Enum.member?(member.role, [:owner, :admin]) do + PlausibleWeb.Email.import_failure(member.user.email, site) + |> Plausible.Mailer.send_email_safe() + end + end) + {:error, error} end end From 8eba6262499eefeee5151b95bb7515abbfce0ce0 Mon Sep 17 00:00:00 2001 From: mcol Date: Sun, 23 Jan 2022 22:28:58 +0000 Subject: [PATCH 105/148] Add handler to insert imported data into tests and imported_browsers_factory --- lib/plausible/stats/query.ex | 2 +- .../api/stats_controller/browsers_test.exs | 21 +++++++++++++++ test/support/factory.ex | 16 +++++++++++- test/support/test_utils.ex | 26 ++++++++++++++++++- 4 files changed, 62 insertions(+), 3 deletions(-) diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 1d994049c537..f26dc688d149 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -258,6 +258,6 @@ defmodule Plausible.Stats.Query do defp parse_goal_filter(event), do: {:is, :event, event} defp include_imported(params) do - params["filters"] == "{}" && params["with_imported"] == "true" + params["filters"] in [nil, "{}"] && params["with_imported"] == "true" end end diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs index 5d362277a90d..6c9d1af3612a 100644 --- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs @@ -40,6 +40,27 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do } ] end + + test "returns top browsers including imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, browser: "Chrome"), + build(:imported_browsers, browser: "Chrome"), + build(:imported_browsers, browser: "Firefox") + ]) + + conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day") + + assert json_response(conn, 200) == [ + %{"name" => "Chrome", "visitors" => 1, "percentage" => 100} + ] + + conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&with_imported=true") + + assert json_response(conn, 200) == [ + %{"name" => "Chrome", "visitors" => 2, "percentage" => 67}, + %{"name" => "Firefox", "visitors" => 1, "percentage" => 33} + ] + end end describe "GET /api/stats/:domain/browser-versions" do diff --git a/test/support/factory.ex b/test/support/factory.ex index 754668f83325..a997f4d8766d 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -26,7 +26,8 @@ defmodule Plausible.Factory do %Plausible.Site{ domain: domain, - timezone: "UTC" + timezone: "UTC", + imported_source: "Google Analytics" } end @@ -180,6 +181,19 @@ defmodule Plausible.Factory do } end + @today Timex.today() |> Timex.to_naive_datetime() + + def imported_browsers_factory do + %Plausible.Imported.Browsers{ + timestamp: @today, + browser: "", + visitors: 1, + visits: 1, + bounces: 0, + visit_duration: 10 + } + end + defp hash_key() do Keyword.fetch!( Application.get_env(:plausible, PlausibleWeb.Endpoint), diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index 616846e7dd0e..e5fc6f400793 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -81,12 +81,32 @@ defmodule Plausible.TestUtils do def populate_stats(site, events) do Enum.map(events, fn event -> - Map.put(event, :domain, site.domain) + case event do + %Plausible.ClickhouseEvent{} -> + Map.put(event, :domain, site.domain) + _ -> + Map.put(event, :site_id, site.id) + end end) |> populate_stats end def populate_stats(events) do + {native, imported} = + Enum.split_with(events, fn event -> + case event do + %Plausible.ClickhouseEvent{} -> + true + _ -> + false + end + end) + + if native, do: populate_native_stats(native) + if imported, do: populate_imported_stats(imported) + end + + defp populate_native_stats(events) do sessions = Enum.reduce(events, %{}, fn event, sessions -> Plausible.Session.Store.reconcile_event(sessions, event) @@ -108,6 +128,10 @@ defmodule Plausible.TestUtils do ) end + defp populate_imported_stats(events) do + Enum.map(events, &Plausible.ClickhouseRepo.insert!/1) + end + def relative_time(shifts) do NaiveDateTime.utc_now() |> Timex.shift(shifts) From 680708fe9e103363858ce93bc9bea060264967a3 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 27 Jan 2022 21:32:08 +0000 Subject: [PATCH 106/148] Add remaining import data test factories --- test/support/factory.ex | 142 +++++++++++++++++++++++++++++++++++++ test/support/test_utils.ex | 2 + 2 files changed, 144 insertions(+) diff --git a/test/support/factory.ex b/test/support/factory.ex index a997f4d8766d..9bc7c3e3c09c 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -183,6 +183,137 @@ defmodule Plausible.Factory do @today Timex.today() |> Timex.to_naive_datetime() + def imported_visitors_factory do + %Plausible.Imported.Visitors{ + timestamp: @today, + visitors: 1, + pageviews: 1, + bounces: 0, + visits: 1, + visit_duration: 10 + } + end + + def imported_sources_factory do + %Plausible.Imported.Sources{ + timestamp: @today, + source: "", + visitors: 1, + visits: 1, + bounces: 0, + visit_duration: 10 + } + end + + def imported_utm_mediums_factory do + %Plausible.Imported.UtmMediums{ + timestamp: @today, + utm_medium: "", + visitors: 1, + visits: 1, + bounces: 0, + visit_duration: 10 + } + end + + def imported_utm_sources_factory do + %Plausible.Imported.UtmSources{ + timestamp: @today, + utm_source: "", + visitors: 1, + visits: 1, + bounces: 0, + visit_duration: 10 + } + end + + def imported_utm_campaigns_factory do + %Plausible.Imported.UtmCampaigns{ + timestamp: @today, + utm_campaign: "", + visitors: 1, + visits: 1, + bounces: 0, + visit_duration: 10 + } + end + + def imported_utm_terms_factory do + %Plausible.Imported.UtmTerms{ + timestamp: @today, + utm_term: "", + visitors: 1, + visits: 1, + bounces: 0, + visit_duration: 10 + } + end + + def imported_utm_contents_factory do + %Plausible.Imported.UtmContents{ + timestamp: @today, + utm_content: "", + visitors: 1, + visits: 1, + bounces: 0, + visit_duration: 10 + } + end + + def imported_pages_factory do + %Plausible.Imported.Pages{ + timestamp: @today, + page: "", + visitors: 1, + pageviews: 1, + time_on_page: 10 + } + end + + def imported_entry_pages_factory do + %Plausible.Imported.EntryPages{ + timestamp: @today, + entry_page: "", + visitors: 1, + entrances: 1, + bounces: 0, + visit_duration: 10 + } + end + + def imported_exit_pages_factory do + %Plausible.Imported.ExitPages{ + timestamp: @today, + exit_page: "", + visitors: 1, + exits: 1 + } + end + + def imported_locations_factory do + %Plausible.Imported.Locations{ + timestamp: @today, + country: "", + region: "", + city: 0, + visitors: 1, + visits: 1, + bounces: 0, + visit_duration: 10 + } + end + + def imported_devices_factory do + %Plausible.Imported.Devices{ + timestamp: @today, + device: "", + visitors: 1, + visits: 1, + bounces: 0, + visit_duration: 10 + } + end + def imported_browsers_factory do %Plausible.Imported.Browsers{ timestamp: @today, @@ -194,6 +325,17 @@ defmodule Plausible.Factory do } end + def imported_operating_systems_factory do + %Plausible.Imported.OperatingSystems{ + timestamp: @today, + operating_system: "", + visitors: 1, + visits: 1, + bounces: 0, + visit_duration: 10 + } + end + defp hash_key() do Keyword.fetch!( Application.get_env(:plausible, PlausibleWeb.Endpoint), diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index e5fc6f400793..e4a7f64228aa 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -84,6 +84,7 @@ defmodule Plausible.TestUtils do case event do %Plausible.ClickhouseEvent{} -> Map.put(event, :domain, site.domain) + _ -> Map.put(event, :site_id, site.id) end @@ -97,6 +98,7 @@ defmodule Plausible.TestUtils do case event do %Plausible.ClickhouseEvent{} -> true + _ -> false end From 4cb02e56d086c6692db2c501360efecfb31cd26b Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 27 Jan 2022 21:36:43 +0000 Subject: [PATCH 107/148] Add imported location data to test --- .../api/stats_controller/countries_test.exs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/plausible_web/controllers/api/stats_controller/countries_test.exs b/test/plausible_web/controllers/api/stats_controller/countries_test.exs index ecfa4f4a8739..4c4255bb7ebc 100644 --- a/test/plausible_web/controllers/api/stats_controller/countries_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/countries_test.exs @@ -15,6 +15,12 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do ), build(:pageview, country_code: "GB" + ), + build(:imported_locations, + country: "EE" + ), + build(:imported_locations, + country: "GB" ) ]) @@ -38,6 +44,27 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do "percentage" => 33 } ] + + conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&with_imported=true") + + assert json_response(conn, 200) == [ + %{ + "code" => "EE", + "alpha_3" => "EST", + "name" => "Estonia", + "flag" => "🇪🇪", + "visitors" => 3, + "percentage" => 60 + }, + %{ + "code" => "GB", + "alpha_3" => "GBR", + "name" => "United Kingdom", + "flag" => "🇬🇧", + "visitors" => 2, + "percentage" => 40 + } + ] end test "calculates conversion_rate when filtering for goal", %{conn: conn, site: site} do From 63095626ab144037846e8b43b8b3a17a95e93e89 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 27 Jan 2022 22:24:16 +0000 Subject: [PATCH 108/148] Test main graph with imported data --- .../api/stats_controller/main_graph_test.exs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index 0707ea921519..70d3feb5ba77 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -66,6 +66,53 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert Enum.count(plot) == 31 assert List.first(plot) == 1 assert List.last(plot) == 1 + assert Enum.sum(plot) == 2 + end + + test "displays visitors for a month with imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-31 00:00:00]), + build(:imported_visitors, timestamp: ~N[2021-01-01 00:00:00]), + build(:imported_visitors, timestamp: ~N[2021-01-31 00:00:00]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true" + ) + + assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200) + + assert Enum.count(plot) == 31 + assert List.first(plot) == 2 + assert List.last(plot) == 2 + assert Enum.sum(plot) == 4 + end + + test "displays visitors for a month with imported data and filter", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00], pathname: "/pageA"), + build(:pageview, timestamp: ~N[2021-01-31 00:00:00], pathname: "/pageA"), + build(:imported_visitors, timestamp: ~N[2021-01-01 00:00:00]), + build(:imported_visitors, timestamp: ~N[2021-01-31 00:00:00]) + ]) + + filters = Jason.encode!(%{page: "/pageA"}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&filters=#{filters}" + ) + + assert %{"plot" => plot, "imported_source" => ""} = json_response(conn, 200) + + assert Enum.count(plot) == 31 + assert List.first(plot) == 1 + assert List.last(plot) == 1 + assert Enum.sum(plot) == 2 end # TODO: missing 6, 12 months, 30 days From 516c84de375c209cc1a7c21f6d6c50cded68b765 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 27 Jan 2022 22:29:04 +0000 Subject: [PATCH 109/148] Add imported data to operating systems tests --- .../operating_systems_test.exs | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs index f1f700e55ddf..59d8182d6a59 100644 --- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs @@ -41,6 +41,57 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do } ] end + + test "returns operating systems by unique visitors with imported data", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, operating_system: "Mac"), + build(:pageview, operating_system: "Mac"), + build(:pageview, operating_system: "Android"), + build(:imported_operating_systems, operating_system: "Mac"), + build(:imported_operating_systems, operating_system: "Android") + ]) + + conn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day") + + assert json_response(conn, 200) == [ + %{"name" => "Mac", "visitors" => 2, "percentage" => 67}, + %{"name" => "Android", "visitors" => 1, "percentage" => 33} + ] + + conn = + get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&with_imported=true") + + assert json_response(conn, 200) == [ + %{"name" => "Mac", "visitors" => 3, "percentage" => 60}, + %{"name" => "Android", "visitors" => 2, "percentage" => 40} + ] + end + + test "imported data is ignored when filtering for goal", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, user_id: 1, operating_system: "Mac"), + build(:pageview, user_id: 2, operating_system: "Mac"), + build(:imported_operating_systems, operating_system: "Mac"), + build(:event, user_id: 1, name: "Signup") + ]) + + filters = Jason.encode!(%{"goal" => "Signup"}) + + conn = + get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&filters=#{filters}") + + assert json_response(conn, 200) == [ + %{ + "name" => "Mac", + "total_visitors" => 2, + "visitors" => 1, + "conversion_rate" => 50.0 + } + ] + end end describe "GET /api/stats/:domain/operating-system-versions" do From 4132f6460b147f178091a1aad47ccb105cd77ed2 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 27 Jan 2022 23:03:03 +0000 Subject: [PATCH 110/148] Add imported data to pages tests --- .../api/stats_controller/pages_test.exs | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index e9db51ec9556..e7227ce4f4bc 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -25,6 +25,35 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ] end + test "returns top pages by visitors with imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/"), + build(:pageview, pathname: "/"), + build(:pageview, pathname: "/"), + build(:imported_pages, page: "/"), + build(:pageview, pathname: "/register"), + build(:pageview, pathname: "/register"), + build(:imported_pages, page: "/register"), + build(:pageview, pathname: "/contact") + ]) + + conn = get(conn, "/api/stats/#{site.domain}/pages?period=day") + + assert json_response(conn, 200) == [ + %{"visitors" => 3, "name" => "/"}, + %{"visitors" => 2, "name" => "/register"}, + %{"visitors" => 1, "name" => "/contact"} + ] + + conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&with_imported=true") + + assert json_response(conn, 200) == [ + %{"visitors" => 4, "name" => "/"}, + %{"visitors" => 3, "name" => "/register"}, + %{"visitors" => 1, "name" => "/contact"} + ] + end + test "calculates bounce rate and time on page for pages", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, @@ -67,6 +96,67 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ] end + test "calculates bounce rate and time on page for pages with imported data", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + pathname: "/", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/some-other-page", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:15:00] + ), + build(:pageview, + pathname: "/", + timestamp: ~N[2021-01-01 00:15:00] + ), + build(:imported_pages, + page: "/", + timestamp: ~N[2021-01-01 00:15:00], + time_on_page: 700 + ), + build(:imported_entry_pages, + entry_page: "/", + timestamp: ~N[2021-01-01 00:15:00], + entrances: 3, + bounces: 1 + ), + build(:imported_pages, + page: "/some-other-page", + timestamp: ~N[2021-01-01 00:15:00], + time_on_page: 60 + ) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "bounce_rate" => 40.0, + "time_on_page" => 800.0, + "visitors" => 3, + "pageviews" => 3, + "name" => "/" + }, + %{ + "bounce_rate" => nil, + "time_on_page" => 60, + "visitors" => 2, + "pageviews" => 2, + "name" => "/some-other-page" + } + ] + end + test "returns top pages in realtime report", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, pathname: "/page1"), From 51ca030f6f7e6806042b71250348862add1f1563 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 27 Jan 2022 23:10:18 +0000 Subject: [PATCH 111/148] Add imported data to entry pages tests --- .../api/stats_controller/pages_test.exs | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index e7227ce4f4bc..90af78c817e0 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -241,6 +241,85 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ] end + test "returns top entry pages by visitors with imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/page1", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/page1", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/page2", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/page2", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:15:00] + ) + ]) + + populate_stats(site, [ + build(:pageview, + pathname: "/page2", + user_id: @user_id, + timestamp: ~N[2021-01-01 23:15:00] + ) + ]) + + populate_stats(site, [ + build(:imported_entry_pages, + entry_page: "/page2", + timestamp: ~N[2021-01-01 00:00:00], + entrances: 3, + visitors: 2, + visit_duration: 300 + ) + ]) + + conn = get(conn, "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01") + + assert json_response(conn, 200) == [ + %{ + "unique_entrances" => 2, + "total_entrances" => 2, + "name" => "/page1", + "visit_duration" => 0 + }, + %{ + "unique_entrances" => 1, + "total_entrances" => 2, + "name" => "/page2", + "visit_duration" => 450 + } + ] + + conn = + get( + conn, + "/api/stats/#{site.domain}/entry-pages?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "unique_entrances" => 3, + "total_entrances" => 5, + "name" => "/page2", + "visit_duration" => 240.0 + }, + %{ + "unique_entrances" => 2, + "total_entrances" => 2, + "name" => "/page1", + "visit_duration" => 0 + } + ] + end + test "calculates conversion_rate when filtering for goal", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, From 2e13a60dd674906082c2165b8b9a7df9248ffd0c Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 27 Jan 2022 23:22:08 +0000 Subject: [PATCH 112/148] Add imported data to exit pages tests --- .../api/stats_controller/pages_test.exs | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index 90af78c817e0..4ad9438432c5 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -416,6 +416,67 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ] end + test "returns top exit pages by visitors with imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/page1", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/page1", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/page1", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/page2", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:15:00] + ) + ]) + + populate_stats(site, [ + build(:imported_pages, + page: "/page2", + timestamp: ~N[2021-01-01 00:00:00], + pageviews: 4, + visitors: 2 + ), + build(:imported_exit_pages, + exit_page: "/page2", + timestamp: ~N[2021-01-01 00:00:00], + exits: 3, + visitors: 2 + ) + ]) + + conn = get(conn, "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01") + + assert json_response(conn, 200) == [ + %{"name" => "/page1", "unique_exits" => 2, "total_exits" => 2, "exit_rate" => 66}, + %{"name" => "/page2", "unique_exits" => 1, "total_exits" => 1, "exit_rate" => 100} + ] + + conn = + get( + conn, + "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "/page2", + "unique_exits" => 3, + "total_exits" => 4, + "exit_rate" => 80.0 + }, + %{"name" => "/page1", "unique_exits" => 2, "total_exits" => 2, "exit_rate" => 66} + ] + end + test "calculates correct exit rate and conversion_rate when filtering for goal", %{ conn: conn, site: site From d369ba31bc7ab531115a66bc874bcda3475886ac Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 27 Jan 2022 23:24:28 +0000 Subject: [PATCH 113/148] Add imported data to devices tests --- .../stats_controller/screen_sizes_test.exs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs index 56751b96e163..97b564b49292 100644 --- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs @@ -20,6 +20,34 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do ] end + test "returns screen sizes by new visitors with imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, screen_size: "Desktop"), + build(:pageview, screen_size: "Desktop"), + build(:pageview, screen_size: "Laptop") + ]) + + populate_stats(site, [ + build(:imported_devices, device: "Mobile"), + build(:imported_devices, device: "Laptop") + ]) + + conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day") + + assert json_response(conn, 200) == [ + %{"name" => "Desktop", "visitors" => 2, "percentage" => 67}, + %{"name" => "Laptop", "visitors" => 1, "percentage" => 33} + ] + + conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&with_imported=true") + + assert json_response(conn, 200) == [ + %{"name" => "Desktop", "visitors" => 2, "percentage" => 40}, + %{"name" => "Laptop", "visitors" => 2, "percentage" => 40}, + %{"name" => "Mobile", "visitors" => 1, "percentage" => 20} + ] + end + test "calculates conversion_rate when filtering for goal", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, user_id: 1, screen_size: "Desktop"), From 16a9d0145bb99fe287c9948d4d7d42dbac3d1cce Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 27 Jan 2022 23:45:08 +0000 Subject: [PATCH 114/148] Add imported data to sources tests --- .../api/stats_controller/sources_test.exs | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index d16636730751..b361cac22af4 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -30,6 +30,39 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ] end + test "returns top sources with imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, referrer_source: "Google", referrer: "google.com"), + build(:pageview, referrer_source: "Google", referrer: "google.com"), + build(:pageview, referrer_source: "DuckDuckGo", referrer: "duckduckgo.com") + ]) + + populate_stats(site, [ + build(:imported_sources, + source: "Google", + visitors: 2 + ), + build(:imported_sources, + source: "DuckDuckGo", + visitors: 1 + ) + ]) + + conn = get(conn, "/api/stats/#{site.domain}/sources") + + assert json_response(conn, 200) == [ + %{"name" => "Google", "visitors" => 2}, + %{"name" => "DuckDuckGo", "visitors" => 1} + ] + + conn = get(conn, "/api/stats/#{site.domain}/sources?with_imported=true") + + assert json_response(conn, 200) == [ + %{"name" => "Google", "visitors" => 4}, + %{"name" => "DuckDuckGo", "visitors" => 2} + ] + end + test "calculates bounce rate and visit duration for sources", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, @@ -73,6 +106,92 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ] end + test "calculates bounce rate and visit duration for sources with imported data", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + referrer_source: "Google", + referrer: "google.com", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Google", + referrer: "google.com", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:15:00] + ), + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + populate_stats(site, [ + build(:imported_sources, + source: "Google", + timestamp: ~N[2021-01-01 00:00:00], + visitors: 2, + visits: 3, + bounces: 1, + visit_duration: 900 + ), + build(:imported_sources, + source: "DuckDuckGo", + timestamp: ~N[2021-01-01 00:00:00], + visitors: 1, + visits: 1, + visit_duration: 100, + bounces: 0 + ) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&detailed=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "Google", + "visitors" => 1, + "bounce_rate" => 0, + "visit_duration" => 900 + }, + %{ + "name" => "DuckDuckGo", + "visitors" => 1, + "bounce_rate" => 100, + "visit_duration" => 0 + } + ] + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&detailed=true&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "Google", + "visitors" => 3, + "bounce_rate" => 25, + "visit_duration" => 450.0 + }, + %{ + "name" => "DuckDuckGo", + "visitors" => 2, + "bounce_rate" => 50, + "visit_duration" => 50 + } + ] + end + test "returns top sources in realtime report", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, @@ -113,6 +232,9 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do build(:pageview, referrer_source: "DuckDuckGo", referrer: "duckduckgo.com" + ), + build(:imported_sources, + source: "DuckDuckGo" ) ]) @@ -121,6 +243,12 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do assert json_response(conn, 200) == [ %{"name" => "DuckDuckGo", "visitors" => 1} ] + + conn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2&with_imported=true") + + assert json_response(conn, 200) == [ + %{"name" => "DuckDuckGo", "visitors" => 2} + ] end test "shows sources for a page", %{conn: conn, site: site} do From 67fba86655ce2bb868521771c2b2bbdcef4fabaa Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 28 Jan 2022 00:01:43 +0000 Subject: [PATCH 115/148] Add imported data to UTM tests --- .../api/stats_controller/sources_test.exs | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index b361cac22af4..801d338930eb 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -289,6 +289,25 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ) ]) + populate_stats(site, [ + build(:imported_utm_mediums, + utm_medium: "social", + timestamp: ~N[2021-01-01 00:00:00], + visit_duration: 700, + bounces: 1, + visits: 1, + visitors: 1 + ), + build(:imported_utm_mediums, + utm_medium: "email", + timestamp: ~N[2021-01-01 00:00:00], + bounces: 0, + visits: 1, + visitors: 1, + visit_duration: 100 + ) + ]) + conn = get( conn, @@ -309,6 +328,27 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do "visit_duration" => 0 } ] + + conn = + get( + conn, + "/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "social", + "visitors" => 2, + "bounce_rate" => 50, + "visit_duration" => 800.0 + }, + %{ + "name" => "email", + "visitors" => 2, + "bounce_rate" => 50, + "visit_duration" => 50 + } + ] end end @@ -337,6 +377,25 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ) ]) + populate_stats(site, [ + build(:imported_utm_campaigns, + utm_campaign: "profile", + timestamp: ~N[2021-01-01 00:00:00], + visit_duration: 700, + bounces: 1, + visits: 1, + visitors: 1 + ), + build(:imported_utm_campaigns, + utm_campaign: "august", + timestamp: ~N[2021-01-01 00:00:00], + bounces: 0, + visits: 1, + visitors: 1, + visit_duration: 900 + ) + ]) + conn = get( conn, @@ -357,6 +416,27 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do "visit_duration" => 900 } ] + + conn = + get( + conn, + "/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "august", + "visitors" => 3, + "bounce_rate" => 67, + "visit_duration" => 300 + }, + %{ + "name" => "profile", + "visitors" => 2, + "bounce_rate" => 50, + "visit_duration" => 800.0 + } + ] end end @@ -385,6 +465,25 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ) ]) + populate_stats(site, [ + build(:imported_utm_sources, + utm_source: "Twitter", + timestamp: ~N[2021-01-01 00:00:00], + visit_duration: 700, + bounces: 1, + visits: 1, + visitors: 1 + ), + build(:imported_utm_sources, + utm_source: "newsletter", + timestamp: ~N[2021-01-01 00:00:00], + bounces: 0, + visits: 1, + visitors: 1, + visit_duration: 900 + ) + ]) + conn = get( conn, @@ -405,6 +504,27 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do "visit_duration" => 900 } ] + + conn = + get( + conn, + "/api/stats/#{site.domain}/utm_sources?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "newsletter", + "visitors" => 3, + "bounce_rate" => 67, + "visit_duration" => 300 + }, + %{ + "name" => "Twitter", + "visitors" => 2, + "bounce_rate" => 50, + "visit_duration" => 800.0 + } + ] end end @@ -429,6 +549,11 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ) ]) + # Imported data is ignored when filtering + populate_stats(site, [ + build(:imported_sources, source: "Twitter") + ]) + filters = Jason.encode!(%{goal: "Signup"}) conn = From 5ac5ba65637037874ad0278898fe258de7b104d5 Mon Sep 17 00:00:00 2001 From: mcol Date: Fri, 28 Jan 2022 01:05:30 +0000 Subject: [PATCH 116/148] Add new test module for the data import step --- test/plausible/imported/imported_test.exs | 48 +++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 test/plausible/imported/imported_test.exs diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs new file mode 100644 index 000000000000..3b8f0e4853cf --- /dev/null +++ b/test/plausible/imported/imported_test.exs @@ -0,0 +1,48 @@ +defmodule Plausible.ImportedTest do + use PlausibleWeb.ConnCase + use Timex + import Plausible.TestUtils + + @utc Timezone.get("UTC") + + describe "Parse and import third party data fetched from Google Analytics" do + setup [:create_user, :log_in, :create_new_site] + + test "Visitors data imported from Google Analytics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-01-31 00:00:00]) + ]) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100"], + "metrics" => [%{"values" => ["1", "1", "0", "1", "60"]}] + }, + %{ + "dimensions" => ["2021013100"], + "metrics" => [%{"values" => ["1", "1", "1", "1", "60"]}] + } + ], + site.id, + "visitors", + @utc + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true" + ) + + assert %{"plot" => plot, "imported_source" => "Google Analytics"} = json_response(conn, 200) + + assert Enum.count(plot) == 31 + assert List.first(plot) == 2 + assert List.last(plot) == 2 + assert Enum.sum(plot) == 4 + end + end +end From cb9ce03ecf2dfec148963e5b3de00a11fea9acd3 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 17:10:03 +0000 Subject: [PATCH 117/148] Test import of sources GA data --- test/plausible/imported/imported_test.exs | 48 +++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 3b8f0e4853cf..2b6efd3a9ed4 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -44,5 +44,53 @@ defmodule Plausible.ImportedTest do assert List.last(plot) == 2 assert Enum.sum(plot) == 4 end + + test "Sources data imported from Google Analytics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer_source: "Google", + referrer: "google.com", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Google", + referrer: "google.com", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "duckduckgo.com"], + "metrics" => [%{"values" => ["1", "1", "0", "60"]}] + }, + %{ + "dimensions" => ["2021013100", "google.com"], + "metrics" => [%{"values" => ["1", "1", "1", "60"]}] + } + ], + site.id, + "sources", + @utc + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=month&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{"name" => "Google", "visitors" => 3}, + %{"name" => "DuckDuckGo", "visitors" => 2} + ] + end end end From de821d62e43a9e35f268492ffae776ffc0b28d14 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 17:41:03 +0000 Subject: [PATCH 118/148] Test import of utm_mediums GA data --- lib/plausible/imported/site.ex | 4 +- test/plausible/imported/imported_test.exs | 55 +++++++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index a001ebcd8aa6..200a27a8c3cb 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -84,7 +84,7 @@ defmodule Plausible.Imported do Imported.UtmMediums.new(%{ site_id: site_id, timestamp: format_timestamp(timestamp, timezone), - medium: medium, + utm_medium: medium, visitors: visitors, visits: visits, bounces: bounces, @@ -106,7 +106,7 @@ defmodule Plausible.Imported do Imported.UtmCampaigns.new(%{ site_id: site_id, timestamp: format_timestamp(timestamp, timezone), - campaign: campaign, + utm_campaign: campaign, visitors: visitors, visits: visits, bounces: bounces, diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 2b6efd3a9ed4..58630e375620 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -92,5 +92,60 @@ defmodule Plausible.ImportedTest do %{"name" => "DuckDuckGo", "visitors" => 2} ] end + + test "UTM mediums data imported from Google Analytics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + utm_medium: "social", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + utm_medium: "social", + timestamp: ~N[2021-01-01 12:00:00] + ), + build(:pageview, + utm_medium: "email", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "social"], + "metrics" => [%{"values" => ["1", "1", "1", "60"]}] + }, + %{ + "dimensions" => ["2021010100", "email"], + "metrics" => [%{"values" => ["1", "1", "0", "100"]}] + } + ], + site.id, + "utm_mediums", + @utc + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/utm_mediums?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "bounce_rate" => 100.0, + "name" => "social", + "visit_duration" => 20, + "visitors" => 3 + }, + %{ + "bounce_rate" => 50.0, + "name" => "email", + "visit_duration" => 50.0, + "visitors" => 2 + } + ] + end end end From d6467573046f1271e35f6cac4a20cd99187106d7 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 17:57:53 +0000 Subject: [PATCH 119/148] Test import of utm_campaigns GA data --- test/plausible/imported/imported_test.exs | 45 +++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 58630e375620..429da63fb37a 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -147,5 +147,50 @@ defmodule Plausible.ImportedTest do } ] end + + test "UTM campaigns data imported from Google Analytics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, utm_campaign: "profile", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, utm_campaign: "august", timestamp: ~N[2021-01-01 00:00:00]) + ]) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "profile"], + "metrics" => [%{"values" => ["1", "1", "1", "100"]}] + }, + %{ + "dimensions" => ["2021010100", "august"], + "metrics" => [%{"values" => ["1", "1", "0", "100"]}] + } + ], + site.id, + "utm_campaigns", + @utc + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/utm_campaigns?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "august", + "visitors" => 2, + "bounce_rate" => 50.0, + "visit_duration" => 50.0 + }, + %{ + "name" => "profile", + "visitors" => 2, + "bounce_rate" => 100.0, + "visit_duration" => 50.0 + } + ] + end end end From cd17fe566f33c7bc37bc4aa3d809ba6b71134805 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 18:14:39 +0000 Subject: [PATCH 120/148] Add tests for UTM terms --- test/plausible/imported/imported_test.exs | 45 ++++++++++ .../api/stats_controller/sources_test.exs | 88 +++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 429da63fb37a..b0ecff0b1436 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -192,5 +192,50 @@ defmodule Plausible.ImportedTest do } ] end + + test "UTM terms data imported from Google Analytics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, utm_term: "oat milk", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00]) + ]) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "oat milk"], + "metrics" => [%{"values" => ["1", "1", "1", "100"]}] + }, + %{ + "dimensions" => ["2021010100", "Sweden"], + "metrics" => [%{"values" => ["1", "1", "0", "100"]}] + } + ], + site.id, + "utm_terms", + @utc + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "Sweden", + "visitors" => 2, + "bounce_rate" => 50.0, + "visit_duration" => 50.0 + }, + %{ + "name" => "oat milk", + "visitors" => 2, + "bounce_rate" => 100.0, + "visit_duration" => 50.0 + } + ] + end end end diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index 801d338930eb..e9985cffe24f 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -801,4 +801,92 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do } end end + + describe "GET /api/stats/:domain/utm_terms" do + setup [:create_user, :log_in, :create_new_site] + + test "returns top utm_terms by unique user ids", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + utm_term: "oat milk", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + utm_term: "oat milk", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:15:00] + ), + build(:pageview, + utm_term: "Sweden", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + utm_term: "Sweden", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + populate_stats(site, [ + build(:imported_utm_terms, + utm_term: "oat milk", + timestamp: ~N[2021-01-01 00:00:00], + visit_duration: 700, + bounces: 1, + visits: 1, + visitors: 1 + ), + build(:imported_utm_terms, + utm_term: "Sweden", + timestamp: ~N[2021-01-01 00:00:00], + bounces: 0, + visits: 1, + visitors: 1, + visit_duration: 900 + ) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "Sweden", + "visitors" => 2, + "bounce_rate" => 100, + "visit_duration" => 0 + }, + %{ + "name" => "oat milk", + "visitors" => 1, + "bounce_rate" => 0, + "visit_duration" => 900 + } + ] + + conn = + get( + conn, + "/api/stats/#{site.domain}/utm_terms?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "Sweden", + "visitors" => 3, + "bounce_rate" => 67, + "visit_duration" => 300 + }, + %{ + "name" => "oat milk", + "visitors" => 2, + "bounce_rate" => 50, + "visit_duration" => 800.0 + } + ] + end + end end From 72303d8465d0e97a55a101696ac29cfc993caf03 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 18:20:40 +0000 Subject: [PATCH 121/148] Add tests for UTM contents --- test/plausible/imported/imported_test.exs | 45 ++++++++++ .../api/stats_controller/sources_test.exs | 88 +++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index b0ecff0b1436..6938311bc9b2 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -237,5 +237,50 @@ defmodule Plausible.ImportedTest do } ] end + + test "UTM contents data imported from Google Analytics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, utm_content: "ad", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, utm_content: "blog", timestamp: ~N[2021-01-01 00:00:00]) + ]) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "ad"], + "metrics" => [%{"values" => ["1", "1", "1", "100"]}] + }, + %{ + "dimensions" => ["2021010100", "blog"], + "metrics" => [%{"values" => ["1", "1", "0", "100"]}] + } + ], + site.id, + "utm_contents", + @utc + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "blog", + "visitors" => 2, + "bounce_rate" => 50.0, + "visit_duration" => 50.0 + }, + %{ + "name" => "ad", + "visitors" => 2, + "bounce_rate" => 100.0, + "visit_duration" => 50.0 + } + ] + end end end diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index e9985cffe24f..9ed1a8f29f90 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -889,4 +889,92 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ] end end + + describe "GET /api/stats/:domain/utm_contents" do + setup [:create_user, :log_in, :create_new_site] + + test "returns top utm_contents by unique user ids", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + utm_content: "ad", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + utm_content: "ad", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:15:00] + ), + build(:pageview, + utm_content: "blog", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + utm_content: "blog", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + populate_stats(site, [ + build(:imported_utm_contents, + utm_content: "ad", + timestamp: ~N[2021-01-01 00:00:00], + visit_duration: 700, + bounces: 1, + visits: 1, + visitors: 1 + ), + build(:imported_utm_contents, + utm_content: "blog", + timestamp: ~N[2021-01-01 00:00:00], + bounces: 0, + visits: 1, + visitors: 1, + visit_duration: 900 + ) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "blog", + "visitors" => 2, + "bounce_rate" => 100, + "visit_duration" => 0 + }, + %{ + "name" => "ad", + "visitors" => 1, + "bounce_rate" => 0, + "visit_duration" => 900 + } + ] + + conn = + get( + conn, + "/api/stats/#{site.domain}/utm_contents?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "blog", + "visitors" => 3, + "bounce_rate" => 67, + "visit_duration" => 300 + }, + %{ + "name" => "ad", + "visitors" => 2, + "bounce_rate" => 50, + "visit_duration" => 800.0 + } + ] + end + end end From 8b9089608b77fcaaa922c6480d8426d2feef4eb1 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 18:28:15 +0000 Subject: [PATCH 122/148] Add test for importing pages and entry pages data from GA --- test/plausible/imported/imported_test.exs | 73 +++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 6938311bc9b2..ede970c1fed1 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -4,6 +4,7 @@ defmodule Plausible.ImportedTest do import Plausible.TestUtils @utc Timezone.get("UTC") + @user_id 123 describe "Parse and import third party data fetched from Google Analytics" do setup [:create_user, :log_in, :create_new_site] @@ -282,5 +283,77 @@ defmodule Plausible.ImportedTest do } ] end + + test "Page event data imported from Google Analytics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/some-other-page", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:15:00] + ), + build(:pageview, + pathname: "/", + timestamp: ~N[2021-01-01 00:15:00] + ) + ]) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "/"], + "metrics" => [%{"values" => ["1", "1", "700"]}] + }, + %{ + "dimensions" => ["2021010100", "/some-other-page"], + "metrics" => [%{"values" => ["1", "1", "60"]}] + } + ], + site.id, + "pages", + @utc + ) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "/"], + "metrics" => [%{"values" => ["1", "3", "10", "1"]}] + } + ], + site.id, + "entry_pages", + @utc + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "bounce_rate" => 40.0, + "time_on_page" => 800.0, + "visitors" => 3, + "pageviews" => 3, + "name" => "/" + }, + %{ + "bounce_rate" => nil, + "time_on_page" => 60, + "visitors" => 2, + "pageviews" => 2, + "name" => "/some-other-page" + } + ] + end end end From 380f9aa168bf1dce3665b954c5047ef15e180f17 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 18:38:51 +0000 Subject: [PATCH 123/148] Add test for importing exit page data --- lib/plausible/imported/site.ex | 8 +-- test/plausible/imported/imported_test.exs | 65 +++++++++++++++++++++++ 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 200a27a8c3cb..e8ee175bb13d 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -160,9 +160,9 @@ defmodule Plausible.Imported do defp new_from_google_analytics(site_id, timezone, "pages", %{ "dimensions" => [timestamp, page], - "metrics" => [%{"values" => [value, pageviews, time_on_page]}] + "metrics" => [%{"values" => [visitors, pageviews, time_on_page]}] }) do - {visitors, ""} = Integer.parse(value) + {visitors, ""} = Integer.parse(visitors) {pageviews, ""} = Integer.parse(pageviews) {time_on_page, _} = Integer.parse(time_on_page) @@ -198,9 +198,9 @@ defmodule Plausible.Imported do defp new_from_google_analytics(site_id, timezone, "exit_pages", %{ "dimensions" => [timestamp, exit_page], - "metrics" => [%{"values" => [value, exits]}] + "metrics" => [%{"values" => [visitors, exits]}] }) do - {visitors, ""} = Integer.parse(value) + {visitors, ""} = Integer.parse(visitors) {exits, ""} = Integer.parse(exits) Imported.ExitPages.new(%{ diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index ede970c1fed1..c26d8574b72b 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -355,5 +355,70 @@ defmodule Plausible.ImportedTest do } ] end + + test "Exit page event data imported from Google Analytics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + pathname: "/page1", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/page1", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/page1", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/page2", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:15:00] + ) + ]) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "/page2"], + "metrics" => [%{"values" => ["2", "4", "10"]}] + } + ], + site.id, + "pages", + @utc + ) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "/page2"], + "metrics" => [%{"values" => ["2", "3"]}] + } + ], + site.id, + "exit_pages", + @utc + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/exit-pages?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "/page2", + "unique_exits" => 3, + "total_exits" => 4, + "exit_rate" => 80.0 + }, + %{"name" => "/page1", "unique_exits" => 2, "total_exits" => 2, "exit_rate" => 66} + ] + end end end From a29a7c1e36e59b7f35de981e546180b3c1df43fa Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 18:43:06 +0000 Subject: [PATCH 124/148] Fix module file name typo --- .../imported/{operation_systems.ex => operating_systems.ex} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/plausible/imported/{operation_systems.ex => operating_systems.ex} (100%) diff --git a/lib/plausible/imported/operation_systems.ex b/lib/plausible/imported/operating_systems.ex similarity index 100% rename from lib/plausible/imported/operation_systems.ex rename to lib/plausible/imported/operating_systems.ex From 99e3c1590fc7b33490336ccf78e1ff0ef4a23515 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 18:50:20 +0000 Subject: [PATCH 125/148] Add test for importing location data from GA --- test/plausible/imported/imported_test.exs | 59 +++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index c26d8574b72b..1c99340aba4c 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -420,5 +420,64 @@ defmodule Plausible.ImportedTest do %{"name" => "/page1", "unique_exits" => 2, "total_exits" => 2, "exit_rate" => 66} ] end + + test "Location data imported from Google Analytics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + country_code: "EE", + timestamp: ~N[2021-01-01 00:15:00] + ), + build(:pageview, + country_code: "EE", + timestamp: ~N[2021-01-01 00:15:00] + ), + build(:pageview, + country_code: "GB", + timestamp: ~N[2021-01-01 00:15:00] + ) + ]) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "EE", "Tartumaa"], + "metrics" => [%{"values" => ["1", "1", "0", "10"]}] + }, + %{ + "dimensions" => ["2021010100", "GB", "Midlothian"], + "metrics" => [%{"values" => ["1", "1", "0", "10"]}] + } + ], + site.id, + "locations", + @utc + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/countries?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "code" => "EE", + "alpha_3" => "EST", + "name" => "Estonia", + "flag" => "🇪🇪", + "visitors" => 3, + "percentage" => 60 + }, + %{ + "code" => "GB", + "alpha_3" => "GBR", + "name" => "United Kingdom", + "flag" => "🇬🇧", + "visitors" => 2, + "percentage" => 40 + } + ] + end end end From 82e50cb8aa62aa398c09021ebd8ed218226199b5 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 18:54:48 +0000 Subject: [PATCH 126/148] Add test for importing devices data from GA --- test/plausible/imported/imported_test.exs | 37 +++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 1c99340aba4c..90a11fa10841 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -479,5 +479,42 @@ defmodule Plausible.ImportedTest do } ] end + + test "Devices data imported from Google Analytics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, screen_size: "Laptop", timestamp: ~N[2021-01-01 00:15:00]) + ]) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "mobile"], + "metrics" => [%{"values" => ["1", "1", "0", "10"]}] + }, + %{ + "dimensions" => ["2021010100", "Laptop"], + "metrics" => [%{"values" => ["1", "1", "0", "10"]}] + } + ], + site.id, + "devices", + @utc + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/screen-sizes?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{"name" => "Desktop", "visitors" => 2, "percentage" => 40}, + %{"name" => "Laptop", "visitors" => 2, "percentage" => 40}, + %{"name" => "Mobile", "visitors" => 1, "percentage" => 20} + ] + end end end From f5603f991a1d9ccce7dbf2dfa4857af8fa3f91a1 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 19:00:17 +0000 Subject: [PATCH 127/148] Add test for importing browsers data from GA --- test/plausible/imported/imported_test.exs | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 90a11fa10841..2a78b5211def 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -516,5 +516,41 @@ defmodule Plausible.ImportedTest do %{"name" => "Mobile", "visitors" => 1, "percentage" => 20} ] end + + test "Browsers data imported from Google Analytics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:15:00]) + ]) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "User-Agent: Mozilla"], + "metrics" => [%{"values" => ["1", "1", "0", "10"]}] + }, + %{ + "dimensions" => ["2021010100", "Android Browser"], + "metrics" => [%{"values" => ["1", "1", "0", "10"]}] + } + ], + site.id, + "browsers", + @utc + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/browsers?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{"name" => "Firefox", "visitors" => 2, "percentage" => 50}, + %{"name" => "Mobile App", "visitors" => 1, "percentage" => 25}, + %{"name" => "Chrome", "visitors" => 1, "percentage" => 25} + ] + end end end From b37e814d6f5f9ff7843d314ab0f7089d3b7a44a1 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 19:04:16 +0000 Subject: [PATCH 128/148] Add test for importing OS data from GA --- test/plausible/imported/imported_test.exs | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 2a78b5211def..31eb49443ba2 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -552,5 +552,41 @@ defmodule Plausible.ImportedTest do %{"name" => "Chrome", "visitors" => 1, "percentage" => 25} ] end + + test "OS data imported from Google Analytics", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, operating_system: "GNU/Linux", timestamp: ~N[2021-01-01 00:15:00]) + ]) + + assert {:ok, _} = + Plausible.Imported.from_google_analytics( + [ + %{ + "dimensions" => ["2021010100", "Macintosh"], + "metrics" => [%{"values" => ["1", "1", "0", "10"]}] + }, + %{ + "dimensions" => ["2021010100", "Linux"], + "metrics" => [%{"values" => ["1", "1", "0", "10"]}] + } + ], + site.id, + "operating_systems", + @utc + ) + + conn = + get( + conn, + "/api/stats/#{site.domain}/operating-systems?period=day&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{"name" => "Mac", "visitors" => 3, "percentage" => 60}, + %{"name" => "GNU/Linux", "visitors" => 2, "percentage" => 40} + ] + end end end From 55da66ac1da7f044ef1fb37470feddbcfbccb5d3 Mon Sep 17 00:00:00 2001 From: mcol Date: Sat, 29 Jan 2022 19:43:00 +0000 Subject: [PATCH 129/148] Paginate GA requests to download all data --- lib/plausible/google/api.ex | 88 +++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 38 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index e6768665759e..863cabd2867b 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -251,11 +251,13 @@ defmodule Plausible.Google.Api do } ] - # batchGet can receive a maximum of 5 requests. responses = - request_data - |> Enum.chunk_every(5) - |> Enum.map(&fetch_analytic_reports(&1, request)) + Enum.map( + request_data, + fn {dataset, dimensions, metrics} -> + fetch_analytic_reports(dataset, dimensions, metrics, request) + end + ) case Keyword.get(responses, :error) do nil -> @@ -295,50 +297,60 @@ defmodule Plausible.Google.Api do end end - defp fetch_analytic_reports(request_data, request) do - reports = - Enum.map(request_data, fn {_, dimensions, metrics} -> + defp fetch_analytic_reports(dataset, dimensions, metrics, request, page_token \\ "") do + report = %{ + viewId: request.profile, + dateRanges: [ + %{ + # The earliest valid date + startDate: "2005-01-01", + endDate: request.end_date + } + ], + dimensions: Enum.map(dimensions, &%{name: &1, histogramBuckets: []}), + metrics: Enum.map(metrics, &%{expression: &1}), + hideTotals: true, + hideValueRanges: true, + orderBys: [ %{ - viewId: request.profile, - dateRanges: [ - %{ - # The earliest valid date - startDate: "2005-01-01", - endDate: request.end_date - } - ], - dimensions: Enum.map(dimensions, &%{name: &1, histogramBuckets: []}), - metrics: Enum.map(metrics, &%{expression: &1}), - hideTotals: true, - hideValueRanges: true, - orderBys: [ - %{ - fieldName: "ga:dateHour", - sortOrder: "DESCENDING" - } - ], - pageSize: 100_000 + fieldName: "ga:dateHour", + sortOrder: "DESCENDING" } - end) + ], + pageSize: 100_00, + pageToken: page_token + } res = HTTPoison.post!( "https://analyticsreporting.googleapis.com/v4/reports:batchGet", - Jason.encode!(%{reportRequests: reports}), + Jason.encode!(%{reportRequests: [report]}), Authorization: "Bearer #{request.auth.access_token}" ) if res.status_code == 200 do - data = - Jason.decode!(res.body)["reports"] - |> Enum.with_index() - |> Enum.map(fn {report, index} -> - {dataset, _, _} = Enum.at(request_data, index) - {dataset, report["data"]["rows"]} - end) - |> Map.new() - - {:ok, data} + report = List.first(Jason.decode!(res.body)["reports"]) + data = report["data"]["rows"] + next_page_token = report["nextPageToken"] + + if next_page_token do + # Recursively make more requests until we run out of next page tokens + case fetch_analytic_reports( + dataset, + dimensions, + metrics, + request, + next_page_token + ) do + {:ok, %{^dataset => remainder}} -> + {:ok, %{dataset => [data | remainder]}} + + error -> + error + end + else + {:ok, %{dataset => data}} + end else {:error, Jason.decode!(res.body)["error"]["message"]} end From 2ff7606ce275eb08d2b3bce5928f612e9301853f Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 7 Feb 2022 22:42:57 +0000 Subject: [PATCH 130/148] Bump clickhouse_ecto version --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 835391726f35..659ee7597b6e 100644 --- a/mix.lock +++ b/mix.lock @@ -9,7 +9,7 @@ "cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"}, "certifi": {:hex, :certifi, "2.8.0", "d4fb0a6bb20b7c9c3643e22507e42f356ac090a1dcea9ab99e27e0376d695eba", [:rebar3], [], "hexpm", "6ac7efc1c6f8600b08d625292d4bbf584e14847ce1b6b5c44d983d273e1097ea"}, "chatterbox": {:hex, :ts_chatterbox, "0.11.0", "b8f372c706023eb0de5bf2976764edb27c70fe67052c88c1f6a66b3a5626847f", [:rebar3], [{:hpack, "~>0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "722fe2bad52913ab7e87d849fc6370375f0c961ffb2f0b5e6d647c9170c382a6"}, - "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "93d86c48230f85797555c348dbe9e8738d3b8cc2", []}, + "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "7bc94cce111d3e9dbd8534fe96bd5195181826a2", []}, "clickhousex": {:git, "https://github.com/plausible/clickhousex", "6405ac09b4fa103644bb4fe7fc0509fb48497927", []}, "combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, From ba286227ca292717b12ded280246ccdf629b3114 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 7 Feb 2022 22:44:18 +0000 Subject: [PATCH 131/148] Move RefInspector wrapper function into module --- lib/plausible/imported/sources.ex | 23 ++----------- .../controllers/api/external_controller.ex | 32 ++----------------- lib/plausible_web/refinspector.ex | 25 +++++++++++++++ 3 files changed, 29 insertions(+), 51 deletions(-) create mode 100644 lib/plausible_web/refinspector.ex diff --git a/lib/plausible/imported/sources.ex b/lib/plausible/imported/sources.ex index 25e3d2aa9b2f..8f9e8fd7f1e0 100644 --- a/lib/plausible/imported/sources.ex +++ b/lib/plausible/imported/sources.ex @@ -54,27 +54,8 @@ defmodule Plausible.Imported.Sources do if se do se else - ref = "https://" <> ref - - case RefInspector.parse(ref).source do - :unknown -> - uri = URI.parse(String.trim(ref)) - - if right_uri?(uri) do - String.replace_leading(uri.host, "www.", "") - end - - source -> - source - end + RefInspector.parse("https://" <> ref) + |> PlausibleWeb.RefInspector.parse() end end - - defp right_uri?(%URI{host: nil}), do: false - - defp right_uri?(%URI{host: host, scheme: scheme}) - when scheme in ["http", "https"] and byte_size(host) > 0, - do: true - - defp right_uri?(_), do: false end diff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.ex index 643f32077250..70fe2460de12 100644 --- a/lib/plausible_web/controllers/api/external_controller.ex +++ b/lib/plausible_web/controllers/api/external_controller.ex @@ -495,7 +495,7 @@ defmodule PlausibleWeb.Api.ExternalController do defp clean_referrer(ref) do uri = URI.parse(ref.referer) - if right_uri?(uri) do + if PlausibleWeb.RefInspector.right_uri?(uri) do host = String.replace_prefix(uri.host, "www.", "") path = uri.path || "" host <> String.trim_trailing(path, "/") @@ -574,37 +574,9 @@ defmodule PlausibleWeb.Api.ExternalController do defp get_referrer_source(query, ref) do source = query["utm_source"] || query["source"] || query["ref"] - source || get_source_from_referrer(ref) + source || PlausibleWeb.RefInspector.parse(ref) end - defp get_source_from_referrer(nil), do: nil - - defp get_source_from_referrer(ref) do - case ref.source do - :unknown -> - clean_uri(ref.referer) - - source -> - source - end - end - - defp clean_uri(uri) do - uri = URI.parse(String.trim(uri)) - - if right_uri?(uri) do - String.replace_leading(uri.host, "www.", "") - end - end - - defp right_uri?(%URI{host: nil}), do: false - - defp right_uri?(%URI{host: host, scheme: scheme}) - when scheme in ["http", "https"] and byte_size(host) > 0, - do: true - - defp right_uri?(_), do: false - defp decode_query_params(nil), do: nil defp decode_query_params(%URI{query: nil}), do: nil diff --git a/lib/plausible_web/refinspector.ex b/lib/plausible_web/refinspector.ex new file mode 100644 index 000000000000..154ab9476b22 --- /dev/null +++ b/lib/plausible_web/refinspector.ex @@ -0,0 +1,25 @@ +defmodule PlausibleWeb.RefInspector do + def parse(nil), do: nil + + def parse(ref) do + case ref.source do + :unknown -> + uri = URI.parse(String.trim(ref.referer)) + + if right_uri?(uri) do + String.replace_leading(uri.host, "www.", "") + end + + source -> + source + end + end + + def right_uri?(%URI{host: nil}), do: false + + def right_uri?(%URI{host: host, scheme: scheme}) + when scheme in ["http", "https"] and byte_size(host) > 0, + do: true + + def right_uri?(_), do: false +end From df6d1bce7dcf9f786cba3dc0016aebf8bb61dbd9 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 7 Feb 2022 23:07:52 +0000 Subject: [PATCH 132/148] Drop timezone transform on import --- lib/plausible/google/api.ex | 63 ++++--------- lib/plausible/imported/browsers.ex | 2 +- lib/plausible/imported/devices.ex | 2 +- lib/plausible/imported/entry_pages.ex | 2 +- lib/plausible/imported/exit_pages.ex | 2 +- lib/plausible/imported/locations.ex | 2 +- lib/plausible/imported/operating_systems.ex | 2 +- lib/plausible/imported/pages.ex | 2 +- lib/plausible/imported/site.ex | 64 +++++++------ lib/plausible/imported/sources.ex | 2 +- lib/plausible/imported/utm_campaigns.ex | 2 +- lib/plausible/imported/utm_contents.ex | 2 +- lib/plausible/imported/utm_mediums.ex | 2 +- lib/plausible/imported/utm_sources.ex | 2 +- lib/plausible/imported/utm_terms.ex | 2 +- lib/plausible/imported/visitors.ex | 2 +- ...0211112130238_create_imported_visitors.exs | 2 +- ...20211118160420_create_imported_sources.exs | 2 +- ...1124135248_create_imported_utm_mediums.exs | 2 +- ...24135252_create_imported_utm_campaigns.exs | 2 +- .../20211129111618_create_imported_pages.exs | 2 +- ...1129111622_create_imported_entry_pages.exs | 2 +- ...11129111630_create_imported_exit_pages.exs | 2 +- ...211129111639_create_imported_locations.exs | 2 +- ...20211129111644_create_imported_devices.exs | 2 +- ...0211129111648_create_imported_browsers.exs | 2 +- ...1653_create_imported_operating_systems.exs | 2 +- ...1129164103_create_imported_utm_sources.exs | 2 +- ...211231120503_create_imported_utm_terms.exs | 2 +- ...231120603_create_imported_utm_contents.exs | 2 +- test/plausible/imported/imported_test.exs | 93 ++++++++----------- .../api/stats_controller/main_graph_test.exs | 8 +- .../api/stats_controller/pages_test.exs | 12 +-- .../api/stats_controller/sources_test.exs | 24 ++--- test/support/factory.ex | 2 +- 35 files changed, 138 insertions(+), 184 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 863cabd2867b..a77c215abe55 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -154,8 +154,7 @@ defmodule Plausible.Google.Api do def import_analytics(site, profile) do with {:ok, auth} <- refresh_if_needed(site.google_auth) do - {:ok, timezone} = get_profile_timezone(auth, profile) - do_import_analytics(site, auth, profile, timezone) + do_import_analytics(site, auth, profile) end end @@ -165,7 +164,7 @@ defmodule Plausible.Google.Api do Dimensions reference: https://ga-dev-tools.web.app/dimensions-metrics-explorer """ - def do_import_analytics(site, auth, profile, timezone) do + def do_import_analytics(site, auth, profile) do end_date = Plausible.Stats.Clickhouse.pageviews_begin(site) |> NaiveDateTime.to_date() @@ -180,7 +179,7 @@ defmodule Plausible.Google.Api do request_data = [ { "visitors", - ["ga:dateHour"], + ["ga:date"], [ "ga:users", "ga:pageviews", @@ -191,62 +190,62 @@ defmodule Plausible.Google.Api do }, { "sources", - ["ga:dateHour", "ga:fullReferrer"], + ["ga:date", "ga:fullReferrer"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { "utm_mediums", - ["ga:dateHour", "ga:medium"], + ["ga:date", "ga:medium"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { "utm_campaigns", - ["ga:dateHour", "ga:campaign"], + ["ga:date", "ga:campaign"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { "utm_terms", - ["ga:dateHour", "ga:keyword"], + ["ga:date", "ga:keyword"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { "utm_contents", - ["ga:dateHour", "ga:adContent"], + ["ga:date", "ga:adContent"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { "pages", - ["ga:dateHour", "ga:pagePath"], + ["ga:date", "ga:pagePath"], ["ga:users", "ga:pageviews", "ga:timeOnPage"] }, { "entry_pages", - ["ga:dateHour", "ga:landingPagePath"], + ["ga:date", "ga:landingPagePath"], ["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"] }, { "exit_pages", - ["ga:dateHour", "ga:exitPagePath"], + ["ga:date", "ga:exitPagePath"], ["ga:users", "ga:exits"] }, { "locations", - ["ga:dateHour", "ga:countryIsoCode", "ga:regionIsoCode"], + ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { "devices", - ["ga:dateHour", "ga:deviceCategory"], + ["ga:date", "ga:deviceCategory"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { "browsers", - ["ga:dateHour", "ga:browser"], + ["ga:date", "ga:browser"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { "operating_systems", - ["ga:dateHour", "ga:operatingSystem"], + ["ga:date", "ga:operatingSystem"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] } ] @@ -271,7 +270,7 @@ defmodule Plausible.Google.Api do results |> Enum.map(fn {dataset, data} -> Task.async(fn -> - Imported.from_google_analytics(data, site.id, dataset, timezone) + Imported.from_google_analytics(data, site.id, dataset) end) end) |> Enum.map(&Task.await(&1, 120_000)) @@ -313,7 +312,7 @@ defmodule Plausible.Google.Api do hideValueRanges: true, orderBys: [ %{ - fieldName: "ga:dateHour", + fieldName: "ga:date", sortOrder: "DESCENDING" } ], @@ -356,34 +355,6 @@ defmodule Plausible.Google.Api do end end - defp get_profile_timezone(auth, profile) do - res = - HTTPoison.get!( - "https://www.googleapis.com/analytics/v3/management/accounts/~all/webproperties/~all/profiles", - Authorization: "Bearer #{auth.access_token}" - ) - - case res.status_code do - 200 -> - timezone_info = - Jason.decode!(res.body) - |> Map.get("items") - |> Enum.map(fn item -> {Map.get(item, "id"), Map.get(item, "timezone")} end) - |> Map.new() - |> Map.get(profile) - |> Timezone.get() - - {:ok, timezone_info} - - _ -> - Sentry.capture_message("Error fetching Google view ID during import", - extra: Jason.decode!(res.body) - ) - - {:error, res.body} - end - end - defp refresh_if_needed(auth) do if Timex.before?(auth.expires, Timex.now() |> Timex.shift(seconds: 30)) do refresh_token(auth) diff --git a/lib/plausible/imported/browsers.ex b/lib/plausible/imported/browsers.ex index a9cd8d9fdd76..bd1d3650b170 100644 --- a/lib/plausible/imported/browsers.ex +++ b/lib/plausible/imported/browsers.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.Browsers do @primary_key false schema "imported_browsers" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :browser, :string field :visitors, :integer field :visits, :integer diff --git a/lib/plausible/imported/devices.ex b/lib/plausible/imported/devices.ex index 576ed79961a3..2c3091317874 100644 --- a/lib/plausible/imported/devices.ex +++ b/lib/plausible/imported/devices.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.Devices do @primary_key false schema "imported_devices" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :device, :string field :visitors, :integer field :visits, :integer diff --git a/lib/plausible/imported/entry_pages.ex b/lib/plausible/imported/entry_pages.ex index 0e5a6363b3c9..c94d1d8b5d13 100644 --- a/lib/plausible/imported/entry_pages.ex +++ b/lib/plausible/imported/entry_pages.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.EntryPages do @primary_key false schema "imported_entry_pages" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :entry_page, :string field :visitors, :integer field :entrances, :integer diff --git a/lib/plausible/imported/exit_pages.ex b/lib/plausible/imported/exit_pages.ex index fbb81bc963d9..5c2df2734fea 100644 --- a/lib/plausible/imported/exit_pages.ex +++ b/lib/plausible/imported/exit_pages.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.ExitPages do @primary_key false schema "imported_exit_pages" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :exit_page, :string field :visitors, :integer field :exits, :integer diff --git a/lib/plausible/imported/locations.ex b/lib/plausible/imported/locations.ex index bbc8f71c4946..6ffeda8d0dc5 100644 --- a/lib/plausible/imported/locations.ex +++ b/lib/plausible/imported/locations.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.Locations do @primary_key false schema "imported_locations" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :country, :string, default: "" field :region, :string, default: "" field :city, :integer, default: 0 diff --git a/lib/plausible/imported/operating_systems.ex b/lib/plausible/imported/operating_systems.ex index e8a0c9c332f4..ed4810b6a721 100644 --- a/lib/plausible/imported/operating_systems.ex +++ b/lib/plausible/imported/operating_systems.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.OperatingSystems do @primary_key false schema "imported_operating_systems" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :operating_system, :string field :visitors, :integer field :visits, :integer diff --git a/lib/plausible/imported/pages.ex b/lib/plausible/imported/pages.ex index c73b66442435..29a4183681e8 100644 --- a/lib/plausible/imported/pages.ex +++ b/lib/plausible/imported/pages.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.Pages do @primary_key false schema "imported_pages" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :page, :string field :visitors, :integer field :pageviews, :integer diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index e8ee175bb13d..65e328f0ffb2 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -8,11 +8,11 @@ defmodule Plausible.Imported do def from_google_analytics(nil, _site_id, _metric, _timezone), do: {:ok, nil} - def from_google_analytics(data, site_id, metric, timezone) do + def from_google_analytics(data, site_id, metric) do maybe_error = data |> Enum.map(fn row -> - new_from_google_analytics(site_id, timezone, metric, row) + new_from_google_analytics(site_id, metric, row) |> Plausible.ClickhouseRepo.insert(on_conflict: :replace_all) end) |> Keyword.get(:error) @@ -26,7 +26,7 @@ defmodule Plausible.Imported do end end - defp new_from_google_analytics(site_id, timezone, "visitors", %{ + defp new_from_google_analytics(site_id, "visitors", %{ "dimensions" => [timestamp], "metrics" => [%{"values" => values}] }) do @@ -37,7 +37,7 @@ defmodule Plausible.Imported do Imported.Visitors.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), visitors: visitors, pageviews: pageviews, bounces: bounces, @@ -46,7 +46,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(site_id, timezone, "sources", %{ + defp new_from_google_analytics(site_id, "sources", %{ "dimensions" => [timestamp, source], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -59,7 +59,7 @@ defmodule Plausible.Imported do Imported.Sources.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), source: Imported.Sources.parse(source), visitors: visitors, visits: visits, @@ -70,7 +70,7 @@ defmodule Plausible.Imported do # TODO: utm_sources. Google reports sources and utm_sources unified. - defp new_from_google_analytics(site_id, timezone, "utm_mediums", %{ + defp new_from_google_analytics(site_id, "utm_mediums", %{ "dimensions" => [timestamp, medium], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -83,7 +83,7 @@ defmodule Plausible.Imported do Imported.UtmMediums.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), utm_medium: medium, visitors: visitors, visits: visits, @@ -92,7 +92,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(site_id, timezone, "utm_campaigns", %{ + defp new_from_google_analytics(site_id, "utm_campaigns", %{ "dimensions" => [timestamp, campaign], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -105,7 +105,7 @@ defmodule Plausible.Imported do Imported.UtmCampaigns.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), utm_campaign: campaign, visitors: visitors, visits: visits, @@ -114,7 +114,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(site_id, timezone, "utm_terms", %{ + defp new_from_google_analytics(site_id, "utm_terms", %{ "dimensions" => [timestamp, term], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -127,7 +127,7 @@ defmodule Plausible.Imported do Imported.UtmTerms.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), utm_term: term, visitors: visitors, visits: visits, @@ -136,7 +136,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(site_id, timezone, "utm_contents", %{ + defp new_from_google_analytics(site_id, "utm_contents", %{ "dimensions" => [timestamp, content], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -149,7 +149,7 @@ defmodule Plausible.Imported do Imported.UtmContents.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), utm_content: content, visitors: visitors, visits: visits, @@ -158,7 +158,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(site_id, timezone, "pages", %{ + defp new_from_google_analytics(site_id, "pages", %{ "dimensions" => [timestamp, page], "metrics" => [%{"values" => [visitors, pageviews, time_on_page]}] }) do @@ -168,7 +168,7 @@ defmodule Plausible.Imported do Imported.Pages.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), page: page, visitors: visitors, pageviews: pageviews, @@ -176,7 +176,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(site_id, timezone, "entry_pages", %{ + defp new_from_google_analytics(site_id, "entry_pages", %{ "dimensions" => [timestamp, entry_page], "metrics" => [%{"values" => [visitors, entrances, visit_duration, bounces]}] }) do @@ -187,7 +187,7 @@ defmodule Plausible.Imported do Imported.EntryPages.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), entry_page: entry_page, visitors: visitors, entrances: entrances, @@ -196,7 +196,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(site_id, timezone, "exit_pages", %{ + defp new_from_google_analytics(site_id, "exit_pages", %{ "dimensions" => [timestamp, exit_page], "metrics" => [%{"values" => [visitors, exits]}] }) do @@ -205,14 +205,14 @@ defmodule Plausible.Imported do Imported.ExitPages.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), exit_page: exit_page, visitors: visitors, exits: exits }) end - defp new_from_google_analytics(site_id, timezone, "locations", %{ + defp new_from_google_analytics(site_id, "locations", %{ "dimensions" => [timestamp, country, region], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -225,7 +225,7 @@ defmodule Plausible.Imported do Imported.Locations.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), country: country, region: region, city: 0, @@ -236,7 +236,7 @@ defmodule Plausible.Imported do }) end - defp new_from_google_analytics(site_id, timezone, "devices", %{ + defp new_from_google_analytics(site_id, "devices", %{ "dimensions" => [timestamp, device], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -247,7 +247,7 @@ defmodule Plausible.Imported do Imported.Devices.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), device: String.capitalize(device), visitors: visitors, visits: visits, @@ -266,7 +266,7 @@ defmodule Plausible.Imported do "(not set)" => "" } - defp new_from_google_analytics(site_id, timezone, "browsers", %{ + defp new_from_google_analytics(site_id, "browsers", %{ "dimensions" => [timestamp, browser], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -277,7 +277,7 @@ defmodule Plausible.Imported do Imported.Browsers.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), browser: Map.get(@browser_google_to_plausible, browser, browser), visitors: visitors, visits: visits, @@ -292,7 +292,7 @@ defmodule Plausible.Imported do "(not set)" => "" } - defp new_from_google_analytics(site_id, timezone, "operating_systems", %{ + defp new_from_google_analytics(site_id, "operating_systems", %{ "dimensions" => [timestamp, operating_system], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -303,7 +303,7 @@ defmodule Plausible.Imported do Imported.OperatingSystems.new(%{ site_id: site_id, - timestamp: format_timestamp(timestamp, timezone), + timestamp: format_timestamp(timestamp), operating_system: Map.get(@os_google_to_plausible, operating_system, operating_system), visitors: visitors, visits: visits, @@ -312,10 +312,8 @@ defmodule Plausible.Imported do }) end - defp format_timestamp(timestamp, timezone) do - Timex.parse!("#{timestamp}", "%Y%m%d%H", :strftime) - |> Timezone.convert(timezone) - |> Timezone.convert("UTC") - |> DateTime.to_naive() + defp format_timestamp(timestamp) do + Timex.parse!("#{timestamp}", "%Y%m%d", :strftime) + |> NaiveDateTime.to_date() end end diff --git a/lib/plausible/imported/sources.ex b/lib/plausible/imported/sources.ex index 8f9e8fd7f1e0..d0bf3dd388e7 100644 --- a/lib/plausible/imported/sources.ex +++ b/lib/plausible/imported/sources.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.Sources do @primary_key false schema "imported_sources" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :source, :string, default: "" field :visitors, :integer field :visits, :integer diff --git a/lib/plausible/imported/utm_campaigns.ex b/lib/plausible/imported/utm_campaigns.ex index 7a534a0cabb0..720348a72d63 100644 --- a/lib/plausible/imported/utm_campaigns.ex +++ b/lib/plausible/imported/utm_campaigns.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.UtmCampaigns do @primary_key false schema "imported_utm_campaigns" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :utm_campaign, :string, default: "" field :visitors, :integer field :visits, :integer diff --git a/lib/plausible/imported/utm_contents.ex b/lib/plausible/imported/utm_contents.ex index 20ad64d270a1..8033ffbc540b 100644 --- a/lib/plausible/imported/utm_contents.ex +++ b/lib/plausible/imported/utm_contents.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.UtmContents do @primary_key false schema "imported_utm_contents" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :utm_content, :string, default: "" field :visitors, :integer field :visits, :integer diff --git a/lib/plausible/imported/utm_mediums.ex b/lib/plausible/imported/utm_mediums.ex index 96e30b82fc6f..f34c3ef02c63 100644 --- a/lib/plausible/imported/utm_mediums.ex +++ b/lib/plausible/imported/utm_mediums.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.UtmMediums do @primary_key false schema "imported_utm_mediums" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :utm_medium, :string, default: "" field :visitors, :integer field :visits, :integer diff --git a/lib/plausible/imported/utm_sources.ex b/lib/plausible/imported/utm_sources.ex index 038953a4fe81..554daa7ae190 100644 --- a/lib/plausible/imported/utm_sources.ex +++ b/lib/plausible/imported/utm_sources.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.UtmSources do @primary_key false schema "imported_utm_sources" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :utm_source, :string, default: "" field :visitors, :integer field :visits, :integer diff --git a/lib/plausible/imported/utm_terms.ex b/lib/plausible/imported/utm_terms.ex index f28db02e34fb..bf3f934db87d 100644 --- a/lib/plausible/imported/utm_terms.ex +++ b/lib/plausible/imported/utm_terms.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.UtmTerms do @primary_key false schema "imported_utm_terms" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :utm_term, :string, default: "" field :visitors, :integer field :visits, :integer diff --git a/lib/plausible/imported/visitors.ex b/lib/plausible/imported/visitors.ex index b639c0fd1541..ea312f5da90e 100644 --- a/lib/plausible/imported/visitors.ex +++ b/lib/plausible/imported/visitors.ex @@ -6,7 +6,7 @@ defmodule Plausible.Imported.Visitors do @primary_key false schema "imported_visitors" do field :site_id, :integer - field :timestamp, :naive_datetime + field :timestamp, :date field :visitors, :integer field :pageviews, :integer field :bounces, :integer diff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs index e0cec007815d..0d99f495098d 100644 --- a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs +++ b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do def change do create_if_not_exists table(:imported_visitors, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :visitors, :UInt64 add :pageviews, :UInt64 add :bounces, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs b/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs index 18dab5895fc4..21b5f9c876bd 100644 --- a/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs +++ b/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedSources do def change do create_if_not_exists table(:imported_sources, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :source, :string add :visitors, :UInt64 add :visits, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs b/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs index ac110e113619..4bb02eb59b05 100644 --- a/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs +++ b/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmMediums do def change do create_if_not_exists table(:imported_utm_mediums, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :utm_medium, :string add :visitors, :UInt64 add :visits, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs b/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs index c9fcfc666b94..9c2a497b2587 100644 --- a/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs +++ b/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmCampaigns do def change do create_if_not_exists table(:imported_utm_campaigns, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :utm_campaign, :string add :visitors, :UInt64 add :visits, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs b/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs index 1a93f3284693..3ed65cc1e0ae 100644 --- a/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedPages do def change do create_if_not_exists table(:imported_pages, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :page, :string add :visitors, :UInt64 add :pageviews, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs index 291e04aeeadf..62e08dafa868 100644 --- a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedEntryPages do def change do create_if_not_exists table(:imported_entry_pages, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :entry_page, :string add :visitors, :UInt64 add :entrances, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs b/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs index 12f73b73bb63..c4e5a244cffd 100644 --- a/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedExitPages do def change do create_if_not_exists table(:imported_exit_pages, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :exit_page, :string add :visitors, :UInt64 add :exits, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs index 9a6dfeb1d363..291318bbbbc1 100644 --- a/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs +++ b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedLocations do def change do create_if_not_exists table(:imported_locations, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :country, :string add :region, :string add :city, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs b/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs index 112de1a3123d..9fd222ea542d 100644 --- a/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs +++ b/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedDevices do def change do create_if_not_exists table(:imported_devices, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :device, :string add :visitors, :UInt64 add :visits, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs index 897d21a4c6db..5b4925796984 100644 --- a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs +++ b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedBrowsers do def change do create_if_not_exists table(:imported_browsers, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :browser, :string add :visitors, :UInt64 add :visits, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs index b18d9db0332d..0eb5cf193888 100644 --- a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs +++ b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedOperatingSystems do def change do create_if_not_exists table(:imported_operating_systems, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :operating_system, :string add :visitors, :UInt64 add :visits, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs b/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs index 7055e276b859..e6af6b6c27da 100644 --- a/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs +++ b/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmSources do def change do create_if_not_exists table(:imported_utm_sources, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :utm_source, :string add :visitors, :UInt64 add :visits, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs b/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs index 795137a2e410..56083b88943b 100644 --- a/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs +++ b/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmTerms do def change do create_if_not_exists table(:imported_utm_terms, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :utm_term, :string add :visitors, :UInt64 add :visits, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs b/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs index 42c5ee98b1a1..07d2d30e660d 100644 --- a/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs +++ b/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs @@ -4,7 +4,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmContents do def change do create_if_not_exists table(:imported_utm_contents, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 - add :timestamp, :naive_datetime + add :timestamp, :date add :utm_content, :string add :visitors, :UInt64 add :visits, :UInt64 diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 31eb49443ba2..180f006c3609 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -3,7 +3,6 @@ defmodule Plausible.ImportedTest do use Timex import Plausible.TestUtils - @utc Timezone.get("UTC") @user_id 123 describe "Parse and import third party data fetched from Google Analytics" do @@ -19,17 +18,16 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100"], + "dimensions" => ["20210101"], "metrics" => [%{"values" => ["1", "1", "0", "1", "60"]}] }, %{ - "dimensions" => ["2021013100"], + "dimensions" => ["20210131"], "metrics" => [%{"values" => ["1", "1", "1", "1", "60"]}] } ], site.id, - "visitors", - @utc + "visitors" ) conn = @@ -69,17 +67,16 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "duckduckgo.com"], + "dimensions" => ["20210101", "duckduckgo.com"], "metrics" => [%{"values" => ["1", "1", "0", "60"]}] }, %{ - "dimensions" => ["2021013100", "google.com"], + "dimensions" => ["20210131", "google.com"], "metrics" => [%{"values" => ["1", "1", "1", "60"]}] } ], site.id, - "sources", - @utc + "sources" ) conn = @@ -114,17 +111,16 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "social"], + "dimensions" => ["20210101", "social"], "metrics" => [%{"values" => ["1", "1", "1", "60"]}] }, %{ - "dimensions" => ["2021010100", "email"], + "dimensions" => ["20210101", "email"], "metrics" => [%{"values" => ["1", "1", "0", "100"]}] } ], site.id, - "utm_mediums", - @utc + "utm_mediums" ) conn = @@ -159,17 +155,16 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "profile"], + "dimensions" => ["20210101", "profile"], "metrics" => [%{"values" => ["1", "1", "1", "100"]}] }, %{ - "dimensions" => ["2021010100", "august"], + "dimensions" => ["20210101", "august"], "metrics" => [%{"values" => ["1", "1", "0", "100"]}] } ], site.id, - "utm_campaigns", - @utc + "utm_campaigns" ) conn = @@ -204,17 +199,16 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "oat milk"], + "dimensions" => ["20210101", "oat milk"], "metrics" => [%{"values" => ["1", "1", "1", "100"]}] }, %{ - "dimensions" => ["2021010100", "Sweden"], + "dimensions" => ["20210101", "Sweden"], "metrics" => [%{"values" => ["1", "1", "0", "100"]}] } ], site.id, - "utm_terms", - @utc + "utm_terms" ) conn = @@ -249,17 +243,16 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "ad"], + "dimensions" => ["20210101", "ad"], "metrics" => [%{"values" => ["1", "1", "1", "100"]}] }, %{ - "dimensions" => ["2021010100", "blog"], + "dimensions" => ["20210101", "blog"], "metrics" => [%{"values" => ["1", "1", "0", "100"]}] } ], site.id, - "utm_contents", - @utc + "utm_contents" ) conn = @@ -306,30 +299,28 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "/"], + "dimensions" => ["20210101", "/"], "metrics" => [%{"values" => ["1", "1", "700"]}] }, %{ - "dimensions" => ["2021010100", "/some-other-page"], + "dimensions" => ["20210101", "/some-other-page"], "metrics" => [%{"values" => ["1", "1", "60"]}] } ], site.id, - "pages", - @utc + "pages" ) assert {:ok, _} = Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "/"], + "dimensions" => ["20210101", "/"], "metrics" => [%{"values" => ["1", "3", "10", "1"]}] } ], site.id, - "entry_pages", - @utc + "entry_pages" ) conn = @@ -382,26 +373,24 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "/page2"], + "dimensions" => ["20210101", "/page2"], "metrics" => [%{"values" => ["2", "4", "10"]}] } ], site.id, - "pages", - @utc + "pages" ) assert {:ok, _} = Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "/page2"], + "dimensions" => ["20210101", "/page2"], "metrics" => [%{"values" => ["2", "3"]}] } ], site.id, - "exit_pages", - @utc + "exit_pages" ) conn = @@ -441,17 +430,16 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "EE", "Tartumaa"], + "dimensions" => ["20210101", "EE", "Tartumaa"], "metrics" => [%{"values" => ["1", "1", "0", "10"]}] }, %{ - "dimensions" => ["2021010100", "GB", "Midlothian"], + "dimensions" => ["20210101", "GB", "Midlothian"], "metrics" => [%{"values" => ["1", "1", "0", "10"]}] } ], site.id, - "locations", - @utc + "locations" ) conn = @@ -491,17 +479,16 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "mobile"], + "dimensions" => ["20210101", "mobile"], "metrics" => [%{"values" => ["1", "1", "0", "10"]}] }, %{ - "dimensions" => ["2021010100", "Laptop"], + "dimensions" => ["20210101", "Laptop"], "metrics" => [%{"values" => ["1", "1", "0", "10"]}] } ], site.id, - "devices", - @utc + "devices" ) conn = @@ -527,17 +514,16 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "User-Agent: Mozilla"], + "dimensions" => ["20210101", "User-Agent: Mozilla"], "metrics" => [%{"values" => ["1", "1", "0", "10"]}] }, %{ - "dimensions" => ["2021010100", "Android Browser"], + "dimensions" => ["20210101", "Android Browser"], "metrics" => [%{"values" => ["1", "1", "0", "10"]}] } ], site.id, - "browsers", - @utc + "browsers" ) conn = @@ -564,17 +550,16 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["2021010100", "Macintosh"], + "dimensions" => ["20210101", "Macintosh"], "metrics" => [%{"values" => ["1", "1", "0", "10"]}] }, %{ - "dimensions" => ["2021010100", "Linux"], + "dimensions" => ["20210101", "Linux"], "metrics" => [%{"values" => ["1", "1", "0", "10"]}] } ], site.id, - "operating_systems", - @utc + "operating_systems" ) conn = diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index 70d3feb5ba77..ba3a5f444e36 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -73,8 +73,8 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do populate_stats(site, [ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), build(:pageview, timestamp: ~N[2021-01-31 00:00:00]), - build(:imported_visitors, timestamp: ~N[2021-01-01 00:00:00]), - build(:imported_visitors, timestamp: ~N[2021-01-31 00:00:00]) + build(:imported_visitors, timestamp: ~D[2021-01-01]), + build(:imported_visitors, timestamp: ~D[2021-01-31]) ]) conn = @@ -95,8 +95,8 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do populate_stats(site, [ build(:pageview, timestamp: ~N[2021-01-01 00:00:00], pathname: "/pageA"), build(:pageview, timestamp: ~N[2021-01-31 00:00:00], pathname: "/pageA"), - build(:imported_visitors, timestamp: ~N[2021-01-01 00:00:00]), - build(:imported_visitors, timestamp: ~N[2021-01-31 00:00:00]) + build(:imported_visitors, timestamp: ~D[2021-01-01]), + build(:imported_visitors, timestamp: ~D[2021-01-31]) ]) filters = Jason.encode!(%{page: "/pageA"}) diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index 4ad9438432c5..bb8b0179e842 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -117,18 +117,18 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ), build(:imported_pages, page: "/", - timestamp: ~N[2021-01-01 00:15:00], + timestamp: ~D[2021-01-01], time_on_page: 700 ), build(:imported_entry_pages, entry_page: "/", - timestamp: ~N[2021-01-01 00:15:00], + timestamp: ~D[2021-01-01], entrances: 3, bounces: 1 ), build(:imported_pages, page: "/some-other-page", - timestamp: ~N[2021-01-01 00:15:00], + timestamp: ~D[2021-01-01], time_on_page: 60 ) ]) @@ -274,7 +274,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do populate_stats(site, [ build(:imported_entry_pages, entry_page: "/page2", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], entrances: 3, visitors: 2, visit_duration: 300 @@ -441,13 +441,13 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do populate_stats(site, [ build(:imported_pages, page: "/page2", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], pageviews: 4, visitors: 2 ), build(:imported_exit_pages, exit_page: "/page2", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], exits: 3, visitors: 2 ) diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index 9ed1a8f29f90..fed047edf2e3 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -133,7 +133,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do populate_stats(site, [ build(:imported_sources, source: "Google", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], visitors: 2, visits: 3, bounces: 1, @@ -141,7 +141,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ), build(:imported_sources, source: "DuckDuckGo", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], visitors: 1, visits: 1, visit_duration: 100, @@ -292,7 +292,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do populate_stats(site, [ build(:imported_utm_mediums, utm_medium: "social", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], visit_duration: 700, bounces: 1, visits: 1, @@ -300,7 +300,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ), build(:imported_utm_mediums, utm_medium: "email", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], bounces: 0, visits: 1, visitors: 1, @@ -380,7 +380,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do populate_stats(site, [ build(:imported_utm_campaigns, utm_campaign: "profile", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], visit_duration: 700, bounces: 1, visits: 1, @@ -388,7 +388,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ), build(:imported_utm_campaigns, utm_campaign: "august", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], bounces: 0, visits: 1, visitors: 1, @@ -468,7 +468,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do populate_stats(site, [ build(:imported_utm_sources, utm_source: "Twitter", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], visit_duration: 700, bounces: 1, visits: 1, @@ -476,7 +476,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ), build(:imported_utm_sources, utm_source: "newsletter", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], bounces: 0, visits: 1, visitors: 1, @@ -830,7 +830,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do populate_stats(site, [ build(:imported_utm_terms, utm_term: "oat milk", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], visit_duration: 700, bounces: 1, visits: 1, @@ -838,7 +838,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ), build(:imported_utm_terms, utm_term: "Sweden", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], bounces: 0, visits: 1, visitors: 1, @@ -918,7 +918,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do populate_stats(site, [ build(:imported_utm_contents, utm_content: "ad", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], visit_duration: 700, bounces: 1, visits: 1, @@ -926,7 +926,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ), build(:imported_utm_contents, utm_content: "blog", - timestamp: ~N[2021-01-01 00:00:00], + timestamp: ~D[2021-01-01], bounces: 0, visits: 1, visitors: 1, diff --git a/test/support/factory.ex b/test/support/factory.ex index 9bc7c3e3c09c..ef2e4317f12a 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -181,7 +181,7 @@ defmodule Plausible.Factory do } end - @today Timex.today() |> Timex.to_naive_datetime() + @today Timex.today() |> Timex.to_date() def imported_visitors_factory do %Plausible.Imported.Visitors{ From 92ffae126130f1cd14b72df899d7cc7b332ccd87 Mon Sep 17 00:00:00 2001 From: mcol Date: Mon, 7 Feb 2022 23:13:01 +0000 Subject: [PATCH 133/148] Order imported by side_id then date --- lib/plausible/stats/breakdown.ex | 4 +-- lib/plausible/stats/imported.ex | 25 ++++++++----------- ...0211112130238_create_imported_visitors.exs | 2 +- ...20211118160420_create_imported_sources.exs | 2 +- ...1124135248_create_imported_utm_mediums.exs | 2 +- ...24135252_create_imported_utm_campaigns.exs | 2 +- .../20211129111618_create_imported_pages.exs | 2 +- ...1129111622_create_imported_entry_pages.exs | 2 +- ...11129111630_create_imported_exit_pages.exs | 2 +- ...211129111639_create_imported_locations.exs | 2 +- ...20211129111644_create_imported_devices.exs | 2 +- ...0211129111648_create_imported_browsers.exs | 2 +- ...1653_create_imported_operating_systems.exs | 2 +- ...1129164103_create_imported_utm_sources.exs | 2 +- ...211231120503_create_imported_utm_terms.exs | 2 +- ...231120603_create_imported_utm_contents.exs | 2 +- 16 files changed, 25 insertions(+), 32 deletions(-) diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index ab6cac7abfe9..35cef5969d9f 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -278,8 +278,6 @@ defmodule Plausible.Stats.Breakdown do if with_imported do # Imported page views have pre-calculated values - {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) - res = res.rows |> Enum.map(fn [page, time, visits] -> {page, {time, visits}} end) @@ -289,7 +287,7 @@ defmodule Plausible.Stats.Breakdown do i in "imported_pages", group_by: i.page, where: i.site_id == ^site.id, - where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, + where: i.timestamp >= ^query.date_range.first and i.timestamp <= ^query.date_range.last, where: i.page in ^pages, select: %{ page: i.page, diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index ba1b68d72cf4..a2a6490e2f79 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -1,27 +1,26 @@ defmodule Plausible.Stats.Imported do use Plausible.ClickhouseRepo - alias Plausible.Stats.{Query, Timeseries} - import Plausible.Stats.Base + alias Plausible.Stats.Query import Ecto.Query @no_ref "Direct / None" def timeseries(site, query) do - {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) - result = from(v in "imported_visitors", group_by: fragment("date"), where: v.site_id == ^site.id, - where: v.timestamp >= ^first_datetime and v.timestamp < ^last_datetime, - select: %{"visitors" => sum(v.visitors)} + where: v.timestamp >= ^query.date_range.first and v.timestamp <= ^query.date_range.last, + select: %{ + visitors: sum(v.visitors), + date: fragment("? as date", v.timestamp) + } ) - |> Timeseries.select_bucket(site, query) |> ClickhouseRepo.all() - |> Enum.map(fn row -> {row["date"], row["visitors"]} end) + |> Enum.map(fn row -> {row[:date], row[:visitors]} end) |> Map.new() - Timeseries.buckets(query) + Enum.into(query.date_range, []) |> Enum.map(fn step -> Map.get(result, step, 0) end) end @@ -47,8 +46,6 @@ defmodule Plausible.Stats.Imported do "visit:os", "event:page" ] do - {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) - {table, dim} = case property do "visit:country" -> @@ -76,7 +73,7 @@ defmodule Plausible.Stats.Imported do i in table, group_by: field(i, ^dim), where: i.site_id == ^site.id, - where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, + where: i.timestamp >= ^query.date_range.first and i.timestamp <= ^query.date_range.last, select: %{} ) |> select_imported_metrics(metrics) @@ -279,13 +276,11 @@ defmodule Plausible.Stats.Imported do end def merge_imported(q, site, query, :aggregate, metrics) do - {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) - imported_q = from( i in "imported_visitors", where: i.site_id == ^site.id, - where: i.timestamp >= ^first_datetime and i.timestamp < ^last_datetime, + where: i.timestamp >= ^query.date_range.first and i.timestamp <= ^query.date_range.last, select: %{} ) |> select_imported_metrics(metrics) diff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs index 0d99f495098d..239b3ddef9de 100644 --- a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs +++ b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do use Ecto.Migration def change do - create_if_not_exists table(:imported_visitors, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_visitors, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :visitors, :UInt64 diff --git a/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs b/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs index 21b5f9c876bd..a8a3e8909290 100644 --- a/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs +++ b/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedSources do use Ecto.Migration def change do - create_if_not_exists table(:imported_sources, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_sources, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :source, :string diff --git a/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs b/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs index 4bb02eb59b05..4e09eadc93a5 100644 --- a/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs +++ b/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmMediums do use Ecto.Migration def change do - create_if_not_exists table(:imported_utm_mediums, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_utm_mediums, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :utm_medium, :string diff --git a/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs b/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs index 9c2a497b2587..57e918598c4f 100644 --- a/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs +++ b/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmCampaigns do use Ecto.Migration def change do - create_if_not_exists table(:imported_utm_campaigns, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_utm_campaigns, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :utm_campaign, :string diff --git a/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs b/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs index 3ed65cc1e0ae..b836d918b17e 100644 --- a/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedPages do use Ecto.Migration def change do - create_if_not_exists table(:imported_pages, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_pages, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :page, :string diff --git a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs index 62e08dafa868..5d72324d9836 100644 --- a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedEntryPages do use Ecto.Migration def change do - create_if_not_exists table(:imported_entry_pages, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_entry_pages, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :entry_page, :string diff --git a/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs b/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs index c4e5a244cffd..1ba293e55cdc 100644 --- a/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs +++ b/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedExitPages do use Ecto.Migration def change do - create_if_not_exists table(:imported_exit_pages, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_exit_pages, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :exit_page, :string diff --git a/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs index 291318bbbbc1..77dd7f53e4a5 100644 --- a/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs +++ b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedLocations do use Ecto.Migration def change do - create_if_not_exists table(:imported_locations, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_locations, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :country, :string diff --git a/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs b/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs index 9fd222ea542d..9dfa4fe5a997 100644 --- a/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs +++ b/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedDevices do use Ecto.Migration def change do - create_if_not_exists table(:imported_devices, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_devices, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :device, :string diff --git a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs index 5b4925796984..4922c2398bc3 100644 --- a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs +++ b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedBrowsers do use Ecto.Migration def change do - create_if_not_exists table(:imported_browsers, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_browsers, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :browser, :string diff --git a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs index 0eb5cf193888..b0f5d25acb1e 100644 --- a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs +++ b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedOperatingSystems do use Ecto.Migration def change do - create_if_not_exists table(:imported_operating_systems, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_operating_systems, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :operating_system, :string diff --git a/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs b/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs index e6af6b6c27da..e5db4b1acbef 100644 --- a/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs +++ b/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmSources do use Ecto.Migration def change do - create_if_not_exists table(:imported_utm_sources, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_utm_sources, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :utm_source, :string diff --git a/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs b/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs index 56083b88943b..6352c87eaa7a 100644 --- a/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs +++ b/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmTerms do use Ecto.Migration def change do - create_if_not_exists table(:imported_utm_terms, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_utm_terms, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :utm_term, :string diff --git a/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs b/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs index 07d2d30e660d..c6c5548dc890 100644 --- a/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs +++ b/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs @@ -2,7 +2,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmContents do use Ecto.Migration def change do - create_if_not_exists table(:imported_utm_contents, engine: "MergeTree() ORDER BY (timestamp) SETTINGS index_granularity = 1") do + create_if_not_exists table(:imported_utm_contents, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do add :site_id, :UInt64 add :timestamp, :date add :utm_content, :string From 0e9c0456086c9d1c6da5cee664f5cfae88014b1d Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 8 Feb 2022 23:04:33 +0000 Subject: [PATCH 134/148] More strings -> atoms Also changes a conditional to be a bit nicer --- lib/plausible/stats/aggregate.ex | 2 +- lib/plausible/stats/breakdown.ex | 27 +- lib/plausible/stats/compare.ex | 8 +- lib/plausible/stats/timeseries.ex | 15 +- .../api/external_stats_controller.ex | 18 +- .../controllers/api/stats_controller.ex | 258 +++++++++--------- .../controllers/stats_controller.ex | 2 +- lib/workers/send_email_report.ex | 6 +- 8 files changed, 169 insertions(+), 167 deletions(-) diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex index f833a5003cf9..dba5023d0cb0 100644 --- a/lib/plausible/stats/aggregate.ex +++ b/lib/plausible/stats/aggregate.ex @@ -23,7 +23,7 @@ defmodule Plausible.Stats.Aggregate do |> Map.merge(Task.await(event_task, 10_000)) |> Map.merge(Task.await(time_on_page_task, 10_000)) |> Enum.map(fn {metric, value} -> - {metric, %{"value" => round(value || 0)}} + {metric, %{value: round(value || 0)}} end) |> Enum.into(%{}) end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 35cef5969d9f..49f33ea1d56c 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -19,7 +19,7 @@ defmodule Plausible.Stats.Breakdown do event_results = if Enum.any?(event_goals) do breakdown(site, event_query, "event:name", metrics, pagination) - |> transform_keys(%{"name" => "goal"}) + |> transform_keys(%{name: :goal}) else [] end @@ -44,13 +44,13 @@ defmodule Plausible.Stats.Breakdown do ), group_by: fragment("index"), select: %{ - "index" => fragment("arrayJoin(indices) as index"), - "goal" => fragment("concat('Visit ', array(?)[index])", ^page_exprs) + index: fragment("arrayJoin(indices) as index"), + goal: fragment("concat('Visit ', array(?)[index])", ^page_exprs) } ) |> select_event_metrics(metrics) |> ClickhouseRepo.all() - |> Enum.map(fn row -> Map.delete(row, "index") end) + |> Enum.map(fn row -> Map.delete(row, :index) end) else [] end @@ -74,11 +74,15 @@ defmodule Plausible.Stats.Breakdown do ) |> select_event_metrics(metrics) |> ClickhouseRepo.all() + |> transform_keys(%{custom_prop => String.to_atom(custom_prop)}) else [] end - results = breakdown_events(site, query, "event:props:" <> custom_prop, metrics, pagination) + results = + breakdown_events(site, query, "event:props:" <> custom_prop, metrics, pagination) + |> transform_keys(%{custom_prop => String.to_atom(custom_prop)}) + zipped = zip_results(none_result, results, custom_prop, metrics) if Enum.find_index(zipped, fn value -> value[custom_prop] == "(none)" end) == limit do @@ -171,6 +175,7 @@ defmodule Plausible.Stats.Breakdown do |> String.trim_leading("event:") |> String.trim_leading("visit:") |> String.trim_leading("props:") + |> String.to_atom() else property end @@ -345,7 +350,7 @@ defmodule Plausible.Stats.Breakdown do from( e in q, group_by: e.name, - select_merge: %{"name" => e.name} + select_merge: %{name: e.name} ) end @@ -370,8 +375,8 @@ defmodule Plausible.Stats.Breakdown do e in q, group_by: fragment("index"), select_merge: %{ - "index" => fragment("arrayJoin(indices) as index"), - "page_match" => fragment("array(?)[index]", ^match_exprs) + index: fragment("arrayJoin(indices) as index"), + page_match: fragment("array(?)[index]", ^match_exprs) } ) end @@ -432,7 +437,7 @@ defmodule Plausible.Stats.Breakdown do s in q, group_by: s.referrer, select_merge: %{ - "referrer" => fragment("if(empty(?), ?, ?)", s.referrer, @no_ref, s.referrer) + referrer: fragment("if(empty(?), ?, ?)", s.referrer, @no_ref, s.referrer) } ) end @@ -507,7 +512,7 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.operating_system_version, - select_merge: %{"os_version" => s.operating_system_version} + select_merge: %{os_version: s.operating_system_version} ) end @@ -523,7 +528,7 @@ defmodule Plausible.Stats.Breakdown do from( s in q, group_by: s.browser_version, - select_merge: %{"browser_version" => s.browser_version} + select_merge: %{browser_version: s.browser_version} ) end diff --git a/lib/plausible/stats/compare.ex b/lib/plausible/stats/compare.ex index 6c98638c61cb..99c6fc0bd976 100644 --- a/lib/plausible/stats/compare.ex +++ b/lib/plausible/stats/compare.ex @@ -1,14 +1,14 @@ defmodule Plausible.Stats.Compare do def calculate_change(:bounce_rate, old_stats, new_stats) do - old_count = old_stats[:bounce_rate]["value"] - new_count = new_stats[:bounce_rate]["value"] + old_count = old_stats[:bounce_rate][:value] + new_count = new_stats[:bounce_rate][:value] if old_count > 0, do: new_count - old_count end def calculate_change(metric, old_stats, new_stats) do - old_count = old_stats[metric]["value"] - new_count = new_stats[metric]["value"] + old_count = old_stats[metric][:value] + new_count = new_stats[metric][:value] percent_change(old_count, new_count) end diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 44d5bdf36eee..90fbd8a3ebe5 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -23,8 +23,8 @@ defmodule Plausible.Stats.Timeseries do Enum.map(steps, fn step -> empty_row(step, metrics) - |> Map.merge(Enum.find(event_result, fn row -> row["date"] == step end) || %{}) - |> Map.merge(Enum.find(session_result, fn row -> row["date"] == step end) || %{}) + |> Map.merge(Enum.find(event_result, fn row -> row[:date] == step end) || %{}) + |> Map.merge(Enum.find(session_result, fn row -> row[:date] == step end) || %{}) end) end @@ -82,8 +82,7 @@ defmodule Plausible.Stats.Timeseries do from( e in q, select_merge: %{ - "date" => - fragment("toStartOfMonth(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone) + date: fragment("toStartOfMonth(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone) } ) end @@ -92,7 +91,7 @@ defmodule Plausible.Stats.Timeseries do from( e in q, select_merge: %{ - "date" => fragment("toDate(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone) + date: fragment("toDate(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone) } ) end @@ -101,7 +100,7 @@ defmodule Plausible.Stats.Timeseries do from( e in q, select_merge: %{ - "date" => fragment("toStartOfHour(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone) + date: fragment("toStartOfHour(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone) } ) end @@ -110,13 +109,13 @@ defmodule Plausible.Stats.Timeseries do from( e in q, select_merge: %{ - "date" => fragment("dateDiff('minute', now(), ?) as date", e.timestamp) + date: fragment("dateDiff('minute', now(), ?) as date", e.timestamp) } ) end defp empty_row(date, metrics) do - Enum.reduce(metrics, %{"date" => date}, fn metric, row -> + Enum.reduce(metrics, %{date: date}, fn metric, row -> case metric do :pageviews -> Map.merge(row, %{pageviews: 0}) :visitors -> Map.merge(row, %{visitors: 0}) diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index e85129731103..3e52e836209c 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -31,13 +31,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController do 10_000 ) - Enum.map(curr_result, fn {metric, %{"value" => current_val}} -> - %{"value" => prev_val} = prev_result[metric] + Enum.map(curr_result, fn {metric, %{value: current_val}} -> + %{value: prev_val} = prev_result[metric] {metric, %{ - "value" => current_val, - "change" => percent_change(prev_val, current_val) + value: current_val, + change: percent_change(prev_val, current_val) }} end) |> Enum.into(%{}) @@ -45,7 +45,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do Plausible.Stats.aggregate(site, query, metrics) end - json(conn, %{"results" => Map.take(results, metrics)}) + json(conn, %{results: Map.take(results, metrics)}) else {:error, msg} -> conn @@ -72,13 +72,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController do prop_names = Props.props(site, query) Enum.map(results, fn row -> - Map.put(row, "props", prop_names[row["goal"]] || []) + Map.put(row, "props", prop_names[row[:goal]] || []) end) else results end - json(conn, %{"results" => results}) + json(conn, %{results: results}) else {:error, msg} -> conn @@ -147,8 +147,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController do query <- Query.from(site.timezone, params), {:ok, metrics} <- parse_metrics(params, nil, query) do graph = Plausible.Stats.timeseries(site, query, metrics) - metrics = metrics ++ ["date"] - json(conn, %{"results" => Enum.map(graph, &Map.take(&1, metrics))}) + metrics = metrics ++ [:date] + json(conn, %{results: Enum.map(graph, &Map.take(&1, metrics))}) else {:error, msg} -> conn diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 802a09d789c8..2bdc93a7d781 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -21,7 +21,7 @@ defmodule PlausibleWeb.Api.StatsController do timeseries_result = Task.await(timeseries) plot = Enum.map(timeseries_result, fn row -> row[:visitors] end) - labels = Enum.map(timeseries_result, fn row -> row["date"] end) + labels = Enum.map(timeseries_result, fn row -> row[:date] end) present_index = present_index_for(site, query, labels) {plot, with_imported, source} = @@ -33,7 +33,7 @@ defmodule PlausibleWeb.Api.StatsController do {plot, true, site.imported_source} else - if Map.get(params, "filters", "{}") != "{}" do + if Enum.any?(query.filters) do # Hiding imported data due to filtering. # Setting source to "" hides imported indicator from main graph. {plot, false, ""} @@ -88,8 +88,8 @@ defmodule PlausibleWeb.Api.StatsController do query_30m = %Query{query | period: "30m"} %{ - visitors: %{"value" => visitors}, - pageviews: %{"value" => pageviews} + visitors: %{value: visitors}, + pageviews: %{value: pageviews} } = Stats.aggregate(site, query_30m, [:visitors, :pageviews]) stats = [ @@ -116,21 +116,21 @@ defmodule PlausibleWeb.Api.StatsController do prev_total_query = Query.shift_back(total_q, site) %{ - visitors: %{"value" => unique_visitors} + visitors: %{value: unique_visitors} } = Stats.aggregate(site, total_q, [:visitors]) %{ - visitors: %{"value" => prev_unique_visitors} + visitors: %{value: prev_unique_visitors} } = Stats.aggregate(site, prev_total_query, [:visitors]) %{ - visitors: %{"value" => converted_visitors}, - events: %{"value" => completions} + visitors: %{value: converted_visitors}, + events: %{value: completions} } = Stats.aggregate(site, query, [:visitors, :events]) %{ - visitors: %{"value" => prev_converted_visitors}, - events: %{"value" => prev_completions} + visitors: %{value: prev_converted_visitors}, + events: %{value: prev_completions} } = Stats.aggregate(site, prev_query, [:visitors, :events]) conversion_rate = calculate_cr(unique_visitors, converted_visitors) @@ -185,15 +185,15 @@ defmodule PlausibleWeb.Api.StatsController do ] |> Enum.filter(& &1) - {stats, current_results[:sample_percent]["value"]} + {stats, current_results[:sample_percent][:value]} end defp top_stats_entry(current_results, prev_results, name, key) do if current_results[key] do %{ name: name, - value: current_results[key]["value"], - change: calculate_change(key, prev_results[key]["value"], current_results[key]["value"]) + value: current_results[key][:value], + change: calculate_change(key, prev_results[key][:value], current_results[key][:value]) } end end @@ -235,15 +235,15 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:source", metrics, pagination) |> maybe_add_cr(site, query, pagination, :source, "visit:source") - |> transform_keys(%{source: "name"}) + |> transform_keys(%{source: :name}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do res - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) + res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration]) end else json(conn, res) @@ -264,16 +264,16 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:utm_medium", metrics, pagination) - |> maybe_add_cr(site, query, pagination, "utm_medium", "visit:utm_medium") - |> transform_keys(%{utm_medium: "name"}) + |> maybe_add_cr(site, query, pagination, :utm_medium, "visit:utm_medium") + |> transform_keys(%{utm_medium: :name}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do res - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) + res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration]) end else json(conn, res) @@ -294,16 +294,16 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:utm_campaign", metrics, pagination) - |> maybe_add_cr(site, query, pagination, "utm_campaign", "visit:utm_campaign") - |> transform_keys(%{utm_campaign: "name"}) + |> maybe_add_cr(site, query, pagination, :utm_campaign, "visit:utm_campaign") + |> transform_keys(%{utm_campaign: :name}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do res - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) + res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration]) end else json(conn, res) @@ -324,15 +324,15 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:utm_content", metrics, pagination) |> maybe_add_cr(site, query, pagination, :utm_content, "visit:utm_content") - |> transform_keys(%{utm_content: "name"}) + |> transform_keys(%{utm_content: :name}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do res - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) + res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration]) end else json(conn, res) @@ -353,15 +353,15 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:utm_term", metrics, pagination) |> maybe_add_cr(site, query, pagination, :utm_term, "visit:utm_term") - |> transform_keys(%{utm_term: "name"}) + |> transform_keys(%{utm_term: :name}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do res - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) + res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration]) end else json(conn, res) @@ -382,16 +382,16 @@ defmodule PlausibleWeb.Api.StatsController do res = Stats.breakdown(site, query, "visit:utm_source", metrics, pagination) - |> maybe_add_cr(site, query, pagination, "utm_source", "visit:utm_source") - |> transform_keys(%{utm_source: "name"}) + |> maybe_add_cr(site, query, pagination, :utm_source, "visit:utm_source") + |> transform_keys(%{utm_source: :name}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do res - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - res |> to_csv(["name", :visitors, :bounce_rate, :visit_duration]) + res |> to_csv([:name, :visitors, :bounce_rate, :visit_duration]) end else json(conn, res) @@ -411,7 +411,7 @@ defmodule PlausibleWeb.Api.StatsController do google_api().fetch_stats(site, query, params["limit"] || 9) end - %{:visitors => %{"value" => total_visitors}} = Stats.aggregate(site, query, [:visitors]) + %{:visitors => %{value: total_visitors}} = Stats.aggregate(site, query, [:visitors]) case search_terms do nil -> @@ -443,11 +443,11 @@ defmodule PlausibleWeb.Api.StatsController do referrers = Stats.breakdown(site, query, "visit:referrer", metrics, pagination) - |> maybe_add_cr(site, query, pagination, "referrer", "visit:referrer") - |> transform_keys(%{"referrer" => "name"}) + |> maybe_add_cr(site, query, pagination, :referrer, "visit:referrer") + |> transform_keys(%{referrer: :name}) |> Enum.map(&Map.drop(&1, [:visits])) - %{:visitors => %{"value" => total_visitors}} = Stats.aggregate(site, query, [:visitors]) + %{:visitors => %{value: total_visitors}} = Stats.aggregate(site, query, [:visitors]) json(conn, %{referrers: referrers, total_visitors: total_visitors}) end @@ -465,15 +465,15 @@ defmodule PlausibleWeb.Api.StatsController do pages = Stats.breakdown(site, query, "event:page", metrics, pagination) |> maybe_add_cr(site, query, pagination, :page, "event:page") - |> transform_keys(%{page: "name"}) + |> transform_keys(%{page: :name}) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do pages - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - pages |> to_csv(["name", :visitors, :bounce_rate, :time_on_page]) + pages |> to_csv([:name, :visitors, :bounce_rate, :time_on_page]) end else json(conn, pages) @@ -490,18 +490,18 @@ defmodule PlausibleWeb.Api.StatsController do Stats.breakdown(site, query, "visit:entry_page", metrics, pagination) |> maybe_add_cr(site, query, pagination, :entry_page, "visit:entry_page") |> transform_keys(%{ - :entry_page => "name", - :visitors => "unique_entrances", - :visits => "total_entrances" + entry_page: :name, + visitors: :unique_entrances, + visits: :total_entrances }) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do entry_pages - |> transform_keys(%{"unique_entrances" => "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{unique_entrances: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - entry_pages |> to_csv(["name", "unique_entrances", "total_entrances", :visit_duration]) + entry_pages |> to_csv([:name, :unique_entrances, :total_entrances, :visit_duration]) end else json(conn, entry_pages) @@ -518,12 +518,12 @@ defmodule PlausibleWeb.Api.StatsController do Stats.breakdown(site, query, "visit:exit_page", metrics, {limit, page}) |> maybe_add_cr(site, query, {limit, page}, :exit_page, "visit:exit_page") |> transform_keys(%{ - :exit_page => "name", - :visitors => "unique_exits", - :visits => "total_exits" + exit_page: :name, + visitors: :unique_exits, + visits: :total_exits }) - pages = Enum.map(exit_pages, & &1["name"]) + pages = Enum.map(exit_pages, & &1[:name]) total_visits_query = Query.put_filter(query, "event:page", {:member, pages}) @@ -538,24 +538,24 @@ defmodule PlausibleWeb.Api.StatsController do exit_pages = Enum.map(exit_pages, fn exit_page -> exit_rate = - case Enum.find(total_pageviews, &(&1[:page] == exit_page["name"])) do + case Enum.find(total_pageviews, &(&1[:page] == exit_page[:name])) do %{pageviews: pageviews} -> - Float.floor(exit_page["total_exits"] / pageviews * 100) + Float.floor(exit_page[:total_exits] / pageviews * 100) nil -> nil end - Map.put(exit_page, "exit_rate", exit_rate) + Map.put(exit_page, :exit_rate, exit_rate) end) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do exit_pages - |> transform_keys(%{"unique_exits" => "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{unique_exits: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - exit_pages |> to_csv(["name", "unique_exits", "total_exits", "exit_rate"]) + exit_pages |> to_csv([:name, :unique_exits, :total_exits, :exit_rate]) end else json(conn, exit_pages) @@ -575,42 +575,42 @@ defmodule PlausibleWeb.Api.StatsController do countries = Stats.breakdown(site, query, "visit:country", [:visitors], pagination) |> maybe_add_cr(site, query, {300, 1}, :country, "visit:country") - |> transform_keys(%{country: "code"}) + |> transform_keys(%{country: :code}) |> maybe_add_percentages(query) if params["csv"] do countries = countries |> Enum.map(fn country -> - country_info = get_country(country["code"]) - Map.put(country, "name", country_info.name) + country_info = get_country(country[:code]) + Map.put(country, :name, country_info.name) end) if Map.has_key?(query.filters, "event:goal") do countries - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - countries |> to_csv(["name", :visitors]) + countries |> to_csv([:name, :visitors]) end else countries = Enum.map(countries, fn row -> - country = get_country(row["code"]) + country = get_country(row[:code]) if country do Map.merge(row, %{ - "name" => country.name, - "flag" => country.flag, - "alpha_3" => country.alpha_3, - "code" => country.alpha_2 + name: country.name, + flag: country.flag, + alpha_3: country.alpha_3, + code: country.alpha_2 }) else Map.merge(row, %{ - "name" => row["code"], - "flag" => "", - "alpha_3" => "", - "code" => "" + name: row[:code], + flag: "", + alpha_3: "", + code: "" }) end end) @@ -631,26 +631,26 @@ defmodule PlausibleWeb.Api.StatsController do regions = Stats.breakdown(site, query, "visit:region", [:visitors], pagination) - |> transform_keys(%{region: "code"}) + |> transform_keys(%{region: :code}) |> Enum.map(fn region -> - region_entry = Location.get_subdivision(region["code"]) + region_entry = Location.get_subdivision(region[:code]) if region_entry do country_entry = get_country(region_entry.country_code) - Map.merge(region, %{"name" => region_entry.name, "country_flag" => country_entry.flag}) + Map.merge(region, %{name: region_entry.name, country_flag: country_entry.flag}) else - Sentry.capture_message("Could not find region info", extra: %{code: region["code"]}) - Map.merge(region, %{"name" => region["code"]}) + Sentry.capture_message("Could not find region info", extra: %{code: region[:code]}) + Map.merge(region, %{name: region[:code]}) end end) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do regions - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - regions |> to_csv(["name", :visitors]) + regions |> to_csv([:name, :visitors]) end else json(conn, regions) @@ -669,31 +669,31 @@ defmodule PlausibleWeb.Api.StatsController do cities = Stats.breakdown(site, query, "visit:city", [:visitors], pagination) - |> transform_keys(%{city: "code"}) + |> transform_keys(%{city: :code}) |> Enum.map(fn city -> - city_info = Location.get_city(city["code"]) + city_info = Location.get_city(city[:code]) if city_info do country_info = get_country(city_info.country_code) Map.merge(city, %{ - "name" => city_info.name, - "country_flag" => country_info.flag + name: city_info.name, + country_flag: country_info.flag }) else - Sentry.capture_message("Could not find city info", extra: %{code: city["code"]}) + Sentry.capture_message("Could not find city info", extra: %{code: city[:code]}) - Map.merge(city, %{"name" => "N/A"}) + Map.merge(city, %{name: "N/A"}) end end) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do cities - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - cities |> to_csv(["name", :visitors]) + cities |> to_csv([:name, :visitors]) end else json(conn, cities) @@ -708,16 +708,16 @@ defmodule PlausibleWeb.Api.StatsController do browsers = Stats.breakdown(site, query, "visit:browser", [:visitors], pagination) |> maybe_add_cr(site, query, pagination, :browser, "visit:browser") - |> transform_keys(%{browser: "name"}) + |> transform_keys(%{browser: :name}) |> maybe_add_percentages(query) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do browsers - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - browsers |> to_csv(["name", :visitors]) + browsers |> to_csv([:name, :visitors]) end else json(conn, browsers) @@ -731,8 +731,8 @@ defmodule PlausibleWeb.Api.StatsController do versions = Stats.breakdown(site, query, "visit:browser_version", [:visitors], pagination) - |> maybe_add_cr(site, query, pagination, "browser_version", "visit:browser_version") - |> transform_keys(%{"browser_version" => "name"}) + |> maybe_add_cr(site, query, pagination, :browser_version, "visit:browser_version") + |> transform_keys(%{browser_version: :name}) |> maybe_add_percentages(query) json(conn, versions) @@ -746,16 +746,16 @@ defmodule PlausibleWeb.Api.StatsController do systems = Stats.breakdown(site, query, "visit:os", [:visitors], pagination) |> maybe_add_cr(site, query, pagination, :os, "visit:os") - |> transform_keys(%{os: "name"}) + |> transform_keys(%{os: :name}) |> maybe_add_percentages(query) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do systems - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - systems |> to_csv(["name", :visitors]) + systems |> to_csv([:name, :visitors]) end else json(conn, systems) @@ -769,8 +769,8 @@ defmodule PlausibleWeb.Api.StatsController do versions = Stats.breakdown(site, query, "visit:os_version", [:visitors], pagination) - |> maybe_add_cr(site, query, pagination, "os_version", "visit:os_version") - |> transform_keys(%{"os_version" => "name"}) + |> maybe_add_cr(site, query, pagination, :os_version, "visit:os_version") + |> transform_keys(%{os_version: :name}) |> maybe_add_percentages(query) json(conn, versions) @@ -784,16 +784,16 @@ defmodule PlausibleWeb.Api.StatsController do sizes = Stats.breakdown(site, query, "visit:device", [:visitors], pagination) |> maybe_add_cr(site, query, pagination, :device, "visit:device") - |> transform_keys(%{device: "name"}) + |> transform_keys(%{device: :name}) |> maybe_add_percentages(query) if params["csv"] do if Map.has_key?(query.filters, "event:goal") do sizes - |> transform_keys(%{visitors: "conversions"}) - |> to_csv(["name", "conversions", :conversion_rate]) + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) else - sizes |> to_csv(["name", :visitors]) + sizes |> to_csv([:name, :visitors]) end else json(conn, sizes) @@ -821,7 +821,7 @@ defmodule PlausibleWeb.Api.StatsController do total_q = Query.remove_goal(query) - %{visitors: %{"value" => total_visitors}} = Stats.aggregate(site, total_q, [:visitors]) + %{visitors: %{value: total_visitors}} = Stats.aggregate(site, total_q, [:visitors]) prop_names = if query.filters["event:goal"] do @@ -833,18 +833,18 @@ defmodule PlausibleWeb.Api.StatsController do conversions = Stats.breakdown(site, query, "event:goal", [:visitors, :events], {100, 1}) |> transform_keys(%{ - "goal" => "name", - :visitors => "unique_conversions", - :events => "total_conversions" + goal: :name, + visitors: :unique_conversions, + events: :total_conversions }) |> Enum.map(fn goal -> goal - |> Map.put(:prop_names, prop_names[goal["name"]]) - |> Map.put("conversion_rate", calculate_cr(total_visitors, goal["unique_conversions"])) + |> Map.put(:prop_names, prop_names[goal[:name]]) + |> Map.put(:conversion_rate, calculate_cr(total_visitors, goal[:unique_conversions])) end) if params["csv"] do - conversions |> to_csv(["name", "unique_conversions", "total_conversions"]) + conversions |> to_csv([:name, :unique_conversions, :total_conversions]) else json(conn, conversions) end @@ -857,22 +857,22 @@ defmodule PlausibleWeb.Api.StatsController do total_q = Query.remove_goal(query) - %{:visitors => %{"value" => unique_visitors}} = Stats.aggregate(site, total_q, [:visitors]) + %{:visitors => %{value: unique_visitors}} = Stats.aggregate(site, total_q, [:visitors]) prop_name = "event:props:" <> params["prop_name"] props = Stats.breakdown(site, query, prop_name, [:visitors, :events], pagination) |> transform_keys(%{ - params["prop_name"] => "name", - :events => "total_conversions", - :visitors => "unique_conversions" + String.to_atom(params["prop_name"]) => :name, + :events => :total_conversions, + :visitors => :unique_conversions }) |> Enum.map(fn prop -> Map.put( prop, - "conversion_rate", - calculate_cr(unique_visitors, prop["unique_conversions"]) + :conversion_rate, + calculate_cr(unique_visitors, prop[:unique_conversions]) ) end) @@ -887,8 +887,6 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = Query.from(site.timezone, params) |> Filters.add_prefix() - headers = ["prop", "name", "unique_conversions", "total_conversions"] - prop_names = if query.filters["event:goal"] do {_, _, goal} = query.filters["event:goal"] @@ -903,11 +901,11 @@ defmodule PlausibleWeb.Api.StatsController do prop_names |> Enum.map(fn prop -> prop_breakdown(conn, Map.put(params, "prop_name", prop)) - |> Enum.map(&Map.put(&1, "prop", prop)) + |> Enum.map(&Map.put(&1, :prop, prop)) end) |> Enum.concat() - to_csv(values, headers) + to_csv(values, [:prop, :name, :unique_conversions, :total_conversions]) end def current_visitors(conn, _) do @@ -950,7 +948,7 @@ defmodule PlausibleWeb.Api.StatsController do total = Enum.reduce(stat_list, 0, fn %{visitors: count}, total -> total + count end) Enum.map(stat_list, fn stat -> - Map.put(stat, "percentage", round(stat[:visitors] / total * 100)) + Map.put(stat, :percentage, round(stat[:visitors] / total * 100)) end) end end diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index b3f9e386fd45..ae11af66a25a 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -52,7 +52,7 @@ defmodule PlausibleWeb.StatsController do metrics = [:visitors, :pageviews, :bounce_rate, :visit_duration] graph = Plausible.Stats.timeseries(site, query, metrics) - headers = ["date" | metrics] + headers = [:date | metrics] visitors = Enum.map(graph, fn row -> Enum.map(headers, &row[&1]) end) diff --git a/lib/workers/send_email_report.ex b/lib/workers/send_email_report.ex index d83520995615..10be2e5f0dc0 100644 --- a/lib/workers/send_email_report.ex +++ b/lib/workers/send_email_report.ex @@ -65,11 +65,11 @@ defmodule Plausible.Workers.SendEmailReport do template = PlausibleWeb.Email.weekly_report(email, site, - unique_visitors: curr_period[:visitors]["value"], + unique_visitors: curr_period[:visitors][:value], change_visitors: change_visitors, - pageviews: curr_period[:pageviews]["value"], + pageviews: curr_period[:pageviews][:value], change_pageviews: change_pageviews, - bounce_rate: curr_period[:bounce_rate]["value"], + bounce_rate: curr_period[:bounce_rate][:value], change_bounce_rate: change_bounce_rate, sources: sources, unsubscribe_link: unsubscribe_link, From fdf4d001f5dbc492c8a3bae167a111a6a812ae83 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 8 Feb 2022 23:58:30 +0000 Subject: [PATCH 135/148] Remove parallelisation of data import --- lib/plausible/google/api.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index a77c215abe55..3f30a83f1f13 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -269,11 +269,8 @@ defmodule Plausible.Google.Api do maybe_error = results |> Enum.map(fn {dataset, data} -> - Task.async(fn -> - Imported.from_google_analytics(data, site.id, dataset) - end) + Imported.from_google_analytics(data, site.id, dataset) end) - |> Enum.map(&Task.await(&1, 120_000)) |> Keyword.get(:error) case maybe_error do From 0cb66d686e97d62975fc7f974a7be20a7eab6c33 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 10 Feb 2022 20:03:46 +0000 Subject: [PATCH 136/148] Split sources and UTM sources from fetched GA data GA has only a "source" dimension and no "UTM source" dimension. Instead it returns these combined. The logic herein to tease these apart is: 1. "(direct)" -> it's a direct source 2. if the source is a domain -> it's a source 3. "google" -> it's from adwords; let's make this a UTM source "adwords" 4. else -> just a UTM source --- lib/plausible/imported/site.ex | 39 ++++++++++++++------ test/plausible/imported/imported_test.exs | 45 ++++++++++++++++++++++- 2 files changed, 72 insertions(+), 12 deletions(-) diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 65e328f0ffb2..c402c735e9f8 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -46,6 +46,9 @@ defmodule Plausible.Imported do }) end + # Credit: https://github.com/kvesteri/validators + @domain ~r/^(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|([a-zA-Z0-9][-_.a-zA-Z0-9]{0,61}[a-zA-Z0-9]))\.([a-zA-Z]{2,13}|[a-zA-Z0-9-]{2,30}.[a-zA-Z]{2,3})$/ + defp new_from_google_analytics(site_id, "sources", %{ "dimensions" => [timestamp, source], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] @@ -57,19 +60,33 @@ defmodule Plausible.Imported do source = if source == "(direct)", do: nil, else: source - Imported.Sources.new(%{ - site_id: site_id, - timestamp: format_timestamp(timestamp), - source: Imported.Sources.parse(source), - visitors: visitors, - visits: visits, - bounces: bounces, - visit_duration: visit_duration - }) + cond do + is_nil(source) || String.match?(source, @domain) -> + Imported.Sources.new(%{ + site_id: site_id, + timestamp: format_timestamp(timestamp), + source: Imported.Sources.parse(source), + visitors: visitors, + visits: visits, + bounces: bounces, + visit_duration: visit_duration + }) + + true -> + utm_source = if source == "google", do: "adwords", else: source + + Imported.UtmSources.new(%{ + site_id: site_id, + timestamp: format_timestamp(timestamp), + utm_source: utm_source, + visitors: visitors, + visits: visits, + bounces: bounces, + visit_duration: visit_duration + }) + end end - # TODO: utm_sources. Google reports sources and utm_sources unified. - defp new_from_google_analytics(site_id, "utm_mediums", %{ "dimensions" => [timestamp, medium], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 180f006c3609..1d435d527612 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -44,7 +44,7 @@ defmodule Plausible.ImportedTest do assert Enum.sum(plot) == 4 end - test "Sources data imported from Google Analytics", %{conn: conn, site: site} do + test "Sources and UTM sources are separated", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, referrer_source: "Google", @@ -73,6 +73,22 @@ defmodule Plausible.ImportedTest do %{ "dimensions" => ["20210131", "google.com"], "metrics" => [%{"values" => ["1", "1", "1", "60"]}] + }, + %{ + "dimensions" => ["20210101", "google"], + "metrics" => [%{"values" => ["1", "1", "1", "60"]}] + }, + %{ + "dimensions" => ["20210101", "Twitter"], + "metrics" => [%{"values" => ["1", "1", "1", "60"]}] + }, + %{ + "dimensions" => ["20210131", "A Nice Newsletter"], + "metrics" => [%{"values" => ["1", "1", "1", "60"]}] + }, + %{ + "dimensions" => ["20210101", "(direct)"], + "metrics" => [%{"values" => ["1", "1", "1", "60"]}] } ], site.id, @@ -89,6 +105,33 @@ defmodule Plausible.ImportedTest do %{"name" => "Google", "visitors" => 3}, %{"name" => "DuckDuckGo", "visitors" => 2} ] + + conn = + get( + conn, + "/api/stats/#{site.domain}/utm_sources?period=month&date=2021-01-01&with_imported=true" + ) + + assert json_response(conn, 200) == [ + %{ + "name" => "A Nice Newsletter", + "visitors" => 1, + "bounce_rate" => 100.0, + "visit_duration" => 60.0 + }, + %{ + "name" => "Twitter", + "visitors" => 1, + "bounce_rate" => 100.0, + "visit_duration" => 60.0 + }, + %{ + "bounce_rate" => 100.0, + "name" => "adwords", + "visit_duration" => 60.0, + "visitors" => 1 + } + ] end test "UTM mediums data imported from Google Analytics", %{conn: conn, site: site} do From 175025b49701ed2d8b21a3cd491272eeb8647d09 Mon Sep 17 00:00:00 2001 From: mcol Date: Thu, 10 Feb 2022 22:00:14 +0000 Subject: [PATCH 137/148] Keep prop names in queries as strings --- lib/plausible/stats/breakdown.ex | 8 ++------ lib/plausible_web/controllers/api/stats_controller.ex | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 49f33ea1d56c..bac7a99c0037 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -55,7 +55,7 @@ defmodule Plausible.Stats.Breakdown do [] end - zip_results(event_results, page_results, "event:goal", metrics) + zip_results(event_results, page_results, :goal, metrics) end def breakdown(site, query, "event:props:" <> custom_prop, metrics, pagination) do @@ -74,14 +74,11 @@ defmodule Plausible.Stats.Breakdown do ) |> select_event_metrics(metrics) |> ClickhouseRepo.all() - |> transform_keys(%{custom_prop => String.to_atom(custom_prop)}) else [] end - results = - breakdown_events(site, query, "event:props:" <> custom_prop, metrics, pagination) - |> transform_keys(%{custom_prop => String.to_atom(custom_prop)}) + results = breakdown_events(site, query, "event:props:" <> custom_prop, metrics, pagination) zipped = zip_results(none_result, results, custom_prop, metrics) @@ -175,7 +172,6 @@ defmodule Plausible.Stats.Breakdown do |> String.trim_leading("event:") |> String.trim_leading("visit:") |> String.trim_leading("props:") - |> String.to_atom() else property end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 2bdc93a7d781..97b52afe5ddf 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -864,7 +864,7 @@ defmodule PlausibleWeb.Api.StatsController do props = Stats.breakdown(site, query, prop_name, [:visitors, :events], pagination) |> transform_keys(%{ - String.to_atom(params["prop_name"]) => :name, + params["prop_name"] => :name, :events => :total_conversions, :visitors => :unique_conversions }) From c574abe8e25f60b51a4516b1d0b169e84ac97bb5 Mon Sep 17 00:00:00 2001 From: mcol Date: Tue, 15 Feb 2022 18:14:19 +0000 Subject: [PATCH 138/148] fix typo --- .../templates/email/google_analytics_import.html.eex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plausible_web/templates/email/google_analytics_import.html.eex b/lib/plausible_web/templates/email/google_analytics_import.html.eex index 65f4e0949c4d..d56f3f0cf407 100644 --- a/lib/plausible_web/templates/email/google_analytics_import.html.eex +++ b/lib/plausible_web/templates/email/google_analytics_import.html.eex @@ -1,7 +1,7 @@ Hey <%= user_salutation(@user) %>,

    <%= if @success do %> - Your Google Analytics import has complete. + Your Google Analytics import has completed.

    View dashboard: @link <% else %> From 9213b29eeccf68cf7733ca8e41f8b77fdc17c0bb Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Wed, 2 Mar 2022 14:06:44 -0600 Subject: [PATCH 139/148] Fix import --- lib/plausible/google/api.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 3f30a83f1f13..815b48ff2c9c 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -169,6 +169,13 @@ defmodule Plausible.Google.Api do Plausible.Stats.Clickhouse.pageviews_begin(site) |> NaiveDateTime.to_date() + end_date = + if end_date == ~D[1970-01-01] do + Timex.today() + else + end_date + end + request = %{ auth: auth, profile: profile, @@ -339,7 +346,7 @@ defmodule Plausible.Google.Api do next_page_token ) do {:ok, %{^dataset => remainder}} -> - {:ok, %{dataset => [data | remainder]}} + {:ok, %{dataset => data ++ remainder}} error -> error From 5073b39122035b12f6353cf0ca81c9e0084b8a11 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Thu, 3 Mar 2022 15:17:05 -0600 Subject: [PATCH 140/148] Insert data to clickhouse in batches --- lib/plausible/google/api.ex | 26 ++-- lib/plausible/imported/browsers.ex | 42 ------ lib/plausible/imported/devices.ex | 42 ------ lib/plausible/imported/entry_pages.ex | 43 ------ lib/plausible/imported/exit_pages.ex | 36 ----- lib/plausible/imported/locations.ex | 46 ------- lib/plausible/imported/operating_systems.ex | 42 ------ lib/plausible/imported/pages.ex | 36 ----- lib/plausible/imported/site.ex | 141 +++++++++----------- lib/plausible/imported/sources.ex | 61 --------- lib/plausible/imported/utm_campaigns.ex | 42 ------ lib/plausible/imported/utm_contents.ex | 42 ------ lib/plausible/imported/utm_mediums.ex | 42 ------ lib/plausible/imported/utm_sources.ex | 42 ------ lib/plausible/imported/utm_terms.ex | 42 ------ lib/plausible/imported/visitors.ex | 43 ------ test/plausible/imported/imported_test.exs | 56 ++++---- test/support/factory.ex | 42 ++++-- test/support/test_utils.ex | 3 +- 19 files changed, 136 insertions(+), 733 deletions(-) delete mode 100644 lib/plausible/imported/browsers.ex delete mode 100644 lib/plausible/imported/devices.ex delete mode 100644 lib/plausible/imported/entry_pages.ex delete mode 100644 lib/plausible/imported/exit_pages.ex delete mode 100644 lib/plausible/imported/locations.ex delete mode 100644 lib/plausible/imported/operating_systems.ex delete mode 100644 lib/plausible/imported/pages.ex delete mode 100644 lib/plausible/imported/sources.ex delete mode 100644 lib/plausible/imported/utm_campaigns.ex delete mode 100644 lib/plausible/imported/utm_contents.ex delete mode 100644 lib/plausible/imported/utm_mediums.ex delete mode 100644 lib/plausible/imported/utm_sources.ex delete mode 100644 lib/plausible/imported/utm_terms.ex delete mode 100644 lib/plausible/imported/visitors.ex diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 815b48ff2c9c..461e986fbae4 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -185,7 +185,7 @@ defmodule Plausible.Google.Api do # Each element is: {dataset, dimensions, metrics} request_data = [ { - "visitors", + "imported_visitors", ["ga:date"], [ "ga:users", @@ -196,62 +196,62 @@ defmodule Plausible.Google.Api do ] }, { - "sources", + "imported_sources", ["ga:date", "ga:fullReferrer"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { - "utm_mediums", + "imported_utm_mediums", ["ga:date", "ga:medium"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { - "utm_campaigns", + "imported_utm_campaigns", ["ga:date", "ga:campaign"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { - "utm_terms", + "imported_utm_terms", ["ga:date", "ga:keyword"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { - "utm_contents", + "imported_utm_contents", ["ga:date", "ga:adContent"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { - "pages", + "imported_pages", ["ga:date", "ga:pagePath"], ["ga:users", "ga:pageviews", "ga:timeOnPage"] }, { - "entry_pages", + "imported_entry_pages", ["ga:date", "ga:landingPagePath"], ["ga:users", "ga:entrances", "ga:sessionDuration", "ga:bounces"] }, { - "exit_pages", + "imported_exit_pages", ["ga:date", "ga:exitPagePath"], ["ga:users", "ga:exits"] }, { - "locations", + "imported_locations", ["ga:date", "ga:countryIsoCode", "ga:regionIsoCode"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { - "devices", + "imported_devices", ["ga:date", "ga:deviceCategory"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { - "browsers", + "imported_browsers", ["ga:date", "ga:browser"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { - "operating_systems", + "imported_operating_systems", ["ga:date", "ga:operatingSystem"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] } diff --git a/lib/plausible/imported/browsers.ex b/lib/plausible/imported/browsers.ex deleted file mode 100644 index bd1d3650b170..000000000000 --- a/lib/plausible/imported/browsers.ex +++ /dev/null @@ -1,42 +0,0 @@ -defmodule Plausible.Imported.Browsers do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_browsers" do - field :site_id, :integer - field :timestamp, :date - field :browser, :string - field :visitors, :integer - field :visits, :integer - field :bounces, :integer - # Sum total - field :visit_duration, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :browser, - :visitors, - :visits, - :bounces, - :visit_duration - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :visitors, - :visits, - :bounces, - :visit_duration - ]) - end -end diff --git a/lib/plausible/imported/devices.ex b/lib/plausible/imported/devices.ex deleted file mode 100644 index 2c3091317874..000000000000 --- a/lib/plausible/imported/devices.ex +++ /dev/null @@ -1,42 +0,0 @@ -defmodule Plausible.Imported.Devices do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_devices" do - field :site_id, :integer - field :timestamp, :date - field :device, :string - field :visitors, :integer - field :visits, :integer - field :bounces, :integer - # Sum total - field :visit_duration, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :device, - :visitors, - :visits, - :bounces, - :visit_duration - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :visitors, - :visits, - :bounces, - :visit_duration - ]) - end -end diff --git a/lib/plausible/imported/entry_pages.ex b/lib/plausible/imported/entry_pages.ex deleted file mode 100644 index c94d1d8b5d13..000000000000 --- a/lib/plausible/imported/entry_pages.ex +++ /dev/null @@ -1,43 +0,0 @@ -defmodule Plausible.Imported.EntryPages do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_entry_pages" do - field :site_id, :integer - field :timestamp, :date - field :entry_page, :string - field :visitors, :integer - field :entrances, :integer - field :bounces, :integer - # Sum total - field :visit_duration, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :entry_page, - :visitors, - :entrances, - :bounces, - :visit_duration - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :entry_page, - :visitors, - :entrances, - :bounces, - :visit_duration - ]) - end -end diff --git a/lib/plausible/imported/exit_pages.ex b/lib/plausible/imported/exit_pages.ex deleted file mode 100644 index 5c2df2734fea..000000000000 --- a/lib/plausible/imported/exit_pages.ex +++ /dev/null @@ -1,36 +0,0 @@ -defmodule Plausible.Imported.ExitPages do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_exit_pages" do - field :site_id, :integer - field :timestamp, :date - field :exit_page, :string - field :visitors, :integer - field :exits, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :exit_page, - :visitors, - :exits - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :exit_page, - :visitors, - :exits - ]) - end -end diff --git a/lib/plausible/imported/locations.ex b/lib/plausible/imported/locations.ex deleted file mode 100644 index 6ffeda8d0dc5..000000000000 --- a/lib/plausible/imported/locations.ex +++ /dev/null @@ -1,46 +0,0 @@ -defmodule Plausible.Imported.Locations do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_locations" do - field :site_id, :integer - field :timestamp, :date - field :country, :string, default: "" - field :region, :string, default: "" - field :city, :integer, default: 0 - field :visitors, :integer - field :visits, :integer - field :bounces, :integer - # Sum total - field :visit_duration, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :country, - :region, - :city, - :visitors, - :visits, - :bounces, - :visit_duration - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :visitors, - :visits, - :bounces, - :visit_duration - ]) - end -end diff --git a/lib/plausible/imported/operating_systems.ex b/lib/plausible/imported/operating_systems.ex deleted file mode 100644 index ed4810b6a721..000000000000 --- a/lib/plausible/imported/operating_systems.ex +++ /dev/null @@ -1,42 +0,0 @@ -defmodule Plausible.Imported.OperatingSystems do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_operating_systems" do - field :site_id, :integer - field :timestamp, :date - field :operating_system, :string - field :visitors, :integer - field :visits, :integer - field :bounces, :integer - # Sum total - field :visit_duration, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :operating_system, - :visitors, - :visits, - :bounces, - :visit_duration - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :visitors, - :visits, - :bounces, - :visit_duration - ]) - end -end diff --git a/lib/plausible/imported/pages.ex b/lib/plausible/imported/pages.ex deleted file mode 100644 index 29a4183681e8..000000000000 --- a/lib/plausible/imported/pages.ex +++ /dev/null @@ -1,36 +0,0 @@ -defmodule Plausible.Imported.Pages do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_pages" do - field :site_id, :integer - field :timestamp, :date - field :page, :string - field :visitors, :integer - field :pageviews, :integer - field :time_on_page, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :page, - :visitors, - :pageviews, - :time_on_page - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :visitors - ]) - end -end diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index c402c735e9f8..db99a63657ed 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -1,5 +1,5 @@ defmodule Plausible.Imported do - alias Plausible.Imported + use Plausible.ClickhouseRepo use Timex def forget(site) do @@ -8,25 +8,19 @@ defmodule Plausible.Imported do def from_google_analytics(nil, _site_id, _metric, _timezone), do: {:ok, nil} - def from_google_analytics(data, site_id, metric) do - maybe_error = - data - |> Enum.map(fn row -> - new_from_google_analytics(site_id, metric, row) - |> Plausible.ClickhouseRepo.insert(on_conflict: :replace_all) + def from_google_analytics(data, site_id, table) do + data = + Enum.map(data, fn row -> + new_from_google_analytics(site_id, table, row) end) - |> Keyword.get(:error) - case maybe_error do - nil -> - {:ok, nil} - - error -> - {:error, error.errors} + case ClickhouseRepo.insert_all(table, data) do + {n_rows, _} when n_rows > 0 -> :ok + error -> error end end - defp new_from_google_analytics(site_id, "visitors", %{ + defp new_from_google_analytics(site_id, "imported_visitors", %{ "dimensions" => [timestamp], "metrics" => [%{"values" => values}] }) do @@ -35,7 +29,7 @@ defmodule Plausible.Imported do |> Enum.map(&Integer.parse/1) |> Enum.map(&elem(&1, 0)) - Imported.Visitors.new(%{ + %{ site_id: site_id, timestamp: format_timestamp(timestamp), visitors: visitors, @@ -43,13 +37,13 @@ defmodule Plausible.Imported do bounces: bounces, visits: visits, visit_duration: visit_duration - }) + } end # Credit: https://github.com/kvesteri/validators @domain ~r/^(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|([a-zA-Z0-9][-_.a-zA-Z0-9]{0,61}[a-zA-Z0-9]))\.([a-zA-Z]{2,13}|[a-zA-Z0-9-]{2,30}.[a-zA-Z]{2,3})$/ - defp new_from_google_analytics(site_id, "sources", %{ + defp new_from_google_analytics(site_id, "imported_sources", %{ "dimensions" => [timestamp, source], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -59,35 +53,20 @@ defmodule Plausible.Imported do {visit_duration, _} = Integer.parse(visit_duration) source = if source == "(direct)", do: nil, else: source + source = if source && String.match?(source, @domain), do: parse_referrer(source), else: source - cond do - is_nil(source) || String.match?(source, @domain) -> - Imported.Sources.new(%{ - site_id: site_id, - timestamp: format_timestamp(timestamp), - source: Imported.Sources.parse(source), - visitors: visitors, - visits: visits, - bounces: bounces, - visit_duration: visit_duration - }) - - true -> - utm_source = if source == "google", do: "adwords", else: source - - Imported.UtmSources.new(%{ - site_id: site_id, - timestamp: format_timestamp(timestamp), - utm_source: utm_source, - visitors: visitors, - visits: visits, - bounces: bounces, - visit_duration: visit_duration - }) - end + %{ + site_id: site_id, + timestamp: format_timestamp(timestamp), + source: parse_referrer(source), + visitors: visitors, + visits: visits, + bounces: bounces, + visit_duration: visit_duration + } end - defp new_from_google_analytics(site_id, "utm_mediums", %{ + defp new_from_google_analytics(site_id, "imported_utm_mediums", %{ "dimensions" => [timestamp, medium], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -98,7 +77,7 @@ defmodule Plausible.Imported do medium = if medium == "(none)", do: "", else: medium - Imported.UtmMediums.new(%{ + %{ site_id: site_id, timestamp: format_timestamp(timestamp), utm_medium: medium, @@ -106,10 +85,10 @@ defmodule Plausible.Imported do visits: visits, bounces: bounces, visit_duration: visit_duration - }) + } end - defp new_from_google_analytics(site_id, "utm_campaigns", %{ + defp new_from_google_analytics(site_id, "imported_utm_campaigns", %{ "dimensions" => [timestamp, campaign], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -120,7 +99,7 @@ defmodule Plausible.Imported do campaign = if campaign == "(not set)", do: "", else: campaign - Imported.UtmCampaigns.new(%{ + %{ site_id: site_id, timestamp: format_timestamp(timestamp), utm_campaign: campaign, @@ -128,10 +107,10 @@ defmodule Plausible.Imported do visits: visits, bounces: bounces, visit_duration: visit_duration - }) + } end - defp new_from_google_analytics(site_id, "utm_terms", %{ + defp new_from_google_analytics(site_id, "imported_utm_terms", %{ "dimensions" => [timestamp, term], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -142,7 +121,7 @@ defmodule Plausible.Imported do term = if term == "(not set)" or term == "(not provided)", do: "", else: term - Imported.UtmTerms.new(%{ + %{ site_id: site_id, timestamp: format_timestamp(timestamp), utm_term: term, @@ -150,10 +129,10 @@ defmodule Plausible.Imported do visits: visits, bounces: bounces, visit_duration: visit_duration - }) + } end - defp new_from_google_analytics(site_id, "utm_contents", %{ + defp new_from_google_analytics(site_id, "imported_utm_contents", %{ "dimensions" => [timestamp, content], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -164,7 +143,7 @@ defmodule Plausible.Imported do content = if content == "(not set)", do: "", else: content - Imported.UtmContents.new(%{ + %{ site_id: site_id, timestamp: format_timestamp(timestamp), utm_content: content, @@ -172,10 +151,10 @@ defmodule Plausible.Imported do visits: visits, bounces: bounces, visit_duration: visit_duration - }) + } end - defp new_from_google_analytics(site_id, "pages", %{ + defp new_from_google_analytics(site_id, "imported_pages", %{ "dimensions" => [timestamp, page], "metrics" => [%{"values" => [visitors, pageviews, time_on_page]}] }) do @@ -183,17 +162,17 @@ defmodule Plausible.Imported do {pageviews, ""} = Integer.parse(pageviews) {time_on_page, _} = Integer.parse(time_on_page) - Imported.Pages.new(%{ + %{ site_id: site_id, timestamp: format_timestamp(timestamp), page: page, visitors: visitors, pageviews: pageviews, time_on_page: time_on_page - }) + } end - defp new_from_google_analytics(site_id, "entry_pages", %{ + defp new_from_google_analytics(site_id, "imported_entry_pages", %{ "dimensions" => [timestamp, entry_page], "metrics" => [%{"values" => [visitors, entrances, visit_duration, bounces]}] }) do @@ -202,7 +181,7 @@ defmodule Plausible.Imported do {bounces, ""} = Integer.parse(bounces) {visit_duration, _} = Integer.parse(visit_duration) - Imported.EntryPages.new(%{ + %{ site_id: site_id, timestamp: format_timestamp(timestamp), entry_page: entry_page, @@ -210,26 +189,26 @@ defmodule Plausible.Imported do entrances: entrances, visit_duration: visit_duration, bounces: bounces - }) + } end - defp new_from_google_analytics(site_id, "exit_pages", %{ + defp new_from_google_analytics(site_id, "imported_exit_pages", %{ "dimensions" => [timestamp, exit_page], "metrics" => [%{"values" => [visitors, exits]}] }) do {visitors, ""} = Integer.parse(visitors) {exits, ""} = Integer.parse(exits) - Imported.ExitPages.new(%{ + %{ site_id: site_id, timestamp: format_timestamp(timestamp), exit_page: exit_page, visitors: visitors, exits: exits - }) + } end - defp new_from_google_analytics(site_id, "locations", %{ + defp new_from_google_analytics(site_id, "imported_locations", %{ "dimensions" => [timestamp, country, region], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -240,7 +219,7 @@ defmodule Plausible.Imported do {bounces, ""} = Integer.parse(bounces) {visit_duration, _} = Integer.parse(visit_duration) - Imported.Locations.new(%{ + %{ site_id: site_id, timestamp: format_timestamp(timestamp), country: country, @@ -250,10 +229,10 @@ defmodule Plausible.Imported do visits: visits, bounces: bounces, visit_duration: visit_duration - }) + } end - defp new_from_google_analytics(site_id, "devices", %{ + defp new_from_google_analytics(site_id, "imported_devices", %{ "dimensions" => [timestamp, device], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -262,7 +241,7 @@ defmodule Plausible.Imported do {bounces, ""} = Integer.parse(bounces) {visit_duration, _} = Integer.parse(visit_duration) - Imported.Devices.new(%{ + %{ site_id: site_id, timestamp: format_timestamp(timestamp), device: String.capitalize(device), @@ -270,7 +249,7 @@ defmodule Plausible.Imported do visits: visits, bounces: bounces, visit_duration: visit_duration - }) + } end @browser_google_to_plausible %{ @@ -283,7 +262,7 @@ defmodule Plausible.Imported do "(not set)" => "" } - defp new_from_google_analytics(site_id, "browsers", %{ + defp new_from_google_analytics(site_id, "imported_browsers", %{ "dimensions" => [timestamp, browser], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -292,7 +271,7 @@ defmodule Plausible.Imported do {bounces, ""} = Integer.parse(bounces) {visit_duration, _} = Integer.parse(visit_duration) - Imported.Browsers.new(%{ + %{ site_id: site_id, timestamp: format_timestamp(timestamp), browser: Map.get(@browser_google_to_plausible, browser, browser), @@ -300,7 +279,7 @@ defmodule Plausible.Imported do visits: visits, bounces: bounces, visit_duration: visit_duration - }) + } end @os_google_to_plausible %{ @@ -309,7 +288,7 @@ defmodule Plausible.Imported do "(not set)" => "" } - defp new_from_google_analytics(site_id, "operating_systems", %{ + defp new_from_google_analytics(site_id, "imported_operating_systems", %{ "dimensions" => [timestamp, operating_system], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do @@ -318,7 +297,7 @@ defmodule Plausible.Imported do {bounces, ""} = Integer.parse(bounces) {visit_duration, _} = Integer.parse(visit_duration) - Imported.OperatingSystems.new(%{ + %{ site_id: site_id, timestamp: format_timestamp(timestamp), operating_system: Map.get(@os_google_to_plausible, operating_system, operating_system), @@ -326,11 +305,21 @@ defmodule Plausible.Imported do visits: visits, bounces: bounces, visit_duration: visit_duration - }) + } end defp format_timestamp(timestamp) do Timex.parse!("#{timestamp}", "%Y%m%d", :strftime) |> NaiveDateTime.to_date() end + + def parse_referrer(nil), do: nil + def parse_referrer("google"), do: "Google" + def parse_referrer("bing"), do: "Bing" + def parse_referrer("duckduckgo"), do: "DuckDuckGo" + + def parse_referrer(ref) do + RefInspector.parse("https://" <> ref) + |> PlausibleWeb.RefInspector.parse() + end end diff --git a/lib/plausible/imported/sources.ex b/lib/plausible/imported/sources.ex deleted file mode 100644 index d0bf3dd388e7..000000000000 --- a/lib/plausible/imported/sources.ex +++ /dev/null @@ -1,61 +0,0 @@ -defmodule Plausible.Imported.Sources do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_sources" do - field :site_id, :integer - field :timestamp, :date - field :source, :string, default: "" - field :visitors, :integer - field :visits, :integer - field :bounces, :integer - # Sum total - field :visit_duration, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :source, - :visitors, - :visits, - :bounces, - :visit_duration - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :visitors, - :visits, - :bounces, - :visit_duration - ]) - end - - @search_engines %{ - "google" => "Google", - "bing" => "Bing", - "duckduckgo" => "DuckDuckGo" - } - - def parse(nil), do: nil - - def parse(ref) do - se = @search_engines[ref] - - if se do - se - else - RefInspector.parse("https://" <> ref) - |> PlausibleWeb.RefInspector.parse() - end - end -end diff --git a/lib/plausible/imported/utm_campaigns.ex b/lib/plausible/imported/utm_campaigns.ex deleted file mode 100644 index 720348a72d63..000000000000 --- a/lib/plausible/imported/utm_campaigns.ex +++ /dev/null @@ -1,42 +0,0 @@ -defmodule Plausible.Imported.UtmCampaigns do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_utm_campaigns" do - field :site_id, :integer - field :timestamp, :date - field :utm_campaign, :string, default: "" - field :visitors, :integer - field :visits, :integer - field :bounces, :integer - # Sum total - field :visit_duration, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :utm_campaign, - :visitors, - :visits, - :bounces, - :visit_duration - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :visitors, - :visits, - :bounces, - :visit_duration - ]) - end -end diff --git a/lib/plausible/imported/utm_contents.ex b/lib/plausible/imported/utm_contents.ex deleted file mode 100644 index 8033ffbc540b..000000000000 --- a/lib/plausible/imported/utm_contents.ex +++ /dev/null @@ -1,42 +0,0 @@ -defmodule Plausible.Imported.UtmContents do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_utm_contents" do - field :site_id, :integer - field :timestamp, :date - field :utm_content, :string, default: "" - field :visitors, :integer - field :visits, :integer - field :bounces, :integer - # Sum total - field :visit_duration, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :utm_content, - :visitors, - :visits, - :bounces, - :visit_duration - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :visitors, - :visits, - :bounces, - :visit_duration - ]) - end -end diff --git a/lib/plausible/imported/utm_mediums.ex b/lib/plausible/imported/utm_mediums.ex deleted file mode 100644 index f34c3ef02c63..000000000000 --- a/lib/plausible/imported/utm_mediums.ex +++ /dev/null @@ -1,42 +0,0 @@ -defmodule Plausible.Imported.UtmMediums do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_utm_mediums" do - field :site_id, :integer - field :timestamp, :date - field :utm_medium, :string, default: "" - field :visitors, :integer - field :visits, :integer - field :bounces, :integer - # Sum total - field :visit_duration, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :utm_medium, - :visitors, - :visits, - :bounces, - :visit_duration - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :visitors, - :visits, - :bounces, - :visit_duration - ]) - end -end diff --git a/lib/plausible/imported/utm_sources.ex b/lib/plausible/imported/utm_sources.ex deleted file mode 100644 index 554daa7ae190..000000000000 --- a/lib/plausible/imported/utm_sources.ex +++ /dev/null @@ -1,42 +0,0 @@ -defmodule Plausible.Imported.UtmSources do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_utm_sources" do - field :site_id, :integer - field :timestamp, :date - field :utm_source, :string, default: "" - field :visitors, :integer - field :visits, :integer - field :bounces, :integer - # Sum total - field :visit_duration, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :utm_source, - :visitors, - :visits, - :bounces, - :visit_duration - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :visitors, - :visits, - :bounces, - :visit_duration - ]) - end -end diff --git a/lib/plausible/imported/utm_terms.ex b/lib/plausible/imported/utm_terms.ex deleted file mode 100644 index bf3f934db87d..000000000000 --- a/lib/plausible/imported/utm_terms.ex +++ /dev/null @@ -1,42 +0,0 @@ -defmodule Plausible.Imported.UtmTerms do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_utm_terms" do - field :site_id, :integer - field :timestamp, :date - field :utm_term, :string, default: "" - field :visitors, :integer - field :visits, :integer - field :bounces, :integer - # Sum total - field :visit_duration, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :utm_term, - :visitors, - :visits, - :bounces, - :visit_duration - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :visitors, - :visits, - :bounces, - :visit_duration - ]) - end -end diff --git a/lib/plausible/imported/visitors.ex b/lib/plausible/imported/visitors.ex deleted file mode 100644 index ea312f5da90e..000000000000 --- a/lib/plausible/imported/visitors.ex +++ /dev/null @@ -1,43 +0,0 @@ -defmodule Plausible.Imported.Visitors do - use Ecto.Schema - use Plausible.ClickhouseRepo - import Ecto.Changeset - - @primary_key false - schema "imported_visitors" do - field :site_id, :integer - field :timestamp, :date - field :visitors, :integer - field :pageviews, :integer - field :bounces, :integer - field :visits, :integer - # Sum total - field :visit_duration, :integer - end - - def new(attrs) do - %__MODULE__{} - |> cast( - attrs, - [ - :site_id, - :timestamp, - :visitors, - :pageviews, - :bounces, - :visits, - :visit_duration - ], - empty_values: [nil, ""] - ) - |> validate_required([ - :site_id, - :timestamp, - :visitors, - :pageviews, - :bounces, - :visits, - :visit_duration - ]) - end -end diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 1d435d527612..e989c608de0f 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -14,7 +14,7 @@ defmodule Plausible.ImportedTest do build(:pageview, timestamp: ~N[2021-01-31 00:00:00]) ]) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -27,7 +27,7 @@ defmodule Plausible.ImportedTest do } ], site.id, - "visitors" + "imported_visitors" ) conn = @@ -63,7 +63,7 @@ defmodule Plausible.ImportedTest do ) ]) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -92,7 +92,7 @@ defmodule Plausible.ImportedTest do } ], site.id, - "sources" + "imported_sources" ) conn = @@ -150,7 +150,7 @@ defmodule Plausible.ImportedTest do ) ]) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -163,7 +163,7 @@ defmodule Plausible.ImportedTest do } ], site.id, - "utm_mediums" + "imported_utm_mediums" ) conn = @@ -194,7 +194,7 @@ defmodule Plausible.ImportedTest do build(:pageview, utm_campaign: "august", timestamp: ~N[2021-01-01 00:00:00]) ]) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -207,7 +207,7 @@ defmodule Plausible.ImportedTest do } ], site.id, - "utm_campaigns" + "imported_utm_campaigns" ) conn = @@ -238,7 +238,7 @@ defmodule Plausible.ImportedTest do build(:pageview, utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00]) ]) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -251,7 +251,7 @@ defmodule Plausible.ImportedTest do } ], site.id, - "utm_terms" + "imported_utm_terms" ) conn = @@ -282,7 +282,7 @@ defmodule Plausible.ImportedTest do build(:pageview, utm_content: "blog", timestamp: ~N[2021-01-01 00:00:00]) ]) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -295,7 +295,7 @@ defmodule Plausible.ImportedTest do } ], site.id, - "utm_contents" + "imported_utm_contents" ) conn = @@ -338,7 +338,7 @@ defmodule Plausible.ImportedTest do ) ]) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -351,10 +351,10 @@ defmodule Plausible.ImportedTest do } ], site.id, - "pages" + "imported_pages" ) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -363,7 +363,7 @@ defmodule Plausible.ImportedTest do } ], site.id, - "entry_pages" + "imported_entry_pages" ) conn = @@ -412,7 +412,7 @@ defmodule Plausible.ImportedTest do ) ]) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -421,10 +421,10 @@ defmodule Plausible.ImportedTest do } ], site.id, - "pages" + "imported_pages" ) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -433,7 +433,7 @@ defmodule Plausible.ImportedTest do } ], site.id, - "exit_pages" + "imported_exit_pages" ) conn = @@ -469,7 +469,7 @@ defmodule Plausible.ImportedTest do ) ]) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -482,7 +482,7 @@ defmodule Plausible.ImportedTest do } ], site.id, - "locations" + "imported_locations" ) conn = @@ -518,7 +518,7 @@ defmodule Plausible.ImportedTest do build(:pageview, screen_size: "Laptop", timestamp: ~N[2021-01-01 00:15:00]) ]) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -531,7 +531,7 @@ defmodule Plausible.ImportedTest do } ], site.id, - "devices" + "imported_devices" ) conn = @@ -553,7 +553,7 @@ defmodule Plausible.ImportedTest do build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:15:00]) ]) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -566,7 +566,7 @@ defmodule Plausible.ImportedTest do } ], site.id, - "browsers" + "imported_browsers" ) conn = @@ -589,7 +589,7 @@ defmodule Plausible.ImportedTest do build(:pageview, operating_system: "GNU/Linux", timestamp: ~N[2021-01-01 00:15:00]) ]) - assert {:ok, _} = + assert :ok = Plausible.Imported.from_google_analytics( [ %{ @@ -602,7 +602,7 @@ defmodule Plausible.ImportedTest do } ], site.id, - "operating_systems" + "imported_operating_systems" ) conn = diff --git a/test/support/factory.ex b/test/support/factory.ex index ef2e4317f12a..173f01a419fb 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -184,7 +184,8 @@ defmodule Plausible.Factory do @today Timex.today() |> Timex.to_date() def imported_visitors_factory do - %Plausible.Imported.Visitors{ + %{ + table: "imported_visitors", timestamp: @today, visitors: 1, pageviews: 1, @@ -195,7 +196,8 @@ defmodule Plausible.Factory do end def imported_sources_factory do - %Plausible.Imported.Sources{ + %{ + table: "imported_sources", timestamp: @today, source: "", visitors: 1, @@ -206,7 +208,8 @@ defmodule Plausible.Factory do end def imported_utm_mediums_factory do - %Plausible.Imported.UtmMediums{ + %{ + table: "imported_utm_mediums", timestamp: @today, utm_medium: "", visitors: 1, @@ -217,7 +220,8 @@ defmodule Plausible.Factory do end def imported_utm_sources_factory do - %Plausible.Imported.UtmSources{ + %{ + table: "imported_utm_sources", timestamp: @today, utm_source: "", visitors: 1, @@ -228,7 +232,8 @@ defmodule Plausible.Factory do end def imported_utm_campaigns_factory do - %Plausible.Imported.UtmCampaigns{ + %{ + table: "imported_utm_campaigns", timestamp: @today, utm_campaign: "", visitors: 1, @@ -239,7 +244,8 @@ defmodule Plausible.Factory do end def imported_utm_terms_factory do - %Plausible.Imported.UtmTerms{ + %{ + table: "imported_utm_terms", timestamp: @today, utm_term: "", visitors: 1, @@ -250,7 +256,8 @@ defmodule Plausible.Factory do end def imported_utm_contents_factory do - %Plausible.Imported.UtmContents{ + %{ + table: "imported_utm_contents", timestamp: @today, utm_content: "", visitors: 1, @@ -261,7 +268,8 @@ defmodule Plausible.Factory do end def imported_pages_factory do - %Plausible.Imported.Pages{ + %{ + table: "imported_pages", timestamp: @today, page: "", visitors: 1, @@ -271,7 +279,8 @@ defmodule Plausible.Factory do end def imported_entry_pages_factory do - %Plausible.Imported.EntryPages{ + %{ + table: "imported_entry_pages", timestamp: @today, entry_page: "", visitors: 1, @@ -282,7 +291,8 @@ defmodule Plausible.Factory do end def imported_exit_pages_factory do - %Plausible.Imported.ExitPages{ + %{ + table: "imported_exit_pages", timestamp: @today, exit_page: "", visitors: 1, @@ -291,7 +301,8 @@ defmodule Plausible.Factory do end def imported_locations_factory do - %Plausible.Imported.Locations{ + %{ + table: "imported_locations", timestamp: @today, country: "", region: "", @@ -304,7 +315,8 @@ defmodule Plausible.Factory do end def imported_devices_factory do - %Plausible.Imported.Devices{ + %{ + table: "imported_devices", timestamp: @today, device: "", visitors: 1, @@ -315,7 +327,8 @@ defmodule Plausible.Factory do end def imported_browsers_factory do - %Plausible.Imported.Browsers{ + %{ + table: "imported_browsers", timestamp: @today, browser: "", visitors: 1, @@ -326,7 +339,8 @@ defmodule Plausible.Factory do end def imported_operating_systems_factory do - %Plausible.Imported.OperatingSystems{ + %{ + table: "imported_operating_systems", timestamp: @today, operating_system: "", visitors: 1, diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index e4a7f64228aa..4f8149575e8e 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -131,7 +131,8 @@ defmodule Plausible.TestUtils do end defp populate_imported_stats(events) do - Enum.map(events, &Plausible.ClickhouseRepo.insert!/1) + Enum.group_by(events, &Map.fetch!(&1, :table), &Map.delete(&1, :table)) + |> Enum.map(fn {table, events} -> Plausible.ClickhouseRepo.insert_all(table, events) end) end def relative_time(shifts) do From 0f99441b792f94060364bd8b1aec1d6fe4b4c561 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Thu, 3 Mar 2022 15:21:28 -0600 Subject: [PATCH 141/148] Fix link when removing imported data --- assets/js/dashboard/stats/visitor-graph.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/assets/js/dashboard/stats/visitor-graph.js b/assets/js/dashboard/stats/visitor-graph.js index 159c133c3dee..d6119f43e510 100644 --- a/assets/js/dashboard/stats/visitor-graph.js +++ b/assets/js/dashboard/stats/visitor-graph.js @@ -356,9 +356,7 @@ class LineGraph extends React.Component { if (source) { const withImported = this.props.graphData.with_imported; const strike = withImported ? "" : " line-through" - const target = withImported || false ? - url.setQuery('with_imported', false) : - window.location.pathname; + const target = url.setQuery('with_imported', !withImported) const tip = withImported ? "" : "do not "; return ( From 16c8cb199c8def98380c443aaddf24f079030be0 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Fri, 4 Mar 2022 13:16:15 -0600 Subject: [PATCH 142/148] Merge source tables --- lib/plausible/clickhouse_repo.ex | 5 - lib/plausible/google/api.ex | 22 +-- lib/plausible/imported/site.ex | 65 +++++---- lib/plausible/stats/breakdown.ex | 2 +- lib/plausible/stats/imported.ex | 23 +++- .../20211112130238_create_imported_tables.exs | 125 ++++++++++++++++++ ...0211112130238_create_imported_visitors.exs | 15 --- ...20211118160420_create_imported_sources.exs | 15 --- ...1124135248_create_imported_utm_mediums.exs | 15 --- ...24135252_create_imported_utm_campaigns.exs | 15 --- .../20211129111618_create_imported_pages.exs | 14 -- ...1129111622_create_imported_entry_pages.exs | 15 --- ...11129111630_create_imported_exit_pages.exs | 13 -- ...211129111639_create_imported_locations.exs | 17 --- ...20211129111644_create_imported_devices.exs | 15 --- ...0211129111648_create_imported_browsers.exs | 15 --- ...1653_create_imported_operating_systems.exs | 15 --- ...1129164103_create_imported_utm_sources.exs | 15 --- ...211231120503_create_imported_utm_terms.exs | 15 --- ...231120603_create_imported_utm_contents.exs | 15 --- test/plausible/imported/imported_test.exs | 106 +++++++-------- .../api/stats_controller/main_graph_test.exs | 8 +- .../api/stats_controller/pages_test.exs | 12 +- .../api/stats_controller/sources_test.exs | 76 +++-------- test/support/factory.ex | 78 ++--------- 25 files changed, 264 insertions(+), 467 deletions(-) create mode 100644 priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs delete mode 100644 priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs delete mode 100644 priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs delete mode 100644 priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs delete mode 100644 priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs delete mode 100644 priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs delete mode 100644 priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs delete mode 100644 priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs delete mode 100644 priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs delete mode 100644 priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs delete mode 100644 priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs delete mode 100644 priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs delete mode 100644 priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs delete mode 100644 priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs delete mode 100644 priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex index a1f57fba13fa..b77ef717f7d9 100644 --- a/lib/plausible/clickhouse_repo.ex +++ b/lib/plausible/clickhouse_repo.ex @@ -22,11 +22,6 @@ defmodule Plausible.ClickhouseRepo do [ "imported_visitors", "imported_sources", - "imported_utm_mediums", - "imported_utm_sources", - "imported_utm_campaigns", - "imported_utm_contents", - "imported_utm_terms", "imported_pages", "imported_entry_pages", "imported_exit_pages", diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 461e986fbae4..105304dddfab 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -197,27 +197,7 @@ defmodule Plausible.Google.Api do }, { "imported_sources", - ["ga:date", "ga:fullReferrer"], - ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] - }, - { - "imported_utm_mediums", - ["ga:date", "ga:medium"], - ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] - }, - { - "imported_utm_campaigns", - ["ga:date", "ga:campaign"], - ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] - }, - { - "imported_utm_terms", - ["ga:date", "ga:keyword"], - ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] - }, - { - "imported_utm_contents", - ["ga:date", "ga:adContent"], + ["ga:date", "ga:source", "ga:medium", "ga:campaign", "ga:adContent", "ga:keyword"], ["ga:users", "ga:sessions", "ga:bounces", "ga:sessionDuration"] }, { diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index db99a63657ed..242ab897188f 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -21,7 +21,7 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(site_id, "imported_visitors", %{ - "dimensions" => [timestamp], + "dimensions" => [date], "metrics" => [%{"values" => values}] }) do [visitors, pageviews, bounces, visits, visit_duration] = @@ -31,7 +31,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), visitors: visitors, pageviews: pageviews, bounces: bounces, @@ -44,7 +44,7 @@ defmodule Plausible.Imported do @domain ~r/^(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|([a-zA-Z0-9][-_.a-zA-Z0-9]{0,61}[a-zA-Z0-9]))\.([a-zA-Z]{2,13}|[a-zA-Z0-9-]{2,30}.[a-zA-Z]{2,3})$/ defp new_from_google_analytics(site_id, "imported_sources", %{ - "dimensions" => [timestamp, source], + "dimensions" => [date, source, medium, campaign, content, term], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do {visitors, ""} = Integer.parse(visitors) @@ -55,10 +55,19 @@ defmodule Plausible.Imported do source = if source == "(direct)", do: nil, else: source source = if source && String.match?(source, @domain), do: parse_referrer(source), else: source + medium = if medium == "(none)", do: nil, else: medium + campaign = if campaign == "(not set)", do: nil, else: campaign + term = if term in ["(not set)", "(not provided)"], do: nil, else: term + content = if content == "(not set)", do: nil, else: content + %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), source: parse_referrer(source), + utm_medium: medium, + utm_campaign: campaign, + utm_content: content, + utm_term: term, visitors: visitors, visits: visits, bounces: bounces, @@ -67,7 +76,7 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(site_id, "imported_utm_mediums", %{ - "dimensions" => [timestamp, medium], + "dimensions" => [date, medium], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do {visitors, ""} = Integer.parse(visitors) @@ -79,7 +88,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), utm_medium: medium, visitors: visitors, visits: visits, @@ -89,7 +98,7 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(site_id, "imported_utm_campaigns", %{ - "dimensions" => [timestamp, campaign], + "dimensions" => [date, campaign], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do {visitors, ""} = Integer.parse(visitors) @@ -101,7 +110,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), utm_campaign: campaign, visitors: visitors, visits: visits, @@ -111,7 +120,7 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(site_id, "imported_utm_terms", %{ - "dimensions" => [timestamp, term], + "dimensions" => [date, term], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do {visitors, ""} = Integer.parse(visitors) @@ -123,7 +132,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), utm_term: term, visitors: visitors, visits: visits, @@ -133,7 +142,7 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(site_id, "imported_utm_contents", %{ - "dimensions" => [timestamp, content], + "dimensions" => [date, content], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do {visitors, ""} = Integer.parse(visitors) @@ -145,7 +154,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), utm_content: content, visitors: visitors, visits: visits, @@ -155,7 +164,7 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(site_id, "imported_pages", %{ - "dimensions" => [timestamp, page], + "dimensions" => [date, page], "metrics" => [%{"values" => [visitors, pageviews, time_on_page]}] }) do {visitors, ""} = Integer.parse(visitors) @@ -164,7 +173,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), page: page, visitors: visitors, pageviews: pageviews, @@ -173,7 +182,7 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(site_id, "imported_entry_pages", %{ - "dimensions" => [timestamp, entry_page], + "dimensions" => [date, entry_page], "metrics" => [%{"values" => [visitors, entrances, visit_duration, bounces]}] }) do {visitors, ""} = Integer.parse(visitors) @@ -183,7 +192,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), entry_page: entry_page, visitors: visitors, entrances: entrances, @@ -193,7 +202,7 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(site_id, "imported_exit_pages", %{ - "dimensions" => [timestamp, exit_page], + "dimensions" => [date, exit_page], "metrics" => [%{"values" => [visitors, exits]}] }) do {visitors, ""} = Integer.parse(visitors) @@ -201,7 +210,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), exit_page: exit_page, visitors: visitors, exits: exits @@ -209,7 +218,7 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(site_id, "imported_locations", %{ - "dimensions" => [timestamp, country, region], + "dimensions" => [date, country, region], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do country = if country == "(not set)", do: "", else: country @@ -221,7 +230,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), country: country, region: region, city: 0, @@ -233,7 +242,7 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(site_id, "imported_devices", %{ - "dimensions" => [timestamp, device], + "dimensions" => [date, device], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do {visitors, ""} = Integer.parse(visitors) @@ -243,7 +252,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), device: String.capitalize(device), visitors: visitors, visits: visits, @@ -263,7 +272,7 @@ defmodule Plausible.Imported do } defp new_from_google_analytics(site_id, "imported_browsers", %{ - "dimensions" => [timestamp, browser], + "dimensions" => [date, browser], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do {visitors, ""} = Integer.parse(visitors) @@ -273,7 +282,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), browser: Map.get(@browser_google_to_plausible, browser, browser), visitors: visitors, visits: visits, @@ -289,7 +298,7 @@ defmodule Plausible.Imported do } defp new_from_google_analytics(site_id, "imported_operating_systems", %{ - "dimensions" => [timestamp, operating_system], + "dimensions" => [date, operating_system], "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] }) do {visitors, ""} = Integer.parse(visitors) @@ -299,7 +308,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, - timestamp: format_timestamp(timestamp), + date: format_date(date), operating_system: Map.get(@os_google_to_plausible, operating_system, operating_system), visitors: visitors, visits: visits, @@ -308,8 +317,8 @@ defmodule Plausible.Imported do } end - defp format_timestamp(timestamp) do - Timex.parse!("#{timestamp}", "%Y%m%d", :strftime) + defp format_date(date) do + Timex.parse!("#{date}", "%Y%m%d", :strftime) |> NaiveDateTime.to_date() end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index bac7a99c0037..38eb3f20a9aa 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -288,7 +288,7 @@ defmodule Plausible.Stats.Breakdown do i in "imported_pages", group_by: i.page, where: i.site_id == ^site.id, - where: i.timestamp >= ^query.date_range.first and i.timestamp <= ^query.date_range.last, + where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last, where: i.page in ^pages, select: %{ page: i.page, diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index a2a6490e2f79..cd2edf2b29fc 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -10,10 +10,10 @@ defmodule Plausible.Stats.Imported do from(v in "imported_visitors", group_by: fragment("date"), where: v.site_id == ^site.id, - where: v.timestamp >= ^query.date_range.first and v.timestamp <= ^query.date_range.last, + where: v.date >= ^query.date_range.first and v.date <= ^query.date_range.last, select: %{ visitors: sum(v.visitors), - date: fragment("? as date", v.timestamp) + date: fragment("? as date", v.date) } ) |> ClickhouseRepo.all() @@ -27,12 +27,13 @@ defmodule Plausible.Stats.Imported do def merge_imported(q, %Plausible.Site{imported_source: nil}, _, _, _), do: q def merge_imported(q, _, %Query{with_imported: false}, _, _), do: q def merge_imported(q, _, _, _, [:events | _]), do: q + # GA only has 'source' + def merge_imported(q, _, _, "utm_source", _), do: q def merge_imported(q, site, query, property, metrics) when property in [ "visit:source", "visit:utm_medium", - "visit:utm_source", "visit:utm_campaign", "visit:utm_term", "visit:utm_content", @@ -57,6 +58,18 @@ defmodule Plausible.Stats.Imported do "visit:city" -> {"imported_locations", :city} + "visit:utm_medium" -> + {"imported_sources", :utm_medium} + + "visit:utm_campaign" -> + {"imported_sources", :utm_campaign} + + "visit:utm_term" -> + {"imported_sources", :utm_term} + + "visit:utm_content" -> + {"imported_sources", :utm_content} + "visit:os" -> {"imported_operating_systems", :operating_system} @@ -73,7 +86,7 @@ defmodule Plausible.Stats.Imported do i in table, group_by: field(i, ^dim), where: i.site_id == ^site.id, - where: i.timestamp >= ^query.date_range.first and i.timestamp <= ^query.date_range.last, + where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last, select: %{} ) |> select_imported_metrics(metrics) @@ -280,7 +293,7 @@ defmodule Plausible.Stats.Imported do from( i in "imported_visitors", where: i.site_id == ^site.id, - where: i.timestamp >= ^query.date_range.first and i.timestamp <= ^query.date_range.last, + where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last, select: %{} ) |> select_imported_metrics(metrics) diff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs b/priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs new file mode 100644 index 000000000000..616698f1b7e1 --- /dev/null +++ b/priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs @@ -0,0 +1,125 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_visitors, + engine: + "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:visitors, :UInt64) + add(:pageviews, :UInt64) + add(:bounces, :UInt64) + add(:visits, :UInt64) + add(:visit_duration, :UInt64) + end + + create_if_not_exists table(:imported_sources, + engine: + "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:source, :string) + add(:utm_medium, :string) + add(:utm_campaign, :string) + add(:utm_content, :string) + add(:utm_term, :string) + add(:visitors, :UInt64) + add(:visits, :UInt64) + add(:visit_duration, :UInt64) + add(:bounces, :UInt32) + end + + create_if_not_exists table(:imported_pages, + engine: + "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:page, :string) + add(:visitors, :UInt64) + add(:pageviews, :UInt64) + add(:time_on_page, :UInt64) + end + + create_if_not_exists table(:imported_entry_pages, + engine: + "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:entry_page, :string) + add(:visitors, :UInt64) + add(:entrances, :UInt64) + add(:visit_duration, :UInt64) + add(:bounces, :UInt32) + end + + create_if_not_exists table(:imported_exit_pages, + engine: + "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:exit_page, :string) + add(:visitors, :UInt64) + add(:exits, :UInt64) + end + + create_if_not_exists table(:imported_locations, + engine: + "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:country, :string) + add(:region, :string) + add(:city, :UInt64) + add(:visitors, :UInt64) + add(:visits, :UInt64) + add(:visit_duration, :UInt64) + add(:bounces, :UInt32) + end + + create_if_not_exists table(:imported_devices, + engine: + "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:device, :string) + add(:visitors, :UInt64) + add(:visits, :UInt64) + add(:visit_duration, :UInt64) + add(:bounces, :UInt32) + end + + create_if_not_exists table(:imported_browsers, + engine: + "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:browser, :string) + add(:visitors, :UInt64) + add(:visits, :UInt64) + add(:visit_duration, :UInt64) + add(:bounces, :UInt32) + end + + create_if_not_exists table(:imported_operating_systems, + engine: + "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:operating_system, :string) + add(:visitors, :UInt64) + add(:visits, :UInt64) + add(:visit_duration, :UInt64) + add(:bounces, :UInt32) + end + end +end diff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs b/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs deleted file mode 100644 index 239b3ddef9de..000000000000 --- a/priv/clickhouse_repo/migrations/20211112130238_create_imported_visitors.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_visitors, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :visitors, :UInt64 - add :pageviews, :UInt64 - add :bounces, :UInt64 - add :visits, :UInt64 - add :visit_duration, :UInt64 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs b/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs deleted file mode 100644 index a8a3e8909290..000000000000 --- a/priv/clickhouse_repo/migrations/20211118160420_create_imported_sources.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedSources do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_sources, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :source, :string - add :visitors, :UInt64 - add :visits, :UInt64 - add :visit_duration, :UInt64 - add :bounces, :UInt32 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs b/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs deleted file mode 100644 index 4e09eadc93a5..000000000000 --- a/priv/clickhouse_repo/migrations/20211124135248_create_imported_utm_mediums.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmMediums do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_utm_mediums, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :utm_medium, :string - add :visitors, :UInt64 - add :visits, :UInt64 - add :visit_duration, :UInt64 - add :bounces, :UInt32 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs b/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs deleted file mode 100644 index 57e918598c4f..000000000000 --- a/priv/clickhouse_repo/migrations/20211124135252_create_imported_utm_campaigns.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmCampaigns do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_utm_campaigns, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :utm_campaign, :string - add :visitors, :UInt64 - add :visits, :UInt64 - add :visit_duration, :UInt64 - add :bounces, :UInt32 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs b/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs deleted file mode 100644 index b836d918b17e..000000000000 --- a/priv/clickhouse_repo/migrations/20211129111618_create_imported_pages.exs +++ /dev/null @@ -1,14 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedPages do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_pages, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :page, :string - add :visitors, :UInt64 - add :pageviews, :UInt64 - add :time_on_page, :UInt64 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs b/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs deleted file mode 100644 index 5d72324d9836..000000000000 --- a/priv/clickhouse_repo/migrations/20211129111622_create_imported_entry_pages.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedEntryPages do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_entry_pages, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :entry_page, :string - add :visitors, :UInt64 - add :entrances, :UInt64 - add :visit_duration, :UInt64 - add :bounces, :UInt32 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs b/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs deleted file mode 100644 index 1ba293e55cdc..000000000000 --- a/priv/clickhouse_repo/migrations/20211129111630_create_imported_exit_pages.exs +++ /dev/null @@ -1,13 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedExitPages do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_exit_pages, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :exit_page, :string - add :visitors, :UInt64 - add :exits, :UInt64 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs b/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs deleted file mode 100644 index 77dd7f53e4a5..000000000000 --- a/priv/clickhouse_repo/migrations/20211129111639_create_imported_locations.exs +++ /dev/null @@ -1,17 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedLocations do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_locations, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :country, :string - add :region, :string - add :city, :UInt64 - add :visitors, :UInt64 - add :visits, :UInt64 - add :visit_duration, :UInt64 - add :bounces, :UInt32 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs b/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs deleted file mode 100644 index 9dfa4fe5a997..000000000000 --- a/priv/clickhouse_repo/migrations/20211129111644_create_imported_devices.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedDevices do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_devices, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :device, :string - add :visitors, :UInt64 - add :visits, :UInt64 - add :visit_duration, :UInt64 - add :bounces, :UInt32 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs b/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs deleted file mode 100644 index 4922c2398bc3..000000000000 --- a/priv/clickhouse_repo/migrations/20211129111648_create_imported_browsers.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedBrowsers do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_browsers, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :browser, :string - add :visitors, :UInt64 - add :visits, :UInt64 - add :visit_duration, :UInt64 - add :bounces, :UInt32 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs b/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs deleted file mode 100644 index b0f5d25acb1e..000000000000 --- a/priv/clickhouse_repo/migrations/20211129111653_create_imported_operating_systems.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedOperatingSystems do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_operating_systems, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :operating_system, :string - add :visitors, :UInt64 - add :visits, :UInt64 - add :visit_duration, :UInt64 - add :bounces, :UInt32 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs b/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs deleted file mode 100644 index e5db4b1acbef..000000000000 --- a/priv/clickhouse_repo/migrations/20211129164103_create_imported_utm_sources.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmSources do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_utm_sources, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :utm_source, :string - add :visitors, :UInt64 - add :visits, :UInt64 - add :visit_duration, :UInt64 - add :bounces, :UInt32 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs b/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs deleted file mode 100644 index 6352c87eaa7a..000000000000 --- a/priv/clickhouse_repo/migrations/20211231120503_create_imported_utm_terms.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmTerms do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_utm_terms, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :utm_term, :string - add :visitors, :UInt64 - add :visits, :UInt64 - add :visit_duration, :UInt64 - add :bounces, :UInt32 - end - end -end diff --git a/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs b/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs deleted file mode 100644 index c6c5548dc890..000000000000 --- a/priv/clickhouse_repo/migrations/20211231120603_create_imported_utm_contents.exs +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedUtmContents do - use Ecto.Migration - - def change do - create_if_not_exists table(:imported_utm_contents, engine: "MergeTree() ORDER BY (site_id, timestamp) SETTINGS index_granularity = 1") do - add :site_id, :UInt64 - add :timestamp, :date - add :utm_content, :string - add :visitors, :UInt64 - add :visits, :UInt64 - add :visit_duration, :UInt64 - add :bounces, :UInt32 - end - end -end diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index e989c608de0f..74e274bc5597 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -44,7 +44,7 @@ defmodule Plausible.ImportedTest do assert Enum.sum(plot) == 4 end - test "Sources and UTM sources are separated", %{conn: conn, site: site} do + test "Sources are imported", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, referrer_source: "Google", @@ -67,27 +67,34 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["20210101", "duckduckgo.com"], + "dimensions" => ["20210101", "duckduckgo.com", "organic", "", "", ""], "metrics" => [%{"values" => ["1", "1", "0", "60"]}] }, %{ - "dimensions" => ["20210131", "google.com"], + "dimensions" => ["20210131", "google.com", "organic", "", "", ""], "metrics" => [%{"values" => ["1", "1", "1", "60"]}] }, %{ - "dimensions" => ["20210101", "google"], + "dimensions" => ["20210101", "google.com", "paid", "", "", ""], "metrics" => [%{"values" => ["1", "1", "1", "60"]}] }, %{ - "dimensions" => ["20210101", "Twitter"], + "dimensions" => ["20210101", "Twitter", "social", "", "", ""], "metrics" => [%{"values" => ["1", "1", "1", "60"]}] }, %{ - "dimensions" => ["20210131", "A Nice Newsletter"], + "dimensions" => [ + "20210131", + "A Nice Newsletter", + "email", + "newsletter", + "", + "" + ], "metrics" => [%{"values" => ["1", "1", "1", "60"]}] }, %{ - "dimensions" => ["20210101", "(direct)"], + "dimensions" => ["20210101", "(direct)", "(none)", "", "", ""], "metrics" => [%{"values" => ["1", "1", "1", "60"]}] } ], @@ -102,35 +109,10 @@ defmodule Plausible.ImportedTest do ) assert json_response(conn, 200) == [ - %{"name" => "Google", "visitors" => 3}, - %{"name" => "DuckDuckGo", "visitors" => 2} - ] - - conn = - get( - conn, - "/api/stats/#{site.domain}/utm_sources?period=month&date=2021-01-01&with_imported=true" - ) - - assert json_response(conn, 200) == [ - %{ - "name" => "A Nice Newsletter", - "visitors" => 1, - "bounce_rate" => 100.0, - "visit_duration" => 60.0 - }, - %{ - "name" => "Twitter", - "visitors" => 1, - "bounce_rate" => 100.0, - "visit_duration" => 60.0 - }, - %{ - "bounce_rate" => 100.0, - "name" => "adwords", - "visit_duration" => 60.0, - "visitors" => 1 - } + %{"name" => "Google", "visitors" => 4}, + %{"name" => "DuckDuckGo", "visitors" => 2}, + %{"name" => "A Nice Newsletter", "visitors" => 1}, + %{"name" => "Twitter", "visitors" => 1} ] end @@ -143,10 +125,6 @@ defmodule Plausible.ImportedTest do build(:pageview, utm_medium: "social", timestamp: ~N[2021-01-01 12:00:00] - ), - build(:pageview, - utm_medium: "email", - timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -154,16 +132,16 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["20210101", "social"], + "dimensions" => ["20210101", "Twitter", "social", "", "", ""], "metrics" => [%{"values" => ["1", "1", "1", "60"]}] }, %{ - "dimensions" => ["20210101", "email"], - "metrics" => [%{"values" => ["1", "1", "0", "100"]}] + "dimensions" => ["20210101", "(direct)", "(none)", "", "", ""], + "metrics" => [%{"values" => ["1", "1", "1", "60"]}] } ], site.id, - "imported_utm_mediums" + "imported_sources" ) conn = @@ -178,12 +156,6 @@ defmodule Plausible.ImportedTest do "name" => "social", "visit_duration" => 20, "visitors" => 3 - }, - %{ - "bounce_rate" => 50.0, - "name" => "email", - "visit_duration" => 50.0, - "visitors" => 2 } ] end @@ -198,16 +170,20 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["20210101", "profile"], + "dimensions" => ["20210101", "Twitter", "social", "profile", "", ""], "metrics" => [%{"values" => ["1", "1", "1", "100"]}] }, %{ - "dimensions" => ["20210101", "august"], + "dimensions" => ["20210101", "Gmail", "email", "august", "", ""], + "metrics" => [%{"values" => ["1", "1", "0", "100"]}] + }, + %{ + "dimensions" => ["20210101", "Gmail", "email", "(not set)", "", ""], "metrics" => [%{"values" => ["1", "1", "0", "100"]}] } ], site.id, - "imported_utm_campaigns" + "imported_sources" ) conn = @@ -242,16 +218,20 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["20210101", "oat milk"], + "dimensions" => ["20210101", "Google", "paid", "", "", "oat milk"], "metrics" => [%{"values" => ["1", "1", "1", "100"]}] }, %{ - "dimensions" => ["20210101", "Sweden"], + "dimensions" => ["20210101", "Google", "paid", "", "", "Sweden"], + "metrics" => [%{"values" => ["1", "1", "0", "100"]}] + }, + %{ + "dimensions" => ["20210101", "Google", "paid", "", "", "(not set)"], "metrics" => [%{"values" => ["1", "1", "0", "100"]}] } ], site.id, - "imported_utm_terms" + "imported_sources" ) conn = @@ -286,16 +266,20 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["20210101", "ad"], + "dimensions" => ["20210101", "Google", "paid", "", "ad", ""], "metrics" => [%{"values" => ["1", "1", "1", "100"]}] }, %{ - "dimensions" => ["20210101", "blog"], + "dimensions" => ["20210101", "Google", "paid", "", "blog", ""], + "metrics" => [%{"values" => ["1", "1", "0", "100"]}] + }, + %{ + "dimensions" => ["20210101", "Google", "paid", "", "(not set)", ""], "metrics" => [%{"values" => ["1", "1", "0", "100"]}] } ], site.id, - "imported_utm_contents" + "imported_sources" ) conn = @@ -383,8 +367,8 @@ defmodule Plausible.ImportedTest do %{ "bounce_rate" => nil, "time_on_page" => 60, - "visitors" => 2, - "pageviews" => 2, + "visitors" => 3, + "pageviews" => 4, "name" => "/some-other-page" } ] diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index ba3a5f444e36..0f5dd0194793 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -73,8 +73,8 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do populate_stats(site, [ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), build(:pageview, timestamp: ~N[2021-01-31 00:00:00]), - build(:imported_visitors, timestamp: ~D[2021-01-01]), - build(:imported_visitors, timestamp: ~D[2021-01-31]) + build(:imported_visitors, date: ~D[2021-01-01]), + build(:imported_visitors, date: ~D[2021-01-31]) ]) conn = @@ -95,8 +95,8 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do populate_stats(site, [ build(:pageview, timestamp: ~N[2021-01-01 00:00:00], pathname: "/pageA"), build(:pageview, timestamp: ~N[2021-01-31 00:00:00], pathname: "/pageA"), - build(:imported_visitors, timestamp: ~D[2021-01-01]), - build(:imported_visitors, timestamp: ~D[2021-01-31]) + build(:imported_visitors, date: ~D[2021-01-01]), + build(:imported_visitors, date: ~D[2021-01-31]) ]) filters = Jason.encode!(%{page: "/pageA"}) diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index bb8b0179e842..fa708720c051 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -117,18 +117,18 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do ), build(:imported_pages, page: "/", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], time_on_page: 700 ), build(:imported_entry_pages, entry_page: "/", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], entrances: 3, bounces: 1 ), build(:imported_pages, page: "/some-other-page", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], time_on_page: 60 ) ]) @@ -274,7 +274,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do populate_stats(site, [ build(:imported_entry_pages, entry_page: "/page2", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], entrances: 3, visitors: 2, visit_duration: 300 @@ -441,13 +441,13 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do populate_stats(site, [ build(:imported_pages, page: "/page2", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], pageviews: 4, visitors: 2 ), build(:imported_exit_pages, exit_page: "/page2", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], exits: 3, visitors: 2 ) diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index fed047edf2e3..873e26edd2f0 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -133,7 +133,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do populate_stats(site, [ build(:imported_sources, source: "Google", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], visitors: 2, visits: 3, bounces: 1, @@ -141,7 +141,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ), build(:imported_sources, source: "DuckDuckGo", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], visitors: 1, visits: 1, visit_duration: 100, @@ -290,17 +290,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ]) populate_stats(site, [ - build(:imported_utm_mediums, + build(:imported_sources, utm_medium: "social", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], visit_duration: 700, bounces: 1, visits: 1, visitors: 1 ), - build(:imported_utm_mediums, + build(:imported_sources, utm_medium: "email", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], bounces: 0, visits: 1, visitors: 1, @@ -378,17 +378,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ]) populate_stats(site, [ - build(:imported_utm_campaigns, + build(:imported_sources, utm_campaign: "profile", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], visit_duration: 700, bounces: 1, visits: 1, visitors: 1 ), - build(:imported_utm_campaigns, + build(:imported_sources, utm_campaign: "august", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], bounces: 0, visits: 1, visitors: 1, @@ -465,25 +465,6 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ) ]) - populate_stats(site, [ - build(:imported_utm_sources, - utm_source: "Twitter", - timestamp: ~D[2021-01-01], - visit_duration: 700, - bounces: 1, - visits: 1, - visitors: 1 - ), - build(:imported_utm_sources, - utm_source: "newsletter", - timestamp: ~D[2021-01-01], - bounces: 0, - visits: 1, - visitors: 1, - visit_duration: 900 - ) - ]) - conn = get( conn, @@ -504,27 +485,6 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do "visit_duration" => 900 } ] - - conn = - get( - conn, - "/api/stats/#{site.domain}/utm_sources?period=day&date=2021-01-01&with_imported=true" - ) - - assert json_response(conn, 200) == [ - %{ - "name" => "newsletter", - "visitors" => 3, - "bounce_rate" => 67, - "visit_duration" => 300 - }, - %{ - "name" => "Twitter", - "visitors" => 2, - "bounce_rate" => 50, - "visit_duration" => 800.0 - } - ] end end @@ -828,17 +788,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ]) populate_stats(site, [ - build(:imported_utm_terms, + build(:imported_sources, utm_term: "oat milk", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], visit_duration: 700, bounces: 1, visits: 1, visitors: 1 ), - build(:imported_utm_terms, + build(:imported_sources, utm_term: "Sweden", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], bounces: 0, visits: 1, visitors: 1, @@ -916,17 +876,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ]) populate_stats(site, [ - build(:imported_utm_contents, + build(:imported_sources, utm_content: "ad", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], visit_duration: 700, bounces: 1, visits: 1, visitors: 1 ), - build(:imported_utm_contents, + build(:imported_sources, utm_content: "blog", - timestamp: ~D[2021-01-01], + date: ~D[2021-01-01], bounces: 0, visits: 1, visitors: 1, diff --git a/test/support/factory.ex b/test/support/factory.ex index 173f01a419fb..e2d5a122f659 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -186,7 +186,7 @@ defmodule Plausible.Factory do def imported_visitors_factory do %{ table: "imported_visitors", - timestamp: @today, + date: @today, visitors: 1, pageviews: 1, bounces: 0, @@ -198,7 +198,7 @@ defmodule Plausible.Factory do def imported_sources_factory do %{ table: "imported_sources", - timestamp: @today, + date: @today, source: "", visitors: 1, visits: 1, @@ -207,70 +207,10 @@ defmodule Plausible.Factory do } end - def imported_utm_mediums_factory do - %{ - table: "imported_utm_mediums", - timestamp: @today, - utm_medium: "", - visitors: 1, - visits: 1, - bounces: 0, - visit_duration: 10 - } - end - - def imported_utm_sources_factory do - %{ - table: "imported_utm_sources", - timestamp: @today, - utm_source: "", - visitors: 1, - visits: 1, - bounces: 0, - visit_duration: 10 - } - end - - def imported_utm_campaigns_factory do - %{ - table: "imported_utm_campaigns", - timestamp: @today, - utm_campaign: "", - visitors: 1, - visits: 1, - bounces: 0, - visit_duration: 10 - } - end - - def imported_utm_terms_factory do - %{ - table: "imported_utm_terms", - timestamp: @today, - utm_term: "", - visitors: 1, - visits: 1, - bounces: 0, - visit_duration: 10 - } - end - - def imported_utm_contents_factory do - %{ - table: "imported_utm_contents", - timestamp: @today, - utm_content: "", - visitors: 1, - visits: 1, - bounces: 0, - visit_duration: 10 - } - end - def imported_pages_factory do %{ table: "imported_pages", - timestamp: @today, + date: @today, page: "", visitors: 1, pageviews: 1, @@ -281,7 +221,7 @@ defmodule Plausible.Factory do def imported_entry_pages_factory do %{ table: "imported_entry_pages", - timestamp: @today, + date: @today, entry_page: "", visitors: 1, entrances: 1, @@ -293,7 +233,7 @@ defmodule Plausible.Factory do def imported_exit_pages_factory do %{ table: "imported_exit_pages", - timestamp: @today, + date: @today, exit_page: "", visitors: 1, exits: 1 @@ -303,7 +243,7 @@ defmodule Plausible.Factory do def imported_locations_factory do %{ table: "imported_locations", - timestamp: @today, + date: @today, country: "", region: "", city: 0, @@ -317,7 +257,7 @@ defmodule Plausible.Factory do def imported_devices_factory do %{ table: "imported_devices", - timestamp: @today, + date: @today, device: "", visitors: 1, visits: 1, @@ -329,7 +269,7 @@ defmodule Plausible.Factory do def imported_browsers_factory do %{ table: "imported_browsers", - timestamp: @today, + date: @today, browser: "", visitors: 1, visits: 1, @@ -341,7 +281,7 @@ defmodule Plausible.Factory do def imported_operating_systems_factory do %{ table: "imported_operating_systems", - timestamp: @today, + date: @today, operating_system: "", visitors: 1, visits: 1, From c324395184b8a66ae03a6ab77e64d087a92cbc22 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Fri, 4 Mar 2022 15:25:40 -0600 Subject: [PATCH 143/148] Import hostname as well as pathname --- lib/plausible/google/api.ex | 4 +- lib/plausible/imported/site.ex | 109 ++---------------- .../20211112130238_create_imported_tables.exs | 28 ++--- test/plausible/imported/imported_test.exs | 32 ++--- 4 files changed, 40 insertions(+), 133 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 105304dddfab..90707ced33f5 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -202,7 +202,7 @@ defmodule Plausible.Google.Api do }, { "imported_pages", - ["ga:date", "ga:pagePath"], + ["ga:date", "ga:hostname", "ga:pagePath"], ["ga:users", "ga:pageviews", "ga:timeOnPage"] }, { @@ -286,7 +286,7 @@ defmodule Plausible.Google.Api do dateRanges: [ %{ # The earliest valid date - startDate: "2005-01-01", + startDate: "2017-01-01", endDate: request.end_date } ], diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 242ab897188f..8e2c42456835 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -55,107 +55,14 @@ defmodule Plausible.Imported do source = if source == "(direct)", do: nil, else: source source = if source && String.match?(source, @domain), do: parse_referrer(source), else: source - medium = if medium == "(none)", do: nil, else: medium - campaign = if campaign == "(not set)", do: nil, else: campaign - term = if term in ["(not set)", "(not provided)"], do: nil, else: term - content = if content == "(not set)", do: nil, else: content - %{ site_id: site_id, date: format_date(date), source: parse_referrer(source), - utm_medium: medium, - utm_campaign: campaign, - utm_content: content, - utm_term: term, - visitors: visitors, - visits: visits, - bounces: bounces, - visit_duration: visit_duration - } - end - - defp new_from_google_analytics(site_id, "imported_utm_mediums", %{ - "dimensions" => [date, medium], - "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] - }) do - {visitors, ""} = Integer.parse(visitors) - {visits, ""} = Integer.parse(visits) - {bounces, ""} = Integer.parse(bounces) - {visit_duration, _} = Integer.parse(visit_duration) - - medium = if medium == "(none)", do: "", else: medium - - %{ - site_id: site_id, - date: format_date(date), - utm_medium: medium, - visitors: visitors, - visits: visits, - bounces: bounces, - visit_duration: visit_duration - } - end - - defp new_from_google_analytics(site_id, "imported_utm_campaigns", %{ - "dimensions" => [date, campaign], - "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] - }) do - {visitors, ""} = Integer.parse(visitors) - {visits, ""} = Integer.parse(visits) - {bounces, ""} = Integer.parse(bounces) - {visit_duration, _} = Integer.parse(visit_duration) - - campaign = if campaign == "(not set)", do: "", else: campaign - - %{ - site_id: site_id, - date: format_date(date), - utm_campaign: campaign, - visitors: visitors, - visits: visits, - bounces: bounces, - visit_duration: visit_duration - } - end - - defp new_from_google_analytics(site_id, "imported_utm_terms", %{ - "dimensions" => [date, term], - "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] - }) do - {visitors, ""} = Integer.parse(visitors) - {visits, ""} = Integer.parse(visits) - {bounces, ""} = Integer.parse(bounces) - {visit_duration, _} = Integer.parse(visit_duration) - - term = if term == "(not set)" or term == "(not provided)", do: "", else: term - - %{ - site_id: site_id, - date: format_date(date), - utm_term: term, - visitors: visitors, - visits: visits, - bounces: bounces, - visit_duration: visit_duration - } - end - - defp new_from_google_analytics(site_id, "imported_utm_contents", %{ - "dimensions" => [date, content], - "metrics" => [%{"values" => [visitors, visits, bounces, visit_duration]}] - }) do - {visitors, ""} = Integer.parse(visitors) - {visits, ""} = Integer.parse(visits) - {bounces, ""} = Integer.parse(bounces) - {visit_duration, _} = Integer.parse(visit_duration) - - content = if content == "(not set)", do: "", else: content - - %{ - site_id: site_id, - date: format_date(date), - utm_content: content, + utm_medium: nil_if_missing(medium), + utm_campaign: nil_if_missing(campaign), + utm_content: nil_if_missing(content), + utm_term: nil_if_missing(term), visitors: visitors, visits: visits, bounces: bounces, @@ -164,9 +71,10 @@ defmodule Plausible.Imported do end defp new_from_google_analytics(site_id, "imported_pages", %{ - "dimensions" => [date, page], + "dimensions" => [date, hostname, page], "metrics" => [%{"values" => [visitors, pageviews, time_on_page]}] }) do + page = URI.parse(page).path {visitors, ""} = Integer.parse(visitors) {pageviews, ""} = Integer.parse(pageviews) {time_on_page, _} = Integer.parse(time_on_page) @@ -174,6 +82,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, date: format_date(date), + hostname: hostname, page: page, visitors: visitors, pageviews: pageviews, @@ -322,6 +231,10 @@ defmodule Plausible.Imported do |> NaiveDateTime.to_date() end + @missing_values ["(none)", "(not set)", "(not provided)"] + def nil_if_missing(value) when value in @missing_values, do: nil + def nil_if_missing(value), do: value + def parse_referrer(nil), do: nil def parse_referrer("google"), do: "Google" def parse_referrer("bing"), do: "Bing" diff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs b/priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs index 616698f1b7e1..5abc84a957bf 100644 --- a/priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs +++ b/priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs @@ -3,8 +3,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do def change do create_if_not_exists table(:imported_visitors, - engine: - "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + engine: "MergeTree() ORDER BY (site_id, date)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -16,8 +15,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_sources, - engine: - "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + engine: "MergeTree() ORDER BY (site_id, date, source)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -33,11 +31,11 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_pages, - engine: - "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + engine: "MergeTree() ORDER BY (site_id, date, hostname, page)" ) do add(:site_id, :UInt64) add(:date, :date) + add(:hostname, :string) add(:page, :string) add(:visitors, :UInt64) add(:pageviews, :UInt64) @@ -45,8 +43,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_entry_pages, - engine: - "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + engine: "MergeTree() ORDER BY (site_id, date, entry_page)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -58,8 +55,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_exit_pages, - engine: - "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + engine: "MergeTree() ORDER BY (site_id, date, exit_page)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -69,8 +65,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_locations, - engine: - "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + engine: "MergeTree() ORDER BY (site_id, date, country, region, city)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -84,8 +79,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_devices, - engine: - "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + engine: "MergeTree() ORDER BY (site_id, date, device)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -97,8 +91,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_browsers, - engine: - "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + engine: "MergeTree() ORDER BY (site_id, date, browser)" ) do add(:site_id, :UInt64) add(:date, :date) @@ -110,8 +103,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do end create_if_not_exists table(:imported_operating_systems, - engine: - "MergeTree() ORDER BY (site_id, date) SETTINGS index_granularity = 1" + engine: "MergeTree() ORDER BY (site_id, date, operating_system)" ) do add(:site_id, :UInt64) add(:date, :date) diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 74e274bc5597..6a67866054fa 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -308,17 +308,15 @@ defmodule Plausible.ImportedTest do populate_stats(site, [ build(:pageview, pathname: "/", + hostname: "host-a.com", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, pathname: "/some-other-page", + hostname: "host-a.com", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] - ), - build(:pageview, - pathname: "/", - timestamp: ~N[2021-01-01 00:15:00] ) ]) @@ -326,11 +324,15 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["20210101", "/"], + "dimensions" => ["20210101", "host-a.com", "/"], "metrics" => [%{"values" => ["1", "1", "700"]}] }, %{ - "dimensions" => ["20210101", "/some-other-page"], + "dimensions" => ["20210101", "host-b.com", "/some-other-page"], + "metrics" => [%{"values" => ["1", "1", "60"]}] + }, + %{ + "dimensions" => ["20210101", "host-b.com", "/some-other-page?wat=wot"], "metrics" => [%{"values" => ["1", "1", "60"]}] } ], @@ -357,19 +359,19 @@ defmodule Plausible.ImportedTest do ) assert json_response(conn, 200) == [ - %{ - "bounce_rate" => 40.0, - "time_on_page" => 800.0, - "visitors" => 3, - "pageviews" => 3, - "name" => "/" - }, %{ "bounce_rate" => nil, "time_on_page" => 60, "visitors" => 3, - "pageviews" => 4, + "pageviews" => 3, "name" => "/some-other-page" + }, + %{ + "bounce_rate" => 25.0, + "time_on_page" => 800.0, + "visitors" => 2, + "pageviews" => 2, + "name" => "/" } ] end @@ -400,7 +402,7 @@ defmodule Plausible.ImportedTest do Plausible.Imported.from_google_analytics( [ %{ - "dimensions" => ["20210101", "/page2"], + "dimensions" => ["20210101", "host-a.com", "/page2"], "metrics" => [%{"values" => ["2", "4", "10"]}] } ], From d4b554b0dede01b77640bcd535c0a26dd8c57d04 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Mon, 7 Mar 2022 11:40:24 -0600 Subject: [PATCH 144/148] Record start and end time of imported data --- config/.env.dev | 3 + lib/plausible/google/api.ex | 2 +- lib/plausible/imported/site.ex | 2 +- lib/plausible/site/schema.ex | 23 ++++++- lib/plausible/stats/breakdown.ex | 2 +- lib/plausible/stats/imported.ex | 2 +- .../controllers/api/stats_controller.ex | 6 +- .../controllers/site_controller.ex | 12 ++-- lib/plausible_web/email.ex | 10 +-- lib/workers/import_google_analytics.ex | 21 +++--- priv/ref_inspector/referers.yml | 2 +- ...0211110174617_add_site_imported_source.exs | 2 +- test/plausible/imported/imported_test.exs | 2 +- .../api/stats_controller/browsers_test.exs | 2 +- .../api/stats_controller/countries_test.exs | 2 +- .../api/stats_controller/main_graph_test.exs | 2 +- .../operating_systems_test.exs | 2 +- .../api/stats_controller/pages_test.exs | 6 +- .../stats_controller/screen_sizes_test.exs | 2 +- .../api/stats_controller/sources_test.exs | 10 +-- .../controllers/site_controller_test.exs | 24 +++++++ test/support/factory.ex | 3 +- test/support/test_utils.ex | 9 +++ test/workers/import_google_analytics_test.exs | 66 +++++++++++++++++++ 24 files changed, 171 insertions(+), 46 deletions(-) create mode 100644 test/workers/import_google_analytics_test.exs diff --git a/config/.env.dev b/config/.env.dev index d224a66b0133..787a5cd69d42 100644 --- a/config/.env.dev +++ b/config/.env.dev @@ -12,4 +12,7 @@ SHOW_CITIES=true PADDLE_VENDOR_AUTH_CODE=895e20d4efaec0575bb857f44b183217b332d9592e76e69b8a PADDLE_VENDOR_ID=3942 +GOOGLE_CLIENT_ID=875387135161-l8tp53dpt7fdhdg9m1pc3vl42si95rh0.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=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 + IP_GEOLOCATION_DB=/home/ukutaht/plausible/analytics/city_database.mmdb diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 90707ced33f5..797cadc5e6b1 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -286,7 +286,7 @@ defmodule Plausible.Google.Api do dateRanges: [ %{ # The earliest valid date - startDate: "2017-01-01", + startDate: "2005-01-01", endDate: request.end_date } ], diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index 8e2c42456835..a418cb78e1fb 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -82,7 +82,7 @@ defmodule Plausible.Imported do %{ site_id: site_id, date: format_date(date), - hostname: hostname, + hostname: String.replace_prefix(hostname, "www.", ""), page: page, visitors: visitors, pageviews: pageviews, diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index 49bf3b71bb75..dde2c6b588c6 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -1,3 +1,12 @@ +defmodule Plausible.Site.ImportedData do + use Ecto.Schema + + embedded_schema do + field :end_date, :date + field :source, :string + end +end + defmodule Plausible.Site do use Ecto.Schema import Ecto.Changeset @@ -11,7 +20,8 @@ defmodule Plausible.Site do field :public, :boolean field :locked, :boolean field :has_stats, :boolean - field :imported_source, :string + + embeds_one :imported_data, Plausible.Site.ImportedData, on_replace: :update many_to_many :members, User, join_through: Plausible.Site.Membership has_many :memberships, Plausible.Site.Membership @@ -49,7 +59,16 @@ defmodule Plausible.Site do end def set_imported_source(site, imported_source) do - change(site, imported_source: imported_source) + change(site, + imported_data: %Plausible.Site.ImportedData{ + end_date: Timex.today(), + source: imported_source + } + ) + end + + def remove_imported_data(site) do + change(site, imported_data: nil) end defp clean_domain(changeset) do diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 38eb3f20a9aa..93c282f936e0 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -246,7 +246,7 @@ defmodule Plausible.Stats.Breakdown do {base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q) - with_imported = query.with_imported && site.imported_source + with_imported = query.with_imported && site.imported_data select = if with_imported do diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index cd2edf2b29fc..444dedec6637 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -24,7 +24,7 @@ defmodule Plausible.Stats.Imported do |> Enum.map(fn step -> Map.get(result, step, 0) end) end - def merge_imported(q, %Plausible.Site{imported_source: nil}, _, _, _), do: q + def merge_imported(q, %Plausible.Site{imported_data: nil}, _, _, _), do: q def merge_imported(q, _, %Query{with_imported: false}, _, _), do: q def merge_imported(q, _, _, _, [:events | _]), do: q # GA only has 'source' diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 97b52afe5ddf..cd559fafd88d 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -25,13 +25,13 @@ defmodule PlausibleWeb.Api.StatsController do present_index = present_index_for(site, query, labels) {plot, with_imported, source} = - if query.with_imported && site.imported_source do + if query.with_imported && site.imported_data do # Showing imported data. plot = Stats.Imported.timeseries(site, timeseries_query) |> Enum.zip_with(plot, &(&1 + &2)) - {plot, true, site.imported_source} + {plot, true, site.imported_data.source} else if Enum.any?(query.filters) do # Hiding imported data due to filtering. @@ -39,7 +39,7 @@ defmodule PlausibleWeb.Api.StatsController do {plot, false, ""} else # Hiding imported data either by request or because there is none. - {plot, false, site.imported_source || ""} + {plot, false, (site.imported_data && site.imported_data.source) || ""} end end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 06b2bb5468fb..59e0eddcfecd 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -173,7 +173,7 @@ defmodule PlausibleWeb.SiteController do |> Repo.preload([:custom_domain, :google_auth]) google_profiles = - if is_nil(site.imported_source) and site.google_auth do + if is_nil(site.imported_data) and site.google_auth do Plausible.Google.Api.get_analytics_view_ids(site) end @@ -182,7 +182,7 @@ defmodule PlausibleWeb.SiteController do |> render("settings_general.html", site: site, google_profiles: google_profiles, - imported_from: site.imported_source, + imported_from: site.imported_data && site.imported_data.source, changeset: Plausible.Site.changeset(site, %{}), layout: {PlausibleWeb.LayoutView, "site_settings.html"} ) @@ -643,9 +643,9 @@ defmodule PlausibleWeb.SiteController do |> Repo.preload(:google_auth) cond do - site.imported_source -> + site.imported_data -> conn - |> put_flash(:error, "Data already imported from: #{site.imported_source}") + |> put_flash(:error, "Data already imported from: #{site.imported_data.source}") |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) profile == "" -> @@ -668,11 +668,11 @@ defmodule PlausibleWeb.SiteController do site = conn.assigns[:site] cond do - site.imported_source -> + site.imported_data -> Plausible.Imported.forget(site) site - |> Plausible.Site.set_imported_source(nil) + |> Plausible.Site.remove_imported_data() |> Repo.update!() conn diff --git a/lib/plausible_web/email.ex b/lib/plausible_web/email.ex index 345dd3ca1b5f..2508babdbbf4 100644 --- a/lib/plausible_web/email.ex +++ b/lib/plausible_web/email.ex @@ -280,24 +280,26 @@ defmodule PlausibleWeb.Email do ) end - def import_success(email, site) do + def import_success(user, site) do base_email() - |> to(email) + |> to(user) |> tag("import-success-email") |> subject("Google Analytics data imported for #{site.domain}") |> render("google_analytics_import.html", %{ site: site, link: PlausibleWeb.Endpoint.url() <> "/" <> URI.encode_www_form(site.domain), + user: user, success: true }) end - def import_failure(email, site) do + def import_failure(user, site) do base_email() - |> to(email) + |> to(user) |> tag("import-failure-email") |> subject("Google Analytics import failed for #{site.domain}") |> render("google_analytics_import.html", %{ + user: user, site: site, success: false }) diff --git a/lib/workers/import_google_analytics.ex b/lib/workers/import_google_analytics.ex index 6900d87b588d..a854fc6f75d8 100644 --- a/lib/workers/import_google_analytics.ex +++ b/lib/workers/import_google_analytics.ex @@ -7,20 +7,23 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do unique: [fields: [:args], period: 60] @impl Oban.Worker - def perform(%Oban.Job{args: %{"site_id" => site_id, "profile" => profile}}) do + def perform( + %Oban.Job{args: %{"site_id" => site_id, "profile" => profile}}, + google_api \\ Plausible.Google.Api + ) do site = Repo.get(Plausible.Site, site_id) - |> Repo.preload([:google_auth, :members]) + |> Repo.preload([:google_auth, [memberships: :user]]) - case Plausible.Google.Api.import_analytics(site, profile) do + case google_api.import_analytics(site, profile) do {:ok, _} -> site |> Plausible.Site.set_imported_source("Google Analytics") |> Repo.update!() - Enum.each(site.members, fn member -> - if Enum.member?(member.role, [:owner, :admin]) do - PlausibleWeb.Email.import_success(member.user.email, site) + Enum.each(site.memberships, fn membership -> + if membership.role in [:owner, :admin] do + PlausibleWeb.Email.import_success(membership.user, site) |> Plausible.Mailer.send_email_safe() end end) @@ -28,9 +31,9 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do :ok {:error, error} -> - Enum.each(site.members, fn member -> - if Enum.member?(member.role, [:owner, :admin]) do - PlausibleWeb.Email.import_failure(member.user.email, site) + Enum.each(site.memberships, fn membership -> + if membership.role in [:owner, :admin] do + PlausibleWeb.Email.import_failure(membership.user, site) |> Plausible.Mailer.send_email_safe() end end) diff --git a/priv/ref_inspector/referers.yml b/priv/ref_inspector/referers.yml index a6f88f3c7085..7b9cc1b60096 100644 --- a/priv/ref_inspector/referers.yml +++ b/priv/ref_inspector/referers.yml @@ -184,7 +184,7 @@ email: Rambler: domains: - - mail.rambler.ru + - mail.rambler.ru Seznam Mail: domains: diff --git a/priv/repo/migrations/20211110174617_add_site_imported_source.exs b/priv/repo/migrations/20211110174617_add_site_imported_source.exs index f04c2899a46a..0b30f982dc60 100644 --- a/priv/repo/migrations/20211110174617_add_site_imported_source.exs +++ b/priv/repo/migrations/20211110174617_add_site_imported_source.exs @@ -3,7 +3,7 @@ defmodule Plausible.Repo.Migrations.GoogleAuthImportedSource do def change do alter table(:sites) do - add :imported_source, :string, null: true, default: nil + add :imported_data, :map end end end diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 6a67866054fa..90bf1d66408f 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -6,7 +6,7 @@ defmodule Plausible.ImportedTest do @user_id 123 describe "Parse and import third party data fetched from Google Analytics" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "Visitors data imported from Google Analytics", %{conn: conn, site: site} do populate_stats(site, [ diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs index 6c9d1af3612a..96fc77441b5d 100644 --- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs @@ -3,7 +3,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do import Plausible.TestUtils describe "GET /api/stats/:domain/browsers" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "returns top browsers by unique visitors", %{conn: conn, site: site} do populate_stats(site, [ diff --git a/test/plausible_web/controllers/api/stats_controller/countries_test.exs b/test/plausible_web/controllers/api/stats_controller/countries_test.exs index 4c4255bb7ebc..dd0485b59b53 100644 --- a/test/plausible_web/controllers/api/stats_controller/countries_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/countries_test.exs @@ -3,7 +3,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do import Plausible.TestUtils describe "GET /api/stats/:domain/countries" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "returns top countries by new visitors", %{conn: conn, site: site} do populate_stats(site, [ diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index 0f5dd0194793..ceefa0f58777 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do @user_id 123 describe "GET /api/stats/main-graph - plot" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "displays pageviews for the last 30 minutes in realtime graph", %{conn: conn, site: site} do populate_stats(site, [ diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs index 59d8182d6a59..f0d025ca8bc0 100644 --- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs @@ -3,7 +3,7 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do import Plausible.TestUtils describe "GET /api/stats/:domain/operating_systems" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "returns operating systems by unique visitors", %{conn: conn, site: site} do populate_stats(site, [ diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index fa708720c051..9cacb00cee3a 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do @user_id 123 describe "GET /api/stats/:domain/pages" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "returns top pages by visitors", %{conn: conn, site: site} do populate_stats(site, [ @@ -191,7 +191,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do end describe "GET /api/stats/:domain/entry-pages" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "returns top entry pages by visitors", %{conn: conn, site: site} do populate_stats(site, [ @@ -384,7 +384,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do end describe "GET /api/stats/:domain/exit-pages" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "returns top exit pages by visitors", %{conn: conn, site: site} do populate_stats(site, [ diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs index 97b564b49292..029546cac413 100644 --- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs @@ -3,7 +3,7 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do import Plausible.TestUtils describe "GET /api/stats/:domain/browsers" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "returns screen sizes by new visitors", %{conn: conn, site: site} do populate_stats(site, [ diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index 873e26edd2f0..97f88ff5917c 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -4,7 +4,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do @user_id 123 describe "GET /api/stats/:domain/sources" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "returns top sources by unique user ids", %{conn: conn, site: site} do populate_stats(site, [ @@ -269,7 +269,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do end describe "GET /api/stats/:domain/utm_mediums" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "returns top utm_mediums by unique user ids", %{conn: conn, site: site} do populate_stats(site, [ @@ -353,7 +353,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do end describe "GET /api/stats/:domain/utm_campaigns" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "returns top utm_campaigns by unique user ids", %{conn: conn, site: site} do populate_stats(site, [ @@ -763,7 +763,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do end describe "GET /api/stats/:domain/utm_terms" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "returns top utm_terms by unique user ids", %{conn: conn, site: site} do populate_stats(site, [ @@ -851,7 +851,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do end describe "GET /api/stats/:domain/utm_contents" do - setup [:create_user, :log_in, :create_new_site] + setup [:create_user, :log_in, :create_new_site, :add_imported_data] test "returns top utm_contents by unique user ids", %{conn: conn, site: site} do populate_stats(site, [ diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs index 37361bd56c63..ad4b745efc93 100644 --- a/test/plausible_web/controllers/site_controller_test.exs +++ b/test/plausible_web/controllers/site_controller_test.exs @@ -2,6 +2,7 @@ defmodule PlausibleWeb.SiteControllerTest do use PlausibleWeb.ConnCase use Plausible.Repo use Bamboo.Test + use Oban.Testing, repo: Plausible.Repo import Plausible.TestUtils describe "GET /sites/new" do @@ -718,4 +719,27 @@ defmodule PlausibleWeb.SiteControllerTest do assert Repo.aggregate(Plausible.Site.CustomDomain, :count, :id) == 0 end end + + describe "POST /:website/settings/google-import" do + setup [:create_user, :log_in, :create_new_site] + + test "schedules an import job in Oban", %{conn: conn, site: site} do + post(conn, "/#{site.domain}/settings/google-import", %{"profile" => "123"}) + + assert_enqueued( + worker: Plausible.Workers.ImportGoogleAnalytics, + args: %{"site_id" => site.id, "profile" => "123"} + ) + end + end + + describe "DELETE /:website/settings/:forget_imported" do + setup [:create_user, :log_in, :create_new_site] + + test "removes imported_data field from site", %{conn: conn, site: site} do + delete(conn, "/#{site.domain}/settings/forget-imported") + + assert Repo.reload(site).imported_data == nil + end + end end diff --git a/test/support/factory.ex b/test/support/factory.ex index e2d5a122f659..2978638cb726 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -26,8 +26,7 @@ defmodule Plausible.Factory do %Plausible.Site{ domain: domain, - timezone: "UTC", - imported_source: "Google Analytics" + timezone: "UTC" } end diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index 4f8149575e8e..2cb6ebfa2123 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -11,6 +11,15 @@ defmodule Plausible.TestUtils do {:ok, site: site} end + def add_imported_data(%{site: site}) do + site = + site + |> Plausible.Site.set_imported_source("Google Analytics") + |> Repo.update!() + + {:ok, site: site} + end + def create_new_site(%{user: user}) do site = Factory.insert(:site, members: [user]) {:ok, site: site} diff --git a/test/workers/import_google_analytics_test.exs b/test/workers/import_google_analytics_test.exs new file mode 100644 index 000000000000..8b5773a73f8c --- /dev/null +++ b/test/workers/import_google_analytics_test.exs @@ -0,0 +1,66 @@ +defmodule Plausible.Workers.ImportGoogleAnalyticsTest do + use Plausible.DataCase + use Bamboo.Test + import Double + alias Plausible.Workers.ImportGoogleAnalytics + + test "sets the imported_data field for the site after succesful import" do + user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1)) + site = insert(:site, members: [user], imported_data: nil) + + api_stub = + stub(Plausible.Google.Api, :import_analytics, fn _site, _profile -> + {:ok, nil} + end) + + ImportGoogleAnalytics.perform( + %Oban.Job{args: %{"site_id" => site.id, "profile" => "profile"}}, + api_stub + ) + + refute Repo.reload!(site).imported_data == %Plausible.Site.ImportedData{ + source: "Google Analytics", + end_date: Timex.today() + } + end + + test "sends email to owner after succesful import" do + user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1)) + site = insert(:site, members: [user], imported_data: nil) + + api_stub = + stub(Plausible.Google.Api, :import_analytics, fn _site, _profile -> + {:ok, nil} + end) + + ImportGoogleAnalytics.perform( + %Oban.Job{args: %{"site_id" => site.id, "profile" => "profile"}}, + api_stub + ) + + assert_email_delivered_with( + to: [user], + subject: "Google Analytics data imported for #{site.domain}" + ) + end + + test "sends email to owner after failed import" do + user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1)) + site = insert(:site, members: [user], imported_data: nil) + + api_stub = + stub(Plausible.Google.Api, :import_analytics, fn _site, _profile -> + {:error, "Something went wrong"} + end) + + ImportGoogleAnalytics.perform( + %Oban.Job{args: %{"site_id" => site.id, "profile" => "profile"}}, + api_stub + ) + + assert_email_delivered_with( + to: [user], + subject: "Google Analytics import failed for #{site.domain}" + ) + end +end From 8a0b5321c71fb48e16505a4eb5757df8baa27f2e Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Mon, 7 Mar 2022 13:32:49 -0600 Subject: [PATCH 145/148] Track import progress --- lib/plausible/site/schema.ex | 19 ++++++++++ .../controllers/site_controller.ex | 23 +++++++++--- .../templates/site/settings_general.html.eex | 8 +++-- lib/workers/import_google_analytics.ex | 6 ++-- .../controllers/site_controller_test.exs | 11 ++++++ test/workers/import_google_analytics_test.exs | 36 ++++++++++++++----- 6 files changed, 86 insertions(+), 17 deletions(-) diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index dde2c6b588c6..f79556d04376 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -4,6 +4,7 @@ defmodule Plausible.Site.ImportedData do embedded_schema do field :end_date, :date field :source, :string + field :status, :string end end @@ -58,6 +59,24 @@ defmodule Plausible.Site do change(site, has_stats: has_stats_val) end + def start_import(site, imported_source) do + change(site, + imported_data: %Plausible.Site.ImportedData{ + end_date: Timex.today(), + source: imported_source, + status: "importing" + } + ) + end + + def import_success(site) do + change(site, imported_data: %{status: "ok"}) + end + + def import_failure(site) do + change(site, imported_data: %{status: "error"}) + end + def set_imported_source(site, imported_source) do change(site, imported_data: %Plausible.Site.ImportedData{ diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index 59e0eddcfecd..7ec776fddd68 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -167,13 +167,19 @@ defmodule PlausibleWeb.SiteController do redirect(conn, to: Routes.site_path(conn, :settings_general, website)) end + defp can_trigger_import(site) do + no_import = is_nil(site.imported_data) || site.imported_data.status == "error" + + no_import && site.google_auth + end + def settings_general(conn, _params) do site = conn.assigns[:site] |> Repo.preload([:custom_domain, :google_auth]) google_profiles = - if is_nil(site.imported_data) and site.google_auth do + if can_trigger_import(site) do Plausible.Google.Api.get_analytics_view_ids(site) end @@ -182,7 +188,7 @@ defmodule PlausibleWeb.SiteController do |> render("settings_general.html", site: site, google_profiles: google_profiles, - imported_from: site.imported_data && site.imported_data.source, + imported_data: site.imported_data, changeset: Plausible.Site.changeset(site, %{}), layout: {PlausibleWeb.LayoutView, "site_settings.html"} ) @@ -654,9 +660,16 @@ defmodule PlausibleWeb.SiteController do |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) true -> - %{"site_id" => site.id, "profile" => profile} - |> Plausible.Workers.ImportGoogleAnalytics.new() - |> Oban.insert() + job = + Plausible.Workers.ImportGoogleAnalytics.new(%{ + "site_id" => site.id, + "profile" => profile + }) + + Ecto.Multi.new() + |> Ecto.Multi.update(:update_site, Plausible.Site.start_import(site, "Google Analytics")) + |> Oban.insert(:oban_job, job) + |> Repo.transaction() conn |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") diff --git a/lib/plausible_web/templates/site/settings_general.html.eex b/lib/plausible_web/templates/site/settings_general.html.eex index 184c656978f4..689f4698878d 100644 --- a/lib/plausible_web/templates/site/settings_general.html.eex +++ b/lib/plausible_web/templates/site/settings_general.html.eex @@ -58,14 +58,18 @@ <%= if Keyword.get(Application.get_env(:plausible, :google), :client_id) do %> <%= cond do %> - <% @imported_from -> %> + <% @imported_data && @imported_data.status == "importing" -> %> +
    +
    We are importing data from <%= @imported_data.source %> in the background... You will receive an email when it's completed
    + + <% @imported_data && @imported_data.status == "ok" -> %>

  • Forget Imported Data

    - Removes all data imported from <%= @imported_from %> + Removes all data imported from <%= @imported_data.source %>

    <%= link("Forget imported stats", to: "/#{URI.encode_www_form(@site.domain)}/settings/forget-imported", method: :delete, class: "inline-block mt-4 px-4 py-2 border border-gray-300 dark:border-gray-500 text-sm leading-5 font-medium rounded-md text-red-700 bg-white dark:bg-gray-800 hover:text-red-500 dark:hover:text-red-400 focus:outline-none focus:border-blue-300 focus:ring active:text-red-800 active:bg-gray-50 transition ease-in-out duration-150") %> diff --git a/lib/workers/import_google_analytics.ex b/lib/workers/import_google_analytics.ex index a854fc6f75d8..0e32b8c109a4 100644 --- a/lib/workers/import_google_analytics.ex +++ b/lib/workers/import_google_analytics.ex @@ -17,8 +17,7 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do case google_api.import_analytics(site, profile) do {:ok, _} -> - site - |> Plausible.Site.set_imported_source("Google Analytics") + Plausible.Site.import_success(site) |> Repo.update!() Enum.each(site.memberships, fn membership -> @@ -31,6 +30,9 @@ defmodule Plausible.Workers.ImportGoogleAnalytics do :ok {:error, error} -> + Plausible.Site.import_failure(site) + |> Repo.update!() + Enum.each(site.memberships, fn membership -> if membership.role in [:owner, :admin] do PlausibleWeb.Email.import_failure(membership.user, site) diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs index ad4b745efc93..3d25b8398d68 100644 --- a/test/plausible_web/controllers/site_controller_test.exs +++ b/test/plausible_web/controllers/site_controller_test.exs @@ -723,6 +723,17 @@ defmodule PlausibleWeb.SiteControllerTest do describe "POST /:website/settings/google-import" do setup [:create_user, :log_in, :create_new_site] + test "adds in-progress imported tag to site", %{conn: conn, site: site} do + post(conn, "/#{site.domain}/settings/google-import", %{"profile" => "123"}) + + imported_data = Repo.reload(site).imported_data + + assert imported_data + assert imported_data.source == "Google Analytics" + assert imported_data.end_date == Timex.today() + assert imported_data.status == "importing" + end + test "schedules an import job in Oban", %{conn: conn, site: site} do post(conn, "/#{site.domain}/settings/google-import", %{"profile" => "123"}) diff --git a/test/workers/import_google_analytics_test.exs b/test/workers/import_google_analytics_test.exs index 8b5773a73f8c..d126144b9920 100644 --- a/test/workers/import_google_analytics_test.exs +++ b/test/workers/import_google_analytics_test.exs @@ -4,9 +4,15 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do import Double alias Plausible.Workers.ImportGoogleAnalytics - test "sets the imported_data field for the site after succesful import" do + @imported_data %Plausible.Site.ImportedData{ + end_date: Timex.today(), + source: "Google Analytics", + status: "importing" + } + + test "updates the imported_data field for the site after succesful import" do user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1)) - site = insert(:site, members: [user], imported_data: nil) + site = insert(:site, members: [user], imported_data: @imported_data) api_stub = stub(Plausible.Google.Api, :import_analytics, fn _site, _profile -> @@ -18,15 +24,12 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do api_stub ) - refute Repo.reload!(site).imported_data == %Plausible.Site.ImportedData{ - source: "Google Analytics", - end_date: Timex.today() - } + assert Repo.reload!(site).imported_data.status == "ok" end test "sends email to owner after succesful import" do user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1)) - site = insert(:site, members: [user], imported_data: nil) + site = insert(:site, members: [user], imported_data: @imported_data) api_stub = stub(Plausible.Google.Api, :import_analytics, fn _site, _profile -> @@ -44,9 +47,26 @@ defmodule Plausible.Workers.ImportGoogleAnalyticsTest do ) end + test "updates site record after failed import" do + user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1)) + site = insert(:site, members: [user], imported_data: @imported_data) + + api_stub = + stub(Plausible.Google.Api, :import_analytics, fn _site, _profile -> + {:error, "Something went wrong"} + end) + + ImportGoogleAnalytics.perform( + %Oban.Job{args: %{"site_id" => site.id, "profile" => "profile"}}, + api_stub + ) + + assert Repo.reload!(site).imported_data.status == "error" + end + test "sends email to owner after failed import" do user = insert(:user, trial_expiry_date: Timex.today() |> Timex.shift(days: 1)) - site = insert(:site, members: [user], imported_data: nil) + site = insert(:site, members: [user], imported_data: @imported_data) api_stub = stub(Plausible.Google.Api, :import_analytics, fn _site, _profile -> From a7bc60f5ba97a8aec25a7bf2fe76d245983b599d Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Tue, 8 Mar 2022 11:31:58 -0600 Subject: [PATCH 146/148] Fix month interval with imported data --- lib/plausible/site/schema.ex | 4 +- lib/plausible/stats/breakdown.ex | 2 - lib/plausible/stats/imported.ex | 60 ++++++++++++++----- lib/plausible/stats/timeseries.ex | 34 ++++++----- .../controllers/api/stats_controller.ex | 22 +------ .../api/stats_controller/main_graph_test.exs | 24 +++++++- test/support/factory.ex | 20 +++---- test/support/test_utils.ex | 2 +- 8 files changed, 101 insertions(+), 67 deletions(-) diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index f79556d04376..07de7266e627 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -59,12 +59,12 @@ defmodule Plausible.Site do change(site, has_stats: has_stats_val) end - def start_import(site, imported_source) do + def start_import(site, imported_source, status \\ "importing") do change(site, imported_data: %Plausible.Site.ImportedData{ end_date: Timex.today(), source: imported_source, - status: "importing" + status: status } ) end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 93c282f936e0..791e040e4f0a 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -205,7 +205,6 @@ defmodule Plausible.Stats.Breakdown do |> merge_imported(site, query, property, metrics) |> apply_pagination(pagination) |> ClickhouseRepo.all() - # TODO: migrate schema field to 'os' |> transform_keys(%{operating_system: :os}) end @@ -221,7 +220,6 @@ defmodule Plausible.Stats.Breakdown do |> merge_imported(site, query, property, metrics) |> apply_pagination(pagination) |> ClickhouseRepo.all() - # TODO: migrate schema field to 'os' |> transform_keys(%{operating_system: :os}) end diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index 444dedec6637..d79aca415a28 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -5,23 +5,46 @@ defmodule Plausible.Stats.Imported do @no_ref "Direct / None" - def timeseries(site, query) do - result = + def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{with_imported: false}, _), + do: native_q + + def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{filters: filters}, _) + when length(filters) > 0, + do: native_q + + def merge_imported_timeseries( + native_q, + %Plausible.Site{id: site_id, imported_data: %{status: "ok"}}, + query, + metrics + ) do + imported_q = from(v in "imported_visitors", - group_by: fragment("date"), - where: v.site_id == ^site.id, + where: v.site_id == ^site_id, where: v.date >= ^query.date_range.first and v.date <= ^query.date_range.last, - select: %{ - visitors: sum(v.visitors), - date: fragment("? as date", v.date) - } + select: %{visitors: sum(v.visitors)} ) - |> ClickhouseRepo.all() - |> Enum.map(fn row -> {row[:date], row[:visitors]} end) - |> Map.new() + |> apply_interval(query) + + from(s in Ecto.Query.subquery(native_q), + full_join: i in subquery(imported_q), + on: field(s, :date) == field(i, :date) + ) + |> select_joined_metrics(metrics) + end + + def merge_imported_timeseries(native_q, _site, _query, _metrics), do: native_q - Enum.into(query.date_range, []) - |> Enum.map(fn step -> Map.get(result, step, 0) end) + defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}) do + imported_q + |> group_by([i], fragment("toStartOfMonth(?)", i.date)) + |> select_merge([i], %{date: fragment("toStartOfMonth(?)", i.date)}) + end + + defp apply_interval(imported_q, _query) do + imported_q + |> group_by([i], i.date) + |> select_merge([i], %{date: i.date}) end def merge_imported(q, %Plausible.Site{imported_data: nil}, _, _, _), do: q @@ -185,6 +208,7 @@ defmodule Plausible.Stats.Imported do on: field(s, ^dim) == field(i, ^dim) ) |> select_joined_metrics(metrics) + |> apply_order_by(metrics) case dim do :source -> @@ -354,9 +378,6 @@ defmodule Plausible.Stats.Imported do |> select_merge([s, i], %{ :visitors => fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) }) - |> order_by([s, i], - desc: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors) - ) |> select_joined_metrics(rest) end @@ -410,4 +431,11 @@ defmodule Plausible.Stats.Imported do q |> select_joined_metrics(rest) end + + defp apply_order_by(q, [:visitors | rest]) do + order_by(q, [s, i], desc: fragment("coalesce(?, 0) + coalesce(?, 0)", s.visitors, i.visitors)) + |> apply_order_by(rest) + end + + defp apply_order_by(q, _), do: q end diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex index 90fbd8a3ebe5..694a7dd21ace 100644 --- a/lib/plausible/stats/timeseries.ex +++ b/lib/plausible/stats/timeseries.ex @@ -28,27 +28,25 @@ defmodule Plausible.Stats.Timeseries do end) end + defp events_timeseries(_, _, []), do: [] + defp events_timeseries(site, query, metrics) do - from(e in base_event_query(site, query), - group_by: fragment("date"), - order_by: fragment("date"), - select: %{} - ) + from(e in base_event_query(site, query), select: %{}) |> select_bucket(site, query) |> select_event_metrics(metrics) + |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics) |> ClickhouseRepo.all() end + defp sessions_timeseries(_, _, []), do: [] + defp sessions_timeseries(site, query, metrics) do query = Query.treat_page_filter_as_entry_page(query) - from(e in query_sessions(site, query), - group_by: fragment("date"), - order_by: fragment("date"), - select: %{} - ) + from(e in query_sessions(site, query), select: %{}) |> select_bucket(site, query) |> select_session_metrics(metrics) + |> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics) |> ClickhouseRepo.all() end @@ -81,8 +79,10 @@ defmodule Plausible.Stats.Timeseries do def select_bucket(q, site, %Query{interval: "month"}) do from( e in q, + group_by: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone), + order_by: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone), select_merge: %{ - date: fragment("toStartOfMonth(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone) + date: fragment("toStartOfMonth(toTimeZone(?, ?))", e.timestamp, ^site.timezone) } ) end @@ -90,8 +90,10 @@ defmodule Plausible.Stats.Timeseries do def select_bucket(q, site, %Query{interval: "date"}) do from( e in q, + group_by: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone), + order_by: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone), select_merge: %{ - date: fragment("toDate(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone) + date: fragment("toDate(toTimeZone(?, ?))", e.timestamp, ^site.timezone) } ) end @@ -99,8 +101,10 @@ defmodule Plausible.Stats.Timeseries do def select_bucket(q, site, %Query{interval: "hour"}) do from( e in q, + group_by: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone), + order_by: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone), select_merge: %{ - date: fragment("toStartOfHour(toTimeZone(?, ?)) as date", e.timestamp, ^site.timezone) + date: fragment("toStartOfHour(toTimeZone(?, ?))", e.timestamp, ^site.timezone) } ) end @@ -108,8 +112,10 @@ defmodule Plausible.Stats.Timeseries do def select_bucket(q, _site, %Query{interval: "minute"}) do from( e in q, + group_by: fragment("dateDiff('minute', now(), ?)", e.timestamp), + order_by: fragment("dateDiff('minute', now(), ?)", e.timestamp), select_merge: %{ - date: fragment("dateDiff('minute', now(), ?) as date", e.timestamp) + date: fragment("dateDiff('minute', now(), ?)", e.timestamp) } ) end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index cd559fafd88d..f7716c5c09c9 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -23,25 +23,7 @@ defmodule PlausibleWeb.Api.StatsController do plot = Enum.map(timeseries_result, fn row -> row[:visitors] end) labels = Enum.map(timeseries_result, fn row -> row[:date] end) present_index = present_index_for(site, query, labels) - - {plot, with_imported, source} = - if query.with_imported && site.imported_data do - # Showing imported data. - plot = - Stats.Imported.timeseries(site, timeseries_query) - |> Enum.zip_with(plot, &(&1 + &2)) - - {plot, true, site.imported_data.source} - else - if Enum.any?(query.filters) do - # Hiding imported data due to filtering. - # Setting source to "" hides imported indicator from main graph. - {plot, false, ""} - else - # Hiding imported data either by request or because there is none. - {plot, false, (site.imported_data && site.imported_data.source) || ""} - end - end + with_imported = query.with_imported && site.imported_data && Enum.empty?(query.filters) json(conn, %{ plot: plot, @@ -51,7 +33,7 @@ defmodule PlausibleWeb.Api.StatsController do interval: query.interval, sample_percent: sample_percent, with_imported: with_imported, - imported_source: source + imported_source: site.imported_data && site.imported_data.source }) end diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index ceefa0f58777..0d8fa0216e5a 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -107,7 +107,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do "/api/stats/#{site.domain}/main-graph?period=month&date=2021-01-01&with_imported=true&filters=#{filters}" ) - assert %{"plot" => plot, "imported_source" => ""} = json_response(conn, 200) + assert %{"plot" => plot} = json_response(conn, 200) assert Enum.count(plot) == 31 assert List.first(plot) == 1 @@ -115,6 +115,28 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do assert Enum.sum(plot) == 2 end + test "displays visitors for 6 months with imported data", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, timestamp: ~N[2021-06-30 00:00:00]), + build(:imported_visitors, date: ~D[2021-01-01]), + build(:imported_visitors, date: ~D[2021-06-30]) + ]) + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=6mo&date=2021-06-30&with_imported=true" + ) + + assert %{"plot" => plot} = json_response(conn, 200) + + assert Enum.count(plot) == 6 + assert List.first(plot) == 2 + assert List.last(plot) == 2 + assert Enum.sum(plot) == 4 + end + # TODO: missing 6, 12 months, 30 days end diff --git a/test/support/factory.ex b/test/support/factory.ex index 2978638cb726..747d888a66ac 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -180,12 +180,10 @@ defmodule Plausible.Factory do } end - @today Timex.today() |> Timex.to_date() - def imported_visitors_factory do %{ table: "imported_visitors", - date: @today, + date: Timex.today(), visitors: 1, pageviews: 1, bounces: 0, @@ -197,7 +195,7 @@ defmodule Plausible.Factory do def imported_sources_factory do %{ table: "imported_sources", - date: @today, + date: Timex.today(), source: "", visitors: 1, visits: 1, @@ -209,7 +207,7 @@ defmodule Plausible.Factory do def imported_pages_factory do %{ table: "imported_pages", - date: @today, + date: Timex.today(), page: "", visitors: 1, pageviews: 1, @@ -220,7 +218,7 @@ defmodule Plausible.Factory do def imported_entry_pages_factory do %{ table: "imported_entry_pages", - date: @today, + date: Timex.today(), entry_page: "", visitors: 1, entrances: 1, @@ -232,7 +230,7 @@ defmodule Plausible.Factory do def imported_exit_pages_factory do %{ table: "imported_exit_pages", - date: @today, + date: Timex.today(), exit_page: "", visitors: 1, exits: 1 @@ -242,7 +240,7 @@ defmodule Plausible.Factory do def imported_locations_factory do %{ table: "imported_locations", - date: @today, + date: Timex.today(), country: "", region: "", city: 0, @@ -256,7 +254,7 @@ defmodule Plausible.Factory do def imported_devices_factory do %{ table: "imported_devices", - date: @today, + date: Timex.today(), device: "", visitors: 1, visits: 1, @@ -268,7 +266,7 @@ defmodule Plausible.Factory do def imported_browsers_factory do %{ table: "imported_browsers", - date: @today, + date: Timex.today(), browser: "", visitors: 1, visits: 1, @@ -280,7 +278,7 @@ defmodule Plausible.Factory do def imported_operating_systems_factory do %{ table: "imported_operating_systems", - date: @today, + date: Timex.today(), operating_system: "", visitors: 1, visits: 1, diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index 2cb6ebfa2123..e0fdeb094664 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -14,7 +14,7 @@ defmodule Plausible.TestUtils do def add_imported_data(%{site: site}) do site = site - |> Plausible.Site.set_imported_source("Google Analytics") + |> Plausible.Site.start_import("Google Analytics", "ok") |> Repo.update!() {:ok, site: site} From 8ddbfb4b5fc185632b529ada92f1ae8c6c8f246c Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Tue, 8 Mar 2022 12:57:38 -0600 Subject: [PATCH 147/148] Do not JOIN when imported date range has no overlap --- lib/plausible/stats/breakdown.ex | 6 +- lib/plausible/stats/imported.ex | 15 +--- lib/plausible/stats/query.ex | 81 +++++++++++-------- .../api/external_stats_controller.ex | 8 +- .../controllers/api/stats_controller.ex | 51 ++++++------ .../controllers/stats_controller.ex | 2 +- lib/workers/send_email_report.ex | 4 +- lib/workers/spike_notifier.ex | 2 +- test/plausible/stats/query_test.exs | 22 ++--- 9 files changed, 96 insertions(+), 95 deletions(-) diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 791e040e4f0a..c9c0a77f7723 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -244,10 +244,8 @@ defmodule Plausible.Stats.Breakdown do {base_query_raw, base_query_raw_params} = ClickhouseRepo.to_sql(:all, q) - with_imported = query.with_imported && site.imported_data - select = - if with_imported do + if query.include_imported do "sum(td), count(case when p2 != p then 1 end)" else "round(sum(td)/count(case when p2 != p then 1 end))" @@ -275,7 +273,7 @@ defmodule Plausible.Stats.Breakdown do {:ok, res} = ClickhouseRepo.query(time_query, base_query_raw_params ++ [pages]) - if with_imported do + if query.include_imported do # Imported page views have pre-calculated values res = res.rows diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex index d79aca415a28..f39e8a27be57 100644 --- a/lib/plausible/stats/imported.ex +++ b/lib/plausible/stats/imported.ex @@ -5,22 +5,18 @@ defmodule Plausible.Stats.Imported do @no_ref "Direct / None" - def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{with_imported: false}, _), + def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _), do: native_q - def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{filters: filters}, _) - when length(filters) > 0, - do: native_q - def merge_imported_timeseries( native_q, - %Plausible.Site{id: site_id, imported_data: %{status: "ok"}}, + site, query, metrics ) do imported_q = from(v in "imported_visitors", - where: v.site_id == ^site_id, + where: v.site_id == ^site.id, where: v.date >= ^query.date_range.first and v.date <= ^query.date_range.last, select: %{visitors: sum(v.visitors)} ) @@ -33,8 +29,6 @@ defmodule Plausible.Stats.Imported do |> select_joined_metrics(metrics) end - def merge_imported_timeseries(native_q, _site, _query, _metrics), do: native_q - defp apply_interval(imported_q, %Plausible.Stats.Query{interval: "month"}) do imported_q |> group_by([i], fragment("toStartOfMonth(?)", i.date)) @@ -47,8 +41,7 @@ defmodule Plausible.Stats.Imported do |> select_merge([i], %{date: i.date}) end - def merge_imported(q, %Plausible.Site{imported_data: nil}, _, _, _), do: q - def merge_imported(q, _, %Query{with_imported: false}, _, _), do: q + def merge_imported(q, _, %Query{include_imported: false}, _, _), do: q def merge_imported(q, _, _, _, [:events | _]), do: q # GA only has 'source' def merge_imported(q, _, _, "utm_source", _), do: q diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index f26dc688d149..525364f8f039 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -4,7 +4,7 @@ defmodule Plausible.Stats.Query do period: nil, filters: %{}, sample_threshold: 20_000_000, - with_imported: true + include_imported: false @default_sample_threshold 20_000_000 @@ -38,8 +38,8 @@ defmodule Plausible.Stats.Query do Map.put(query, :date_range, Date.range(new_first, new_last)) end - def from(tz, %{"period" => "realtime"} = params) do - date = today(tz) + def from(site, %{"period" => "realtime"} = params) do + date = today(site.timezone) %__MODULE__{ period: "realtime", @@ -47,25 +47,25 @@ defmodule Plausible.Stats.Query do date_range: Date.range(date, date), filters: parse_filters(params), sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: false + include_imported: false } end - def from(tz, %{"period" => "day"} = params) do - date = parse_single_date(tz, params) + def from(site, %{"period" => "day"} = params) do + date = parse_single_date(site.timezone, params) %__MODULE__{ period: "day", date_range: Date.range(date, date), interval: "hour", filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(params) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } + |> maybe_include_imported(site, params) end - def from(tz, %{"period" => "7d"} = params) do - end_date = parse_single_date(tz, params) + def from(site, %{"period" => "7d"} = params) do + end_date = parse_single_date(site.timezone, params) start_date = end_date |> Timex.shift(days: -6) %__MODULE__{ @@ -73,13 +73,13 @@ defmodule Plausible.Stats.Query do date_range: Date.range(start_date, end_date), interval: "date", filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(params) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } + |> maybe_include_imported(site, params) end - def from(tz, %{"period" => "30d"} = params) do - end_date = parse_single_date(tz, params) + def from(site, %{"period" => "30d"} = params) do + end_date = parse_single_date(site.timezone, params) start_date = end_date |> Timex.shift(days: -30) %__MODULE__{ @@ -87,13 +87,13 @@ defmodule Plausible.Stats.Query do date_range: Date.range(start_date, end_date), interval: "date", filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(params) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } + |> maybe_include_imported(site, params) end - def from(tz, %{"period" => "month"} = params) do - date = parse_single_date(tz, params) + def from(site, %{"period" => "month"} = params) do + date = parse_single_date(site.timezone, params) start_date = Timex.beginning_of_month(date) end_date = Timex.end_of_month(date) @@ -103,14 +103,14 @@ defmodule Plausible.Stats.Query do date_range: Date.range(start_date, end_date), interval: "date", filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(params) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } + |> maybe_include_imported(site, params) end - def from(tz, %{"period" => "6mo"} = params) do + def from(site, %{"period" => "6mo"} = params) do end_date = - parse_single_date(tz, params) + parse_single_date(site.timezone, params) |> Timex.end_of_month() start_date = @@ -122,14 +122,14 @@ defmodule Plausible.Stats.Query do date_range: Date.range(start_date, end_date), interval: Map.get(params, "interval", "month"), filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(params) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } + |> maybe_include_imported(site, params) end - def from(tz, %{"period" => "12mo"} = params) do + def from(site, %{"period" => "12mo"} = params) do end_date = - parse_single_date(tz, params) + parse_single_date(site.timezone, params) |> Timex.end_of_month() start_date = @@ -141,22 +141,22 @@ defmodule Plausible.Stats.Query do date_range: Date.range(start_date, end_date), interval: Map.get(params, "interval", "month"), filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(params) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } + |> maybe_include_imported(site, params) end - def from(tz, %{"period" => "custom", "from" => from, "to" => to} = params) do + def from(site, %{"period" => "custom", "from" => from, "to" => to} = params) do new_params = params |> Map.delete("from") |> Map.delete("to") |> Map.put("date", Enum.join([from, to], ",")) - from(tz, new_params) + from(site, new_params) end - def from(_tz, %{"period" => "custom", "date" => date} = params) do + def from(site, %{"period" => "custom", "date" => date} = params) do [from, to] = String.split(date, ",") from_date = Date.from_iso8601!(String.trim(from)) to_date = Date.from_iso8601!(String.trim(to)) @@ -166,9 +166,9 @@ defmodule Plausible.Stats.Query do date_range: Date.range(from_date, to_date), interval: Map.get(params, "interval", "date"), filters: parse_filters(params), - sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold), - with_imported: include_imported(params) + sample_threshold: Map.get(params, "sample_threshold", @default_sample_threshold) } + |> maybe_include_imported(site, params) end def from(tz, params) do @@ -257,7 +257,18 @@ defmodule Plausible.Stats.Query do defp parse_goal_filter("Visit " <> page), do: {:is, :page, page} defp parse_goal_filter(event), do: {:is, :event, event} - defp include_imported(params) do - params["filters"] in [nil, "{}"] && params["with_imported"] == "true" + defp maybe_include_imported(query, site, params) do + imported_data_requested = params["with_imported"] == "true" + has_imported_data = site.imported_data && site.imported_data.status == "ok" + + date_range_overlaps = + has_imported_data && !Timex.after?(query.date_range.first, site.imported_data.end_date) + + no_filters_applied = Enum.empty?(query.filters) + + include_imported = + imported_data_requested && has_imported_data && date_range_overlaps && no_filters_applied + + %{query | include_imported: !!include_imported} end end diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index 3e52e836209c..554a1e50919e 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -6,7 +6,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do def realtime_visitors(conn, _params) do site = conn.assigns[:site] - query = Query.from(site.timezone, %{"period" => "realtime"}) + query = Query.from(site, %{"period" => "realtime"}) json(conn, Plausible.Stats.Clickhouse.current_visitors(site, query)) end @@ -16,7 +16,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do with :ok <- validate_period(params), :ok <- validate_date(params), - query <- Query.from(site.timezone, params), + query <- Query.from(site, params), {:ok, metrics} <- parse_metrics(params, nil, query) do results = if params["compare"] == "previous_period" do @@ -61,7 +61,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do with :ok <- validate_period(params), :ok <- validate_date(params), {:ok, property} <- validate_property(params), - query <- Query.from(site.timezone, params), + query <- Query.from(site, params), {:ok, metrics} <- parse_metrics(params, property, query) do limit = String.to_integer(Map.get(params, "limit", "100")) page = String.to_integer(Map.get(params, "page", "1")) @@ -144,7 +144,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do with :ok <- validate_period(params), :ok <- validate_date(params), :ok <- validate_interval(params), - query <- Query.from(site.timezone, params), + query <- Query.from(site, params), {:ok, metrics} <- parse_metrics(params, nil, query) do graph = Plausible.Stats.timeseries(site, query, metrics) metrics = metrics ++ [:date] diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index f7716c5c09c9..943aa98e331b 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -7,7 +7,7 @@ defmodule PlausibleWeb.Api.StatsController do def main_graph(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() timeseries_query = if query.period == "realtime" do @@ -23,7 +23,6 @@ defmodule PlausibleWeb.Api.StatsController do plot = Enum.map(timeseries_result, fn row -> row[:visitors] end) labels = Enum.map(timeseries_result, fn row -> row[:date] end) present_index = present_index_for(site, query, labels) - with_imported = query.with_imported && site.imported_data && Enum.empty?(query.filters) json(conn, %{ plot: plot, @@ -32,7 +31,7 @@ defmodule PlausibleWeb.Api.StatsController do top_stats: top_stats, interval: query.interval, sample_percent: sample_percent, - with_imported: with_imported, + with_imported: query.include_imported, imported_source: site.imported_data && site.imported_data.source }) end @@ -205,7 +204,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = - Query.from(site.timezone, params) + Query.from(site, params) |> Filters.add_prefix() |> maybe_hide_noref("visit:source", params) @@ -236,7 +235,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = - Query.from(site.timezone, params) + Query.from(site, params) |> Filters.add_prefix() |> maybe_hide_noref("visit:utm_medium", params) @@ -266,7 +265,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = - Query.from(site.timezone, params) + Query.from(site, params) |> Filters.add_prefix() |> maybe_hide_noref("visit:utm_campaign", params) @@ -296,7 +295,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = - Query.from(site.timezone, params) + Query.from(site, params) |> Filters.add_prefix() |> maybe_hide_noref("visit:utm_content", params) @@ -325,7 +324,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = - Query.from(site.timezone, params) + Query.from(site, params) |> Filters.add_prefix() |> maybe_hide_noref("visit:utm_term", params) @@ -354,7 +353,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = - Query.from(site.timezone, params) + Query.from(site, params) |> Filters.add_prefix() |> maybe_hide_noref("visit:utm_source", params) @@ -384,7 +383,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] |> Repo.preload(:google_auth) query = - Query.from(site.timezone, params) + Query.from(site, params) |> Query.put_filter("source", "Google") |> Filters.add_prefix() @@ -414,7 +413,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = - Query.from(site.timezone, params) + Query.from(site, params) |> Query.put_filter("source", referrer) |> Filters.add_prefix() @@ -435,7 +434,7 @@ defmodule PlausibleWeb.Api.StatsController do def pages(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() metrics = if params["detailed"], @@ -464,7 +463,7 @@ defmodule PlausibleWeb.Api.StatsController do def entry_pages(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() pagination = parse_pagination(params) metrics = [:visitors, :visits, :visit_duration] @@ -492,7 +491,7 @@ defmodule PlausibleWeb.Api.StatsController do def exit_pages(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() {limit, page} = parse_pagination(params) metrics = [:visitors, :visits] @@ -548,7 +547,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = - Query.from(site.timezone, params) + Query.from(site, params) |> Filters.add_prefix() |> Query.put_filter("visit:country", {:is_not, "\0\0"}) @@ -605,7 +604,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = - Query.from(site.timezone, params) + Query.from(site, params) |> Filters.add_prefix() |> Query.put_filter("visit:region", {:is_not, ""}) @@ -643,7 +642,7 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] query = - Query.from(site.timezone, params) + Query.from(site, params) |> Filters.add_prefix() |> Query.put_filter("visit:city", {:is_not, 0}) @@ -684,7 +683,7 @@ defmodule PlausibleWeb.Api.StatsController do def browsers(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() pagination = parse_pagination(params) browsers = @@ -708,7 +707,7 @@ defmodule PlausibleWeb.Api.StatsController do def browser_versions(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() pagination = parse_pagination(params) versions = @@ -722,7 +721,7 @@ defmodule PlausibleWeb.Api.StatsController do def operating_systems(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() pagination = parse_pagination(params) systems = @@ -746,7 +745,7 @@ defmodule PlausibleWeb.Api.StatsController do def operating_system_versions(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() pagination = parse_pagination(params) versions = @@ -760,7 +759,7 @@ defmodule PlausibleWeb.Api.StatsController do def screen_sizes(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() pagination = parse_pagination(params) sizes = @@ -792,7 +791,7 @@ defmodule PlausibleWeb.Api.StatsController do def conversions(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() query = if query.period == "realtime" do @@ -834,7 +833,7 @@ defmodule PlausibleWeb.Api.StatsController do def prop_breakdown(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() pagination = parse_pagination(params) total_q = Query.remove_goal(query) @@ -867,7 +866,7 @@ defmodule PlausibleWeb.Api.StatsController do def all_props_breakdown(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() prop_names = if query.filters["event:goal"] do @@ -903,7 +902,7 @@ defmodule PlausibleWeb.Api.StatsController do def filter_suggestions(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() json(conn, Stats.filter_suggestions(site, query, params["filter_name"], params["q"])) end diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index ae11af66a25a..e925904e6a04 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -48,7 +48,7 @@ defmodule PlausibleWeb.StatsController do """ def csv_export(conn, params) do site = conn.assigns[:site] - query = Query.from(site.timezone, params) |> Filters.add_prefix() + query = Query.from(site, params) |> Filters.add_prefix() metrics = [:visitors, :pageviews, :bounce_rate, :visit_duration] graph = Plausible.Stats.timeseries(site, query, metrics) diff --git a/lib/workers/send_email_report.ex b/lib/workers/send_email_report.ex index 10be2e5f0dc0..5a7eb477b9bc 100644 --- a/lib/workers/send_email_report.ex +++ b/lib/workers/send_email_report.ex @@ -9,7 +9,7 @@ defmodule Plausible.Workers.SendEmailReport do site = Repo.get(Plausible.Site, site_id) |> Repo.preload(:weekly_report) today = Timex.now(site.timezone) |> DateTime.to_date() date = Timex.shift(today, weeks: -1) |> Timex.end_of_week() |> Date.to_iso8601() - query = Query.from(site.timezone, %{"period" => "7d", "date" => date}) + query = Query.from(site, %{"period" => "7d", "date" => date}) for email <- site.weekly_report.recipients do unsubscribe_link = @@ -32,7 +32,7 @@ defmodule Plausible.Workers.SendEmailReport do |> Timex.beginning_of_month() query = - Query.from(site.timezone, %{ + Query.from(site, %{ "period" => "month", "date" => Timex.format!(last_month, "{ISOdate}") }) diff --git a/lib/workers/spike_notifier.ex b/lib/workers/spike_notifier.ex index a336b34e5d47..52cde6cf1a7f 100644 --- a/lib/workers/spike_notifier.ex +++ b/lib/workers/spike_notifier.ex @@ -19,7 +19,7 @@ defmodule Plausible.Workers.SpikeNotifier do ) for notification <- notifications do - query = Query.from(notification.site.timezone, %{"period" => "realtime"}) + query = Query.from(notification.site, %{"period" => "realtime"}) current_visitors = clickhouse.current_visitors(notification.site, query) if current_visitors >= notification.threshold do diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index 71830ea8b2cb..061989db248f 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -2,10 +2,10 @@ defmodule Plausible.Stats.QueryTest do use ExUnit.Case, async: true alias Plausible.Stats.Query - @tz "UTC" + @site %Plausible.Site{timezone: "UTC"} test "parses day format" do - q = Query.from(@tz, %{"period" => "day", "date" => "2019-01-01"}) + q = Query.from(@site, %{"period" => "day", "date" => "2019-01-01"}) assert q.date_range.first == ~D[2019-01-01] assert q.date_range.last == ~D[2019-01-01] @@ -13,7 +13,7 @@ defmodule Plausible.Stats.QueryTest do end test "day fromat defaults to today" do - q = Query.from(@tz, %{"period" => "day"}) + q = Query.from(@site, %{"period" => "day"}) assert q.date_range.first == Timex.today() assert q.date_range.last == Timex.today() @@ -21,7 +21,7 @@ defmodule Plausible.Stats.QueryTest do end test "parses realtime format" do - q = Query.from(@tz, %{"period" => "realtime"}) + q = Query.from(@site, %{"period" => "realtime"}) assert q.date_range.first == Timex.today() assert q.date_range.last == Timex.today() @@ -29,7 +29,7 @@ defmodule Plausible.Stats.QueryTest do end test "parses month format" do - q = Query.from(@tz, %{"period" => "month", "date" => "2019-01-01"}) + q = Query.from(@site, %{"period" => "month", "date" => "2019-01-01"}) assert q.date_range.first == ~D[2019-01-01] assert q.date_range.last == ~D[2019-01-31] @@ -37,7 +37,7 @@ defmodule Plausible.Stats.QueryTest do end test "parses 6 month format" do - q = Query.from(@tz, %{"period" => "6mo"}) + q = Query.from(@site, %{"period" => "6mo"}) assert q.date_range.first == Timex.shift(Timex.today(), months: -5) |> Timex.beginning_of_month() @@ -47,7 +47,7 @@ defmodule Plausible.Stats.QueryTest do end test "parses 12 month format" do - q = Query.from(@tz, %{"period" => "12mo"}) + q = Query.from(@site, %{"period" => "12mo"}) assert q.date_range.first == Timex.shift(Timex.today(), months: -11) |> Timex.beginning_of_month() @@ -57,11 +57,11 @@ defmodule Plausible.Stats.QueryTest do end test "defaults to 30 days format" do - assert Query.from(@tz, %{}) == Query.from(@tz, %{"period" => "30d"}) + assert Query.from(@site, %{}) == Query.from(@site, %{"period" => "30d"}) end test "parses custom format" do - q = Query.from(@tz, %{"period" => "custom", "from" => "2019-01-01", "to" => "2019-01-15"}) + q = Query.from(@site, %{"period" => "custom", "from" => "2019-01-01", "to" => "2019-01-15"}) assert q.date_range.first == ~D[2019-01-01] assert q.date_range.last == ~D[2019-01-15] @@ -71,14 +71,14 @@ defmodule Plausible.Stats.QueryTest do describe "filters" do test "parses goal filter" do filters = Jason.encode!(%{"goal" => "Signup"}) - q = Query.from(@tz, %{"period" => "6mo", "filters" => filters}) + q = Query.from(@site, %{"period" => "6mo", "filters" => filters}) assert q.filters["goal"] == "Signup" end test "parses source filter" do filters = Jason.encode!(%{"source" => "Twitter"}) - q = Query.from(@tz, %{"period" => "6mo", "filters" => filters}) + q = Query.from(@site, %{"period" => "6mo", "filters" => filters}) assert q.filters["source"] == "Twitter" end From bd7e40b7b2a290667cb0ef32f8bab199914cc729 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Thu, 10 Mar 2022 14:52:39 -0600 Subject: [PATCH 148/148] Fix time on page using exits --- lib/plausible/google/api.ex | 2 +- lib/plausible/imported/site.ex | 4 +++- lib/plausible/stats/breakdown.ex | 7 ++++--- .../20211112130238_create_imported_tables.exs | 1 + test/plausible/imported/imported_test.exs | 10 +++++----- test/support/factory.ex | 1 + 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 797cadc5e6b1..44372da25048 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -203,7 +203,7 @@ defmodule Plausible.Google.Api do { "imported_pages", ["ga:date", "ga:hostname", "ga:pagePath"], - ["ga:users", "ga:pageviews", "ga:timeOnPage"] + ["ga:users", "ga:pageviews", "ga:exits", "ga:timeOnPage"] }, { "imported_entry_pages", diff --git a/lib/plausible/imported/site.ex b/lib/plausible/imported/site.ex index a418cb78e1fb..d137fb667656 100644 --- a/lib/plausible/imported/site.ex +++ b/lib/plausible/imported/site.ex @@ -72,11 +72,12 @@ defmodule Plausible.Imported do defp new_from_google_analytics(site_id, "imported_pages", %{ "dimensions" => [date, hostname, page], - "metrics" => [%{"values" => [visitors, pageviews, time_on_page]}] + "metrics" => [%{"values" => [visitors, pageviews, exits, time_on_page]}] }) do page = URI.parse(page).path {visitors, ""} = Integer.parse(visitors) {pageviews, ""} = Integer.parse(pageviews) + {exits, ""} = Integer.parse(exits) {time_on_page, _} = Integer.parse(time_on_page) %{ @@ -86,6 +87,7 @@ defmodule Plausible.Imported do page: page, visitors: visitors, pageviews: pageviews, + exits: exits, time_on_page: time_on_page } end diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index c9c0a77f7723..ccef6a49a72d 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -288,7 +288,7 @@ defmodule Plausible.Stats.Breakdown do where: i.page in ^pages, select: %{ page: i.page, - pageviews: sum(i.pageviews), + pageviews: fragment("sum(?) - sum(?)", i.pageviews, i.exits), time_on_page: sum(i.time_on_page) } ) @@ -297,8 +297,9 @@ defmodule Plausible.Stats.Breakdown do {restime, resviews} = Map.get(res, page, {0, 0}) Map.put(res, page, {restime + time, resviews + pageviews}) end) - |> Enum.map(fn {page, {time, pageviews}} -> - {page, time / pageviews} + |> Enum.map(fn + {page, {_, 0}} -> {page, nil} + {page, {time, pageviews}} -> {page, time / pageviews} end) |> Enum.into(%{}) else diff --git a/priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs b/priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs index 5abc84a957bf..33bc8b23c4d4 100644 --- a/priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs +++ b/priv/clickhouse_repo/migrations/20211112130238_create_imported_tables.exs @@ -39,6 +39,7 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do add(:page, :string) add(:visitors, :UInt64) add(:pageviews, :UInt64) + add(:exits, :UInt64) add(:time_on_page, :UInt64) end diff --git a/test/plausible/imported/imported_test.exs b/test/plausible/imported/imported_test.exs index 90bf1d66408f..d211d2215e0e 100644 --- a/test/plausible/imported/imported_test.exs +++ b/test/plausible/imported/imported_test.exs @@ -325,15 +325,15 @@ defmodule Plausible.ImportedTest do [ %{ "dimensions" => ["20210101", "host-a.com", "/"], - "metrics" => [%{"values" => ["1", "1", "700"]}] + "metrics" => [%{"values" => ["1", "1", "0", "700"]}] }, %{ "dimensions" => ["20210101", "host-b.com", "/some-other-page"], - "metrics" => [%{"values" => ["1", "1", "60"]}] + "metrics" => [%{"values" => ["1", "2", "1", "60"]}] }, %{ "dimensions" => ["20210101", "host-b.com", "/some-other-page?wat=wot"], - "metrics" => [%{"values" => ["1", "1", "60"]}] + "metrics" => [%{"values" => ["1", "1", "0", "60"]}] } ], site.id, @@ -363,7 +363,7 @@ defmodule Plausible.ImportedTest do "bounce_rate" => nil, "time_on_page" => 60, "visitors" => 3, - "pageviews" => 3, + "pageviews" => 4, "name" => "/some-other-page" }, %{ @@ -403,7 +403,7 @@ defmodule Plausible.ImportedTest do [ %{ "dimensions" => ["20210101", "host-a.com", "/page2"], - "metrics" => [%{"values" => ["2", "4", "10"]}] + "metrics" => [%{"values" => ["2", "4", "0", "10"]}] } ], site.id, diff --git a/test/support/factory.ex b/test/support/factory.ex index 747d888a66ac..30ca268069f1 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -211,6 +211,7 @@ defmodule Plausible.Factory do page: "", visitors: 1, pageviews: 1, + exits: 0, time_on_page: 10 } end