From dce165f4e0441d077bda161c2d059d9704f4139e Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Fri, 28 May 2021 15:30:08 +0300 Subject: [PATCH 1/2] Support IN filter expression --- lib/plausible/stats/base.ex | 72 ++++---- lib/plausible/stats/breakdown.ex | 4 +- lib/plausible/stats/query.ex | 13 +- mix.exs | 2 +- .../breakdown_test.exs | 169 ++++++++++++++++++ 5 files changed, 222 insertions(+), 38 deletions(-) diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 68a7fbefe0da..311766b83c40 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -47,24 +47,24 @@ defmodule Plausible.Stats.Base do ) q = - if query.filters["event:page"] do - page = query.filters["event:page"] - from(e in q, where: e.pathname == ^page) - else - q + case query.filters["event:page"] do + {:is, page} -> from(e in q, where: e.pathname == ^page) + {:member, list} -> from(e in q, where: e.pathname in ^list) + _ -> q end q = - if query.filters["event:name"] do - name = query.filters["event:name"] - from(e in q, where: e.name == ^name) - else - q + case query.filters["event:name"] do + {:is, name} -> from(e in q, where: e.name == ^name) + {:member, list} -> from(e in q, where: e.name in ^list) + _ -> q end Enum.reduce(query.filters, q, fn {filter_key, filter_value}, query -> case filter_key do "event:props:" <> prop_name -> + filter_value = elem(filter_value, 1) + if filter_value == "(none)" do from( e in query, @@ -85,6 +85,14 @@ defmodule Plausible.Stats.Base do end) end + @api_prop_name_to_db %{ + "source" => "referrer_source", + "device" => "screen_size", + "os" => "operating_system", + "os_version" => "operating_system_version", + "country" => "country_code" + } + def query_sessions(site, query) do {first_datetime, last_datetime} = utc_boundaries(query, site.timezone) @@ -96,36 +104,40 @@ defmodule Plausible.Stats.Base do sessions_q = if query.filters["event:page"] do - page = query.filters["event:page"] - from(e in sessions_q, where: e.entry_page == ^page) + case query.filters["event:page"] do + {:is, page} -> + from(e in sessions_q, where: e.entry_page == ^page) + + {:member, list} -> + from(e in sessions_q, where: e.entry_page in ^list) + end else sessions_q end Enum.reduce(@session_props, sessions_q, fn prop_name, sessions_q -> - prop_val = query.filters["visit:" <> prop_name] - prop_name = if prop_name == "source", do: "referrer_source", else: prop_name - prop_name = if prop_name == "device", do: "screen_size", else: prop_name - prop_name = if prop_name == "os", do: "operating_system", else: prop_name - prop_name = if prop_name == "os_version", do: "operating_system_version", else: prop_name - prop_name = if prop_name == "country", do: "country_code", else: prop_name - - prop_val = - if prop_name == "referrer_source" && prop_val == @no_ref do - "" - else - prop_val - end + filter = query.filters["visit:" <> prop_name] + prop_name = Map.get(@api_prop_name_to_db, prop_name, prop_name) - if prop_val do - where_target = [{String.to_existing_atom(prop_name), prop_val}] - from(s in sessions_q, where: ^where_target) - else - sessions_q + case filter do + {:is, value} -> + where_target = [{String.to_existing_atom(prop_name), db_prop_val(prop_name, value)}] + from(s in sessions_q, where: ^where_target) + + {:member, values} -> + list = Enum.map(values, &db_prop_val(prop_name, &1)) + fragment_data = [{String.to_existing_atom(prop_name), {:in, list}}] + from(s in sessions_q, where: fragment(^fragment_data)) + + _ -> + sessions_q end end) end + defp db_prop_val("referrer_source", @no_ref), do: "" + defp db_prop_val(_, val), do: val + defp utc_boundaries(%Query{date_range: date_range}, timezone) do {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00]) diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex index 42eb8074bbbe..bb335c77af53 100644 --- a/lib/plausible/stats/breakdown.ex +++ b/lib/plausible/stats/breakdown.ex @@ -71,9 +71,7 @@ defmodule Plausible.Stats.Breakdown do end defp filter_converted_sessions(db_query, site, query) do - event = query.filters["event:name"] - - if is_binary(event) do + if query.filters["event:name"] do converted_sessions = from(e in query_events(site, query), select: %{session_id: fragment("DISTINCT ?", e.session_id)} diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index e9de1e0d47d8..acac183b5c22 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -182,9 +182,14 @@ defmodule Plausible.Stats.Query do end defp parse_single_filter(str) do - String.trim(str) - |> String.split("==") - |> Enum.map(&String.trim/1) - |> List.to_tuple() + [key, val] = + String.trim(str) + |> String.split("==") + |> Enum.map(&String.trim/1) + + case String.split(val, "|") do + [single_value] -> {key, {:is, single_value}} + list -> {key, {:member, list}} + end end end diff --git a/mix.exs b/mix.exs index 6ba5e37fe058..c009f903711e 100644 --- a/mix.exs +++ b/mix.exs @@ -87,7 +87,7 @@ defmodule Plausible.MixProject do {:oban, "~> 2.0"}, {:sshex, "2.2.1"}, {:geolix, "~> 1.0"}, - {:clickhouse_ecto, git: "https://github.com/plausible/clickhouse_ecto.git"}, + {:clickhouse_ecto, path: "/home/uku/plausible/clickhouse_ecto"}, {:geolix_adapter_mmdb2, "~> 0.5.0"}, {:mix_test_watch, "~> 1.0", only: :dev, runtime: false}, {:cachex, "~> 3.3"}, diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs index 7b07e648fbba..6688ae1b090f 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs @@ -776,6 +776,175 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do ] } end + + test "IN filter for event:page", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + pathname: "/ignore", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/important-page", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/v1/stats/breakdown", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "property" => "event:page", + "filters" => "event:page == /plausible.io|/important-page" + }) + + assert json_response(conn, 200) == %{ + "results" => [ + %{"page" => "/plausible.io", "visitors" => 2}, + %{"page" => "/important-page", "visitors" => 1} + ] + } + end + + test "IN filter for visit:browser", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + pathname: "/ignore", + browser: "Firefox", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + browser: "Chrome", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + browser: "Safari", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/important-page", + browser: "Safari", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/v1/stats/breakdown", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "property" => "event:page", + "filters" => "visit:browser == Chrome|Safari" + }) + + assert json_response(conn, 200) == %{ + "results" => [ + %{"page" => "/plausible.io", "visitors" => 2}, + %{"page" => "/important-page", "visitors" => 1} + ] + } + end + + test "IN filter for visit:entry_page", %{conn: conn, site: site} do + populate_stats([ + build(:pageview, + pathname: "/ignore", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/plausible.io", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + pathname: "/important-page", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/v1/stats/breakdown", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "property" => "event:page", + "filters" => "event:page == /plausible.io|/important-page", + "metrics" => "bounce_rate" + }) + + assert json_response(conn, 200) == %{ + "results" => [ + %{"page" => "/plausible.io", "bounce_rate" => 100}, + %{"page" => "/important-page", "bounce_rate" => 100} + ] + } + end + + test "IN filter for event:name", %{conn: conn, site: site} do + populate_stats([ + build(:event, + name: "Signup", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Login", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Irrelevant", + domain: site.domain, + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + get(conn, "/api/v1/stats/breakdown", %{ + "site_id" => site.domain, + "period" => "day", + "date" => "2021-01-01", + "property" => "event:name", + "filters" => "event:name == Signup|Login" + }) + + assert json_response(conn, 200) == %{ + "results" => [ + %{"name" => "Signup", "visitors" => 2}, + %{"name" => "Login", "visitors" => 1} + ] + } + end end describe "pagination" do From 081ec5a05daafee2f6625dc9b332b90a62697804 Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Wed, 2 Jun 2021 15:13:19 +0300 Subject: [PATCH 2/2] Update clickhouse_ecto --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index c009f903711e..6ba5e37fe058 100644 --- a/mix.exs +++ b/mix.exs @@ -87,7 +87,7 @@ defmodule Plausible.MixProject do {:oban, "~> 2.0"}, {:sshex, "2.2.1"}, {:geolix, "~> 1.0"}, - {:clickhouse_ecto, path: "/home/uku/plausible/clickhouse_ecto"}, + {:clickhouse_ecto, git: "https://github.com/plausible/clickhouse_ecto.git"}, {:geolix_adapter_mmdb2, "~> 0.5.0"}, {:mix_test_watch, "~> 1.0", only: :dev, runtime: false}, {:cachex, "~> 3.3"}, diff --git a/mix.lock b/mix.lock index f34cb0a6c915..4194ec35d709 100644 --- a/mix.lock +++ b/mix.lock @@ -12,7 +12,7 @@ "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, "cachex": {:hex, :cachex, "3.3.0", "6f2ebb8f27491fe39121bd207c78badc499214d76c695658b19d6079beeca5c2", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "d90e5ee1dde14cef33f6b187af4335b88748b72b30c038969176cd4e6ccc31a1"}, "certifi": {:hex, :certifi, "2.6.1", "dbab8e5e155a0763eea978c913ca280a6b544bfa115633fa20249c3d396d9493", [:rebar3], [], "hexpm", "524c97b4991b3849dd5c17a631223896272c6b0af446778ba4675a1dff53bb7e"}, - "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "1969f14ecef7c357b2bd8bdc3e566234269de58c", []}, + "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "07adb8da725346e4de6d376069192e9942fe7a5b", []}, "clickhousex": {:git, "https://github.com/plausible/clickhousex", "0832dd4b1af1f0eba1d1018c231bf0d8d281f031", []}, "combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},