Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Stats API: Support IN filter expression #1090

Merged
merged 2 commits into from
Jun 2, 2021
Merged

Stats API: Support IN filter expression #1090

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dce165f
Support IN filter expression
ukutaht May 28, 2021
081ec5a
Update clickhouse_ecto
ukutaht Jun 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 42 additions & 30 deletions lib/plausible/stats/base.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,24 +47,24 @@ defmodule Plausible.Stats.Base do
)

q =
if query.filters["event:page"] do
page = query.filters["event:page"]
from(e in q, where: e.pathname == ^page)
else
q
case query.filters["event:page"] do
{:is, page} -> from(e in q, where: e.pathname == ^page)
{:member, list} -> from(e in q, where: e.pathname in ^list)
_ -> q
end

q =
if query.filters["event:name"] do
name = query.filters["event:name"]
from(e in q, where: e.name == ^name)
else
q
case query.filters["event:name"] do
{:is, name} -> from(e in q, where: e.name == ^name)
{:member, list} -> from(e in q, where: e.name in ^list)
_ -> q
end

Enum.reduce(query.filters, q, fn {filter_key, filter_value}, query ->
case filter_key do
"event:props:" <> prop_name ->
filter_value = elem(filter_value, 1)

if filter_value == "(none)" do
from(
e in query,
Expand All @@ -85,6 +85,14 @@ defmodule Plausible.Stats.Base do
end)
end

@api_prop_name_to_db %{
"source" => "referrer_source",
"device" => "screen_size",
"os" => "operating_system",
"os_version" => "operating_system_version",
"country" => "country_code"
}

def query_sessions(site, query) do
{first_datetime, last_datetime} = utc_boundaries(query, site.timezone)

Expand All @@ -96,36 +104,40 @@ defmodule Plausible.Stats.Base do

sessions_q =
if query.filters["event:page"] do
page = query.filters["event:page"]
from(e in sessions_q, where: e.entry_page == ^page)
case query.filters["event:page"] do
{:is, page} ->
from(e in sessions_q, where: e.entry_page == ^page)

{:member, list} ->
from(e in sessions_q, where: e.entry_page in ^list)
end
else
sessions_q
end

Enum.reduce(@session_props, sessions_q, fn prop_name, sessions_q ->
prop_val = query.filters["visit:" <> prop_name]
prop_name = if prop_name == "source", do: "referrer_source", else: prop_name
prop_name = if prop_name == "device", do: "screen_size", else: prop_name
prop_name = if prop_name == "os", do: "operating_system", else: prop_name
prop_name = if prop_name == "os_version", do: "operating_system_version", else: prop_name
prop_name = if prop_name == "country", do: "country_code", else: prop_name

prop_val =
if prop_name == "referrer_source" && prop_val == @no_ref do
""
else
prop_val
end
filter = query.filters["visit:" <> prop_name]
prop_name = Map.get(@api_prop_name_to_db, prop_name, prop_name)

if prop_val do
where_target = [{String.to_existing_atom(prop_name), prop_val}]
from(s in sessions_q, where: ^where_target)
else
sessions_q
case filter do
{:is, value} ->
where_target = [{String.to_existing_atom(prop_name), db_prop_val(prop_name, value)}]
from(s in sessions_q, where: ^where_target)

{:member, values} ->
list = Enum.map(values, &db_prop_val(prop_name, &1))
fragment_data = [{String.to_existing_atom(prop_name), {:in, list}}]
from(s in sessions_q, where: fragment(^fragment_data))

_ ->
sessions_q
end
end)
end

defp db_prop_val("referrer_source", @no_ref), do: ""
defp db_prop_val(_, val), do: val

defp utc_boundaries(%Query{date_range: date_range}, timezone) do
{:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00])

Expand Down
4 changes: 1 addition & 3 deletions lib/plausible/stats/breakdown.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,7 @@ defmodule Plausible.Stats.Breakdown do
end

defp filter_converted_sessions(db_query, site, query) do
event = query.filters["event:name"]

if is_binary(event) do
if query.filters["event:name"] do
converted_sessions =
from(e in query_events(site, query),
select: %{session_id: fragment("DISTINCT ?", e.session_id)}
Expand Down
13 changes: 9 additions & 4 deletions lib/plausible/stats/query.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,9 +182,14 @@ defmodule Plausible.Stats.Query do
end

defp parse_single_filter(str) do
String.trim(str)
|> String.split("==")
|> Enum.map(&String.trim/1)
|> List.to_tuple()
[key, val] =
String.trim(str)
|> String.split("==")
|> Enum.map(&String.trim/1)

case String.split(val, "|") do
[single_value] -> {key, {:is, single_value}}
list -> {key, {:member, list}}
end
end
end
2 changes: 1 addition & 1 deletion mix.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"},
"cachex": {:hex, :cachex, "3.3.0", "6f2ebb8f27491fe39121bd207c78badc499214d76c695658b19d6079beeca5c2", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "d90e5ee1dde14cef33f6b187af4335b88748b72b30c038969176cd4e6ccc31a1"},
"certifi": {:hex, :certifi, "2.6.1", "dbab8e5e155a0763eea978c913ca280a6b544bfa115633fa20249c3d396d9493", [:rebar3], [], "hexpm", "524c97b4991b3849dd5c17a631223896272c6b0af446778ba4675a1dff53bb7e"},
"clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "1969f14ecef7c357b2bd8bdc3e566234269de58c", []},
"clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "07adb8da725346e4de6d376069192e9942fe7a5b", []},
"clickhousex": {:git, "https://github.com/plausible/clickhousex", "0832dd4b1af1f0eba1d1018c231bf0d8d281f031", []},
"combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
Expand Down
169 changes: 169 additions & 0 deletions test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -776,6 +776,175 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do
]
}
end

test "IN filter for event:page", %{conn: conn, site: site} do
populate_stats([
build(:pageview,
pathname: "/ignore",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/important-page",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])

conn =
get(conn, "/api/v1/stats/breakdown", %{
"site_id" => site.domain,
"period" => "day",
"date" => "2021-01-01",
"property" => "event:page",
"filters" => "event:page == /plausible.io|/important-page"
})

assert json_response(conn, 200) == %{
"results" => [
%{"page" => "/plausible.io", "visitors" => 2},
%{"page" => "/important-page", "visitors" => 1}
]
}
end

test "IN filter for visit:browser", %{conn: conn, site: site} do
populate_stats([
build(:pageview,
pathname: "/ignore",
browser: "Firefox",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
browser: "Chrome",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
browser: "Safari",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/important-page",
browser: "Safari",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])

conn =
get(conn, "/api/v1/stats/breakdown", %{
"site_id" => site.domain,
"period" => "day",
"date" => "2021-01-01",
"property" => "event:page",
"filters" => "visit:browser == Chrome|Safari"
})

assert json_response(conn, 200) == %{
"results" => [
%{"page" => "/plausible.io", "visitors" => 2},
%{"page" => "/important-page", "visitors" => 1}
]
}
end

test "IN filter for visit:entry_page", %{conn: conn, site: site} do
populate_stats([
build(:pageview,
pathname: "/ignore",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/plausible.io",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:pageview,
pathname: "/important-page",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])

conn =
get(conn, "/api/v1/stats/breakdown", %{
"site_id" => site.domain,
"period" => "day",
"date" => "2021-01-01",
"property" => "event:page",
"filters" => "event:page == /plausible.io|/important-page",
"metrics" => "bounce_rate"
})

assert json_response(conn, 200) == %{
"results" => [
%{"page" => "/plausible.io", "bounce_rate" => 100},
%{"page" => "/important-page", "bounce_rate" => 100}
]
}
end

test "IN filter for event:name", %{conn: conn, site: site} do
populate_stats([
build(:event,
name: "Signup",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Signup",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Login",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
),
build(:event,
name: "Irrelevant",
domain: site.domain,
timestamp: ~N[2021-01-01 00:00:00]
)
])

conn =
get(conn, "/api/v1/stats/breakdown", %{
"site_id" => site.domain,
"period" => "day",
"date" => "2021-01-01",
"property" => "event:name",
"filters" => "event:name == Signup|Login"
})

assert json_response(conn, 200) == %{
"results" => [
%{"name" => "Signup", "visitors" => 2},
%{"name" => "Login", "visitors" => 1}
]
}
end
end

describe "pagination" do
Expand Down