Skip to content

Commit f0104cb

Browse files
apataapata
authored
Expand segments in queries (#4982)
* Refactor segments model * Fix inconsistent code * Remove superfluous error case * Beautify Plausible.Segments module * Expand segments in filters * Add tests * Generate types * Remove extraneous newlines * Move Segment filters logic away from QueryParser * Add moduledoc * Add tests for /v2/query-internal-test * Refactor max segment filters count to module attribute * Add more parser tests and unify asserts in query
1 parent 006e460 commit f0104cb

File tree

9 files changed

+502
-12
lines changed

9 files changed

+502
-12
lines changed

‎assets/js/types/query-api.d.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ export type CustomPropertyFilterDimensions = string;
6464
export type GoalDimension = "event:goal";
6565
export type TimeDimensions = "time" | "time:month" | "time:week" | "time:day" | "time:hour";
6666
export type FilterTree = FilterEntry | FilterAndOr | FilterNot;
67-
export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern;
67+
export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern | FilterForSegment;
6868
/**
6969
* @minItems 3
7070
* @maxItems 4
@@ -115,6 +115,11 @@ export type FilterWithPattern = [
115115
* filter operation
116116
*/
117117
export type FilterOperationRegex = "matches" | "matches_not";
118+
/**
119+
* @minItems 3
120+
* @maxItems 3
121+
*/
122+
export type FilterForSegment = ["is", "segment", number[]];
118123
/**
119124
* @minItems 2
120125
* @maxItems 2

‎lib/plausible/segments/filters.ex‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
defmodule Plausible.Segments.Filters do
2+
@moduledoc """
3+
This module contains functions that enable resolving segments in filters.
4+
"""
5+
alias Plausible.Segments
6+
alias Plausible.Stats.Filters
7+
8+
@max_segment_filters_count 10
9+
10+
@doc """
11+
Finds unique segment IDs used in query filters.
12+
13+
## Examples
14+
iex> get_segment_ids([[:not, [:is, "segment", [10, 20]]], [:contains, "visit:entry_page", ["blog"]]])
15+
{:ok, [10, 20]}
16+
17+
iex> get_segment_ids([[:and, [[:is, "segment", Enum.to_list(1..6)], [:is, "segment", Enum.to_list(1..6)]]]])
18+
{:error, "Invalid filters. You can only use up to 10 segment filters in a query."}
19+
"""
20+
def get_segment_ids(filters) do
21+
ids =
22+
filters
23+
|> Filters.traverse()
24+
|> Enum.flat_map(fn
25+
{[_operation, "segment", clauses], _depth} -> clauses
26+
_ -> []
27+
end)
28+
29+
if length(ids) > @max_segment_filters_count do
30+
{:error,
31+
"Invalid filters. You can only use up to #{@max_segment_filters_count} segment filters in a query."}
32+
else
33+
{:ok, Enum.uniq(ids)}
34+
end
35+
end
36+
37+
def preload_needed_segments(%Plausible.Site{} = site, filters) do
38+
with {:ok, segment_ids} <- get_segment_ids(filters),
39+
{:ok, segments} <-
40+
Segments.get_many(
41+
site,
42+
segment_ids,
43+
fields: [:id, :segment_data]
44+
),
45+
{:ok, segments_by_id} <-
46+
{:ok,
47+
Enum.into(
48+
segments,
49+
%{},
50+
fn %Segments.Segment{id: id, segment_data: segment_data} ->
51+
case Filters.QueryParser.parse_filters(segment_data["filters"]) do
52+
{:ok, filters} -> {id, filters}
53+
_ -> {id, nil}
54+
end
55+
end
56+
)},
57+
:ok <-
58+
if(Enum.any?(segment_ids, fn id -> is_nil(Map.get(segments_by_id, id)) end),
59+
do: {:error, "Invalid filters. Some segments don't exist or aren't accessible."},
60+
else: :ok
61+
) do
62+
{:ok, segments_by_id}
63+
end
64+
end
65+
66+
defp replace_segment_with_filter_tree([_, "segment", clauses], preloaded_segments) do
67+
if length(clauses) === 1 do
68+
[[:and, Map.get(preloaded_segments, Enum.at(clauses, 0))]]
69+
else
70+
[[:or, Enum.map(clauses, fn id -> [:and, Map.get(preloaded_segments, id)] end)]]
71+
end
72+
end
73+
74+
defp replace_segment_with_filter_tree(_filter, _preloaded_segments) do
75+
nil
76+
end
77+
78+
@doc """
79+
## Examples
80+
81+
iex> resolve_segments([[:is, "visit:entry_page", ["/home"]]], %{})
82+
{:ok, [[:is, "visit:entry_page", ["/home"]]]}
83+
84+
iex> resolve_segments([[:is, "visit:entry_page", ["/home"]], [:is, "segment", [1]]], %{1 => [[:contains, "visit:entry_page", ["blog"]], [:is, "visit:country", ["PL"]]]})
85+
{:ok, [
86+
[:is, "visit:entry_page", ["/home"]],
87+
[:and, [[:contains, "visit:entry_page", ["blog"]], [:is, "visit:country", ["PL"]]]]
88+
]}
89+
90+
iex> resolve_segments([[:is, "segment", [1, 2]]], %{1 => [[:contains, "event:goal", ["Singup"]], [:is, "visit:country", ["PL"]]], 2 => [[:contains, "event:goal", ["Sauna"]], [:is, "visit:country", ["EE"]]]})
91+
{:ok, [
92+
[:or, [
93+
[:and, [[:contains, "event:goal", ["Singup"]], [:is, "visit:country", ["PL"]]]],
94+
[:and, [[:contains, "event:goal", ["Sauna"]], [:is, "visit:country", ["EE"]]]]]
95+
]
96+
]}
97+
"""
98+
def resolve_segments(original_filters, preloaded_segments) do
99+
if map_size(preloaded_segments) > 0 do
100+
{:ok,
101+
Filters.transform_filters(original_filters, fn f ->
102+
replace_segment_with_filter_tree(f, preloaded_segments)
103+
end)}
104+
else
105+
{:ok, original_filters}
106+
end
107+
end
108+
end

‎lib/plausible/segments/segments.ex‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,21 @@ defmodule Plausible.Segments do
3737
end
3838
end
3939

40+
@spec get_many(Plausible.Site.t(), list(pos_integer()), Keyword.t()) ::
41+
{:ok, [Segment.t()]}
42+
def get_many(%Plausible.Site{} = site, segment_ids, opts) when is_list(segment_ids) do
43+
fields = Keyword.get(opts, :fields, [:id])
44+
45+
query =
46+
from(segment in Segment,
47+
select: ^fields,
48+
where: segment.site_id == ^site.id,
49+
where: segment.id in ^segment_ids
50+
)
51+
52+
{:ok, Repo.all(query)}
53+
end
54+
4055
@spec get_one(pos_integer(), Plausible.Site.t(), atom(), pos_integer() | nil) ::
4156
{:ok, Segment.t()}
4257
| error_not_enough_permissions()

‎lib/plausible/stats/filters/filters.ex‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ defmodule Plausible.Stats.Filters do
5757
5858
Returns an empty list when argument type is unexpected (e.g. `nil`).
5959
60-
### Examples:
60+
## Examples:
6161
6262
iex> Filters.parse("visit:browser!=Chrome")
6363
[[:is_not, "visit:browser", ["Chrome"]]]
@@ -128,6 +128,13 @@ defmodule Plausible.Stats.Filters do
128128
Transformer will receive each node (filter, and/or/not subtree) of
129129
query and must return a list of nodes to replace it with or nil
130130
to ignore and look deeper.
131+
132+
## Examples
133+
iex> Filters.transform_filters([[:is, "visit:os", ["Linux"]], [:and, [[:is, "segment", [1]], [:is, "segment", [2]]]]], fn
134+
...> [_, "segment", _] -> [[:is, "segment", ["changed"]]]
135+
...> _ -> nil
136+
...> end)
137+
[[:is, "visit:os", ["Linux"]], [:and, [[:is, "segment", ["changed"]], [:is, "segment", ["changed"]]]]]
131138
"""
132139
def transform_filters(filters, transformer) do
133140
filters
@@ -146,15 +153,15 @@ defmodule Plausible.Stats.Filters do
146153

147154
# Reached a leaf node, return existing value
148155
{nil, filter} ->
149-
[[filter]]
156+
[filter]
150157

151158
# Transformer returned a value - don't transform that subtree
152159
{transformed_filters, _filter} ->
153160
transformed_filters
154161
end
155162
end
156163

157-
defp traverse(filters, depth \\ -1) do
164+
def traverse(filters, depth \\ -1) do
158165
filters
159166
|> Enum.flat_map(&traverse_tree(&1, depth + 1))
160167
end

‎lib/plausible/stats/filters/query_parser.ex‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ defmodule Plausible.Stats.Filters.QueryParser do
3434
utc_time_range = raw_time_range |> DateTimeRange.to_timezone("Etc/UTC"),
3535
{:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),
3636
{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),
37+
{:ok, preloaded_segments} <-
38+
Plausible.Segments.Filters.preload_needed_segments(site, filters),
39+
{:ok, filters} <-
40+
Plausible.Segments.Filters.resolve_segments(filters, preloaded_segments),
3741
{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),
3842
{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),
3943
{:ok, include} <- parse_include(site, Map.get(params, "include", %{})),
@@ -161,7 +165,10 @@ defmodule Plausible.Stats.Filters.QueryParser do
161165
"Invalid visit:country filter, visit:country needs to be a valid 2-letter country code."}
162166
end
163167

164-
{_, true} ->
168+
{"segment", _} when all_integers? ->
169+
{:ok, list}
170+
171+
{_, true} when filter_key !== "segment" ->
165172
{:ok, list}
166173

167174
_ ->
@@ -396,6 +403,9 @@ defmodule Plausible.Stats.Filters.QueryParser do
396403
{:error, error_message}
397404
end
398405

406+
"segment" ->
407+
{:ok, filter_key}
408+
399409
_ ->
400410
{:error, error_message}
401411
end

‎priv/json-schemas/query-api-schema.json‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,11 @@
9898
"minItems": 2,
9999
"maxItems": 2,
100100
"items": {
101-
"type": "string",
102-
"format": "date"
101+
"type": "string",
102+
"format": "date"
103103
},
104104
"description": "If custom period. A list of two ISO8601 dates or timestamps to compare against.",
105-
"examples": [
106-
["2024-01-01", "2024-01-31"]
107-
]
105+
"examples": [["2024-01-01", "2024-01-31"]]
108106
}
109107
},
110108
"required": ["mode", "date_range"],
@@ -439,11 +437,35 @@
439437
}
440438
]
441439
},
440+
"filter_for_segment": {
441+
"type": "array",
442+
"additionalItems": false,
443+
"minItems": 3,
444+
"maxItems": 3,
445+
"items": [
446+
{
447+
"const": "is"
448+
},
449+
{
450+
"const": "segment"
451+
},
452+
{
453+
"type": "array",
454+
"items": {
455+
"type": ["integer"]
456+
}
457+
}
458+
]
459+
},
442460
"filter_entry": {
443461
"oneOf": [
444462
{ "$ref": "#/definitions/filter_without_goals" },
445463
{ "$ref": "#/definitions/filter_with_goals" },
446-
{ "$ref": "#/definitions/filter_with_pattern" }
464+
{ "$ref": "#/definitions/filter_with_pattern" },
465+
{
466+
"$ref": "#/definitions/filter_for_segment",
467+
"$comment": "only :internal"
468+
}
447469
]
448470
},
449471
"filter_tree": {

‎test/plausible/segments/filters_test.exs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
defmodule Plausible.Segments.FiltersTest do
2+
use ExUnit.Case, async: true
3+
4+
doctest Plausible.Segments.Filters, import: true
5+
end

0 commit comments

Comments
 (0)