Skip to content

Commit 6c676bc

Browse files
ShourieGShourieG
authored
[Akamai] - Migrated SIEM data stream from HTTPJSON to CEL input (elastic#15713)
Migrated SIEM data stream from HTTPJSON to CEL input with with necessary reworks and working system tests. Updated minimum stack version to 8.18 to allow usage of required CEL functions.
1 parent 450b8fa commit 6c676bc

File tree

7 files changed

+193
-112
lines changed

7 files changed

+193
-112
lines changed

‎packages/akamai/changelog.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,15 @@
11
# newer versions go on top
2+
- version: "3.0.0"
3+
changes:
4+
- description: |
5+
Migrated SIEM data stream from HTTPJSON to CEL.
6+
type: enhancement
7+
link: https://github.com/elastic/integrations/pull/15713
8+
- description: |
9+
Credentials will likely need to be re-configured since the integration has been
10+
updated to use the new CEL input.
11+
type: breaking-change
12+
link: https://github.com/elastic/integrations/pull/15713
213
- version: "2.28.2"
314
changes:
415
- description: Remove empty HTTP message headers placeholder.

‎packages/akamai/data_stream/siem/_dev/test/pipeline/test-http-json.log-config.yml‎ renamed to ‎packages/akamai/data_stream/siem/_dev/test/pipeline/test-cel.log-config.yml‎

File renamed without changes.

‎packages/akamai/data_stream/siem/_dev/test/system/test-emulator-config.yml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
input: httpjson
1+
input: cel
22
service: akamai-siem-emulator
33
vars: ~
44
data_stream:
@@ -12,10 +12,9 @@ data_stream:
1212
access_token: at-6b8c7217-8748-490d-b0f5-bfeb72b2e7cd
1313
config_ids: 123456
1414
event_limit: 20
15+
# The akamai-siem emulator does not limit the number of events or pages returned, so we set a large number of max_executions.
16+
max_executions: 50000
1517
enable_request_tracer: true
1618
assert:
1719
# 12 hours at 5 minutes between events.
1820
hit_count: 144 # = 12 * 60/5
19-
skip:
20-
reason: "The fleet health status changes to degraded when the HTTPJSON template's value evaluation comes up empty, which leads to system test failures but does not interrupt the data flow."
21-
link: https://github.com/elastic/beats/issues/45664

‎packages/akamai/data_stream/siem/agent/stream/cel.yml.hbs‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
config_version: 2
2+
interval: {{interval}}
3+
resource:
4+
url: {{api_host}}/siem/v1/configs/{{config_ids}}
5+
{{#if ssl}}
6+
ssl: {{ssl}}
7+
{{/if}}
8+
{{#if http_client_timeout}}
9+
timeout: {{http_client_timeout}}
10+
{{/if}}
11+
{{#if proxy_url }}
12+
proxy_url: {{proxy_url}}
13+
{{/if}}
14+
tracer:
15+
enabled: {{enable_request_tracer}}
16+
filename: "../../logs/cel/http-request-trace-*.ndjson"
17+
maxbackups: 5
18+
{{#if max_executions}}
19+
max_executions: {{max_executions}}
20+
{{/if}}
21+
22+
state:
23+
client_token: {{client_token}}
24+
access_token: {{access_token}}
25+
client_secret: {{client_secret}}
26+
initial_interval: {{initial_interval}}
27+
event_limit: {{event_limit}}
28+
29+
redact:
30+
fields:
31+
- client_secret
32+
- access_token
33+
- client_token
34+
35+
program: |-
36+
state.with(
37+
(
38+
state.?cursor.recovery_mode.orValue(false) ?
39+
{
40+
"from": int(now - duration("12h")),
41+
"to": int(now - duration("1m")),
42+
}
43+
: state.?cursor.last_offset.hasValue() ?
44+
{
45+
"offset": state.cursor.last_offset,
46+
}
47+
:
48+
{
49+
"from": max(int(now - duration(state.initial_interval)), int(now - duration("12h"))),
50+
"to": int(now - duration("1m")),
51+
}
52+
).as(params,
53+
(
54+
state.url.trim_right("/") + "?" + {
55+
"limit": [string(state.event_limit)],
56+
?"from": params.?from.optMap(v, [string(v)]),
57+
?"to": params.?to.optMap(v, [string(v)]),
58+
?"offset": params.?offset.optMap(v, [string(v)]),
59+
}.format_query()
60+
).as(request_url,
61+
now.format("20060102T15:04:05-0700").as(timestamp,
62+
uuid().as(nonce,
63+
sprintf(
64+
"EG1-HMAC-SHA256 client_token=%s;access_token=%s;timestamp=%s;nonce=%s;",
65+
[state.client_token, state.access_token, timestamp, nonce]
66+
).as(sig_base,
67+
base64(hmac(timestamp, "sha256", bytes(state.client_secret))).as(sig_key,
68+
request_url.parse_url().as(u,
69+
sprintf(
70+
"GET\t%s\t%s\t%s?%s\t\t\t%s",
71+
[
72+
u.Scheme,
73+
u.Host,
74+
u.Path,
75+
u.RawQuery,
76+
sig_base,
77+
]
78+
).as(to_sign,
79+
base64(hmac(to_sign, "sha256", bytes(sig_key))).as(signature,
80+
sig_base + "signature=" + signature
81+
)
82+
)
83+
)
84+
)
85+
).as(auth_header,
86+
request(
87+
"GET",
88+
request_url
89+
).with(
90+
{
91+
"Header": {
92+
"Authorization": [auth_header],
93+
},
94+
}
95+
).do_request().as(resp,
96+
(resp.StatusCode == 200) ?
97+
string(resp.Body).split("\n").filter(line, line != "").as(lines,
98+
{
99+
"events": lines.map(line, {"message": line}),
100+
"cursor": {
101+
?"last_offset": (lines.size() > 0) ?
102+
lines[lines.size() - 1].decode_json().as(lastEvent,
103+
(has(lastEvent.offset) && lastEvent.offset != "") ?
104+
optional.of(lastEvent.offset)
105+
:
106+
optional.none()
107+
)
108+
:
109+
optional.none(),
110+
"recovery_mode": false,
111+
},
112+
"want_more": (lines.size() > 0) ?
113+
lines[lines.size() - 1].decode_json().as(lastEvent,
114+
has(lastEvent.offset) && lastEvent.offset != ""
115+
)
116+
:
117+
false,
118+
}
119+
)
120+
: (resp.StatusCode == 416) ?
121+
{
122+
"events": [
123+
{
124+
"error": {
125+
"code": string(resp.StatusCode),
126+
"id": string(resp.Status),
127+
"message": "GET " + request_url + (
128+
(size(resp.Body) != 0) ?
129+
string(resp.Body)
130+
:
131+
string(resp.Status) + " (" + string(resp.StatusCode) + ")"
132+
),
133+
},
134+
}
135+
],
136+
"cursor": state.cursor.drop("last_offset").with(
137+
{
138+
"recovery_mode": true,
139+
}
140+
),
141+
"want_more": true,
142+
}
143+
:
144+
{
145+
"events": {
146+
"error": {
147+
"code": string(resp.StatusCode),
148+
"id": string(resp.Status),
149+
"message": "GET " + request_url + (
150+
(size(resp.Body) != 0) ?
151+
string(resp.Body)
152+
:
153+
string(resp.Status) + " (" + string(resp.StatusCode) + ")"
154+
),
155+
},
156+
},
157+
"want_more": false,
158+
}
159+
)
160+
)
161+
)
162+
)
163+
)
164+
)
165+
)

‎packages/akamai/data_stream/siem/agent/stream/httpjson.yml.hbs‎

Lines changed: 0 additions & 103 deletions
This file was deleted.

‎packages/akamai/data_stream/siem/manifest.yml‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
type: logs
22
title: Akamai SIEM Logs
33
streams:
4-
- input: httpjson
5-
template_path: httpjson.yml.hbs
4+
- input: cel
5+
template_path: cel.yml.hbs
66
title: Akamai SIEM logs
77
description: Collect Akamai logs via the SIEM API
88
vars:
@@ -76,6 +76,15 @@ streams:
7676
show_user: false
7777
title: Event Limit
7878
description: Defines the approximate maximum number of security events each fetch returns, in both offset and time-based modes. The default limit is 10000 and the maximum limit available is 600000. Listing an unlimited number of logs isn't possible. Expect requests to return a slightly higher number of security events than you set in the limit parameter, because data is stored in different buckets.
79+
default: 10000
80+
- name: max_executions
81+
type: integer
82+
title: Maximum Pages Per Interval
83+
description: Maximum Pages Per Interval is the maximum number of pages that can be collected at each interval.
84+
multi: false
85+
required: false
86+
show_user: false
87+
default: 5000
7988
- name: proxy_url
8089
type: text
8190
title: Proxy URL

‎packages/akamai/manifest.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
name: akamai
22
title: Akamai
3-
version: "2.28.2"
3+
version: "3.0.0"
44
description: Collect logs from Akamai with Elastic Agent.
55
type: integration
66
format_version: "3.0.2"
77
categories: [security, cdn_security]
88
conditions:
99
kibana:
10-
version: "^8.13.0 || ^9.0.0"
10+
version: "^8.18.0 || ^9.0.0"
1111
icons:
1212
- src: /img/akamai_logo.svg
1313
title: Akamai
@@ -18,7 +18,7 @@ policy_templates:
1818
title: Akamai logs
1919
description: Collect SIEM logs from Akamai
2020
inputs:
21-
- type: httpjson
21+
- type: cel
2222
title: "Collect Akamai SIEM logs via API"
2323
description: "Collecting SIEM logs from Akamai via API"
2424
- type: gcs

0 commit comments

Comments
 (0)