From 4b4db87720c7c8d03573d59c4db466e8e6d6002d Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Tue, 4 Nov 2025 11:37:29 +0100 Subject: [PATCH 01/11] rsyslog: Only opdate the lastseen tabel for newer dates --- roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 b/roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 index b37f4720c..7e0bc7bcb 100644 --- a/roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 +++ b/roles/rsyslog/templates/parse_ebauth_to_mysql.py.j2 @@ -21,11 +21,17 @@ cursor = db.cursor() def update_lastseen(user_id, date): query = """ - REPLACE INTO last_login (userid, lastseen) + INSERT INTO last_login (userid, lastseen) VALUES (%s, %s) + ON DUPLICATE KEY UPDATE + lastseen = GREATEST(lastseen, VALUES(lastseen)) """ - cursor.execute(query, (user_id, date)) - db.commit() + try: + cursor.execute(query, (user_id, date)) + db.commit() + except Exception as e: + db.rollback() + print(f"Error updating last_login for user {user_id}: {e}") def load_in_mysql(a,b,c,d,e,f,g,h): sql = """insert into log_logins(idpentityid,spentityid,loginstamp,userid,keyid,sessionid,requestid,trustedproxyentityid) values(%s,%s,%s,%s,%s,%s,%s,%s)""" @@ -73,4 +79,3 @@ for filename in os.listdir(workdir): cursor.close() db.close() - From 0cff50e61faac0cb32f8366f5cd8fd8b30fa7641 Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Tue, 4 Nov 2025 12:50:09 +0100 Subject: [PATCH 02/11] rsyslog: Also rotate and parse stepup-logs --- .../rsyslog/templates/logrotate_stepupauth.j2 | 16 +++ .../parse_stepupauthauth_to_mysql.py.j2 | 133 ++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 roles/rsyslog/templates/logrotate_stepupauth.j2 create mode 100644 roles/rsyslog/templates/parse_stepupauthauth_to_mysql.py.j2 diff --git a/roles/rsyslog/templates/logrotate_stepupauth.j2 b/roles/rsyslog/templates/logrotate_stepupauth.j2 new file mode 100644 index 000000000..be1a50652 --- /dev/null +++ b/roles/rsyslog/templates/logrotate_stepupauth.j2 @@ -0,0 +1,16 @@ +{{ rsyslog_dir }}/log_logins/{{ item.name }}/stepup-authentication.log +{ + missingok + daily + rotate 180 + sharedscripts + dateext + dateyesterday + compress + delaycompress + create 0640 root {{ rsyslog_read_group }} + postrotate + /usr/local/sbin/parse_stepupauth_to_mysql_{{ item.name }}.py > /dev/null + systemctl kill -s HUP rsyslog.service + endscript +} diff --git a/roles/rsyslog/templates/parse_stepupauthauth_to_mysql.py.j2 b/roles/rsyslog/templates/parse_stepupauthauth_to_mysql.py.j2 new file mode 100644 index 000000000..2fe55d6ab --- /dev/null +++ b/roles/rsyslog/templates/parse_stepupauthauth_to_mysql.py.j2 @@ -0,0 +1,133 @@ +#!/usr/bin/python3 +# This script parses rotated stepup-authentication.log files produced by engineblock. +# It filters for successful logins (authentication_result:OK) and inserts the data +# into the log_logins and last_login MySQL tables. +# This script is intended to be run separately during logrotate. + +import os +import sys +import json +import MySQLdb +from dateutil.parser import parse + +# Configuration variables (to be injected by Ansible/Jinja2) +mysql_host="{{ item.db_loglogins_host }}" +mysql_user="{{ item.db_loglogins_user }}" +mysql_password="{{ item.db_loglogins_password }}" +mysql_db="{{ item.db_loglogins_name }}" +workdir="{{ rsyslog_dir }}/log_logins/{{ item.name}}/" + +# Establish database connection +try: + db = MySQLdb.connect(mysql_host,mysql_user,mysql_password,mysql_db ) + cursor = db.cursor() +except Exception as e: + print(f"Error connecting to MySQL: {e}") + sys.exit(1) + +# --- Database Functions --- + +def update_lastseen(user_id, date): + """ + Updates the last_login table. + Uses GREATEST() to ensure only newer dates overwrite the existing 'lastseen' value. + """ + query = """ + INSERT INTO last_login (userid, lastseen) + VALUES (%s, %s) + ON DUPLICATE KEY UPDATE + lastseen = GREATEST(lastseen, VALUES(lastseen)) + """ + try: + cursor.execute(query, (user_id, date)) + db.commit() + except Exception as e: + db.rollback() + print(f"Error updating last_login for user {user_id}: {e}") + +def load_stepup_in_mysql(idp, sp, loginstamp, userid, requestid): + """ + Inserts Step-up login data into the log_logins table. + Fills keyid, sessionid, and trustedproxyentityid with NULL. + """ + # Columns in log_logins: idpentityid, spentityid, loginstamp, userid, keyid, sessionid, requestid, trustedproxyentityid + + keyid = None + sessionid = None + trustedproxyentityid = None + + sql = """ + INSERT INTO log_logins(idpentityid, spentityid, loginstamp, userid, keyid, sessionid, requestid, trustedproxyentityid) + VALUES(%s, %s, %s, %s, %s, %s, %s, %s) + """ + try: + cursor.execute(sql, (idp, sp, loginstamp, userid, keyid, sessionid, requestid, trustedproxyentityid)) + db.commit() + except Exception as e: + db.rollback() + print(f"Error inserting stepup data: {e}") + # Print the data that failed insertion + print((idp, sp, loginstamp, userid, keyid, sessionid, requestid, trustedproxyentityid)) + +# --- Parsing Function --- + +def parse_stepup_lines(a): + """ + Opens the stepup log file, parses each line, filters for successful logins, + and loads the data into MySQL. + """ + input_file = open((a), 'r') + for line in input_file: + try: + # Assumes JSON data starts after the first ']:' + jsonline = line.split(']:',2)[1] + data = json.loads(jsonline) + except: + continue + + # 1. Filtering condition: Only parse logs having authentication_result:OK + if data.get("authentication_result") != "OK": + continue + + # 2. Extract required fields + user_id = data.get("identity_id") + timestamp = data.get("datetime") + request_id = data.get("request_id") + sp_entity_id = data.get("requesting_sp") + idp_entity_id = data.get("authenticating_idp") + + # Basic data validation + if not user_id or not timestamp: + continue + + try: + # 3. Format date and time for MySQL + loginstamp = parse(timestamp).strftime("%Y-%m-%d %H:%M:%S") + last_login_date = parse(timestamp).strftime("%Y-%m-%d") + except: + continue + + # 4. Insert into MySQL + load_stepup_in_mysql(idp_entity_id, sp_entity_id, loginstamp, user_id, request_id) + + # 5. Update last login date + update_lastseen(user_id, last_login_date) + + +# --- Main Execution --- + +## Loop over the files and parse them one by one +for filename in os.listdir(workdir): + filetoparse=(os.path.join(workdir, filename)) + + # Check for Stepup files, ignore compressed files + if os.path.isfile(filetoparse) and filename.startswith("stepup-authentication.log-") and not filename.endswith(".gz"): + print(f"Parsing stepup log file: {filename}") + parse_stepup_lines(filetoparse) + else: + continue + +# Close database connection +cursor.close() +db.close() +print("Stepup log parsing complete.") From c255aa48e62fa9b47af185621d3a026669a091b9 Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Tue, 4 Nov 2025 12:53:04 +0100 Subject: [PATCH 03/11] rsyslog: Add ansible tasks for stepup log parsing --- roles/rsyslog/tasks/process_auth_logs.yml | 24 +++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/roles/rsyslog/tasks/process_auth_logs.yml b/roles/rsyslog/tasks/process_auth_logs.yml index e62027530..e123fb33f 100644 --- a/roles/rsyslog/tasks/process_auth_logs.yml +++ b/roles/rsyslog/tasks/process_auth_logs.yml @@ -39,7 +39,7 @@ state: present when: ansible_os_family == "Debian" -- name: Create a python script that parses log_logins per environment +- name: Create a python script that parses eb log_logins per environment ansible.builtin.template: src: parse_ebauth_to_mysql.py.j2 dest: /usr/local/sbin/parse_ebauth_to_mysql_{{ item.name }}.py @@ -49,7 +49,17 @@ with_items: "{{ rsyslog_environments }}" when: item.db_loglogins_name is defined -- name: Put log_logins logrotate scripts +- name: Create a python script that parses stepup log_logins per environment + ansible.builtin.template: + src: parse_ebauth_to_mysql.py.j2 + dest: /usr/local/sbin/parse_stepupauth_to_mysql_{{ item.name }}.py + mode: 0740 + owner: root + group: root + with_items: "{{ rsyslog_environments }}" + when: item.db_loglogins_name is defined + +- name: Put log_logins logrotate scripts for eb ansible.builtin.template: src: logrotate_ebauth.j2 dest: /etc/logrotate.d/logrotate_ebauth_{{ item.name }} @@ -59,6 +69,16 @@ with_items: "{{ rsyslog_environments }}" when: item.db_loglogins_name is defined +- name: Put log_logins logrotate scripts for stepup + ansible.builtin.template: + src: logrotate_ebauth.j2 + dest: /etc/logrotate.d/logrotate_stepupauth_{{ item.name }} + mode: 0644 + owner: root + group: root + with_items: "{{ rsyslog_environments }}" + when: item.db_loglogins_name is defined + - name: Create logdirectory for log_logins cleanup script ansible.builtin.file: path: "{{ rsyslog_dir }}/apps/{{ item.name }}/loglogins_cleanup/" From ff6862d2eade667d5d060d98d12e40705e387e0e Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Fri, 10 Apr 2026 13:31:18 +0200 Subject: [PATCH 04/11] Fix name for parse_stepupauth template --- roles/rsyslog/tasks/process_auth_logs.yml | 2 +- ...pauthauth_to_mysql.py.j2 => parse_stepupauth_to_mysql.py.j2} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename roles/rsyslog/templates/{parse_stepupauthauth_to_mysql.py.j2 => parse_stepupauth_to_mysql.py.j2} (100%) diff --git a/roles/rsyslog/tasks/process_auth_logs.yml b/roles/rsyslog/tasks/process_auth_logs.yml index e123fb33f..808d8fa9f 100644 --- a/roles/rsyslog/tasks/process_auth_logs.yml +++ b/roles/rsyslog/tasks/process_auth_logs.yml @@ -51,7 +51,7 @@ - name: Create a python script that parses stepup log_logins per environment ansible.builtin.template: - src: parse_ebauth_to_mysql.py.j2 + src: parse_stepupauth_to_mysql.py.j2 dest: /usr/local/sbin/parse_stepupauth_to_mysql_{{ item.name }}.py mode: 0740 owner: root diff --git a/roles/rsyslog/templates/parse_stepupauthauth_to_mysql.py.j2 b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 similarity index 100% rename from roles/rsyslog/templates/parse_stepupauthauth_to_mysql.py.j2 rename to roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 From 4089da9cddacd192a7bfeb21b3a8a4aad084abf1 Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Fri, 10 Apr 2026 13:48:43 +0200 Subject: [PATCH 05/11] Fix template name --- roles/rsyslog/tasks/process_auth_logs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rsyslog/tasks/process_auth_logs.yml b/roles/rsyslog/tasks/process_auth_logs.yml index 808d8fa9f..804bf629b 100644 --- a/roles/rsyslog/tasks/process_auth_logs.yml +++ b/roles/rsyslog/tasks/process_auth_logs.yml @@ -71,7 +71,7 @@ - name: Put log_logins logrotate scripts for stepup ansible.builtin.template: - src: logrotate_ebauth.j2 + src: logrotate_stepupauth.j2 dest: /etc/logrotate.d/logrotate_stepupauth_{{ item.name }} mode: 0644 owner: root From 5e0ec792b6d1fd30c569caefdfa593ce13edb346 Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Fri, 10 Apr 2026 13:49:02 +0200 Subject: [PATCH 06/11] Disable check for authentication_result --- roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 index 2fe55d6ab..a6aa8b707 100644 --- a/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 +++ b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 @@ -86,8 +86,13 @@ def parse_stepup_lines(a): continue # 1. Filtering condition: Only parse logs having authentication_result:OK - if data.get("authentication_result") != "OK": - continue + # Only successful authentications are logged, so this check is not + # necessary. There is currently a bug in the Stepup-Gateway where + # FAILED is logged, even though the result is OK, making this check + # do the wrong thing now. + # + #if data.get("authentication_result") != "OK": + # continue # 2. Extract required fields user_id = data.get("identity_id") From 5a7b328be91cd9ecae7fb228cdf0d78672265f65 Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Fri, 10 Apr 2026 13:52:23 +0200 Subject: [PATCH 07/11] Get data from the context object --- .../rsyslog/templates/parse_stepupauth_to_mysql.py.j2 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 index a6aa8b707..cef39191e 100644 --- a/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 +++ b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 @@ -95,11 +95,11 @@ def parse_stepup_lines(a): # continue # 2. Extract required fields - user_id = data.get("identity_id") - timestamp = data.get("datetime") - request_id = data.get("request_id") - sp_entity_id = data.get("requesting_sp") - idp_entity_id = data.get("authenticating_idp") + user_id = data.get("context").("identity_id") + timestamp = data.get("context").("datetime") + request_id = data.get("context").("request_id") + sp_entity_id = data.get("context").("requesting_sp") + idp_entity_id = data.get("context").("authenticating_idp") # Basic data validation if not user_id or not timestamp: From adf12d2352e036dcc13892d6ba5f85cc0130cccb Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Fri, 10 Apr 2026 13:53:41 +0200 Subject: [PATCH 08/11] Get data from the context object --- roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 index cef39191e..3893cc5a1 100644 --- a/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 +++ b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 @@ -91,7 +91,7 @@ def parse_stepup_lines(a): # FAILED is logged, even though the result is OK, making this check # do the wrong thing now. # - #if data.get("authentication_result") != "OK": + #if data.get("context").("authentication_result") != "OK": # continue # 2. Extract required fields From a7aa40d279c09213b4b858bd04f91a6bff30c706 Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Fri, 10 Apr 2026 14:00:20 +0200 Subject: [PATCH 09/11] Log if data checks failed --- roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 index 3893cc5a1..eef7fdff5 100644 --- a/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 +++ b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 @@ -103,6 +103,10 @@ def parse_stepup_lines(a): # Basic data validation if not user_id or not timestamp: + print( + "Skipping line: validation failed " + f"(user_id={user_id!r}, timestamp={timestamp!r}, request_id={request_id!r})" + ) continue try: @@ -110,6 +114,10 @@ def parse_stepup_lines(a): loginstamp = parse(timestamp).strftime("%Y-%m-%d %H:%M:%S") last_login_date = parse(timestamp).strftime("%Y-%m-%d") except: + print( + "Skipping line: timestamp parsing failed " + f"(timestamp={timestamp!r}, user_id={user_id!r}, error={e})" + ) continue # 4. Insert into MySQL From 6669321a8932cdd5a5de68d26155c08701730594 Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Fri, 10 Apr 2026 14:12:24 +0200 Subject: [PATCH 10/11] more robust parsing of data --- .../templates/parse_stepupauth_to_mysql.py.j2 | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 index eef7fdff5..843fe44bc 100644 --- a/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 +++ b/roles/rsyslog/templates/parse_stepupauth_to_mysql.py.j2 @@ -95,11 +95,17 @@ def parse_stepup_lines(a): # continue # 2. Extract required fields - user_id = data.get("context").("identity_id") - timestamp = data.get("context").("datetime") - request_id = data.get("context").("request_id") - sp_entity_id = data.get("context").("requesting_sp") - idp_entity_id = data.get("context").("authenticating_idp") + context = data.get("context") + + if not isinstance(context, dict): + print("Skipping line: context is missing or invalid") + continue + + user_id = context.get("identity_id") + timestamp = context.get("datetime") + request_id = context.get("request_id") + sp_entity_id = context.get("requesting_sp") + idp_entity_id = context.get("authenticating_idp") # Basic data validation if not user_id or not timestamp: From 0107394b42fab8fe83a48ecf2f5c5202d48c41ca Mon Sep 17 00:00:00 2001 From: Peter Havekes Date: Fri, 10 Apr 2026 15:05:27 +0200 Subject: [PATCH 11/11] Add python3-dateutil to rsyslog role --- roles/rsyslog/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/rsyslog/tasks/main.yml b/roles/rsyslog/tasks/main.yml index 1fc0608dc..a531fd677 100644 --- a/roles/rsyslog/tasks/main.yml +++ b/roles/rsyslog/tasks/main.yml @@ -1,9 +1,10 @@ -- name: Install rsyslog +- name: Install rsyslog and python modules ansible.builtin.package: name: - rsyslog - rsyslog-gnutls - rsyslog-relp + - python3-dateutil state: present notify: - "restart rsyslog"