# HG changeset patch # User Chaiwat Suttipongsakul # Date 1687254280 -25200 # Tue Jun 20 16:44:40 2023 +0700 # Node ID e40518e6f6aa49dff7317a3a15ebb5556d91c342 # Parent 0c1d950b3a2c149e661b04663c21385b670ed2e8 Handle both RFC-3164 and RFC-5424 diff --git a/rfc5424.py b/rfc5424.py new file mode 100644 --- /dev/null +++ b/rfc5424.py @@ -0,0 +1,43 @@ +import re +from datetime import datetime +from itertools import zip_longest + +def convert_rfc5424_to_rfc3164(message): + # Extract the required fields from the RFC-5424 message + # Example RFC-5424 message: '<189>Jun 20 06:31:32 FortiGate-101F_02 - eventtime=1687242692932574340 ... + pattern = r'<(\d+)>(\d+ ){0,1}(\S+ \d+ \d+:\d+:\d+) ([\S\s]+) - ([\S\s]+)' + match = re.match(pattern, message) + + if not match: + return message + + # Extract the necessary fields + priority = match.group(1) + version = match.group(2) + timestamp = match.group(3) + hostname = match.group(4) + message = match.group(5) + + # Hostname may contains application name and process id (space separated) + hostname, appname, procid = [x or y for x, y in zip_longest(['', '', ''], hostname.split(' '), fillvalue='')] + syslog_tag = appname + if procid: + syslog_tag += '-{}'.format(procid) + syslog_tag = syslog_tag or '-' # In case that there is no appname and procid + + # Convert the priority to RFC-3164 format + rfc5424_pri = int(priority) + facility = rfc5424_pri // 8 + severity = rfc5424_pri % 8 + rfc3164_pri = facility * 8 + severity + + + # Convert the timestamp to RFC-3164 format (MMM DD HH:MM:SS) + timestamp_dt = datetime.strptime(timestamp, '%b %d %H:%M:%S') + timestamp_rfc3164 = timestamp_dt.strftime('%b %d %H:%M:%S') + + # Rearrange the fields according to RFC-3164 format + rfc3164_message = '<{}>{} {} {}: {}'.format(rfc3164_pri, timestamp_rfc3164, hostname, syslog_tag, message) + + return rfc3164_message + diff --git a/syslogd.py b/syslogd.py --- a/syslogd.py +++ b/syslogd.py @@ -6,6 +6,8 @@ # Configuration import os DEBUG = os.environ.get('DEBUG') == 'True' +SQL_DUMP = os.environ.get('SQL_DUMP') == 'True' +SQL_WRITE = os.environ.get('SQL_WRITE') == 'True' BINDING_IP = os.environ.get('BINDING_IP', '0.0.0.0') BINDING_PORT = os.environ.get('BINDING_PORT', '5140') MYSQL_SERVER = os.environ.get('MYSQL_SERVER', '127.0.0.1') @@ -28,6 +30,7 @@ from gevent.server import DatagramServer from priority import SyslogMatrix from mysqlx.helpers import escape as pure_python_escape +from rfc5424 import convert_rfc5424_to_rfc3164 class SyslogUDPHandler(DatagramServer): @@ -41,30 +44,36 @@ print(f'Listening on UDP {args[0]}') def __del__(self): - self.db.close() + if SQL_WRITE: + self.db.close() def setup(self): - self.db = mysql.connector.connect( - user=MYSQL_USER, - password=MYSQL_PASSWORD, - host=MYSQL_SERVER, - database=MYSQL_DB - ) - self.cursor = self.db.cursor() + if SQL_WRITE: + self.db = mysql.connector.connect( + user=MYSQL_USER, + password=MYSQL_PASSWORD, + host=MYSQL_SERVER, + database=MYSQL_DB + ) + self.cursor = self.db.cursor() def escape(self, msg): - try: - return self.db._cmysql.escape_string(msg).decode() - except AttributeError: + if SQL_WRITE: + try: + return self.db._cmysql.escape_string(msg).decode() + except AttributeError: + return pure_python_escape(msg) + except Exception as e: + if DEBUG: + print('Cannot escape string') + raise(e) + else: return pure_python_escape(msg) - except Exception as e: - if DEBUG: - print('Cannot escape string') - raise(e) def handle(self, data, address): ReceivedAt = datetime.now() # Generated data = data.decode() + data = convert_rfc5424_to_rfc3164(data) datetime_hostname_program = data[data.find('>')+1:data.find(': ')] m,d,t = datetime_hostname_program.split()[:3] formatted_datetime = '%s %s %s %s' % ( @@ -85,7 +94,7 @@ if abs(time_delta.days) > 1: pass # Something wrong, just ignore it. else: - program = datetime_hostname_program.split()[-1] + hostname, program = datetime_hostname_program.split()[-2:] if '[' in program: SysLogTag = program[:program.find('[')] else: @@ -97,7 +106,7 @@ Message = data[data.find(': ')+2:] code = data[data.find('<')+1:data.find('>')] Facility, Priority = self.syslog_matrix.decode_int(code) - FromHost = address[0] + FromHost = hostname or address[0] InfoUnitID = 1 # Hardcoded sql_command = SQL % ( ReceivedAt.strftime('%Y%m'), @@ -107,29 +116,31 @@ SysLogTag, ProcessID, self.escape(Message) ) - try: - self.cursor.execute(sql_command) - except Exception as e: - if DEBUG: - print(sql_command) - print(e) + + if SQL_DUMP: + print('\nSQL:', sql_command) + if SQL_WRITE: try: - self.db.rollback() - except Exception as ee: + self.cursor.execute(sql_command) + except Exception as e: if DEBUG: - print('Fatal Exception! Restarting Process') - print(ee) - self.setup() - else: - if DEBUG: - print(data) - try: - self.db.commit() - except Exception as ee: - if DEBUG: - print('Fatal Exception! Restarting Process') - print(ee) - self.setup() + print(sql_command) + print(e) + try: + self.db.rollback() + except Exception as ee: + if DEBUG: + print('Fatal Exception! Restarting Process') + print(ee) + self.setup() + else: + try: + self.db.commit() + except Exception as ee: + if DEBUG: + print('Fatal Exception! Restarting Process') + print(ee) + self.setup() if __name__ == "__main__":