Assuming syslog guarantees message deliver is a common pitfall.
Neither syslog over TCP nor syslog over UDP implement any application layer confirmation of the receipt of any log messages. If anything occurs between the writing to the buffer or the sending of the log over the socket and the logs having been written to the disk on the receiver, those logs are permanently lost.
In order to reliably transport logs between hosts, application layer confirmations have to be sent and received by the syslog daemons on both hosts. The syslog protocol itself does not implement it. Only RELP (Reliable Event Logging Protocol) of rsyslog, implemented in librelp, implements this. For authentication, privacy and replay protection, rsyslog allows the usage of GnuTLS and OpenSSL certificate based authentication either by using a name (SAN field) or hash (SHA1 hash) whitelist. Check the authmode description for details
librelp is available under the GPLv3 on Github.
Rsyslog and Logstash implement support for RELP. Following are links to documentation regarding support of it.
omrelp (Output Module for RELP in rsyslog)
imrelp (Input Module for RELP in rsyslog)
Relp input plugin (Input plugin for RELP in Logstash)
Following are example configurations taken out of my own logging setup for home usage.
$template authedFiles,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/%PROGRAMNAME%.log"
$template authedCritFiles,"/mnt/logs/authed/crit/%HOSTNAME%-%fromhost-ip%/%PROGRAMNAME%.log"
$template LogLine,"%timegenerated% %HOSTNAME% %pri-text%: %syslogtag% %msg:::drop-last-lf%\n"
$template dnsLine,"%msg:::drop-last-lf%\n"
$template auditFiles,"/mnt/logs/audit/%HOSTNAME%-%fromhost-ip%/audit.log"
$template dnsLogs,"/mnt/logs/dnslog.log"
$template authPrivLog,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/auth.log"
$template syslog,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/syslog"
$template cronLog,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/cron.log"
$template daemonLog,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/daemon.log"
$template kernLog,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/kern.log"
$template lprLog,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/lpr.log"
$template mailLog,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/mail.log"
$template userLog,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/user.log"
$template mailLogInfo,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/mail.info"
$template mailLogWarn,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/mail.warn"
$template mailLogErr,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/mail.err"
$template newsLogCrit,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/news/news.crit"
$template newsLogErr,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/news/news.err"
$template newsLogNotice,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/news/news.notice"
$template debugLog,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/debug"
$template messagesLog,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/syslog-categories/messages"
$template smartdLogs,"/mnt/logs/authed/%HOSTNAME%-%fromhost-ip%/smartd.log
ruleset(name="relp-ruleset" queue.type="fixedArray"){
local0.* -?auditFiles;LogLine
local0.* stop
if ($fromhost == "storage.thermicorp-mgm.lan" or $fromhost == "storage.thermicorp.lan" or $fromhost-ip == "192.168.178.75" or $fromhost-ip == "192.168.180.75") then {
local1.* /mnt/logs/dnslog.log;dnsLine
local1.* stop
}
local2.* ?smartdLogs
*.crit ?authedCritFiles;LogLine
auth,authpriv.* ?authPrivLog
*.*;auth,authpriv.none -?syslog
cron.* -?cronLog
daemon.* -?daemonLog
kern.* -?kernLog
lpr.* -?lprLog
mail.* -?mailLog
user.* -?userLog
mail.info -?mailLogInfo
mail.warn -?mailLogWarn
mail.err -?mailLogErr
news.crit -?newsLogCrit
news.err -?newsLogErr
news.notice -?newsLogNotice
*.=debug;\
auth,authpriv.none;\
news.none;mail.none -?debugLog
*.=info;*.=notice;*.=warn;\
auth,authpriv.none;\
cron,daemon.none;\
mail,news.none -?messagesLog
stop
}
module(load="imrelp")
input(type="imrelp"
port="20514"
maxDataSize="10k"
tls="on"
tls.dhbits="2048"
tls.authmode="name"
tls.PermittedPeer=["*.thermicorp.lan", "*thermicorp-mgm.lan"]
tls.cacert="/etc/ssl/thermicorp-ca-bundle.pem"
tls.mycert="/etc/ssl/syslog_stuff/syslog-1.pem"
tls.myprivkey="/etc/ssl/syslog_stuff/syslog-1.key"
keepalive="on"
tls.prioritystring="SECURE128:SECURE192:-VERS-ALL:+VERS-TLS1.2"
ruleset="relp-ruleset")
module (load="imfile")
module (load="omrelp")
input(type="imfile"
File="/var/log/audit/audit.log"
tag="audit"
Severity="info"
Facility="local0"
PersistStateInterval="60"
)
local2.err action(type="omfwd"
target="netconsole.thermicorp.lan"
port="6666"
protocol="udp"
queue.type="linkedlist"
queue.filename="udp-netconsole-queue"
queue.maxdiskspace="200m"
queue.saveonshutdown="on"
action.copyMsg="on"
action.resumeretrycount="5"
action.resumeInterval="60"
)
local2.err stop
*.* action(type="omrelp"
target="syslog.mgm.thermicorp.lan"
port="20514"
tls="on"
tls.AuthMode="name"
tls.PermittedPeer=["syslog.mgm.thermicorp.lan"]
tls.CaCert="/etc/ssl/all-cas.pem"
tls.MyCert="/etc/ssl/this-host.pem"
tls.MyPrivKey="/etc/ssl/private/this-host.key"
tls.prioritystring="SECURE128:SECURE192:-VERS-ALL:+VERS-TLS1.2"
queue.type="linkedlist"
queue.filename="relp-queue"
queue.maxdiskspace="200m"
queue.saveonshutdown="on"
#action.resumeretrycount="5"
#action.resumeInterval="60"
)
*.* action(type="omfwd"
target="syslog.mgm.thermicorp.lan"
port="514"
protocol="tcp"
queue.type="linkedlist"
queue.filename="tcp-queue"
queue.maxdiskspace="200m"
queue.saveonshutdown="on"
action.resumeretrycount="5"
action.resumeInterval="60"
action.execOnlyWhenPreviousIsSuspended="on"
)