Browse Source

New upstream version 8.30.0

upstream upstream/8.30.0
Michael Biebl 5 years ago
parent
commit
c29173b526
  1. 2
      .tarball-version
  2. 208
      ChangeLog
  3. 3
      Makefile.in
  4. 280
      action.c
  5. 3
      config.h.in
  6. 380
      configure
  7. 14
      configure.ac
  8. 115
      grammar/rainerscript.c
  9. 5
      grammar/rainerscript.h
  10. 4
      plugins/imdiag/imdiag.c
  11. 23
      plugins/imfile/imfile.c
  12. 87
      plugins/imjournal/imjournal.c
  13. 6
      plugins/imkafka/imkafka.c
  14. 31
      plugins/imptcp/imptcp.c
  15. 4
      plugins/imtcp/imtcp.c
  16. 953
      plugins/mmanon/mmanon.c
  17. 2
      plugins/mmnormalize/mmnormalize.c
  18. 16
      plugins/omelasticsearch/omelasticsearch.c
  19. 16
      plugins/omkafka/omkafka.c
  20. 71
      plugins/ommysql/ommysql.c
  21. 6
      plugins/omrelp/omrelp.c
  22. 9
      plugins/omuxsock/omuxsock.c
  23. 53
      runtime/datetime.c
  24. 5
      runtime/datetime.h
  25. 11
      runtime/debug.c
  26. 21
      runtime/debug.h
  27. 64
      runtime/glbl.c
  28. 22
      runtime/msg.c
  29. 1
      runtime/msg.h
  30. 7
      runtime/netstrm.c
  31. 5
      runtime/netstrm.h
  32. 31
      runtime/netstrms.c
  33. 3
      runtime/netstrms.h
  34. 5
      runtime/nsd.h
  35. 27
      runtime/nsd_gtls.c
  36. 27
      runtime/nsd_ptcp.c
  37. 19
      runtime/ratelimit.c
  38. 1
      runtime/rsconf.c
  39. 1
      runtime/rsconf.h
  40. 1
      runtime/rsyslog.h
  41. 65
      runtime/stream.c
  42. 9
      runtime/stream.h
  43. 12
      runtime/strmsrv.c
  44. 2
      runtime/strmsrv.h
  45. 19
      runtime/tcps_sess.c
  46. 13
      runtime/tcpsrv.c
  47. 7
      runtime/tcpsrv.h
  48. 10
      runtime/wti.h
  49. 7
      runtime/wtp.c
  50. 1
      template.c
  51. 49
      tests/Makefile.am
  52. 214
      tests/Makefile.in
  53. 32
      tests/action-tx-single-processing.sh
  54. 10
      tests/chkseq.c
  55. 55
      tests/diag.sh
  56. 21
      tests/imfile-wildcards.sh
  57. 4
      tests/imptcp-oversize-message-display.sh
  58. 35
      tests/mmanon_both_modes_compatible.sh
  59. 64
      tests/mmanon_random_128_ipv6.sh
  60. 64
      tests/mmanon_random_32_ipv4.sh
  61. 81
      tests/mmanon_random_cons_128_ipv6.sh
  62. 91
      tests/mmanon_random_cons_32_ipv4.sh
  63. 79
      tests/mmanon_recognize_ipv4.sh
  64. 59
      tests/mmanon_recognize_ipv6.sh
  65. 43
      tests/mmanon_simple_12_ipv4.sh
  66. 87
      tests/mmanon_simple_33_ipv4.sh
  67. 35
      tests/mmanon_simple_8_ipv4.sh
  68. 50
      tests/mmanon_zero_128_ipv6.sh
  69. 35
      tests/mmanon_zero_12_ipv4.sh
  70. 45
      tests/mmanon_zero_33_ipv4.sh
  71. 42
      tests/mmanon_zero_50_ipv6.sh
  72. 41
      tests/mmanon_zero_64_ipv6.sh
  73. 35
      tests/mmanon_zero_8_ipv4.sh
  74. 42
      tests/mmanon_zero_96_ipv6.sh
  75. 44
      tests/rscript_format_time.sh
  76. 84
      tests/rscript_trim.sh
  77. 1
      tests/sndrcv_kafka.sh
  78. 13
      tests/sndrcv_relp_rebind.sh
  79. 1
      tests/sndrcv_tls_priorityString.sh
  80. 4
      tests/testsuites/imfile-wildcards.conf
  81. 1
      tests/testsuites/json_var_case.conf
  82. 9
      tests/testsuites/sndrcv_relp_rebind_rcvr.conf
  83. 12
      tests/testsuites/sndrcv_relp_rebind_sender.conf
  84. 2
      tests/testsuites/sndrcv_tls_priorityString_sender.conf
  85. 13
      tools/iminternal.c
  86. 124
      tools/omfwd.c
  87. 2
      tools/rscryutil.c
  88. 49
      tools/rsyslogd.c

2
.tarball-version

@ -1 +1 @@
8.29.0
8.30.0

208
ChangeLog

@ -1,4 +1,212 @@
------------------------------------------------------------------------------
Version 8.30.0 [v8-stable] 2017-10-17
- NEW BUILD REQUIREMENTS
* libfastjson 0.99.7 is now mandatory
the new version is required to support case-insensitive variable
comparisons, which are now the default
* when building imjournal, libsystemd-journal >= 234 is now recommended
This is to support the imjournal enhancement. Note that it is possible
to build with lower version, but this will degrade imjournal functionality.
- CHANGE OF BEHAVIOUR: all variables are now case-insensitive by default
Formerly, JSON based variables ($!, $., $/) were case-sensitive.
Turn old default back on: global(variables.casesensitve="on")
See ChangeLog entry below for more details.
- core: handle (JSON) variables in case-insensitive way
The variable system inside rsyslog is JSON based (for easy consumption
of JSON input, the prime source of structured data). In JSON, keys
("variable names") are case-sensitive. This causes constant problems
inside rsyslog configurations. A major nit is that case-insensitivity
option inside templates (even if turned on) does not work with JSON
keys because they of inner workings*1.
It is much more natural to treat keys in a case-INsensitive way (e.g.
"$!Key" and "$!key" are the same). We do not expect any real problems
out of this, as key names only differing in case is highly unlikely.
However, as it is possible we provide a way to enable case-sensitivity
via the global(variables.casesensitve="on") global config object.
Note that the default is to do case-insensitive matches. The reason
is that this is the right thing to do in almost all cases, and we do
not envision any problems at all in existing deployments.
*1 Note: case-insensitivity in rsyslog is achieved by converting all
names to lower case. So that the higher speed of strcmp() can be used.
The template option does actually that, convert the template keys to
lower case. Unfortunately, this does not work with JSON, as JSON keys
are NOT converted to lower case.
closes https://github.com/rsyslog/rsyslog/issues/1805
- imjournal: made switching to persistent journal in runtime possible
with this patch imjournal can continue logging after switch to
persistent journal without need to restart rsyslog service
Thanks to github user jvymazal for the patch
- mmanon: complete refactor and enhancements
- add pseudonymization mode
- add address randomization mode
- add support for IPv6 (this also supports various replacement modes)
closes https://github.com/rsyslog/rsyslog/issues/1614
also fixes bugs
- in IPv4 address recognition
closes https://github.com/rsyslog/rsyslog/issues/1720
- in IPv4 simple mode to-be-anonymized bits can get wrong
closes https://github.com/rsyslog/rsyslog/issues/1717
- imfile: add "fileoffset" metadata
This permits to send the offset from which the message was read together
with the message text.
Thanks to github user derekjsonar for the initial implementation which
we extended to use the message start offset.
- RainerScript: add ltrim and rtrim functions
closes https://github.com/rsyslog/rsyslog/issues/1711
- core: report module name when suspending action
Thanks to Orion Poplawski for the patch.
- core: add ability to limit number of error messages going to stderr
This permits to put a hard limit on the number of messages that can
go to stderr. If for nothing else, this capability is helpful for the
testbench. It permits to reduce spamming the test log while still
providing the ability to see initial error messages. Might also be
useful for some practical deployments.
global parameter: global(errorMessagesToStderr.maxNumber)
- tcpsrv subsystem: improvate clarity of some error messages
operating system error message are added to some messages, providing
better advise of what actually is the error cause
- imptcp: include module name in error msg
- imtcp: include module name in error msg
- tls improvement: better error message if certificate file cannot be read
- omfwd: slightly improved error messages during config parsing
They now contain config file/line number information where this was missing.
- ommysql improvements
* Return specifc code for unrecoverable errors. This makes retry processing
more performant and robust.
* error messages improved
* Update to utilize native v8 transactional interface. Previously, it used
the v7 interface with v8 emulation.
* treat server and client-generated messages differently
Server-generated error messages are considered non-recoverable, while
client generated once point into connection problems (which simply can
be retried). This is part of the improvements towards better
message-induced errors. Previous commits did only handle SQL parsing
errors, now we aim to address all of the message-induced error. We assume
that all server-generated errors are such - mysql API unfortunately does
not provide a clear indication of what type if error this is and it is
out of question to check for hundereds of error codes.
closes https://github.com/rsyslog/rsyslog/issues/1830
- ommysql bugfix: do not duplicate entries on failed transaction
If a multi-message batch contains data errors, messages may be
duplicated as connection close is implicit commit (not rollback).
This patch introduces a specific rollback request in those cases.
closes https://github.com/rsyslog/rsyslog/issues/1829
- imtcp bugfix: parameter priorityString was ignored
defaults were always used
- template/bugfix: invalid template option conflict detection
This prevented "option.casesenstive" to be used with the SQL and JSON
options.
- core/actions: fix handling of data-induced errors
Rsyslog core should try to detect data-induced (unrecoverable) errors
during action processing. An example of such is invalid SQL statements.
If the action returns a data error, rsyslog shall retry individual
messages from a batch in an effort to log those without errors. The others
shall be dropped.
This logic was temporarily disabled after the switch to the new v8
transaction interface. Now this bug is fixed and the testbench has been
ammended to detect problems in the future.
closes https://github.com/rsyslog/rsyslog/issues/974
- core/action bugfix: no "action suspended" message during retry processing
The action engine does not emit "action suspended" messages but "resumed"
ones in retry processing. This is wrong, as they are a strong indication
that something does not work correctly. Nevertheless, "resumed" messages
were emitted, which was very puzzling for the user.
This patch fixes it so that "suspend" messages are given during retry
processing. These do not contain a retry timestamp, providing evidence
that a retry is currently being tried.
coses https://github.com/rsyslog/rsyslog/issues/1069
- core/ratelimit bugfix: race can lead to segfault
There was a race in iminternalAddMsg(), where the mutex is
released and after that the passed message object is accessed.
If the mainloop iterates in between, the msg may have already
been deleted by this time, resulting in a segfault.
Most importantly, there is no need to release the mutex lock
early, as suggested by current code. Inside mainloop the mutex
is acquired when it is time to do so, so at worst we'll have a
very slight wait there (which really does not matter at all).
This only happens if a large number of internal messages are emitted.
closes https://github.com/rsyslog/rsyslog/issues/1828
- core bugfix: rsyslog aborts if errmsg is generated in early startup
Note that the segfault can occur only during early startup. Once
rsyslog has started, everything works reliably. This condition can
especially be triggerred by specifying invalid TLS default certificates.
closes https://github.com/rsyslog/rsyslog/issues/1783
closes https://github.com/rsyslog/rsyslog/issues/1786
- core bugfix: informational messages was logged with error severity
When the main loop reaped a child process (a normal action), this was
reported as an error. This caused user confusion. Now it is reported as
an informational message.
- core bugfix: --enable-debugless build was broken
This was a regression from the v8.29.0 debug enhancements
Thanks to Stephen Workman for the patch.
- queue bugfix: file write error message was incorrect
when a queue was restarted from disk file, it almost always
emitted a message claiming
"file opened for non-append write, but already contains xxx bytes"
This message was wrong and did not indicate a real error condition.
The predicate check was incorrect.
closes https://github.com/rsyslog/rsyslog/issues/170 (kind of)
- omrelp bugfix: segfault when rebindinterval parameter is used
- omkafka bugfix: invalid load of failedmsg file on startup if disabled
error "rsyslogd: omkafka: could not load failed messages from "
"file (null) error -2027 - failed messages will not be resend."
occurs because, depite `keepFailedMessages="off"` as a default,
omkafka still tries to check for and load a `(none)` file which
triggers an IO error of sorts according to the 2027 link above.
Obviously, it probably shouldn't try load the file if
`keepFailedMessages` is not enabled.
Thanks to github user JPvRiel for a very good error report and
analysis.
closes https://github.com/rsyslog/rsyslog/issues/1765
- kafka bugfix: problem on invalid kafka configuration values
omkafka ended up in an endless loop and high cpu.
imkafka tried to subscribe to a not connected kafka server.
closes https://github.com/rsyslog/rsyslog/issues/1806
- imudp bugfix: UDP oversize message not properly handled
When a message larger than supported by the UDP stack is to be sent,
EMSGSIZE is returned, but not specifically handled. That in turn
will lead to action suspension. However, this does not make sense
because messages over the UDP max message size simply cannot be sent.
closes https://github.com/rsyslog/rsyslog/issues/1654
- core bugfix: memory corruption during configuration parsing
when omfwd is used with the $streamdriverpermittedpeers legacy
parameter, a memory corruption can occur. This depends on the
length of the provided strings and probably the malloc subsystem.
Once config parsing succeeds, no problem can happen.
Thanks to Brent Douglas for initially reporting this issue and
providing great analysis.
Thanks to github user bwdoll for analyzing this bug and providing
a suggested fix (which is almost what this commit includes).
closes https://github.com/rsyslog/rsyslog/issues/1408
closes https://github.com/rsyslog/rsyslog/issues/1474
- core bugfix: race on worker thread termination during shutdown
The testbench got some occasionally failing tests. Review of
them brought up the idea that there is a race during worker
threat termination. Further investigation showed that this
might be a long-standing issue, but so far did not really
surface as the timing was almost always correct. However,
with the new functionality to emit a message on worker
shutdown (v8.29), the timing got more complex and now this
seemed to occasionally surface.
closes https://github.com/rsyslog/rsyslog/issues/1754
- omelasticsearch: avoid ES5 warnings while sending json in bulkmode
do this by adding proper content type header to ES request
Thanks to William Dauchy for the patch
- omelasticsearch bugfix: incompatibility with newer ElasticSearch version
ElasticSearch changed its API in newer versions. When "bulkmode" is enabled
in omelasticsearch, rsyslog seems to consider all responses from Elasticsearch
as errors, even the successful ones. As a consequence, every single request
ends up being logged into the error file.
closes https://github.com/rsyslog/rsyslog/issues/1731
Thanks to Vincent Quéméner for the patch.
- imptcp bugfix: invalid mutex addressing on some platforms
code did not compile on platforms without atomic instructions
Thanks to github user petiepooo for the patch
- imptcp bugfix: do not accept missing port in legacy listener definition
If legacy listener definition was used, a missing port was accepted during
the config read phase but errored out upon listener activation. This now
errors out directly when processing the config directive.
------------------------------------------------------------------------------
Version 8.29.0 [v8-stable] 2017-08-08
- imptcp: add experimental parameter "multiline"
This enables a heuristic to support multiline messages on raw tcp syslog

3
Makefile.in

@ -286,8 +286,7 @@ DIST_SUBDIRS = compat runtime grammar . plugins/immark \
contrib/omtcl tests
am__DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/config.h.in AUTHORS \
COPYING COPYING.LESSER ChangeLog INSTALL NEWS README compile \
config.guess config.sub depcomp install-sh ltmain.sh missing \
ylwrap
config.guess config.sub install-sh ltmain.sh missing ylwrap
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
distdir = $(PACKAGE)-$(VERSION)
top_distdir = $(distdir)

280
action.c

@ -63,7 +63,7 @@
* beast.
* rgerhards, 2011-06-15
*
* Copyright 2007-2016 Rainer Gerhards and Adiscon GmbH.
* Copyright 2007-2017 Rainer Gerhards and Adiscon GmbH.
*
* This file is part of rsyslog.
*
@ -594,8 +594,8 @@ static uchar *getActStateName(action_t * const pThis, wti_t * const pWti)
return (uchar*) "rtry";
case ACT_STATE_SUSP:
return (uchar*) "susp";
case ACT_STATE_COMM:
return (uchar*) "comm";
case ACT_STATE_DATAFAIL:
return (uchar*) "datafail";
default:
return (uchar*) "ERROR/UNKNWON";
}
@ -609,7 +609,6 @@ static rsRetVal getReturnCode(action_t * const pThis, wti_t * const pWti)
{
DEFiRet;
ASSERT(pThis != NULL);
switch(getActionState(pWti, pThis)) {
case ACT_STATE_RDY:
iRet = RS_RET_OK;
@ -628,6 +627,9 @@ static rsRetVal getReturnCode(action_t * const pThis, wti_t * const pWti)
case ACT_STATE_SUSP:
iRet = RS_RET_ACTION_FAILED;
break;
case ACT_STATE_DATAFAIL:
iRet = RS_RET_DATAFAIL;
break;
default:
DBGPRINTF("Invalid action engine state %u, program error\n",
getActionState(pWti, pThis));
@ -646,8 +648,8 @@ static inline void
actionSetState(action_t * const pThis, wti_t * const pWti, uint8_t newState)
{
setActionState(pWti, pThis, newState);
DBGPRINTF("Action %d transitioned to state: %s\n",
pThis->iActionNbr, getActStateName(pThis, pWti));
DBGPRINTF("action[%s] transitioned to state: %s\n",
pThis->pszName, getActStateName(pThis, pWti));
}
/* Handles the transient commit state. So far, this is
@ -660,12 +662,38 @@ static void actionCommitted(action_t * const pThis, wti_t * const pWti)
}
/* we need to defer setting the action's own bReportSuspension state until
* after the full config has been processed. So the most simple case to do
* that is here. It's not a performance problem, as it happens infrequently.
* it's not a threading race problem, as always the same value will be written.
* As we need to do this in several places, we have moved the code to its own
* helper function.
*/
static void
setSuspendMessageConfVars(action_t *__restrict__ const pThis)
{
if(pThis->bReportSuspension == -1)
pThis->bReportSuspension = bActionReportSuspension;
if(pThis->bReportSuspensionCont == -1) {
pThis->bReportSuspensionCont = bActionReportSuspensionCont;
if(pThis->bReportSuspensionCont == -1)
pThis->bReportSuspensionCont = 1;
}
}
/* set action to "rtry" state.
* rgerhards, 2007-08-02
*/
static void actionRetry(action_t * const pThis, wti_t * const pWti)
{
setSuspendMessageConfVars(pThis);
actionSetState(pThis, pWti, ACT_STATE_RTRY);
LogMsg(0, RS_RET_SUSPENDED, LOG_WARNING,
"action '%s' suspended (module '%s'), retry %d. There should "
"be messages before this one giving the reason for suspension.",
pThis->pszName, pThis->pMod->pszName,
getActionNbrResRtry(pWti, pThis));
incActionResumeInRow(pWti, pThis);
}
@ -684,18 +712,7 @@ actionSuspend(action_t * const pThis, wti_t * const pWti)
int suspendDuration;
char timebuf[32];
/* we need to defer setting the action's own bReportSuspension state until
* after the full config has been processed. So the most simple case to do
* that is here. It's not a performance problem, as it happens infrequently.
* it's not a threading race problem, as always the same value will be written.
*/
if(pThis->bReportSuspension == -1)
pThis->bReportSuspension = bActionReportSuspension;
if(pThis->bReportSuspensionCont == -1) {
pThis->bReportSuspensionCont = bActionReportSuspensionCont;
if(pThis->bReportSuspensionCont == -1)
pThis->bReportSuspension = 1;
}
setSuspendMessageConfVars(pThis);
/* note: we can NOT use a cached timestamp, as time may have evolved
* since caching, and this would break logic (and it actually did so!)
@ -714,8 +731,10 @@ actionSuspend(action_t * const pThis, wti_t * const pWti)
ctime_r(&pThis->ttResumeRtry, timebuf);
timebuf[strlen(timebuf)-1] = '\0'; /* strip LF */
errmsg.LogMsg(0, RS_RET_SUSPENDED, LOG_WARNING,
"action '%s' suspended, next retry is %s",
pThis->pszName, timebuf);
"action '%s' suspended (module '%s'), next retry is %s, retry nbr %d. "
"There should be messages before this one giving the reason for suspension.",
pThis->pszName, pThis->pMod->pszName, timebuf,
getActionNbrResRtry(pWti, pThis));
}
DBGPRINTF("action '%s' suspended, earliest retry=%lld (now %lld), iNbrResRtry %d, "
"duration %d\n",
@ -770,7 +789,6 @@ actionDoRetry(action_t * const pThis, wti_t * const pWti)
"resumed (module '%s')",
pThis->pszName, pThis->pMod->pszName);
}
setActionJustResumed(pWti, pThis, 1);
actionSetState(pThis, pWti, ACT_STATE_RDY);
} else if(iRet == RS_RET_SUSPENDED || bTreatOKasSusp) {
/* max retries reached? */
@ -862,14 +880,16 @@ actionTryResume(action_t * const pThis, wti_t * const pWti)
}
if(getActionState(pWti, pThis) == ACT_STATE_RTRY) {
if(ttNow == NO_TIME_PROVIDED) /* use cached result if we have it */
datetime.GetTime(&ttNow);
CHKiRet(actionDoRetry(pThis, pWti));
}
if(Debug && (getActionState(pWti, pThis) == ACT_STATE_RTRY ||getActionState(pWti, pThis) == ACT_STATE_SUSP)) {
DBGPRINTF("actionTryResume: action %p state: %s, next retry (if applicable): %u [now %u]\n",
pThis, getActStateName(pThis, pWti), (unsigned) pThis->ttResumeRtry, (unsigned) ttNow);
if(Debug && (getActionState(pWti, pThis) == ACT_STATE_RTRY ||
getActionState(pWti, pThis) == ACT_STATE_SUSP)) {
if(ttNow == NO_TIME_PROVIDED) /* use cached result if we have it */
datetime.GetTime(&ttNow);
dbgprintf("actionTryResume: action[%s] state: %s, next retry (if applicable): %u [now %u]\n",
pThis->pszName, getActStateName(pThis, pWti),
(unsigned) pThis->ttResumeRtry, (unsigned) ttNow);
}
finalize_it:
@ -913,44 +933,6 @@ finalize_it:
}
#if 0 // TODO: remove?
/* debug-print the contents of an action object
* rgerhards, 2007-08-02
*/
static rsRetVal actionDbgPrint(action_t *pThis)
{
DEFiRet;
char *sz;
dbgprintf("%s: ", module.GetStateName(pThis->pMod));
pThis->pMod->dbgPrintInstInfo(pThis->pModData);
dbgprintf("\n");
dbgprintf("\tInstance data: 0x%lx\n", (unsigned long) pThis->pModData);
dbgprintf("\tResume Interval: %d\n", pThis->iResumeInterval);
#if 0 // do we need this ???
if(getActionState(pWti, pThis) == ACT_STATE_SUSP) {
dbgprintf("\tresume next retry: %u, number retries: %d",
(unsigned) pThis->ttResumeRtry, pThis->iNbrResRtry);
}
#endif
dbgprintf("\tExec only when previous is suspended: %d\n", pThis->bExecWhenPrevSusp);
if(pThis->submitToActQ == doSubmitToActionQComplexBatch) {
sz = "slow, but feature-rich";
} else if(pThis->submitToActQ == doSubmitToActionQNotAllMarkBatch) {
sz = "fast, but supports partial mark messages";
} else if(pThis->submitToActQ == doSubmitToActionQBatch) {
sz = "firehose (fastest)";
} else {
sz = "unknown (need to update debug display?)";
}
dbgprintf("\tsubmission mode: %s\n", sz);
dbgprintf("\n");
RETiRet;
}
#endif
/* prepare the calling parameters for doAction()
* rgerhards, 2009-05-07
*/
@ -1057,16 +1039,6 @@ static void
actionSetActionWorked(action_t *__restrict__ const pThis, wti_t *__restrict__ const pWti)
{
setActionResumeInRow(pWti, pThis, 0);
if(getActionJustResumed(pWti, pThis)) {
/* OK, we *really* could resume, so tell user! */
if(pThis->bReportSuspension) {
errmsg.LogMsg(0, RS_RET_RESUMED, LOG_INFO, "action '%s' "
"resumed (module '%s')",
pThis->pszName, pThis->pMod->pszName);
}
setActionJustResumed(pWti, pThis, 0);
}
}
static rsRetVal
@ -1093,9 +1065,18 @@ handleActionExecResult(action_t *__restrict__ const pThis,
actionDisable(pThis);
break;
case RS_RET_SUSPENDED:
default:/* error happened - if it hits us here, we treat it as suspension */
actionRetry(pThis, pWti);
break;
default:/* error happened - if it hits us here, we assume the message cannot
* be processed but an retry makes no sense. Usually, this should be
* return code RS_RET_DATAFAIL. -- rgerhards, 2017-10-06
*/
LogError(0, ret, "action '%s' (module '%s') "
"message lost, could not be processed. Check for "
"additional error messages before this one.",
pThis->pszName, pThis->pMod->pszName);
actionSetState(pThis, pWti, ACT_STATE_DATAFAIL);
break;
}
iRet = getReturnCode(pThis, pWti);
@ -1135,21 +1116,20 @@ actionCallDoAction(action_t *__restrict__ const pThis,
/* call the commitTransaction output plugin entry point */
static rsRetVal
actionCallCommitTransaction(action_t * const pThis,
const actWrkrInfo_t *const wrkrInfo,
wti_t *const pWti)
wti_t *const pWti,
actWrkrIParams_t *__restrict__ const iparams, const int nparams)
{
DEFiRet;
ASSERT(pThis != NULL);
DBGPRINTF("entering actionCallCommitTransaction(), state: %s, actionNbr %d, "
"nMsgs %u\n",
getActStateName(pThis, pWti), pThis->iActionNbr,
wrkrInfo->p.tx.currIParam);
DBGPRINTF("entering actionCallCommitTransaction[%s], state: %s, nMsgs %u\n",
pThis->pszName, getActStateName(pThis, pWti), nparams);
iRet = pThis->pMod->mod.om.commitTransaction(
pWti->actWrkrInfo[pThis->iActionNbr].actWrkrData,
wrkrInfo->p.tx.iparams, wrkrInfo->p.tx.currIParam);
iparams, nparams);
DBGPRINTF("actionCallCommitTransaction[%s] state: %s "
"mod commitTransaction returned %d\n",
pThis->pszName, getActStateName(pThis, pWti), iRet);
iRet = handleActionExecResult(pThis, pWti, iRet);
RETiRet;
}
@ -1176,9 +1156,10 @@ finalize_it:
}
/* the following functions simulates a potential future new omo callback */
/* the following function uses the new-style transactional interface */
static rsRetVal
doTransaction(action_t *__restrict__ const pThis, wti_t *__restrict__ const pWti)
doTransaction(action_t *__restrict__ const pThis, wti_t *__restrict__ const pWti,
actWrkrIParams_t *__restrict__ const iparams, const int nparams)
{
actWrkrInfo_t *wrkrInfo;
int i;
@ -1187,16 +1168,16 @@ doTransaction(action_t *__restrict__ const pThis, wti_t *__restrict__ const pWti
wrkrInfo = &(pWti->actWrkrInfo[pThis->iActionNbr]);
if(pThis->pMod->mod.om.commitTransaction != NULL) {
DBGPRINTF("doTransaction: have commitTransaction IF, using that, pWrkrInfo %p\n", wrkrInfo);
CHKiRet(actionCallCommitTransaction(pThis, wrkrInfo, pWti));
CHKiRet(actionCallCommitTransaction(pThis, pWti, iparams, nparams));
} else { /* note: this branch is for compatibility with old TX modules */
DBGPRINTF("doTransaction: action %d, currIParam %d\n",
pThis->iActionNbr, wrkrInfo->p.tx.currIParam);
DBGPRINTF("doTransaction: action '%s', currIParam %d\n",
pThis->pszName, wrkrInfo->p.tx.currIParam);
for(i = 0 ; i < wrkrInfo->p.tx.currIParam ; ++i) {
/* Note: we provide the message's base iparam - actionProcessMessage()
* uses this as *base* address.
*/
iRet = actionProcessMessage(pThis,
&actParam(wrkrInfo->p.tx.iparams, pThis->iNumTpls, i, 0), pWti);
&actParam(iparams, pThis->iNumTpls, i, 0), pWti);
if(iRet != RS_RET_DEFER_COMMIT && iRet != RS_RET_PREVIOUS_COMMITTED &&
iRet != RS_RET_OK)
--i; /* we need to re-submit */
@ -1213,13 +1194,16 @@ finalize_it:
/* Commit try committing (do not handle retry processing and such) */
static rsRetVal
actionTryCommit(action_t *__restrict__ const pThis, wti_t *__restrict__ const pWti)
actionTryCommit(action_t *__restrict__ const pThis, wti_t *__restrict__ const pWti,
actWrkrIParams_t *__restrict__ const iparams, const int nparams)
{
DEFiRet;
DBGPRINTF("actionTryCommit[%s] enter\n", pThis->pszName);
CHKiRet(actionPrepare(pThis, pWti));
CHKiRet(doTransaction(pThis, pWti));
CHKiRet(doTransaction(pThis, pWti, iparams, nparams));
DBGPRINTF("actionTryCommit[%s] past doTransaction\n", pThis->pszName);
if(getActionState(pWti, pThis) == ACT_STATE_ITX) {
iRet = pThis->pMod->mod.om.endTransaction(pWti->actWrkrInfo[pThis->iActionNbr].actWrkrData);
@ -1246,12 +1230,15 @@ actionTryCommit(action_t *__restrict__ const pThis, wti_t *__restrict__ const pW
default:/* permanent failure of this message - no sense in retrying. This is
* not yet handled (but easy TODO)
*/
DBGPRINTF("action[%s]: actionTryCommit receveived iRet %d\n",
pThis->pszName, iRet);
FINALIZE;
}
}
iRet = getReturnCode(pThis, pWti);
finalize_it:
DBGPRINTF("actionTryCommit[%s] exit %d\n", pThis->pszName, iRet);
RETiRet;
}
@ -1275,6 +1262,30 @@ actionWriteErrorFile(action_t *__restrict__ const pThis, wti_t *__restrict__ con
}
static rsRetVal
actionTryRemoveHardErrorsFromBatch(action_t *__restrict__ const pThis, wti_t *__restrict__ const pWti,
actWrkrIParams_t *const new_iparams, unsigned *new_nMsgs)
{
actWrkrInfo_t *const wrkrInfo = &(pWti->actWrkrInfo[pThis->iActionNbr]);
const unsigned nMsgs = wrkrInfo->p.tx.currIParam;
actWrkrIParams_t oneParam;
rsRetVal ret;
DEFiRet;
*new_nMsgs = 0;
for(unsigned i = 0 ; i < nMsgs ; ++i) {
setActionResumeInRow(pWti, pThis, 0); // make sure we do not trigger OK-as-SUSPEND handling
memcpy(&oneParam, &actParam(wrkrInfo->p.tx.iparams, 1, i, 0), sizeof(oneParam));
ret = actionTryCommit(pThis, pWti, &oneParam, 1);
DBGPRINTF("msg %d, iRet %d, content: '%s'\n", i, ret, oneParam.param);
if(ret == RS_RET_SUSPENDED) {
memcpy(new_iparams + *new_nMsgs, &oneParam, sizeof(oneParam));
++(*new_nMsgs);
}
}
RETiRet;
}
/* Note: we currently need to return an iRet, as this is used in
* direct mode. TODO: However, it may be worth further investigating this,
* as it looks like there is no ultimate consumer of this code.
@ -1283,38 +1294,81 @@ actionWriteErrorFile(action_t *__restrict__ const pThis, wti_t *__restrict__ con
static rsRetVal
actionCommit(action_t *__restrict__ const pThis, wti_t *__restrict__ const pWti)
{
sbool bDone;
actWrkrInfo_t *const wrkrInfo = &(pWti->actWrkrInfo[pThis->iActionNbr]);
/* Variables that permit us to override the batch of messages */
unsigned nMsgs;
actWrkrIParams_t *iparams = NULL;
int needfree_iparams = 0; // work-around for clang static analyzer false positive
DEFiRet;
DBGPRINTF("actionCommit[%s]: enter, %d msgs\n", pThis->pszName, wrkrInfo->p.tx.currIParam);
if(!pThis->isTransactional ||
pWti->actWrkrInfo[pThis->iActionNbr].p.tx.currIParam == 0 ||
getActionState(pWti, pThis) == ACT_STATE_SUSP
) {
FINALIZE;
}
DBGPRINTF("actionCommit[%s]: processing...\n", pThis->pszName);
/* even more TODO:
This is the place where retry processing needs to go in. If the action
permanently fails, we should - as a new feature - add the capability to
write an error file. This is already done be omelasticsearch, and IMHO
pretty useful.
rgerhards, 2013-11-04
*/
/* we now do one try at commiting the whole batch. Usually, this will
* succeed. If so, we are happy and done. If not, we dig into the details
* of finding out if we have a non-temporary error and try to handle this
* as well as retry processing. Due to this logic we do a bit more retries
* than configured (if temporary failure), but this unavoidable and should
* do no real harm. - rgerhards, 2017-10-06
*/
iRet = actionTryCommit(pThis, pWti, wrkrInfo->p.tx.iparams, wrkrInfo->p.tx.currIParam);
if(iRet == RS_RET_OK) {
FINALIZE;
}
/* even more TODO:
This is the place where retry processing needs to go in. If the action
permanently fails, we should - as a new feature - add the capability to
write an error file. This is already done be omelasticsearch, and IMHO
pretty useful.
For the time being, I do NOT implement all of this (not even retry!)
as I want to get the rest of the engine to SISD (non-SIMD ;)) so that
I know any potential suprises and complications that arise out of this.
When this is done, I can come back here and complete this work. Obviously,
many features do not work in the mean time (but it is not planned to release
any of these partial implementations).
rgerhards, 2013-11-04
/* check if this was a single-message batch. If it had a datafail error, we
* are done. If it is a multi-message batch, we need to sort out the individual
* message states.
*/
bDone = 0;
if(wrkrInfo->p.tx.currIParam == 1) {
needfree_iparams = 0;
iparams = wrkrInfo->p.tx.iparams;
nMsgs = wrkrInfo->p.tx.currIParam;
if(iRet == RS_RET_DATAFAIL) {
FINALIZE;
}
} else {
DBGPRINTF("actionCommit[%s]: somewhat unhappy, full batch of %d msgs returned "
"status %d. Trying messages as individual actions.\n",
pThis->pszName, wrkrInfo->p.tx.currIParam, iRet);
CHKmalloc(iparams = malloc(sizeof(actWrkrIParams_t) * wrkrInfo->p.tx.currIParam));
needfree_iparams = 1;
actionTryRemoveHardErrorsFromBatch(pThis, pWti, iparams, &nMsgs);
}
if(nMsgs == 0) {
ABORT_FINALIZE(RS_RET_OK); // here, we consider everyting OK
}
/* We still have some messages with suspend error. So now let's do our
* "regular" retry and suspend processing.
*/
DBGPRINTF("actionCommit[%s]: unhappy, we still have %d uncommited messages.\n",
pThis->pszName, nMsgs);
int bDone = 0;
do {
iRet = actionTryCommit(pThis, pWti);
DBGPRINTF("actionCommit, action %d, in retry loop, iRet %d\n",
pThis->iActionNbr, iRet);
iRet = actionTryCommit(pThis, pWti, iparams, nMsgs);
DBGPRINTF("actionCommit[%s]: in retry loop, iRet %d\n",
pThis->pszName, iRet);
if(iRet == RS_RET_FORCE_TERM) {
ABORT_FINALIZE(RS_RET_FORCE_TERM);
} else if(iRet == RS_RET_SUSPENDED) {
iRet = actionDoRetry(pThis, pWti);
DBGPRINTF("actionCommit[%s]: actionDoRetry returned %d\n",
pThis->pszName, iRet);
if(iRet == RS_RET_FORCE_TERM) {
ABORT_FINALIZE(RS_RET_FORCE_TERM);
} else if(iRet != RS_RET_OK) {
@ -1327,13 +1381,17 @@ actionCommit(action_t *__restrict__ const pThis, wti_t *__restrict__ const pWti)
iRet == RS_RET_ACTION_FAILED) {
bDone = 1;
}
if(getActionState(pWti, pThis) == ACT_STATE_RDY ||
if(getActionState(pWti, pThis) == ACT_STATE_RDY ||
getActionState(pWti, pThis) == ACT_STATE_SUSP) {
bDone = 1;
}
} while(!bDone);
finalize_it:
pWti->actWrkrInfo[pThis->iActionNbr].p.tx.currIParam = 0; /* reset to beginning */
DBGPRINTF("actionCommit[%s]: done, iRet %d\n", pThis->pszName, iRet);
if(needfree_iparams) {
free(iparams);
}
wrkrInfo->p.tx.currIParam = 0; /* reset to beginning */
RETiRet;
}

3
config.h.in

@ -387,6 +387,9 @@
/* Defined if debug mode is disabled. */
#undef NDEBUG
/* new systemd present */
#undef NEW_JOURNAL
/* Indicator for a AIX OS */
#undef OS_AIX

380
configure

@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for rsyslog 8.29.0.
# Generated by GNU Autoconf 2.69 for rsyslog 8.30.0.
#
# Report bugs to <rsyslog@lists.adiscon.com>.
#
@ -590,8 +590,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='rsyslog'
PACKAGE_TARNAME='rsyslog'
PACKAGE_VERSION='8.29.0'
PACKAGE_STRING='rsyslog 8.29.0'
PACKAGE_VERSION='8.30.0'
PACKAGE_STRING='rsyslog 8.30.0'
PACKAGE_BUGREPORT='rsyslog@lists.adiscon.com'
PACKAGE_URL=''
@ -1744,7 +1744,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures rsyslog 8.29.0 to adapt to many kinds of systems.
\`configure' configures rsyslog 8.30.0 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1815,7 +1815,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of rsyslog 8.29.0:";;
short | recursive ) echo "Configuration of rsyslog 8.30.0:";;
esac
cat <<\_ACEOF
@ -2133,7 +2133,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
rsyslog configure 8.29.0
rsyslog configure 8.30.0
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
@ -2713,7 +2713,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by rsyslog $as_me 8.29.0, which was
It was created by rsyslog $as_me 8.30.0, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@ -3608,7 +3608,7 @@ fi
# Define the identity of the package.
PACKAGE='rsyslog'
VERSION='8.29.0'
VERSION='8.30.0'
cat >>confdefs.h <<_ACEOF
@ -15203,12 +15203,12 @@ if test -n "$LIBFASTJSON_CFLAGS"; then
pkg_cv_LIBFASTJSON_CFLAGS="$LIBFASTJSON_CFLAGS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libfastjson >= 0.99.3\""; } >&5
($PKG_CONFIG --exists --print-errors "libfastjson >= 0.99.3") 2>&5
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libfastjson >= 0.99.7\""; } >&5
($PKG_CONFIG --exists --print-errors "libfastjson >= 0.99.7") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
pkg_cv_LIBFASTJSON_CFLAGS=`$PKG_CONFIG --cflags "libfastjson >= 0.99.3" 2>/dev/null`
pkg_cv_LIBFASTJSON_CFLAGS=`$PKG_CONFIG --cflags "libfastjson >= 0.99.7" 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
@ -15220,12 +15220,12 @@ if test -n "$LIBFASTJSON_LIBS"; then
pkg_cv_LIBFASTJSON_LIBS="$LIBFASTJSON_LIBS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libfastjson >= 0.99.3\""; } >&5
($PKG_CONFIG --exists --print-errors "libfastjson >= 0.99.3") 2>&5
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libfastjson >= 0.99.7\""; } >&5
($PKG_CONFIG --exists --print-errors "libfastjson >= 0.99.7") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
pkg_cv_LIBFASTJSON_LIBS=`$PKG_CONFIG --libs "libfastjson >= 0.99.3" 2>/dev/null`
pkg_cv_LIBFASTJSON_LIBS=`$PKG_CONFIG --libs "libfastjson >= 0.99.7" 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
@ -15246,14 +15246,14 @@ else
_pkg_short_errors_supported=no
fi
if test $_pkg_short_errors_supported = yes; then
LIBFASTJSON_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libfastjson >= 0.99.3" 2>&1`
LIBFASTJSON_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libfastjson >= 0.99.7" 2>&1`
else
LIBFASTJSON_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libfastjson >= 0.99.3" 2>&1`
LIBFASTJSON_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libfastjson >= 0.99.7" 2>&1`
fi
# Put the nasty error message in config.log where it belongs
echo "$LIBFASTJSON_PKG_ERRORS" >&5
as_fn_error $? "Package requirements (libfastjson >= 0.99.3) were not met:
as_fn_error $? "Package requirements (libfastjson >= 0.99.7) were not met:
$LIBFASTJSON_PKG_ERRORS
@ -18071,6 +18071,66 @@ fi
if test "x$enable_imjournal" = "xyes"; then
pkg_failed=no
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBSYSTEMD_JOURNAL" >&5
$as_echo_n "checking for LIBSYSTEMD_JOURNAL... " >&6; }
if test -n "$LIBSYSTEMD_JOURNAL_CFLAGS"; then
pkg_cv_LIBSYSTEMD_JOURNAL_CFLAGS="$LIBSYSTEMD_JOURNAL_CFLAGS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libsystemd >= 234 \""; } >&5
($PKG_CONFIG --exists --print-errors "libsystemd >= 234 ") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
pkg_cv_LIBSYSTEMD_JOURNAL_CFLAGS=`$PKG_CONFIG --cflags "libsystemd >= 234 " 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
fi
else
pkg_failed=untried
fi
if test -n "$LIBSYSTEMD_JOURNAL_LIBS"; then
pkg_cv_LIBSYSTEMD_JOURNAL_LIBS="$LIBSYSTEMD_JOURNAL_LIBS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libsystemd >= 234 \""; } >&5
($PKG_CONFIG --exists --print-errors "libsystemd >= 234 ") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
pkg_cv_LIBSYSTEMD_JOURNAL_LIBS=`$PKG_CONFIG --libs "libsystemd >= 234 " 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
fi
else
pkg_failed=untried
fi
if test $pkg_failed = yes; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
_pkg_short_errors_supported=yes
else
_pkg_short_errors_supported=no
fi
if test $_pkg_short_errors_supported = yes; then
LIBSYSTEMD_JOURNAL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libsystemd >= 234 " 2>&1`
else
LIBSYSTEMD_JOURNAL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libsystemd >= 234 " 2>&1`
fi
# Put the nasty error message in config.log where it belongs
echo "$LIBSYSTEMD_JOURNAL_PKG_ERRORS" >&5
pkg_failed=no
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBSYSTEMD_JOURNAL" >&5
$as_echo_n "checking for LIBSYSTEMD_JOURNAL... " >&6; }
@ -18324,6 +18384,276 @@ else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
fi
elif test $pkg_failed = untried; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
pkg_failed=no
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBSYSTEMD_JOURNAL" >&5
$as_echo_n "checking for LIBSYSTEMD_JOURNAL... " >&6; }
if test -n "$LIBSYSTEMD_JOURNAL_CFLAGS"; then
pkg_cv_LIBSYSTEMD_JOURNAL_CFLAGS="$LIBSYSTEMD_JOURNAL_CFLAGS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libsystemd >= 209 \""; } >&5
($PKG_CONFIG --exists --print-errors "libsystemd >= 209 ") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
pkg_cv_LIBSYSTEMD_JOURNAL_CFLAGS=`$PKG_CONFIG --cflags "libsystemd >= 209 " 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
fi
else
pkg_failed=untried
fi
if test -n "$LIBSYSTEMD_JOURNAL_LIBS"; then
pkg_cv_LIBSYSTEMD_JOURNAL_LIBS="$LIBSYSTEMD_JOURNAL_LIBS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libsystemd >= 209 \""; } >&5
($PKG_CONFIG --exists --print-errors "libsystemd >= 209 ") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
pkg_cv_LIBSYSTEMD_JOURNAL_LIBS=`$PKG_CONFIG --libs "libsystemd >= 209 " 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
fi
else
pkg_failed=untried
fi
if test $pkg_failed = yes; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
_pkg_short_errors_supported=yes
else
_pkg_short_errors_supported=no
fi
if test $_pkg_short_errors_supported = yes; then
LIBSYSTEMD_JOURNAL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libsystemd >= 209 " 2>&1`
else
LIBSYSTEMD_JOURNAL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libsystemd >= 209 " 2>&1`
fi
# Put the nasty error message in config.log where it belongs
echo "$LIBSYSTEMD_JOURNAL_PKG_ERRORS" >&5
pkg_failed=no
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBSYSTEMD_JOURNAL" >&5
$as_echo_n "checking for LIBSYSTEMD_JOURNAL... " >&6; }
if test -n "$LIBSYSTEMD_JOURNAL_CFLAGS"; then
pkg_cv_LIBSYSTEMD_JOURNAL_CFLAGS="$LIBSYSTEMD_JOURNAL_CFLAGS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libsystemd-journal >= 197\""; } >&5
($PKG_CONFIG --exists --print-errors "libsystemd-journal >= 197") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
pkg_cv_LIBSYSTEMD_JOURNAL_CFLAGS=`$PKG_CONFIG --cflags "libsystemd-journal >= 197" 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
fi
else
pkg_failed=untried
fi
if test -n "$LIBSYSTEMD_JOURNAL_LIBS"; then
pkg_cv_LIBSYSTEMD_JOURNAL_LIBS="$LIBSYSTEMD_JOURNAL_LIBS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libsystemd-journal >= 197\""; } >&5
($PKG_CONFIG --exists --print-errors "libsystemd-journal >= 197") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
pkg_cv_LIBSYSTEMD_JOURNAL_LIBS=`$PKG_CONFIG --libs "libsystemd-journal >= 197" 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
fi
else
pkg_failed=untried
fi
if test $pkg_failed = yes; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
_pkg_short_errors_supported=yes
else
_pkg_short_errors_supported=no
fi
if test $_pkg_short_errors_supported = yes; then
LIBSYSTEMD_JOURNAL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libsystemd-journal >= 197" 2>&1`
else
LIBSYSTEMD_JOURNAL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libsystemd-journal >= 197" 2>&1`
fi
# Put the nasty error message in config.log where it belongs
echo "$LIBSYSTEMD_JOURNAL_PKG_ERRORS" >&5
as_fn_error $? "Package requirements (libsystemd-journal >= 197) were not met:
$LIBSYSTEMD_JOURNAL_PKG_ERRORS
Consider adjusting the PKG_CONFIG_PATH environment variable if you
installed software in a non-standard prefix.
Alternatively, you may set the environment variables LIBSYSTEMD_JOURNAL_CFLAGS
and LIBSYSTEMD_JOURNAL_LIBS to avoid the need to call pkg-config.
See the pkg-config man page for more details." "$LINENO" 5
elif test $pkg_failed = untried; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
is in your PATH or set the PKG_CONFIG environment variable to the full
path to pkg-config.
Alternatively, you may set the environment variables LIBSYSTEMD_JOURNAL_CFLAGS
and LIBSYSTEMD_JOURNAL_LIBS to avoid the need to call pkg-config.
See the pkg-config man page for more details.
To get pkg-config, see <http://pkg-config.freedesktop.org/>.
See \`config.log' for more details" "$LINENO" 5; }
else
LIBSYSTEMD_JOURNAL_CFLAGS=$pkg_cv_LIBSYSTEMD_JOURNAL_CFLAGS
LIBSYSTEMD_JOURNAL_LIBS=$pkg_cv_LIBSYSTEMD_JOURNAL_LIBS
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
fi
elif test $pkg_failed = untried; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
pkg_failed=no
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBSYSTEMD_JOURNAL" >&5
$as_echo_n "checking for LIBSYSTEMD_JOURNAL... " >&6; }
if test -n "$LIBSYSTEMD_JOURNAL_CFLAGS"; then
pkg_cv_LIBSYSTEMD_JOURNAL_CFLAGS="$LIBSYSTEMD_JOURNAL_CFLAGS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libsystemd-journal >= 197\""; } >&5
($PKG_CONFIG --exists --print-errors "libsystemd-journal >= 197") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
pkg_cv_LIBSYSTEMD_JOURNAL_CFLAGS=`$PKG_CONFIG --cflags "libsystemd-journal >= 197" 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
fi
else
pkg_failed=untried
fi
if test -n "$LIBSYSTEMD_JOURNAL_LIBS"; then
pkg_cv_LIBSYSTEMD_JOURNAL_LIBS="$LIBSYSTEMD_JOURNAL_LIBS"
elif test -n "$PKG_CONFIG"; then
if test -n "$PKG_CONFIG" && \
{ { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libsystemd-journal >= 197\""; } >&5
($PKG_CONFIG --exists --print-errors "libsystemd-journal >= 197") 2>&5
ac_status=$?
$as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
test $ac_status = 0; }; then
pkg_cv_LIBSYSTEMD_JOURNAL_LIBS=`$PKG_CONFIG --libs "libsystemd-journal >= 197" 2>/dev/null`
test "x$?" != "x0" && pkg_failed=yes
else
pkg_failed=yes
fi
else
pkg_failed=untried
fi
if test $pkg_failed = yes; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
_pkg_short_errors_supported=yes
else
_pkg_short_errors_supported=no
fi
if test $_pkg_short_errors_supported = yes; then
LIBSYSTEMD_JOURNAL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libsystemd-journal >= 197" 2>&1`
else
LIBSYSTEMD_JOURNAL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libsystemd-journal >= 197" 2>&1`
fi
# Put the nasty error message in config.log where it belongs
echo "$LIBSYSTEMD_JOURNAL_PKG_ERRORS" >&5
as_fn_error $? "Package requirements (libsystemd-journal >= 197) were not met:
$LIBSYSTEMD_JOURNAL_PKG_ERRORS
Consider adjusting the PKG_CONFIG_PATH environment variable if you
installed software in a non-standard prefix.
Alternatively, you may set the environment variables LIBSYSTEMD_JOURNAL_CFLAGS
and LIBSYSTEMD_JOURNAL_LIBS to avoid the need to call pkg-config.
See the pkg-config man page for more details." "$LINENO" 5
elif test $pkg_failed = untried; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
is in your PATH or set the PKG_CONFIG environment variable to the full
path to pkg-config.
Alternatively, you may set the environment variables LIBSYSTEMD_JOURNAL_CFLAGS
and LIBSYSTEMD_JOURNAL_LIBS to avoid the need to call pkg-config.
See the pkg-config man page for more details.
To get pkg-config, see <http://pkg-config.freedesktop.org/>.
See \`config.log' for more details" "$LINENO" 5; }
else
LIBSYSTEMD_JOURNAL_CFLAGS=$pkg_cv_LIBSYSTEMD_JOURNAL_CFLAGS
LIBSYSTEMD_JOURNAL_LIBS=$pkg_cv_LIBSYSTEMD_JOURNAL_LIBS
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
fi
else
LIBSYSTEMD_JOURNAL_CFLAGS=$pkg_cv_LIBSYSTEMD_JOURNAL_CFLAGS
LIBSYSTEMD_JOURNAL_LIBS=$pkg_cv_LIBSYSTEMD_JOURNAL_LIBS
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
fi
else
LIBSYSTEMD_JOURNAL_CFLAGS=$pkg_cv_LIBSYSTEMD_JOURNAL_CFLAGS
LIBSYSTEMD_JOURNAL_LIBS=$pkg_cv_LIBSYSTEMD_JOURNAL_LIBS
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
$as_echo "#define NEW_JOURNAL 1" >>confdefs.h
fi
fi
if test x$enable_imjournal = xyes; then
@ -23307,7 +23637,7 @@ else
JAVA_TEST=Test.java
CLASS_TEST=Test.class
cat << \EOF > $JAVA_TEST
/* #line 23310 "configure" */
/* #line 23640 "configure" */
public class Test {
}
EOF
@ -23482,7 +23812,7 @@ EOF
if $UUDECODE Test.uue; then
ac_cv_prog_uudecode_base64=yes
else
echo "configure: 23485: uudecode had trouble decoding base 64 file 'Test.uue'" >&5
echo "configure: 23815: uudecode had trouble decoding base 64 file 'Test.uue'" >&5
echo "configure: failed file was:" >&5
cat Test.uue >&5
ac_cv_prog_uudecode_base64=no
@ -23599,7 +23929,7 @@ else
JAVA_TEST=Test.java
CLASS_TEST=Test.class
cat << \EOF > $JAVA_TEST
/* #line 23602 "configure" */
/* #line 23932 "configure" */
public class Test {
}
EOF
@ -23634,7 +23964,7 @@ JAVA_TEST=Test.java
CLASS_TEST=Test.class
TEST=Test
cat << \EOF > $JAVA_TEST
/* [#]line 23637 "configure" */
/* [#]line 23967 "configure" */
public class Test {
public static void main (String args[]) {
System.exit (0);
@ -26249,7 +26579,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by rsyslog $as_me 8.29.0, which was
This file was extended by rsyslog $as_me 8.30.0, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -26315,7 +26645,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
rsyslog config.status 8.29.0
rsyslog config.status 8.30.0
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
@ -28087,9 +28417,7 @@ echo " Log file signing support: $enable_guardtime"
echo " Log file signing support via KSI LS12: $enable_ksi_ls12"
echo " Log file encryption support: $enable_libgcrypt"
echo " anonymization support enabled: $enable_mmanon"
echo " mmrm1stspace module enabled: $enable_mmrm1stspace"
echo " message counting support enabled: $enable_mmcount"
echo " mmfields enabled: $enable_mmfields"
echo " liblogging-stdlog support enabled: $enable_liblogging_stdlog"
echo " kafka static linking enabled: $enable_kafka_static"
echo
@ -28145,6 +28473,8 @@ echo " mmrfc5424addhmac enabled: $enable_mmrfc5424addhmac"
echo " mmpstrucdata enabled: $enable_mmpstrucdata"
echo " mmsequence enabled: $enable_mmsequence"
echo " mmdblookup enabled: $enable_mmdblookup"
echo " mmfields enabled: $enable_mmfields"
echo " mmrm1stspace module enabled: $enable_mmrm1stspace"
echo
echo "---{ database support }---"
echo " MySql support enabled: $enable_mysql"

14
configure.ac

@ -2,7 +2,7 @@
# Process this file with autoconf to produce a configure script.
AC_PREREQ(2.61)
AC_INIT([rsyslog],[8.29.0],[rsyslog@lists.adiscon.com])
AC_INIT([rsyslog],[8.30.0],[rsyslog@lists.adiscon.com])
# AIXPORT START: Detect the underlying OS
unamestr=$(uname)
@ -75,7 +75,7 @@ PKG_PROG_PKG_CONFIG
# modules we require
PKG_CHECK_MODULES(LIBESTR, libestr >= 0.1.9)
PKG_CHECK_MODULES([LIBFASTJSON], [libfastjson >= 0.99.3],,)
PKG_CHECK_MODULES([LIBFASTJSON], [libfastjson >= 0.99.7],,)
AC_DEFINE_UNQUOTED([PLATFORM_ID], ["${host}"], [platform id for display purposes])
# we don't mind if we don't have the lsb_release utility. But if we have, it's
@ -460,8 +460,10 @@ AC_ARG_ENABLE(imjournal,
[enable_imjournal="no"]
)
if test "x$enable_imjournal" = "xyes"; then
PKG_CHECK_MODULES([LIBSYSTEMD_JOURNAL], [libsystemd >= 209] ,, [
PKG_CHECK_MODULES([LIBSYSTEMD_JOURNAL], [libsystemd-journal >= 197])
PKG_CHECK_MODULES([LIBSYSTEMD_JOURNAL], [libsystemd >= 234] , [AC_DEFINE(NEW_JOURNAL, 1, [new systemd present])] , [
PKG_CHECK_MODULES([LIBSYSTEMD_JOURNAL], [libsystemd >= 209] , , [
PKG_CHECK_MODULES([LIBSYSTEMD_JOURNAL], [libsystemd-journal >= 197])
])
])
fi
AM_CONDITIONAL(ENABLE_IMJOURNAL, test x$enable_imjournal = xyes)
@ -2140,9 +2142,7 @@ echo " Log file signing support: $enable_guardtime"
echo " Log file signing support via KSI LS12: $enable_ksi_ls12"
echo " Log file encryption support: $enable_libgcrypt"
echo " anonymization support enabled: $enable_mmanon"
echo " mmrm1stspace module enabled: $enable_mmrm1stspace"
echo " message counting support enabled: $enable_mmcount"
echo " mmfields enabled: $enable_mmfields"
echo " liblogging-stdlog support enabled: $enable_liblogging_stdlog"
echo " kafka static linking enabled: $enable_kafka_static"
echo
@ -2198,6 +2198,8 @@ echo " mmrfc5424addhmac enabled: $enable_mmrfc5424addhmac"
echo " mmpstrucdata enabled: $enable_mmpstrucdata"
echo " mmsequence enabled: $enable_mmsequence"
echo " mmdblookup enabled: $enable_mmdblookup"
echo " mmfields enabled: $enable_mmfields"
echo " mmrm1stspace module enabled: $enable_mmrm1stspace"
echo
echo "---{ database support }---"
echo " MySql support enabled: $enable_mysql"

115
grammar/rainerscript.c

@ -45,12 +45,14 @@
#include "queue.h"
#include "srUtils.h"
#include "regexp.h"
#include "datetime.h"
#include "obj.h"
#include "modules.h"
#include "ruleset.h"
#include "msg.h"
#include "wti.h"
#include "unicode-helper.h"
#include "errmsg.h"
#if !defined(_AIX)
#pragma GCC diagnostic ignored "-Wswitch-enum"
@ -58,6 +60,7 @@
DEFobjCurrIf(obj)
DEFobjCurrIf(regexp)
DEFobjCurrIf(datetime)
struct cnfexpr* cnfexprOptimize(struct cnfexpr *expr);
static void cnfstmtOptimizePRIFilt(struct cnfstmt *stmt);
@ -1720,6 +1723,46 @@ doRandomGen(struct svar *__restrict__ const sourceVal) {
return x % max;
}
static es_str_t*
lTrim(char *str)
{
int len = strlen(str);
int i;
es_str_t *estr = NULL;
for(i = 0; i < len; i++) {
if(str[i] != ' ') {
break;
}
}
if(i != (len - 1)) {
estr = es_newStrFromCStr(str + i, strlen(str) - i);
}
return(estr);
}
static es_str_t*
rTrim(char *str)
{
int len = strlen(str);
int i;
es_str_t *estr = NULL;
for(i = (len - 1); i > -1; i--) {
if(str[i] != ' ') {
break;
}
}
if(i != 0) {
estr = es_newStrFromCStr(str, (i + 1));
}
return(estr);
}
static long long
ipv42num(char *str)
@ -1899,6 +1942,24 @@ doFuncCall(struct cnffunc *__restrict__ const func, struct svar *__restrict__ co
ret->datatype = 'S';
varFreeMembers(&r[0]);
break;
case CNFFUNC_LTRIM:
cnfexprEval(func->expr[0], &r[0], usrptr);
str = (char*)var2CString(&r[0], &bMustFree);
ret->datatype = 'S';
ret->d.estr = lTrim(str);
varFreeMembers(&r[0]);
if(bMustFree)
free(str);
break;
case CNFFUNC_RTRIM:
cnfexprEval(func->expr[0], &r[0], usrptr);
str = (char*)var2CString(&r[0], &bMustFree);
ret->datatype = 'S';
ret->d.estr = rTrim(str);
varFreeMembers(&r[0]);
if(bMustFree)
free(str);
break;
case CNFFUNC_GETENV:
/* note: the optimizer shall have replaced calls to getenv()
* with a constant argument to a single string (once obtained via
@ -2064,6 +2125,54 @@ doFuncCall(struct cnffunc *__restrict__ const func, struct svar *__restrict__ co
if(bMustFree) free(str);
varFreeMembers(&r[1]);
break;
case CNFFUNC_FORMAT_TIME: {
long long unixtime;
const int resMax = 64;
char result[resMax];
char *formatstr = NULL;
cnfexprEval(func->expr[0], &r[0], usrptr);
cnfexprEval(func->expr[1], &r[1], usrptr);
unixtime = var2Number(&r[0], &retval);
// Make sure that the timestamp we got can fit into
// time_t on older systems.
if (sizeof(time_t) == sizeof(int)) {
if (unixtime < INT_MIN || unixtime > INT_MAX) {
LogMsg(
0, RS_RET_VAL_OUT_OF_RANGE, LOG_WARNING,
"Timestamp value %lld is out of range for this system (time_t is 32bits)!\n", unixtime
);
retval = 0;
}
}
// We want the string form too so we can return it as the
// default if we run into problems parsing the number.
str = (char*) var2CString(&r[0], &bMustFree);
formatstr = (char*) es_str2cstr(r[1].d.estr, NULL);
ret->datatype = 'S';
if (objUse(datetime, CORE_COMPONENT) != RS_RET_OK) {
ret->d.estr = es_newStr(0);
} else {
if (!retval || datetime.formatUnixTimeFromTime_t(unixtime, formatstr, result, resMax) == -1) {
strncpy(result, str, resMax);
result[resMax - 1] = '\0';
}
ret->d.estr = es_newStrFromCStr(result, strlen(result));
}
if (bMustFree) free(str);
free(formatstr);
varFreeMembers(&r[0]);
varFreeMembers(&r[1]);
break;
}
default:
if(Debug) {
char *fname = es_str2cstr(func->fname, NULL);
@ -4132,6 +4241,10 @@ funcName2ID(es_str_t *fname, unsigned short nParams)
GENERATE_FUNC("getenv", 1, CNFFUNC_GETENV);
} else if(FUNC_NAME("num2ipv4")) {
GENERATE_FUNC("num2ipv4", 1, CNFFUNC_NUM2IPV4);
} else if(FUNC_NAME("ltrim")) {
GENERATE_FUNC("ltrim", 1, CNFFUNC_LTRIM);
} else if(FUNC_NAME("rtrim")) {
GENERATE_FUNC("rtrim", 1, CNFFUNC_RTRIM);
} else if(FUNC_NAME("tolower")) {
GENERATE_FUNC