File 0011-imfile-proper-handling-of-multiline-messag... of Package rsyslog.5379

Overview Repositories Revisions Requests Users Attributes Meta

File 0011-imfile-proper-handling-of-multiline-messages-after-r.patch of Package rsyslog.5379

From dd546f82a51bf2e71a97383fce88fffaa3b2a611 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 24 Mar 2015 11:35:20 +0100
Subject: [PATCH] imfile: proper handling of multiline messages after rsyslog
 restart

handling of persisted info (and polling loops) was not correct.

This needs some cleanup but I hope to find some folks for
independent testing before I declare victory.

see also https://github.com/rsyslog/rsyslog/issues/144

From ed8d132e3c7adaff715ebb12043a63c7b96a7436 Mon Sep 17 00:00:00 2001
From: Nirmoy Das <ndas@suse.de>
Date: Mon, 7 Aug 2017 17:31:09 +0200
Subject: [PATCH] remove debug prints from upstream patch

---
 runtime/stream.c                                   |  8 ++-
 tests/imfile-readmode2-with-persists.sh            | 65 ++++++++++++++++++++++
 .../testsuites/imfile-readmode2-with-persists.conf | 21 +++++++
 3 files changed, 92 insertions(+), 2 deletions(-)
 create mode 100644 tests/imfile-readmode2-with-persists.sh
 create mode 100644 tests/testsuites/imfile-readmode2-with-persists.conf

diff --git a/runtime/stream.c b/runtime/stream.c
index 1ee872c..1bc03c0 100644
--- a/runtime/stream.c
+++ b/runtime/stream.c
@@ -634,7 +634,7 @@ static rsRetVal strmReadChar(strm_t *pThis, uchar *pC)
 	ASSERT(pThis != NULL);
 	ASSERT(pC != NULL);
 
-	/* DEV debug only: DBGOPRINT((obj_t*) pThis, "strmRead index %d, max %d\n", pThis->iBufPtr, pThis->iBufPtrMax); */
+	/* DEV debug only: DBGOPRINT((obj_t*) pThis, "strmRead index %zd, max %zd\n", pThis->iBufPtr, pThis->iBufPtrMax); */
 	if(pThis->iUngetC != -1) {	/* do we have an "unread" char that we need to provide? */
 		*pC = pThis->iUngetC;
 		++pThis->iCurrOffs; /* one more octet read */
@@ -790,7 +790,10 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr, uint8_t mode, sbool bEscapeLF)
 
 finalize_it:
         if(iRet != RS_RET_OK && *ppCStr != NULL) {
+			if(cstrLen(*ppCStr) > 0) {
+				/* we may have an empty string in an unsuccsfull poll or after restart! */
 				rsCStrConstructFromCStr(&pThis->prevLineSegment, *ppCStr);
+			}
                 cstrDestruct(ppCStr);
 	}
 
@@ -1854,7 +1857,8 @@ static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm)
 	l = pThis->inode;
 	objSerializeSCALAR_VAR(pStrm, inode, INT64, l);
 
-	objSerializePTR(pStrm, prevLineSegment, PSZ);
+	cstrFinalize(pThis->prevLineSegment);
+	objSerializePTR(pStrm, prevLineSegment, CSTR);
 
 	i = pThis->bPrevWasNL;
 	objSerializeSCALAR_VAR(pStrm, bPrevWasNL, INT, i);
diff --git a/tests/imfile-readmode2-with-persists.sh b/tests/imfile-readmode2-with-persists.sh
new file mode 100644
index 0000000..3063484
--- /dev/null
+++ b/tests/imfile-readmode2-with-persists.sh
@@ -0,0 +1,65 @@
+# This is part of the rsyslog testbench, licensed under ASL 2.0
+echo ======================================================================
+echo [imfile-readmode2-with-persists.sh]
+source $srcdir/diag.sh init
+source $srcdir/diag.sh startup imfile-readmode2-with-persists.conf
+
+# write the beginning of the file
+echo 'msgnum:0
+ msgnum:1' > rsyslog.input
+echo 'msgnum:2' >> rsyslog.input
+
+# sleep a little to give rsyslog a chance to begin processing
+sleep 1
+
+# now stop and restart rsyslog so that the file info must be
+# persisted and read again on startup. Results should still be
+# correct ;)
+echo stopping rsyslog
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown    # we need to wait until rsyslogd is finished!
+echo restarting rsyslog
+source $srcdir/diag.sh startup imfile-readmode2-with-persists.conf
+echo restarted rsyslog, continuing with test
+
+# write some more lines (see https://github.com/rsyslog/rsyslog/issues/144)
+echo 'msgnum:3
+ msgnum:4' >> rsyslog.input
+echo 'msgnum:5' >> rsyslog.input # this one shouldn't be written to the output file because of ReadMode 2
+
+# give it time to finish
+sleep 1
+
+source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages
+source $srcdir/diag.sh wait-shutdown    # we need to wait until rsyslogd is finished!
+
+# give it time to write the output file
+sleep 1
+
+## check if we have the correct number of messages
+
+NUMLINES=$(grep -c HEADER $srcdir/rsyslog.out.log 2>/dev/null)
+
+if [ -z $NUMLINES ]; then
+  echo "ERROR: expecting at least a match for HEADER, maybe rsyslog.out.log wasn't even written?"
+  exit 1
+else
+  if [ ! $NUMLINES -eq 3 ]; then
+    echo "ERROR: expecting 3 headers, got $NUMLINES"
+    exit 1
+  fi
+fi
+
+## check if all the data we expect to get in the file is there
+
+for i in {1..4}; do
+  grep msgnum:$i $srcdir/rsyslog.out.log > /dev/null 2>&1
+  if [ ! $? -eq 0 ]; then
+    echo "ERROR: expecting the string 'msgnum:$i', it's not there"
+    exit 1
+  fi
+done
+
+## if we got here, all is good :)
+
+source $srcdir/diag.sh exit
diff --git a/tests/testsuites/imfile-readmode2-with-persists.conf b/tests/testsuites/imfile-readmode2-with-persists.conf
new file mode 100644
index 0000000..5d44641
--- /dev/null
+++ b/tests/testsuites/imfile-readmode2-with-persists.conf
@@ -0,0 +1,21 @@
+$IncludeConfig diag-common.conf
+global(workDirectory="test-spool")
+module(load="../plugins/imfile/.libs/imfile")
+
+input(type="imfile"
+      File="./rsyslog.input"
+      Tag="file:"
+      ReadMode="2")
+
+template(name="outfmt" type="list") {
+  constant(value="HEADER ")
+  property(name="msg" format="json")
+  constant(value="\n")
+}
+
+if $msg contains "msgnum:" then
+ action(
+   type="omfile"
+   file="rsyslog.out.log"
+   template="outfmt"
+ )
-- 
2.13.2

Places

File 0011-imfile-proper-handling-of-multiline-messages-after-r.patch of Package rsyslog.5379

Places