File 0007-mmkubernetes-fix-lnrules-add-defaults-add-... of Package rsyslog

Overview Repositories Revisions Requests Users Attributes Meta

File 0007-mmkubernetes-fix-lnrules-add-defaults-add-test.patch of Package rsyslog (Revision b72e4865a68404dda31625da1f7fed0a)

Currently displaying revision b72e4865a68404dda31625da1f7fed0a , Show latest

From 2b7e62786a0771de3a82d6d477ff33f10b5b8816 Mon Sep 17 00:00:00 2001
From: Rich Megginson <rmeggins@redhat.com>
Date: Thu, 5 Apr 2018 21:57:17 -0600
Subject: [PATCH] mmkubernetes: fix lnrules, add defaults, add test

Fix lnrules for CONTAINER_NAME

Add pkg check for lognorm >= 2.0.3 so we can set the macro
to enable ln_loadSamplesFromString

Add some reasonable default values for parameters, such as
kubernetesurl https://kubernetes.default.svc.cluster.local:443

Clean up sample.conf configuration file

Add test for mmkubernetes, including mock kubernetes service

(cherry picked from commit 1d49aac5cb101704486bfb065fac362ca69f06bc)
---
 configure.ac                        |  14 +++++
 contrib/mmkubernetes/mmkubernetes.c |  69 ++++++++++++++++----
 contrib/mmkubernetes/sample.conf    |  22 ++-----
 tests/Makefile.am                   |  13 +++-
 tests/mmkubernetes-basic-vg.sh      |  89 ++++++++++++++++++++++++++
 tests/mmkubernetes-basic.out.json   | 110 ++++++++++++++++++++++++++++++++
 tests/mmkubernetes-basic.sh         |  88 ++++++++++++++++++++++++++
 tests/mmkubernetes_test_server.py   | 121 ++++++++++++++++++++++++++++++++++++
 8 files changed, 495 insertions(+), 31 deletions(-)
 create mode 100755 tests/mmkubernetes-basic-vg.sh
 create mode 100644 tests/mmkubernetes-basic.out.json
 create mode 100755 tests/mmkubernetes-basic.sh
 create mode 100644 tests/mmkubernetes_test_server.py

diff --git a/configure.ac b/configure.ac
index de679b3ef..17548b9fe 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2099,6 +2099,20 @@ AC_ARG_ENABLE(mmkubernetes,
 )
 if test "x$enable_mmkubernetes" = "xyes"; then
         PKG_CHECK_MODULES([CURL], [libcurl])
+        PKG_CHECK_MODULES(LIBLOGNORM, lognorm >= 2.0.3)
+
+        save_CFLAGS="$CFLAGS"
+        save_LIBS="$LIBS"
+
+        CFLAGS="$CFLAGS $LIBLOGNORM_CFLAGS"
+        LIBS="$LIBS $LIBLOGNORM_LIBS"
+
+        AC_CHECK_FUNC([ln_loadSamplesFromString],
+                      [AC_DEFINE([HAVE_LOADSAMPLESFROMSTRING], [1], [Define if ln_loadSamplesFromString exists.])],
+                      [AC_DEFINE([NO_LOADSAMPLESFROMSTRING], [1], [Define if ln_loadSamplesFromString does not exist.])])
+
+        CFLAGS="$save_CFLAGS"
+        LIBS="$save_LIBS"
 fi
 AM_CONDITIONAL(ENABLE_MMKUBERNETES, test x$enable_mmkubernetes = xyes)
 
diff --git a/contrib/mmkubernetes/mmkubernetes.c b/contrib/mmkubernetes/mmkubernetes.c
index 5012c54f6..6aba930e3 100644
--- a/contrib/mmkubernetes/mmkubernetes.c
+++ b/contrib/mmkubernetes/mmkubernetes.c
@@ -75,10 +75,10 @@ DEFobjCurrIf(regexp)
  * this is for _tag_ match, not actual filename match - in_tail turns filename
  * into a fluentd tag
  */
-#define DFLT_FILENAME_LNRULES ":/var/log/containers/%pod_name:char-to:.%."\
+#define DFLT_FILENAME_LNRULES "rule=:/var/log/containers/%pod_name:char-to:.%."\
 	"%container_hash:char-to:_%_"\
 	"%namespace_name:char-to:_%_%container_name:char-to:-%-%container_id:char-to:.%.log\n"\
-	":/var/log/containers/%pod_name:char-to:_%_"\
+	"rule=:/var/log/containers/%pod_name:char-to:_%_"\
 	"%namespace_name:char-to:_%_%container_name:char-to:-%-%container_id:char-to:.%.log"
 #define DFLT_FILENAME_RULEBASE "/etc/rsyslog.d/k8s_filename.rulebase"
 /* original from fluentd plugin:
@@ -86,10 +86,10 @@ DEFobjCurrIf(regexp)
  *     (\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_\
  *     (?<namespace>[^_]+)_[^_]+_[^_]+$'
  */
-#define DFLT_CONTAINER_LNRULES ":%k8s_prefix:char-to:_%_%container_name:char-to:.%."\
-	"%container_hash:char-to:_%_%"\
+#define DFLT_CONTAINER_LNRULES "rule=:%k8s_prefix:char-to:_%_%container_name:char-to:.%."\
+	"%container_hash:char-to:_%_"\
 	"%pod_name:char-to:_%_%namespace_name:char-to:_%_%not_used_1:char-to:_%_%not_used_2:rest%\n"\
-	":%k8s_prefix:char-to:_%_%container_name:char-to:_%_"\
+	"rule=:%k8s_prefix:char-to:_%_%container_name:char-to:_%_"\
 	"%pod_name:char-to:_%_%namespace_name:char-to:_%_%not_used_1:char-to:_%_%not_used_2:rest%"
 #define DFLT_CONTAINER_RULEBASE "/etc/rsyslog.d/k8s_container_name.rulebase"
 #define DFLT_SRCMD_PATH "$!metadata!filename"
@@ -98,6 +98,7 @@ DEFobjCurrIf(regexp)
 #define DFLT_DE_DOT_SEPARATOR "_"
 #define DFLT_CONTAINER_NAME "$!CONTAINER_NAME" /* name of variable holding CONTAINER_NAME value */
 #define DFLT_CONTAINER_ID_FULL "$!CONTAINER_ID_FULL" /* name of variable holding CONTAINER_ID_FULL value */
+#define DFLT_KUBERNETES_URL "https://kubernetes.default.svc.cluster.local:443"
 
 static struct cache_s {
 	const uchar *kbUrl;
@@ -953,9 +954,11 @@ CODESTARTnewActInst
 			loadModConf->contRules, loadModConf->contRulebase));
 
 	if(pData->kubernetesUrl == NULL) {
-		if(loadModConf->kubernetesUrl == NULL)
-			ABORT_FINALIZE(RS_RET_CONFIG_ERROR);
-		pData->kubernetesUrl = (uchar *) strdup((char *) loadModConf->kubernetesUrl);
+		if(loadModConf->kubernetesUrl == NULL) {
+			CHKmalloc(pData->kubernetesUrl = (uchar *) strdup(DFLT_KUBERNETES_URL));
+		} else {
+			CHKmalloc(pData->kubernetesUrl = (uchar *) strdup((char *) loadModConf->kubernetesUrl));
+		}
 	}
 	if(pData->srcMetadataDescr == NULL) {
 		CHKmalloc(pData->srcMetadataDescr = MALLOC(sizeof(msgPropDescr_t)));
@@ -1125,10 +1128,10 @@ extractMsgMetadata(smsg_t *pMsg, instanceData *pData, struct json_object **json)
 
 	/* extract metadata from the file name */
 	filename = MsgGetProp(pMsg, NULL, pData->srcMetadataDescr, &fnLen, &freeFn, NULL);
-	dbgprintf("mmkubernetes: filename: '%s'.\n", filename);
-	if(filename == NULL)
+	if((filename == NULL) || (fnLen == 0))
 		ABORT_FINALIZE(RS_RET_NOT_FOUND);
 
+	dbgprintf("mmkubernetes: filename: '%s' len %d.\n", filename, fnLen);
 	if ((lnret = ln_normalize(pData->fnCtxln, (char*)filename, fnLen, json))) {
 		ABORT_FINALIZE(RS_RET_ERR);
 	}
@@ -1173,18 +1176,58 @@ queryKB(wrkrInstanceData_t *pWrkrData, char *url, struct json_object **rply)
 	CURLcode ccode;
 	struct json_tokener *jt = NULL;
 	struct json_object *jo;
+	long resp_code = 400;
 
 	/* query kubernetes for pod info */
 	ccode = curl_easy_setopt(pWrkrData->curlCtx, CURLOPT_URL, url);
 	if(ccode != CURLE_OK)
 		ABORT_FINALIZE(RS_RET_ERR);
-	if (CURLE_OK != (ccode = curl_easy_perform(pWrkrData->curlCtx))) {
+	if(CURLE_OK != (ccode = curl_easy_perform(pWrkrData->curlCtx))) {
 		errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
 			      "mmkubernetes: failed to connect to [%s] - %d:%s\n",
 			      url, ccode, curl_easy_strerror(ccode));
 		ABORT_FINALIZE(RS_RET_ERR);
 	}
-
+	if(CURLE_OK != (ccode = curl_easy_getinfo(pWrkrData->curlCtx,
+					CURLINFO_RESPONSE_CODE, &resp_code))) {
+		errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
+			      "mmkubernetes: could not get response code from query to [%s] - %d:%s\n",
+			      url, ccode, curl_easy_strerror(ccode));
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+	if(resp_code == 401) {
+		errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
+			      "mmkubernetes: Unauthorized: not allowed to view url - "
+			      "check token/auth credentials [%s]\n",
+			      url);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+	if(resp_code == 403) {
+		errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
+			      "mmkubernetes: Forbidden: no access - "
+			      "check permissions to view url [%s]\n",
+			      url);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+	if(resp_code == 404) {
+		errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
+			      "mmkubernetes: Not Found: the resource does not exist at url [%s]\n",
+			      url);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+	if(resp_code == 429) {
+		errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
+			      "mmkubernetes: Too Many Requests: the server is too heavily loaded "
+			      "to provide the data for the requested url [%s]\n",
+			      url);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+	if(resp_code != 200) {
+		errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR,
+			      "mmkubernetes: server returned unexpected code [%ld] for url [%s]\n",
+			      resp_code, url);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
 	/* parse retrieved data */
 	jt = json_tokener_new();
 	json_tokener_reset(jt);
@@ -1384,7 +1427,7 @@ CODESTARTdoAction
 	msgAddJSON(pMsg, (uchar *) pWrkrData->pData->dstMetadataPath + 1, jMetadataCopy, 0, 0);
 
 finalize_it:
-    json_object_put(jMsgMeta);
+	json_object_put(jMsgMeta);
 	free(mdKey);
 ENDdoAction
 
diff --git a/contrib/mmkubernetes/sample.conf b/contrib/mmkubernetes/sample.conf
index 4c400ed51..55946003a 100644
--- a/contrib/mmkubernetes/sample.conf
+++ b/contrib/mmkubernetes/sample.conf
@@ -1,19 +1,7 @@
-module(load="imfile" mode="inotify")
-module(load="mmkubernetes" kubernetesurl="https://localhost:8443"
-       tls.cacert="/etc/rsyslog.d/mmk8s.ca.crt"
-       tokenfile="/etc/rsyslog.d/mmk8s.token" annotation_match=["."])
+module(load="mmkubernetes") # see docs for all module and action parameters
 
-template(name="tpl" type="list") {
-    property(name="jsonmesg")
-    constant(value="\n")
-}
+# $!metadata!filename added by imfile using addmetadata="on"
+# e.g. input(type="imfile" file="/var/log/containers/*.log" tag="kubernetes" addmetadata="on")
+# $!CONTAINER_NAME and $!CONTAINER_ID_FULL added by imjournal
 
-ruleset(name="k8s") {
-    action(type="mmkubernetes")
-    action(type="omfile" file="/var/log/k8s.log" template="tpl")
-}
-
-input(type="imfile" file="/var/log/containers/*.log" tag="kubernetes" addmetadata="on" ruleset="k8s")
-if ($!_SYSTEMD_UNIT == "docker.service") and (strlen($!CONTAINER_NAME) > 0) then {
-    call k8s
-}
+action(type="mmkubernetes")
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 4386c626d..bdee87134 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -810,6 +810,15 @@ TESTS += \
 	 omtcl.sh
 endif
 
+if ENABLE_MMKUBERNETES
+TESTS += \
+	mmkubernetes-basic.sh
+if HAVE_VALGRIND
+TESTS += \
+	mmkubernetes-basic-vg.sh
+endif
+endif
+
 endif # if ENABLE_TESTBENCH
 
 TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/
@@ -1841,7 +1850,9 @@ EXTRA_DIST= \
 	pgsql-basic-cnf6-vg.sh \
 	pgsql-template-cnf6-vg.sh \
 	pgsql-actq-mt-withpause-vg.sh \
-	../devtools/prep-mysql-db.sh
+	../devtools/prep-mysql-db.sh \
+	mmkubernetes-basic.sh \
+	mmkubernetes-basic-vg.sh
 
 ourtail_SOURCES = ourtail.c
 msleep_SOURCES = msleep.c
diff --git a/tests/mmkubernetes-basic-vg.sh b/tests/mmkubernetes-basic-vg.sh
new file mode 100755
index 000000000..33f3d3588
--- /dev/null
+++ b/tests/mmkubernetes-basic-vg.sh
@@ -0,0 +1,89 @@
+#!/bin/bash
+# added 2018-04-06 by richm, released under ASL 2.0
+#export RSYSLOG_DEBUG="debug"
+. $srcdir/diag.sh init
+
+testsrv=mmk8s-test-server
+python ./mmkubernetes_test_server.py 18443 rsyslog${testsrv}.pid rsyslogd${testsrv}.started > mmk8s_srv.log 2>&1 &
+BGPROCESS=$!
+. $srcdir/diag.sh wait-startup $testsrv
+echo background mmkubernetes_test_server.py process id is $BGPROCESS
+
+pwd=$( pwd )
+. $srcdir/diag.sh generate-conf
+. $srcdir/diag.sh add-conf '
+module(load="../plugins/imfile/.libs/imfile")
+module(load="../plugins/mmjsonparse/.libs/mmjsonparse")
+module(load="../contrib/mmkubernetes/.libs/mmkubernetes" token="dummy" kubernetesurl="http://localhost:18443"
+       filenamerules=["rule=:'$pwd'/%pod_name:char-to:.%.%container_hash:char-to:_%_%namespace_name:char-to:_%_%container_name_and_id:char-to:.%.log",
+	                  "rule=:'$pwd'/%pod_name:char-to:_%_%namespace_name:char-to:_%_%container_name_and_id:char-to:.%.log"]
+)
+
+template(name="mmk8s_template" type="list") {
+    property(name="$!all-json-plain")
+    constant(value="\n")
+}
+
+input(type="imfile" file="'$pwd'/pod-*.log" tag="kubernetes" addmetadata="on")
+action(type="mmjsonparse" cookie="")
+action(type="mmkubernetes")
+action(type="omfile" file="rsyslog.out.log" template="mmk8s_template")
+'
+cat > pod-name1_namespace-name1_container-name1-id1.log <<EOF
+{"log":"{\"type\":\"response\",\"@timestamp\":\"2018-04-06T17:26:34Z\",\"tags\":[],\"pid\":75,\"method\":\"head\",\"statusCode\":200,\"req\":{\"url\":\"/\",\"method\":\"head\",\"headers\":{\"user-agent\":\"curl/7.29.0\",\"host\":\"localhost:5601\",\"accept\":\"*/*\"},\"remoteAddress\":\"127.0.0.1\",\"userAgent\":\"127.0.0.1\"},\"res\":{\"statusCode\":200,\"responseTime\":1,\"contentLength\":9},\"message\":\"HEAD1 / 200 1ms - 9.0B\"}\n","stream":"stdout","time":"2018-04-06T17:26:34.492083106Z"}
+EOF
+cat > pod-name2.container-hash2_namespace-name2_container-name2-id2.log <<EOF
+{"log":"{\"type\":\"response\",\"@timestamp\":\"2018-04-06T17:26:34Z\",\"tags\":[],\"pid\":75,\"method\":\"head\",\"statusCode\":200,\"req\":{\"url\":\"/\",\"method\":\"head\",\"headers\":{\"user-agent\":\"curl/7.29.0\",\"host\":\"localhost:5601\",\"accept\":\"*/*\"},\"remoteAddress\":\"127.0.0.1\",\"userAgent\":\"127.0.0.1\"},\"res\":{\"statusCode\":200,\"responseTime\":1,\"contentLength\":9},\"message\":\"HEAD2 / 200 1ms - 9.0B\"}\n","stream":"stdout","time":"2018-04-06T17:26:34.492083106Z"}
+EOF
+cat > pod-name3.log <<EOF
+{"message":"a message from container 3","CONTAINER_NAME":"some-prefix_container-name3.container-hash3_pod-name3_namespace-name3_unused3_unused33","CONTAINER_ID_FULL":"id3"}
+EOF
+cat > pod-name4.log <<EOF
+{"message":"a message from container 4","CONTAINER_NAME":"some-prefix_container-name4_pod-name4_namespace-name4_unused4_unused44","CONTAINER_ID_FULL":"id4"}
+EOF
+rm -f imfile-state\:*
+. $srcdir/diag.sh startup-vg-noleak
+sleep 10 || :
+. $srcdir/diag.sh shutdown-when-empty
+. $srcdir/diag.sh wait-shutdown-vg
+. $srcdir/diag.sh check-exit-vg
+
+kill $BGPROCESS
+. $srcdir/diag.sh wait-pid-termination rsyslog${testsrv}.pid
+cat mmk8s_srv.log
+
+# for each record in mmkubernetes-basic.out.json, see if the matching
+# record is found in rsyslog.out.log
+python -c 'import sys,json
+expected = {}
+for hsh in json.load(open(sys.argv[1])):
+	if "kubernetes" in hsh and "pod_name" in hsh["kubernetes"]:
+		expected[hsh["kubernetes"]["pod_name"]] = hsh
+rc = 0
+actual = {}
+for line in open(sys.argv[2]):
+	hsh = json.loads(line)
+	if "kubernetes" in hsh and "pod_name" in hsh["kubernetes"]:
+		actual[hsh["kubernetes"]["pod_name"]] = hsh
+for pod,hsh in expected.items():
+	if not pod in actual:
+		print("Error: record for pod {0} not found in output".format(pod))
+		rc = 1
+	else:
+		for kk,vv in hsh.items():
+			if not kk in actual[pod]:
+				print("Error: key {0} in record for pod {1} not found in output".format(kk, pod))
+				rc = 1
+			elif not vv == actual[pod][kk]:
+				print("Error: value {0} for key {1} in record for pod {2} does not match the expected value {3}".format(actual[pod][kk], kk, pod, vv))
+				rc = 1
+sys.exit(rc)
+' mmkubernetes-basic.out.json rsyslog.out.log
+if [ $? -ne 0 ]; then
+	echo
+	echo "FAIL: expected data not found. rsyslog.out.log is:"
+	cat rsyslog.out.log
+	. $srcdir/diag.sh error-exit 1
+fi
+
+. $srcdir/diag.sh exit
diff --git a/tests/mmkubernetes-basic.out.json b/tests/mmkubernetes-basic.out.json
new file mode 100644
index 000000000..e5876ef21
--- /dev/null
+++ b/tests/mmkubernetes-basic.out.json
@@ -0,0 +1,110 @@
+[{
+  "kubernetes": {
+    "namespace_id": "namespace-name2-id",
+    "namespace_labels": {
+      "label_1_key": "label 1 value",
+      "label_with_empty_value": "",
+      "label_2_key": "label 2 value"
+    },
+    "creation_timestamp": "2018-04-09T21:56:39Z",
+    "pod_id": "pod-name2-id",
+    "labels": {
+      "custom_label": "pod-name2-label-value",
+      "deploymentconfig": "pod-name2-dc",
+      "component": "pod-name2-component",
+      "label_with_empty_value": "",
+      "deployment": "pod-name2-deployment"
+    },
+    "pod_name": "pod-name2",
+    "namespace_name": "namespace-name2",
+    "container_name": "container-name2",
+    "master_url": "http://localhost:18443"
+  },
+  "docker": {
+    "container_id": "id2"
+  }
+},
+{
+  "message": "a message from container 4",
+  "CONTAINER_NAME": "some-prefix_container-name4_pod-name4_namespace-name4_unused4_unused44",
+  "CONTAINER_ID_FULL": "id4",
+  "kubernetes": {
+    "namespace_id": "namespace-name4-id",
+    "namespace_labels": {
+      "label_1_key": "label 1 value",
+      "label_with_empty_value": "",
+      "label_2_key": "label 2 value"
+    },
+    "creation_timestamp": "2018-04-09T21:56:39Z",
+    "pod_id": "pod-name4-id",
+    "labels": {
+      "custom_label": "pod-name4-label-value",
+      "deploymentconfig": "pod-name4-dc",
+      "component": "pod-name4-component",
+      "label_with_empty_value": "",
+      "deployment": "pod-name4-deployment"
+    },
+    "pod_name": "pod-name4",
+    "namespace_name": "namespace-name4",
+    "container_name": "container-name4",
+    "master_url": "http://localhost:18443"
+  },
+  "docker": {
+    "container_id": "id4"
+  }
+},
+{
+  "kubernetes": {
+    "namespace_id": "namespace-name1-id",
+    "namespace_labels": {
+      "label_1_key": "label 1 value",
+      "label_with_empty_value": "",
+      "label_2_key": "label 2 value"
+    },
+    "creation_timestamp": "2018-04-09T21:56:39Z",
+    "pod_id": "pod-name1-id",
+    "labels": {
+      "custom_label": "pod-name1-label-value",
+      "deploymentconfig": "pod-name1-dc",
+      "component": "pod-name1-component",
+      "label_with_empty_value": "",
+      "deployment": "pod-name1-deployment"
+    },
+    "pod_name": "pod-name1",
+    "namespace_name": "namespace-name1",
+    "container_name": "container-name1",
+    "master_url": "http://localhost:18443"
+  },
+  "docker": {
+    "container_id": "id1"
+  }
+},
+{
+  "message": "a message from container 3",
+  "CONTAINER_NAME": "some-prefix_container-name3.container-hash3_pod-name3_namespace-name3_unused3_unused33",
+  "CONTAINER_ID_FULL": "id3",
+  "kubernetes": {
+    "namespace_id": "namespace-name3-id",
+    "namespace_labels": {
+      "label_1_key": "label 1 value",
+      "label_with_empty_value": "",
+      "label_2_key": "label 2 value"
+    },
+    "creation_timestamp": "2018-04-09T21:56:39Z",
+    "pod_id": "pod-name3-id",
+    "labels": {
+      "custom_label": "pod-name3-label-value",
+      "deploymentconfig": "pod-name3-dc",
+      "component": "pod-name3-component",
+      "label_with_empty_value": "",
+      "deployment": "pod-name3-deployment"
+    },
+    "pod_name": "pod-name3",
+    "namespace_name": "namespace-name3",
+    "container_name": "container-name3",
+    "master_url": "http://localhost:18443"
+  },
+  "docker": {
+    "container_id": "id3"
+  }
+}]
diff --git a/tests/mmkubernetes-basic.sh b/tests/mmkubernetes-basic.sh
new file mode 100755
index 000000000..0bbfd08ca
--- /dev/null
+++ b/tests/mmkubernetes-basic.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+# added 2018-04-06 by richm, released under ASL 2.0
+#export RSYSLOG_DEBUG="debug"
+. $srcdir/diag.sh init
+
+testsrv=mmk8s-test-server
+python ./mmkubernetes_test_server.py 18443 rsyslog${testsrv}.pid rsyslogd${testsrv}.started > mmk8s_srv.log 2>&1 &
+BGPROCESS=$!
+. $srcdir/diag.sh wait-startup $testsrv
+echo background mmkubernetes_test_server.py process id is $BGPROCESS
+
+pwd=$( pwd )
+. $srcdir/diag.sh generate-conf
+. $srcdir/diag.sh add-conf '
+module(load="../plugins/imfile/.libs/imfile")
+module(load="../plugins/mmjsonparse/.libs/mmjsonparse")
+module(load="../contrib/mmkubernetes/.libs/mmkubernetes" token="dummy" kubernetesurl="http://localhost:18443"
+       filenamerules=["rule=:'$pwd'/%pod_name:char-to:.%.%container_hash:char-to:_%_%namespace_name:char-to:_%_%container_name_and_id:char-to:.%.log",
+	                  "rule=:'$pwd'/%pod_name:char-to:_%_%namespace_name:char-to:_%_%container_name_and_id:char-to:.%.log"]
+)
+
+template(name="mmk8s_template" type="list") {
+    property(name="$!all-json-plain")
+    constant(value="\n")
+}
+
+input(type="imfile" file="'$pwd'/pod-*.log" tag="kubernetes" addmetadata="on")
+action(type="mmjsonparse" cookie="")
+action(type="mmkubernetes")
+action(type="omfile" file="rsyslog.out.log" template="mmk8s_template")
+'
+cat > pod-name1_namespace-name1_container-name1-id1.log <<EOF
+{"log":"{\"type\":\"response\",\"@timestamp\":\"2018-04-06T17:26:34Z\",\"tags\":[],\"pid\":75,\"method\":\"head\",\"statusCode\":200,\"req\":{\"url\":\"/\",\"method\":\"head\",\"headers\":{\"user-agent\":\"curl/7.29.0\",\"host\":\"localhost:5601\",\"accept\":\"*/*\"},\"remoteAddress\":\"127.0.0.1\",\"userAgent\":\"127.0.0.1\"},\"res\":{\"statusCode\":200,\"responseTime\":1,\"contentLength\":9},\"message\":\"HEAD1 / 200 1ms - 9.0B\"}\n","stream":"stdout","time":"2018-04-06T17:26:34.492083106Z"}
+EOF
+cat > pod-name2.container-hash2_namespace-name2_container-name2-id2.log <<EOF
+{"log":"{\"type\":\"response\",\"@timestamp\":\"2018-04-06T17:26:34Z\",\"tags\":[],\"pid\":75,\"method\":\"head\",\"statusCode\":200,\"req\":{\"url\":\"/\",\"method\":\"head\",\"headers\":{\"user-agent\":\"curl/7.29.0\",\"host\":\"localhost:5601\",\"accept\":\"*/*\"},\"remoteAddress\":\"127.0.0.1\",\"userAgent\":\"127.0.0.1\"},\"res\":{\"statusCode\":200,\"responseTime\":1,\"contentLength\":9},\"message\":\"HEAD2 / 200 1ms - 9.0B\"}\n","stream":"stdout","time":"2018-04-06T17:26:34.492083106Z"}
+EOF
+cat > pod-name3.log <<EOF
+{"message":"a message from container 3","CONTAINER_NAME":"some-prefix_container-name3.container-hash3_pod-name3_namespace-name3_unused3_unused33","CONTAINER_ID_FULL":"id3"}
+EOF
+cat > pod-name4.log <<EOF
+{"message":"a message from container 4","CONTAINER_NAME":"some-prefix_container-name4_pod-name4_namespace-name4_unused4_unused44","CONTAINER_ID_FULL":"id4"}
+EOF
+rm -f imfile-state\:*
+. $srcdir/diag.sh startup
+sleep 10 || :
+. $srcdir/diag.sh shutdown-when-empty
+. $srcdir/diag.sh wait-shutdown
+
+kill $BGPROCESS
+. $srcdir/diag.sh wait-pid-termination rsyslog${testsrv}.pid
+cat mmk8s_srv.log
+
+# for each record in mmkubernetes-basic.out.json, see if the matching
+# record is found in rsyslog.out.log
+python -c 'import sys,json
+expected = {}
+for hsh in json.load(open(sys.argv[1])):
+	if "kubernetes" in hsh and "pod_name" in hsh["kubernetes"]:
+		expected[hsh["kubernetes"]["pod_name"]] = hsh
+rc = 0
+actual = {}
+for line in open(sys.argv[2]):
+	hsh = json.loads(line)
+	if "kubernetes" in hsh and "pod_name" in hsh["kubernetes"]:
+		actual[hsh["kubernetes"]["pod_name"]] = hsh
+for pod,hsh in expected.items():
+	if not pod in actual:
+		print("Error: record for pod {0} not found in output".format(pod))
+		rc = 1
+	else:
+		for kk,vv in hsh.items():
+			if not kk in actual[pod]:
+				print("Error: key {0} in record for pod {1} not found in output".format(kk, pod))
+				rc = 1
+			elif not vv == actual[pod][kk]:
+				print("Error: value {0} for key {1} in record for pod {2} does not match the expected value {3}".format(actual[pod][kk], kk, pod, vv))
+				rc = 1
+sys.exit(rc)
+' mmkubernetes-basic.out.json rsyslog.out.log
+if [ $? -ne 0 ]; then
+	echo
+	echo "FAIL: expected data not found. rsyslog.out.log is:"
+	cat rsyslog.out.log
+	. $srcdir/diag.sh error-exit 1
+fi
+
+. $srcdir/diag.sh exit
diff --git a/tests/mmkubernetes_test_server.py b/tests/mmkubernetes_test_server.py
new file mode 100644
index 000000000..0de215603
--- /dev/null
+++ b/tests/mmkubernetes_test_server.py
@@ -0,0 +1,121 @@
+# Used by the mmkubernetes tests
+# This is a simple http server which responds to kubernetes api requests
+# and responds with kubernetes api server responses
+# added 2018-04-06 by richm, released under ASL 2.0
+import os
+import json
+import sys
+
+try:
+    from http.server import HTTPServer, BaseHTTPRequestHandler
+except ImportError:
+    from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
+
+ns_template = '''{{
+  "kind": "Namespace",
+  "apiVersion": "v1",
+  "metadata": {{
+    "name": "{namespace_name}",
+    "selfLink": "/api/v1/namespaces/{namespace_name}",
+    "uid": "{namespace_name}-id",
+    "resourceVersion": "2988",
+    "creationTimestamp": "2018-04-09T21:56:39Z",
+    "labels": {{
+      "label.1.key":"label 1 value",
+      "label.2.key":"label 2 value",
+      "label.with.empty.value":""
+    }},
+    "annotations": {{
+      "k8s.io/description": "",
+      "k8s.io/display-name": "",
+      "k8s.io/node-selector": "",
+      "k8s.io/sa.scc.mcs": "s0:c9,c4",
+      "k8s.io/sa.scc.supplemental-groups": "1000080000/10000",
+      "k8s.io/sa.scc.uid-range": "1000080000/10000",
+      "quota.k8s.io/cluster-resource-override-enabled": "false"
+    }}
+  }},
+  "spec": {{
+    "finalizers": [
+      "openshift.io/origin",
+      "kubernetes"
+    ]
+  }},
+  "status": {{
+    "phase": "Active"
+  }}
+}}'''
+
+pod_template = '''{{
+  "kind": "Pod",
+  "apiVersion": "v1",
+  "metadata": {{
+    "name": "{pod_name}",
+    "generateName": "{pod_name}-prefix",
+    "namespace": "{namespace_name}",
+    "selfLink": "/api/v1/namespaces/{namespace_name}/pods/{pod_name}",
+    "uid": "{pod_name}-id",
+    "resourceVersion": "3486",
+    "creationTimestamp": "2018-04-09T21:56:39Z",
+    "labels": {{
+      "component": "{pod_name}-component",
+      "deployment": "{pod_name}-deployment",
+      "deploymentconfig": "{pod_name}-dc",
+      "custom.label": "{pod_name}-label-value",
+      "label.with.empty.value":""
+    }},
+    "annotations": {{
+      "k8s.io/deployment-config.latest-version": "1",
+      "k8s.io/deployment-config.name": "{pod_name}-dc",
+      "k8s.io/deployment.name": "{pod_name}-deployment",
+      "k8s.io/custom.name": "custom value",
+      "annotation.with.empty.value":""
+    }}
+  }},
+  "status": {{
+    "phase": "Running",
+    "hostIP": "172.18.4.32",
+    "podIP": "10.128.0.14",
+    "startTime": "2018-04-09T21:57:39Z"
+  }}
+}}'''
+
+class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
+
+    def do_GET(self):
+        # "http://localhost:18443/api/v1/namespaces/namespace-name2"
+        # parse url - either /api/v1/namespaces/$ns_name
+        # or
+        # /api/v1/namespaces/$ns_name/pods/$pod_name
+        comps = self.path.split('/')
+        status = 400
+        if len(comps) >= 5 and comps[1] == 'api' and comps[2] == 'v1':
+            if len(comps) == 5 and comps[3] == 'namespaces': # namespace
+                resp = ns_template.format(namespace_name=comps[4])
+                status = 200
+            elif len(comps) == 7 and comps[3] == 'namespaces' and comps[5] == 'pods':
+                resp = pod_template.format(namespace_name=comps[4], pod_name=comps[6])
+                status = 200
+            else:
+                resp = '{{"error":"do not recognize {0}"}}'.format(self.path)
+        else:
+            resp = '{{"error":"do not recognize {0}"}}'.format(self.path)
+        if not status == 200:
+            self.log_error(resp)
+        self.send_response(status)
+        self.end_headers()
+        self.wfile.write(json.dumps(json.loads(resp), separators=(',',':')))
+
+port = int(sys.argv[1])
+
+httpd = HTTPServer(('localhost', port), SimpleHTTPRequestHandler)
+
+# write "started" to file named in argv[3]
+with open(sys.argv[3], "w") as ff:
+    ff.write("started\n")
+
+# write pid to file named in argv[2]
+with open(sys.argv[2], "w") as ff:
+    ff.write('{0}\n'.format(os.getpid()))
+
+httpd.serve_forever()
-- 
2.16.4

Places

File 0007-mmkubernetes-fix-lnrules-add-defaults-add-test.patch of Package rsyslog (Revision b72e4865a68404dda31625da1f7fed0a)

Places