Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
SUSE:SLE-15-SP1:GA
rsyslog.18313
0007-mmkubernetes-fix-lnrules-add-defaults-add-...
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 0007-mmkubernetes-fix-lnrules-add-defaults-add-test.patch of Package rsyslog.18313
From 2b7e62786a0771de3a82d6d477ff33f10b5b8816 Mon Sep 17 00:00:00 2001 From: Rich Megginson <rmeggins@redhat.com> Date: Thu, 5 Apr 2018 21:57:17 -0600 Subject: [PATCH] mmkubernetes: fix lnrules, add defaults, add test Fix lnrules for CONTAINER_NAME Add pkg check for lognorm >= 2.0.3 so we can set the macro to enable ln_loadSamplesFromString Add some reasonable default values for parameters, such as kubernetesurl https://kubernetes.default.svc.cluster.local:443 Clean up sample.conf configuration file Add test for mmkubernetes, including mock kubernetes service (cherry picked from commit 1d49aac5cb101704486bfb065fac362ca69f06bc) --- configure.ac | 14 +++++ contrib/mmkubernetes/mmkubernetes.c | 69 ++++++++++++++++---- contrib/mmkubernetes/sample.conf | 22 ++----- tests/Makefile.am | 13 +++- tests/mmkubernetes-basic-vg.sh | 89 ++++++++++++++++++++++++++ tests/mmkubernetes-basic.out.json | 110 ++++++++++++++++++++++++++++++++ tests/mmkubernetes-basic.sh | 88 ++++++++++++++++++++++++++ tests/mmkubernetes_test_server.py | 121 ++++++++++++++++++++++++++++++++++++ 8 files changed, 495 insertions(+), 31 deletions(-) create mode 100755 tests/mmkubernetes-basic-vg.sh create mode 100644 tests/mmkubernetes-basic.out.json create mode 100755 tests/mmkubernetes-basic.sh create mode 100644 tests/mmkubernetes_test_server.py diff --git a/configure.ac b/configure.ac index de679b3ef..17548b9fe 100644 --- a/configure.ac +++ b/configure.ac @@ -2099,6 +2099,20 @@ AC_ARG_ENABLE(mmkubernetes, ) if test "x$enable_mmkubernetes" = "xyes"; then PKG_CHECK_MODULES([CURL], [libcurl]) + PKG_CHECK_MODULES(LIBLOGNORM, lognorm >= 2.0.3) + + save_CFLAGS="$CFLAGS" + save_LIBS="$LIBS" + + CFLAGS="$CFLAGS $LIBLOGNORM_CFLAGS" + LIBS="$LIBS $LIBLOGNORM_LIBS" + + AC_CHECK_FUNC([ln_loadSamplesFromString], + [AC_DEFINE([HAVE_LOADSAMPLESFROMSTRING], [1], [Define if ln_loadSamplesFromString exists.])], + [AC_DEFINE([NO_LOADSAMPLESFROMSTRING], [1], [Define if ln_loadSamplesFromString does not exist.])]) + + CFLAGS="$save_CFLAGS" + LIBS="$save_LIBS" fi AM_CONDITIONAL(ENABLE_MMKUBERNETES, test x$enable_mmkubernetes = xyes) diff --git a/contrib/mmkubernetes/mmkubernetes.c b/contrib/mmkubernetes/mmkubernetes.c index 5012c54f6..6aba930e3 100644 --- a/contrib/mmkubernetes/mmkubernetes.c +++ b/contrib/mmkubernetes/mmkubernetes.c @@ -75,10 +75,10 @@ DEFobjCurrIf(regexp) * this is for _tag_ match, not actual filename match - in_tail turns filename * into a fluentd tag */ -#define DFLT_FILENAME_LNRULES ":/var/log/containers/%pod_name:char-to:.%."\ +#define DFLT_FILENAME_LNRULES "rule=:/var/log/containers/%pod_name:char-to:.%."\ "%container_hash:char-to:_%_"\ "%namespace_name:char-to:_%_%container_name:char-to:-%-%container_id:char-to:.%.log\n"\ - ":/var/log/containers/%pod_name:char-to:_%_"\ + "rule=:/var/log/containers/%pod_name:char-to:_%_"\ "%namespace_name:char-to:_%_%container_name:char-to:-%-%container_id:char-to:.%.log" #define DFLT_FILENAME_RULEBASE "/etc/rsyslog.d/k8s_filename.rulebase" /* original from fluentd plugin: @@ -86,10 +86,10 @@ DEFobjCurrIf(regexp) * (\.(?<container_hash>[^_]+))?_(?<pod_name>[^_]+)_\ * (?<namespace>[^_]+)_[^_]+_[^_]+$' */ -#define DFLT_CONTAINER_LNRULES ":%k8s_prefix:char-to:_%_%container_name:char-to:.%."\ - "%container_hash:char-to:_%_%"\ +#define DFLT_CONTAINER_LNRULES "rule=:%k8s_prefix:char-to:_%_%container_name:char-to:.%."\ + "%container_hash:char-to:_%_"\ "%pod_name:char-to:_%_%namespace_name:char-to:_%_%not_used_1:char-to:_%_%not_used_2:rest%\n"\ - ":%k8s_prefix:char-to:_%_%container_name:char-to:_%_"\ + "rule=:%k8s_prefix:char-to:_%_%container_name:char-to:_%_"\ "%pod_name:char-to:_%_%namespace_name:char-to:_%_%not_used_1:char-to:_%_%not_used_2:rest%" #define DFLT_CONTAINER_RULEBASE "/etc/rsyslog.d/k8s_container_name.rulebase" #define DFLT_SRCMD_PATH "$!metadata!filename" @@ -98,6 +98,7 @@ DEFobjCurrIf(regexp) #define DFLT_DE_DOT_SEPARATOR "_" #define DFLT_CONTAINER_NAME "$!CONTAINER_NAME" /* name of variable holding CONTAINER_NAME value */ #define DFLT_CONTAINER_ID_FULL "$!CONTAINER_ID_FULL" /* name of variable holding CONTAINER_ID_FULL value */ +#define DFLT_KUBERNETES_URL "https://kubernetes.default.svc.cluster.local:443" static struct cache_s { const uchar *kbUrl; @@ -953,9 +954,11 @@ CODESTARTnewActInst loadModConf->contRules, loadModConf->contRulebase)); if(pData->kubernetesUrl == NULL) { - if(loadModConf->kubernetesUrl == NULL) - ABORT_FINALIZE(RS_RET_CONFIG_ERROR); - pData->kubernetesUrl = (uchar *) strdup((char *) loadModConf->kubernetesUrl); + if(loadModConf->kubernetesUrl == NULL) { + CHKmalloc(pData->kubernetesUrl = (uchar *) strdup(DFLT_KUBERNETES_URL)); + } else { + CHKmalloc(pData->kubernetesUrl = (uchar *) strdup((char *) loadModConf->kubernetesUrl)); + } } if(pData->srcMetadataDescr == NULL) { CHKmalloc(pData->srcMetadataDescr = MALLOC(sizeof(msgPropDescr_t))); @@ -1125,10 +1128,10 @@ extractMsgMetadata(smsg_t *pMsg, instanceData *pData, struct json_object **json) /* extract metadata from the file name */ filename = MsgGetProp(pMsg, NULL, pData->srcMetadataDescr, &fnLen, &freeFn, NULL); - dbgprintf("mmkubernetes: filename: '%s'.\n", filename); - if(filename == NULL) + if((filename == NULL) || (fnLen == 0)) ABORT_FINALIZE(RS_RET_NOT_FOUND); + dbgprintf("mmkubernetes: filename: '%s' len %d.\n", filename, fnLen); if ((lnret = ln_normalize(pData->fnCtxln, (char*)filename, fnLen, json))) { ABORT_FINALIZE(RS_RET_ERR); } @@ -1173,18 +1176,58 @@ queryKB(wrkrInstanceData_t *pWrkrData, char *url, struct json_object **rply) CURLcode ccode; struct json_tokener *jt = NULL; struct json_object *jo; + long resp_code = 400; /* query kubernetes for pod info */ ccode = curl_easy_setopt(pWrkrData->curlCtx, CURLOPT_URL, url); if(ccode != CURLE_OK) ABORT_FINALIZE(RS_RET_ERR); - if (CURLE_OK != (ccode = curl_easy_perform(pWrkrData->curlCtx))) { + if(CURLE_OK != (ccode = curl_easy_perform(pWrkrData->curlCtx))) { errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR, "mmkubernetes: failed to connect to [%s] - %d:%s\n", url, ccode, curl_easy_strerror(ccode)); ABORT_FINALIZE(RS_RET_ERR); } - + if(CURLE_OK != (ccode = curl_easy_getinfo(pWrkrData->curlCtx, + CURLINFO_RESPONSE_CODE, &resp_code))) { + errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR, + "mmkubernetes: could not get response code from query to [%s] - %d:%s\n", + url, ccode, curl_easy_strerror(ccode)); + ABORT_FINALIZE(RS_RET_ERR); + } + if(resp_code == 401) { + errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR, + "mmkubernetes: Unauthorized: not allowed to view url - " + "check token/auth credentials [%s]\n", + url); + ABORT_FINALIZE(RS_RET_ERR); + } + if(resp_code == 403) { + errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR, + "mmkubernetes: Forbidden: no access - " + "check permissions to view url [%s]\n", + url); + ABORT_FINALIZE(RS_RET_ERR); + } + if(resp_code == 404) { + errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR, + "mmkubernetes: Not Found: the resource does not exist at url [%s]\n", + url); + ABORT_FINALIZE(RS_RET_ERR); + } + if(resp_code == 429) { + errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR, + "mmkubernetes: Too Many Requests: the server is too heavily loaded " + "to provide the data for the requested url [%s]\n", + url); + ABORT_FINALIZE(RS_RET_ERR); + } + if(resp_code != 200) { + errmsg.LogMsg(0, RS_RET_ERR, LOG_ERR, + "mmkubernetes: server returned unexpected code [%ld] for url [%s]\n", + resp_code, url); + ABORT_FINALIZE(RS_RET_ERR); + } /* parse retrieved data */ jt = json_tokener_new(); json_tokener_reset(jt); @@ -1384,7 +1427,7 @@ CODESTARTdoAction msgAddJSON(pMsg, (uchar *) pWrkrData->pData->dstMetadataPath + 1, jMetadataCopy, 0, 0); finalize_it: - json_object_put(jMsgMeta); + json_object_put(jMsgMeta); free(mdKey); ENDdoAction diff --git a/contrib/mmkubernetes/sample.conf b/contrib/mmkubernetes/sample.conf index 4c400ed51..55946003a 100644 --- a/contrib/mmkubernetes/sample.conf +++ b/contrib/mmkubernetes/sample.conf @@ -1,19 +1,7 @@ -module(load="imfile" mode="inotify") -module(load="mmkubernetes" kubernetesurl="https://localhost:8443" - tls.cacert="/etc/rsyslog.d/mmk8s.ca.crt" - tokenfile="/etc/rsyslog.d/mmk8s.token" annotation_match=["."]) +module(load="mmkubernetes") # see docs for all module and action parameters -template(name="tpl" type="list") { - property(name="jsonmesg") - constant(value="\n") -} +# $!metadata!filename added by imfile using addmetadata="on" +# e.g. input(type="imfile" file="/var/log/containers/*.log" tag="kubernetes" addmetadata="on") +# $!CONTAINER_NAME and $!CONTAINER_ID_FULL added by imjournal -ruleset(name="k8s") { - action(type="mmkubernetes") - action(type="omfile" file="/var/log/k8s.log" template="tpl") -} - -input(type="imfile" file="/var/log/containers/*.log" tag="kubernetes" addmetadata="on" ruleset="k8s") -if ($!_SYSTEMD_UNIT == "docker.service") and (strlen($!CONTAINER_NAME) > 0) then { - call k8s -} +action(type="mmkubernetes") diff --git a/tests/Makefile.am b/tests/Makefile.am index 4386c626d..bdee87134 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -810,6 +810,15 @@ TESTS += \ omtcl.sh endif +if ENABLE_MMKUBERNETES +TESTS += \ + mmkubernetes-basic.sh +if HAVE_VALGRIND +TESTS += \ + mmkubernetes-basic-vg.sh +endif +endif + endif # if ENABLE_TESTBENCH TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ @@ -1841,7 +1850,9 @@ EXTRA_DIST= \ pgsql-basic-cnf6-vg.sh \ pgsql-template-cnf6-vg.sh \ pgsql-actq-mt-withpause-vg.sh \ - ../devtools/prep-mysql-db.sh + ../devtools/prep-mysql-db.sh \ + mmkubernetes-basic.sh \ + mmkubernetes-basic-vg.sh ourtail_SOURCES = ourtail.c msleep_SOURCES = msleep.c diff --git a/tests/mmkubernetes-basic-vg.sh b/tests/mmkubernetes-basic-vg.sh new file mode 100755 index 000000000..33f3d3588 --- /dev/null +++ b/tests/mmkubernetes-basic-vg.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# added 2018-04-06 by richm, released under ASL 2.0 +#export RSYSLOG_DEBUG="debug" +. $srcdir/diag.sh init + +testsrv=mmk8s-test-server +python ./mmkubernetes_test_server.py 18443 rsyslog${testsrv}.pid rsyslogd${testsrv}.started > mmk8s_srv.log 2>&1 & +BGPROCESS=$! +. $srcdir/diag.sh wait-startup $testsrv +echo background mmkubernetes_test_server.py process id is $BGPROCESS + +pwd=$( pwd ) +. $srcdir/diag.sh generate-conf +. $srcdir/diag.sh add-conf ' +module(load="../plugins/imfile/.libs/imfile") +module(load="../plugins/mmjsonparse/.libs/mmjsonparse") +module(load="../contrib/mmkubernetes/.libs/mmkubernetes" token="dummy" kubernetesurl="http://localhost:18443" + filenamerules=["rule=:'$pwd'/%pod_name:char-to:.%.%container_hash:char-to:_%_%namespace_name:char-to:_%_%container_name_and_id:char-to:.%.log", + "rule=:'$pwd'/%pod_name:char-to:_%_%namespace_name:char-to:_%_%container_name_and_id:char-to:.%.log"] +) + +template(name="mmk8s_template" type="list") { + property(name="$!all-json-plain") + constant(value="\n") +} + +input(type="imfile" file="'$pwd'/pod-*.log" tag="kubernetes" addmetadata="on") +action(type="mmjsonparse" cookie="") +action(type="mmkubernetes") +action(type="omfile" file="rsyslog.out.log" template="mmk8s_template") +' +cat > pod-name1_namespace-name1_container-name1-id1.log <<EOF +{"log":"{\"type\":\"response\",\"@timestamp\":\"2018-04-06T17:26:34Z\",\"tags\":[],\"pid\":75,\"method\":\"head\",\"statusCode\":200,\"req\":{\"url\":\"/\",\"method\":\"head\",\"headers\":{\"user-agent\":\"curl/7.29.0\",\"host\":\"localhost:5601\",\"accept\":\"*/*\"},\"remoteAddress\":\"127.0.0.1\",\"userAgent\":\"127.0.0.1\"},\"res\":{\"statusCode\":200,\"responseTime\":1,\"contentLength\":9},\"message\":\"HEAD1 / 200 1ms - 9.0B\"}\n","stream":"stdout","time":"2018-04-06T17:26:34.492083106Z"} +EOF +cat > pod-name2.container-hash2_namespace-name2_container-name2-id2.log <<EOF +{"log":"{\"type\":\"response\",\"@timestamp\":\"2018-04-06T17:26:34Z\",\"tags\":[],\"pid\":75,\"method\":\"head\",\"statusCode\":200,\"req\":{\"url\":\"/\",\"method\":\"head\",\"headers\":{\"user-agent\":\"curl/7.29.0\",\"host\":\"localhost:5601\",\"accept\":\"*/*\"},\"remoteAddress\":\"127.0.0.1\",\"userAgent\":\"127.0.0.1\"},\"res\":{\"statusCode\":200,\"responseTime\":1,\"contentLength\":9},\"message\":\"HEAD2 / 200 1ms - 9.0B\"}\n","stream":"stdout","time":"2018-04-06T17:26:34.492083106Z"} +EOF +cat > pod-name3.log <<EOF +{"message":"a message from container 3","CONTAINER_NAME":"some-prefix_container-name3.container-hash3_pod-name3_namespace-name3_unused3_unused33","CONTAINER_ID_FULL":"id3"} +EOF +cat > pod-name4.log <<EOF +{"message":"a message from container 4","CONTAINER_NAME":"some-prefix_container-name4_pod-name4_namespace-name4_unused4_unused44","CONTAINER_ID_FULL":"id4"} +EOF +rm -f imfile-state\:* +. $srcdir/diag.sh startup-vg-noleak +sleep 10 || : +. $srcdir/diag.sh shutdown-when-empty +. $srcdir/diag.sh wait-shutdown-vg +. $srcdir/diag.sh check-exit-vg + +kill $BGPROCESS +. $srcdir/diag.sh wait-pid-termination rsyslog${testsrv}.pid +cat mmk8s_srv.log + +# for each record in mmkubernetes-basic.out.json, see if the matching +# record is found in rsyslog.out.log +python -c 'import sys,json +expected = {} +for hsh in json.load(open(sys.argv[1])): + if "kubernetes" in hsh and "pod_name" in hsh["kubernetes"]: + expected[hsh["kubernetes"]["pod_name"]] = hsh +rc = 0 +actual = {} +for line in open(sys.argv[2]): + hsh = json.loads(line) + if "kubernetes" in hsh and "pod_name" in hsh["kubernetes"]: + actual[hsh["kubernetes"]["pod_name"]] = hsh +for pod,hsh in expected.items(): + if not pod in actual: + print("Error: record for pod {0} not found in output".format(pod)) + rc = 1 + else: + for kk,vv in hsh.items(): + if not kk in actual[pod]: + print("Error: key {0} in record for pod {1} not found in output".format(kk, pod)) + rc = 1 + elif not vv == actual[pod][kk]: + print("Error: value {0} for key {1} in record for pod {2} does not match the expected value {3}".format(actual[pod][kk], kk, pod, vv)) + rc = 1 +sys.exit(rc) +' mmkubernetes-basic.out.json rsyslog.out.log +if [ $? -ne 0 ]; then + echo + echo "FAIL: expected data not found. rsyslog.out.log is:" + cat rsyslog.out.log + . $srcdir/diag.sh error-exit 1 +fi + +. $srcdir/diag.sh exit diff --git a/tests/mmkubernetes-basic.out.json b/tests/mmkubernetes-basic.out.json new file mode 100644 index 000000000..e5876ef21 --- /dev/null +++ b/tests/mmkubernetes-basic.out.json @@ -0,0 +1,110 @@ +[{ + "kubernetes": { + "namespace_id": "namespace-name2-id", + "namespace_labels": { + "label_1_key": "label 1 value", + "label_with_empty_value": "", + "label_2_key": "label 2 value" + }, + "creation_timestamp": "2018-04-09T21:56:39Z", + "pod_id": "pod-name2-id", + "labels": { + "custom_label": "pod-name2-label-value", + "deploymentconfig": "pod-name2-dc", + "component": "pod-name2-component", + "label_with_empty_value": "", + "deployment": "pod-name2-deployment" + }, + "pod_name": "pod-name2", + "namespace_name": "namespace-name2", + "container_name": "container-name2", + "master_url": "http://localhost:18443" + }, + "docker": { + "container_id": "id2" + } +}, +{ + "message": "a message from container 4", + "CONTAINER_NAME": "some-prefix_container-name4_pod-name4_namespace-name4_unused4_unused44", + "CONTAINER_ID_FULL": "id4", + "kubernetes": { + "namespace_id": "namespace-name4-id", + "namespace_labels": { + "label_1_key": "label 1 value", + "label_with_empty_value": "", + "label_2_key": "label 2 value" + }, + "creation_timestamp": "2018-04-09T21:56:39Z", + "pod_id": "pod-name4-id", + "labels": { + "custom_label": "pod-name4-label-value", + "deploymentconfig": "pod-name4-dc", + "component": "pod-name4-component", + "label_with_empty_value": "", + "deployment": "pod-name4-deployment" + }, + "pod_name": "pod-name4", + "namespace_name": "namespace-name4", + "container_name": "container-name4", + "master_url": "http://localhost:18443" + }, + "docker": { + "container_id": "id4" + } +}, +{ + "kubernetes": { + "namespace_id": "namespace-name1-id", + "namespace_labels": { + "label_1_key": "label 1 value", + "label_with_empty_value": "", + "label_2_key": "label 2 value" + }, + "creation_timestamp": "2018-04-09T21:56:39Z", + "pod_id": "pod-name1-id", + "labels": { + "custom_label": "pod-name1-label-value", + "deploymentconfig": "pod-name1-dc", + "component": "pod-name1-component", + "label_with_empty_value": "", + "deployment": "pod-name1-deployment" + }, + "pod_name": "pod-name1", + "namespace_name": "namespace-name1", + "container_name": "container-name1", + "master_url": "http://localhost:18443" + }, + "docker": { + "container_id": "id1" + } +}, +{ + "message": "a message from container 3", + "CONTAINER_NAME": "some-prefix_container-name3.container-hash3_pod-name3_namespace-name3_unused3_unused33", + "CONTAINER_ID_FULL": "id3", + "kubernetes": { + "namespace_id": "namespace-name3-id", + "namespace_labels": { + "label_1_key": "label 1 value", + "label_with_empty_value": "", + "label_2_key": "label 2 value" + }, + "creation_timestamp": "2018-04-09T21:56:39Z", + "pod_id": "pod-name3-id", + "labels": { + "custom_label": "pod-name3-label-value", + "deploymentconfig": "pod-name3-dc", + "component": "pod-name3-component", + "label_with_empty_value": "", + "deployment": "pod-name3-deployment" + }, + "pod_name": "pod-name3", + "namespace_name": "namespace-name3", + "container_name": "container-name3", + "master_url": "http://localhost:18443" + }, + "docker": { + "container_id": "id3" + } +}] diff --git a/tests/mmkubernetes-basic.sh b/tests/mmkubernetes-basic.sh new file mode 100755 index 000000000..0bbfd08ca --- /dev/null +++ b/tests/mmkubernetes-basic.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# added 2018-04-06 by richm, released under ASL 2.0 +#export RSYSLOG_DEBUG="debug" +. $srcdir/diag.sh init + +testsrv=mmk8s-test-server +python ./mmkubernetes_test_server.py 18443 rsyslog${testsrv}.pid rsyslogd${testsrv}.started > mmk8s_srv.log 2>&1 & +BGPROCESS=$! +. $srcdir/diag.sh wait-startup $testsrv +echo background mmkubernetes_test_server.py process id is $BGPROCESS + +pwd=$( pwd ) +. $srcdir/diag.sh generate-conf +. $srcdir/diag.sh add-conf ' +module(load="../plugins/imfile/.libs/imfile") +module(load="../plugins/mmjsonparse/.libs/mmjsonparse") +module(load="../contrib/mmkubernetes/.libs/mmkubernetes" token="dummy" kubernetesurl="http://localhost:18443" + filenamerules=["rule=:'$pwd'/%pod_name:char-to:.%.%container_hash:char-to:_%_%namespace_name:char-to:_%_%container_name_and_id:char-to:.%.log", + "rule=:'$pwd'/%pod_name:char-to:_%_%namespace_name:char-to:_%_%container_name_and_id:char-to:.%.log"] +) + +template(name="mmk8s_template" type="list") { + property(name="$!all-json-plain") + constant(value="\n") +} + +input(type="imfile" file="'$pwd'/pod-*.log" tag="kubernetes" addmetadata="on") +action(type="mmjsonparse" cookie="") +action(type="mmkubernetes") +action(type="omfile" file="rsyslog.out.log" template="mmk8s_template") +' +cat > pod-name1_namespace-name1_container-name1-id1.log <<EOF +{"log":"{\"type\":\"response\",\"@timestamp\":\"2018-04-06T17:26:34Z\",\"tags\":[],\"pid\":75,\"method\":\"head\",\"statusCode\":200,\"req\":{\"url\":\"/\",\"method\":\"head\",\"headers\":{\"user-agent\":\"curl/7.29.0\",\"host\":\"localhost:5601\",\"accept\":\"*/*\"},\"remoteAddress\":\"127.0.0.1\",\"userAgent\":\"127.0.0.1\"},\"res\":{\"statusCode\":200,\"responseTime\":1,\"contentLength\":9},\"message\":\"HEAD1 / 200 1ms - 9.0B\"}\n","stream":"stdout","time":"2018-04-06T17:26:34.492083106Z"} +EOF +cat > pod-name2.container-hash2_namespace-name2_container-name2-id2.log <<EOF +{"log":"{\"type\":\"response\",\"@timestamp\":\"2018-04-06T17:26:34Z\",\"tags\":[],\"pid\":75,\"method\":\"head\",\"statusCode\":200,\"req\":{\"url\":\"/\",\"method\":\"head\",\"headers\":{\"user-agent\":\"curl/7.29.0\",\"host\":\"localhost:5601\",\"accept\":\"*/*\"},\"remoteAddress\":\"127.0.0.1\",\"userAgent\":\"127.0.0.1\"},\"res\":{\"statusCode\":200,\"responseTime\":1,\"contentLength\":9},\"message\":\"HEAD2 / 200 1ms - 9.0B\"}\n","stream":"stdout","time":"2018-04-06T17:26:34.492083106Z"} +EOF +cat > pod-name3.log <<EOF +{"message":"a message from container 3","CONTAINER_NAME":"some-prefix_container-name3.container-hash3_pod-name3_namespace-name3_unused3_unused33","CONTAINER_ID_FULL":"id3"} +EOF +cat > pod-name4.log <<EOF +{"message":"a message from container 4","CONTAINER_NAME":"some-prefix_container-name4_pod-name4_namespace-name4_unused4_unused44","CONTAINER_ID_FULL":"id4"} +EOF +rm -f imfile-state\:* +. $srcdir/diag.sh startup +sleep 10 || : +. $srcdir/diag.sh shutdown-when-empty +. $srcdir/diag.sh wait-shutdown + +kill $BGPROCESS +. $srcdir/diag.sh wait-pid-termination rsyslog${testsrv}.pid +cat mmk8s_srv.log + +# for each record in mmkubernetes-basic.out.json, see if the matching +# record is found in rsyslog.out.log +python -c 'import sys,json +expected = {} +for hsh in json.load(open(sys.argv[1])): + if "kubernetes" in hsh and "pod_name" in hsh["kubernetes"]: + expected[hsh["kubernetes"]["pod_name"]] = hsh +rc = 0 +actual = {} +for line in open(sys.argv[2]): + hsh = json.loads(line) + if "kubernetes" in hsh and "pod_name" in hsh["kubernetes"]: + actual[hsh["kubernetes"]["pod_name"]] = hsh +for pod,hsh in expected.items(): + if not pod in actual: + print("Error: record for pod {0} not found in output".format(pod)) + rc = 1 + else: + for kk,vv in hsh.items(): + if not kk in actual[pod]: + print("Error: key {0} in record for pod {1} not found in output".format(kk, pod)) + rc = 1 + elif not vv == actual[pod][kk]: + print("Error: value {0} for key {1} in record for pod {2} does not match the expected value {3}".format(actual[pod][kk], kk, pod, vv)) + rc = 1 +sys.exit(rc) +' mmkubernetes-basic.out.json rsyslog.out.log +if [ $? -ne 0 ]; then + echo + echo "FAIL: expected data not found. rsyslog.out.log is:" + cat rsyslog.out.log + . $srcdir/diag.sh error-exit 1 +fi + +. $srcdir/diag.sh exit diff --git a/tests/mmkubernetes_test_server.py b/tests/mmkubernetes_test_server.py new file mode 100644 index 000000000..0de215603 --- /dev/null +++ b/tests/mmkubernetes_test_server.py @@ -0,0 +1,121 @@ +# Used by the mmkubernetes tests +# This is a simple http server which responds to kubernetes api requests +# and responds with kubernetes api server responses +# added 2018-04-06 by richm, released under ASL 2.0 +import os +import json +import sys + +try: + from http.server import HTTPServer, BaseHTTPRequestHandler +except ImportError: + from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler + +ns_template = '''{{ + "kind": "Namespace", + "apiVersion": "v1", + "metadata": {{ + "name": "{namespace_name}", + "selfLink": "/api/v1/namespaces/{namespace_name}", + "uid": "{namespace_name}-id", + "resourceVersion": "2988", + "creationTimestamp": "2018-04-09T21:56:39Z", + "labels": {{ + "label.1.key":"label 1 value", + "label.2.key":"label 2 value", + "label.with.empty.value":"" + }}, + "annotations": {{ + "k8s.io/description": "", + "k8s.io/display-name": "", + "k8s.io/node-selector": "", + "k8s.io/sa.scc.mcs": "s0:c9,c4", + "k8s.io/sa.scc.supplemental-groups": "1000080000/10000", + "k8s.io/sa.scc.uid-range": "1000080000/10000", + "quota.k8s.io/cluster-resource-override-enabled": "false" + }} + }}, + "spec": {{ + "finalizers": [ + "openshift.io/origin", + "kubernetes" + ] + }}, + "status": {{ + "phase": "Active" + }} +}}''' + +pod_template = '''{{ + "kind": "Pod", + "apiVersion": "v1", + "metadata": {{ + "name": "{pod_name}", + "generateName": "{pod_name}-prefix", + "namespace": "{namespace_name}", + "selfLink": "/api/v1/namespaces/{namespace_name}/pods/{pod_name}", + "uid": "{pod_name}-id", + "resourceVersion": "3486", + "creationTimestamp": "2018-04-09T21:56:39Z", + "labels": {{ + "component": "{pod_name}-component", + "deployment": "{pod_name}-deployment", + "deploymentconfig": "{pod_name}-dc", + "custom.label": "{pod_name}-label-value", + "label.with.empty.value":"" + }}, + "annotations": {{ + "k8s.io/deployment-config.latest-version": "1", + "k8s.io/deployment-config.name": "{pod_name}-dc", + "k8s.io/deployment.name": "{pod_name}-deployment", + "k8s.io/custom.name": "custom value", + "annotation.with.empty.value":"" + }} + }}, + "status": {{ + "phase": "Running", + "hostIP": "172.18.4.32", + "podIP": "10.128.0.14", + "startTime": "2018-04-09T21:57:39Z" + }} +}}''' + +class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): + + def do_GET(self): + # "http://localhost:18443/api/v1/namespaces/namespace-name2" + # parse url - either /api/v1/namespaces/$ns_name + # or + # /api/v1/namespaces/$ns_name/pods/$pod_name + comps = self.path.split('/') + status = 400 + if len(comps) >= 5 and comps[1] == 'api' and comps[2] == 'v1': + if len(comps) == 5 and comps[3] == 'namespaces': # namespace + resp = ns_template.format(namespace_name=comps[4]) + status = 200 + elif len(comps) == 7 and comps[3] == 'namespaces' and comps[5] == 'pods': + resp = pod_template.format(namespace_name=comps[4], pod_name=comps[6]) + status = 200 + else: + resp = '{{"error":"do not recognize {0}"}}'.format(self.path) + else: + resp = '{{"error":"do not recognize {0}"}}'.format(self.path) + if not status == 200: + self.log_error(resp) + self.send_response(status) + self.end_headers() + self.wfile.write(json.dumps(json.loads(resp), separators=(',',':'))) + +port = int(sys.argv[1]) + +httpd = HTTPServer(('localhost', port), SimpleHTTPRequestHandler) + +# write "started" to file named in argv[3] +with open(sys.argv[3], "w") as ff: + ff.write("started\n") + +# write pid to file named in argv[2] +with open(sys.argv[2], "w") as ff: + ff.write('{0}\n'.format(os.getpid())) + +httpd.serve_forever() -- 2.16.4
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
