File syslog-ng-summarize.dif of Package syslog-ng

Overview Repositories Revisions Requests Users Attributes Meta

File syslog-ng-summarize.dif of Package syslog-ng (Revision 5f456e1ce7151f61509b636869fe5110)

Currently displaying revision 5f456e1ce7151f61509b636869fe5110 , Show latest

--- doc/sgml/syslog-ng.txt
+++ doc/sgml/syslog-ng.txt	2007/07/25 13:02:40
@@ -1146,6 +1146,12 @@ Table 3.15. List of supported global o
 │                       │          │containing statistics about       │       │
 │                       │          │dropped log messages.             │       │
 ├───────────────────────┼──────────┼──────────────────────────────────┼───────┤
+│                       │          │The number of seconds to wait for │       │
+│                       │          │a differing message, before the   │       │
+│summary()              │number    │"last message repeated N times" is│60     │
+│                       │          │generated. Set to 0 to disable and│       │
+│                       │          │log all messages.                 │       │
+├───────────────────────┼──────────┼──────────────────────────────────┼───────┤
 │log_fifo_size()        │number    │The number of lines fitting to the│100    │
 │                       │          │output queue.                     │       │
 ├───────────────────────┼──────────┼──────────────────────────────────┼───────┤
--- doc/sgml/syslog-ng.xml
+++ doc/sgml/syslog-ng.xml	2007/07/25 13:06:36
@@ -1413,6 +1413,12 @@ Declaration: 
               <entry>600</entry>
             </row>
             <row>
+              <entry>summary()</entry>
+              <entry>number</entry>
+              <entry>The number of seconds to wait for a differing message, before the "last message repeated N times" is generated. Set to 0 to disable and log all messages.</entry>
+              <entry>60</entry>
+            </row>
+            <row>
               <entry>log_fifo_size()</entry>
               <entry>number</entry>
               <entry>The number of lines fitting to the output queue.</entry>
--- doc/syslog-ng.conf.5
+++ doc/syslog-ng.conf.5	2007/07/25 12:54:08
@@ -198,6 +198,15 @@ NOTE: not implemented yet.
 The number of seconds between two STATS messages.
 
 .TP
+.B summary(n)
+This syslog-ng version contains an extension to avoid logging
+of repeated occurrences of the same message.
+This option allows to specify the number of seconds after the
+"last message repeated N times" is generated, when no differing
+message arrived in the meantime. Default value is 60 seconds,
+set to 0 to disable this extension and log all messages.
+
+.TP
 .B sync(n)
 The number of lines buffered before written to file
 (can be overridden locally).
--- src/center.c
+++ src/center.c	2007/07/25 12:30:31
@@ -36,6 +36,8 @@
 #undef CLASS_DEFINE
 
 #include "center.c.x"
+#include <string.h>
+
 
 /* log_endpoint_info */
 
@@ -57,6 +59,8 @@ struct log_connection *make_log_connecti
 	struct log_endpoint_info *last_src, *last_filter, *last_dest;
 	
 	self->flags = flags;
+	self->last_log_cnt = 0;
+	self->last_log_msg = NULL;
 	
 	/* Preseve the order of endpoints, it might be important */
 	for (last_src = last_filter = last_dest = NULL; nodes; nodes = nodes->next) {
@@ -102,9 +106,66 @@ struct log_connection *make_log_connecti
        (name log_center)
        (super log_handler)
        (vars
+         (summarize simple UINT32)
          (connections object log_connection)))
 */
 
+static int ol_string_equal(const struct ol_string *l, const struct ol_string *r)
+{
+	return	(l == r) ||
+		(l != NULL && r != NULL &&
+		l->length == r->length &&
+		memcmp(l->data, r->data, r->length) == 0);
+}
+
+static int log_info_equal(const struct log_info *l, const struct log_info *r)
+{
+	return  ol_string_equal(l->msg,  r->msg) &&
+		ol_string_equal(l->host, r->host);
+}
+
+static void do_handle_log_msg(struct log_endpoint_info *destinations,
+                              struct log_info *msg)
+{
+	struct log_endpoint_info *d;
+	for (d = destinations; d; d = d->next) {
+		assert(d->ref);
+		HANDLE_LOG(d->ref, log_info_use(msg));
+	}
+}
+
+void flush_connection_log(struct log_connection *p)
+{
+	if (p->last_log_msg != NULL) {
+		if (p->last_log_cnt > 1) {
+			// generate repeat count message
+			struct ol_string *str = c_format_cstring(
+				"last message repeated %i times",
+				p->last_log_cnt
+			);
+			struct log_info  *msg = make_internal_message(
+				p->last_log_msg->pri,
+				str->length, str->data
+			);
+
+			do_handle_log_msg(p->destinations, msg);
+
+			ol_string_free(str);
+			log_info_free(msg);
+		}
+		else
+		if (p->last_log_cnt > 0) {
+			// just handle it a second time
+			do_handle_log_msg(p->destinations, p->last_log_msg);
+		}
+
+		// forget remembered message
+		log_info_free(p->last_log_msg);
+		p->last_log_msg = NULL;
+		p->last_log_cnt = 0;
+	}
+}
+
 static void do_distribute_log(struct log_handler *c, struct log_info *msg)
 {
 	CAST(log_center, self, c);
@@ -114,7 +175,7 @@ static void do_distribute_log(struct log
 	for (match = 0, fallbacks = 0; !match && have_fallbacks && (fallbacks <= 1); fallbacks++) {
 		have_fallbacks = 1;
 		for (p = self->connections; p; p = p->next) {
-			struct log_endpoint_info *s, *f, *d;
+			struct log_endpoint_info *s, *f;
 	
 			if ((!fallbacks && (p->flags & LOG_CONN_FALLBACK))) {
 				have_fallbacks = 1;
@@ -144,9 +205,27 @@ static void do_distribute_log(struct log
 	
 			/* Matched, let's distribute it */
 			match = 1;
-			for (d = p->destinations; d; d = d->next) {
-				assert(d->ref);
-				HANDLE_LOG(d->ref, log_info_use(msg));
+			if (self->summarize > 0)
+			{
+				if (p->last_log_msg != NULL &&
+				    log_info_equal(p->last_log_msg, msg)) {
+					// count repeated messages
+					p->last_log_cnt++;
+				} else {
+					// flush remembered messages
+					flush_connection_log(p);
+
+					// remember new message
+					p->last_log_cnt = 0;
+					p->last_log_msg = log_info_use(msg);
+
+					// handle new message
+					do_handle_log_msg(p->destinations, msg);
+				}
+			}
+			else
+			{
+				do_handle_log_msg(p->destinations, msg);
 			}
 	
 			if (p->flags & LOG_CONN_FINAL) {
@@ -161,11 +240,13 @@ static void do_distribute_log(struct log
 	log_info_free(msg);
 }
 
-struct log_handler *make_log_center(struct log_connection *connections)
+struct log_handler *make_log_center(struct log_connection *connections,
+				    UINT32 summarize)
 {
 	NEW(log_center, self);
 	
 	self->super.handler = do_distribute_log;
+	self->summarize = summarize;
 	self->connections = connections;
 	return &self->super;
 }
--- src/center.c.x
+++ src/center.c.x	2007/07/25 12:30:31
@@ -2,6 +2,7 @@
 struct log_center
 {
   struct log_handler super;
+  UINT32 summarize;
   struct log_connection *connections;
 };
 extern struct ol_class log_center_class;
--- src/center.h
+++ src/center.h	2007/07/25 12:30:31
@@ -62,13 +62,18 @@ struct log_endpoint_info *make_log_endpo
          (sources object log_endpoint_info)
          (filters object log_endpoint_info)
          (destinations object log_endpoint_info)
+         (last_log_cnt simple UINT32)
+         (last_log_msg simple "struct log_info *")
          (next object log_connection)))
 */
 
 struct log_connection *make_log_connection(struct log_endpoint_info *nodes,
 	UINT32 flags);
 
-struct log_handler *make_log_center(struct log_connection *connections);
+struct log_handler *make_log_center(struct log_connection *connections,
+                                    UINT32 summarize);
+
+void flush_connection_log(struct log_connection *connection);
 
 struct callback *
 make_driver_reinit(struct log_handler *handler, struct syslog_config *cfg);
--- src/center.h.x
+++ src/center.h.x	2007/07/25 12:30:31
@@ -41,6 +41,8 @@ struct log_connection
   struct log_endpoint_info *sources;
   struct log_endpoint_info *filters;
   struct log_endpoint_info *destinations;
+  UINT32 last_log_cnt;
+  struct log_info *last_log_msg;
   struct log_connection *next;
 };
 extern struct ol_class log_connection_class;
--- src/cfgfile.c
+++ src/cfgfile.c	2007/07/25 12:30:31
@@ -219,7 +219,7 @@ static int do_init_config(struct syslog_
 
 	gc_idle_threshold = self->gc_idle_threshold;
 	gc_busy_threshold = self->gc_busy_threshold;
-	center = make_log_center(self->connections);
+	center = make_log_center(self->connections, self->summary_freq);
 
 	if (self->dns_cache)
 		self->cache = nscache_new(self->dns_cache_size,
@@ -267,8 +267,14 @@ static int do_destroy_config(struct sysl
 {
 	struct log_dest_group *d;
 	struct log_source_group *s;
+	struct log_connection *c;
 
 	self->living = 1;
+
+	for(c = self->connections; c; c = c->next) {
+		flush_connection_log(c);
+	}
+
 	for (s = self->sources; s; s = s->next_source_group) {
 		if (s->super.super.destroy)
 			LOG_HANDLER_DESTROY(s, self, persistent);
@@ -309,6 +315,7 @@ struct syslog_config *make_syslog_config
 	self->sync_freq = 0;
 	self->mark_freq = 1200; /* 20 minutes */
 	self->stats_freq = 600;
+	self->summary_freq = 60;
 	self->chain_hostnames = 1;
 	self->use_fqdn = 0;
 	self->use_dns = 1;
--- src/cfgfile.h
+++ src/cfgfile.h	2007/07/25 12:30:31
@@ -50,6 +50,7 @@
 	(mark_freq simple UINT32)
 	(sync_freq simple UINT32)
 	(stats_freq simple UINT32)
+	(summary_freq simple UINT32)
 	(time_sleep simple UINT32)
 	(time_reopen simple UINT32)
 	(time_reap simple UINT32)
--- src/cfgfile.h.x
+++ src/cfgfile.h.x	2007/07/25 12:30:31
@@ -12,6 +12,7 @@ struct syslog_config
   UINT32 mark_freq;
   UINT32 sync_freq;
   UINT32 stats_freq;
+  UINT32 summary_freq;
   UINT32 time_sleep;
   UINT32 time_reopen;
   UINT32 time_reap;
--- src/cfg-grammar.y
+++ src/cfg-grammar.y	2007/07/25 12:29:59
@@ -76,7 +76,7 @@ get_last_log_prefix(void)
 
 /* option items */
 %token KW_FLAGS KW_CATCHALL KW_FALLBACK KW_FINAL
-%token KW_FSYNC KW_MARK_FREQ KW_SYNC_FREQ KW_STATS_FREQ KW_CHAIN_HOSTNAMES KW_KEEP_HOSTNAME KW_CHECK_HOSTNAME KW_BAD_HOSTNAME
+%token KW_FSYNC KW_MARK_FREQ KW_SYNC_FREQ KW_STATS_FREQ KW_SUMMARY_FREQ KW_CHAIN_HOSTNAMES KW_KEEP_HOSTNAME KW_CHECK_HOSTNAME KW_BAD_HOSTNAME
 %token KW_LOG_FIFO_SIZE KW_LOG_MSG_SIZE
 %token KW_TIME_REOPEN KW_TIME_REAP KW_TIME_SLEEP KW_USE_TIME_RECVD
 %token KW_USE_DNS KW_USE_FQDN KW_GC_BUSY_THRESHOLD 
@@ -623,6 +623,7 @@ options_item
 	: KW_MARK_FREQ '(' NUMBER ')'		{ configuration->mark_freq = $3; }
 	| KW_SYNC_FREQ '(' NUMBER ')'		{ configuration->sync_freq = $3; }
 	| KW_STATS_FREQ '(' NUMBER ')'		{ configuration->stats_freq = $3; }
+	| KW_SUMMARY_FREQ '(' NUMBER ')'	{ configuration->summary_freq = $3; }
 	| KW_CHAIN_HOSTNAMES '(' yesno ')'	{ configuration->chain_hostnames = $3; }
 	| KW_KEEP_HOSTNAME '(' yesno ')'	{ configuration->keep_hostname = $3; }
 	| KW_CHECK_HOSTNAME '(' yesno ')'	{ configuration->check_hostname = $3; }
--- src/cfg-lex.l
+++ src/cfg-lex.l	2007/07/25 12:28:40
@@ -53,6 +53,7 @@ static struct keyword keywords[] = {
 	{ "mark",		KW_MARK_FREQ },
 	{ "sync", 		KW_SYNC_FREQ },
 	{ "stats",		KW_STATS_FREQ },
+	{ "summary",		KW_SUMMARY_FREQ },
 	{ "fsync",		KW_FSYNC },
 	{ "long_hostnames",	KW_CHAIN_HOSTNAMES },
         { "chain_hostnames",    KW_CHAIN_HOSTNAMES },
--- src/main.c
+++ src/main.c	2007/07/25 12:30:31
@@ -151,6 +151,30 @@ static void do_flush_wakeup(struct callb
 
 /* CLASS:
    (class
+     (name flush_summary)
+     (super callback)
+     (vars
+       (timeout simple int)
+       (backend object syslog_backend)))
+*/
+static void do_flush_summary(struct callback *c)
+{
+	CAST(flush_summary, self, c);
+
+	if (!self->backend->super.reloading && self->backend->configuration) {
+		struct log_connection    *con;
+
+		for(con = self->backend->configuration->connections;
+		    con != NULL; con = con->next) {
+			flush_connection_log(con);
+		}
+	}
+
+	io_callout(&self->backend->super, self->timeout, &self->super);
+}
+
+/* CLASS:
+   (class
      (name syslog_backend)
      (super io_backend)
      (vars
@@ -188,14 +212,20 @@ int main_loop(struct syslog_backend *bac
 		{ { STATIC_HEADER, do_report_stats }, 0, NULL };
 	static struct reread_config r = 
 		{ { STATIC_HEADER, do_reread_config }, NULL, 0 };
+	static struct flush_summary f = 
+		{ { STATIC_HEADER, do_flush_summary }, 0, NULL };
 
 	int exit_main_loop = 0, reload_config = 0;
 
 	s.timeout = backend->configuration->stats_freq;
 	s.backend = backend;
 	r.backend = backend;
+	f.timeout = backend->configuration->summary_freq;
+	f.backend = backend;
 	if (s.timeout > 0)
 		io_callout(&backend->super, s.timeout, &s.super);
+	if (f.timeout > 0)
+		io_callout(&backend->super, f.timeout, &f.super);
 
 	while (io_iter(&backend->super)) {
 		
@@ -219,7 +249,10 @@ int main_loop(struct syslog_backend *bac
 				s.timeout = backend->configuration->stats_freq;
 				if (s.timeout > 0)
 					io_callout(&backend->super, s.timeout, &s.super);
-				
+				f.timeout = backend->configuration->summary_freq;
+				if (f.timeout > 0)
+					io_callout(&backend->super, f.timeout, &f.super);
+
 				r.state = 0;
 				io_callout(&backend->super, 0, &r.super);
 			}
--- src/main.c.x
+++ src/main.c.x	2007/07/25 12:30:31
@@ -51,6 +51,32 @@ struct ol_class report_stats_class =
 #endif /* !CLASS_DECLARE */
 
 #ifndef CLASS_DEFINE
+struct flush_summary
+{
+  struct callback super;
+  int timeout;
+  struct syslog_backend *backend;
+};
+extern struct ol_class flush_summary_class;
+#endif /* !CLASS_DEFINE */
+
+#ifndef CLASS_DECLARE
+static void do_flush_summary_mark(struct ol_object *o, 
+void (*mark)(struct ol_object *o))
+{
+  struct flush_summary *i = (struct flush_summary *) o;
+  mark((struct ol_object *) i->backend);
+}
+
+struct ol_class flush_summary_class =
+{ STATIC_HEADER,
+  &callback_class, "flush_summary", sizeof(struct flush_summary),
+  do_flush_summary_mark,
+  NULL
+};
+#endif /* !CLASS_DECLARE */
+
+#ifndef CLASS_DEFINE
 struct syslog_backend
 {
   struct io_backend super;

Places

File syslog-ng-summarize.dif of Package syslog-ng (Revision 5f456e1ce7151f61509b636869fe5110)

Places