Jean Faye | 23 Jul 15:13 2014
Picon

Plugin module not found and Error parsing source, source plugin pipe not found

confirm a6d843f7ad7d1dbf1fefb6f12432e54941a680a9

Hi all,

When I run syslog-ng on my platform I faced the issue below. Could you please tell me why this issues happen and how can I fixed them?

I am using syslog-ng-3.2.5 and eventlog-0.2.13.

See in below the syslog-ng.conf I used.

Thank you in advance

Ismael

/**************Error Logs*******************/
Starting syslog-ng:Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='syslogformat'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='basicfuncs'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='afsocket'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='affile'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='afprog'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='afuser'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='dbparser'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='csvparser'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='syslogformat'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='basicfuncs'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='afsocket'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='affile'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='afprog'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='afuser'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='dbparser'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='csvparser'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='confgen'
Plugin module not found in 'module-path'; module-path='/usr/lib/syslog-ng', module='confgen'
Error parsing source, source plugin pipe not found in /etc/syslog-ng.conf at line 26, column 2:

        pipe("/tmp/pipe" pad_size(2048));
        ^^^^

syslog-ng documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
mailing list: https://lists.balabit.hu/mailman/listinfo/syslog-ng



confirm a6d843f7ad7d1dbf1fefb6f12432e54941a680a9


syslog-ng.conf

<at> version: 3.2
<at> include "scl.conf"
#
# Syslog-ng configuration file, compatible with default Debian syslogd
# installation. Originally written by anonymous (I can't find his name)
# Revised, and rewrited by me (SZALAY Attila <sasa <at> debian.org>)

# First, set some global options.
options { long_hostnames(off); flush_lines(0); use_dns(no); use_fqdn(no);
      owner("root"); group("adm"); perm(0640); stats_freq(0);
      bad_hostname("^gconfd$");create_dirs(yes);
};

########################
# Sources
########################
# This is the default behavior of sysklogd package
# Logs may come from unix stream, but not from another machine.
#
#source s_src { unix-dgram("/dev/log"); internal();
#                file("/proc/kmsg" program_override("kernel"));
#};

######FIJ######
source s_mysource {
    pipe("/tmp/pipe" pad_size(2048));
    #file("/var/log/ldb/fij_source.log" create_dirs(yes));
};
###############

# If you wish to get logs from remote machine you should uncomment
# this and comment the above source line.
#
#source s_net { tcp(ip(127.0.0.1) port(1000) authentication(required) encrypt(allow)); };

########################
# Destinations
########################
# First some standard logfile
#
#destination d_auth { file("/var/log/auth.log"); };
#destination d_cron { file("/var/log/cron.log"); };
#destination d_daemon { file("/var/log/daemon.log"); };
#destination d_kern { file("/var/log/kern.log"); };
#destination d_lpr { file("/var/log/lpr.log"); };
#destination d_mail { file("/var/log/mail.log"); };
#destination d_syslog { file("/var/log/syslog"); };
#destination d_user { file("/var/log/user.log"); };
#destination d_uucp { file("/var/log/uucp.log"); };

# This files are the log come from the mail subsystem.
#
#destination d_mailinfo { file("/var/log/mail/mail.info"); };
#destination d_mailwarn { file("/var/log/mail/mail.warn"); };
#destination d_mailerr { file("/var/log/mail/mail.err"); };

# Logging for INN news system
#
#destination d_newscrit { file("/var/log/news/news.crit"); };
#destination d_newserr { file("/var/log/news/news.err"); };
#destination d_newsnotice { file("/var/log/news/news.notice"); };

# Some `catch-all' logfiles.
#
#destination d_debug { file("/var/log/debug"); };
#destination d_error { file("/var/log/error"); };
#destination d_messages { file("/var/log/messages"); };

# The root's console.
#
#destination d_console { usertty("root"); };

# Virtual console.
#
#destination d_console_all { file("/dev/tty10"); };

# The named pipe /dev/xconsole is for the nsole' utility.  To use it,
# you must invoke nsole' with the -file' option:
#
#    $ xconsole -file /dev/xconsole [...]
#
#destination d_xconsole { pipe("/dev/xconsole"); };

# Send the messages to an other host
#
#destination d_net { tcp("127.0.0.1" port(1000) authentication(on) encrypt(on) log_fifo_size(1000)); };

# Debian only
#destination d_ppp { file("/var/log/ppp.log"); };

######FIJ######
########################Vérifier l'option create_dirs(yes)######################
destination d_GEN {
                #file("/var/log/ldb/GENTrace.log" create_dirs(yes));
                file("/var/log/ldb/GENTrace.log");
};

destination d_SU {
                #file("/var/log/ldb/SUTrace.log" create_dirs(yes));
                file("/var/log/ldb/SUTrace.log");
};

destination d_WAN {
                #file("/var/log/ldb/WANTrace.log" create_dirs(yes));
                file("/var/log/ldb/WANTrace.log");
};

destination d_CPL {
                #file("/var/log/ldb/CPLTrace.log" create_dirs(yes));
                file("/var/log/ldb/CPLTrace.log");
};
###############

########################
# Filters
########################
# Here's come the filter options. With this rules, we can set which
# message go where.

#filter f_dbg { level(debug); };
#filter f_info { level(info); };
#filter f_notice { level(notice); };
#filter f_warn { level(warn); };
#filter f_err { level(err); };
#filter f_crit { level(crit .. emerg); };

#filter f_debug { level(debug) and not facility(auth, authpriv, news, mail); };
#filter f_error { level(err .. emerg) ; };
#filter f_messages { level(info,notice,warn) and
#                    not facility(auth,authpriv,cron,daemon,mail,news); };

#filter f_auth { facility(auth, authpriv) and not filter(f_debug); };
#filter f_cron { facility(cron) and not filter(f_debug); };
#filter f_daemon { facility(daemon) and not filter(f_debug); };
#filter f_kern { facility(kern) and not filter(f_debug); };
#filter f_lpr { facility(lpr) and not filter(f_debug); };
#filter f_local { facility(local0, local1, local3, local4, local5,
#                        local6, local7) and not filter(f_debug); };
#filter f_mail { facility(mail) and not filter(f_debug); };
#filter f_news { facility(news) and not filter(f_debug); };
#filter f_syslog3 { not facility(auth, authpriv, mail) and not filter(f_debug); };
#filter f_user { facility(user) and not filter(f_debug); };
#filter f_uucp { facility(uucp) and not filter(f_debug); };

#filter f_cnews { level(notice, err, crit) and facility(news); };
#filter f_cother { level(debug, info, notice, warn) or facility(daemon, mail); };

#filter f_ppp { facility(local2) and not filter(f_debug); };
#filter f_console { level(warn .. emerg); };

######FIJ######
filter f_GEN {
                facility(local0) and filter(nom_du_composant_applicatif);
};

filter f_SU {
                facility(local0) and filter(nom_du_composant_applicatif);
};

filter f_WAN {
                facility(local0) and filter(nom_du_composant_applicatif);
};

filter f_CPL {
                facility(local0) and filter(nom_du_composant_applicatif);
};
###############

########################
# Log paths
########################
#log { source(s_src); filter(f_auth); destination(d_auth); };
#log { source(s_src); filter(f_cron); destination(d_cron); };
#log { source(s_src); filter(f_daemon); destination(d_daemon); };
#log { source(s_src); filter(f_kern); destination(d_kern); };
#log { source(s_src); filter(f_lpr); destination(d_lpr); };
#log { source(s_src); filter(f_syslog3); destination(d_syslog); };
#log { source(s_src); filter(f_user); destination(d_user); };
#log { source(s_src); filter(f_uucp); destination(d_uucp); };

#log { source(s_src); filter(f_mail); destination(d_mail); };
#log { source(s_src); filter(f_mail); filter(f_info); destination(d_mailinfo); };
#log { source(s_src); filter(f_mail); filter(f_warn); destination(d_mailwarn); };
#log { source(s_src); filter(f_mail); filter(f_err); destination(d_mailerr); };

#log { source(s_src); filter(f_news); filter(f_crit); destination(d_newscrit); };
#log { source(s_src); filter(f_news); filter(f_err); destination(d_newserr); };
#log { source(s_src); filter(f_news); filter(f_notice); destination(d_newsnotice); };
#log { source(s_src); filter(f_cnews); destination(d_console_all); };
#log { source(s_src); filter(f_cother); destination(d_console_all); };

#log { source(s_src); filter(f_ppp); destination(d_ppp); };

#log { source(s_src); filter(f_debug); destination(d_debug); };
#log { source(s_src); filter(f_error); destination(d_error); };
#log { source(s_src); filter(f_messages); destination(d_messages); };

#log { source(s_src); filter(f_console); destination(d_console_all);
                    destination(d_xconsole); };
#log { source(s_src); filter(f_crit); destination(d_console); };

# All messages send to a remote site
#
#log { source(s_src); destination(d_net); };

######FIJ######
log { source(s_mysource); filter(f_GEN); destination(d_GEN); };
log { source(s_mysource); filter(f_SU); destination(d_SU); };
log { source(s_mysource); filter(f_WAN); destination(d_WAN); };
log { source(s_mysource); filter(f_CPL); destination(d_CPL); };
###############
______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

Peter Czanik | 23 Jul 11:06 2014
Picon

syslog-ng is now available in EPEL7

Hi,
syslog-ng 3.5.5 just arrived to EPEL7: 
https://czanik.blogs.balabit.com/2014/07/epel-7-now-contains-syslog-ng/ 
As far as I can see, not all mirrors carry it yet...
Bye,

--

-- 
Peter Czanik (CzP) <peter.czanik <at> balabit.com>
BalaBit IT Security / syslog-ng upstream
http://czanik.blogs.balabit.com/
https://twitter.com/PCzanik

______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

John Cole | 21 Jul 22:20 2014

syslog driver parse bug?

Apologies if I missed a bugfix in my web searching and manually scanning the changelogs.

 

I’m running syslog-ng (syslog-ng-3.2.5-3.el6.x86_64) on EL6 with the syslog driver as my network source:

source s_network {

        syslog(ip(0.0.0.0) transport("udp") port(514));

        syslog(ip(0.0.0.0) transport("tcp") port(514));

};

 

I have an application that does not have an internal synchronized clock source.  Per RFC5424, “A syslog application MUST use the NILVALUE as TIMESTAMP if the syslog application is incapable of obtaining system time.”   And, the grammar shows TIMESTAMP = NILVALUE / FULL-DATE "T" FULL-TIME

 

When I specify the “-“ NILVALUE in the syslog message, the syslog driver does not seem to be able to parse the message and does not log anything.

 

If I hardcode a time value, all message fields seem to post appropriately.

 

Is this a new issue, or did I miss a version that addressed the handling of NILVALUE?  Given RedHat lagging on versions, and Fedora’s subsequent changes, I haven’t yet begun the effort of retrofitting the RPM source in RAWHIDE to test, with the hope that someone might recognize the bug, or have a pointer to a EL6 SRPM so I can test against latest…

 

Thanks for any/all pointers for a quick resolution!

 

John

 

______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

jrhendri | 21 Jul 22:15 2014

compiling problems --enable-mongodb --enable-json --enable-redis

Hi,

  I am trying to get syslog-ng to be the front-end for a larger log collection architecture including
mongodb, redis, elasticsearch, possibly logstash & kibana.

  Have been trying to build syslog-ng with the necessary modules and failing miserably. 

  I know it must be me - that said, I have spent several hours trying to get this to compile on both RHEL 5 and
Ubuntu 14.04

I ran into a few things that might be useful (or not), but since the platforms and libraries installed are
different, yet the end error is the same I thought I would ask for other eyes / brains to help here...

Thanks for any assistance!

Jim

RHEL 5:
I have installed json-c & redis-stable which provides these libraries:

./redis-stable/deps/hiredis/hiredis.h
./redis-stable/deps/hiredis/libhiredis.a

and created links here to try and get around issues:
/usr/lib/hiredis
/usr/lib/hiredis/libhiredis.a
/usr/include/hiredis
/usr/include/hiredis/hiredis.h

configure succeeds with this:
 Modules:
  Module search path          : /usr/local/lib/syslog-ng
  Sun STREAMS support (module): no
  SSL support (module)        : no
  SQL support (module)        : no
  PACCT module (EXPERIMENTAL) : no
  MongoDB destination (module): yes
  JSON support (module)       : yes
  SMTP support (module)       : no
  AMQP destination (module)   : yes
  STOMP destination (module)  : yes
  GEOIP support (module)      : no
  Redis support (module)      : yes

Make fails here:

  CC       modules/redis/modules_redis_libredis_la-redis-grammar.lo
  CC       modules/redis/modules_redis_libredis_la-redis.lo
  CC       modules/redis/modules_redis_libredis_la-redis-parser.lo
  CCLD     modules/redis/libredis.la
/usr/bin/ld: cannot find -lhiredis
collect2: ld returned 1 exit status
make[2]: *** [modules/redis/libredis.la] Error 1
make[1]: *** [all-recursive] Error 1
make: *** [all] Error 2
[n0142566 <at> VDDP13E-F1A47ED syslog-ng-3.5.4.1]$ 

Ubuntu:
I have installed json-c, redis-stable & libredis:
/usr/local/lib/libredis.so.1.0.0
/usr/local/lib/libredis.1
/usr/local/lib/libredis.la
/usr/local/include/redis.h

configure succeeds with this:

 Modules:
  Module search path          : /usr/local/lib/syslog-ng
  Sun STREAMS support (module): no
  SSL support (module)        : yes
  SQL support (module)        : no
  PACCT module (EXPERIMENTAL) : no
  MongoDB destination (module): yes
  JSON support (module)       : yes
  SMTP support (module)       : no
  AMQP destination (module)   : yes
  STOMP destination (module)  : yes
  GEOIP support (module)      : no
  Redis support (module)      : yes

Make fails with:

  CC       modules/redis/modules_redis_libredis_la-redis-grammar.lo
  CC       modules/redis/modules_redis_libredis_la-redis.lo
  CC       modules/redis/modules_redis_libredis_la-redis-parser.lo
  CCLD     modules/redis/libredis.la
/usr/bin/ld: cannot find -lhiredis
collect2: ld returned 1 exit status
make[2]: *** [modules/redis/libredis.la] Error 1
make[1]: *** [all-recursive] Error 1
make: *** [all] Error 2
[n0142566 <at> picard:~/src/syslog-ng-3.5.5$ 

===========================================

______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

Nicholas Radonicich | 21 Jul 15:53 2014

redis/tcp destination throughput and dropped messages

Hello, hoping someone might be able to help... I am attempting to build a syslog server with outputs to Logstash/Elasticsearch via redis but am having issues with dropped messages in such that i cannot get over ~4000 messages per second out to a destination (TCP or redis) without the rest being dropped. I have tried various tuning on the tcp output with flush_lines() and threaded() to no avail, as well as some things on the inputs. Both redis and logstash (used for tcp inputs) were on the localhost.

 

redis-benchmark has no issues doing >70000 LPUSH/second and the file based destinations of syslog-ng are not dropping messages (we are sending almost 80k/s) so I don’t know where I am going wrong.

 

loggen was accepting > 110000mps its just on the outputs it seems.

 

 

 

Thanks, Nick


The information in this message, including in all attachments, is confidential or privileged. In the event you have received this message in error and are not the intended recipient, you are hereby advised that any use, copying or reproduction of this document is strictly forbidden. Please notify immediately the sender of this error and destroy this message, including its attachments, as the case may be.

L'information apparaissant dans ce message électronique et dans les documents qui y sont joints est de nature confidentielle ou privilégiée. Si ce message vous est parvenu par erreur et que vous n'en êtes pas le destinataire visé, vous êtes par les présentes avisé que toute utilisation, copie ou distribution de ce message est strictement interdite. Vous êtes donc prié d’en informer immédiatement l’expéditeur et de détruire ce message, ainsi que les documents qui y sont joints, le cas échéant.

 

Ce courriel provient de Nicholas.Radonicich <at> cogeco.com . Pour assurer la livraison de futurs envois, veuillez inclure la présente adresse courriel à votre carnet d’adresses ou votre liste d’expéditeurs autorisés.

Si vous ne souhaitez plus recevoir de messages promotionnels de la part de Cogeco, veuillez envoyer un courriel à desabonnement <at> cogeco.com.

Politique en matière de protection des renseignements personnels de Cogeco et Engagement en matière d’anti-spamContactez-nous

Cogeco Câble Canada, 5 Place Ville-Marie, Bureau 1700, Montréal, Québec, H3B 0B3

--

This email is from Nicholas.Radonicich <at> cogeco.com . To ensure the delivery of future emails, please add the current email address to your address book or safe senders list.

If you no longer wish to receive promotional emails from Cogeco, please send an email to unsubscribe <at> cogeco.com.

Privacy Policy and Anti-spam Commitment - Contact us

Cogeco Cable Canada, 5 Place Ville-Marie, Suite 1700, Montreal, Quebec, H3B 0B3

 

______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

Anwar El fatayri | 21 Jul 10:47 2014
Picon

Syslog-ng using an old Template

<!-- .hmmessage P { margin:0px; padding:0px } body.hmmessage { font-size: 12pt; font-family:Calibri } -->
Hey,

I was using this template for my tests : 

template t_lyraJBossFormat {

                template("${DATE} ${FULLHOST} jboss_vad - ------------ - SD : ${SDATA} - ------------ - MSG :
 ${MSGONLY}\n");
                template_escape(no);
};

This template is a Format that I use for JBoss Logs. The syslog packet (IETF) is created and then sent to syslog-ng via a named pipe. 

After Debugging all the problems, the following template was created to be used for all Jboss Logs :

template t_lyraJBossFormat {

                template("${DATE} ${FULLHOST} jboss_vad - ${SDATA} - ${MSGONLY}\n");
                template_escape(no);
};


The problem is that syslog-ng is still using the old template. (I restarted the syslog-ng service of course) 

Any ideas ?

Thanks in advance.

El Fatayri Anwar


______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

Radu Gheorghe | 18 Jul 18:57 2014

(no subject)

Hi,

This is my first post here, so I have to start by thanking all the contributors for an awesome product :)

My question is about adding an array to a JSON document. What I'm trying to do is to send a message like this:

<at> cee: {"message": "test message", "tags":["test", "message"]}

My template looks a like this:

template(" <at> cee: $(format-json --pair message=\"$MSG\" --pair tags="test")\n")

This works fine for a single tag, but how can I add multiple ones?

The broader use-case is that I want to add tags to logs matching a specific filter. For example:
----------------------
filter user_tests { facility(user) and message(test) };

destination logsene_tests {
      transport("tcp")
      port(514)
      template(" <at> cee: $(format-json --pair message=\"$MSG\" --pair tags=\"test\")\n")
    );
};

log { source(all_syslog); filter(user_tests); destination(logsene_tests); flags(final); };
----------------------

If there's a better way to add multiple tags to a log, please tell me - I'm good with making big changes if it leads to a cleaner/better config.

Best regards,
Radu
--
Performance Monitoring * Log Analytics * Search Analytics
Solr & Elasticsearch Support * http://sematext.com/
______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

Rafał Radecki | 18 Jul 10:14 2014
Picon

syslog-ng 2.1.4 - file sources are read only when reload or restart is performed?

Hi All ;)

I am using a virtual machine with:
Red Hat Enterprise Linux Server release 5.7 (Tikanga)
Linux logserver01 2.6.18-274.el5 #1 SMP Fri Jul 8 17:36:59 EDT 2011 x86_64 x86_64 x86_64 GNU/Linux
syslog-ng 2.1.4

I have following configuration:

...
  9 options {
 10         create_dirs (yes);
 11         dir_group (root);
 12         dir_owner (root);
 13         dir_perm (0700);
 14         group (root);
 15         owner (root);
 16         perm (0600);
 17         flush_lines(1);
 18         flush_timeout (1000);
 19         keep_hostname (yes);
 20         log_fifo_size (1);
 21         use_dns (no);
 22         use_fqdn (no);
 23 };
...
 39 source s_stdout {
 40 #       file ("/logs/stdout.log" flags(no-parse) follow_freq(1));
 41         file ("/logs/stdout.log" flags(no-parse));
 42 };
...
61 destination d_stdout         { file("/var/log/$YEAR$MONTH$DAY/stdout"); };
...
80 log { source(s_stdout); destination(d_stdout); };

The problem is that changes in /logs/stdout.log are only visible in /var/log/20140717/stdout when I perform /etc/init.d/syslog-ng reload or restart. I tried several settings of flush_*, log_fifo_size and follow_freq but with no luck :D

Is it a problem withe the version that I use (quite old :D ) or is there a mistake in my configuration maybe?

BR,
Rafal.
______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

Renato Bezerra | 17 Jul 21:09 2014
Picon

Logs sent to wrong destination

Hi,

I'm using syslog-ng in a long time, but recently i noted that, in some cases, the log has sent to a wrong destination.

I have many devices sending logs to my host, the problem appears when the server receive webservers logs, they are delivered to a different destination and I don't known how.

here is the configuration:

destination apache {
       file("/var/log/webserver/$R_YEAR-$R_MONTH-$R_DAY-$R_HOUR"
       owner(ll)
       group(ll)
       perm(0644)
       dir_perm(0755)
       create_dirs(yes));
};

filter f_apache {
    (
        host("xxx.xxx.xxx.82") or
        host("xxx.xxx.xxx.137")
    );
};

log {
    source(aaa);
    filter(f_apache);
    destination(apache);
};

The ip address xxx.xxx.xxx.137 send a duplicate log event to another directory, without any other configuration.

Have you seen this?
______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

Pierre-Yves Ritschard | 11 Jul 10:11 2014

Apache Kafka output module

Hi list,

I submitted a PR for an Apache Kafka [1] output module. This is my first
syslog-ng module, so it might need a bit of guidance.

This provides the ability to create a partitioned stream of logs based
on a message field or static key.

A typical destination configuration will look like:

destination d_kafka {
  kafka( properties(metadata.broker.list("localhost:9092"))
       topic("syslog")
       payload("$(format-json --scope all-nv-pairs --scope core)")

       partition("static-key")
#      partition(random)
#      partition(field("$PROGRAM"))
); 

};

For reference, this allows us to work on logs locally with syslog,
extract fields, then publish on a queue whose consumers will index logs
in our data stores.

[1]. http://kafka.apache.org

______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq

James Lay | 9 Jul 20:10 2014
Picon

Syslog-ng completely unusable after upgrade

Well shoot...here's what I got...I'm using the stock syslog-ng.conf 
installed with 3.5.4.1 just for testing:

#############################################################################
# Default syslog-ng.conf file which collects all local logs into a
# single file called /var/log/messages.
#

 <at> version: 3.3
 <at> include "scl.conf"

source s_local {
	system();
	internal();
};

source s_network {
	udp();
};

destination d_local {
	file("/var/log/messages");
};

log {
	source(s_local);

	# uncomment this line to open port 514 to receive messages
	#source(s_network);
	destination(d_local);
};

Running strace here's the last bit:

/usr/sbin/syslog-ng -t -d -v -F -f /usr/etc/syslog-ng.conf

open("/usr/lib/libsyslog-ng-3.3.11.so", O_RDONLY) = 10
read(10, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 <at> 0\1\000"..., 
512) = 512
fstat64(10, {st_mode=S_IFREG|0755, st_size=1427313, ...}) = 0
mmap2(NULL, 447784, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 10, 
0) = 0xb79d7000
mmap2(0xb7a40000, 16384, PROT_READ|PROT_WRITE, 
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 10, 0x68) = 0xb7a40000
mmap2(0xb7a44000, 1320, PROT_READ|PROT_WRITE, 
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7a44000
close(10)                               = 0
write(2, "iv_tls_user_register: called aft"..., 43iv_tls_user_register: 
called after iv_init
) = 43
rt_sigprocmask(SIG_UNBLOCK, [ABRT], NULL, 8) = 0
tgkill(5172, 5172, SIGABRT)             = 0
--- SIGABRT (Aborted)  <at>  0 (0) ---
+++ killed by SIGABRT +++
Process 5172 detached

And that's it....I'm glad I'm doing this on a virtual machine.  I've 
not been able to anything on this online...thoughts or help are 
appreciated.

James
______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq


Gmane