Запись syslog на внешний USB-диск с помощю Syslog-ng

_alex_ · 18 февраля

В итоге получилось так:

/opt/etc/init.d/S01syslog-ng

#!/bin/sh

ENABLED=yes
PROCS=syslog-ng
ARGS=""
PREARGS=""
DESC=$PROCS
PATH=/opt/sbin:/opt/bin:/opt/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

ACTION=$1
BASENAME="$(basename $0)"
FLAGFILE="/opt/var/syslog-ng.flag"
NDMCFILE="/tmp/mnt/USB/log/ndmc.log"
SYSNGFILE="/opt/var/syslog-ng.persist"
PARTLINE="$(echo `yes =|head -50`|tr -d ' ')"

start() {
    rm -f $FLAGFILE 2>/dev/null
    return 0
}
stop() {
    echo -n "" > $FLAGFILE
    rm -f $NDMCFILE
    rm -f $SYSNGFILE
    return 0
}

case $ACTION in
start)
    # opkg mount
    while [ ! -f /opt/bin/mount  ]; do
         sleep 1
    done

    # USB mount
    while [ "`/opt/bin/mount | grep -qi /sda1 || echo 1`" ]; do
        sleep 1
    done

    # wait 1 minute or until ntp client completed (normal timestamp will be in the log)
    cDate="$(date +%s)";
    while [ "$(($(date +%s) - $cDate))" -lt "$((1*60))" ]
    do
        sleep 1;
    done;

    if [ -f $FLAGFILE ]; then
        FIRSTLINE="kernel: ${PARTLINE} warm reboot ${PARTLINE}"
    else
        FIRSTLINE="kernel: ${PARTLINE} cold reboot ${PARTLINE}"
    fi
    
    # IETF log format message
    ndmc -c show log | sed ':a;N;$!ba;s/\n \{1,\}/ /g' | cut -c -190 | sed '1,/kernel: Booting Linux/ {/kernel: Booting Linux/!d}' | sed 's/. \[.*:..:..\] //' | sed "1 i\\${FIRSTLINE}" | awk -F CGG '/^>/ {print; next} {for (i=1; i<=NF; ++i) {printf "%s %s%s", length($i), $i, (i==NF)?"\n":" "}}' | sed '$ d' > ${NDMCFILE}
            
    start
    ;;
stop|kill)
    stop
    ;;
restart)
    stop
    start
    ;;
*)
    echo "ACTION=unknown" | logger -p emerg -t kernel
    ;;
esac


. /opt/etc/init.d/rc.func

/opt/etc/syslog-ng.conf

# Collect all local logs into a single file /var/log/messages.
# See https://www.syslog-ng.com/technical-documents/list/syslog-ng-open-source-edition
# See https://github.com/balabit/syslog-ng-ose-guides

@version: 4.7
@include "scl.conf"

# cat /opt/etc/syslog-ng.conf | nocomments
###############################################################################
# /opt/etc/init.d/S01syslog-ng stop
# /opt/etc/init.d/S01syslog-ng start
# /opt/etc/init.d/S01syslog-ng restart
###############################################################################
# Source drivers available in syslog-ng (syslog-ng PE 7.0.34 Administration Guide)
#
# file()             Opens the specified file and reads messages.
# internal()         Messages generated internally in syslog-ng.
# linux-audit()      Reads the logfiles of the auditd application.
# network()          Receives messages from remote hosts using the BSD-syslog protocol over IPv4 and IPv6. Supports the TCP, UDP, ALTP, and TLS network protocols.
# pipe()             Opens the specified named pipe and reads messages.
# program()          Opens the specified application and reads messages from its standard output.
# python() and
# python-fetcher()   Receive or fetch messages using a custom source written in Python.
# syslog()           Listens for incoming messages using the new IETF-standard syslog protocol.
# system()           Automatically detects which platform syslog-ng PE is running on, and collects the native log messages of that platform.
# systemd-journal()  Collects messages directly from the journal of platforms that use systemd.
# systemd-syslog()   Collects messages from the journal using a socket on platforms that use systemd.
# unix-dgram()       Opens the specified unix socket in SOCK_DGRAM mode and listens for incoming messages.
# unix-stream()      Opens the specified unix socket in SOCK_STREAM mode and listens for incoming messages.
# windowsevent()     Reads messages from the Windows Event Collector tool.
###############################################################################

options {
  chain-hostnames(no);      # Enable or disable the chained hostname format.
  create-dirs(yes);         # Enable creating non-existing directories when creating files or socket files.
  keep-hostname(yes);       # Enable or disable hostname rewriting.
  log-fifo-size(10000);    # The number of messages that the output queue can store.
  log-msg-size(200);        # Maximum length of a message in bytes.
  stats(freq(0));           # The period between two STATS messages (sent by syslog-ng, containing statistics about dropped logs) in seconds.
  flush-lines(0);           # How many lines are flushed to a destination at a time.
  trim-large-messages(yes); # trims the incoming log message to the size set in log-msg-size()
  use-fqdn(no);             # Add Fully Qualified Domain Name instead of short hostname.
  ts-format(iso);           # To convert all timestamps to the ISODATE format
};

source src {
  internal();
  unix-dgram("/dev/log");
#  unix-stream("/dev/log");
};

source net {
  network( ip("127.0.0.1") transport("udp") port(3127) keep-timestamp(no) );
};

source kernel {
  file("/proc/kmsg" program_override("kernel"));
};

source syslog {
  syslog();
};

source ndm {
  program("/bin/ndm");
};

source ndmc {
#  file( "/opt/var/log/ndmc.log" flags(no-parse) );
  file( "/tmp/mnt/USB/log/ndmc.log" );
};

destination messages {
  file("/tmp/mnt/USB/log/messages" template("${ISODATE} (${PRIORITY}) ${PROGRAM}: ${MESSAGE}\n") time_zone("Europe/Moscow"));
};

destination debug {
  file("/tmp/mnt/USB/log/debuglog");
};

filter f_kernel       { not program(kernel) or (program(kernel) and match ("reboot|error|Linux|Memory|Boot CPU|Machine|Found.*storage" value("MESSAGE"))); };
filter f_ndm          { not program(ndm) or (program(ndm) and match("Network Device Manager|Ntp" value("MESSAGE"))); };
filter f_cron         { not program(cron) or (program(cron) and match("started|stopped|Hello" value("MESSAGE")) ); };
filter f_ni_ndhcps    { not program(ndhcps) or (program(ndhcps) and not match("sending ACK|received ACK|DHCPDISCOVER|DHCPREQUEST|making OFFER|NDM DHCP" value("MESSAGE")) ); };
filter f_ni_dhcpd     { not program(dhcpd) or (program(dhcpd) and not match("Information-request|Sending Reply" value("MESSAGE")) ); };
filter f_no_radvd     { not program(radvd); };
filter f_no_telnetd   { not program(telnetd); };
filter f_no_bndstrg   { not program(bndstrg); };
filter f_no_ndnproxy  { not program(ndnproxy); };
filter f_no_avahi     { not program(avahi-daemon); };
filter f_no_nimproxy  { not program(nimproxy); };
filter f_no_ndhcpc    { not program(ndhcpc); };
filter f_no_upnp      { not program(upnp); };
filter f_no_coalagent { not program(coalagent); };
filter f_no_dhcpd     { not program(dhcpd); };
filter f_no_wind      { not program(wind); };

log {
  source(src);
  source(net);
  source(kernel);
  source(syslog);
  source(ndmc);
  filter(f_kernel);
  filter(f_ndm);
  filter(f_cron);
  filter(f_ni_ndhcps);
  filter(f_ni_dhcpd);
  filter(f_no_radvd);
  filter(f_no_telnetd);
  filter(f_no_bndstrg);
  filter(f_no_ndnproxy);
  filter(f_no_avahi);
  filter(f_no_nimproxy);
  filter(f_no_ndhcpc);
  filter(f_no_upnp);
  filter(f_no_coalagent);
  filter(f_no_dhcpd);
  filter(f_no_wind);
  destination(messages);
};

log {
  source(src);
  source(net);
  source(kernel);
  source(syslog);
  source(ndmc);
#  source(ndm);
  destination(debug);
};


# Include any user settings last so that s/he can override or
# supplement all "canned" settings inherited from the distribution.
@include "/opt/etc/syslog-ng.d/" # Put any customization files in this directory

_alex_ · 21 февраля

Поправил /opt/etc/init.d/S01syslog-ng

чтобы не зависал намертво при проблемах со storage и usb

#!/bin/sh

ENABLED=yes
PROCS=syslog-ng
ARGS=""
PREARGS=""
DESC=$PROCS
PATH=/opt/sbin:/opt/bin:/opt/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

ACTION=$1
BASENAME="$(basename $0)"
FLAGFILE="/opt/var/syslog-ng.flag"
NDMCFILE="/tmp/mnt/USB/log/ndmc.log"
SYSNGFILE="/opt/var/syslog-ng.persist"
PARTLINE="$(echo `yes =|head -50`|tr -d ' ')"

start() {
    rm -f $FLAGFILE 2>/dev/null
    return 0
}
stop() {
    echo -n "" > $FLAGFILE
    rm -f $NDMCFILE
    rm -f $SYSNGFILE
    return 0
}

case $ACTION in
start)
    # opkg mount
    cDate="$(date +%s)";
    while [ ! -f /opt/bin/mount  ]; do
        if [ "$(($(date +%s) - $cDate))" -gt "$((3*60))" ]; then
            # protection for infinite loop
            logger -s -p err -t opkg "opkg is destroyed"
            exit 1
        fi
        sleep 1
    done

    # USB mount
    cDate="$(date +%s)";
    while [ "`/opt/bin/mount | grep -qi /sda1 || echo 1`" ]; do
        if [ "$(($(date +%s) - $cDate))" -gt "$((3*60))" ]; then
            # protection for infinite loop
            logger -s -p err -t opkg "USB Disk is not mounted"
            exit 1
        fi
        sleep 1
    done

    # waiting 1 minute or until ntp client completed (normal timestamp will be in the log)
    cDate="$(date +%s)";
    while [ "$(($(date +%s) - $cDate))" -lt "$((1*60))" ]; do
        sleep 1;
    done;

    if [ -f $FLAGFILE ]; then
        FIRSTLINE="kernel: ${PARTLINE} warm reboot ${PARTLINE}"
    else
        FIRSTLINE="kernel: ${PARTLINE} cold reboot ${PARTLINE}"
    fi
    
    # IETF log format message
    ndmc -c show log | sed ':a;N;$!ba;s/\n \{1,\}/ /g' | cut -c -190 | sed '1,/kernel: Booting Linux/ {/kernel: Booting Linux/!d}' | sed 's/. \[.*:..:..\] //' | sed "1 i\\${FIRSTLINE}" | awk -F CGG '/^>/ {print; next} {for (i=1; i<=NF; ++i) {printf "%s %s%s", length($i), $i, (i==NF)?"\n":" "}}' | sed '$ d' > ${NDMCFILE}
            
    start
    ;;
stop|kill)
    stop
    ;;
restart)
    stop
    start
    ;;
*)
    echo "ACTION=unknown" | logger -p emerg -t kernel
    ;;
esac


. /opt/etc/init.d/rc.func

_alex_ · 21 февраля

Установил и настроил logrotate и cron

logrotate

/opt/etc/logrotate.d/syslog-ng-message

/tmp/mnt/USB/log/messages
{
  weekly
  rotate 4
  copytruncate
  missingok
  compress
  compresscmd /opt/bin/gzip
  delaycompress
}

/opt/etc/logrotate.d/syslog-ng-debug

/tmp/mnt/USB/log/debuglog
{
  daily
  rotate 7
  copytruncate
  missingok
  compress
  compresscmd /opt/bin/gzip
  delaycompress
}

crontab -l

@weekly /opt/sbin/logrotate -f /opt/etc/logrotate.d/syslog-ng-message
@daily /opt/sbin/logrotate -f /opt/etc/logrotate.d/syslog-ng-debug

_alex_ · 25 февраля

Добавил обработку Severity

/opt/etc/init.d/S01syslog-ng:

#!/bin/sh

ENABLED=yes
PROCS=syslog-ng
ARGS=""
PREARGS=""
DESC=$PROCS
PATH=/opt/sbin:/opt/bin:/opt/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

ACTION=$1
BASENAME="$(basename $0)"
FLAGFILE="/opt/var/syslog-ng.flag"
NDMCFILE="/tmp/mnt/USB/log/ndmc.log"
SYSNGFILE="/opt/var/syslog-ng.persist"
PARTLINE="$(echo `yes =|head -50`|tr -d ' ')"

start() {
    rm -f $FLAGFILE 2>/dev/null
    return 0
}
stop() {
    echo -n "" > $FLAGFILE
    rm -f $NDMCFILE
    rm -f $SYSNGFILE
    return 0
}

case $ACTION in
start)
    # opkg mount
    cDate="$(date +%s)";
    while [ ! -f /opt/bin/mount  ]; do
        if [ "$(($(date +%s) - $cDate))" -gt "$((3*60))" ]; then
            # protection for infinite loop
            logger -s -p err -t opkg "opkg is destroyed"
            exit 1
        fi
        sleep 1
    done

    # USB mount
    cDate="$(date +%s)";
    while [ "`/opt/bin/mount | grep -qi /sda1 || echo 1`" ]; do
        if [ "$(($(date +%s) - $cDate))" -gt "$((3*60))" ]; then
            # protection for infinite loop
            logger -s -p err -t opkg "USB Disk is not mounted"
            exit 1
        fi
        sleep 1
    done

    # waiting 1 minute or until ntp client completed (normal timestamp will be in the log)
    cDate="$(date +%s)";
    while [ "$(($(date +%s) - $cDate))" -lt "$((1*60))" ]; do
        sleep 1;
    done;

    if [ -f $FLAGFILE ]; then
        #FIRSTLINE="kernel: ${PARTLINE} warm reboot ${PARTLINE}"
        FIRSTLINE="<14> kernel: ${PARTLINE} warm reboot ${PARTLINE}"
    else
        #FIRSTLINE="kernel: ${PARTLINE} cold reboot ${PARTLINE}"
        FIRSTLINE="<14> kernel: ${PARTLINE} cold reboot ${PARTLINE}"
    fi

    # IETF log format message
    ndmc -c show log | sed ':a;N;$!ba;s/\n \{1,\}/ /g' | cut -c -190 | sed '1,/kernel: Booting Linux/ {/kernel: Booting Linux/!d}' \
        | sed 's/^C/<10>/' | sed 's/^E/<11>/' | sed 's/^W/<12>/' | sed 's/^I/<14>/' \
        | sed 's/\[.*:..:..\] //' | sed "1 i\\${FIRSTLINE}" | sed '$ d' > ${NDMCFILE}

    ndmc -c system log server 127.0.0.1:3127

    start
    ;;
stop|kill)
    stop
    ;;
restart)
    stop
    start
    ;;
*)
    echo "ACTION=unknown" | logger -p emerg -t kernel
    ;;
esac


. /opt/etc/init.d/rc.func

/opt/etc/syslog-ng.conf:

# Collect all local logs into a single file /var/log/messages.
# See https://www.syslog-ng.com/technical-documents/list/syslog-ng-open-source-edition
# See https://github.com/balabit/syslog-ng-ose-guides

@version: 4.7
@include "scl.conf"

# cat /opt/etc/syslog-ng.conf | nocomments
###############################################################################
# /opt/etc/init.d/S01syslog-ng stop
# /opt/etc/init.d/S01syslog-ng start
# /opt/etc/init.d/S01syslog-ng restart
###############################################################################
# Source drivers available in syslog-ng (syslog-ng PE 7.0.34 Administration Guide)
#
# file()             Opens the specified file and reads messages.
# internal()         Messages generated internally in syslog-ng.
# linux-audit()      Reads the logfiles of the auditd application.
# network()          Receives messages from remote hosts using the BSD-syslog protocol over IPv4 and IPv6. Supports the TCP, UDP, ALTP, and TLS network protocols.
# pipe()             Opens the specified named pipe and reads messages.
# program()          Opens the specified application and reads messages from its standard output.
# python() and
# python-fetcher()   Receive or fetch messages using a custom source written in Python.
# syslog()           Listens for incoming messages using the new IETF-standard syslog protocol.
# system()           Automatically detects which platform syslog-ng PE is running on, and collects the native log messages of that platform.
# systemd-journal()  Collects messages directly from the journal of platforms that use systemd.
# systemd-syslog()   Collects messages from the journal using a socket on platforms that use systemd.
# unix-dgram()       Opens the specified unix socket in SOCK_DGRAM mode and listens for incoming messages.
# unix-stream()      Opens the specified unix socket in SOCK_STREAM mode and listens for incoming messages.
# windowsevent()     Reads messages from the Windows Event Collector tool.
###############################################################################

options {
  chain-hostnames(no);      # Enable or disable the chained hostname format.
  create-dirs(yes);         # Enable creating non-existing directories when creating files or socket files.
  keep-hostname(yes);       # Enable or disable hostname rewriting.
  log-fifo-size(10000);     # The number of messages that the output queue can store.
  log-msg-size(200);        # Maximum length of a message in bytes.
  stats(freq(0));           # The period between two STATS messages (sent by syslog-ng, containing statistics about dropped logs) in seconds.
  flush-lines(0);           # How many lines are flushed to a destination at a time.
  trim-large-messages(yes); # trims the incoming log message to the size set in log-msg-size()
  use-fqdn(no);             # Add Fully Qualified Domain Name instead of short hostname.
  ts-format(iso);           # To convert all timestamps to the ISODATE format
};

source src {
  internal();
  unix-dgram("/dev/log");
#  unix-stream("/dev/log");
};

source kernel {
  file("/proc/kmsg" program_override("kernel"));
};

source syslog {
  syslog();
};

source ndmc {
  file( "/tmp/mnt/USB/log/ndmc.log" );
};

# ndmc -c system log server 127.0.0.1:3127
source net {
  network( ip("127.0.0.1") transport("udp") port(3127) keep-timestamp(no) );
};

destination messages {
  file("/tmp/mnt/USB/log/messages" template("${ISODATE} (${PRIORITY}) ${PROGRAM}: ${MESSAGE}\n") time_zone("Europe/Moscow"));
};

destination debug {
#  file("/tmp/mnt/USB/log/debuglog");
  file("/tmp/mnt/USB/log/debuglog" template("${ISODATE} (${PRIORITY}) Keenetic: ${PROGRAM}: ${MESSAGE}\n") time_zone("Europe/Moscow"));
};

filter f_kernel       { not program(kernel) or (program(kernel) and match ("reboot|error|Linux|Memory|Boot CPU|Machine|Found.*storage" value("MESSAGE"))); };
filter f_ndm          { not program(ndm) or (program(ndm) and match("Network Device Manager|Ntp" value("MESSAGE"))); };
filter f_cron         { not program(cron) or (program(cron) and match("started|stopped|Hello" value("MESSAGE")) ); };
filter f_ni_ndhcps    { not program(ndhcps) or (program(ndhcps) and not match("sending INFORM|sending ACK|received ACK|DHCPDISCOVER|DHCPREQUEST|DHCPINFORM|making OFFER|NDM DHCP" value("MESSAGE")) ); };
filter f_ni_dhcpd     { not program(dhcpd) or (program(dhcpd) and not match("Information-request|Sending Reply" value("MESSAGE")) ); };
filter f_no_radvd     { not program(radvd); };
filter f_no_telnetd   { not program(telnetd); };
filter f_no_bndstrg   { not program(bndstrg); };
filter f_no_ndnproxy  { not program(ndnproxy); };
filter f_no_avahi     { not program(avahi-daemon); };
filter f_no_nimproxy  { not program(nimproxy); };
filter f_no_ndhcpc    { not program(ndhcpc); };
filter f_no_upnp      { not program(upnp); };
filter f_no_coalagent { not program(coalagent); };
filter f_no_dhcpd     { not program(dhcpd); };
filter f_no_wind      { not program(wind); };

log {
  source(src);
  source(net);
  source(kernel);
  source(syslog);
  source(ndmc);
  filter(f_kernel);
  filter(f_ndm);
  filter(f_cron);
  filter(f_ni_ndhcps);
  filter(f_ni_dhcpd);
  filter(f_no_radvd);
  filter(f_no_telnetd);
  filter(f_no_bndstrg);
  filter(f_no_ndnproxy);
  filter(f_no_avahi);
  filter(f_no_nimproxy);
  filter(f_no_ndhcpc);
  filter(f_no_upnp);
  filter(f_no_coalagent);
  filter(f_no_dhcpd);
  filter(f_no_wind);
  destination(messages);
};

log {
  source(src);
  source(net);
  source(kernel);
  source(syslog);
  source(ndmc);
  destination(debug);
};


# Include any user settings last so that s/he can override or
# supplement all "canned" settings inherited from the distribution.
@include "/opt/etc/syslog-ng.d/" # Put any customization files in this directory

_alex_ · 25 февраля

В какой-то момент настройки opkg перестал стартовать.

Спасло то, что сохранил исходные файлы:

cp /opt/etc/init.d/S01syslog-ng /opt/etc/init.d/K01syslog-ng
cp /opt/etc/syslog-ng.conf /opt/etc/syslog-ng.conf.orig

Подключился по ssh на 22 порт, выполнил:

ls storage:etc/init.d/
more storage:etc/init.d/S01syslog-ng
erase storage:etc/init.d/S01syslog-ng
copy storage:etc/init.d/K01syslog-ng storage:etc/init.d/S01syslog-ng

ls storage:etc
more storage:etc/syslog-ng.conf
erase storage:etc/syslog-ng.conf
copy storage:etc/syslog-ng.conf.orig storage:etc/syslog-ng.conf

И перезагрузился из web.

Не пришлось заморачиваться с повторной установкой opkg...

TheBB · 25 февраля

1 час назад, _alex_ сказал:

В какой-то момент настройки opkg перестал стартовать.

Запустите скрипт с отладкой (измените шебанг на `#!/opt/bin/sh -xv`) и посмотрите, что там он делает.

_alex_ · 27 февраля

Скорее всего, не поможет.

То ли /opt/sbin/syslog-ng зависает при старте из-за ошибок в конфиге /opt/etc/syslog-ng.conf

То ли /opt/etc/init.d/rc.unslung прекращает работу если что-то не стартует

Изменено 27 февраля пользователем _alex_

TheBB · 27 февраля

7 часов назад, _alex_ сказал:

Скорее всего, не поможет.

Как гласит мудрость народная: "Не попробуешь - не узнаешь."

Как минимум, дублируются функции - собственные start/stop и из подключаемого файла /opt/etc/init.d/rc.func.

Войти

Запись syslog на внешний USB-диск с помощю Syslog-ng

Рекомендуемые сообщения

_alex_

Топ авторов темы

Популярные дни

Топ авторов темы

Популярные дни

Популярные посты

Roman_Petrov

gvan

Roman_Petrov

Изображения в теме

_alex_

_alex_

_alex_

_alex_

TheBB

_alex_

TheBB

Присоединяйтесь к обсуждению

Последние посетители 0 пользователей онлайн

Roman_Petrov

gvan

Roman_Petrov

Обзор

Активность

Магазин

My Details

Важная информация