logstash1.5.5测试笔记(4)

云计算 2023-07-15 大白菜程序猿 手机阅读

YUM安装

rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch
Add the following in your /etc/yum.repos.d/ directory in a file with a .repo suffix, for example logstash.repo
[logstash-2.2]
name=Logstash repository for 2.2.x packages
baseurl=http://packages.elastic.co/logstash/2.2/centos
gpgcheck=1
gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1

借鉴与：http://udn.yyuap.com/doc/logstash-best-practice-cn/output/elasticsearch.html

由于之前用2.2.2.1很多不熟悉，导致很多问题，这次试用1.5.5借鉴与西门飞冰，也是我的好友的文章，感谢编译安装：

wget https://download.elastic.co/logstash/logstash/logstash-1.5.5.tar.gz
yum -y install java-1.8.0
tar zxf logstash-1.5.4.tar.gz
mv logstash-1.5.4 /usr/local/
ln -s /usr/local/logstash-1.5.4/ /usr/local/logstash

启动脚本：

vim /etc/init.d/logstash
#!/bin/sh
1. Init script for logstash
1. Maintained by Elasticsearch
1. Generated by pleaserun.
1. Implemented based on LSB Core 3.1:
1.   * Sections: 20.2, 20.3
1. ### BEGIN INIT INFO
1. Provides:          logstash
1. Required-Start:    $remote_fs $syslog
1. Required-Stop:     $remote_fs $syslog
1. Default-Start:     2 3 4 5
1. Default-Stop:      0 1 6
1. Short-Description:
1. Description:        Starts Logstash as a daemon.
### END INIT INFO

PATH=/sbin:/usr/sbin:/bin:/usr/bin
export PATH

if [ `id -u` -ne 0 ]; then
   echo "You need root privileges to run this script"
   exit 1
fi

name=logstash
pidfile="/var/run/$name.pid"
export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin
LS_USER=logstash
LS_GROUP=logstash
LS_HOME=/usr/local/logstash
LS_HEAP_SIZE="500m"
LS_JAVA_OPTS="-Djava.io.tmpdir=${LS_HOME}"
LS_LOG_DIR=/usr/local/logstash
LS_LOG_FILE="${LS_LOG_DIR}/$name.log"
LS_CONF_FILE=/etc/logstash.conf
LS_OPEN_FILES=16384
LS_NICE=19
LS_OPTS=""

[ -r /etc/default/$name ] && . /etc/default/$name
[ -r /etc/sysconfig/$name ] && . /etc/sysconfig/$name

program=/usr/local/logstash/bin/logstash
args="agent -f ${LS_CONF_FILE} -l ${LS_LOG_FILE} ${LS_OPTS}"

start() {

  JAVA_OPTS=${LS_JAVA_OPTS}
  HOME=${LS_HOME}
  export PATH HOME JAVA_OPTS LS_HEAP_SIZE LS_JAVA_OPTS LS_USE_GC_LOGGING

  1. set ulimit as (root, presumably) first, before we drop privileges
  ulimit -n ${LS_OPEN_FILES}

  1. Run the program!
  nice -n ${LS_NICE} sh -c "
    cd $LS_HOME
    ulimit -n ${LS_OPEN_FILES}
    exec "$program" $args
  " > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &

  1. Generate the pidfile from here. If we instead made the forked process
  1. generate it there will be a race condition between the pidfile writing
  1. and a process possibly asking for status.
  echo $! > $pidfile

  echo "$name started."
  return 0
}

stop() {
  1. Try a few times to kill TERM the program
  if status ; then
    pid=`cat "$pidfile"`
    echo "Killing $name (pid $pid) with SIGTERM"
    kill -TERM $pid
    1. Wait for it to exit.
    for i in 1 2 3 4 5 ; do
      echo "Waiting $name (pid $pid) to die..."
      status || break
      sleep 1
    done
    if status ; then
      echo "$name stop failed; still running."
    else
      echo "$name stopped."
    fi
  fi
}

status() {
  if [ -f "$pidfile" ] ; then
    pid=`cat "$pidfile"`
    if kill -0 $pid > /dev/null 2> /dev/null ; then
      1. process by this pid is running.
      1. It may not be our pid, but that's what you get with just pidfiles.
      1. TODO(sissel): Check if this process seems to be the same as the one we
      1. expect. It'd be nice to use flock here, but flock uses fork, not exec,
      1. so it makes it quite awkward to use in this case.
      return 0
    else
      return 2 # program is dead but pid file exists
    fi
  else
    return 3 # program is not running
  fi
}

force_stop() {
  if status ; then
    stop
    status && kill -KILL `cat "$pidfile"`
  fi
}

case "$1" in
  start)
    status
    code=$?
    if [ $code -eq 0 ]; then
      echo "$name is already running"
    else
      start
      code=$?
    fi
    exit $code
    ;;
  stop) stop ;;
  force-stop) force_stop ;;
  status)
    status
    code=$?
    if [ $code -eq 0 ] ; then
      echo "$name is running"
    else
      echo "$name is not running"
    fi
    exit $code
    ;;
  restart)

    stop && start
    ;;
  reload)
    stop && start
    ;;
  *)
    echo "Usage: $SCRIPTNAME {start|stop|force-stop|status|restart}" >&2
    exit 3
  ;;
esac

exit $?

执行权限和开机启动

chkconfig --add logstash
chkconfig logstash on
chkconfig --list logstash

配置文件：

[root@elk1 ~]# cat /etc/logstash.conf 
input {
 file {
    path => "/var/log/messages"
    type => "system-log" #指定日志类型，以便在一个配置文件中收集多个日志，用来区别输出
    }

 file {
    path => "/root/test.log"
    type => "test.log" #指定日志类型，以便在一个配置文件中收集多个日志，用来区别输出 
    }
}
output {
    if [type] == "system-log" {   
   elasticsearch {
    host => ["192.168.1.4:9200","192.168.1.5:9200"]
    index => "system-messages-%{+YYYY.MM.dd.HH}"
        protocol => "http"
        workers => 5
        template_overwrite => true
    }
}
    if [type] == "test.log" {     #对input中的输入进行判断，如果日志类型为nginx-access则执行以下输出，否则不执行
   elasticsearch {    
    host => ["192.168.1.4:9200","192.168.1.5:9200"]
        index => "test.log-%{+YYYY.MM.dd.HH}"
        protocol => "http"
        workers => 5
        template_overwrite => true

}
}
}
[root@elk1 ~]#

启动

[root@elk1 ~]# /usr/local/logstash/bin/logstash -f /etc/logstash.conf
Logstash startup completed

导入日志测试：

[root@elk1 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0  >> /root/test.log 
[root@elk1 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0 >> /var/log/messages

logstashsystem-messages.png