Difference between revisions of "Logstash"

(Created page with "Category:Linux =Installation= Source: http://logstash.net/docs/latest/repositories * '''Add Logstash repository''': see Sources#ELK * Install application <syntaxh...")
 
 
(41 intermediate revisions by the same user not shown)
Line 1: Line 1:
 
[[Category:Linux]]
 
[[Category:Linux]]
 +
 +
[[File:Icon logstash.png|32px|caption|Logstash]] ''Logstash'' retrieves, extracts and sends data to the ''ElasticSearch'' server.
 +
 +
''Logstash'' needs to be install on all the servers / clients you want to get logs from.
 +
 +
 +
Related topics:
 +
* [[File:Grok icon.png|link=Logstash grok expressions|160px|caption|Logstash grok expressions]] [[Logstash grok expressions]]
 +
* [[File:Configuration examples.png|link=Logstash configuration examples|64px|caption|Logstash configuration examples]] [[logstash configuration examples]]
 +
  
 
=Installation=
 
=Installation=
  
 +
You can install logstash either manually or as an APT-GET package.
 +
 +
I recommend you to use the ''manual'' installation because the ''automatic'' one will chroot you in /var/log. If your application is using logs that are somewhere else, then you'll be screwed.
 +
 +
 +
==Manual installation (recommended)==
 +
 +
'''Be careful''': Logstash version must match the ElasticSearch version for better performances.
 +
 +
 +
* Get Logstash from the official website: http://logstash.net/
 +
* Install it and unpack it into /opt/
 +
 +
<syntaxhighlight lang="bash">
 +
cd /tmp
 +
wget https://download.elasticsearch.org/logstash/logstash/logstash-1.4.2.tar.gz
 +
tar xzvf logstash-1.4.2.tar.gz
 +
rm logstash-1.4.2.tar.gz
 +
mv logstash-1.4.2/ /opt/
 +
cd /opt
 +
ln -s /opt/logstash-1.4.2 /opt/logstash
 +
</syntaxhighlight>
 +
 +
 +
* Create configuration directories
 +
 +
<syntaxhighlight lang="bash">
 +
mkdir -p /etc/logstash/conf.d
 +
mkdir /etc/logstash/grok
 +
mkdir /etc/logstash/db
 +
chmod -R 777 /etc/logstash
 +
</syntaxhighlight>
 +
 +
 +
* touch log file
 +
 +
<syntaxhighlight lang="bash">
 +
touch /var/log/logstash.log
 +
chmod -R 777 /var/log/logstash.log
 +
</syntaxhighlight>
 +
 +
 +
* Create an init.d script
 +
 +
<syntaxhighlight lang="bash">
 +
cd /etc/init.d
 +
vim logstash.sh
 +
</syntaxhighlight>
 +
 +
 +
Parse the following content:
 +
 +
<syntaxhighlight lang="bash">
 +
#!/bin/sh
 +
### BEGIN INIT INFO
 +
# Provides: logstash
 +
# Required-Start: $remote_fs $syslog
 +
# Required-Stop: $remote_fs $syslog
 +
# Default-Start: 2 3 4 5
 +
# Default-Stop: 0 1 6
 +
# Short-Description: Start daemon at boot time
 +
# Description: Enable service provided by daemon.
 +
### END INIT INFO
 +
 +
. /lib/lsb/init-functions
 +
 +
if [ $(id -u) -ne 0 ]; then
 +
echo -e " "
 +
echo -e "!!!!!!!!!!!!!!!!!!!!"
 +
echo -e "!! Security alert !!"
 +
echo -e "!!!!!!!!!!!!!!!!!!!!"
 +
echo -e "You need to be root or have root privileges to run this script!\n\n"
 +
echo -e " "
 +
exit 1
 +
fi
 +
 +
# Where should Logstash keep track of each file?
 +
export SINCEDB_DIR="/etc/logstash/db"
 +
 +
# Logstash params
 +
name="logstash"
 +
logstash_bin="/opt/logstash/bin/logstash"
 +
logstash_conf="/etc/logstash/conf.d/"
 +
logstash_log="/var/log/logstash.log"
 +
pid_file="/var/run/$name.pid"
 +
 +
start () {
 +
commandOpts="agent -f $logstash_conf --log ${logstash_log} --verbose"
 +
log_daemon_msg "Starting $name" "$name"
 +
if start-stop-daemon --start --quiet --oknodo --pidfile "$pid_file" -b -m --exec $logstash_bin -- $commandOpts; then
 +
log_end_msg 0
 +
else
 +
log_end_msg 1
 +
fi
 +
}
 +
testConfig () {
 +
echo "#############################"
 +
echo " Logstash configuration test"
 +
echo "#############################"
 +
command="${logstash_bin} -f $logstash_conf --verbose -t"
 +
$command
 +
}
 +
stop () {
 +
log_daemon_msg "Stopping $name" "$name"
 +
start-stop-daemon --stop --quiet --oknodo --pidfile "$pid_file"
 +
}
 +
status () {
 +
status_of_proc -p $pid_file "" "$name"
 +
}
 +
 +
case $1 in
 +
start)
 +
if status; then exit 0; fi
 +
start
 +
;;
 +
stop)
 +
stop
 +
;;
 +
reload)
 +
stop
 +
start
 +
;;
 +
restart)
 +
stop
 +
start
 +
;;
 +
status)
 +
status && exit 0 || exit $?
 +
;;
 +
testConfig)
 +
testConfig
 +
;;
 +
*)
 +
echo "Usage: $0 {start|stop|restart|reload|status|testConfig}"
 +
exit 1
 +
;;
 +
esac
 +
exit 0
 +
 +
</syntaxhighlight>
 +
 +
 +
* Create symlinks
 +
 +
<syntaxhighlight lang="bash">
 +
ln -s /etc/init.d/logstash.sh /usr/bin/logstash
 +
</syntaxhighlight>
 +
 +
 +
* Register application as a service (optional)
 +
 +
<syntaxhighlight lang="bash">
 +
cd /etc/init.d
 +
update-rc.d logstash.sh defaults
 +
</syntaxhighlight>
 +
 +
 +
 +
==Automatic installation==
  
 
Source: http://logstash.net/docs/latest/repositories
 
Source: http://logstash.net/docs/latest/repositories
Line 12: Line 181:
 
apt-get install logstash logstash-contrib
 
apt-get install logstash logstash-contrib
 
</syntaxhighlight>
 
</syntaxhighlight>
 +
 +
>> Binaries in ''/opt/logstash''
 +
 +
>> Configuration in ''/etc/logstash/conf.d/''
 +
 +
>> Logs in ''/var/log/logstash/
 +
 +
 +
* Create a folder for logstash to keep track of each file
 +
 +
<syntaxhighlight lang="bash">
 +
mkdir -p /etc/logstash/db
 +
chmod -R 777 /etc/logstash/
 +
</syntaxhighlight>
 +
 +
 +
* Add a new environment variable in your <code>/etc/profile</code> || <code>/etc/environment</code>
 +
 +
Put:
 +
 +
<syntaxhighlight lang="bash">
 +
SINCEDB_DIR=/etc/logstash/db
 +
</syntaxhighlight>
 +
 +
 +
Apply changes:
 +
 +
<syntaxhighlight lang="bash">
 +
source /etc/environment
 +
</syntaxhighlight>
 +
 +
 +
 +
 +
=Manual commands=
 +
 +
The following command(s) are just here for my personal reference:
 +
 +
<syntaxhighlight lang="bash">
 +
cd /opt/logstash/bin
 +
./logstash -f /etc/logstash/conf.d/ -t --verbose
 +
</syntaxhighlight>
 +
 +
 +
 +
=Grok expressions=
 +
 +
See [[Logstash grok expressions]]
 +
 +
 +
 +
=Configuration examples=
 +
 +
See [[logstash configuration examples]]
 +
 +
 +
 +
=References=
 +
 +
* Very good webinar from the ElasticSearch team: http://www.elasticsearch.org/webinars/introduction-to-logstash/?watch=1
 +
* Very good blog article: https://home.regit.org/2014/01/a-bit-of-logstash-cooking/
 +
* Grok on-line debugger: http://grokdebug.herokuapp.com/

Latest revision as of 16:45, 5 February 2015


Logstash Logstash retrieves, extracts and sends data to the ElasticSearch server.

Logstash needs to be install on all the servers / clients you want to get logs from.


Related topics:


Installation

You can install logstash either manually or as an APT-GET package.

I recommend you to use the manual installation because the automatic one will chroot you in /var/log. If your application is using logs that are somewhere else, then you'll be screwed.


Manual installation (recommended)

Be careful: Logstash version must match the ElasticSearch version for better performances. 


cd /tmp
wget https://download.elasticsearch.org/logstash/logstash/logstash-1.4.2.tar.gz
tar xzvf logstash-1.4.2.tar.gz
rm logstash-1.4.2.tar.gz
mv logstash-1.4.2/ /opt/
cd /opt
ln -s /opt/logstash-1.4.2 /opt/logstash


  • Create configuration directories
mkdir -p /etc/logstash/conf.d
mkdir /etc/logstash/grok
mkdir /etc/logstash/db
chmod -R 777 /etc/logstash


  • touch log file
touch /var/log/logstash.log
chmod -R 777 /var/log/logstash.log


  • Create an init.d script
cd /etc/init.d
vim logstash.sh


Parse the following content: 

#!/bin/sh
### BEGIN INIT INFO
# Provides: logstash
# Required-Start: $remote_fs $syslog
# Required-Stop: $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Start daemon at boot time
# Description: Enable service provided by daemon.
### END INIT INFO

. /lib/lsb/init-functions

if [ $(id -u) -ne 0 ]; then
	echo -e " " 
	echo -e "!!!!!!!!!!!!!!!!!!!!" 
	echo -e "!! Security alert !!" 
	echo -e "!!!!!!!!!!!!!!!!!!!!" 
	echo -e "You need to be root or have root privileges to run this script!\n\n"
	echo -e " " 
	exit 1
fi

# Where should Logstash keep track of each file?
export SINCEDB_DIR="/etc/logstash/db"

# Logstash params
name="logstash"
logstash_bin="/opt/logstash/bin/logstash"
logstash_conf="/etc/logstash/conf.d/"
logstash_log="/var/log/logstash.log"
pid_file="/var/run/$name.pid"

start () {
	commandOpts="agent -f $logstash_conf --log ${logstash_log} --verbose"
	log_daemon_msg "Starting $name" "$name"
	if start-stop-daemon --start --quiet --oknodo --pidfile "$pid_file" -b -m --exec $logstash_bin -- $commandOpts; then
		log_end_msg 0
	else
		log_end_msg 1
	fi
}
testConfig () {
	echo "#############################"
	echo " Logstash configuration test"
	echo "#############################"
	command="${logstash_bin} -f $logstash_conf --verbose -t"
	$command
}
stop () {
	log_daemon_msg "Stopping $name" "$name"
	start-stop-daemon --stop --quiet --oknodo --pidfile "$pid_file"
}
status () {
	status_of_proc -p $pid_file "" "$name"
}

case $1 in
	start)
		if status; then exit 0; fi
		start
		;;
	stop)
		stop
		;;
	reload)
		stop
		start
		;;
	restart)
		stop
		start
		;;
	status)
		status && exit 0 || exit $?
		;;
	testConfig)
		testConfig
		;;
	*)
		echo "Usage: $0 {start|stop|restart|reload|status|testConfig}"
		exit 1
		;;
esac
exit 0


  • Create symlinks
ln -s /etc/init.d/logstash.sh /usr/bin/logstash


  • Register application as a service (optional)
cd /etc/init.d
update-rc.d logstash.sh defaults


Automatic installation

Source: http://logstash.net/docs/latest/repositories

  • Add Logstash repository: see Sources#ELK
  • Install application
apt-get install logstash logstash-contrib

>> Binaries in /opt/logstash

>> Configuration in /etc/logstash/conf.d/

>> Logs in /var/log/logstash/


  • Create a folder for logstash to keep track of each file
mkdir -p /etc/logstash/db
chmod -R 777 /etc/logstash/


  • Add a new environment variable in your /etc/profile || /etc/environment

Put: 

SINCEDB_DIR=/etc/logstash/db


Apply changes: 

source /etc/environment



Manual commands

The following command(s) are just here for my personal reference: 

cd /opt/logstash/bin
./logstash -f /etc/logstash/conf.d/ -t --verbose


Grok expressions

See Logstash grok expressions


Configuration examples

See logstash configuration examples


References

Retrieved from "http://www.daxiongmao.eu/wiki/index.php?title=Logstash&oldid=2096"