recepies/lxc/lxc-watchdog.sh

#! /bin/sh
### BEGIN INIT INFO
# Provides:          lxc-watchdog
# Required-Start:    $remote_fs $named $network $time
# Required-Stop:     $remote_fs $named $network
# Required-Start:    
# Required-Stop:     
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Manage Linux Containers startup/shutdown
# Description:       Uses clever inotify hack to monitor container's
#                    halt/reboot events watching /var/run/utmp
### END INIT INFO

# Author: Dobrica Pavlinusic <dpavlin@rot13.org>
#
# based on Tony Risinger post to lxc-users mailing list
# http://www.mail-archive.com/lxc-users@lists.sourceforge.net/msg00074.html
#
# Install with:
# ln -sf /srv/sysadmin-cookbook/recepies/lxc/lxc-watchdog.sh /etc/init.d/lxc-watchdog
# update-rc.d lxc-watchdog defaults


which inotifywait >/dev/null || apt-get install inotify-tools


lxc_exists() {
        name=$1

        if [ ! -e /var/lib/lxc/$name/config ] ; then
                echo "Usage: $0 name"
                lxc_status
                exit 1
        fi
}


lxc_rootfs() {
        grep '^ *lxc\.rootfs *=' "/var/lib/lxc/$1/config" | cut -d= -f2 | sed 's/^ *//'
}

lxc_hostname() {
        inside=`cat $(lxc_rootfs $1)/etc/hostname`
        config=`grep lxc.utsname /var/lib/lxc/$name/config | cut -d= -f2`
        echo "$config [$inside]";
}

lxc_ip() {
        ( grep lxc.network.ipv4 /var/lib/lxc/$name/config | grep -v '^#' | cut -d= -f2 || \
        grep address $(lxc_rootfs $name)/etc/network/interfaces | grep -v '^#' | sed 's/.*address //' ) | \
        head -1 | \
        sed -e 's/ *//g' -e 's/\/.*$//'
}

lxc_status() {
        ( find /var/lib/lxc/ -name "config" | cut -d/ -f5 | sort -u | while read name ; do
                status=`lxc-info -n $name 2>/dev/null | grep state: | cut -d: -f2`     # 0.7.5
                test -z "$status" && status=`lxc-info -n $name | sed -e 's/^.* is //'` # 0.7.2
                boot="-"
                test -s /var/lib/lxc/$name/on_boot && boot="boot"
                echo "$name $status $boot $(lxc_rootfs $name) $(lxc_ip $name) $(lxc_hostname $name)"
        done ) | column -t
}


cleanup_init_scripts() {
        rootfs=$(lxc_rootfs $1)

        ls \
                $rootfs/etc/rc?.d/*checkroot* \
                $rootfs/etc/rc?.d/*umountfs \
                $rootfs/etc/rc?.d/*umountroot \
                $rootfs/etc/rc?.d/*hwclock* \
                $rootfs/etc/rc?.d/*udev* \
                $rootfs/etc/rc?.d/*checkfs* \
        2>/dev/null | xargs -i rm -v {}

        echo $1 > $rootfs/etc/hostname
        grep $1 $rootfs/etc/hosts || echo "$(lxc_ip $1) $1" >> $rootfs/etc/hosts
}


setup_inittab() {
        rootfs=$(lxc_rootfs $1)
        remove=$2
        add=$3

        # let container respond to kill -SIGPWR
        inittab=$rootfs/etc/inittab
        if test -e $inittab && ! grep "$add" ${inittab} >/dev/null ; then
                grep -v "$remove" ${inittab} > ${inittab}.new
                echo $add >> ${inittab}.new
                mv ${inittab}.new ${inittab}
                echo "$inittab modified with $add"
        fi
}


lxc_log() {
        echo `date +%Y-%m-%dT%H:%M:%S` $*
}


lxc_kill() {
        name=$1
        sig=$2

        ver=`lxc-version | cut -d: -f2 | sed 's/\.//g'`
        opts=''
        test $ver -ge 075 && opts='--'

        init_pid=`lxc-ps $opts -C init -o pid | grep "^$name" | cut -d" " -f2-`
        if [ -z "$init_pid" ] ; then
                lxc-info -n $name
                exit 1
        fi
        lxc_log "$name kill $sig $init_pid"
        /bin/kill $sig $init_pid
}

lxc_stop() {
        lxc_log "$name stop"
        lxc_kill $name -SIGPWR
        lxc-wait -n $name -s STOPPED
        lxc_log "$name stoped"
#       rm -f /var/lib/lxc/${name}/on_boot
}


lxc_start() {
        name=$1
        rootfs=$(lxc_rootfs $1)

        if [ ! -e $rootfs ] ; then
                echo "ERROR $name rootfs $rootfs not found"
                return
        fi

        if ! lxc-info -n $name | grep RUNNING ; then
                lxc_log "$name start"
                dev=`df -P $rootfs | tail -1 | cut -d" " -f1`
                mount $dev -o remount,rw # fix debian upgrade which remounts dir ro
                lxc-start -n $name -o /tmp/${name}.log -d
                lxc-wait  -n $name -s RUNNING
                lxc-info  -n $name
                test -f /var/lib/lxc/${name}/on_boot || echo $name > /var/lib/lxc/${name}/on_boot
        fi
}

lxc_watchdog() {
name=$1
cgroup=$(mount -t cgroup | awk '{ print $3 }')
test -d "$cgroup/lxc/$1" && cgroup="$cgroup/lxc"
rootfs=$(lxc_rootfs $1)
run=$rootfs/var/run
test -L $run && run=$rootfs/`readlink $run` # recent Debian have symlink to /run
cd $run || echo "can't cd watchdog into $run"

while true; do
        tasks=`wc -l < $cgroup/${name}/tasks`
        stop_on=1 # init
        sulogins=`lxc-ps --name $name | grep sulogin | wc -l`
        if [ "$sulogins" -gt 0 ] ; then
                stop_on=`expr $stop_on + $sulogins`
        fi

        test -z "$tasks" && exit 1
        if [ "$tasks" -eq $stop_on ]; then

                runlevel="$(runlevel utmp)"
                lxc_log "$name runlevel $runlevel"

                case $runlevel in
                N*)
                        # nothing for new boot state
                ;;
                ??0|unknown)
                        lxc_log "$name halt"
                        lxc-stop -n "${name}"
                        lxc-wait -n ${name} -s STOPPED
                        break
                ;;
                ??6)
                        lxc_log "$name reboot";
                        lxc-stop -n ${name}
                        lxc-wait -n ${name} -s STOPPED
                        lxc-start -d -n ${name} -o /tmp/${name}.log
                ;;
                *)
                        # make sure vps is still running
                        state="$(lxc-info -n "${name}" | sed -e 's/.* is //')"
                        [ "$state" = "RUNNING" ] || break
                ;;
                esac
        else
                lxc_log "$name $tasks tasks $sulogins console"
        fi

        # time of 5 minutes on it JUST IN CASE...
        inotifywait -qqt 300 utmp
done

lxc_log "$name watchdog exited"

}


usage() {
        echo "Usage: $0 {start|stop|restart|status|boot|disable} [name name ... ]" >&2
        exit 3
}

command_on_lxc() {
command=$1
shift

echo "# $command $1"

case "$command" in

start)
        lxc_exists $1
        cleanup_init_scripts $1
        setup_inittab $1 ":respawn:/sbin/getty.*tty1"   "c1:12345:respawn:/sbin/getty 38400 tty1 linux"
        setup_inittab $1 "::power"                      "p0::powerfail:/sbin/init 0"
        setup_inittab $1 "::ctrlaltdel"                 "p6::ctrlaltdel:/sbin/init 6"
        lxc_start $1
        # give container 5 seconds to start more than one process
        ( sleep 5 ; nohup $0 watchdog $1 >> /tmp/$1.log 2>/dev/null ) &
        ;;
stop|halt)
        lxc_exists $1
        lxc_stop $1
        ;;
reload|force-reload|restart|reboot)
        lxc_kill $1 -SIGINT
        ;;
watchdog)
        lxc_watchdog $1
        ;;
boot)
        echo $1 > /var/lib/lxc/$1/on_boot
        ;;
disable)
        echo -n > /var/lib/lxc/$1/on_boot
        ;;
*)
        usage
        ;;

esac

}

command=$1
test -z "$command" && usage
test "$command" = "status" && lxc_status && exit
shift

if [ -z "$1" ] ; then
        ls /var/lib/lxc/*/on_boot | while read path ; do
                name=`echo $path | cut -d/ -f5`
                if [ "$command" != "start" -o "$command" = "start" -a -s $path ] ; then
                        command_on_lxc $command $name
                else
                        echo "# skip $command $name"
                fi
        done
else
        while [ ! -z "$1" ] ; do
                command_on_lxc $command $1
                shift
        done
fi

1	dpavlin	184	#! /bin/sh
2			### BEGIN INIT INFO
3			# Provides: lxc-watchdog
4			# Required-Start: $remote_fs $named $network $time
5			# Required-Stop: $remote_fs $named $network
6			# Required-Start:
7			# Required-Stop:
8			# Default-Start: 2 3 4 5
9			# Default-Stop: 0 1 6
10			# Short-Description: Manage Linux Containers startup/shutdown
11			# Description: Uses clever inotify hack to monitor container's
12			# halt/reboot events watching /var/run/utmp
13			### END INIT INFO
14	root	175
15	dpavlin	184	# Author: Dobrica Pavlinusic <dpavlin@rot13.org>
16	dpavlin	181	#
17	dpavlin	184	# based on Tony Risinger post to lxc-users mailing list
18	root	175	# http://www.mail-archive.com/lxc-users@lists.sourceforge.net/msg00074.html
19	root	190	#
20			# Install with:
21			# ln -sf /srv/sysadmin-cookbook/recepies/lxc/lxc-watchdog.sh /etc/init.d/lxc-watchdog
22			# update-rc.d lxc-watchdog defaults
23	root	175
24	dpavlin	181
25	dpavlin	177	which inotifywait >/dev/null \|\| apt-get install inotify-tools
26	root	175
27
28	dpavlin	181	lxc_exists() {
29			name=$1
30	root	175
31	dpavlin	181	if [ ! -e /var/lib/lxc/$name/config ] ; then
32			echo "Usage: $0 name"
33			lxc_status
34			exit 1
35			fi
36			}
37	root	175
38	dpavlin	179
39	dpavlin	181	lxc_rootfs() {
40	dpavlin	194	grep '^ lxc\.rootfs =' "/var/lib/lxc/$1/config" \| cut -d= -f2 \| sed 's/^ *//'
41	dpavlin	181	}
42	dpavlin	178
43	dpavlin	232	lxc_hostname() {
44			inside=`cat $(lxc_rootfs $1)/etc/hostname`
45			config=`grep lxc.utsname /var/lib/lxc/$name/config \| cut -d= -f2`
46			echo "$config [$inside]";
47			}
48	root	175
49	dpavlin	232	lxc_ip() {
50	dpavlin	235	( grep lxc.network.ipv4 /var/lib/lxc/$name/config \| grep -v '^#' \| cut -d= -f2 \|\| \
51			grep address $(lxc_rootfs $name)/etc/network/interfaces \| grep -v '^#' \| sed 's/.*address //' ) \| \
52			head -1 \| \
53	dpavlin	233	sed -e 's/ //g' -e 's/\/.$//'
54	dpavlin	232	}
55
56	dpavlin	183	lxc_status() {
57	dpavlin	271	( find /var/lib/lxc/ -name "config" \| cut -d/ -f5 \| sort -u \| while read name ; do
58	dpavlin	272	status=`lxc-info -n $name 2>/dev/null \| grep state: \| cut -d: -f2` # 0.7.5
59			test -z "$status" && status=`lxc-info -n $name \| sed -e 's/^.* is //'` # 0.7.2
60	dpavlin	196	boot="-"
61	dpavlin	189	test -s /var/lib/lxc/$name/on_boot && boot="boot"
62	dpavlin	232	echo "$name $status $boot $(lxc_rootfs $name) $(lxc_ip $name) $(lxc_hostname $name)"
63	dpavlin	196	done ) \| column -t
64	dpavlin	183	}
65
66
67	dpavlin	181	cleanup_init_scripts() {
68			rootfs=$(lxc_rootfs $1)
69	dpavlin	179
70	dpavlin	181	ls \
71	dpavlin	234	$rootfs/etc/rc?.d/checkroot \
72	dpavlin	181	$rootfs/etc/rc?.d/*umountfs \
73			$rootfs/etc/rc?.d/*umountroot \
74			$rootfs/etc/rc?.d/hwclock \
75	root	218	$rootfs/etc/rc?.d/udev \
76	dpavlin	237	$rootfs/etc/rc?.d/checkfs \
77	dpavlin	181	2>/dev/null \| xargs -i rm -v {}
78	dpavlin	226
79			echo $1 > $rootfs/etc/hostname
80	dpavlin	233	grep $1 $rootfs/etc/hosts \|\| echo "$(lxc_ip $1) $1" >> $rootfs/etc/hosts
81	dpavlin	181	}
82	dpavlin	179
83	dpavlin	181
84			setup_inittab() {
85			rootfs=$(lxc_rootfs $1)
86	dpavlin	183	remove=$2
87			add=$3
88	dpavlin	181
89			# let container respond to kill -SIGPWR
90			inittab=$rootfs/etc/inittab
91	dpavlin	227	if test -e $inittab && ! grep "$add" ${inittab} >/dev/null ; then
92	dpavlin	183	grep -v "$remove" ${inittab} > ${inittab}.new
93			echo $add >> ${inittab}.new
94	dpavlin	181	mv ${inittab}.new ${inittab}
95	dpavlin	183	echo "$inittab modified with $add"
96	dpavlin	181	fi
97			}
98
99
100	dpavlin	189	lxc_log() {
101			echo `date +%Y-%m-%dT%H:%M:%S` $*
102			}
103
104
105	dpavlin	183	lxc_kill() {
106	dpavlin	181	name=$1
107	dpavlin	183	sig=$2
108	dpavlin	181
109	dpavlin	284	ver=`lxc-version \| cut -d: -f2 \| sed 's/\.//g'`
110			opts=''
111			test $ver -ge 075 && opts='--'
112
113			init_pid=`lxc-ps $opts -C init -o pid \| grep "^$name" \| cut -d" " -f2-`
114	dpavlin	181	if [ -z "$init_pid" ] ; then
115			lxc-info -n $name
116			exit 1
117			fi
118	dpavlin	189	lxc_log "$name kill $sig $init_pid"
119	dpavlin	183	/bin/kill $sig $init_pid
120			}
121
122			lxc_stop() {
123	dpavlin	189	lxc_log "$name stop"
124	dpavlin	183	lxc_kill $name -SIGPWR
125	dpavlin	179	lxc-wait -n $name -s STOPPED
126	dpavlin	189	lxc_log "$name stoped"
127	dpavlin	183	# rm -f /var/lib/lxc/${name}/on_boot
128	dpavlin	181	}
129	dpavlin	179
130
131	dpavlin	181	lxc_start() {
132			name=$1
133	dpavlin	227	rootfs=$(lxc_rootfs $1)
134	dpavlin	181
135	dpavlin	227	if [ ! -e $rootfs ] ; then
136			echo "ERROR $name rootfs $rootfs not found"
137			return
138			fi
139
140	dpavlin	181	if ! lxc-info -n $name \| grep RUNNING ; then
141	dpavlin	189	lxc_log "$name start"
142	dpavlin	244	dev=`df -P $rootfs \| tail -1 \| cut -d" " -f1`
143			mount $dev -o remount,rw # fix debian upgrade which remounts dir ro
144	dpavlin	181	lxc-start -n $name -o /tmp/${name}.log -d
145			lxc-wait -n $name -s RUNNING
146			lxc-info -n $name
147	dpavlin	185	test -f /var/lib/lxc/${name}/on_boot \|\| echo $name > /var/lib/lxc/${name}/on_boot
148	dpavlin	181	fi
149			}
150
151			lxc_watchdog() {
152			name=$1
153	dpavlin	301	cgroup=$(mount -t cgroup \| awk '{ print $3 }')
154			test -d "$cgroup/lxc/$1" && cgroup="$cgroup/lxc"
155	dpavlin	181	rootfs=$(lxc_rootfs $1)
156	root	264	run=$rootfs/var/run
157			test -L $run && run=$rootfs/`readlink $run` # recent Debian have symlink to /run
158			cd $run \|\| echo "can't cd watchdog into $run"
159	dpavlin	181
160	root	175	while true; do
161	dpavlin	301	tasks=`wc -l < $cgroup/${name}/tasks`
162	dpavlin	236	stop_on=1 # init
163	dpavlin	245	sulogins=`lxc-ps --name $name \| grep sulogin \| wc -l`
164	dpavlin	236	if [ "$sulogins" -gt 0 ] ; then
165			stop_on=`expr $stop_on + $sulogins`
166			fi
167
168	dpavlin	181	test -z "$tasks" && exit 1
169	dpavlin	236	if [ "$tasks" -eq $stop_on ]; then
170	root	175
171	root	264	runlevel="$(runlevel utmp)"
172	dpavlin	187	lxc_log "$name runlevel $runlevel"
173	root	175
174			case $runlevel in
175			N*)
176			# nothing for new boot state
177			;;
178	root	264	??0\|unknown)
179	dpavlin	187	lxc_log "$name halt"
180	root	175	lxc-stop -n "${name}"
181	dpavlin	181	lxc-wait -n ${name} -s STOPPED
182	root	175	break
183			;;
184			??6)
185	dpavlin	187	lxc_log "$name reboot";
186	root	175	lxc-stop -n ${name}
187			lxc-wait -n ${name} -s STOPPED
188			lxc-start -d -n ${name} -o /tmp/${name}.log
189			;;
190			*)
191			# make sure vps is still running
192			state="$(lxc-info -n "${name}" \| sed -e 's/.* is //')"
193			[ "$state" = "RUNNING" ] \|\| break
194			;;
195			esac
196	dpavlin	181	else
197	dpavlin	236	lxc_log "$name $tasks tasks $sulogins console"
198	root	175	fi
199	dpavlin	182
200			# time of 5 minutes on it JUST IN CASE...
201	root	264	inotifywait -qqt 300 utmp
202	root	175	done
203
204	dpavlin	189	lxc_log "$name watchdog exited"
205	dpavlin	181
206			}
207
208
209	root	190	usage() {
210			echo "Usage: $0 {start\|stop\|restart\|status\|boot\|disable} [name name ... ]" >&2
211			exit 3
212			}
213
214	dpavlin	183	command_on_lxc() {
215			command=$1
216			shift
217	dpavlin	181
218	dpavlin	183	echo "# $command $1"
219
220			case "$command" in
221
222	dpavlin	181	start)
223	dpavlin	183	lxc_exists $1
224			cleanup_init_scripts $1
225	dpavlin	193	setup_inittab $1 ":respawn:/sbin/getty.*tty1" "c1:12345:respawn:/sbin/getty 38400 tty1 linux"
226			setup_inittab $1 "::power" "p0::powerfail:/sbin/init 0"
227			setup_inittab $1 "::ctrlaltdel" "p6::ctrlaltdel:/sbin/init 6"
228	dpavlin	183	lxc_start $1
229	dpavlin	186	# give container 5 seconds to start more than one process
230			( sleep 5 ; nohup $0 watchdog $1 >> /tmp/$1.log 2>/dev/null ) &
231	dpavlin	181	;;
232	dpavlin	183	stop\|halt)
233			lxc_exists $1
234			lxc_stop $1
235	dpavlin	181	;;
236	dpavlin	183	reload\|force-reload\|restart\|reboot)
237			lxc_kill $1 -SIGINT
238	dpavlin	181	;;
239			watchdog)
240	dpavlin	183	lxc_watchdog $1
241	dpavlin	181	;;
242	dpavlin	188	boot)
243			echo $1 > /var/lib/lxc/$1/on_boot
244			;;
245			disable)
246			echo -n > /var/lib/lxc/$1/on_boot
247			;;
248	dpavlin	181	*)
249	root	190	usage
250	dpavlin	181	;;
251
252			esac
253
254	dpavlin	183	}
255
256			command=$1
257	root	190	test -z "$command" && usage
258			test "$command" = "status" && lxc_status && exit
259	dpavlin	189	shift
260	dpavlin	183
261			if [ -z "$1" ] ; then
262			ls /var/lib/lxc/*/on_boot \| while read path ; do
263			name=`echo $path \| cut -d/ -f5`
264	dpavlin	189	if [ "$command" != "start" -o "$command" = "start" -a -s $path ] ; then
265			command_on_lxc $command $name
266			else
267			echo "# skip $command $name"
268			fi
269	dpavlin	183	done
270			else
271			while [ ! -z "$1" ] ; do
272			command_on_lxc $command $1
273			shift
274			done
275			fi
276