ToFoIn – Toggle Failover of Internet или переключение между двумя внешними каналами в FreeBSD

1. Создать в /usr/local/etc основной файл настроек dhcpd.conf, на который ссылаемся в rc.conf. У меня выглядит следующим образом:
   /usr/local/etc/dhcpd.conf

   
   # dhcpd.conf
   #
   
   # option definitions common to all supported networks...
   option domain-name "companyname.local";
   option domain-name-servers 10.0.0.2, 10.0.0.1;
   option ntp-servers 10.0.0.2, 10.0.0.1;
   option log-servers 10.0.0.1;
   update-static-leases on;
   
   # 1 hour
   default-lease-time 3600;
   
   # 1 day
   max-lease-time 86400;
   
   # Use this to enable / disable dynamic dns updates globally.
   ddns-update-style interim;
   
   # If this DHCP server is the official DHCP server for the local
   # network, the authoritative directive should be uncommented.
   authoritative;
   
   # Use this to send dhcp log messages to a different log file (you also
   # have to hack syslog.conf to complete the redirection).
   log-facility local7;
   set vendorclass = option vendor-class-identifier;
   
   # DNS key
   include "/usr/local/etc/dhcpd/dns.key";
   
   zone companyname.local.{
   	primary 127.0.0.1;
   	key DHCP_UPDATER;
   }
   
   zone 0.0.10.in-addr.arpa.{
   	primary 127.0.0.1;
   	key DHCP_UPDATER;
   }
   
   # DHCP Failover, Primary
   include "/usr/local/etc/dhcpd/dhcpd.conf_primary";
   
   # Subnet declaration
   include "/usr/local/etc/dhcpd/dhcpd.subnet";
   
   # Static IP addresses
   include "/usr/local/etc/dhcpd/dhcpd.static";
   

   Здесь dns.key – ключ для связи с dns сервером, данные вопросы подробно рассмотрены в статьях по настройке dns+dhcp.

2. Создать папку /usr/local/etc/dhcpd. Создать в ней следующие файлы, содержащие примерно следующее:
   /usr/local/etc/dhcpd/dhcpd.conf_primary

   
   ##########################
   # DHCP Failover, Primary #
   ##########################
   
   failover peer "dhcpdpeer" {              # Failover configuration
   	primary;                         # I am the primary
           address 10.0.0.1;                # My IP address
           port 1111;
           peer address 10.0.0.2;           # Peer's IP address
           peer port 2222;
           max-response-delay 60;
           max-unacked-updates 10;
           mclt 3600;
           split 128;                       # Leave this at 128, only defined on Primary
           load balance max seconds 3;
   }
   

   /usr/local/etc/dhcpd/dhcpd.subnet

   
   subnet 10.0.0.0 netmask 255.255.255.0 {
   	pool {
   		failover peer "dhcpdpeer";
   		range 10.0.0.15 10.0.0.240;
   	}
   	option subnet-mask 255.255.255.0;
   	option routers 10.0.0.2, 10.0.0.1;
   	option broadcast-address 10.0.0.255;
   	option netbios-name-servers 10.0.0.3;
   	option netbios-dd-server 10.0.0.3;
   	option netbios-node-type 8;
   }
   

   В данном случае netbios name server – windows сервер с запущенной службой wins сервера, также в этой роли может выступать samba.

   /usr/local/etc/dhcpd/dhcpd.static

   
   host SERVER3 {
     hardware ethernet 11:11:11:11:11:11;
     fixed-address 10.0.0.3;
   }  	
   
   host SERVER4 {
     hardware ethernet 22:22:22:22:22:22;
     fixed-address 10.0.0.4;
   }
   

   Данный файл, как нетрудно догадаться, для статических адресов.

3. На втором «маршрутизаторе» файлы выглядят следующим образом:
   /usr/local/etc/dhcpd.conf

   
   # dhcpd.conf
   #
   
   # option definitions common to all supported networks...
   option domain-name "companyname.local ";
   option domain-name-servers 10.0.0.2, 10.0.0.1;
   option ntp-servers 10.0.0.2, 10.0.0.1;
   option log-servers 10.0.0.1;
   update-static-leases on;
   
   # 1 hour
   default-lease-time 3600;
   
   # 1 day
   max-lease-time 86400;
   
   # Use this to enable / disable dynamic dns updates globally.
   ddns-update-style interim;
   
   # If this DHCP server is the official DHCP server for the local
   # network, the authoritative directive should be uncommented.
   authoritative;
   
   # Use this to send dhcp log messages to a different log file (you also
   # have to hack syslog.conf to complete the redirection).
   log-facility local7;
   set vendorclass = option vendor-class-identifier;
   
   # DNS key
   include "/usr/local/etc/dhcpd/dns.key";
   
   zone companyname.local.{
   	secondary 127.0.0.1;
   	key DHCP_UPDATER;
   }
   
   zone 0.0.10.in-addr.arpa.{
   	secondary 127.0.0.1;
   	key DHCP_UPDATER;
   }
   
   # DHCP Failover, Primary
   include "/usr/local/etc/dhcpd/dhcpd.conf_secondary";
   
   # Subnet declaration
   include "/usr/local/etc/dhcpd/dhcpd.subnet.DONOTEDIT";
   
   # Static IP addresses
   include "/usr/local/etc/dhcpd/dhcpd.static.DONOTEDIT";
   

   /usr/local/etc/dhcpd/dhcpd.conf_secondary

   
   ###########################
   # DHCP Failover,Secondary #
   ###########################
   
   failover peer "dhcpdpeer" {              # Failover configuration
   	secondary;                       # I am the secondary
   	address 10.0.0.2;                # My IP address
   	port 2222;
   	peer address 10.0.0.1;           # Peer's IP address
   	peer port 1111;
   	max-response-delay 60;
   	max-unacked-updates 10;
   	mclt 3600;
   	load balance max seconds 3;
   }
   

   Остальные файлы можно взять от первого «маршрутизатора», только изменив название, либо настроить до конца и файлы переместятся автоматически при перезапуске isc-dhcpd (о том, как именно – ниже).

4. Создать исполняемый файл со следующим содержимым:
   /usr/local/bin/dhcpd-sync

   
   #!/bin/sh
   # backup generation
   date=`date -v-1d '+%Y%m%d-%H%M%s'`
   month=`date '+%m%Y'`
   sudo -u dhcp-updater cp -f /usr/local/etc/dhcpd/dhcpd.subnet /var/dhcp-backup/dhcpd.subnet.$date
   sudo -u dhcp-updater bzip2 -f -k -z /var/dhcp-backup/dhcpd.subnet.$date
   sudo -u dhcp-updater tar -r -f /var/dhcp-backup/dhcpd.subnet.$month.tar -C /var/dhcp-backup dhcpd.subnet.$date.bz2
   
   sudo -u dhcp-updater cp -f /usr/local/etc/dhcpd/dhcpd.static /var/dhcp-backup/dhcpd.static.$date
   sudo -u dhcp-updater bzip2 -f -k -z /var/dhcp-backup/dhcpd.static.$date
   sudo -u dhcp-updater tar -r -f /var/dhcp-backup/dhcpd.static.$month.tar -C /var/dhcp-backup dhcpd.static.$date.bz2
   
   sudo -u dhcp-updater scp -P 22 -q /var/dhcp-backup/dhcpd.subnet.$date.bz2 dhcp-updater@10.0.0.2:/var/dhcp-backup
   sudo -u dhcp-updater ssh -p 22 10.0.0.2 tar -r -f /var/dhcp-backup/dhcpd.subnet.$month.tar -C /var/dhcp-backup dhcpd.subnet.$date.bz2
   
   sudo -u dhcp-updater scp -P 22 -q /var/dhcp-backup/dhcpd.static.$date.bz2 dhcp-updater@10.0.0.2:/var/dhcp-backup
   sudo -u dhcp-updater ssh -p 22 10.0.0.2 tar -r -f /var/dhcp-backup/dhcpd.static.$month.tar -C /var/dhcp-backup dhcpd.static.$date.bz2
   
   sudo -u dhcp-updater ssh -p 22 10.0.0.2 rm /var/dhcp-backup/dhcpd.subnet.$date.bz2
   sudo -u dhcp-updater ssh -p 22 10.0.0.2 rm /var/dhcp-backup/dhcpd.static.$date.bz2
   
   sudo -u dhcp-updater rm /var/dhcp-backup/dhcpd.subnet.$date
   sudo -u dhcp-updater rm /var/dhcp-backup/dhcpd.static.$date
   sudo -u dhcp-updater rm /var/dhcp-backup/dhcpd.subnet.$date.bz2
   sudo -u dhcp-updater rm /var/dhcp-backup/dhcpd.static.$date.bz2
   
   # sync and restart secondary DHCP
   sudo -u dhcp-updater scp -P 22 -q /usr/local/etc/dhcpd/dhcpd.subnet dhcp-updater@10.0.0.2:/usr/local/etc/dhcpd/dhcpd.subnet.DONOTEDIT
   sudo -u dhcp-updater scp -P 22 -q /usr/local/etc/dhcpd/dhcpd.static dhcp-updater@10.0.0.2:/usr/local/etc/dhcpd/dhcpd.static.DONOTEDIT
   sudo -u dhcp-updater ssh -p 22 10.0.0.2 sudo /usr/local/etc/rc.d/isc-dhcpd restart
   

5. Создать пользователя dhcp-updater с соответствующими правами на обоих серверах, прописать его в настройках sudo, настроить подключение по ssh по ключу с основного на вторичный «маршрутизатор», удалить пароль. Возможно нужно будет еще создать папку /var/dhcp-backup/ на обоих машинах.
6. Изменить кусок файла /usr/local/etc/rc.d/isc-dhcpd следующим образом:
   До:

   
   dhcpd_checkconfig ()
   {
           local rc_flags_mod
           setup_flags
   	rc_flags_mod="$rc_flags"
           # Eliminate '-q' flag if it is present
   	case "$rc_flags" in
   	*-q*)	rc_flags_mod=`echo "${rc_flags}" | sed -Ee 's/(^-q | -q | -q$)//'` ;;
   	esac
           if ! ${command} -t -q ${rc_flags_mod}; then
                   err 1 "`${command} -t ${rc_flags_mod}` Configuration file sanity check failed"
           fi
   }
   

   После:

   
   dhcpd_checkconfig ()
   {
           local rc_flags_mod
           setup_flags
   	rc_flags_mod="$rc_flags"
           # Eliminate '-q' flag if it is present
   	case "$rc_flags" in
   	*-q*)	rc_flags_mod=`echo "${rc_flags}" | sed -Ee 's/(^-q | -q | -q$)//'` ;;
   	esac
           if ! ${command} -t -q ${rc_flags_mod}; then
                   err 1 "`${command} -t ${rc_flags_mod}` Configuration file sanity check failed"
   	else sh /usr/local/bin/dhcpd-sync	
           fi
   }
   

7. Если все настройки произведены правильно, при перезапуске dhcp-сервера на основной машине текущая конфигурация будет архивироваться, синхронизироваться со вторым сервером, и перезапуск будет происходить на обеих машинах.
8. Нелишним было бы добавить в crontab следующее задание:

   0	0	*	*	*	root	/usr/local/etc/rc.d/isc-dhcpd restart

9. На этом настройка failover dhcpd закончена.

#!/bin/sh
#
# PROVIDE: SETFIB1
# REQUIRE: NETWORKING
# BEFORE: DAEMON
#
# Add the following lines to /etc/rc.conf to enable setfib -1 at startup
# setfib1 (bool): Set to "NO" by default.
#                Set it to "YES" to enable setfib1
# setfib1_defaultroute (str): Set to "" by default
#       Set it to ip address of default gateway for use in fib 1
. /etc/rc.subr
name="setfib1"
rcvar=`set_rcvar`
load_rc_config $name
[ -z "$setfib1_enable" ] && setfib1_enable="NO"
[ -z "$setfib1_defaultrouter" ] && setfib1_defaultrouter=""
start_cmd="${name}_start"
stop_cmd="${name}_stop"
setfib1_start()
{
	if [ ${setfib1_defaultrouter} ]
	then
		setfib 1 route add -net default ${setfib1_defaultrouter}
	else
		echo "Can not set default route for fib 1 - setfib1_defaultrouter is not assigned in rc.conf!"
	fi
}
setfib1_stop()
{
	setfib 1 route del -net default
}
run_rc_command "$1"

hostname="SERVER1.companyname.local"
keymap="ru.koi8-r"
font8x8="cp866-8x8"
font8x14="cp866-8x14"
font8x16="cp866-8x16"
scrnmap="koi8-r2cp866"
cursor="destructive"
ifconfig_eth0="up"
vlans_eth0="vlan1 vlan3"
create_args_vlan1="vlan 1"
create_args_vlan3="vlan 3"
ifconfig_eth1="up"
vlans_eth1="vlan2 vlan4"
create_args_vlan2="vlan 2"
create_args_vlan4="vlan 4"
ifconfig_eth2="inet 10.0.0.1 netmask 255.255.255.0"
ifconfig_vlan1="inet 1.1.1.2/24"
ifconfig_vlan3="inet 10.0.1.1/30"
ifconfig_vlan2="inet 2.2.2.2/24"
ifconfig_vlan4="inet 10.0.2.1/30"
defaultrouter="1.1.1.1"
setfib1_enable="YES"
setfib1_defaultrouter="2.2.2.1"
gateway_enable="YES"
sshd_enable="YES"
moused_enable="YES"
ntpd_enable="YES"
powerd_enable="YES"
hald_enable="YES"
dbus_enable="YES"
dumpdev="AUTO"
firewall_enable="YES"
firewall_logging="YES" 
firewall_script="/etc/firewall.sh"
named_enable="YES"
named_program="/usr/sbin/named"
named_flags="-u bind -c /etc/namedb/named.conf"
dhcpd_enable="YES"
dhcpd_conf="/usr/local/etc/dhcpd.conf"
dhcpd_ifaces="eth2"

#!/bin/sh
# Delete all rules
/sbin/ipfw -q -f flush
/sbin/ipfw -q -f pipe flush
/sbin/ipfw -q -f queue flush
/sbin/ipfw -q -f nat 1 delete
/sbin/ipfw -q -f table all flush
# Parameters
ipfw="/sbin/ipfw -q add"
extM_if="vlan1"
extM_ip="1.1.1.2"
extS_if="vlan2"
extS_ip="2.2.2.2"
int_if="eth2"
int_ip="10.0.0.1"
lan_net="10.0.0.0/24"
odmin="10.0.0.111"
# Tables
# Table 1 - non-routes networks
/sbin/ipfw table 1 add 192.168.0.0/16
/sbin/ipfw table 1 add 172.16.0.0/12
/sbin/ipfw table 1 add 10.0.0.0/8
/sbin/ipfw table 1 add 127.0.0.0/8
/sbin/ipfw table 1 add 0.0.0.0/8
/sbin/ipfw table 1 add 169.254.0.0/16
/sbin/ipfw table 1 add 192.0.2.0/24
/sbin/ipfw table 1 add 204.152.64.0/23
/sbin/ipfw table 1 add 224.0.0.0/3
# Choose route table
$ipfw setfib 0 all from any to any via $int_if 
# Allow all traffic on loopback
$ipfw allow all from any to any via lo0
# Deny access to lo0 from out
$ipfw deny log all from any to 127.0.0.0/8
# Deny outcome packets from lo0
$ipfw deny log all from 127.0.0.0/8 to any
# Allow returning 
$ipfw check-state
# Deny IPv6
$ipfw deny log ipv6 from any to any
# Antispoofing
$ipfw deny log all from any to any not antispoof in
# Block any delayed packets (fragments)
$ipfw deny all from any to any frag
#########################################
# Internal interface, outcoming traffic #
#########################################
# Allow all traffic from gateway to lan
$ipfw allow all from any to $lan_net out via $int_if
# Deny and log other
$ipfw deny log all from any to any out via $int_if
########################################
# Internal interface, incoming traffic #
########################################
# Deny all Netbios 
$ipfw deny tcp from any to any 81,137,138,139 in via $int_if
# Allow traffic on internal interface
# DHCP
$ipfw allow udp from any to me 67,68,1515,1516 in via $int_if
# Mail
$ipfw allow tcp from $lan_net to any 25,110,143,465,993,995 in via $int_if
# Time
$ipfw allow tcp from $lan_net to any 37 in via $int_if
$ipfw allow udp from $lan_net to any 123 in via $int_if
# ICQ
$ipfw allow tcp from $lan_net to any 443,5190,5222 in via $int_if
# FTP and some other
$ipfw allow tcp from $lan_net to any 21,22,49152-65535 in via $int_if
# HTTP
$ipfw allow tcp from $lan_net to any 80 in via $int_if
# Output whois
$ipfw allow tcp from $lan_net to any 43 in via $int_if
# DNS
$ipfw allow udp from $lan_net to any 53 in via $int_if
$ipfw allow tcp from $lan_net 53 to $int_ip in via $int_if
$ipfw allow tcp from $lan_net to $int_ip 53 in via $int_if
# Ping
$ipfw allow icmp from $lan_net to any icmptypes 0,3,8,11 in via $int_if
# For admin
$ipfw allow all from $odmin 1025-6000,11111,22222,50000-60000 to any in via $int_if
$ipfw allow all from 10.0.0.2 22 to $int_ip in via $int_if
$ipfw 55100 allow all from any to $int_ip 22 in via $int_if
# Deny and log other
$ipfw deny log all from any to any in via $int_if
#########################################
# External interface, outcoming traffic #
#########################################
# Deny all outcoming traffic to non-route networks
$ipfw deny log all from any to 'table(1)' out via $extM_if
$ipfw deny log all from any to 'table(1)' out via $extS_if
# Deny broadcast ICMP on ext interface
$ipfw deny icmp from any to 255.255.255.255 out via $extM_if
$ipfw deny icmp from any to 255.255.255.255 out via $extS_if
# Deny multicast on ext interface
$ipfw deny all from 224.0.0.0/4 to any out via $extM_if
$ipfw deny all from 224.0.0.0/4 to any out via $extS_if
# Allow me go to internet
$ipfw allow all from $extM_ip to any out via $extM_if setup keep-state 
$ipfw allow all from $extS_ip to any out via $extS_if setup keep-state
# DNS BIND
$ipfw allow udp from $extM_ip to any 53 out via $extM_if keep-state
$ipfw allow udp from $extS_ip to any 53 out via $extS_if keep-state
# Time
$ipfw allow udp from $extM_ip to any 123 out via $extM_if keep-state
$ipfw allow tcp from $extM_ip to any 37 out via $extM_if setup keep-state
# Output whois
$ipfw allow tcp from $extM_ip to any 43 out via $extM_if setup keep-state
# NAT
/sbin/ipfw -q nat 1 config log if $extM_if reset same_ports deny_in unreg_only redirect_port tcp 10.0.0.111:33333 33333 redirect_port udp 10.0.0.111:11111 11111 redirect_port tcp 10.0.0.111:22222 22222 redirect_port udp 10.0.0.111:22222 22222
# NAT outcoming traffic
$ipfw nat 1 ip from any to any out via $extM_if
# Allow traffic on outcoming interface
# Mail
$ipfw allow tcp from any to any 25,110,143,465,993,995 out via $extM_if
# ICQ
$ipfw allow tcp from any to any 443,5190,5222 out via $extM_if
# FTP and some other
$ipfw allow tcp from any to any 21,22,49152-65535 out via $extM_if 
# HTTP
$ipfw allow tcp from any to any 80 out via $extM_if
# Ping
$ipfw allow icmp from any to any icmptypes 0,3,8,11 out via $extM_if
$ipfw allow icmp from any to any icmptypes 0,3,8,11 out via $extS_if
# For admin
$ipfw allow tcp from any 1025-6000 to any out via $extM_if
$ipfw allow all from any 11111,22222,50000-60000 to any out via $extM_if
# Deny and log other
$ipfw deny log all from any to any out via $extM_if
$ipfw deny log all from any to any out via $extS_if
########################################
# External interface, incoming traffic #
########################################
# Deny all incoming traffic from non-route networks
$ipfw deny log all from 'table(1)' to any in via $extM_if
$ipfw deny log all from 'table(1)' to any in via $extS_if
# Deny ident
$ipfw deny tcp from any to any 113 in via $extM_if
$ipfw deny tcp from any to any 113 in via $extS_if
# Deny all Netbios
$ipfw deny tcp from any to any 81,137,138,139 in via $extM_if
$ipfw deny tcp from any to any 81,137,138,139 in via $extS_if
# SSH (also for internal network)
$ipfw allow all from any to me 22 in via $extM_if
$ipfw allow all from any to me 22 in via $extS_if
# NAT incoming traffic
$ipfw nat 1 ip from any to any in via $extM_if
# Allow traffic on outcoming interface
# Mail
$ipfw allow tcp from any 25,110,143,465,993,995 to any in via $extM_if
# ICQ
$ipfw allow tcp from any 443,5190,5222 to any in via $extM_if
# FTP and some other
$ipfw allow tcp from any 21,22,49152-65535 to any in via $extM_if
# HTTP
$ipfw allow tcp from any 80 to any in via $extM_if
# Ping
$ipfw allow icmp from any to any icmptypes 0,3,8,11 in via $extM_if
$ipfw allow icmp from any to any icmptypes 0,3,8,11 in via $extS_if
# For admin
$ipfw allow tcp from any to $odmin 1025-6000 in via $extM_if
$ipfw allow all from any to $odmin 11111,22222,50000-60000 in via $extM_if
# Deny and log other
$ipfw deny log all from any to any in via $extM_if
$ipfw deny log all from any to any in via $extS_if

$ipfw deny log all from any to any

# Parameters
ipfw="/sbin/ipfw -q add"
extM_if="vlan2"
extM_ip="2.2.2.2"
extS_if="vlan1"
extS_ip="1.1.1.1"
int_if="eth2"
int_ip="10.0.0.1"
lan_net="10.0.0.0/24"
odmin="10.0.0.111"
serv="10.0.0.4

##         tofoin.conf        ##
## by LordNicky v0.6 20140719 ##
## Little about the modules and about what function they perform.
## Tester - Testing the availability of the Internet on selected channel.
## Judge - Test results analysis, the decision to switch 
## from one channel to another.
## Logger - Event logging.
## Watchdog - Testing and debugging of the scripts.
## Configuration.
## Amouth of the Internet channels.
CNUMBER=2
## Main Internet channel properties.
## Interface name.
EXT_0_IF=vlan10
## Id number of the routing table.
RTABLE_0=0
## Reserve Internet channel properties.
## Interface name.
EXT_1_IF=vlan20
## Id number of the routing table
RTABLE_1=1
## URL's supposed to be used for diagnostic of the availability
## of the Internet channel. PTARGET_0 should be domain name, and
## PTARGET_1 should be IP address.
## Attention: The resources should be different.
PTARGET_0=ya.ru
PTARGET_1=8.8.8.8
## Count of icmp packets used for testing one resource.
PNUMBER=2
## Period of launching of the module "Tester" (in seconds).
## Strongly not recomended to set a value less than 60.
TESTERPERIOD=240
## Period of launching of the module "Judge" (in seconds).
## Strongly not recomended to set a value less than TESTERPERIOD.
## Usually enough TESTERPERIOD + 60.
JUDGEPERIOD=300
## Launching sensitivity for the modules Tester and Judge.
## Usually enough 60.
SENSITIVITY=60
## The maximum operating time for the module Tester.
TESTERMAXDELAY=40
## The maximum operating time for the module Judge.
JUDGEMAXDELAY=30
## The maximum operating time for the module Logger.
LOGGERMAXDELAY=20
## Amount of tests that successfully passed before returning 
## to the main channel. Thereby, time elapsed since the restore
## the work main channel is approximately (WNUMBER+1)*JUDGEPERIOD
## seconds.
WNUMBER=3
## The frequency of writing error message into the log file.
## The main idea is the following. At first time the message 
## is written completely. After LOGFREQ1 repetitions logger 
## writes the only message about LOGFREQ1 the same messages.
## Later in each LOGFREQ2 repetitions logger writes the only 
## message about LOGFREQ2 the same messages. This algorithm
## works only if the same messages are following after each other.
LOGFREQ1=5
LOGFREQ2=20
## File paths.
## Paths for configuration script files IPFW.
## Default file. (It is written in the rc.conf)
FIRESETDEF=/etc/firewall.sh
## Settings for main Internet channel.
FIRESET_0=/etc/rules.firewall0
## Settings for reserve Internet channel.
FIRESET_1=/etc/rules.firewall1
## Paths for all ToFoIn files.
## Daemon.
DAEMON=/path/to/file/tofoin_daemon.sh
## Tester.
TESTER=/path/to/file/tofoin_tester.sh
## Judge.
JUDGE=/path/to/file/tofoin_judge.sh
## Logger.
LOGGER=/path/to/file/tofoin_logger.sh
## Watchdog.
WATCHDOG=/path/to/file/tofoin_watchdog.sh
## Log file. It is recommended to locate it into the /var/log.
LOGFILE=/path/to/file/tofoin.log
## The directory supposed for test results. It is recomended
## to locate it into the /tmp.
TESTER_RESULT=/path/to/directory
## Auxiliary module file Judge. It is recommended to locate
## it into the /tmp.
JUDGEMETER=/path/to/file/judgemeter
## Auxiliary module file Logger. It is recommended to locate
## it into the /tmp.
LOGTMP=/path/to/file/logger.tmp
LOGMETER=/path/to/file/logmeter
## PID files for all executable modules. It is recommended
## to locate it into /var/run.
DAEMON_PID=/path/to/file/tofoin_daemon.pid
TESTER_PID=/path/to/directory
JUDGE_PID=/path/to/file/tofoin_judge.pid
LOGGER_PID=/path/to/file/tofoin_logger.pid
WATCHDOG_PID=/path/to/file/tofoin_watchdog.pid

#!/usr/local/bin/bash
# by LordNicky v0.5 20140717
. /root/ToFoIn/tofoin.conf

test_time=`date +%s`;
judge_time=`date +%s`;

echo $$ > $DAEMON_PID;
$LOGGER "DAEMON: start successfully with pid $$" &
tester_0="$TESTER $RTABLE_0 10 0";
tester_1="$TESTER $RTABLE_1 10 1";
$tester_0 & $tester_1 &

while true
do
  current_time=`date +%s`;
  if [ "`expr $current_time - $test_time`" -ge "$TESTERPERIOD" ]
  then $tester_0 & $tester_1 & test_time=`date +%s`;
  else :;
  fi
  if [ "`expr $current_time - $judge_time`" -ge "$JUDGEPERIOD" ]
  then $JUDGE & judge_time=`date +%s`;
  else :;
  fi
  sleep $SENSITIVITY;
done  	

#!/usr/local/bin/bash
# by LordNicky v0.7 20140717
. /root/ToFoIn/tofoin.conf

exit_function () {
rm $tester_pid; 
exit $exit_code;
}

tester_pid=$TESTER_PID/tofoin_test_$3.pid;
if [ -e $tester_pid ];
then $WATCHDOG "tofoin_test" "$tester_pid" "$3" & exit 0;
else echo `date +%s` $$ > $tester_pid;
     if [ "$2" -eq 10 ];
     then if setfib $1 ping -c $PNUMBER $PTARGET_0 > /dev/null;
          then echo `date +%s` "0 0" > $TESTER_RESULT/result_$3;
          exit_code=0; exit_function;
          else if setfib $1 ping -c $PNUMBER $PTARGET_1 > /dev/null;
               then echo `date +%s` "0 1" > $TESTER_RESULT/result_$3;
	       exit_code=0; exit_function;
	       else echo `date +%s` "1 1" > $TESTER_RESULT/result_$3;
	       exit_code=0; exit_function;
	       fi
          fi
     elif [ "$2" -eq 0 ];
     then setfib $1 ping -c $PNUMBER $PTARGET_0;
     exit_code=0; exit_function;
     elif [ "$2" -eq 1 ];
     then setfib $1 ping -c $PNUMBER $PTARGET_1;
     exit_code=0; exit_function;
     else setfib $1 ping -c $PNUMBER $2;
     exit_code=1; exit_function;
     fi
fi     

#!/usr/local/bin/bash
# by LordNicky v0.7 20140717
. /root/ToFoIn/tofoin.conf

exit_function () {
rm $JUDGE_PID; 
exit $exit_code;
}

decision_function () {
if [ "$actualchan" -eq "$prefchan" ];
then if [ "$actualchan" -eq 0 ];
     then $LOGGER "JUDGE: No problems detected" & 
     exit_code=0; exit_function;
     elif [ "$actualchan" -eq 1 ];
     then echo -e "0" > $JUDGEMETER; 
     $LOGGER "JUDGE: No problems detected at channel $actualchan" & 
     exit_code=0; exit_function;
     else $LOGGER "JUDGE(decision): Invalid actualchan = $actualchan" & 
     exit_code=1; exit_function;
     fi
else if [ "$prefchan" -eq 1 ];
     then switch_function; exit_code=0; exit_function;
     elif [ "$prefchan" -eq 0 ];
     then if [ "$actualstate" -eq 0 ]
          then meter=`cat $JUDGEMETER`;
               if [ "$meter" -eq "$WNUMBER" ];
	       then switch_function; exit_code=0; exit_function;
	       elif [ "$meter" -lt "$WNUMBER" ];
	       then expr $meter + 1 > $JUDGEMETER; 
	       exit_code=0; exit_function;
	       else echo -e "0" > $JUDGEMETER; exit_code=0; exit_function;
	       fi
	  elif [ "$actualstate" -eq 1 ]
	  then $LOGGER "JUDGE: Emergency switch to $prefchan"; 
	  switch_function; exit_code=0; exit_function;
	  else $LOGGER "JUDGE(decision): Invalid actualstate = $actualstate" & exit_code=1; exit_function;
	  fi     	
     else $LOGGER "JUDGE(decision): Invalid prefchan = $prefchan" & 
     exit_code=1; exit_function;
     fi	   	 
fi
} 

switch_function () {
echo -e "0" > $JUDGEMETER;
if [ "$prefchan" -eq 0 ];
then /etc/rc.d/named stop; 
cp $FIRESET_0 $FIRESETDEF; 
/etc/rc.d/ipfw restart; 
setfib $RTABLE_0 /etc/rc.d/named start; 
$LOGGER "JUDGE: Now switching on channel $RTABLE_0" & 
exit_code=0; exit_function;
elif [ "$prefchan" -eq 1 ]
then /etc/rc.d/named stop;
cp $FIRESET_1 $FIRESETDEF;
/etc/rc.d/ipfw restart;
setfib $RTABLE_1 /etc/rc.d/named start;
$LOGGER "JUDGE: Now switching on channel $RTABLE_1" & 
exit_code=0; exit_function;
else $LOGGER "JUDGE(switch): Invalid prefchan = $prefchan" & 
exit_code=1; exit_function;
fi
}	

createarea_function () {
for ((a=0; a < CNUMBER ; a++))
do
  current_time=`date +%s`
  timearea[$a]=`cut -c 1-10 $TESTER_RESULT/result_$a`;
  if [ "`expr $current_time - ${timearea[$a]}`" -ge 0 ];
  then if [ "`expr $current_time - ${timearea[$a]}`" -lt "`expr $TESTERPERIOD + 120`" ];
       then :;
       else $LOGGER "JUDGE: MAX period" & 
       $WATCHDOG & 
       exit_code=1; exit_function;
       fi
  else $LOGGER "JUDGE: testmodule $a in future" & 
  $WATCHDOG & 
  exit_code=1; exit_function;
  fi	
  statearea[$a]=`cut -c 12 $TESTER_RESULT/result_$a`;
  if [ "$actualchan" -eq "$a" ]
  then actualstate=${statearea[$a]};
  else :;   
  fi	  
done
}

findarea_function () {
for ((a=0; a < CNUMBER ; a++))
do
  if [ "${statearea[$a]}" -eq 0 ]
  then prefchan=$a; decision_function; 
  exit_code=0; exit_function; 
  else if [ "${statearea[$a]}" -eq 1 ]
       then continue 
       else $LOGGER "JUDGE: Invalid channel state" & 
       exit_code=1; exit_function;
       fi  
  fi
done
}

if [ -e $JUDGE_PID ]
then $WATCHDOG "tofoin_judge" "$JUDGE_PID" & exit 0;
else echo `date +%s` $$ > $JUDGE_PID;	
     if ipfw list | grep nat | egrep -q $EXT_0_IF;
     then actualchan=0;
     elif ipfw list | grep nat | egrep -q $EXT_1_IF;
     then actualchan=1;
     else $LOGGER "JUDGE: NAT error" & 
     prefchan=0; switch_function; 
     exit_code=1; exit_function;
     fi
     createarea_function;
     findarea_function;
     $LOGGER "JUDGE: All channels down" & 
     exit_code=1; exit_function;
fi     

#!/usr/local/bin/bash
# by LordNicky v0.5 20140713
. /root/ToFoIn/tofoin.conf

exit_function () {
rm $LOGGER_PID; 
exit $exit_code;
}

main_function () {
if [[ `tail -n 1 $LOGFILE | grep -o "$1" | grep -o "JUDGE: No problems detected"` = "JUDGE: No problems detected" ]];
then exit_code=0; exit_function;
else if [[ `cat $LOGTMP` = $1 ]];
     then meter=`cat $LOGMETER`;
          if [ "$meter" -ge "$LOGFREQ2" ];
	  then echo -e "0" > $LOGMETER; 
	  echo -e "`date -j +%Y%m%d%H%M` last message repeat $LOGFREQ2 times" >> $LOGFILE; 
	  exit_code=0; exit_function;
	  elif [ "$meter" -ge "$LOGFREQ1" ];
	  then if [[ `tail -n 1 $LOGFILE | grep -o "last message repeat $LOGFREQ1 times"` = "last message repeat $LOGFREQ1 times" ]];
	       then expr $meter + 1 > $LOGMETER; 
	       exit_code=0; exit_function;
	       elif [[ `tail -n 1 $LOGFILE | grep -o "last message repeat $LOGFREQ2 times"` = "last message repeat $LOGFREQ2 times" ]];
               then expr $meter + 1 > $LOGMETER; 
	       exit_code=0; exit_function;
	       else echo -e "`date -j +%Y%m%d%H%M` last message repeat $LOGFREQ1 times" >> $LOGFILE; 
	       exit_code=0; exit_function;
	       fi
	  elif [ "$meter" -ge 0 ];
	  then expr $meter + 1 > $LOGMETER; 
	  exit_code=0; exit_function;
	  else echo -e "0" > $LOGMETER; 
	  echo -e "`date -j +%Y%m%d%H%M` LOGGER: logmeter index error, write 0" >> $LOGFILE; 
	  exit_code=1; exit_function;
	  fi
     else if [ `cat $LOGMETER` -eq 0 ];
	  then echo -e "$1" > $LOGTMP; 
	  echo -e "`date -j +%Y%m%d%H%M` $1" >> $LOGFILE; 
	  exit_code=0; exit_function;
	  else echo -e "0" > $LOGMETER; 
	  echo -e "$1" > $LOGTMP; 
	  echo -e "`date -j +%Y%m%d%H%M` $1 ; LOGMETER now zero" >> $LOGFILE;
	  exit_code=0; exit_function;
	  fi
     fi
fi
}

if [ -e $LOGGER_PID ];
then sleep $((RANDOM%5+1)); 
     if [ -e $LOGGER_PID ];
     then $WATCHDOG "tofoin_logger" "$LOGGER_PID" & exit 0;
     else echo `date +%s` $$ > $LOGGER_PID;
     main_function "$1";
     fi
else echo `date +%s` $$ > $LOGGER_PID;
main_function "$1";
fi

#!/usr/local/bin/bash
# by LordNicky v0.5 20140713
. /root/ToFoIn/tofoin.conf

exit_function () {
rm $WATCHDOG_PID; 
exit $exit_code;
}

kill_function () {
if [[ "`ps -o command -p $proc_pid | grep -o "$proc_name"`" = "$proc_name" ]];
then $LOGGER "WATCHDOG: Other $proc_s_name working during $diff, kill him" &
kill $proc_pid; 
else $LOGGER "WATCHDOG: None or other process on $proc_s_name pid, cleaning pid file" &
fi
if [[ "$proc_name" = "tofoin_watchdog" ]];
then main_function;
else rm $proc_pid_file;
fi
}	

main_function () {
echo `date +%s` $$ > $WATCHDOG_PID;
proc_name=${one:-all};
return_wait=10
if [[ "$proc_name" = "all" ]];
b=0; c=0
then for ((a=0; a < CNUMBER ; a++))
     do 
       current_time=`date +%s`;
       tester_result=$TESTER_RESULT/result_$a;
       tester_time=`cut -c 1-10 $tester_result`;
       diff=`expr $current_time - $tester_time`;
       if [ "$diff" -ge 0 ]
       then if [ "$diff" -lt "`expr $TESTERPERIOD + 120`" ];
            then :;
	    else proc_name=tofoin_daemon; proc_pid=`cat $DAEMON_PID`;
		 if  [[ "`ps -o command -p $proc_pid | grep -o "$proc_name"`" = "$proc_name" ]];
                 then $LOGGER "WATCHDOG: Restart daemon" & 
                 kill $proc_pid; $DAEMON & 
                 else $LOGGER "WATCHDOG: None daemon process, start" &
		 $DAEMON & 
                 fi
		 exit_code=0; exit_function;
            fi
       else $LOGGER "WATCHDOG: Check date" &
       fi
     done
elif [[ "$proc_name" = "tofoin_test" ]];
then proc_pid_file=$two; cnumber=$three; 
test_function; return_val=$?;
     if [[ "$return_val" = "$return_wait" ]];
     then sleep $TESTERMAXDELAY; test_function "nowait";
     else :;
     fi
elif [[ "$proc_name" = "tofoin_judge" ]];
then proc_pid_file=$JUDGE_PID;
judge_function; return_val=$?;
     if [[ "$return_val" = "$return_wait" ]];
     then sleep $JUDGEMAXDELAY; judge_function "nowait";
     else :;
     fi
elif [[ "$proc_name" = "tofoin_logger" ]];
then proc_pid_file=$LOGGER_PID;
logger_function; return_val=$?;
     if [[ "$return_val" = "$return_wait" ]];
     then sleep $LOGGERMAXDELAY; logger_function "nowait";
     else :;
     fi
else $LOGGER "WATCHDOG: Incorrect process name";
fi
exit_code=0; exit_function;
}	

test_function () {
if [ -e $proc_pid_file ];
then proc_pid=`cut -c 12-18 $proc_pid_file`;
     proc_s_name="tester $cnumber";
     start_time=`cut -c 1-10 $proc_pid_file`;
     current_time=`date +%s`;
     diff=`expr $current_time - $start_time`;
     if [ "$diff" -ge 0 ];
     then if [ "$diff" -lt "$TESTERMAXDELAY" ];
          then if [[ "$1" = "nowait" ]];
	       then if [ "$proc_pid" = "$proc_temp_pid" ];
	            then kill_function; return 0;
		    else $LOGGER "WATCHDOG: Pid of $proc_s_name was changed, exit" &
		    fi
	       else $LOGGER "WATCHDOG: $proc_s_name now working, try wait" &
	       proc_temp_pid=$proc_pid;
	       return $return_wait;
	       fi
	  else kill_function; return 0;
	  fi
     else $LOGGER "WATCHDOG: Time error in $proc_s_name = $diff" &
     kill_function; return 0;
     fi 
else return 0;
fi
}	

judge_function () {
if [ -e $proc_pid_file ];
then proc_pid=`cut -c 12-18 $proc_pid_file`;
     proc_s_name="judge";
     start_time=`cut -c 1-10 $proc_pid_file`;
     current_time=`date +%s`;
     diff=`expr $current_time - $start_time`;
     if [ "$diff" -ge 0 ];
     then if [ "$diff" -lt "$JUDGEMAXDELAY" ];
          then if [[ "$1" = "nowait" ]];
	       then if [ "$proc_pid" = "$proc_temp_pid" ];
	            then kill_function; return 0;
		    else $LOGGER "WATCHDOG: Pid of $proc_s_name was changed, exit" &
		    fi
	       else $LOGGER "WATCHDOG: $proc_s_name now working, try wait" &
	       proc_temp_pid=$proc_pid;
	       return $return_wait;
	       fi
	  else kill_function; return 0;
	  fi
     else $LOGGER "WATCHDOG: Time error in $proc_s_name = $diff" &
     kill_function; return 0;
     fi 
else return 0;
fi
}	

logger_function () {
if [ -e $proc_pid_file ];
then proc_pid=`cut -c 12-18 $proc_pid_file`;
     proc_s_name="logger";
     start_time=`cut -c 1-10 $proc_pid_file`;
     current_time=`date +%s`;
     diff=`expr $current_time - $start_time`;
     if [ "$diff" -ge 0 ];
     then if [ "$diff" -lt "$LOGGERMAXDELAY" ];
          then if [[ "$1" = "nowait" ]];
	       then if [ "$proc_pid" = "$proc_temp_pid" ];
	            then kill_function; return 0;
		    else echo -e "`date -j +%Y%m%d%H%M` WATCHDOG: Pid of $proc_s_name was changed, exit" >> $LOGFILE;
		    fi
	       else echo -e "`date -j +%Y%m%d%H%M` WATCHDOG: $proc_s_name now working, try wait" >> $LOGFILE;
	       proc_temp_pid=$proc_pid;
	       return $return_wait;
	       fi
	  else kill_function; return 0;
	  fi
     else echo -e "`date -j +%Y%m%d%H%M` WATCHDOG: Time error in $proc_s_name = $diff" >> $LOGFILE;
     kill_function; return 0;
     fi 
else return 0;
fi
}	

one=$1;
two=$2;
three=$3;

if [ -e $WATCHDOG_PID ];
then proc_pid=`cut -c 12-18 $WATCHDOG_PID`;
     proc_name="tofoin_watchdog";
     proc_s_name="watchdog";
     start_time=`cut -c 1-10 $WATCHDOG_PID`;
     current_time=`date +%s`;
     diff=`expr $current_time - $start_time`;
     if [ "$diff" -ge 0 ];
     then if [ "$diff" -lt "`expr $TESTERMAXDELAY + $JUDGEMAXDELAY + $LOGGERMAXDELAY + 30`" ];
          then $LOGGER "WATCHDOG: Other $proc_s_name already working, exit" & exit 0;
	  else kill_function;
	  fi
     else $LOGGER "WATCHDOG: Time error in $proc_s_name = $diff" &
     kill_function; 
     fi 
else main_function;
fi