Автоматизация IP-сети. Часть3 – Мониторинг TCP аномалий

======================================================================================================
| IO Statistics                                                                                      |
|                                                                                                    |
| Duration: 5. 40977 secs                                                                            |
| Interval: 5 secs                                                                                   |
|                                                                                                    |
| Col 1: (ip.addr==1.1.1.1) && tcp                                                            |
|     2: COUNT(tcp.analysis.retransmission)(ip.addr==1.1.1.1) && tcp.analysis.retransmission  |
|     3: COUNT(tcp.analysis.duplicate_ack)(ip.addr==1.1.1.1) && tcp.analysis.duplicate_ack    |
|     4: COUNT(tcp.analysis.lost_segment)(ip.addr==1.1.1.1) && tcp.analysis.lost_segment      |
|     5: COUNT(tcp.analysis.fast_retransmission)(ip.addr==1.1.1.1) &&                         |
|        tcp.analysis.fast_retransmission                                                            |
|     6: COUNT(tcp.analysis.lost_segment)(ip.addr==1.1.1.1) && tcp.analysis.ack_lost_segment  |
|----------------------------------------------------------------------------------------------------|
|                     |1                |2      |3      |4      |5      |6      |                    |
| Date and time       | Frames |  Bytes | COUNT | COUNT | COUNT | COUNT | COUNT |                    |
|-------------------------------------------------------------------------------|                    |
| 2017-07-10 15:00:45 |    507 | 481496 |     1 |     0 |     2 |     0 |     0 |                    |
| 2017-07-10 15:00:50 |      0 |      0 |     0 |     0 |     0 |     0 |     0 |                    |
======================================================================================================

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import datetime
import re
import os
import subprocess
import argparse
import time
import signal

parser = argparse.ArgumentParser()
parser.add_argument("-h_page", "--hostname_page", dest = "hostname_page")
args = parser.parse_args()

curent_time=str(datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S_"))
pid=os.getpid()


##########start Table parser###############
def parser_tshark_output(open_time, parser_file):
        table_data = {'date':[],'1 frame':[],'2 frame':[],'retr':[],'dup_ack':[],'lost_seg':[],'fast_retr':[],'ack_lost_seg':[]}
        lines = open(parser_file, 'r').readlines()
        lookup = 'tcp.analysis.ack_lost_segment '
        number = 0
        for num in lines:
                if lookup in num:
                        number+=lines.index(num)
                        try:
                                del (lines[-1])
                        except:
                                pass
        L=open("/usr/TEST/TMP_FILES/test_tshark_temp_"+str(os.getpid())+".txt", 'w')
        L.writelines(lines)
        L.close()
        with open("/usr/TEST/TMP_FILES/test_tshark_temp_"+str(os.getpid())+".txt", 'r') as table:
                if number != 0:
                        for _ in range(int(number+5)):
                                next(table)  #skip header
                        for row in table:
                                row=row.strip('n').split('|')
                                values = [r.strip() for r in row if r != '']
                                table_data['date'].append(values[0])
                                table_data['1 frame'].append(int(values[1]))
                                table_data['2 frame'].append(int(values[2]))
                                table_data['retr'].append(int(values[3]))
                                table_data['dup_ack'].append(int(values[4]))
                                table_data['lost_seg'].append(int(values[5]))
                                table_data['fast_retr'].append(int(values[6]))
                                table_data['ack_lost_seg'].append(int(values[7]))
                        else:
                                        pass
        if number !=0:
                frames=sum(table_data['2 frame'])
                print ('start-frames_' + str.format("{0:.2f}", frames)+'_end-frames')
                tcp_errors=sum(table_data['retr'])+sum(table_data['dup_ack'])++sum(table_data['fast_retr'])+sum(table_data['ack_lost_seg'])
                print ('start-tcp_errors_' + str(tcp_errors)+'_end-tcp_errors')
                if open_time != 'open_error' and frames != 0:
                        k=tcp_errors
                else:
                        k= 'no data'
        else:
                k= 'no data'
        os.remove("/usr/TEST/TMP_FILES/test_tshark_temp_"+str(os.getpid())+".txt")
        return (k)

###########end table parser###############
###########start tshark part 1###########
resault_temp=subprocess.Popen(['nslookup '+str(args.hostname_page)], bufsize=0, shell=True, stdout = subprocess.PIPE, stderr=subprocess.PIPE)
data=resault_temp.communicate()
ip_adr_temp=re.findall(r'(d{1,3}.d{1,3}.d{1,3}.d{1,3})', str(re.findall(r'Address: (d{1,3}.d{1,3}.d{1,3}.d{1,3})', str(data))))
if ip_adr_temp != []:
        pass
else:
        ip_adr_temp = ['1.1.1.1']
ip_adr=''
for z in ip_adr_temp:
        if len(ip_adr_temp)-ip_adr_temp.index(z)==1:
                ip_adr+='ip.addr=='+str(z)+')'
        else:
                ip_adr+='ip.addr=='+str(z)+' or '

ftcp=open('/usr/TEST/TMP_FILES/1tshark_temp'+curent_time+str(pid)+'.txt', 'w')
tcp=subprocess.Popen(['timeout 180 tshark -i bce0 -t ad -qz io,stat,5,"('+str(ip_adr)+' && tcp","COUNT(tcp.analysis.retransmission)('+str(ip_adr)+' && tcp.analysis.retransmission","COUNT(tcp.analysis.duplicate_ack)('
+str(ip_adr)+' && tcp.analysis.duplicate_ack","COUNT(tcp.analysis.lost_segment)('+str(ip_adr)+' && tcp.analysis.lost_segment","COUNT(tcp.analysis.fast_retransmission)('+str(ip_adr)+
' && tcp.analysis.fast_retransmission","COUNT(tcp.analysis.lost_segment)('+str(ip_adr)+' && tcp.analysis.ack_lost_segment"']
, bufsize=0, shell=True, stdout=(ftcp))#stdout = subprocess.PIPE, stderr=subprocess.PIPE)
time.sleep(2)

############start wget ################


fweb=open('/usr/TEST/TMP_FILES/web_temp'+curent_time+str(pid)+'.txt', 'w')
web=subprocess.call(["timeout 120 wget -E -H -p -Q300K --user-agent=Mozilla --no-cache --no-cookies --delete-after --timeout=15 --tries=2 "+args.hostname_page+" 2>&1 | grep '([0-9.]+ [KM]B/s)'"], bufsize=0, shell=True, stdout=(fweb))
fweb.close()
fweb=open('/usr/TEST/TMP_FILES/web_temp'+curent_time+str(pid)+'.txt', 'r')
data=fweb.read()
os.remove('/usr/TEST/TMP_FILES/web_temp'+curent_time+str(pid)+'.txt')
speed_temp=re.findall(r's ((.*?)B/s', str(data))#[KM]B/s', str(data)))
speed_temp_si=re.findall(r's ((.*?) [KM]B/s', str(data))
try:
        if re.findall(r'M', str(speed_temp))==[] and re.findall(r'K', str(speed_temp))==[]:
                speed_="{0:.3f}".format(float(speed_temp_si[0])*0.001*8)
        elif re.findall(r'M', str(speed_temp))!=[]:
                speed_="{0:.3f}".format(float(speed_temp_si[0])*1000*8)
        elif re.findall(r'K', str(speed_temp))!=[]:
                speed_="{0:.3f}".format(float(speed_temp_si[0])*1*8)
except:
        speed_='no_data'
##############stop wget##############
##############start tshark part2#######

os.kill(tcp.pid, signal.SIGINT)
ftcp.close()
time.sleep(0.3)
tcp_error=parser_tshark_output('1', '/usr/TEST/TMP_FILES/1tshark_temp'+curent_time+str(pid)+'.txt')
os.remove('/usr/TEST/TMP_FILES/1tshark_temp'+curent_time+str(pid)+'.txt')
#########resault to DB###########
print ('web_speed_test:'+str(speed_)+' tcp_error:'+str(tcp_error))

1. Добавляем дополнительную output Fields в Data Input Methods:

2. Добавляем в Data Temlate дополнительную область tcp_error

3. Копируем Graph template и добавляем в него дополнительную информацию, при заведении новых графиков изначально нужно использовать этот Template:

4. Заводим график как показано в предыдущей статье с использование Template выше, график дублируем:

5. в итоге должно получится следующее:

6. Меняем данные в первом графике на правельные. Сначала меняем Template жмем save, затем проверяем что Data Source верный, при необходимости выбираем правильный:

7. Меняем данные для второго графика: