Netflow version 5 collector in python

I am going to provide brief code snippet of netflow version 5 collector in python.
In netflow version 5, the template of data is fixed. It simply means that you know what will come at what place. Netflow version 5 packet structure is as following:-

Packet header Structure

Bytes	Content label	Brief Description
0-1	version	NetFlow export format version number
2-3	count	Number of flows exported in this packet (1-30)
4-7	sys_uptime	Current time in milliseconds since the export device booted
8-11	unix_secs	Current count of seconds since 0000 UTC 1970
12-15	unix_nsecs	Residual nanoseconds since 0000 UTC 1970
16-19	flow_sequence	Sequence counter of total flows seen
20	engine_type	Type of flow-switching engine
21	engine_id	Slot number of the flow-switching engine
22-23	sampling_interval	First two bits hold the sampling mode; remaining 14 bits hold value of sampling interval

Packet record format

Bytes	Content Label	Brief Description
0-3	srcaddr	Source IP address
4-7	dstaddr	Destination IP address
8-11	nexthop	IP address of next hop router
12-13	input	SNMP index of input interface
14-15	output	SNMP index of output interface
16-19	dPkts	Packets in the flow
20-23	dOctets	Total number of Layer 3 bytes in the packets of the flow
24-27	first	SysUptime at start of flow
28-31	last	SysUptime at the time the last packet of the flow was received
32-33	srcport	TCP/UDP source port number or equivalent
34-35	dstport	TCP/UDP destination port number or equivalent
36	pad1	Unused (zero) bytes
37	tcp_flags	Cumulative OR of TCP flags
38	prot	IP protocol type (for example, TCP = 6; UDP = 17)
39	tos	IP type of service (ToS)
40-41	src_as	Autonomous system number of the source, either origin or peer
42-43	dst_as	Autonomous system number of the destination, either origin or peer
44	src_mask	Source address prefix mask bits
45	dst_mask	Destination address prefix mask bits
46-47	pad2	Unused (zero) bytes

A network device like router, switch etc can send netflow data to a netflow collector. Netflow is proprietary protocol of Cisco but many other devices also support it. So if you have a cisco router and you want to write your own netflow collector, you can do in following ways :-

1. Using standard python library
 
import socket, struct
import threading
from socket import inet_ntoa

SIZE_OF_HEADER = 24
SIZE_OF_RECORD = 48

sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind(('0.0.0.0', 9999))

def processPacket(buf,addr):
        (version, count) = struct.unpack('!HH',buf[0:4])
        if version != 5:
                print "Not NetFlow v5!"
                #continue

        # It's pretty unlikely you'll ever see more then 1000 records in a 1500 byte  UDP packet
        if count <= 0 or count >= 1000:
                print "Invalid count %s" % count
                #continue

        uptime = socket.ntohl(struct.unpack('I',buf[4:8])[0])
        epochseconds = socket.ntohl(struct.unpack('I',buf[8:12])[0])

        for i in range(0, count):
                try:
                        base = SIZE_OF_HEADER+(i*SIZE_OF_RECORD)

                        data = struct.unpack('!IIIIHH',buf[base+16:base+36])

                        nfdata = {}
                        nfdata['saddr'] = inet_ntoa(buf[base+0:base+4])
                        nfdata['daddr'] = inet_ntoa(buf[base+4:base+8])
                        nfdata['pcount'] = data[0]
                        nfdata['bcount'] = data[1]
                        nfdata['stime'] = data[2]
                        nfdata['etime'] = data[3]
                        nfdata['sport'] = data[4]
                        nfdata['dport'] = data[5]
                        nfdata['protocol'] = ord(buf[base+38])
                except:
                        continue

        # Do something with the netflow record..
        print "%s:%s -> %s:%s" % (nfdata['saddr'],nfdata['sport'],nfdata['daddr'],nfdata['dport'])


while True:
      buf, addr = sock.recvfrom(1500)
      t = threading.Thread(target=processPacket, args=(buf,addr))
      t.start()
               


2. Using twisted library

     
from twisted.internet.protocol import DatagramProtocol
from twisted.internet import reactor
import socket
from socket import inet_ntoa
import struct
SIZE_OF_HEADER = 24
SIZE_OF_RECORD = 48


class CaptureNetflowPacket(DatagramProtocol):

    def datagramReceived(self, data, addr):
        #print("received %r from %s" % (data, addr))
        #self.transport.write(data, addr)
         (version, count) = struct.unpack('!HH',data[0:4])
         print "Version %s, count %s "%(version,count)
         if version != 5:
            print "Not NetFlow v5!"
         uptime = socket.ntohl(struct.unpack('I',data[4:8])[0])
         epochseconds = socket.ntohl(struct.unpack('I',data[8:12])[0])
         print "Uptime %s , epochseconds %s "% (uptime, epochseconds)
         for i in range(0, count):
                #try:
                base = SIZE_OF_HEADER+(i*SIZE_OF_RECORD)

                fdata = struct.unpack('!IIIIHH',data[base+16:base+36])

                nfdata = {}
                nfdata['saddr'] = inet_ntoa(data[base+0:base+4])
                nfdata['daddr'] = inet_ntoa(data[base+4:base+8])
                nfdata['pcount'] = fdata[0]
                nfdata['bcount'] = fdata[1]
                nfdata['stime'] = fdata[2]
                nfdata['etime'] = fdata[3]
                nfdata['sport'] = fdata[4]
                nfdata['dport'] = fdata[5]
                nfdata['protocol'] = ord(data[base+38])
                #except:
                        #continue
                        #print "Exception occured"

                # Do something with the netflow record..
                print "%s:%s -> %s:%s" % (nfdata['saddr'],nfdata['sport'],nfdata['daddr'],nfdata['dport'])
                print "%s : %s : %s " %(nfdata['pcount'],nfdata['bcount'],nfdata['protocol'])




reactor.listenUDP(9999, CaptureNetflowPacket())
reactor.run()
    

Please note that collector is listening on port 9999 over UDP socket.
This code simply prints packet data on console but one can extend to save the packet data in database.