#! /usr/bin/env python
###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


##############################################################################
## 
##
## Extract probe activity and resource usage from Nprobe logs
## 
## 
############################################################################ 

import string
import os
import sys
import nprobe
from sys import argv
import getopt
from signal import *
import socket

from traceback import print_stack

import np_file_util
import np_http_util
from  np_longutil import *

from np_plot import *

############################################################################

def usage(scriptname):
    print "usage: %s [-h(elp)] <rep-file-list>" % (scriptname)

    sys.exit(1)

#############################################################################

def handle_sigint(n, f):
    print 'SIGINT'
    print_stack()
    sys.exit(0)

#############################################################################

def places(f):
    
    places = 1
    while modf(f)[0]:
        places += 1
        f *= 10
    return places

############################################################################	
	

scriptname = os.path.basename(argv[0])
to_read = None

try:
    optlist, args = getopt.getopt(sys.argv[1:], 'hr:')

except getopt.error, s:
    print '%s: %s' % (scriptname, s)
    usage(scriptname)
    sys.exit(1)

optstr = ''

for opt in optlist:
    if opt[0] == "-h":
	usage(scriptname)
    if opt[0] == "-r":
	to_read = int(opt[1])
        

signal(SIGINT, handle_sigint)

REC_WIRE_PERIOD_REPORT = nprobe.REC_WIRE_PERIOD_REPORT
REC_PERIOD_REPORT = nprobe.REC_PERIOD_REPORT
REC_OTHER = nprobe.REC_OTHER_ALL
REC_PROCSTAT = nprobe.REC_OTHER_PROCSTAT
REC_WRAPPER = nprobe.REC_OTHER_WRAPPER
WRAPPER_TIME_CALLS = nprobe.WRAPPER_TIME_CALLS
WRITE_BLKSZ = nprobe.WRITE_BLKSZ

openfilelist, counters, basepath = np_file_util.get_files(args)

nread = 0

first_ts = 0 # time at start

last_uts = 0 # time of last rusage record
last_utm = 0 # amount of last user rusage
last_stm = 0 # amount of last system rusage

## last_majflt = 0
## last_nswap = 0
## last_blkio = 0
## last_cswitches = 0


last_times_ip_uusage = 0 # amount of last IP call rusage
last_times_ip_susage = 0
last_times_tcp_uusage = 0 # amount of last TCP call rusage
last_times_tcp_susage = 0

last_rts = 0 # time of last period record
last_w_rts = 0 # time of last pkt driven period record
last_times_ts = 0 # time of last call rusage record

last_tous_in = 0
last_tous_out = 0
last_frus_in = 0
last_frus_out = 0

lastrtm = 0

# where to collect the stats
urusage = []
srusage = []
trusage = []
## maxrss = []
## majflt = []
## nswap = []
## blkio = []
## cswitches = []

## times_ip_uusage = []
## times_ip_susage = []
## times_tcp_uusage = []
## times_tcp_susage = []
## times_udp_uusage = []
## times_udp_susage = []
## times_tcp_inseq_uusage = []
## times_tcp_inseq_susage = []
## times_tcp_catchup_uusage = []
## times_tcp_catchup_susage = []
## times_tcp_serv_uusage = []
## times_tcp_serv_susage = []
## times_http_uusage = []
## times_http_susage = []
## times_html_uusage = []
## times_html_susage = []

wire_bw = []
tcp_bw = []
udp_bw = []
http_bw = []
html_parse = []
wire_pkts = []
tcp_pkts = []
udp_pkts = []
http_pkts = []
rwritten = []
dwritten = []
twritten = []
rdumped = []
ddumped = []
tdumped = []
max_fetch_int = []
bheld = []
fails = []
IP_ulen = []

tcp_to = []
udp_to = []
tcp_seq_to = []

tous_in = []
tous_out = []
frus_in = []
frus_out = []

# do but packet arrival driven
w_wire_bw = []
w_tcp_bw = []
w_udp_bw = []
w_http_bw = []
w_html_parse = []
w_wire_pkts = []
w_tcp_pkts = []
w_udp_pkts = []
w_http_pkts = []

# stats from /proc/self
proc_utm = []
proc_stm = []
proc_ttm = []
proc_majflt = []
proc_nswap = []

# stats from /proc/stats
proc_itot = []
proc_csw = []


# running maximum for scaling
## max_maxrss = 0
## max_majflt = 0
## max_nswap = 0
## max_blkio = 0
## max_cswitches = 0

## max_times_ip_uusage = 0
## max_times_ip_susage = 0
## max_times_tcp_uusage = 0
## max_times_tcp_susage = 0
## max_times_udp_uusage = 0
## max_times_udp_susage = 0
## max_times_tcp_inseq_uusage = 0
## max_times_tcp_inseq_susage = 0
## max_times_tcp_catchup_uusage = 0
## max_times_tcp_catchup_susage = 0
## max_times_tcp_serv_uusage = 0
## max_times_tcp_serv_susage = 0
## max_times_http_uusage = 0
## max_times_http_susage = 0
## max_times_html_uusage = 0
## max_times_html_susage = 0

max_wmbs = 0
max_tmbs = 0
max_umbs = 0
max_hmbs = 0 
max_wpps = 0
max_tpps = 0
max_upps = 0
max_hpps = 0
max_hparsed = 0
max_repwritten = 0
max_dumpwritten = 0
max_twritten = 0
max_repdumped = 0
max_dumpdumped = 0
max_tdumped = 0
max_mfi = 0
max_bheld = 0
max_fails = 0

max_IP_ulen = 0

max_tcp_to = 0
max_udp_to = 0
max_tcp_seq_to = 0

max_tous_in = 0
max_tous_out = 0
max_frus_in = 0
max_frus_out = 0

# do but packet arrival driven
w_max_wmbs = 0
w_max_tmbs = 0
w_max_umbs = 0
w_max_hmbs = 0
w_max_wpps = 0
w_max_tpps = 0
w_max_upps = 0
w_max_hpps = 0
w_max_hparsed = 0

# stats from /proc/self
max_proc_utm = 0
max_proc_stm = 0
max_proc_ttm = 0
max_proc_majflt = 0
max_proc_nswap = 0

# stats from /proc/stats
max_proc_itot = 0
max_proc_csw = 0

ru = nprobe.np_rusage()
pr = nprobe.period_report()
wr = nprobe.wrapper_record()
pcr = nprobe.procstat_rec()

file_closes = []

for file in openfilelist:
    
    fnm = os.path.basename(file.fnm)
    hz = file.get_hz()
    ncpus = file.get_ncpus()
    #if len(openfilelist) > 1:
    if 1:
	print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	print "File %s - %d records" % (fnm, file.counters.nrecords)
	print
	sys.stdout.flush()
        
    while 1:
        
        nread += 1
        if to_read and nread > to_read:
            break
         
        #rectype = file.find_next_period_record()
        rectype = file.next_type_rec(REC_OTHER)
        #print rectype
	if rectype == -1: #EOF
	    break

        elif rectype == REC_WIRE_PERIOD_REPORT:
            pr.read_reprec(file, ru, 0)

            #
            # stats stuff - but driven by packet arrival times
            #
            rts = tv2l(pr.get_ts())
            if first_ts == 0:
                first_ts = rts
            if rts == last_w_rts:
                print 'rusage - duplicated time stamp',
                #print float(rts-first_ts)/1000000
                continue
            per = rts - last_w_rts
            last_w_rts = rts
            ts = float(rts - first_ts)/1000000
            #print ts, per

            # bandwidths
            wmbs = pr.wire_mbs
            w_max_wmbs = max([wmbs, w_max_wmbs])
            w_wire_bw.append([ts, wmbs])
            
            tmbs = pr.tcp_mbs
            w_max_tmbs = max([tmbs, w_max_tmbs])
            w_tcp_bw.append([ts, tmbs])
            
            umbs = pr.udp_mbs
            w_max_umbs = max([umbs, w_max_umbs])
            w_udp_bw.append([ts, umbs])
            
            hmbs = pr.http_mbs
            #print hmbs
            w_max_hmbs = max([hmbs, w_max_hmbs])
            w_http_bw.append([ts, hmbs])
            
            hparsed  = float(ul2l(pr.get_html_parsed()))/per
            #print hparsed
            w_max_hparsed = max([hparsed, w_max_hparsed])
            w_html_parse.append([ts, hparsed])

            # packet rates
            p = float(ul2l(pr.get_wire_pkts()))
            w_max_wpps = max([w_max_wpps, p])
            w_wire_pkts.append([ts, p])
            
            p = float(ul2l(pr.get_tcp_pkts()))
            w_max_tpps = max([w_max_tpps, p])
            w_tcp_pkts.append([ts, p])
            
            p = float(ul2l(pr.get_udp_pkts()))
            w_max_upps = max([w_max_upps, p])
            w_udp_pkts.append([ts, p])
            
            p = float(ul2l(pr.get_http_pkts()))
            w_max_hpps = max([w_max_hpps, p])
            w_http_pkts.append([ts, p])

            

        elif rectype == REC_PERIOD_REPORT:
            pr.read_reprec(file, ru, 1)

            #
            # rusage stuff
            #
            uts = tv2l(ru.get_ts())
            if first_ts == 0:
                first_ts = uts
            utm = tv2l(ru.get_utime()) # user time
            stm = tv2l(ru.get_stime()) # system time

            this_utm = utm - last_utm
            this_stm = stm - last_stm
            ru_per = uts - last_uts

            upct = (float(this_utm)*100)/ru_per
            spct = (float(this_stm)*100)/ru_per

            #print '%.2f' % (pct)
            tm = (uts-first_ts)/1000000
            urusage.append([tm, upct])
            srusage.append([tm, spct])
            trusage.append([tm, upct+spct])

            last_utm = utm
            last_stm = stm
            last_uts = uts

##
## Linux rusage doesn't record these
##

##             m = float(ru.max_rss())
##             #print 'rss ', ; print m
##             m = m/1000
##             max_maxrss = max([m, max_maxrss])
##             maxrss.append([tm, m])

##             m = ru.majflt()
##             #print 'majflt ', ; print m
##             n = m - last_majflt
##             last_majflt = m
##             max_majflt = max([n, max_majflt])
##             majflt.append([tm, n])

##             m = ru.nswap()
##             #print 'swap ', ; print m
##             n = m - last_nswap
##             last_nswap = m
##             max_nswap = max([n, max_nswap])
##             nswap.append([tm, n])

##             m = ru.blkio()
##             #print 'blkio ', ; print m
##             n = m - last_blkio
##             last_blkio = m
##             max_blkio = max([n, max_blkio])
##             blkio.append([tm, n])

##             m = ru.cswitches()
##             #print 'cswitches ', ; print m
##             n = m - last_cswitches
##             last_cswitches = m
##             max_cswitches = max([n, max_cswitches])
##             cswitches.append([tm, n])

            #
            # stats stuff
            #
            rts = tv2l(pr.get_ts())
            if first_ts == 0:
                first_ts = rts
            per = rts - last_rts
            last_rts = rts
            ts = float(rts - first_ts)/1000000
            #print ts

            # bandwidths
            wmbs = pr.wire_mbs
            max_wmbs = max([wmbs, max_wmbs])
            wire_bw.append([ts, wmbs])
            
            tmbs = pr.tcp_mbs
            max_tmbs = max([tmbs, max_tmbs])
            tcp_bw.append([ts, tmbs])
            
            umbs = pr.udp_mbs
            max_umbs = max([umbs, max_umbs])
            udp_bw.append([ts, umbs])
            #print umbs, max_umbs
            
            hmbs = pr.http_mbs
            #print hmbs
            max_hmbs = max([hmbs, max_hmbs])
            http_bw.append([ts, hmbs])
            
            hparsed  = float(ul2l(pr.get_html_parsed()))/per
            #print hparsed
            max_hparsed = max([hparsed, max_hparsed])
            html_parse.append([ts, hparsed])

            # packet rates
            p = float(ul2l(pr.get_wire_pkts()))
            max_wpps = max([max_wpps, p])
            wire_pkts.append([ts, p])
            
            p = float(ul2l(pr.get_tcp_pkts()))
            max_tpps = max([max_tpps, p])
            tcp_pkts.append([ts, p])
            
            p = float(ul2l(pr.get_udp_pkts()))
            max_upps = max([max_upps, p])
            udp_pkts.append([ts, p])
            
            p = float(ul2l(pr.get_http_pkts()))
            max_hpps = max([max_hpps, p])
            http_pkts.append([ts, p])

            #write_rates
            repwritten = float(ul2l(pr.get_rep_bytes_written()))/1000
            max_repwritten = max([repwritten, max_repwritten])
            rwritten.append([ts, repwritten])
            #print repwritten,  max_repwritten 
            
            dumpwritten = float(ul2l(pr.get_dump_bytes_written()))/1000
            max_dumpwritten = max([dumpwritten, max_dumpwritten])
            dwritten.append([ts, dumpwritten])
            #print dumpwritten,  max_dumpwritten
            
            d = (repwritten+dumpwritten)
            max_twritten = max([d, max_twritten])
            twritten.append([ts, d])
            #print d,  max_twritten
            #print '-----------'

            # dump rates
            repdumped = (float(ul2l(pr.get_rep_blks_dumped()))*WRITE_BLKSZ)/1000
            max_repdumped = max([repdumped, max_repdumped])
            rdumped.append([ts, repdumped])
            #print repdumped,  max_repdumped 
            
            dumpdumped = (float(ul2l(pr.get_dump_blks_dumped()))*WRITE_BLKSZ)/1000
            max_dumpdumped = max([dumpdumped, max_dumpdumped])
            ddumped.append([ts, dumpdumped])
            #print dumpdumped,  max_dumpdumped
            
            d = (repdumped+dumpdumped)
            max_tdumped = max([d, max_tdumped])
            tdumped.append([ts, d])
            #print d,  max_tdumped
            #print '-----------'
            
            # other stuff
            mfi = float(ul2l(pr.get_max_fetch_interval()))/1000
            max_mfi = max([mfi, max_mfi])
            max_fetch_int.append([ts, ul2l(mfi)])

            bh = pr.buff_held
            max_bheld = max([bh, max_bheld])
            bheld.append([ts, bh])

            nf = pr.buf_alloc_fails
            max_fails = max([max_fails, nf])
            fails.append([ts, nf])

            v = pr.get_tcp_to()
            if v < 0:
                v = ul2l(v)
            max_tcp_to = max([max_tcp_to, v])
            tcp_to.append([ts, v])

            v = pr.get_udp_to()
            if v < 0:
                v = ul2l(v)
            max_udp_to = max([max_udp_to, v])
            udp_to.append([ts, v])

            seq_to = 0L
            v = pr.get_tcp_seq_to()
            if v < 0:
                v = ul2l(v)
            seq_to += v
            v = pr.get_tcp_seq_to_forced()
            if v < 0:
                v = ul2l(v)
            seq_to += v
            v = pr.get_tcp_seq_to_q()
            if v < 0:
                v = ul2l(v)
            seq_to += v
            v = pr.get_tcp_seq_to_ack()
            if v < 0:
                v = ul2l(v)
            seq_to += v

            max_tcp_seq_to = max([max_tcp_seq_to, seq_to]) 
            tcp_seq_to.append([ts, float(seq_to)])
            
            v = float(ul2l(pr.get_IP_ulen()))
            #print v
            max_IP_ulen = max([max_IP_ulen, v])
            IP_ulen.append([ts, v*1000000/per])

            v = ul2l(pr.get_tous_in())
            p = (float(v - last_tous_in)*1000000)/per
            last_tous_in = v
            max_tous_in = max([p, max_tous_in])
            tous_in.append([ts, p])

            v = float(ul2l(pr.get_tous_out()))
            p = (float(v - last_tous_out)*1000000)/per
            last_tous_out = v
            max_tous_out = max([p, max_tous_out])
            tous_out.append([ts, p])

            v = float(ul2l(pr.get_frus_in()))
            p = (float(v - last_frus_in)*1000000)/per
            last_frus_in = v
            max_frus_in = max([p, max_frus_in])
            frus_in.append([ts, p])

            v = float(ul2l(pr.get_frus_out()))
            p = (float(v - last_frus_out)*1000000)/per
            last_frus_out = v
            max_frus_out = max([p, max_frus_out])
            frus_out.append([ts, p])
            
            

        elif rectype == REC_WRAPPER:
            #print 'wrapper'
            wr.read(file)
##             if wr.type == WRAPPER_TIME_CALLS:
##                 #
##                 # Call times stuff
##                 #
##  ##                ts = tv2l(wr.get_ts())
##                 per = ts - last_times_ts
##                 last_times_ts = ts
##                 ts = float(ts - first_ts)/1000000

##                 # IP
##                 v = ull2l(wr.get_ip_usrtm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_ip_uusage = max([max_times_ip_uusage, pc])
##                 times_ip_uusage.append([ts, pc])
                
##                 v = ull2l(wr.get_ip_systm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_ip_susage = max([max_times_ip_susage, pc])
##                 times_ip_susage.append([ts, pc])

##                 # UDP
##                 v = ull2l(wr.get_udp_usrtm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_udp_uusage = max([max_times_udp_uusage, pc])
##                 times_udp_uusage.append([ts, pc])
                
##                 v = ull2l(wr.get_udp_systm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_udp_susage = max([max_times_udp_susage, pc])
##                 times_udp_susage.append([ts, pc])

##                 # TCP
##                 v = ull2l(wr.get_tcp_usrtm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_tcp_uusage = max([max_times_tcp_uusage, pc])
##                 times_tcp_uusage.append([ts, pc])
                
##                 v = ull2l(wr.get_tcp_systm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_tcp_susage = max([max_times_tcp_susage, pc])
##                 times_tcp_susage.append([ts, pc])

##                 # TCP_INSEQ
##                 v = ull2l(wr.get_tcp_inseq_usrtm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_tcp_inseq_uusage = max([max_times_tcp_inseq_uusage, pc])
##                 times_tcp_inseq_uusage.append([ts, pc])
                
##                 v = ull2l(wr.get_tcp_inseq_systm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_tcp_inseq_susage = max([max_times_tcp_inseq_susage, pc])
##                 times_tcp_inseq_susage.append([ts, pc])

##                 # TCP_CATCHUP
##                 v = ull2l(wr.get_tcp_catchup_usrtm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_tcp_catchup_uusage = max([max_times_tcp_catchup_uusage, pc])
##                 times_tcp_catchup_uusage.append([ts, pc])
                
##                 v = ull2l(wr.get_tcp_catchup_systm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_tcp_catchup_susage = max([max_times_tcp_catchup_susage, pc])
##                 times_tcp_catchup_susage.append([ts, pc])

##                 # TCP_SERV
##                 v = ull2l(wr.get_tcp_serv_usrtm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_tcp_serv_uusage = max([max_times_tcp_serv_uusage, pc])
##                 times_tcp_serv_uusage.append([ts, pc])
                
##                 v = ull2l(wr.get_tcp_serv_systm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_tcp_serv_susage = max([max_times_tcp_serv_susage, pc])
##                 times_tcp_serv_susage.append([ts, pc])

##                 # HTTP
##                 v = ull2l(wr.get_http_usrtm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_http_uusage = max([max_times_http_uusage, pc])
##                 times_http_uusage.append([ts, pc])
                
##                 v = ull2l(wr.get_http_systm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_http_susage = max([max_times_http_susage, pc])
##                 times_http_susage.append([ts, pc])

##                 # HTML
##                 v = ull2l(wr.get_html_usrtm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_html_uusage = max([max_times_html_uusage, pc])
##                 times_html_uusage.append([ts, pc])
                
##                 v = ull2l(wr.get_html_systm())
##                 #print v
##                 pc = (float(v)*100)/per
##                 #print pc
##                 max_times_html_susage = max([max_times_html_susage, pc])
##                 times_html_susage.append([ts, pc])

        elif rectype == REC_PROCSTAT:
            # these are all recorded per period
            pcr.read(file)
            ts = float(tv2l(pcr.get_ts()) - first_ts)/1000000
            per_jiff = pcr.get_s_tcpu()/ncpus
            per_s = float(per_jiff)/hz

            utm = ul2l(pcr.get_p_utm())
            pc = (float(utm)*100)/per_jiff
            max_proc_utm = max([max_proc_utm, pc])
            proc_utm.append([ts, pc])

            stm = ul2l(pcr.get_p_stm())
            pc = (float(stm)*100)/per_jiff
            max_proc_stm = max([max_proc_stm, pc])
            proc_stm.append([ts, pc])

            ttm = utm + stm
            pc = (float(ttm)*100)/per_jiff
            max_proc_ttm = max([max_proc_ttm, pc])
            proc_ttm.append([ts, pc])

            majflt = ul2l(pcr.get_p_majflt())
            v = majflt/per_s
            max_proc_majflt = max([max_proc_majflt, v])
            proc_majflt.append([ts, v])

            nswap = ul2l(pcr.get_p_nswap())
            v = nswap/per_s
            max_proc_nswap = max([max_proc_nswap, v])
            proc_nswap.append([ts, v])

            itot = ul2l(pcr.get_s_itot())
            v = itot/per_s
            max_proc_itot = max([max_proc_itot, v])
            proc_itot.append([ts, v])

            csw = ul2l(pcr.get_s_csw())
            v = csw/per_s
            max_proc_csw = max([max_proc_csw, v])
            proc_csw.append([ts, v])
            
        
        else:
            file.advance()

        # end while
    
    file_closes.append(float(tv2l(file.close()) - first_ts)/1000000)

# end for file in files


sets = [
    [trusage, 50.0, 1.0, '%CPU', '', 1],
    [urusage, 50.0, 1.0, '%CPU - user', '', 1],
    [srusage, 50.0, 1.0, '%CPU - system', '', 1],

##     [maxrss, max_maxrss, 1.0, 'Max RSS MB', ''],
##     [majflt, max_majflt, 1.0, 'Major page faults', ''],
##     [nswap, max_nswap, 1.0, 'Swaps', ''],
##     [blkio, max_blkio, 1.0, 'In/out blocks', ''],
##     [cswitches, max_cswitches, 1.0, 'Context switches', ''],

##     [times_ip_uusage, max_times_ip_uusage, 1.0, 'IP call usr %CPU', ''],
##     [times_ip_susage, max_times_ip_susage, 1.0, 'IP call sys %CPU', ''],
##     [times_udp_uusage, max_times_udp_uusage, 1.0, 'UDP call usr %CPU', ''],
##     [times_udp_susage, max_times_udp_susage, 1.0, 'UDP call sys %CPU', ''],
##     [times_tcp_uusage, max_times_tcp_uusage, 1.0, 'TCP call usr %CPU', ''],
##     [times_tcp_susage, max_times_tcp_susage, 1.0, 'TCP call sys %CPU', ''],
##     [times_tcp_inseq_uusage, max_times_tcp_inseq_uusage, 1.0, 'TCP_INSEQ call usr %CPU', ''],
##     [times_tcp_inseq_susage, max_times_tcp_inseq_susage, 1.0, 'TCP_INSEQ call sys %CPU', ''],
##     [times_tcp_catchup_uusage, max_times_tcp_catchup_uusage, 1.0, 'TCP_CATCHUP call usr %CPU', ''],
##     [times_tcp_catchup_susage, max_times_tcp_catchup_susage, 1.0, 'TCP_CATCHUP call sys %CPU', ''],
##     [times_tcp_serv_uusage, max_times_tcp_serv_uusage, 1.0, 'TCP_SERV call usr %CPU', ''],
##     [times_tcp_serv_susage, max_times_tcp_serv_susage, 1.0, 'TCP_SERV call sys %CPU', ''],
##     [times_http_uusage, max_times_http_uusage, 1.0, 'HTTP call usr %CPU', ''],
##     [times_http_susage, max_times_http_susage, 1.0, 'HTTP call sys %CPU', ''],
##     [times_html_uusage, max_times_html_uusage, 1.0, 'HTML call usr %CPU', ''],
##     [times_html_susage, max_times_html_susage, 1.0, 'HTML call sys %CPU', ''],
    
    [wire_bw, max_wmbs, 1.0, 'Wire Mbs', '', 2],
    [tcp_bw, max_tmbs, 1.0, 'TCP Mbs', '', 3],
    [udp_bw, max_umbs, 1.0, 'UDP Mbs', '', 3],
    [http_bw, max_hmbs, 1.0,  'HTTP Mbs', '', 4],
    [html_parse, max_hparsed, 1.0, 'HTML Octets parsed MBs', '', 5],
    
    [wire_pkts, max_wpps, 1.0, 'Wire Pkts', '', 2],
    [tcp_pkts, max_tpps, 1.0, 'TCP Pkts', '', 3],
    [udp_pkts, max_upps, 1.0, 'UDP Pkts', '', 3],
    [http_pkts, max_hpps, 1.0,  'HTTP Pkts', '', 4],
    
    [w_wire_bw, w_max_wmbs, 1.0, 'Wire (wire driven) Mbs', '', 2],
    [w_tcp_bw, w_max_tmbs, 1.0, 'TCP (wire driven) Mbs', '', 3],
    [w_udp_bw, w_max_umbs, 1.0, 'UDP (wire driven) Mbs', '', 3],
    [w_http_bw, w_max_hmbs, 1.0,  'HTTP (wire driven) Mbs', '', 4],
    [w_html_parse, w_max_hparsed, 1.0, 'HTML Octets parsed (wire driven) MBs ', '', 5],
    
    [w_wire_pkts, w_max_wpps, 1.0, 'Wire (wire driven) Pkts', '', 2],
    [w_tcp_pkts, w_max_tpps, 1.0, 'TCP (wire driven) Pkts', '', 3],
    [w_udp_pkts, w_max_upps, 1.0, 'UDP (wire driven) Pkts', '', 3],
    [w_http_pkts, w_max_hpps, 1.0,  'HTTP (wire driven) Pkts', '', 4],
    
    [rwritten, max_repwritten, 1.0,  'Log bytes written KB', '', 4],
    [dwritten, max_dumpwritten, 1.0,  'Error bytes written KB', '', 4],
    [twritten, max_twritten, 1.0,  'Total bytes written KB', '', 4],
    
    [rdumped, max_repdumped, 1.0,  'Log bytes dumped KB', '', 4],
    [ddumped, max_dumpdumped, 1.0,  'Error bytes dumped KB', '', 4],
    [tdumped, max_tdumped, 1.0,  'Total bytes dumped KB', '', 4],
    
    [max_fetch_int, max_mfi, 1.0, 'Max fetch interval ms', '', 6],
    [bheld, max_bheld, 1.0, 'Maximum buffers held', '', 8],
    [fails, max_fails, 1.0, 'Buffer alloc fails', '', 0],
    [tcp_to, max_tcp_to, 1.0, 'TCP flows timed out', ''],
    [udp_to, max_udp_to, 1.0, 'UDP flows timed out', ''],
    [tcp_seq_to, max_tcp_seq_to, 1.0, 'TCP pkts timed out', ''],
    
    [IP_ulen, max_IP_ulen, 1.0, 'Pkts < IP min len p//s', ''],
    
    [tous_in, max_tous_in, 1.0, 'tous_in buffers//s', ''],
    [tous_out, max_tous_in, 1.0, 'tous_out buffers//s', ''],
    [frus_in, max_frus_in, 1.0, 'frus_in buffers//s', ''],
    [frus_out, max_frus_in, 1.0, 'frus_out buffers//s', ''],
    
    [proc_utm, 50.0, 1.0, '%CPU - usr (proc)', ''],
    [proc_stm, 50.0, 1.0, '%CPU - sys (proc)', ''],
    [proc_ttm, 50.0, 1.0, '%CPU - total (proc)', ''],
    [proc_majflt, max_proc_majflt, 1.0, 'Major page faults//s (proc)', ''],
    [proc_nswap, max_proc_nswap, 1.0, 'Page swaps//s (proc)', ''],
    [proc_itot, max_proc_itot, 1.0, 'Interrupts//s (proc)', ''],
    [proc_csw, max_proc_csw, 1.0, 'Context switches//s (proc)', '']
    ]

datasets = []
shows = []

#
# scale e'm all to range 0-100
#
i = 0
for data in sets:
    #print '%s %f' % (data[3], data[1])
    if data[1] == 0:
        continue
    
    while data[1] > 100.0:
        #print 'down'
        data[2] /= 10
        data[1] /= 10
        
    while data[1] < 10.0:
        #print 'up'
        data[2] *= 10
        data[1] *= 10

    #print data[1], data[2]
    
    if data[2] != 1.0:
        for dep in data[0]:
            #print dep[1]
            dep[1] *= data[2]
            #print dep[1]
        fmt = ' X%%.%df' % (places(data[2]))
        data[4] = fmt % (data[2])
    datasets.append(DataSet(data[0], DATA_TS, '%s%s' % (data[3], data[4]),
                            i+1, None))
    shows.append(data[3])
    print '%s %f %f' % (data[3], data[1], data[2])
    i += 1

#
# select what shown on first plot
#
showlist = []
wantlist = [
    '%CPU',
    '%CPU - user',
    # '%CPU - system',

##     #'Max RSS MB',
##     #'Major page faults',
##     #'Swaps',
##     #'In/out blocks',
##     #'Context switches',

##    # 'IP call %CPU',
##    # 'IP call sys %CPU',
##    # 'UDP call %CPU',
##    # 'UDP call sys %CPU',
##     #'TCP call %CPU',
##    # 'TCP call sys %CPU',
##     #'TCP_INSEQ call %CPU',
##    # 'TCP_INSEQ call sys %CPU',
##    # 'TCP_CATCHUP call %CPU',
##    # 'TCP_CATCHUP call sys %CPU',
##     #'TCP_SERV call %CPU',
##     #'TCP_SERV call sys %CPU',
##     #'HTTP call %CPU',
##     #'HTTP call sys %CPU',
##     #'HTML call %CPU',
##    # 'HTML call sys %CPU',
    
    #'Wire Mbs',
    #'TCP Mbs',
    #'UDP Mbs',
    #'HTTP Mbs',
    #'HTML Octets parsed MBs',

    #'Wire Pkts',
    #'TCP Pkts',
    #'UDP Pkts',
    #'HTTP Pkts',
    
    #'Wire (wire driven) Mbs',
    #'TCP (wire driven) Mbs',
    #'UDP (wire driven) Mbs',
    #'HTTP (wire driven) Mbs',
    #'HTML (wire driven) Octets parsed MBs',
    
    #'Wire (wire driven) Pkts',
    #'TCP (wire driven) Pkts',
    #'UDP (wire driven) Pkts',
    #'HTTP (wire driven) Pkts',
    
    # 'Log bytes written KB',
    # 'Error bytes written KB',
    # 'Total bytes written KB',
    
    # 'Log bytes dumped KB',
    # 'Error bytes dumped KB',
    # 'Total bytes dumped KB',
    
    # 'Max fetch interval ms', 
    # 'Maximum buffers held', 
    # 'Buffer alloc fails',
    
    'Pkts < IP min len p//s',
    
    'tous_in buffers//s',
    'tous_out buffers//s',
    'frus_in buffers//s',
    'frus_out buffers//s',

    'TCP flows timed out',
    'UDP flows timed out',
    'TCP pkts timed out',

    '%CPU - usr (proc)',
    '%CPU - sys (proc)',
    '%CPU - total (proc)',
    'Major page faults//s (proc)',
    'Page swaps//s (proc)',
    'Interrupts//s (proc)',
    'Context switches//s (proc)'
    ]
for s in shows:
    if wantlist.count(s):
        showlist.append(shows.index(s))
   

try:
    plot = np_Plot(datasets, basepath, standalone='yes',
           title='Nprobe activity', xlab='Running time %s', ylab='',
           raw_data=0, top=0, showlist=showlist, width=1500, height=1000,
           decorations=[Plot_Decoration(XLINE, file_closes)])
except EmptyDataSetError, p:
    print '%s - empty data set' % (p)
    sys.exit(1)


#decs = plot.add_Decoration([Plot_Decoration(XLINE, xlines)])

#plot.activate()
        
