###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


##############################################################################
## 
##
## Stats collector for HTTP reference tree analysis
## 
##
## 
############################################################################
import types
from nprobe import intoa_string, _inet_aton
from np_filerec import FileRec
from np_notesel import Sellist, CallBackError
from np_TCPDisplay import *
from np_TScreen import Tscreen
from np_statscollector import StatsCollector
from np_plot import DataSet, np_Plot, DATA_TS
from ts_smooth import TsSmooth, NoDataError, SmoothError
from nprobe import TCP_SERV_SYN, TCP_CLI_SYN, TCP_SERV_HTTP, SERVER, CLIENT, \
     TRANS_VAL, TRANS_DUMMY_UNSYNCH, TRANS_DUMMY_ERR, TRANS_ERR, \
     TRANS_INCOMPLETE, TRANS_FINISHED
from nprobe import TSP_SYN, TSP_DUP_SYN, TSP_FIN, TSP_ACKSYN, TSP_RST, \
     CT_NON_OBJECT
from np_TCPConn import D_DUPCSYN, D_DUPFREQ, D_CRETRANS, D_NOT_CONNECTED, \
     D_REQNOTSEEN, D_REPNOTSEEN, D_SLOSS, D_BIGDEL
from np_btypes import AgentTypes, SA, UA
from TCP_Imp import TCP_Imps
from np_atypes import Atype

from array import *
from print_col import whoops, inform

############################################################################

#
# Constants
#

N_TO_RANK = 50 # Top N to rank
NOB_THRESH = 5 # Threshold for no. objects to count a page

PPEROB_TOP = 1000

############################################################################


class TreeStats(StatsCollector):

    def __init__(self, args, trace=0, quiet=0, savedata=1, rec_obs=0, logpath=None):

        StatsCollector.__init__(self, args, quiet=quiet, trace=trace,
                                logpath=logpath)

        self.nob_thresh = NOB_THRESH
        self.nlpages = 0
        self.nspages = 0
        self.spdict = {}
        self.pagelist = []
        self.npages = 0
        self.npagesb = 0
        self.npagesd = 0
        self.npagesdb = 0
        self.cdelays = {}
        self.del_only = 0
        self.ssdeld = None
        self.serv_assoc = {}

        self.nclients = 0
        self.nclients_constructed = 0
        self.nservers = 0

        self.ppers = 0
        self.cpers = 0
        self.nob = 0
        self.nobpers = 0
        self.nconns = 0
        self.nurl = 0

        self.uconns = 0
        self.uconns_inc = 0
        self.ob_uninc = 0
        self.c_uninc = 0
        self.userv = 0
        self.uservd = {}

        self.pperob = array('i', [0]*(PPEROB_TOP+2))
        self.pperob_c = array('i', [0]*(PPEROB_TOP+2))

        self.agtypes = AgentTypes(self.write_log, N_TO_RANK)
        self.reg_b = self.agtypes.reg_b

        self.savedata = savedata
        self.robs = rec_obs
        #self.quiet = quiet

        nppstr = '-(>%dOb/P)' % (self.nob_thresh)

        self.writepage = self.writeconn = self.writepperob = self.nullf

        self.tcp_imps = TCP_Imps()
        self.adict = Atype()

        tree_and_conngroups = [
            'Rank Client-by-#objects',
            'Rank Client-by-#bytes',
            'Rank Client-by-#conns',
            'Rank Client-by-#pages',
            
            'Rank Server-by-#objects',
            'Rank Server-by-#bytes',
            'Rank Server-by-#conns',
            'Rank Server-by-#pages',
            
            'Rank Client/Server-by-#objects',
            'Rank Client/Server-by-#bytes',
            'Rank Client/Server-by-#nconns',
            'Rank Server-by-delay/connection',
            'Rank Server-by-delay/request',
            'Rank Server-by-pdt-per-object' + nppstr + '',
            'Rank Server-by-pdt-per-object' + nppstr + '-less-delays',
            'Rank Server-by-pdt-per-object' + nppstr + '-85%',
            'Rank Server-by-pdt-per-object' + nppstr + '-85%-less-delays',
            'Rank server-by-%-pages-delayed' + nppstr + '',
            'Rank server-by-%-pages-delayed' + nppstr + '-85%'
            ]

        #for w in tree_and_conngroups:
            #print w

        conngroup = ['TCPConn'] + tree_and_conngroups

        treegroup = ['WebClient'] + tree_and_conngroups
        
        nppstr = 'Rank Pages-(>%dob)' % (self.nob_thresh)
        pagegroup = [
            nppstr + '-by-frequency',
            nppstr + '-by-download-time-per-object',
            nppstr + '-by-download-time-per-object-less-delays',
            nppstr + '-by-85%-download-time-per-object',
            nppstr + '-by-85%-download-time-per-object-less-delays',
            nppstr + '-by-average-page-size',
            nppstr + '-by-delay-per-object',
            nppstr + '-by-85%-delay-per-object'
            ]

        allpagegroup = pagegroup + treegroup

        ptgroup = [ 
            'All page times', 
            'All page times - pages <= %d' % (self.nob_thresh), 
            'All page times - pages > %d' % (self.nob_thresh)
            ] 
        
        dtmgroup = tree_and_conngroups + ptgroup + pagegroup

        dpgroup = tree_and_conngroups + pagegroup

        rtmtgroup = ['Retransmits'] + dpgroup
        delgroup = ['Delays', 'Delays-ob'] + dpgroup
        attemptsgroup = ['Attempts']  + dpgroup

        #
        # This is the mapping between packet triggers, stats draw (extract)
        # buttons, and button groups
        # fields: [0] type/button text, [1] groups list, [2] trigger,
        #  [3] delimiter after button flag, [4] plot title
        #
        self.extract_what = [
            ('Connections', conngroup,  None, 0, ''),
            #('Trees', treegroup,  None, 1, ''),
            ('Page', allpagegroup, None, 0, ''),
            ('All Pages', allpagegroup, None, 1, ''),

            
            ('Page Times', dtmgroup, None, 0, 'Page download times'),
            ('85% Page Times', dtmgroup, None, 0, '85% Page download times'),
            ('Page Times less delays', dtmgroup, None, 0,
             'Page download times less delays'),
            ('85% Page Times less delays', dtmgroup, None, 1,
             '85% Page download times less delays'),

            
            ('Page Times per-object', dtmgroup, None, 0,
             'Page download times per-object'),
            ('85% Page Times per-object', dtmgroup, None, 0,
             '85% Page download times per-object'),
            ('Page Times less delays per-object', dtmgroup, None, 0,
             'Page download times less delays per-object'),
            ('85% Page Times less delays per-object', dtmgroup, None, 1,
             '85% Page download times less delays per-object'),

            
            ('Retransmits', rtmtgroup, None, 0, 'Retransmits'),
            ('Retransmits-P-over-1s', rtmtgroup, None, 0, 'Retransmits'),
            ('Retransmits-P-over-5s', rtmtgroup, None, 0, 'Retransmits'),
            ('Retransmits-P-over-10s', rtmtgroup, None, 0, 'Retransmits'),
            ('Retransmits-P-over-60s', rtmtgroup, None, 1, 'Retransmits'),
            ('Delays', delgroup, None, 0, 'Delays'),
            ('Page-delays', dtmgroup, None, 0, 'Delays per page'),
            ('Page-delays-per-object', dtmgroup, None, 0, 'Delays per object per page'),
            ('Attempts', attemptsgroup, 1, 'Attempts')
            ]

############################################################################


    def conn_dels(self, cli):

        #
        # find delays associated with each connection
        #

        def add_ppo(ppo,np):

            try:
                ppo[np] += 1
            except IndexError:
                ppo[-1] += 1
               
            

        self.nconns += len(cli.connlist)
        uconns = []
        dflags = 0
        pperob = self.pperob
        pperob_c = self.pperob_c
        for c in cli.connlist:
            c.pageno = 0
            gotreq, df = c.calc_delays(cli.addr, self.tcp_imps, self.adict)
            dflags |= df
            if not gotreq or not c.translist:
                uconns.append(c)
                continue


            if c.nsgood > 1 or c.ncgood > 1:
                c.persist = max(c.nsgood, c.ncgood)
                self.cpers += 1
                self.nobpers += c.persist
                contlist = []
                i = 0
                for t in c.translist:
                    if t.iscontainer:
                        contlist.append(i)
                    i += 1
                # this is a quick hack - no good for pipelined conns
                first = 1
                to = 0
                opkt = 0
                for p in c.pktlist:
                    if not p.len:
                        continue
                    way = p.dir
                    if way == CLIENT:
                        if first:
                            first = 0
                        else:
                            add_ppo(pperob, opkt)
                            if to in contlist:
                                add_ppo(pperob_c, opkt)
                                
                        opkt = 0
                        to += 1
                    else:
                        opkt += 1
                # collect last
                add_ppo(pperob, opkt)
                if to in contlist:
                    add_ppo(pperob_c, opkt)
                        
            else:
                add_ppo(pperob, c.sdpkts)
                if c.translist[0].iscontainer:
                    add_ppo(pperob_c, c.sdpkts)

        return (uconns, dflags)

############################################################################

    def remdels(self, conns):

        def add(a, b):
            return a+b

        def insert(l, t):
            i = 0
            for lt in l:
                if lt[0] < t[0]:
                    break
                i += 1
            l.insert(i, t)

        trace = 0 and self.trace
        #trace = 1

        if trace:
            print 'new'
        conni = 0
        closes = []
        ncloses = []
        nop = 0
        oop = 0
        tdel = 0

        c = conns[0]
        start = c.open/1000
        nend = 0
        tt = ntt = c.open/1000
        trig = ntrig = 0

        for c in conns:
            id = c.indx
            d =  c.get_delays(1)
            cop = c.cop2
            ot = c.open/1000
            #ct = c.close/1000
            ct = c.close1/1000
            if d and d[0] & D_REPNOTSEEN:
                norep = 1
                if trace:
                    print id, 'no response'
            else:
                norep = 0
            dur = ct-ot
            #print c.id, cop, ot
            while oop > cop:
                gr = closes.pop()
                tt = gr[0]
                trig = gr[1]
                oop -= 1
                #print 'old pop', c.id, tt,
            lag = ot - tt
            #print id, 'trig', trig
            insert(closes, (ot+dur, id))
            oop += 1
            if d and d[0] & D_BIGDEL:
                #print d
                dr = reduce(add, d[1:])
                tdel += dr
                if norep:
                    dur = dr
                dur -= dr
            else:
                dr = 0
            while nop > cop:
                ngr = ncloses.pop()
                ntt = ngr[0]
                ntrig = ngr[1]
                nop -= 1
                #print 'pop', c.id, ot
            newot = ntt + lag
            nc = newot+dur
            if trace:
                print 'dur-dr=', dur+dr, dr, 'newot+dur=nc', newot, dur, nc
            insert(ncloses, (nc, id))
            nop +=1
            c.dsave = ot - newot
            if trace:
                #print '%3d %2d %4d %6d->%6d (%5d+%5d) %6d->%6d %2d->%2d %d %d' % (id, cop, dr,  c.open/1000, newot, ntt, lag,  c.close/1000, nc, trig, ntrig, nc-c.close/1000, nend)
                print '%3d %2d %4d %6d->%6d (%5d+%5d) %6d->%6d %2d->%2d %d' % (id, cop, dr,  c.open/1000, newot, ntt, lag,  c.close/1000, nc, trig, ntrig, nc-c.close/1000)
            
        return tdel    

############################################################################
        

    def suck_tree(self, client, uconns):

        def add(a, b):
            return a+b

        def by_nobs(a, b):
            return b[1][0] - a[1][0]
        
        def by_1(a, b):
            return b[1] - a[1]

        def by_bytes(a, b):
            return b[1][1] - a[1][1]

        def by_nconns(a, b):
            return len(b[1][2]) - len(a[1][2])

        def by_id(a, b):
            return a.id - b.id

        def attach_uconns():
            #
            # Find any transaction-less connections belonging to this tree
            # - won't be linked in as no transaction
            # - assume belong if connid is between first and last or last+1
            #

            gotone = 0
            first = connsf[0].id
            last = connsf[-1].id
            for c in uconns: # small list so linear scan
                id = c.id
                if first < id < last or id == last+1:
                    uconns.remove(c)
                    connsf.append(c)
                    cdict[id] = c
                    last = max(last, id)
                    gotone += 1
                    c.intree = 1
                #elif c > last:
                    #break

            if gotone:
                connsf.sort(by_id)

        def write_linkdata(t):
            ld = {} # dictionary of link types
            #ndlinks = nalinks = nduplinks = 0
            v = t.ob.ldict.values()
            nlinks = len(v) # no of discrete URL links of all types derived from all sights of this object for this client
                
            #for type, links in v:
            for ent in v:
                type = ent[0]
                links = ent[1]
                e = ld.setdefault(type, [0, 0, 0, 0])
                e[0] += 1 # no of discrete URL links of this type derived from all sights of object
                got = 0
                for l in links:
                    if l.trans == t:
                        got = 1
                        break
                if got:
                    #ndlinks += 1
                    e[1] += 1 # no of discrete URL links of this type in this instance of the object
                    if l.target:
                        e[2] += 1 # ditto followed
                    #nduplinks += len(l.subs)
                    e[3] += len(l.subs) # no of duplicated links for this type in this instance

            #print nlinks, ndlinks, nduplinks, ld
            tl = ld.items()
            write(' %d %d' % (nlinks, len(tl)))
            for t, d in tl:
                write(' %d %d %d %d %d' % (t, d[0], d[1], d[2], d[3]))
                        
                    
                    

        #
        #
        # Main fn starts here
        #

        A_INTREE = 0x1
        A_UNLINKED = 0x2
        A_REFRESH = 0x4
        A_INVAL = 0x8
        A_REVISIT = 0x10


        trace = 1 and self.trace
        #trace = 1

        if trace:
            print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
            print 'Suck_tree for %s %s' % (client.Class, client.addr_str())

        spd = self.spdict
        cid = client.id
        self.nob += len(client.translist)
        write = self.writepage

        # tmp
        tdict = {}
        for t in client.translist:
            tdict[t] = 0
        # end tmp
        
        td = client.find_trees2()

        # iterate over trees - g is generator for iteration over transactions
        for g, url, rsid, rconnid, order, start, ua, sa, ltype, ctype in td:
            #tid = (url, rconnid)
            tlist = []
            sdict = {}
            cdict = {}
            nob = 0
            bytes = 0
            persist = 0
            dflags = 0

            if trace:
                print 'tree root', url
                if start != None:
                    print 'start', start/1000
                    
            
            # iterate over transactions
            for t in g:
                connid = t.connid
                tc = t.TConn
                if connid < 0:
                    # dummy - don't count
                    continue
                tdict[t] |= A_INTREE # tmp
                tc.intree = 1
                tc.pageno = self.npages
                t.intree = 1
                if start == None:
                    # root was a dummy - use first real
                    if t.connorder == 0:
                        start = tc.open
                    else:
                        start = t.reqstart
                    rsid = tc.server
                    ua = t.uagent
                    sa = t.server
                    if trace:
                        print 'start', start/1000
                tlist.append(t)
                nb = t.nbytes
                
                nob += 1
                bytes += nb
                cdict[connid] = tc
                sid = t.servaddr
                sdict[sid] = t.server
                spd[sid] = None
                
                persist |= tc.persist
                if tc.delays:
                    dflags |= tc.delays[-1]
                
            if nob == 0:
                continue
            if nob > self.nob_thresh:
                self.npagesb += 1
            #conns = cdict.keys()
            connsf = cdict.values()
            connsf.sort(by_id)
            if uconns:
                attach_uconns()
            tdel = self.remdels(connsf)
            conns = cdict.keys()
            #print conns
            for c in conns:
                cdict[c] = 1

            ends = []
            nends = []
            acc_del = 0
            delv = 0
            for t in tlist:
                tend = t.repend/1000
                ends.append(tend)
                dsave = t.TConn.dsave
                nends.append(tend-dsave)
                acc_del += dsave
                delv += dsave*dsave

            ends.sort()
            end = ends[-1]
            if nob > self.nob_thresh:
                self.nlpages += 1
                #end85 = ends[int((nob*0.85)-0.5)]
            else:
                #end85 = end
                self.nspages += 1
            end85 = ends[int((nob*0.85)-0.5)]
            if tdel:
                self.npagesd += 1
                nends.sort()
                nend = nends[-1]
                if nob > self.nob_thresh:
                    self.npagesdb += 1
                    #nend85 = nends[int((nob*0.85)-0.5)]
                #else:
                    #nend85 = nend
                nend85 = nends[int((nob*0.85)-0.5)]
            else:
                nend = end
                nend85 = end85
            start /= 1000
            #print 'start', start, 'end', end
            tdur = int(end-start)
            tdur85 = int(end85-start)
            ndur = int(nend-start)
            ndur85 = int(nend85-start)
            dsave = ndur-tdur
            dtm =  (start/1000.0, tdur, ndur, tdur85, ndur85, dflags, acc_del,
                    delv, tlist[0].TConn.dsave)
            if ndur - tdur > 500 or ndur85 < 0 or ndur < 0:
                str = 'WebClient #%s delay calc bad: %d/%d %d->%d %d->%d %s %d' % \
                      (client.addr_str(), dsave, tdel, tdur, ndur,
                       tdur85, ndur85, 
                       url, rconnid)
                self.write_log(str)
                self.write_log(str[3:]+'         ')
                whoops(str)
                print 'start', start
                print
                self.trace = 1
                self.remdels(connsf)
                self.trace = 0
                #raw_input('...')

            if trace:
                print 'end', end/1000, end%1000
                print conns
                #raw_input('...')

            self.reg_b(ua, UA)
            self.reg_b(sa, SA)

            nservs = len(sdict)
            if nservs > 1:
                #note any associated servers for the page
                asd = self.serv_assoc
                for s in sdict.items():
                    if s[0] != rsid:
                        try:
                            asd[rsid][s[0]] = 1
                        except KeyError:
                            asd[rsid] = {s[0]: 1}
                            str = 'WebClient #%s Server-assoc: %s->%s' % (client.addr_str(), intoa_string(rsid), intoa_string(s[0]))
                            self.write_log(str +'         ')
                            inform(str)
                        self.reg_b(s[1], SA)
                            
            self.ppers += persist
            
            #url = url.replace(' ', '\32')
            #ltype = (ctype << 16) | ltype 
            write('P %d %s %s %s %d %d %d 0x%x %d %d ' % (self.npages, url,
                           intoa_string(client.id), intoa_string(rsid),
                           bytes, nob, len(conns), ltype, ctype, nservs))
            write('%.3f ' % (dtm[0]))
            for v in dtm[1:]:
                write('%d ' % (v))
            write('\n')
            if not self.quiet:
                self.pagelist.append((url, rconnid, order, client.id, rsid,
                                      bytes, nob, dtm, conns, ltype))
            self.npages += 1

            if self.robs:
                tn = 0
                for t in tlist:
                    fng = t.finger
                    if t.connorder == 0:
                        ostart = t.TConn.open/1000000.0
                    else:
                        ostart = t.reqstart/1000000.0
                    if t.reflink:
                        olink_type = t.reflink.type
                    else:
                        olink_type = 0
                    write('T %d %s %s %d %d 0x%x %d %d %f %d %d %d %d' % (tn, t.absurl, intoa_string(t.servaddr), t.connorder, t.nbytes,  olink_type, t.sobtype, t.retcode, ostart, fng[0], fng[1], fng[2], t.iscontainer))
                    tn += 1
                    if t.iscontainer:
                        write_linkdata(t)
                    write('\n')

                write('##\n')


        #for pre, l in [['U', client.unlinkedlist], ['R', client.refreshlist],
                       #['I', client.invalidlist], ['V', client.revisitlist]]:
        for pre, l, key in [['U', client.unlinkedlist, A_UNLINKED], ['R', client.refreshlist, A_REFRESH], ['I', client.invalidlist, A_INVAL], ['V', client.revisitlist, A_REVISIT]]:
            for t in l:
                tdict[t] |= key # tmp
                write('%s %s %s %s %d %d ' % (pre, t.absurl,
                   intoa_string(client.id), intoa_string(t.servaddr),
                                                 t.nbytes, t.sobtype))
                tc = t.TConn
                d = tc.get_delays(1)
                if d and d[0] & D_BIGDEL:
                    dr = reduce(add, d[1:])
                    df = d[0]
                else:
                    dr = 0
                    df = 0
                dur = tc.close-tc.open
                write('%.3f %d %d %d\n' % (tc.open/1000000.0, dur/1000,
                                           (dur-dr)/1000, df))
        write('##\n##\n')

        # tmp
        for t, v in tdict.items():
            if not v:
                str = 'WebClient #%s unaccounted ob: conn %d' % \
                      (client.addr_str(), t.connid)
                self.write_log(str)
                whoops(str)
            ct = 0
            while v:
                if v & 0x1:
                    ct += 1
                v = v >>1
            if ct > 1:
                str = 'WebClient #%s multiply accounted ob: conn %d refs %d' % \
                      (client.addr_str(), t.connid, v)
                self.write_log(str)
                whoops(str)

############################################################################

    def get_data(self, client):

        trace = 1 and self.trace

        try:

            uconns, dflags = self.conn_dels(client)
            if self.del_only and not dflags:
                return
            nuconns = len(uconns)
            self.uconns += 1
            self.suck_tree(client, uconns)
            nattached = nuconns - len(uconns)
            self.uconns_inc += nattached

            if nuconns:
                if nattached:
                    str = 'WebClient #%s %d/%d Uconns incorporated:' % (client.addr_str(), nattached, nuconns)
                    self.write_log(str)
                    if trace:
                        inform(str)
                else:
                    str = 'WebClient #%s 0/%d Uconns incorporated:' % (client.addr_str(), nuconns)
                    self.write_log(str)
                    if trace:
                        inform(str)

            self.nclients_constructed += 1

        #
        # save connection data
        #

            write =  self.writeconn
            for c in client.connlist:
                write('%s %d ' % (c.id, c.pageno))
                dly = c.dly
                write('%f %d %d %d %d %d' % (dly[0], dly[1], dly[2], dly[3],
                                             c.intree, c.persist))
                if len(dly) == 5:
                    for v in dly[4]:
                        write(' %d' % (v))
                write('\n')
                if not self.quiet:
                    self.cdelays[c.id] = [c.dly, c.intree, c.persist]

                if not c.intree:
                    self.c_uninc += 1
                    if c.server not in self.spdict:
                        self.uservd[c.server] = None


            for t in client.translist:
                if not t.intree:
                    self.ob_uninc += 1

        except:
            str = 'WebClient #%s get_data failure:' % (client.addr_str())
            whoops(str)
            self.write_log(str)
            if trace:
                inform(str)



############################################################################

    def rank(self):

        self.rank_volume()
        self.rank_pages()
        self.rank_servers()
        self.agtypes.rank(self.rank_str)

############################################################################

    def rank_servers(self):

        #
        # Rank most delayed servers by delay per request/connection
        #

        def add(a, b):
            return a+b

        def by_len1(a, b):
            return len(b[1]) - len(a[1])

        odels = []
        cdels = []
        ns = 0
        nsdel = 0
        nsdelb = 0
        totconn = 0
        ndconn = 0
        nbdconn = 0
        sloss = 0


        d_dupcsyn = 0
        d_dupfreq = 0
        d_cretrans = 0
        d_not_connected = 0
        d_reqnotseen = 0
        d_repnotseen = 0
        serv_oop = 0
        serv_rtmtp = 0

        fnm = self.basepath + 'Server-Conns-Summary'
        try:
            f = open(fnm, 'w')
        except IOError, s:
            print 'Couldn\'t open summary file', fnm, s
            sys.exit(1)
        write = f.write
        
        #print 'Rank most delayed servers by delay per request'
        sdl = self.sdict.items()
        sdl.sort(by_len1)
        #self.nservers = len(sdl)
        for s, clist in sdl:
            #print intoa_string(s), clist
            ns += 1
            delay = 0
            npkts = 0
            nconn = 0
            nd = 0
            nbd = 0
            dflags = 0
            for id in clist:
                dels = self.cdelays[id][0]
                npkts += dels[1]
                nconn += 1
                if len(dels) == 3:
                    ndconn += 1
                    nd += 1
                    dd = dels[2]
                    flags = dd[-1]
                    dflags |= flags
                    for i in range(1, len(dd), 2):
                        delay += dd[i]
                    if flags & D_DUPCSYN:
                        d_dupcsyn += 1
                    if flags & D_DUPFREQ:
                        d_dupfreq += 1
                    if flags & D_CRETRANS:
                        d_cretrans += 1
                    if flags & D_NOT_CONNECTED:
                        d_not_connected += 1
                    if flags & D_REQNOTSEEN:
                        d_reqnotseen += 1
                    if flags & D_REPNOTSEEN:
                        d_repnotseen += 1
                    if flags & D_SLOSS:
                        serv_oop += dd[-3]
                        serv_rtmtp += dd[-2]
                        if dd[-3] or dd[-2]:
                            sloss += 1
                        str = 'TCPConn #%d Server loss:' % (id)
                        self.write_log(str)
                    if flags & D_BIGDEL:
                        nbdconn += 1
                        nbd += 1
                            
            if delay and dflags & D_BIGDEL:
                nsdel += 1
                if npkts > self.nob_thresh:
                    nsdelb += 1
                    odels.append((delay/npkts, s, npkts, clist))
                    cdels.append((delay/nconn, s, nconn, clist))
                    #print intoa_string(s), delay, nob, delay/nob
            totconn += nconn
            #addr, npkts, nconns, nconns-del, nconns-bigdel, delay, dflags 
            write('%s %d %d %d %d %d %d\n' \
                  % (intoa_string(s), npkts, nconn, nd, nbd, delay, dflags))

        f.close()

        for what, list in (('packet', odels), ('connection', cdels)):
            list.sort()
            list.reverse()

            for i in range(min(N_TO_RANK, len(list))):
                rank = self.rank_str(i)
                s = list[i]
                str = 'Rank Server-by-delay/%s %s %s > %d requests (%d - %dms):' % \
                      (what, rank, intoa_string(s[1]), self.nob_thresh, s[2], s[0])

                self.write_log(str)
                self.obdict[str] = (s[1], s[3])

        s = '%d/%d servers showing delays (%d > %d pkts) %d/%d conns with all delays %d with large delays %d persistent conns' \
            % (nsdel, ns, nsdelb, self.nob_thresh, ndconn, totconn, nbdconn,
               self.cpers)
        print s
        self.write_log(s)

        s = '%d dupcsyn %d dupfreq %d cretrans %d not connected %d no req %d no rep %d server loss    ' % (d_dupcsyn, d_dupfreq, d_cretrans, d_not_connected, d_reqnotseen, d_repnotseen, sloss)
        print s
        self.write_log(s)

        

############################################################################

    def rank_volume(self):

        def by_0(a, b):
            return b[0] - a[0]

        def by_1(a, b):
            return b[1] - a[1]

        def by_2(a, b):
            return b[2] - a[2]

        def by_3(a, b):
            return b[3] - a[3]
        
        def by_5(a, b):
            return b[5] - a[5]
        
        def by_6(a, b):
            return b[6] - a[6]
        
        def by_7(a, b):
            return b[7] - a[7]
        
        def by_8(a, b):
            return b[8] - a[8]
        
        def by_9(a, b):
            return int((b[9] - a[9])*100)
        
        def by_10(a, b):
            return int((b[10] - a[10])*100)

        def ustr(q):
            if type(q) == types.FloatType:
                return '%.2f%%' % (q)
            else:
                return '%dms' % (q)

        #
        # Rank clients/servers by pages/connections/objects/bytes
        #


        p_by_c_d = {}
        p_by_s_d = {}

        ranks = [
            [by_0, 'objects', 0],
            [by_1, 'bytes', 1],
            [by_2, 'conns', 2],
            [by_3, 'pages', 3]
            ]

        ends = [[3, 'Client', p_by_c_d, 0], [4, 'Server', p_by_s_d, 1]]

        for ki, end , d , dotm in ends:
            for p in self.pagelist:
                e = d.setdefault(p[ki], [0, 0, 0, 0, [], 0, 0, 0, 0, 0, 0])
                e[0] += p[6] #accumulate nob
                e[1] += p[5] #do bytes
                e[2] += len(p[8]) #do conns
                e[3] += 1 #count page
                e[4].append(p) #add page
                if dotm:
                    dtm = p[7]
                    e[5] += dtm[1]
                    e[6] += dtm[2]
                    e[7] += dtm[3]
                    e[8] += dtm[4]
                    if dtm[1] > dtm[2]:
                        e[9] += 1
                        if dtm[3] > dtm[4]:
                            e[10] += 1

            cl = d.values()

            for sortfn, what, di in ranks:

                cl.sort(sortfn)

                for i in range(min(N_TO_RANK, len(cl))):
                    rank = self.rank_str(i)
                    stuff = cl[i]
                    adr = stuff[4][0][ki]
                    #print stuff[:4]
                    #print 'rank', end, what, i, cli[0]
                    str = 'Rank %s-by-#%s %s %s (%d):' % (end, what, rank , intoa_string(adr), stuff[di])
                    self.write_log(str)
                    #self.obdict[str] = (adr, stuff[4])
                    self.obdict[str] = stuff[4]


        
        sv = p_by_s_d.values()
        sv.sort(by_3)
        svl = []
        for s in sv:
            nob = s[0]
            npages = s[3]
            if nob/npages > self.nob_thresh:
                svl.append(s)
            #else:
                #continue
            s[9] *= (100.0/npages)
            s[10] *= (100.0/npages)
            s[5] /= nob
            s[6] /= nob
            s[7] /= nob
            s[8] /= nob

        nppstr = '-(>%dOb/P)' % (self.nob_thresh)
        s1 = 'Rank Server-by-pdt-per-object-(>%dOb/P)' % (self.nob_thresh)
        s2 = 'Rank server-by-%%-pages-delayed-(>%dOb/P)' % (self.nob_thresh)

        ranks = [
            [by_5, s1, '', 0, 5],
            [by_6, s1, '-less-delays', 0, 6],
            [by_7, s1, '-85%', 0, 7],
            [by_8, s1, '-85%-less-delays', 0, 8],
            [by_9, s2, '', 3, 9],
            [by_10, s2, '-85%', 3, 10]
            ]

        for sortfn, rs, what, ti, di in ranks:

            svl.sort(sortfn)

            for i in range(min(N_TO_RANK, len(svl))):
                rank = self.rank_str(i)
                stuff = svl[i]
                if stuff[di] == 0:
                    break
                adr = stuff[4][0][4]
                #print stuff[:4]
                #print 'rank', end, what, i, cli[0]
                str = '%s%s %s %s (%d - %s):' % (rs, what, rank , intoa_string(adr), stuff[ti], ustr(stuff[di]))
                self.write_log(str)
                #self.obdict[str] = (adr, stuff[4])
                self.obdict[str] = stuff[4]

        self.save_delayed_servers([s for s in svl if s[9]])
            
            

############################################################################

    def rank_pages(self):

        #
        # Rank pages by frequency, download times per object
        #


        def by_5_len(a, b):
            return len(b[5]) -len(a[5])

        def by_0(a, b):
            return b[0] -a[0]

        def by_1(a, b):
            return b[1] -a[1]

        def by_2(a, b):
            return b[2] -a[2]

        def by_3(a, b):
            return b[3] -a[3]

        def by_6(a, b):
            return b[6] -a[6]

        def by_7(a, b):
            return b[7] -a[7]

        def by_8(a, b):
            return b[8] -a[8]

        d = {}
        for p in self.pagelist:
            e = d.setdefault(p[0], [0, 0, 0, 0, 0, [], 0, 0, 0])
            e[5].append(p) # add page
            tms = p[7]
            dflags = tms[5]
            e[0] += tms[1] #accumulate dur
            e[1] += tms[2] # do less delays
            e[2] += tms[3] # do 85%
            e[3] += tms[4] # do less delays
            e[4] += p[6] # do no. obs
            e[7] |= dflags # delay flags
            if dflags & D_BIGDEL:
                e[8] += 1

        allpl = d.values()
        self.nurl = len(allpl)
        pl = []
        allpl.sort(by_5_len)           

        # by number of times seen
        pl = [p for p in allpl if p[4]/len(p[5]) > self.nob_thresh]
        #pl.sort(by_5_len)

        for i in range(min(N_TO_RANK, len(pl))):
            rank = self.rank_str(i)
            pages = pl[i][5]
            url = pages[0][0]
            str = 'Rank Pages-(>%dob)-by-frequency %s %s (%d):      ' % (self.nob_thresh, rank , url, len(pages))
            self.write_log(str)
            self.obdict[str] = pages

        # by download time per object
        for p in pl:
            p.append(p[0]-p[1])
            p.append(p[2]-p[3])
            nob = p[4]
            p[0] /= nob
            p[1] /= nob
            p[2] /= nob
            p[3] /= nob
            p[6] = nob/len(p[5])
            p[7] /= nob
            p[8] /= nob

        ranks = [
            ['download-time-per-object', by_0, 0, 'ms'],
            ['download-time-per-object-less-delays', by_1, 1, 'ms'],
            ['85%-download-time-per-object', by_2, 2, 'ms'],
            ['85%-download-time-per-object-less-delays', by_3, 3, 'ms'],
            ['average-page-size', by_6, 6, 'objects'],
            ['delay-per-object', by_7, 7, 'ms'],
            ['85%-delay-per-object', by_8, 8, 'ms']
            ]

        
        for what, sfn, indx, dim in ranks:
            pl.sort(sfn)
            for i in range(min(N_TO_RANK, len(pl))):
                if pl[i][indx] == 0:
                    break
                rank = self.rank_str(i)
                pages = pl[i][5]
                url = pages[0][0]
                str = 'Rank Pages-(>%dob)-by-%s %s %s (%d %d %s):' % (self.nob_thresh, what, rank , url,
                                                   len(p[5]), pl[i][indx], dim)
                self.write_log(str)
                self.obdict[str] = pages

############################################################################

    def write_pperob(self):

        for a, suff in [(self.pperob, ''),
                            (self.pperob_c, '_containers')]:

            fnm = self.basepath + 'Pkts-per_ob' + suff + '.hist'
            try:
                f = open(fnm, 'w')
            except IOError, s:
                print 'Couldn\'t open Pkts-per_ob file', fnm, s
                sys.exit(1)
            write = f.write

            po = a
            for i in range(1, len(po)-1):
                n = po[i]
                if n:
                    write('%d %d\n' % (i, n))
            if po[-1]:
                write('#gt %d %d' % (PPEROB_TOP, po[-1]))

            try:
                f.close()
            except IOError, s:
                print 'Couldn\'t close Pkts-per_ob file', fnm, s
                

############################################################################

    def other_results(self):

        #
        # Enter selections for other results into notes
        #
        
        def pass_lte(p):
            return p[6] <= self.nob_thresh

        def pass_gt(p):
            return p[6] > self.nob_thresh
        
        logfn = self.write_log

        #
        # Entries for page download times (all pages)
        #

        # all pages seen
        str = 'XMsg All Page Downloads: All page times: All page downloads:  All page downloads (%d)' % (len(self.pagelist)) 
        logfn(str)
        self.obdict['All page downloads (%d)' \
                    % (len(self.pagelist))] = None

        # all pages seen <= N objects
        str = 'XMsg All Page Downloads: All page times - pages <= %d: All page downloads - pages <= %d:  All page downloads - pages <= %d (%d)' \
              % (self.nob_thresh, self.nob_thresh, self.nob_thresh, self.nspages) 
        logfn(str)
        self.obdict['All page downloads - pages <= %d (%d)' \
                    % (self.nob_thresh, self.nspages)] = pass_lte

        # all pages seen > N objects
        str = 'XMsg All Page Downloads: All page times - pages > %d: All page downloads - pages > %d:  All page downloads - pages > %d (%d)' \
              % (self.nob_thresh, self.nob_thresh, self.nob_thresh, self.nlpages) 
        logfn(str)
        self.obdict['All page downloads - pages > %d (%d)' \
                    % (self.nob_thresh, self.nlpages)] = pass_gt

        #
        # Entries for Connection delays etc.
        #
        logfn('XMsg Retransmits/connection over time: Retransmits: X: X')
        logfn('XMsg Delays/connection over time: Delays: X: X')
        logfn('XMsg Delays/object over time: Delays-ob: X: X')
        logfn('XMsg No. attempts/connection: Attempts: X: X')

        #
        # Other stuff
        #

        str = '%d Pages seen (%d > %d objects) %d with delays (%d > %d objects) %d persistent  ' % (self.npages, self.npagesb, self.nob_thresh, self.npagesd, self.npagesdb, self.nob_thresh, self.ppers)
        print str
        self.write_log(str)

        
##         #
##         # Save page info
##         #
##         if not self.quiet:
##             fnm = self.basepath + 'NOBT%d.Pages' % (self.nob_thresh)
##             try:
##                 f = open(fnm, 'w')
##             except IOError, s:
##                 print 'Page Summary - couldn\'t open', fnm, s
##                 sys.exit(1)
##             write = f.write

##             for p in self.pagelist:
##                 url = p[0].replace(' ', '\32')
##                 write('%s %s %s %d %d %d ' % (url, intoa_string(p[3]),
##                                       intoa_string(p[4]), p[5], p[6], len(p[8])))
##                 dtm = p[7]
##                 write('%.3f ' % (dtm[0]))
##                 for v in dtm[1:]:
##                     write('%d ' % (v))
##                 write('\n')
##             f.close()

##             fnm = self.basepath + 'Connections'
##             try:
##                 f = open(fnm, 'w')
##             except IOError, s:
##                 print 'open save files - couldn\'t open', fnm, s
##                 sys.exit(1)
##             write = f.write

##             for c in self.cdelays.items():
##                 write('%s ' % (c[0]))
##                 dly = c[1][0]
##                 write('%d %d %d' % (dly[0], dly[1], c[1][1]))
##                 if len(dly) == 3:
##                     for v in dly[2]:
##                         write(' %d' % (v))
##                 write('\n')
                    




############################################################################

    def save_delayed_servers(self, sl):

        sd = self.load_delayed_servers()

        fnm = self.basedir + 'delayed_servers_NOBT%d' % (self.nob_thresh)
        try:
            f = open(fnm, 'w')
        except IOError, s:
            print 'save_delayed_servers - couldn\'t open', fnm, s
            sys.exit(1)
            
        for s in sl:
            pages = s[4]
            first = pages[0]
            d = sd.setdefault(first[4], {})
            for p in pages:
                d[p[3]] = 1
            
        write = f.write
        for s in sd.items():
            write('%s:' % (intoa_string(s[0])))
            for c in s[1].keys():
                write('%s ' % (intoa_string(c)))
            write('\n')

        f.close()

############################################################################

    def load_delayed_servers(self):

        if self.ssdeld != None:
            # already loaded
            return self.ssdeld
        
        self.ssdeld = sd = {}

        fnm = self.basedir + 'NOBT%d.delayed_servers' % (self.nob_thresh)
        try:
            f = open(fnm, 'r')
        except IOError, s:
            print 'get_delayed_servers - couldn\'t open', fnm, s
            if s.__str__().find('No such file or directory') >= 0:
                print '\t- will create'
                return sd
            else:
                sys.exit(1)

        #self.ssdeld = sd = {}
        for l in f.readlines():
            s, cs = l.split(':')
            d = sd[_inet_aton(s)] = {}
            for c in cs.split(' '):
                d[_inet_aton(c)] = 1
        
        f.close()

        return sd

############################################################################

    def get_delayed_servers(self):

        sdict = {}
        cdict = {}
        sd = self.load_delayed_servers()
        for s in sd.items():
            sdict[s[0]] = 1
            for c in s[1]:
                cdict[c] = 1

        return (sdict, cdict)

############################################################################
    

    def save_server_assoc(self):

        fnm = self.basedir + 'server_associations'
        try:
            f = open(fnm, 'w')
        except IOError, s:
            print 'save_server_assoc - couldn\'t open', fnm, s
            sys.exit(1)

        for s in self.serv_assoc.items():
            f.write(intoa_string(s[0]) + ': ')
            for a in s[1].keys():
                if a:
                    f.write(intoa_string(a) + ' ')
            f.write('\n')

        f.close()


############################################################################

    def load_server_assoc(self):

        fnm = self.basedir + 'server_associations'
        try:
            f = open(fnm, 'r')
        except IOError, s:
            #print 'load_server_assoc - couldn\'t open', fnm, s
            print s.__str__()
            if s.__str__().find('No such file or directory') >= 0:
                print '\t- will create'
                return
            else:
                sys.exit(1)

        self.serv_assoc = sa = {}
        for li in f.readlines():
            b, al = li.split(':')
            d = sa[_inet_aton(b)] = {}
            for s in al.split(' '):
                d[_inet_aton(s)] = 1
                         
        f.close()

        return sa

############################################################################

    def add_server_assoc(self, sdict):

        ad = self.serv_assoc

        for sa in ad.keys():
            if sa in sdict:
                for s in ad[sa].keys():
                    sdict[s] = 1

############################################################################

    def add_strict_server_assoc(self, sdict):

        ad = self.serv_assoc

        for sa in ad.keys():
            if sa in sdict:
                b = sdict[sa]
                for add in ad[sa].keys():
                    d = sdict.setdefault(add, {})
                    d.update(b)
            


############################################################################

    #
    # Call-back for Sellist if desired to examine object(s)
    #   - reconstructs from file records and displays

    def draw_objects(self, what, oblist, obdict):

        def recons_type(k):

            for key, type in [ \
            ('Client/Server', 'WebHosts'), \
            ('Client-', 'WebClient'), \
            ('Server-', 'WebServer'), \
            ('Pages-', 'WebClient')
            ]:
                if k.find(key) > 0:
                    return type

            
            print 'Unknown Rank Client filerec type \'%s\'' % (k)
            sys.exit(1)

        def get_conns_from_plist(pl):
            conns = []
            for p in pl:
                conns.extend(p[8])
            return conns
        
        def get_ptlist_from_plist(pl, pfn):

            def pass_all(p):
                return 1

            if not pfn:
                pfn = pass_all
                
            d = {}
            for p in pl:
                if pfn(p):
                    e = d.setdefault(p[3], [])
                    e.append(p)
            ptl = []
            for e in d.items():
                ptl.append(('WebClient', e[1], e[0]))
            return ptl

        def make_pages(plist):
            for p in pages:
                ad = p[3]
                tlist.append(FileRec('WebClient',
                     [obdict[c].recsets for c in p[8]], addr=ad,
                                         rooturl=None, lookup=lookup))

        def make_all_pages(plist):
            for p in plist:
                ad = p[3]
                rooturl = (p[0], p[1])
                conns = self.cdict[ad]
                tlist.append(FileRec('WebClient',
                             [obdict[c].recsets for c in conns], addr=ad,
                             rooturl=rooturl, lookup=lookup))

        lookup = self.lookup
        lookup2 = self.lookup2
        
        tlist = []
        clist = []
        ptlist = []
        ptdiv = 0

        print 'what=', what
        #print oblist

        for m in oblist:
            print 'm=', m
            if m[2] == 'TCPConn':
                clist.append(obdict[m[0]].reconstruct(trace=1))
            elif m[2] == 'WebClient':
                tlist.append(FileRec('WebClient',
                       [obdict[c].recsets for c in self.cdict[m[0]]],
                                     lookup=lookup))
            elif m[2].find('Pages-by-frequency') > 0 \
                     or m[2].find('Pages-by-download-time-per') >= 0 \
                     or m[2].find('Pages-by-85%-download-time-per') >= 0 \
                     or m[2].find('Pages-by-average-page-size') >= 0:
                pages = obdict[m[0]]

                if what == 'Connections':
                    conns = get_conns_from_plist(pages)
                    clist.extend(\
                    [obdict[c].reconstruct(trace=1) for c in conns])
                elif what == 'Page':
                    make_pages(pages)
                elif what == 'All Pages':
                    make_all_pages(pages)
                elif what.find('Page-delays') >= 0:
                    if what.find('per_object') >= 0:
                        po = 1
                    else:
                        po = 0
                    self.show_page_delays(pages, po)
                elif what.find('Page Times') >= 0:
                    if what.find('less delays') >= 0:
                        pti = 2
                        label = 'Page download times less delays'
                    else:
                        pti = 1
                        label = 'Page download times'
                    if what.find('85%') >= 0:
                        pti += 2
                        label = '85% ' + label
                    ptlist.extend(get_ptlist_from_plist(pages, None))

                elif what.find('Retransmits') >= 0 \
                     or what.find('Delays') >= 0 \
                     or  what.find('Attempts') >= 0:
                    dels = self.cdelays
                    conns = get_conns_from_plist(pages)
                    print conns
                    print
                    print
                    self.show_delays([(c, dels[c][0]) for c in conns], what)
                else:
                    raise CallBackError( 'Unknown Pages-by-frequency draw type \'%s\'' % (what))

            elif m[2].find('Server-by-delay') >= 0:
                ad = obdict[m[0]][0]
                conns = obdict[m[0]][1]
                if what == 'Connections':
                    #conns = self.sdict[ad]
                    clist.extend(\
                        [obdict[c].reconstruct(trace=1) for c in conns])
                else:
                    pages = [p for p in self.pagelist if p[4] == ad]

                    if what.find('Page-delays') >= 0:
                        if what.find('per_object') >= 0:
                            po = 1
                        else:
                            po = 0
                        self.show_page_delays(pages, po)
                    elif what == 'Page':
                        make_pages(pages)
                    elif what == 'All Pages':
                        make_all_pages(pages)
                    elif what.find('Page Times') >= 0:
                        if what.find('less delays') >= 0:
                            pti = 2
                            label = 'Page download times less delays'
                        else:
                            pti = 1
                            label = 'Page download times'
                        if what.find('85%') >= 0:
                            pti += 2
                            label = '85% ' + label
                        ptlist.extend(get_ptlist_from_plist(pages, None))
                    elif what.find('Retransmits') >= 0 \
                         or what.find('Delays') >= 0 \
                         or  what.find('Attempts') >= 0:
                        #conns = get_conns_from_plist(pages)
                        dels = self.cdelays
                        self.show_delays([(c, dels[c][0]) for c in conns], what)
                    else:
                        raise CallBackError('Unknown Server-by-delay draw type \'%s\''  % (what))

            
            elif m[2].find('Rank') == 0:
                #raise CallBackError('Rank')
                pages = obdict[m[0]]
                #rtype = recons_type(m[2])
                #conns = get_conns_from_plist(obdict[m[0]][1])
                #ad = obdict[m[0]][0]
                #print conns
                if what == 'Connections':
                    conns = get_conns_from_plist(pages)
                    clist.extend(\
                        [obdict[id].reconstruct(trace=1) for id in conns])
                elif what == 'Page':
                    make_pages(pages)
                elif what == 'All Pages':
                    make_all_pages(pages)   
                elif what.find('Page Times') >= 0:
                    if what.find('less delays') >= 0:
                        pti = 2
                        label = 'Page download times less delays'
                    else:
                        pti = 1
                        label = 'Page download times'
                    if what.find('85%') >= 0:
                        pti += 2
                        label = '85% ' + label
                    ptlist.extend(get_ptlist_from_plist(self.pagelist, None))

                elif what.find('Page-delays') >= 0:
                    if what.find('per_object') >= 0:
                        po = 1
                    else:
                        po = 0
                    self.show_page_delays(pages, po)
                    
                elif what.find('Retransmits') >= 0 \
                     or what.find('Delays') >= 0 \
                     or  what.find('Attempts') >= 0:
                    dels = self.cdelays
                    conns = get_conns_from_plist(pages)
                    self.show_delays([(c, dels[c][0]) for c in conns], what)
                    
                else:
                    raise CallBackError('Unknown Rank Client draw type \'%s\'' % (what))

            elif m[2].find('All page times') == 0:
                if what.find('Page-delays') >= 0:
                    if what.find('per_object') >= 0:
                        po = 1
                    else:
                        po = 0
                    self.show_page_delays(self.pagelist, po)
                    return
                passfn = self.obdict[m[1]]
                ptlist.extend(get_ptlist_from_plist(self.pagelist, passfn))
                label = m[2]
                if what.find('Page Times less delays') >= 0:
                    pti = 2
                elif what.find('Page Times') >= 0:
                    pti = 1
                if what.find('85%') >= 0:
                    pti += 2
                    label = '85% ' + label
                if what.find('per-object') >= 0:
                    ptdiv = 1

            elif m[2].find('Delays') >= 0 or m[2].find('Retransmits') >= 0 \
                     or  m[2].find('Attempts') >= 0:
                #self.show_delays(self.cdelays, what)
                self.show_delays([(c[0], c[1][0]) for c in self.cdelays.items()], what)

            else:
                raise CallBackError('Unknown draw type \'%s\'' % (m[2]))

        if len(tlist) and what in ['Page', 'All Pages']:
            Tscreen(tlist, lookup=lookup2, standalone='no', trace=1)

        if len(clist) and what == 'Connections':
            for c in clist:
                c.adjust_tm_offsets(c.abstart)
            TcpDisplay(clist, standalone='no', logfun=self.nullf, trace=0)

        if len(ptlist):
            data = []
            i = 0
            for e in ptlist:
                rtype = e[0]
                ad = e[2]
                for page in e[1]:
                    #print 'page=', page
                    tms = page[7]
                    wtm = tms[pti]
                    if ptdiv:
                        wtm /= page[6]
                    conns = tuple(page[8])
                    url = (page[0], page[1])
                    #
                    # make the taglist element of tuple to avoid
                    # nplot drag-select from coalescing values
                    data.append([tms[0], wtm, i,
                                 ((rtype, conns, url, ad),)])
                i += 1
            #print '\n\nData\n', data
            ds = DataSet(data, DATA_TS, '', None, callback=self.show_page)
            np_Plot([ds], standalone='no', path=self.obdict['filepath'],
                    title=label, xlab='elapsed time s',
                    ylab='Page\ndownload\ns')
                    

        #print self.curr_show[2]

#############################################################################

    def prob_per(self, data, per, label, nbts, low, firstag):

        #
        # Compute probability of N occurences over period per
        #

        data.sort()
        #print data
        

        nbkts = 10
        nbkts = nbts
        dout = [[], [], [], [], [], [], [], [], [], [], []]
        show = array('f', [0]*(nbkts+1))
        cts = array('i', [0]*(nbkts+1))
        #gt = 0

        #print dout

        start = data[0][0]
        xt = start + per
        n = 0
        for d in data:
            tm = d[0]
            v = d[1]
            ns = d[2]

            if tm > xt:
                #print cts, n
                for i in range(nbkts+1):
                    if n:
                        p = float(cts[i])/n
                    else:
                        p = 0.0
                    dout[i].append([xt, p])
                    show[i] += p
                    cts[i] = 0
                #for i in range(nbkts+1):
                    #print dout[i]
                #raw_input('...')
                n = 0
                xt += per
                while xt < tm:
                    for i in range(nbkts+1):
                        dout[i].append([xt, 0.0])
                    xt += per

            n += ns
            if v < nbkts:
                cts[v] += 1
            else:
                cts[nbkts] += 1

        # any in final interval?
        if n:
            for i in range(nbkts+1):
                p = float(cts[i])/n
                dout[i].append([xt, p])
                show[i] += p


        data = []
        for i in range(low, nbkts):
            if show[i]:
                lab = '%d %s' % (i, label) # -1 as want retransmits
                data.append(DataSet(dout[i], DATA_TS, lab, firstag))
                firstag += 1
        if show[i+1]:
            lab = 'gt. %d %s' % (i, label)
            data.append(DataSet(dout[i], DATA_TS, lab, firstag))
            firstag += 1
                
        return (firstag, data)
        

#############################################################################

    def do_del_prob(self, delay_list, what):
        
        #
        # Plot P connection delays
        #

        syns = []
        syns_nc = []
        freq = []
        req = []
        noreq = []
        norep = []

        dups = 0
        freqr = 0
        reqr = 0
        nreq = 0
        synnc = 0
        nrep = 0
        

        for d in delay_list:
            dd = d[1]
            tm = dd[0]
            nfpkts = dd[1]
            if len(dd) == 3:
                dels = dd[2]
                flags = dels[-1]
                if not (flags & D_BIGDEL):
                    continue
                reqri = 0
                syni = 0
                norepi = 0
                if flags & D_DUPFREQ:
                    freq.append([tm, dels[0]+1, 1])
                    reqri += 2
                    syni += 2
                    freqr = 1
                else:
                    freq.append([tm, 1, 1])
                    if flags & D_REQNOTSEEN and not (flags & D_NOT_CONNECTED):
                        noreq.append([tm, dels[syni], 0, [id]])
                        reqri += 2
                        syni += 2
                        norepi += 2
                        nreq = 1
                    else:
                        noreq.append([tm, 0, 1])
                if flags & D_CRETRANS:
                    req.append([tm, dels[reqri], nfpkts])
                    syni += 2
                    norepi += 2
                    reqr = 1
                else:
                    req.append([tm, 0, nfpkts])
                    
                if flags & D_DUPCSYN:
                    norepi += 2
                    if flags & D_NOT_CONNECTED:
                        syns_nc.append([tm, dels[syni]+1, 1])
                        syns.append([tm, 1, 1])
                        synnc = 1
                    else:
                        syns.append([tm, dels[syni]+1, 1])
                        syns_nc.append([tm, 0, 1])
                        dups = 1
                else:
                    syns.append([tm, 1, 1])
                    #syns_nc.append([tm, 0, 1])
                    if flags & D_NOT_CONNECTED:
                        norepi += 2
                        syns_nc.append([tm, 1, 1])
                        synnc = 1
                    else:
                        syns_nc.append([tm, 0, 1])

                if flags & D_REPNOTSEEN and not (flags & D_REQNOTSEEN):
                    norep.append([tm, dels[norepi], 1])
                else:
                    norep.append([tm, 0, 1])

                
            else:
               freq.append([tm, 1, 1])
               req.append([tm, 0, nfpkts])
               syns.append([tm, 1, 1])
               syns_nc.append([tm, 1, 1])
               noreq.append([tm, 0, 1])
               norep.append([tm, 0, 1])
               
        data = []

        if what.find('Retransmits-P') >= 0:

            try:
                per = self.get_per(what)
            except:
                print 'Don\'t understand P period in \'%s\' - using 1s' % (what)
                per = 1

            firstag = 0
            nbkts = 3
            low = 1

            if freqr:
                nbkts = 3
                firstag, d = self.prob_per(freq, per,
                               'first object requests', nbkts, low, firstag)
                data.extend(d)
            if reqr:
                nbkts = 2
                firstag, d = self.prob_per(req, per,
                               'following object packet retransmission', nbkts,
                                           low, firstag)
                data.extend(d)
            if dups:
                nbkts = 4
                firstag, d = self.prob_per(syns, per, 'SYNs', nbkts, low, firstag)
                data.extend(d)
            if synnc:
                nbkts = 3
                firstag, d = self.prob_per(syns_nc, per,
                           'SYNs - connection not accepted', nbkts, low, firstag)
                data.extend(d)

            nbkts = 2

            if nreq:
                nbkts = 2
                firstag, d = self.prob_per(noreq, per,
                           'No request made on connection', nbkts, low, firstag)
                data.extend(d)

            if nrep:
                nbkts = 2
                firstag, d = self.prob_per(norep, per,
                           'No server response', nbkts, low, firstag)
                data.extend(d)



            if data:
                title = 'Probability over %ds periods' % (per)
                np_Plot(data, standalone='no', path=self.obdict['filepath'],
                        title=title, xlab='elapsed time s',
                        ylab='Probability')

        #elif what.find('Attempts-all') >= 0:
        elif what.find('Attempts') >= 0:
            tag = 0
            for d, lab in [
                (syns, 'SYNS to establish connection'),
                (syns_nc, 'SYNS no connection'),
                (freq, 'First requests'),
                (req, 'Following requests'),
                (noreq, 'No request'),
                (norep, 'No response')
                ]:
                data.append(DataSet(d, DATA_TS, lab, tag, callback=None))
                tag += 1
                
            np_Plot(data, standalone='no', path=self.obdict['filepath'],
                    title='No of:', xlab='elapsed time s',
                    ylab='N')
                
                
            
#############################################################################
    #
    # Plot connection delays
    #

    def show_delays(self, delay_list, what):

        print what
        print delay_list

        syns = []
        syns_nc = []
        nc = []
        freq = []
        req = []
        noreq = []
        norep = []
        gotone = 0

        if what == 'Retransmits':
            ioff = 0
            ylab = 'No'
            deft = 1
            title = 'No. of retransmits per connection'
        elif what.find('Delays') >= 0:
            print 'doing delays'
            ioff = 1
            ylab = 'Delay\nms'
            title = 'Delays and cause'
        elif what.find('Retransmits-P') >= 0 \
                 or what.find('Attempts') >= 0:
            self.do_del_prob(delay_list, what)
            return
        else:
            print 'Unknown delay report spec', what
            sys.exit(1)

        for d in delay_list:
            id = d[0]
            dd = d[1]
            tm = dd[0]
            if len(dd) == 3:
                dels = dd[2]
                flags = dels[-1]
                if not (flags & D_BIGDEL):
                    continue
                gotone = 1
                reqri = ioff
                syni = ioff
                norepi = ioff

                try:
                    if flags & D_DUPFREQ:
                        freq.append([tm, dels[ioff], 0, [id]])
                        reqri += 2
                        syni += 2
                        norepi += 2
                        
                    elif flags & D_REQNOTSEEN and not (flags & D_NOT_CONNECTED):
                        noreq.append([tm, dels[syni], 0, [id]])
                        reqri += 2
                        syni += 2
                        norepi += 2
                        
                    if flags & D_CRETRANS:
                        req.append([tm, dels[reqri], 0, [id]])
                        syni += 2
                        norepi += 2
                        
                    if flags & D_DUPCSYN:
                        norepi += 2
                        if flags & D_NOT_CONNECTED:
                            syns_nc.append([tm, dels[syni], 0, [id]])
                        else:
                            syns.append([tm, dels[syni], 0, [id]])
                    elif flags & D_NOT_CONNECTED:
                        norepi += 2
                        nc.append([tm, dels[syni], 0, [id]])
                        #if dels[syni] > 1:
                            #print '%x' % (flags), dels
                            #raw_input('...')

                    if flags & D_REPNOTSEEN and not (flags & D_REQNOTSEEN):
                        norep.append([tm, dels[norepi], 0, [id]])

                except IndexError:
                    print 'flags %x' % (flags), dels
            #else:
                #print 'No delays for', id

        if not gotone:
            print 'No', what

        data = []
        draws = [
            [syns,'SYNS'],
            [syns_nc,'SYNS - connection refused'],
            [nc,'Single SYN - connection refused'],
            [freq, 'First requests'],
            [req, 'Following requests'],
            [noreq, 'No request on connection'],
            [norep, 'No server response']
            ]
        tag = 0

        for d, lab in draws:
            if d:
                #if per:
                    #d = smoother.smooth_data(d)
                data.append(DataSet(d, DATA_TS, lab, tag, callback=self.show_conn))
                tag += 1

        if data:
            np_Plot(data, standalone='no', path=self.obdict['filepath'],
                    title=title, xlab='elapsed time s',
                    ylab=ylab)
        else:
            print 'No', what
                
                
            
#############################################################################
    #
    # Plot page delays
    #

    def show_page_delays(self, plist, po):

        if po:
            postr = ' per object'
        else:
            postr = ''

        d = {}
        for p in plist:
            e = d.setdefault(p[3], [])
            e.append(p)
            
        data = []
        data85 = []
        for e in d.items():
            ad = e[0]
            for p in e[1]:
                tms = p[7]
                tm = tms[0]
                dl = tms[1] - tms[2]
                dl85 = tms[3] - tms[4]
                conns = tuple(p[8])
                url = (p[0], p[1])
                if po:
                    nob = p[6]
                    dl /= nob
                    dl85 /= nob
                data.append([tm, dl, 0, (('Webclient', conns, url, ad),)])
                data85.append([tm, dl85, 0, (('Webclient', conns, url, ad),)])

        ds = DataSet(data, DATA_TS, 'Page', 0, callback=self.show_page)
        ds85 = DataSet(data85, DATA_TS, '85% Page', 1, callback=self.show_page)
        
        np_Plot([ds, ds85], standalone='no', path=self.obdict['filepath'],
        title='Page delays' + postr, xlab='elapsed time s',
        ylab='Page\ndelay\nms')
            

  
#############################################################################


#
# Call-back for plotter if desired to recreate data point object
# - list will be of conn ids
#                    
    def show_conn(self, cl):

        obdict = self.obdict
        clist = []
        for id in cl:
            C = obdict[id].reconstruct(trace=1)
            C.adjust_tm_offsets(C.abstart)
            clist.append(C)
            
        TcpDisplay(clist, standalone='no', logfun=self.nullf, trace=1)

  
#############################################################################


#
# Call-back for plotter if desired to recreate data point object
# - list will be of pages
#                    
    def show_page(self, ptlist):

        obdict = self.obdict
        tlist = []

        for pt in ptlist:
            rtype = pt[0]
            conns = pt[1]
            url = pt[2]
            ad = pt[3]
            tlist.append( \
                 FileRec(rtype, [obdict[c].recsets for c in conns],
                         addr =ad, rooturl=url, lookup=self.lookup))
        Tscreen(tlist, lookup=self.lookup2, standalone='no', trace=1)


#############################################################################

    def results(self):

        self.in_results = 1
        if not self.quiet:
            self.rank()
            self.other_results()
            
        self.write_pperob()

        str = '%d clients ' % (self.nclients)
        self.write_log(str)
        print str
        str = '%d servers ' % (len(self.spdict))
        self.write_log(str)
        print str
        str =  '%d connections %d objects, %d on persistent connections %d URLs' % (self.nconns, self.nob, self.nobpers, self.nurl)
        self.write_log(str)
        print str

        str = '%d no request connections - %d incorporated' % (self.uconns,
                                                               self.uconns_inc)
        self.write_log(str)
        print str

        sd = self.spdict
        userv = 0
        for s in self.uservd:
            if not sd.has_key(s):
                userv += 1

        str = '%d objects - %d conns %d servers not in trees' % (self.ob_uninc,
                                              self.c_uninc, userv)
        self.write_log(str)
        print str


        self.save_server_assoc()
        self.pagefile.close()
        self.connfile.close()
        
        if self.quiet:
            self.ectrs.printself(self.tfilt, f=self.logf, leader='')
            self.adict.printself_tofile(self.logf)
            self.dump_log()
            self.close_log()
        else:
            self.ectrs.printself_tolist(self.log, self.tfilt)
            self.adict.printself_tolist(self.log)
            self.save_log()
            draw_menu = [[e[0], e[1], e[3]] for e in self.extract_what]
            Sellist(draw_menu, self.log, self.obdict, self.draw_objects)


#############################################################################

    def open_savefiles(self):

        def backup(fnm):
            if not self.savedata:
                return
            if os.access(fnm, os.F_OK):
                print 'backing up', fnm
                os.rename(fnm, fnm + '~')


        fnm = self.basepath + 'NOBT%d.Pages' % (self.nob_thresh)
        backup(fnm)
        try:
            f = open(fnm, 'w')
        except IOError, s:
            print 'open save files - couldn\'t open', fnm, s
            sys.exit(1)
        self.writepage = f.write
        self.pagefile = f
        f.write('# Run start = %d\n' % self.run_start)

        fnm = self.basepath + 'Connections'
        backup(fnm)
        try:
            f = open(fnm, 'w')
        except IOError, s:
            print 'open save files - couldn\'t open', fnm, s
            sys.exit(1)
        self.writeconn = f.write
        self.connfile = f
        f.write('# Run start = %d\n' % self.run_start)






        
        
            
