#! /usr/bin/env python
###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


##############################################################################
## 
##  np_rets.py
## 
##  Inputs series of Nprobe logs, reads in HTTP transactions, counts 
##  frequency of various server return codes, object sizes, etc
## 
############################################################################ 

from string import center
import glob
import os
from os.path import split
import sys
from types import StringType
from sys import argv
import getopt
import re

import Numeric

from nprobe import REC_TCP_HTTP, TRANS_FINISHED, TRANS_INCOMPLETE, \
TCP_SERV_FIN, TCP_FULL_CLOSE, TCP_EFFECTIVE_CLOSE, TCP_QUICK_CLOSE, \
http_server_returncode_string, http_server_objtype_string, accept_conn, filter_help, HTTP_METHOD_GET, http_client_method_string, MAX_NTRANS

from np_file_util import get_files
from np_http_util import allocate_http_reusable_objects, \
     get_http_rec_and_trans, get_http_rec
#from np_http_util import *

from np_plot import np_Plot, DataSet, DATA_HIST, DATA_PDF, DATA_CDF, MODE_HIST, MODE_PDF, MODE_CDF, STYLE_LINES, STYLE_BARS
from minmax import MIN, MAX

from np_WebAgents import WebAgents
from np_FileTypes import FileTypes

from  np_longutil import ull2l

############################################################################

MAX_RETCODE = 600
MAX_METHOD = 256

OBSZ_MIN = 0
OBSZ_MAX = 1000*100
OBSZ_BIN = 50

NBINS = ((OBSZ_MAX-OBSZ_MIN)/OBSZ_BIN) + 1

# Threshold for individula object reporting
VERY_LARGE_OB_THRESH = 10*OBSZ_MAX

BIGNUMBER = 100000000

############################################################################

def usage(scriptname):

    print
    print scriptname + ':', 'Inputs series of Nprobe logs, reads in HTTP transactions, counts\n  frequency of various server return codes, object sizes, etc.\n  Places results in repfile directory np_rets subdir.'
    
    print "usage: " + scriptname + " [flag(s)] <rep-file-list | rep directory>"
    print 'Flags:\n'
    print '\t-F Apply simple filter to input (use -Fh for details)'
    print '\t-s<spec> Show object size distributions\n\t\tr - as histogram\n\t\tp pdf\n\t\tc cdf'
    print '\t-a Save list of all User Agents and Servers encountered'
    print '\t-t<dir> Check HTTP claimed object types against (partial or complete)\n\t  saved objects using file magic.'
    print '\t-h This help.'

    sys.exit(1)

#############################################################################

def openf(fnm, mode):

    try:
        f = open(fnm, mode)
    except IOError, s:
        print 'Error'
        print str(s)
        sys.exit(1)

    return f

############################################################################# 

def sort_by_occurences_rev(a, b):

    return b[1] - a[1]

#############################################################################

def record(f, s):

    print s
    f.write(s + '\n')

#############################################################################

def write_file(f, s):

    f.write(s + '\n')

#############################################################################

def object_type_string(otype):
    
    return http_server_objtype_string(otype).replace('/', '-')

#############################################################################

def get_mode(flag):

    if flag == 'r':
        return (DATA_HIST, MODE_HIST, STYLE_BARS, '.hist')
    elif flag == 'p':
        return(DATA_PDF, MODE_PDF, STYLE_BARS, '.pdf')
    elif flag == 'c':
        return(DATA_CDF, MODE_CDF, STYLE_LINES, '.cdf')
    else:
        print 'Unrecognised drawing mode'
        sys.exit(1)

#############################################################################

def do_methods(filepath, methods, ntrans):

    filepath = filepath + 'request_methods'
    f = open(filepath, 'w')

    record(f, "\nHTTP request methods (total %d valid objects)\n" \
           % (ntrans))

    sortlist = []
    for i in range(MAX_METHOD):
        n = methods[i]
        if n:
            sortlist.append((n, i))

    sortlist.sort()
    sortlist.reverse()

    record(f, "%s%s%s\n\n" % (center("Method", 15), center("Total", 15),
                           center("% of Whole", 12)))

    for n, meth in sortlist:
        record(f, "%15s%15d%12.3f" % (http_client_method_string(meth).ljust(15), n, (n*100.00)/ntrans))

    record(f, '\n')
    sumfile.write('%d HTTP request methods seen - see %s\n' % (len(sortlist),
                                                               filepath))
    record(sumfile, '\n================================================================\n\n')

    f.close
        

    


#############################################################################

def do_retcodes(filepath, sumfile, vals, totrans):

    for v, label in vals:
        ntrans = v[0]
        rvals = v[4]

        filepath = filepath + label + '.retcodes'
        f = open(filepath, 'w')

        rlist = []

        f.write('HTTP server return codes for %s - by code (%d transactions %.3f%% of total transactions)\n\n' \
               % (label, ntrans, (ntrans*100.00)/totrans))

        f.write("%s%s%s\n\n" % (center("Code", 5), center("Text", 30),
                               center("% Total", 12)))

        for i in range(MAX_RETCODE):
            if rvals[i] > 0:
                f.write("%5d%30s%12.3f\n" % (i, http_server_returncode_string(i),
                                             (rvals[i]*100.0)/totrans))
                rlist.append((rvals[i], i))


        f.write("\n\n\nHTTP server return codes for %s - by frequency\n\n" % (label))

        f.write("%s%s%s\n\n" % (center("Code", 5), center("Text", 30),
                                  center("% Total", 12)))

        rlist.sort()
        rlist.reverse()

        for n, code in rlist:
            f.write("%5d%30s%12.3f\n" % (code,
               http_server_returncode_string(code), (n*100.0)/totrans))


        f.close()
        
    record(sumfile, 'For analysis of server return codes  - see %s\n' % (filepath))
    record(sumfile, '\n================================================================\n\n')


#############################################################################


def do_obtypes(filepath, sumfile, typelist, filetypes, ntrans):

    filepath = filepath + 'object_types'
    f = open(filepath, 'w')

    hdstr = "\nHTTP object types with return code 200 (total %d valid objects)\n" % (ntrans)
    record(sumfile, hdstr)
    f.write(hdstr + '\n')

    record(sumfile, "%s%s%s\n\n" \
           % (center("Type", 30), center("Total", 15),
              center("% of Whole", 16)))

    f.write("%s%s%s%s%s%s\n\n\n" \
           % (center("Type", 30), center("Total", 15),
              center("% of Whole", 16), center('Unfinished', 18),
              center('Non-200 responses', 18), center('Min/Max (Finished)', 25)))

    for fins, unfins, type in typelist:
        nfins = fins[0]
        nunfins = unfins[0]
        tot = nfins + nunfins
        non_200 = fins[3] + unfins[3]

        maxsz = max(fins[6], unfins[6])
        if fins[0]:
            minstr = '%d' % fins[5]
        else:
            minstr = '-'
            
        f.write("%-30s%15d%12.3f%10d (%7.3f%%) %10d (%7.3f%%) %8s %10d\n" % \
               (type, tot, (tot*100.00)/ntrans, nunfins, (nunfins*100.00)/tot, non_200,
                (non_200*100.00)/tot, minstr, fins[6]))
            
        record(sumfile, "%-30s%15d%12.3f" % \
               (type, tot, (tot*100.00)/ntrans))

    record(f, '\n')
    if filetypes:
        filetypes.report_diffs(file=f)
    record(sumfile, '\nFor greater detail see %s\n' % (filepath))
    record(sumfile, '\n================================================================\n\n')

    f.close


#############################################################################
    

def do_obsz(filepath, sumfile, vals, totrans, non_200, modeflag, showflag):
        
    szdir = filepath + 'ob_size_dists'
    try:
        os.mkdir(szdir)
        #print 'Created sizes directory %s' % (szdir)
    except OSError, s:
        if str(s).find('exists'):
            pass
            #print 'Results directory %s already exists' % (szdir)
        else:
            print str(s)
            sys.exit(1)
        
    dtype, mode, style, fsuff = get_mode(modeflag)

    showsets = []
    cycle = 0

    for fin, unfin, type in vals:

        nfin = fin[0]
        nfin_notok = fin[3]
        nfinok = nfin - nfin_notok
        
        if nfinok == 0:
            continue
        
        nunfin = unfin[0]
        nunfin_notok = unfin[3]
        nunfinok = nunfin - nunfin_notok

        fp = szdir + '/' + type
        hf = open(fp + '.hist', 'w')
        pf = open(fp + '.pdf', 'w')
        cf = open(fp + '.cdf', 'w')

        data = fin
        bigs = data[2]  
        bigs.sort()
        minsz = data[5]
        maxsz = data[6]
        osz = data[1]
        zerolens = osz[0]

        typetot = nfin + nunfin
        nt = totrans - nunfin - non_200

        for f in [hf, pf, cf]:

            write_file(f, '# Object size distribution for HTTP %s objects\n#\t%d complete objects sized %d - %d bytes\n#\t(%.3f%% of all completed ok transactions, %.3f%% of all transactions)'\
                   % (type, nfinok, minsz, maxsz, (100.0*nfinok)/nt, (100.0*nfinok)/totrans))
            write_file(f, '\n# %d Total transactions of this type' % (typetot))
            write_file(f, '# Of these:\n')
            totok = nfinok + nunfinok
            write_file(f, '#\t%d Retcode 200 - ok (%.3f%%)' % (totok, (totok*100.0)/typetot))
            totnotok = nfin_notok + nunfin_notok
            write_file(f, '#\t%d Not retcode 200 (%.3f%%)' % (totnotok, (totnotok*100.0)/typetot))
            write_file(f, '#\t%d Finished ok (%.3f%%)' % (nfinok, (nfinok*100.0)/typetot))
            write_file(f, '#\t%d Finished not ok (%.3f%%)' % (nfin_notok, (nfin_notok*100.0)/typetot))
            write_file(f, '#\t%d Unfinished ok (%.3f%%)' % (nunfinok, (nunfinok*100.0)/typetot))
            write_file(f, '#\t%d Unfinished not ok (%.3f%%)\n' % (nunfin_notok, (nunfin_notok*100.0)/typetot))


            write_file(f, '\n# %d Zero length completed ok (%.3f%% of completed ok)' % (zerolens, (zerolens*100.00)/nfinok))
            write_file(f, '\n\n')

        bin = 0
        pts = []
        accum = 0.0


        for i in range(NBINS):
            n = osz[i]
            if n:
                p = float(n)/nfinok
                accum += p
                write_file(hf, '%15d %15d' % (bin, n))
                write_file(pf, '%15d %7.6f' % (bin, p))
                write_file(cf, '%15d %7.6f' % (bin, accum))
                if dtype == DATA_PDF:
                    v = p
                elif dtype == DATA_CDF:
                    v = accum
                else:
                    v = n
                pts.append([bin, v, cycle])
            bin += OBSZ_BIN

        if bigs:
            bigs.append(bigs[-1] + OBSZ_BIN)
            while bin < bigs[0]:
                bin += OBSZ_BIN
            n = 0
            for sz in bigs:
                if sz <= bin:
                    n += 1
                    continue
                else:
                    p = float(n)/nfinok
                    accum += p
                    write_file(hf, '%15d %15d' % (bin, n))
                    write_file(pf, '%15d %7.6f' % (bin, p))
                    write_file(cf, '%15d %7.6f' % (bin, accum))
                    if dtype == DATA_PDF:
                        v = p
                    elif dtype == DATA_CDF:
                        v = accum
                    else:
                        v = n
                    pts.append([bin, v, cycle])
                    n = 1
                    while bin < sz:
                        bin += OBSZ_BIN

        
        for f in [hf, pf, cf]:
            f.close() 

        if pts:
            showsets.append(DataSet(pts, dtype, type, cycle, mode))
            cycle += 1

    if showflag:

        totypes = len(showsets)
        ntypes = 0
        nshown = 0
        grogs = []
        prompt = 'first'

        TYPES_AT_ONCE = 5

        for set in showsets:
            grogs.append(set)
            ntypes += 1
            nshown += 1
            if nshown == TYPES_AT_ONCE or ntypes == totypes:
                rep = raw_input('Show size distribution for %s %d types? y/n ' % (prompt, nshown))
                if rep and rep[0] == 'y':
                    np_Plot(grogs, filepath, title='Distribution of HTTP object sizes %s' % (fsuff.replace('.', ' - ')), style=style, mode=mode)
                grogs = []
                nshown = 0
                prompt = 'next'

    record(sumfile, 'Object size distributions written to %s' % (szdir+'/'))
    record(sumfile, '\n================================================================\n\n')

#############################################################################

def do_agents(filepath, sumfile, save_agents, agents):

    f = openf(filepath + 'agents', 'w')
    agents.aggregate_h()
    agents.report_h(file=f)
    if save_agents:
        agents.save_h(filepath)

    record(sumfile, 'Full server and user agent analysis written to %s' % (filepath + 'agents'))
    record(sumfile, '\n================================================================\n\n')

#############################################################################

def do_transconns(filepath, sumfile, transperconn, npers, npers1):
    
    tcdir = filepath + 'transconns'
    
    try:
        os.mkdir(tcdir)
        #print 'Created sizes directory %s' % (szdir)
    except OSError, s:
        if str(s).find('exists'):
            pass
            #print 'Results directory %s already exists' % (szdir)
        else:
            print str(s)
            sys.exit(1)

    totconn = 0
    allconntms = Numeric.zeros(N_DUR_BINS+1,)
    for n, a in transperconn:
        totconn += n
        if n:
            for i in range(N_DUR_BINS+1):
                allconntms[i] += a[i]

    tpf = []
    for suff in ['.hist', '.pdf', '.cdf']:
        f = openf(tcdir + '/transperconns' + suff, 'w')
        tpf.append(f)
        s = '\n#\n# Distribution of # transactions per TCP connection (%s)\n#\n' % (suff.replace('.', ''))
        f.write(s)

    accum = 0.0
    for i in range(MAX_NTRANS+1):
        n = transperconn[i][0]
        if n:
            p = float(n)/totconn
            accum += p
            write_file(tpf[0], '%10d %10d' % (i, n))
            write_file(tpf[1], '%10d %7.6f' % (i, p))
            write_file(tpf[2], '%10d %7.6f' % (i, accum))

    tpf = []
    for suff in ['.hist', '.pdf', '.cdf']:
        f = openf(tcdir + '/allconndurs' + suff, 'w')
        tpf.append(f)
        s = '\n#\n# Distribution of TCP connection durations ms (%s)\n#\n' % (suff.replace('.', ''))
        f.write(s)

    accum = 0.0
    for i in range(N_DUR_BINS):
        n = allconntms[i]
        if n:
            p = float(n)/totconn
            accum += p
            write_file(tpf[0], '%10d %10d' % (i, n))
            write_file(tpf[1], '%10d %7.6f' % (i, p))
            write_file(tpf[2], '%10d %7.6f' % (i, accum))
    n = allconntms[N_DUR_BINS]
    if n:
        p = float(n)/totconn
        accum += p
        write_file(tpf[0], '# %10d+ %10d' % (MAX_DUR, n))
        write_file(tpf[1], '# %10d+ %7.6f' % (MAX_DUR, p))
        write_file(tpf[2], '# %10d+ %7.6f' % (MAX_DUR, accum))

    ntdf = openf(tcdir + '/trans-v-avedur', 'w')
    ntdf.write('\n#\n# Average connection duration *per transaction* for connections carrying x transactions ms\n#\n')
        

    for  i in range(MAX_NTRANS):
        t, a = transperconn[i]
        if t:
            totms = totc = 0
            tpf = []
            for suff in ['.hist', '.pdf', '.cdf']:
                f = openf(tcdir + '/conndurs-%d-trans%s' % (i, suff), 'w')
                tpf.append(f)
                s = '\n#\n# Distribution of TCP connection durations carrying %d transactions ms (%s)\n#\n' % (i, suff.replace('.', ''))
                f.write(s)
            accum = 0.0
            for j in range(N_DUR_BINS):
                n = a[j]
                if n:
                    p = float(n)/t
                    accum += p
                    write_file(tpf[0], '%10d %10d' % (j, n))
                    write_file(tpf[1], '%10d %7.6f' % (j, p))
                    write_file(tpf[2], '%10d %7.6f' % (j, accum))
                    totc += n
                    totms += j*n
            n = a[N_DUR_BINS]
            if n:
                p = float(n)/totconn
                accum += p
                write_file(tpf[0], '# %10d+ %10d' % (MAX_DUR, n))
                write_file(tpf[1], '# %10d+ %7.6f' % (MAX_DUR, p))
                write_file(tpf[2], '# %10d+ %7.6f' % (MAX_DUR, accum))

            if i and totc:
                write_file(ntdf, '%5d %5d' % (i, totms/(totc*i)))

    record(sumfile, 'TCP connection duration and transactions per connection data written to %s\n' % (tcdir+'/'))
    record(sumfile, '%d connections\n%d persistent connections\n%d persistent connections carrying single transaction' % (totconn, npers, npers1))
    record(sumfile, '\n================================================================\n\n')

    

    

#############################################################################
    

def do_output(filepath, sumfile, obs, methods, agents, filetypes, totrans, non_200, unfin, not_gets, save_agents, modeflag, showflag, transperconn, npers, npers1):

    def sum_bins(a, b):
        s = [0, Numeric.zeros(NBINS,), [], 0, Numeric.zeros(MAX_RETCODE,), \
                                           BIGNUMBER, 0]
        s[0] = a[0] + b[0]
        sb = s[1]
        ab = a[1]
        bb = b[1]
        for i in range(NBINS):
            sb[i] = ab[i] + bb[i]
        
        s[2].extend(a[2])
        s[2].extend(b[2])

        s[3] = a[3] + b[3]

        sb = s[4]
        ab = a[4]
        bb = b[4]
        for i in range(MAX_RETCODE):
            sb[i] = ab[i] + bb[i]
        s[5] = min(a[5], b[5])
        s[6] = max(a[6], b[6])
            
        return s
        

    finobs = obs[0]
    unfinobs = obs[1]

    # coalesce all dict entries for total everything
    all_finished = reduce(sum_bins, finobs.values())
    all_unfinished = reduce(sum_bins, unfinobs.values())
    all = reduce(sum_bins, [all_finished, all_unfinished])

    # combine finished/unfinished to sort types by frequency over both
    sortdict = {}
    
    for (type, v) in finobs.iteritems():
        sortdict[type] = v[0]
    
    for (type, v) in unfinobs.iteritems():
        if sortdict.has_key(type):
            sortdict[type] += v[0]
        else:
            sortdict[type] = v[0]

    sortlist = [(tot, type) for (type, tot) in sortdict.iteritems()]
    sortlist.sort()
    sortlist.reverse()

    n = 0
    for tot, type in sortlist:
        n += tot
    #print 'totrans', totrans, 'accumtot', n, 'coalesce tot', all[0]
           

    # construct data list of ordered finished/unfinished pairs
    oblist = [(all_finished, all_unfinished, 'all-types')]
    for tot, key in sortlist:
        fin = finobs.get(key, [0, Numeric.zeros(NBINS,), [], 0, \
                               Numeric.zeros(MAX_RETCODE,), BIGNUMBER, 0])
        unfin = unfinobs.get(key, [0, Numeric.zeros(NBINS,), [], 0, \
                               Numeric.zeros(MAX_RETCODE,), BIGNUMBER, 0])
        oblist.append((fin, unfin, object_type_string(key)))

    do_methods(filepath, methods, totrans)
    do_obtypes(filepath, sumfile, oblist, filetypes, totrans)

    do_retcodes(filepath, sumfile, [(all, 'all_types')], totrans)

    do_obsz(filepath, sumfile, oblist, totrans, non_200, modeflag, showflag)

    do_agents(filepath, sumfile, save_agents, agents)

    do_transconns(filepath, sumfile, transperconn, npers, npers1)


#############################################################################
    


scriptname = os.path.basename(argv[0])
optstr = ''
fspec = 0
modeflag = 'r'
showflag = 0
save_agents = 0
check_obtypes = 0

try:
    optlist, args = getopt.getopt(sys.argv[1:], 'hF:s:at:')

except getopt.error, s:
    print scriptname + ": " + str(s)
    usage(scriptname)
    sys.exit(1)

for opt in optlist:
    if opt[0] == "-h":
	usage(scriptname)
    elif opt[0] == "-s":
        if not opt[1] in ['r', 'p', 'c']:
            print 'unrecognised mode arg \'%s\'' % (opt[1])
            usage(scriptname)
	modeflag = opt[1]
        showflag = 1
    elif opt[0] == "-F":
        if opt[1] == 'h':
            filter_help()
            sys.exit(0)
        fspec = int(opt[1])
        optstr = optstr + '-F' + opt[1]
    elif opt[0] == '-a':
        save_agents = 1
    elif opt[0] == '-t':
        check_obtypes = 1
        objdir = opt[1] + '/'
        
openfilelist, counters, basepath = get_files(args)

counters.printself("")

if optstr:
    optstr += '.'
basepath += '%snp_rets' % (optstr)
try:
    os.mkdir(basepath)
    print 'Created results directory %s' % (basepath)
except OSError, s:
    if str(s).find('exists'):
        print 'Results directory %s already exists' % (basepath)
    else:
        print str(s)
        sys.exit(1)

filepath = basepath + '/'

counters.printself_tofile(filepath + 'counters', '')


# array for retvals etc
retvals = Numeric.zeros(MAX_RETCODE,)
methods = Numeric.zeros(MAX_METHOD,)

MAX_DUR = 10*60*1000 #5mins in ms
N_DUR_BINS = MAX_DUR+1 # 1ms bins
transperconn = []
for i in range(MAX_NTRANS+1):
    transperconn.append([0, Numeric.zeros(N_DUR_BINS+1,)])
    
totrans = 0
non_200 = 0
not_gets = 0
npers = npers1 = 0


obs = [{}, {}]
very_large_obs = []

errs = 0
notboth = 0
unfin = 0 #unfinished trans
nconns = 0

agents = WebAgents()
addagents = agents.add_h

if check_obtypes:
    filetypes = FileTypes(objdir=objdir, corr_by_magic=0, mlengths=[2,20,1], verbose=1, report_diff_files=0)
    checktype = filetypes.check_obtype
else:
    filetypes = checktype = None

reportpath = filepath + 'summary'
sumfile = open(reportpath, 'w')

sumfile.write('Summary of transaction analysis\n\n')
sumfile.write('===============================\n\n\n')

# re-usable
connrec, translist = allocate_http_reusable_objects()

for file in openfilelist:

    if len(openfilelist) > 1:
	print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	print "File %s - %d records" % (file.fnm, file.counters.nrecords)
	print
	sys.stdout.flush()

    while 1:
        
        rtype = file.next_type_rec(REC_TCP_HTTP)
        
        if rtype == -1: # EOF
            break
        elif rtype == REC_TCP_HTTP:
            translen = get_http_rec_and_trans(file, connrec, translist)
            #index, connrec, translist = get_http_rec(file)
        else:
            file.advance()
            continue

        if not accept_conn(connrec.flow_inner, fspec):
            continue

        nconns += 1

	if (not connrec.server_seen()) or (not connrec.client_seen()):
            notboth += 1
	    continue

        server = client = None
        
        dur = (ull2l(connrec.close()) - ull2l(connrec.open()))/1000
        if dur > MAX_DUR:
            bin = MAX_DUR+1
        else:
            bin = int(dur)
        transperconn[translen][0] += 1
        transperconn[translen][1][bin] += 1

        if connrec.http_persistent():
            npers += 1
            if translen == 1:
                npers1 += 1
        
        
        for i in range(translen):
            t = translist[i]
        
            
	    if t.http_serv_isdummytrans():
		break


            totrans += 1
            
	    if (not t.http_serv_isvalid()) or t.http_serv_iserr() or (not t.http_cli_isvalid()) or t.http_cli_iserr():
                errs += 1
		continue

            if not server:
                shost = connrec.dhost()
                sa = t.get_server()
            if not client:
                chost = connrec.shost()
                ua = t.get_uagent()

            addagents(shost, chost, sa, ua)
                    
            retcode = t.http_server_retcode()
            if retcode >= MAX_RETCODE:
                print 'Bad retcode', retcode
                errs += 1
                connrec.printself()
                for trans in translist:
			trans.printself(connrec)

            if retcode == 200:
                ok = 1
            else:
                ok = 0
                non_200 +=1

            method = t.http_meth()
            methods[method] += 1
            if method !=  HTTP_METHOD_GET:
                not_gets += 1


                
            obsz = t.http_obj_bytes()
        ##     if obsz == 0:
##                 #connrec.printself(0)
##                 #t.printself(connrec)
##                 zerolens += 1
##                 continue
 
            obtype = t.http_rep_objtype()

            if check_obtypes and obsz:
                checktype(obtype, '%d.%d' % (connrec.get_conn_id(), i))

            
            sstatus = t.http_serv_status()
            if not (sstatus & TRANS_FINISHED) and (sstatus & TRANS_INCOMPLETE):
                unfin += 1
                osz = obs[1]
            else:
                osz = obs[0]

            #
            # for each object type dict entry (keyed by type):
            # [total trans, size array, list of sizes > array max,
            #    tot non 200 responses, server response array, minsz, maxsz]
            #
            bins = osz.setdefault(obtype, [0, Numeric.zeros(NBINS,), [], \
                                           0, Numeric.zeros(MAX_RETCODE,), \
                                           BIGNUMBER, 0])
            bins[0] += 1
            if  ok:
                if obsz == 0:
                    bins[1][0] += 1
                elif obsz < OBSZ_MAX:
                    bins[1][(obsz/OBSZ_BIN) + 1] += 1
                else:
                    bins[2].append(obsz)
            else:
                bins[3] += 1

            bins[4][retcode] += 1
            bins[5] = min(bins[5], obsz)
            bins[6] = max(bins[6], obsz)

            if obsz > VERY_LARGE_OB_THRESH:
                very_large_obs.append((split(file.fnm)[1], connrec.get_conn_id(), i, center(http_server_objtype_string(obtype), 30), obsz))

#filetypes.report_diffs()

print 'Summary of transaction analysis'
print '===============================\n'


record(sumfile, '%10d %-15s' % (nconns, 'Total TCP/HTTP connections'))
record(sumfile, '%10d %-15s (%2.3f%% of total)\n' % (notboth, 'Unidirectional (rejected)', (notboth*100.0)/nconns))

record(sumfile, '%10d %-15s' % (totrans, 'Total transactions'))
record(sumfile, '%10d %-15s (%2.3f%%)' % (errs, 'Errors', (errs*100.0)/totrans))

nvalid = totrans - errs

record(sumfile, '%10d %-15s (%2.3f%%)\n' % (nvalid, 'Valid', (nvalid*100.0)/totrans))

record(sumfile, 'Of %d valid transactions:-\n' % (nvalid))
record(sumfile, '%10d %-15s (%2.3f%%)' % (unfin, 'Incomplete', (unfin*100.0)/nvalid))
record(sumfile, '%10d %-15s (%2.3f%%)' % (non_200, 'Non 200 (OK) return codes', (non_200*100.0)/nvalid))

ngets = totrans - not_gets

record(sumfile, '%10d %-15s (%2.3f%%)' % (ngets, 'GET requests', (ngets*100.0)/nvalid))
record(sumfile, '\n================================================================\n\n')

do_output(filepath, sumfile, obs, methods, agents, filetypes, nvalid, non_200, unfin, not_gets, save_agents, modeflag, showflag, transperconn, npers, npers1)

if very_large_obs:
    record(sumfile, '\n %d Very large objects (> %d bytes):-\n' % (len(very_large_obs), VERY_LARGE_OB_THRESH))
    print '(Listed in Summary)'
    write_file(sumfile, '%s%s%s%s\n' % (center('File', 20), center('Conn/trans', 15), center('Ob-type', 30), center('Size', 10)))
    for detail in very_large_obs:
        write_file(sumfile, '%-20s%11d/%-3d%s%10d' % detail)

print '\nAnalysis results written to ', filepath
