###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


##############################################################################
## 
##
## 
## 
##
## 

############################################################################


from __future__ import generators

import os
import sys
import getopt
import socket
from struct import unpack as up
from time import asctime, sleep, time
import traceback
from commands import getstatusoutput

import re
import pcap

from DMuxer import FlowDmuxer, FlowDmuxerException
from pcap import ntoa, aton
from pcap import _ntohs as ntohs
from pcap import _ether_ntoa as entoa
from pcap import _ether_aton as eaton
from pcap import py_pcap, nprobe_errstring

from dirwalk import walk



############################################################################
#############################################################################

#
# Link types recognised
#

DLT_EN10MB = 1
DLT_IEEE802 = 6
DLT_RAW = 12
DLT_RAW_HIGH = 101

DLT_NPROBE = 12321

#
# Dictionary of link types and functions to extract 'atuple' used in
# identifying flows
#

linktypes = {
    DLT_EN10MB: 'get_eth_atuple',
    DLT_IEEE802: 'get_eth_atuple',
    DLT_NPROBE: 'get_eth_atuple',
    DLT_RAW: 'get_raw_ip_atuple',
    DLT_RAW_HIGH: 'get_raw_ip_atuple'
    }

#
# IP and TCP flag values
#

IP_MF = 0x2000

TH_FIN = 0x01
TH_SYN = 0x02
TH_RST = 0x04
TH_PUSH = 0x08
TH_ACK = 0x10
TH_URG = 0x20

#
# packet direction indicators
#

CLIENT = 0
SERVER = 1

#
# Connection state flags used in spotting TCP closes
#

NO_STATE = 0
CSYN = 0x1
SSYN = 0x2
CSYNA = 0x4
SSYNA = 0x8
CFIN = 0x10
SFIN = 0x20
CFINA = 0x40
SFINA = 0x80

SYNS = (CSYN, SSYN)
SYNAS = (CSYNA, SSYNA)
FINS = (CFIN, SFIN)
FINAS = (CFINA, SFINA)

FINACKED = CFINA | SFINA

#
# IP protocols recognised
#

IPPROTO_ICMP = 1
IPPROTO_IGMP = 2
IPPROTO_IPIP = 4
IPPROTO_TCP = 6
IPPROTO_UDP = 17

#
# Dictionary of IP protocols and functions to build paths to flow directories
#

IP_PROTS = {
    IPPROTO_ICMP: ('ICMP', 'ip_path'),
    IPPROTO_IGMP: ('IGMP', 'ip_path'),
    IPPROTO_IPIP: ('IPIP', 'ip_path'),
    IPPROTO_TCP: ('TCP', 'tcpudp_path'),
    IPPROTO_UDP: ('UDP', 'tcpudp_path'),
    'FRAGS': ('FRAGS', 'ip_fragpath'),
    'OTHER': ('OTHER', 'ip_path')
    }

#
# Ethernet protocols recognised
#

ETHERTYPE_IP = 0x0800
ETHERTYPE_ARP = 0x0806
ETHERTYPE_REVARP = 0x8035

#
# Dictionary of Ethernet protocols and functions to build paths to
# flow directories
#

ETHERTYPES_D = {
    ETHERTYPE_IP: ('IP', IP_PROTS, 'nullpath'),
    ETHERTYPE_ARP: ('ARP', {}, 'arp_path'),
    ETHERTYPE_REVARP: ('REVARP', {}, 'arp_path'),
    'OTHER': ('OTHER', {}, 'arp_path')
    }

ETHER_HDR_LEN = 14
ETHERMTU = 1500

LLC_SNAP_HDR_LEN = 8
ETH_LLC_HDR_LEN = ETHER_HDR_LEN + LLC_SNAP_HDR_LEN
# Relative to packet start
#MAC_OFF = 0
#MAC_END = 14
ETH_PROT_OFF = 12
ETH_PROT_END = ETH_PROT_OFF + 2
SNAP_PROT_OFF = 20
SNAP_PROT_END = SNAP_PROT_OFF + 2

ETH_DH_OFF = 0
ETH_DH_END = ETH_DH_OFF + 6
ETH_SH_OFF = 6
ETH_SH_END = ETH_SH_OFF + 6

#
# Constant offsets and byte lengths (relative to hdr start) of IP header fields
#

IP_HDR_LEN = 20
IP_HL_OFF = 0
IP_IPLEN_OFF = 2
IP_IPLEN_END = IP_IPLEN_OFF + 2
IP_IPID_OFF = 4
IP_IPID_END = IP_IPID_OFF + 2
IP_IPOFF_OFF = 6
IP_IPOFF_END = IP_IPOFF_OFF + 2
IP_PROT_OFF = 9
IP_SA_OFF = 12
IP_SA_END = IP_SA_OFF + 4
IP_DA_OFF = 16
IP_DA_END = IP_DA_OFF + 4

#
# Constant offsets and byte lengths (relative to hdr start) of TCP/UDP
# port fields
#

TCPUDP_HDR_LEN = 20

TCPUDP_SP_OFF = 0
TCPUDP_SP_END = TCPUDP_SP_OFF + 2
TCPUDP_DP_OFF = 2
TCPUDP_DP_END = TCPUDP_DP_OFF + 2

#
# Constant offsets and byte lengths (relative to hdr start) of other TCP
# header fields used
#

TCP_SEQ_OFF = 4
TCP_SEQ_END = TCP_SEQ_OFF + 4
TCP_ACK_OFF = 8
TCP_ACK_END = TCP_ACK_OFF + 4

TCP_FLAGS_OFF = 13

ERRCODE_OFF = 4



############################################################################
#############################################################################

#
# A DLL to implement the flow timeout queue
#

class TimeoDLL:

    def __init__(self):

        self.n = self.p = self
        self.last = None
        self.len = 0

    def __len__(self):

        return self.len

    def ins_a(self, preob, ob):

        ob.n = preob.n
        preob.n = ob
        ob.p = preob
        ob.n.p = ob
        self.len += 1

    def ins_b(self, postob, ob):

        ob.n = postob
        ob.p = postob.p
        postob.p = ob
        ob.p.n = ob
        self.len += 1

    def prepend(self, ob):

        self.n.p = ob
        ob.n = self.n
        ob.p = self
        self.n = ob
        self.len += 1

    def append(self, ob):

        self.p.n = ob
        ob.p = self.p
        ob.n = self
        self.p = ob
        self.len += 1

    def rem(self, ob):

        
        ob.n.p = ob.p
        ob.p.n = ob.n
        self.len -= 1

    def toend(self, ob):

        self.rem(ob)
        self.append(ob)

    def isempty(self):

        return self.n == self

    def head(self):

        return self.n

    def tail(self):

        return self.p

    def pop(self):

        o = self.head()
        self.rem(o)
        return (o)

    def items(self):

        c = self.n
        while c != self:
            yield c
            c = c.n

    def check(self):

        for o in self.items():
            try:
                assert o.n.p == o
                assert o.p.n == o
            except AssertionError:
                print 'obj', o, 'o.n', o.n, 'o.p', o.p, 'p.n', o.p.n, 'n.p', o.n.p, 'otq', o.otq, 'ontimes', o.onotq, 'offtimes', o.offotq
                raise
            #if o.p.last:
                #assert o.p.last < o.last



############################################################################
#############################################################################

#
# A dummy timeout queue for operation without flow timeouts
#
                

class TimeoNull:

    def __init__(self):

        self.ins_a = self.ins_b = self.prepend = self.append = self.nullf
        self.rem = self.toend = self.nullf

        self.head = self.tail = self.pop = self.check = self.nullf

    def nullf(self, *args):

        return None

    def isempty(self):

        return 1

    def items(self):

        return []

    def __len__(self):

        return 0

############################################################################
#############################################################################

#
# One of these for each flow
#
        
class FlowOb:

    def __init__(self, atuple, ts):

        self.atuple = atuple # flow identifier
        self.start = ts
        self.file = None
        self.written = 0
        self.last = 0
        self.pn = -1
        
        self.state = NO_STATE   # used for TCP conns
        self.seqs = [None, None]# ditto
        
        self.held = [] # file/offset or un-dumped packets
        self.alts = [] # alternative identifiers to this flow (e.g. for IP frags)


############################################################################
#############################################################################

class FlowDmuxerDmux(FlowDmuxer):

    '''
    Class FlowDmuxerDmux (file FlowDmuxerDmux.py) - sub-class of FlowDmuxer

    Dmux all flows passing packet filter into tree (one file per flow) ordered
    by protocol, address, port etc. as appropriate.

    Options:

      --timeout=<t[us/ms/s/m/h]> Time out quiescent flows after t us/ms/s/m/h
                                 (no unit - us) - default is no timeout.
      --accept_nosyns            Accept TCP flows without SYN (default reject
                                 flows not commencing with at least one SYN).
      --check                    Check consistency of dmux tree on completion.
      --nprobe-errors=<n>        Denotes input file is Nprobe error packet dump
                                 - dmuxes by Nprobe error No. and originating
                                 host. Rank top n occurences by error, host and
                                 (error, host)

    A log file is written into the dmux tree base directory.
    
      '''

    def __init__(self, infile, filtexp, opath=None, raise_natural_exceptions=0, save_check_info=0, nprobe_errors=0):

        FlowDmuxer.__init__(self, infile, filtexp, opath, raise_natural_exceptions=raise_natural_exceptions)

        self.accept_nosyns = 0
        self.save_check_info = save_check_info
        self.npe = nprobe_errors

        self.pcap_next = self.pcap.next_o_tts

        self.flowdir = {}
        self.fragdir = {}
        self.pending_dumps = []

        self.ttimeo = TimeoNull()
        self.timeout = long(time())*1000000

        self.odir = os.path.join(self.odir, self.ofile) #always a dir
        if self.npe:
            str = '_nprobe_errors'
            self.errd = {}
        else:
            str = '_dmux'
        if not self.ofile:
            self.odir = os.path.join(self.odir, os.path.split(self.infile)[1]+str)
        else:
            self.odir = os.path.join(self.odir, os.path.split(self.infile)[1]+str)

        self.make_proto_dirs(self.odir)

        self.open_log()

        self.set_link()

        self.ipkts = 0
        self.dpkts = 0
        self.nconns = 0
        self.nosyn = 0
        self.truncated = 0 
        self.nopen = 0
        self.to = 0
        self.resumed = 0
        self.closed = 0
        self.frags = 0
        self.active_at_end = 0
        if self.save_check_info:
            self.oots = {}

        
############################################################################

    def __del__(self):
        
        
        FlowDmuxer.__del__(self)

        
############################################################################

    def set_link(self):

     try:
         self.get_atuple = getattr(self, linktypes[self.linktype])
     except KeyError:
         s = 'Can\'t recognise data link type (0x%x)' % (self.linktype)
         self.log(s)
         print s
         sys.exit(1)

     self.snaplen = self.pcap.snaplen()

     if self.npe:
         if self.linktype != DLT_NPROBE:
             s = 'Nprobe-errors specified but linktype is not DLT_NPROBE'
             print s
             self.log(s)
             sys.exit(1)
         else:
             self.get_atuple = self.get_err_atuple

        
############################################################################ 

    def make_proto_dirs(self, root):

        join = os.path.join
        dd = self.protd = {}
        dirs = []

        for p0, (s0, d, mp0) in ETHERTYPES_D.items():
            path0 = join(root, s0)
            dd[(p0, None)] = (path0, getattr(self, mp0)) 
            dirs.append(path0)
            for p1, (s1, mp1) in d.items():
                path1 = join(path0, str(p1))
                dd[(p0, p1)] = (path1, getattr(self, mp1)) 
                dirs.append(path1)

        for dir in dirs:
            try:
                os.makedirs(dir)
            except OSError, s:
                if str(s).find('File exists') < 0:
                    self.exception('can\'t create dir %s (%s)' % (dir, str(s)))
            
        
        
############################################################################

    def open_log(self):
        
        self.logfile = os.path.join(self.odir, 'log')
        try:
            self.logf = open(self.logfile, 'w')
        except IOError, s:
            self.exception('can\'t create  %s (%s)' % (logfile, str(s)))
        
        self.logf.write('\n\nLog for FlowDmuxerDmux:\n\n')
        self.logf.write('Run on file %s  by %s@%s on %s\n\n' % (self.infile, os.environ['LOGNAME'], os.environ['HOSTNAME'], asctime()))

        
############################################################################

    def log(self, s):

        self.logf.write(s + '\n')

        
############################################################################

    def report(self):

        for a in ('ipkts', 'dpkts', 'nosyn', 'truncated', 'frags', 'nconns', 'closed', 'to', 'resumed', 'active_at_end'):

            s = '%-15s %d' % (a, getattr(self, a))
            self.log(s)
            print s

        if self.npe:
            # rank errors
            by_e = {}
            by_h = {}
            all = []
            toterrs = 0

            for (host, err), n in self.errd.items():
                toterrs += n
                try:
                    by_e[err] += n
                except KeyError:
                    by_e[err] = n
                ## try:
##                     by_h[host] += n
##                 except KeyError:
##                     by_h[host] = n
                ehd = by_h.setdefault(host, [0, {}])
                ehd[0] += n
                ehd[1][err] = ehd[1].setdefault(err, 0) + n
                all.append((n, err, host))

            by_e = [(n, e) for e, n in by_e.items()]
            by_h_n = [(n, h, errs) for h, (n, errs) in by_h.items()]
            
            for l in [by_e, by_h_n, all]:
                l.sort()
                l.reverse()

            N = min(self.npe, len(by_e))
            s = '\nNprobe errors by error (top %d shown):\n\t%s %s %%\n' % (N, 'Errno.', 'No.'.center(6))
            print s
            self.log(s)
            for n, e in by_e[:N]:
                s = '\t%4d %6d %6.2f (%-s)' % (-e, n, (n*100.0)/toterrs, nprobe_errstring(e))
                print s
                self.log(s)

            N = min(self.npe, len(by_h_n))
            s = '\nNprobe errors by host (top %d shown):\n\t%s %s %s\n' % (N, 'Host'.center(18), 'Total'.center(6), 'Errors and number'.center(24))
            print s
            self.log(s)
            for n, h, errs in by_h_n[:self.npe]:
                s = '\t%18s %6d' % (ntoa(h), n)
                el = [(n, e) for e, n in errs.items()]
                el.sort()
                el.reverse()
                for ne, e in el:
                    s += ' (%d %d)' % (-e, ne)
                print s
                self.log(s)

            N = min(self.npe, len(all))
            s = '\nNprobe errors by host and error (top %d shown):\n\t%s %s %s\n' % (N, 'Errno.', 'Host'.center(18), 'No.'.center(6))
            print s
            self.log(s)
                
            for n, e, h in all[:self.npe]:
                s = '\t%4d %18s %6d' % (-e, ntoa(h), n)
                print s
                self.log(s)
            print

        print 'Dmux tree written in', self.odir

############################################################################

    def getopts(self, opts):

        
        dmuxer_opts = []

        try:
            optlist, args = getopt.getopt(opts, '', ['accept_nosyns', 'timeout=', 'float_timestamps', 'nprobe-errors'])
        except getopt.error, s:
            #print 'ouch'
            self.exception('FlowDmuxerDmux getopt() ' + str(s))

        for opt, par in optlist:

            if opt == '--accept_nosyns':
                self.accept_nosyns = 1

            elif opt == '--timeout':
                if self.npe:
                    raise FlowDmuxerException('Timeout specified with Nprobe error dump input')
                for unit, mult in (('us', 1), ('ms', 1e3), ('s', 1e6), ('m', 60*1e6), ('h', 60*60*1e6)):
                    pos = par.find(unit)
                    if pos >=0:
                        break
                try:
                    if pos > 0:
                        self.timeout = long(par[:pos])*long(mult)
                    else:
                        self.timeout = long(par)
                except ValueError, s:
                    self.exception('getopt() ' + str(s))
                self.ttimeo = TimeoDLL()
            else:
                if par:
                    eq = '='
                else:
                    eq = ''
                    dmuxer_opts.append(opt+eq+par)
                
        FlowDmuxer.getopts(self, dmuxer_opts)
        
############################################################################

    def make_path(self, t, ts):

        #
        # Call one of the fn.s below (as determined by protd dir lookup)
        # to construct the appropriate dir name for a flow
        #

        ph =  self.protd[t[0]]

        return ph[1](ph[0], t, ts)

    #
    # create protocol-dependant dmux tree dir names 
    #
    

    def tcpudp_path(self, dir, t, ts):

        if self.npe:
            dir = '%s/%d' % (dir, -t[2])
            p = '%s/%s' % (dir, ntoa(t[1]))
        else:
            dir = '%s/%s/%s' % (dir, ntoa(t[1]), ntoa(t[2]))
            p = '%s/%u.%06u-%u-%u' % (dir, ts/1000000, ts%1000000, t[3], t[4])
            
        if not os.path.isdir(dir):
            os.makedirs(dir)
            
        return p

    def ip_fragpath(self, dir, t, ts):

        dir = '%s/%s/%d' % (dir, ntoa(t[1]), t[2])
        if not os.path.isdir(dir):
            os.makedirs(dir)
        p = '%s/%u.%06u' % (dir, ts/1000000, ts%1000000)
        return p

    def ip_path(self, dir, t, ts):

        if self.npe:
            dir = '%s/%d' % (dir, -t[2])
            p = '%s/%s' % (dir, ntoa(t[1]))
        else:
            dir = '%s/%s/%s' % (dir, ntoa(t[1]), ntoa(t[2]))
            p = '%s/%u.%06u' % (dir, ts/1000000, ts%1000000)
            
        if not os.path.isdir(dir):
            os.makedirs(dir)
            
        return p

    def arp_path(self, dir, t, ts):
        dir = '%s/%s/%s' % (dir, entoa(t[1]), entoa(t[2]))
        #print dir
        if not os.path.isdir(dir):
            os.makedirs(dir)
        p = '%s/%u.%u' % (dir, ts/1000000, ts%1000000)
        return p
            

    def nullpath(dir, t, ts):

        return None
        
############################################################################

    def tidy(self):

        # dump any timed out flows
        if len(self.ttimeo):
            ts = self.ttimeo.tail().last
            self.timeo(ts)
            self.active_at_end = len(self.ttimeo)
            remainder = self.ttimeo.items()
        else:
            self.active_at_end = len(self.flowdir)
            remainder = self.flowdir.itervalues()
            

        freeable = 0
        #for o in self.ttimeo.items():
        for o in remainder:
            if o.file:
                freeable = 1
                break

        if len(self.ttimeo):
            ts = self.tm + self.timeout + 1
            self.timeo(ts)
            self.to -= self.active_at_end
        else:
            self.dump_flowdir()

        if freeable:
            assert len(self.pending_dumps) == 0
        else:
            for flow in self.pending_dumps:
                self.dump_flow(flow)

        self.pending_dumps = []

        try:
            assert len(self.flowdir) == len(self.ttimeo)
            assert not self.flowdir
            assert self.ttimeo.isempty()
            assert not self.pending_dumps
            assert self.nopen == 0
            assert self.dpkts == self.ipkts - self.nosyn -self.truncated
        except AssertionError, s:
            tbs = traceback.format_list(traceback.extract_tb(sys.exc_info()[2]))
            self.exception('tidy - assertion error\n%s' % (tbs[0]))


        fragsdir = self.protd[(ETHERTYPE_IP, 'FRAGS')][0]
        frags = 0
        for root, dirs, files in walk(fragsdir, topdown=0):
            if not (files or dirs):
                os.rmdir(root)
            else:
                frags += len(files)

        if frags:
            str = 'Caution - unresolved IP fragments (%d)' % (frags)
            print str
            self.log(str)


        resdir = self.odir
        for root, dirs, files in walk(resdir, topdown=0):
            if not (files or dirs):
                os.rmdir(root)

            
            

        self.logf.flush()

        
############################################################################

    def dump_held(self, flow, f):

        pcap = self.pcap
        seek = pcap.fseek
        next = self.pcap_next
        dump = pcap.alt_dump

        osave = pcap.ftell()

        for off in flow.held:
            try:
                #print 'off', off
                seek(off)
                next()
                dump(f)
                self.dpkts += 1
            except (TypeError, IOError), s:
                self.exception('dump_held() loop ' + str(s))

        seek(osave)
        flow.held = []
        

        
############################################################################

    def dump_flow(self, flow):

        try:
            if flow.written:
                f = open(flow.path, 'a')
            else:
                f = self.pcap.alt_dump_open(flow.path)
                flow.written = 1
        except IOError, s:
            self.exception(str(s))

        self.dump_held(flow, f)
        
        f.close()

        
############################################################################

    def dump_flowdir(self): 

        freed = 0
        dump = self.dump_flow
        dir = self.flowdir
        pending = self.pending_dumps

        npe = self.npe
        if npe:
            ed = self.errd

        for flow in dir.itervalues():

            try:
                if flow.dumped:
                    continue
            except AttributeError:
                flow.dumped = 1

            if npe:
                fa = flow.atuple
                ed[(fa[1], fa[2])] = flow.pn + 1
            
            if flow.file:
                flow.file.close()
                flow.file = None
                self.nopen -= 1
                freed += 1
            elif flow.held:
                if freed > 0:
                    dump(flow)
                else:
                    pending.append(flow)

        if freed:
            for fl in pending:
                dump(fl)
            pending = []

        self.flowdir = {}

        return freed

        
############################################################################

    def timeo(self, ts): 

        tto = self.ttimeo
        empty = tto.isempty
        head = tto.head
        rem = tto.rem
        to = self.timeout
        freed = 0
        dump = self.dump_flow
        dir = self.flowdir
        pending = self.pending_dumps

        while not empty():

            flow = head()
            if flow.last > ts - to:
                break
            
            rem(flow)
            del dir[flow.atuple]
            for ref in flow.alts:
                del dir[ref]
            self.to += 1
            
            if flow.file:
                flow.file.close()
                flow.file = None
                self.nopen -= 1
                freed += 1
            elif flow.held:
                if freed > 0:
                    dump(flow)
                else:
                    pending.append(flow)

        if freed:
            for fl in pending:
                dump(fl)
            pending = []

        return freed
                

        
        
############################################################################
            
   
    def timesubst(self, f1, f2):

        if f1.atuple[0][1] == 'FRAGS':
            fragpath = f1.path
            flowpath = f2.path
        elif f2.atuple[0][1] == 'FRAGS':
            fragpath = f2.path
            flowpath = f1.path
        else:
            raise FlowDmuxerException('timesubst(): can\'t identify fragments file')

        m = re.match('.*/IP/FRAGS/(\d+\.){3}\d+/\d+/(?P<ts>\d+\.\d+)', fragpath)
        if not m:
            raise FlowDmuxerException('timesubst() - couldn\'t extract timestamp from frags file path')
        frag_ts = m.group('ts')

        m = re.match('.*/IP/(6|(17))/(\d+\.){3}\d+/(\d+\.){3}\d+/(?P<ts>\d+\.\d+)-\d+-\d+', flowpath)
        if not m:
            raise FlowDmuxerException('timesubst() - couldn\'t extract timestamp from flow file path')
        flow_ts = m.group('ts')

        fgts = float(frag_ts)
        flts = float(flow_ts)

        if flts > fgts:
            return re.sub(flow_ts, frag_ts, flowpath)
        else:
            return flowpath 
                

        
        
############################################################################
            
   
    def subst_flow(self, toflow, flow):

       topath = toflow.path
       fpath = flow.path
       newpath = fpath

       subst_frags = toflow.atuple[0] == (ETHERTYPE_IP, 'FRAGS')
        
       if flow.file:
           flow.file.close()
           self.nopen -= 1
           flow.file = None
       if subst_frags:
           if toflow.written:
               newpath = self.timesubst(toflow, flow)
               os.rename(toflow.path, newpath)
               os.remove(flow.path)
           toflow.path = newpath
           toflow.alts.append(toflow.atuple)
           for a in flow.alts:
               self.flowdir[a] = toflow
           toflow.atuple = flow.atuple
            
       self.flowdir[flow.atuple] = toflow    
       self.ttimeo.rem(flow)
       self.ttimeo.toend(toflow)
       for alt in flow.alts:
           if alt != toflow.atuple:
               toflow.alts.append(alt)

       self.log('Flow %s subst for %s as %s' % (topath, fpath, newpath))
       
       return toflow  
        
        
############################################################################
            
   
    def pre_frags(self, fragflow, flow, ts):
        
        if not (flow.written or flow.held):
            res = self.subst_flow(fragflow, flow)
            return res
        else:
            res = self.frags_merge(fragflow, flow, ts)
            return res
            
        
############################################################################
            
   
    def frags_merge(self, fragflow, flow, ts):

        def new_pf(infnm):

            pf = py_pcap()
            while 1:
                try:
                    pf.open_offline(infnm)
                    self.nopen += 1
                    break
                except TypeError, s:
                    print 'ex', str(s),
                    if str(s).find('Too many open files') >= 0:
                        self.close_a_flow(force=1)
                    else:
                        if str(s).find('empty') >= 0:
                            sleep(30)
                            continue
                        else:
                            raise
           #print 'got'
            return pf

        def copy(fr, to): 
            pf = new_pf(fr)
            next = pf.next_tts
            dump = pf.alt_dump
            n = 0
            while 1:
                try:
                    sys.stdout.flush()
                    next()
                    sys.stdout.flush()
                    dump(to)
                    sys.stdout.flush()
                    n += 1
                    sys.stdout.flush()
                except EOFError:
                    break
                except TypeError, s:
                    self.exception('frags_merge() copy loop: ' + str(s))
            del pf
            self.nopen -= 1
            
            return n

        def append_merge(toflow, aflow):
            
            if aflow.written:
                if aflow.file:
                    aflow.file.close()
                    aflow.file = None
                    self.nopen -= 1
                if not toflow.file:
                    toflow.file = self.reopen_flow(toflow)
                toflow.pn += copy(aflow.path, toflow.file)

            toflow.pn += len(aflow.held)
            if not self.stdin:
                if toflow.file:
                    self.dump_held(aflow, toflow.file)
                else:
                    toflow.held.extend(aflow.held)
            else:
                assert not aflow.held
            self.subst_flow(toflow, aflow)

            return toflow

        def full_merge(flow, fragflow):

            def next(pc):

                if pc == pc0 or pc == pc1:
                    if pc == pc0:
                        held = held0
                    else:
                        held = held1
                    try:
                        off = held.pop(0)
                    except IndexError:
                        return None
                    try:
                        pc.fseek(off)
                        return pc.next_fts()
                    except (TypeError, IOError), s:
                        self.exception('full_merge() pcap offset-read ' + str(s))

                else:
                    try:
                        return pc.next_fts()
                    except (TypeError, IOError), s:
                        self.exception('full_merge() pcap read ' + str(s))
                    except EOFError:
                        return None

            def pop_first(plist):
                last = plist[0][0]
                for p in plist:
                    if p[0] <= last:
                        it = p
                plist.remove(it)
                return it

            pcaps = []
            held0 = [off for off in flow.held]
            held1 = [off for off in fragflow.held]
            if held0:
                pc0 = new_pf(self.infile)
                pcaps.append(pc0)
                flow.held = []
            else:
                pc0 = None
            if held1:
                pc1 = new_pf(self.infile)
                pcaps.append(pc1)
            else:
                pc1 = None
            if flow.written:
                if flow.file:
                    flow.file.close()
                    flow.file = None
                    self.nopen -= 1
                pct = new_pf(flow.path)
                pcaps.append(pct)
            if fragflow.written:
                if fragflow.file:
                    os.fsync(fragflow.file.fileno())
                    fragflow.file.close()
                    fragflow.file = None
                    self.nopen -= 1
                pca = new_pf(fragflow.path)
                pcaps.append(pca)

            newfnm = self.timesubst(flow, fragflow)

            tmpfnm = os.tmpnam()
            try:
                f = self.pcap.alt_dump_open(tmpfnm)
                self.nopen += 1
            except (TypeError, IOError), s:
                self.exception('full_merge() ' + str(s))
            
            pkts = []
            npcaps = len(pcaps)
            for p in pcaps:
                stuff = next(p)
                if stuff:
                    pkts.append((stuff[2], p))
                else:
                    npcaps -= 1
                    
            pn = 0
            while npcaps:
                ts, p = pop_first(pkts)
                p.alt_dump(f)
                pn += 1
                stuff = next(p)
                if stuff:
                    pkts.append((stuff[2], p))
                else:
                    #pcaps.remove(p)
                    npcaps -= 1
                    del p

            for p in pcaps:
                del p
                self.nopen -= 1

            os.remove(fragflow.path)
            if flow.path != newfnm:
                os.remove(flow.path)
            f.close()
            self.nopen -= 1
            try:
                os.rename(tmpfnm, newfnm)
            except OSError, s:
                if str(s).find('Invalid cross-device link') >= 0:
                    status, output = getstatusoutput('mv %s %s' % (tmpfnm, newfnm))
                    if status:
                        raise FlowDmuxerError('full_merge() mv failure (\'%s\')' % (output))
            self.log('merged %s and %s as %s' % (flow.path, fragflow.path, newfnm))
            flow.path = newfnm
            self.reopen_flow(flow)
            flow.written = 1
            flow.pn = pn
            self.flowdir[fragflow.atuple] = flow
            flow.alts.append(fragflow.atuple)
            self.ttimeo.rem(fragflow)
            self.ttimeo.toend(flow)
            del fragflow

            return flow

        ffrag_tm = fragflow.start
        lfrag_tm = fragflow.last

        fflow_tm = flow.start
        lflow_tm = flow.last

        if lfrag_tm < fflow_tm:
            return append_merge(fragflow, flow)   
        elif lflow_tm < ffrag_tm:
            return append_merge(flow, fragflow)
        else:
            return full_merge(flow, fragflow)
        
############################################################################

    def new_flow(self, t, ts):

        flow = FlowOb(t, ts)

        self.nconns += 1
        p = self.make_path(t, ts)
        while 1:
            try:
                flow.file = self.pcap.alt_dump_open(p)
                flow.written += 1
                self.nopen += 1
                break
            except IOError, s:
                if str(s).find('Too many open files') >= 0:
                    if not self.close_a_flow():
                        break
                else:
                    raise

        flow.path = p
        self.flowdir[t] = flow
        self.ttimeo.append(flow)

        return flow
            
        
############################################################################

    def get_flow(self, ts, atuple, SYN, tcp):

        flow = None
        way = 2
        for t in [(atuple[0], atuple[2], atuple[1], atuple[4], atuple[3]), atuple]:
            try:
                way -= 1
                flow = self.flowdir[t]
                break
            except KeyError:
                pass
        if flow:
            if flow.last < ts - self.timeout:
                self.to += 1
                if tcp and not SYN:
                    self.resumed += 1
                    self.nconns -= 1
                else:
                    pass
                self.end_flow(flow)
                flow = self.new_flow(atuple, ts)
            else:
                self.ttimeo.toend(flow)

        else:
            if SYN or self.accept_nosyns:
                flow = self.new_flow(t, ts)
                way = CLIENT
            else:
                self.nosyn += 1

        return flow, way
            
        
############################################################################

    def close_a_flow(self, force=0):

        if self.stdin:
            force = 1

        closed = self.timeo(self.tm)

        if (not closed) and force:
            for flow in self.ttimeo.items():
                if flow.file:
                    flow.file.close()
                    flow.file = None
                    self.nopen -= 1
                    closed = 1
                    break

        if (not closed) and force:
            for flow in self.flowdir.itervalues():
                if flow.file:
                    flow.file.close()
                    flow.file = None
                    self.nopen -= 1
                    closed = 1
                    break
                

        return closed
            
############################################################################

    def end_flow(self, flow):

        self.ttimeo.rem(flow)
        del self.flowdir[flow.atuple]
        for ref in flow.alts:
            del self.flowdir[ref]
        if flow.file:
            flow.file.close()
            flow.file = None
            self.nopen -= 1
        elif flow.held:
            self.pending_dumps.append(flow)


############################################################################

    def reopen_flow(self, flow):

        while 1:
            try:
                flow.file = open(flow.path, 'a')
                self.nopen += 1
                break
            except IOError, s:
                if str(s).find('Too many open files') >= 0:
                    self.close_a_flow(force=1)
                else:
                    raise

        if flow.held:
            self.dump_held(flow, flow.file)

        return flow.file
        
############################################################################

    def get_eth_atuple(self, pkt):

        len = min(pkt[1], self.snaplen)
        if len < ETHER_HDR_LEN:
            self.truncated += 1
            return (None, None, None, None)
        
        d = pkt[2]

        p0 = up('!H', d[ETH_PROT_OFF:ETH_PROT_END])[0]
        if p0 < ETHERMTU:
            if len < ETH_LLC_HDR_LEN:
                self.truncated += 1
                return (None, None, None, None)
            p0 = up('!H', d[SNAP_PROT_OFF:SNAP_PROT_END])[0]
            start = ETH_LLC_HDR_LEN
        else:
            start = ETHER_HDR_LEN
        
        if p0 == ETHERTYPE_IP:

            if len < start + IP_HDR_LEN:
                self.truncated += 1
                return (None, None, None, None)
            
            sa = up('i', d[start+IP_SA_OFF:start+IP_SA_END])[0]
            da = up('i',d[start+IP_DA_OFF:start+IP_DA_END])[0]
            iphl = (ord(d[start+IP_HL_OFF]) & 0x0f) << 2
            ipo = up('!H',d[start+IP_IPOFF_OFF:start+IP_IPOFF_END])[0]
            ipoff = (ipo & 0x1fff) << 3
            if ipoff != 0:
                id = up('!H',d[start+IP_IPID_OFF:start+IP_IPID_END])[0]
                iplen = up('!H', d[start+IP_IPLEN_OFF:start+IP_IPLEN_END])[0]
                return (((ETHERTYPE_IP, 'FRAGS'), sa, id, None, None), 0, None, (da, id, ipoff, iplen-iphl, ipo & IP_MF))

            elif ipo & IP_MF:
                id = up('!H',d[start+IP_IPID_OFF:start+IP_IPID_END])[0]
                iplen = up('!H', d[start+IP_IPLEN_OFF:start+IP_IPLEN_END])[0]
                fragt = (da, id, ipoff, iplen-iphl, 1)
            else:
                fragt = None
            
            p1 = ord(d[start+IP_PROT_OFF])
            
            if p1 == IPPROTO_TCP or p1 == IPPROTO_UDP:
                prot = (ETHERTYPE_IP, p1)
                start += iphl
                if len < start + TCPUDP_HDR_LEN:
                    self.truncated += 1
                    return (None, None, None, None)
                sp = up('!H', d[start+TCPUDP_SP_OFF: start+TCPUDP_SP_END])[0]
                dp = up('!H', d[start+TCPUDP_DP_OFF: start+TCPUDP_DP_END])[0]
            elif self.protd.has_key((ETHERTYPE_IP, p1)):
                prot = (ETHERTYPE_IP, p1)
                sp = dp = None
            else:
                prot = (ETHERTYPE_IP, 'OTHER')
                sp = dp = None
                
            return ((prot, sa, da, sp, dp), p1 == IPPROTO_TCP, start, fragt)

        elif self.protd.has_key((p0, None)):
            prot = (p0, None)
        else:
            prot = ('OTHER', None)
            
        dh = d[ETH_DH_OFF:ETH_DH_END]
        sh = d[ETH_SH_OFF:ETH_SH_END]
        
        return ((prot, sh, dh, None, None), 0, 0, None)      
        
############################################################################
            
   
    def get_err_atuple(self, pkt):

        len = min(pkt[1], self.snaplen)
        if len < ETHER_HDR_LEN:
            self.truncated += 1
            return (None, None, None, None)
        
        d = pkt[2]

        p0 = up('!H', d[ETH_PROT_OFF:ETH_PROT_END])[0]
        start = ETHER_HDR_LEN

        err = -ord(d[ERRCODE_OFF])
        
        if p0 == ETHERTYPE_IP:

            if len < start + IP_HDR_LEN:
                self.truncated += 1
                return (None, None, None, None)
            
            sa = up('i', d[start+IP_SA_OFF:start+IP_SA_END])[0]
            iphl = (ord(d[start+IP_HL_OFF]) & 0x0f) << 2
            
            p1 = ord(d[start+IP_PROT_OFF])
            
            if p1 == IPPROTO_TCP or p1 == IPPROTO_UDP:
                prot = (ETHERTYPE_IP, p1)
                start += iphl
            elif self.protd.has_key((ETHERTYPE_IP, p1)):
                prot = (ETHERTYPE_IP, p1)
            else:
                prot = (ETHERTYPE_IP, 'OTHER')
                
            return ((prot, sa, err, None, None), 0, start, None)

        else:
            #
            # No point in looking at others as no MAC-level addressing
            # supplied
            #
            return (None, None, None, None)
        
############################################################################

    def get_raw_ip_atuple(self, pkt):
        
        d = pkt[2]
        start = 0

        len = min(pkt[1], self.snaplen)
        
        if len < IP_HDR_LEN:
            self.truncated += 1
            return (None, None, None, None)
            
        sa = up('i', d[IP_SA_OFF:IP_SA_END])[0]
        da = up('i',d[IP_DA_OFF:IP_DA_END])[0]
        iphl = (ord(d[IP_HL_OFF]) & 0x0f) << 2
        ipo = up('!H',d[IP_IPOFF_OFF:IP_IPOFF_END])[0]
        ipoff = (ipo & 0x1fff) << 3
        if ipoff != 0:
            id = up('!H',d[IP_IPID_OFF:IP_IPID_END])[0]
            iplen = up('!H', d[IP_IPLEN_OFF:IP_IPLEN_END])[0]
            return (((ETHERTYPE_IP, 'FRAGS'), sa, id, None, None), 0, None, (da, id, ipoff, iplen-iphl, ipo & IP_MF))

        elif ipo & IP_MF:
            id = up('!H',d[IP_IPID_OFF:IP_IPID_END])[0]
            iplen = up('!H', d[IP_IPLEN_OFF:IP_IPLEN_END])[0]
            fragt = (da, id, ipoff, iplen-iphl, 1)
        else:
            fragt = None

        p1 = ord(d[IP_PROT_OFF])

        if p1 == IPPROTO_TCP or p1 == IPPROTO_UDP:
            prot = (ETHERTYPE_IP, p1)
            start = iphl
            if len < start + TCPUDP_HDR_LEN:
                self.truncated += 1
                return (None, None, None, None)
            sp = up('!H', d[start+TCPUDP_SP_OFF: start+TCPUDP_SP_END])[0]
            dp = up('!H', d[start+TCPUDP_DP_OFF: start+TCPUDP_DP_END])[0]
        elif self.protd.has_key((ETHERTYPE_IP, p1)):
            prot = (ETHERTYPE_IP, p1)
            sp = dp = None
        else:
            prot = (ETHERTYPE_IP, 'OTHER')
            sp = dp = None

        return ((prot, sa, da, sp, dp), p1 == IPPROTO_TCP, start, fragt)
        
############################################################################
            
   
    def process_pkt(self, pkt):

        self.ipkts += 1

        ts = pkt[3]
        ts = ts[0]*1000000 + ts[1]
        self.tm = ts
        d = pkt[2]

        atuple, tcp, start, fragt = self.get_atuple(pkt)
        if not atuple:
            return

        if tcp:
            flags = ord(d[start+TCP_FLAGS_OFF]) & 0x3f
            SYN = flags & TH_SYN
        else:
            SYN = 1

        flow, way = self.get_flow(ts, atuple, SYN, tcp)

        if fragt: # fragmented IP datagram
            self.frags += 1
            sa = atuple[0][1]
            da, id, ipoff, len, MF = fragt
            if not ipoff: # first frag
                at = ((ETHERTYPE_IP, 'FRAGS'), sa, id, None, None)
                if self.flowdir.has_key(at): # some trailing frags already got
                    fragflow = self.flowdir[at]
                    if fragflow != flow and ts - self.timeout < fragflow.last:
                        # not already merged and not timed out
                        # merge received frags
                        flow = self.pre_frags(self.flowdir[at], flow, ts)
                        self.nconns -= 1
                else:
                    # this is first to arrive - mark for trailing frags
                    self.flowdir[at] = flow
                    flow.alts.append(at)
                    
        if not flow:
            return

        flow.pn += 1
        if flow.last > ts:
            self.log('Unordered time stamp %s Pkt. No. %d' % (flow.path, flow.pn))
            if self.save_check_info:
                self.oots[hash(tuple(d))] = 1
        flow.last = ts
        f = flow.file

        if f:
            self.pcap.alt_dump(f)
            flow.written = 1
            self.dpkts += 1
        else:
            if self.stdin:
                f = self.reopen_flow(flow)
                self.pcap.alt_dump(f)
                self.dpkts += 1
            else:
                flow.held.append(pkt[0])

        if tcp:

            if flags & TH_FIN:
                flow.state |= FINS[way]
                flow.seqs[way] = up('!I', d[start+TCP_SEQ_OFF:start+TCP_SEQ_END])[0]

            if (flow.state & FINS[not way]) and (flags & TH_ACK):
                if (up('!I', d[start+TCP_ACK_OFF:start+TCP_ACK_END])[0] == flow.seqs[not way] + 1):
                    flow.state |= FINAS[not way]

            if flow.state & FINACKED:
                self.closed += 1
                self.end_flow(flow)
        
############################################################################

    #
    # Optional fn called at end to check consistency of dmux tree contents
    #
            
    def check(self, quiet=1):

        #
        # Find any log entries pertinent to a check fail
        #

        def get_log(ff, quiet=1):

            try:
                lf = open(self.logfile, 'r')
            except IOError, s:
                self.exception('Check get_log() ' + str(s))
            lines = []
            for l in lf.readlines():
                if l.find(ff) >= 0:
                    lines.append(l)

            if not quiet:
                if lines:
                    print 'Pertinent log entries:-'
                    for l in lines:
                        print l
                else:
                    print 'No pertinent log entries'

            return lines

        #
        # Translate directory path matching re groups to consistency check data
        #

        def base_ip_addrs_ports(m):

            return (aton(m.group('a1')), aton(m.group('a2')), int(m.group('tss'))*1000000 +  int(m.group('tsus')), int(m.group('p1')), int(m.group('p2')))

        def base_ip_addrs(m):

            return (aton(m.group('a1')), aton(m.group('a2')), int(m.group('tss'))*1000000 +  int(m.group('tsus')))

        def base_ip_frags(m):

            return (aton(m.group('a1')), int(m.group('a2')), int(m.group('tss'))*1000000 +  int(m.group('tsus')))

        def base_eth_addrs(m):

            return (eaton(m.group('a1')), eaton(m.group('a2')), int(m.group('tss'))*1000000 +  int(m.group('tsus')))

        def base_errs(m):
            
            return(int(m.group('err')), aton(m.group('a')))


        #
        # Flow-type consistency checking routines
        #

        def check_ip_addrs_ports((a1, a2, fts, p1, p2), pkt, ts, np):

            t = self.get_atuple(pkt)
            sa = t[0][1]
            da = t[0][2]
            fragt = t[3]

            if fragt:
                ipoff = fragt[2]
                da = fragt[0]
            else:
                ipoff = 0
                
            if ipoff == 0:
                sp = t[0][3]
                dp = t[0][4]
                
            try:
                if np == 0 and ipoff == 0:
                    assert ts == fts
                    assert sa == a1
                    assert da == a2
                    assert sp == p1
                    assert dp == p2
                elif ipoff == 0:
                    assert sa == a1 or sa == a2
                    assert da == a1 or da == a2
                    assert sp ==p1 or sp == p2
                    assert dp == p1 or dp == p2
                else:
                    assert sa == a1 or sa == a2
                    assert da == a1 or da == a2
                    
            except AssertionError:
                tbs = traceback.format_list(traceback.extract_tb(sys.exc_info()[2]))
                #self.exception('Check failed file %s packet No. %d\n%s' % (p, np, tbs[0]))
                print 'Check failed file %s packet No. %d\n%s' % (p, np, tbs[0])
                get_log(p)
                return 1
            return 0

        def check_ip_addrs((a1, a2, fts), pkt, ts, np):

            t = self.get_atuple(pkt)
            sa = t[0][1]
            da = t[0][2]

            try:
                if np == 0:
                    assert ts == fts
                    assert sa == a1
                    assert da == a2
                else:
                    assert sa == a1 or sa == a2
                    assert da == a1 or da == a2
            except AssertionError:
                tbs = traceback.format_list(traceback.extract_tb(sys.exc_info()[2]))
                print 'Check failed file %s packet No. %d\n%s' % (p, np, tbs[0])
                get_log(p)
                return 1
            return 0

        def check_ip_frags((a, id, fts), pkt, ts, np):

            t = self.get_atuple(pkt)
            sa = t[0][1]
            sid = t[3][1]

            try:
                assert sa == a
                assert sid == id
                if np == 0:
                    assert ts == fts
            except AssertionError:
                tbs = traceback.format_list(traceback.extract_tb(sys.exc_info()[2]))
                print 'Check failed file %s packet No. %d\n%s' % (p, np, tbs[0])
                get_log(p)
                return 1
            return 0

        def check_eth_addrs((a1, a2, fts), pkt, ts, np):


            d = pkt[2]
            da = d[ETH_DH_OFF:ETH_DH_END]
            sa = d[ETH_SH_OFF:ETH_SH_END]

            try:
                if np == 0:
                    assert ts == fts
                    assert sa == a1
                    assert da == a2
                else:
                    assert sa == a1 or sa == a2
                    assert da == a1 or da == a2
            except AssertionError:
                tbs = traceback.format_list(traceback.extract_tb(sys.exc_info()[2]))
                print 'Check failed file %s packet No. %d\n%s' % (p, np, tbs[0])
                get_log(p)
                raw_input('...')
                return 1
            return 0

        def check_errs((err, addr), pkt, ts, np):

            t = self.get_atuple(pkt)
            try:
                assert err == -t[0][2]
                assert addr == t[0][1]
            except AssertionError:
                tbs = traceback.format_list(traceback.extract_tb(sys.exc_info()[2]))
                print 'Check failed file %s packet No. %d\n%s' % (p, np, tbs[0])
                return 1
            return 0

        #
        # Main check fn starts here
        #

        print 'Checking dmux tree for consistency'

        nconns = 0
        npkts = 0
        nerrors = 0

        frag_nto = []

        #
        # re.s to match flow-type dependant directory paths
        #
        
        ip_addrs_ports = re.compile('.*/IP/(6|(17))/(?P<a1>(\d+\.){3}\d+)/(?P<a2>(\d+\.){3}\d+)/(?P<tss>\d+)\.(?P<tsus>\d+)-(?P<p1>\d+)-(?P<p2>\d+)')
        ip_addrs = re.compile('.*/IP/(\d+|OTHER)/(?P<a1>(\d+\.){3}\d+)/(?P<a2>(\d+\.){3}\d+)/(?P<tss>\d+)\.(?P<tsus>\d+)(?P<p1>)(?P<p2>)')
        ip_frags = re.compile('.*/IP/FRAGS/(?P<a1>(\d+\.){3}\d+)/(?P<a2>\d+)/(?P<tss>\d+)\.(?P<tsus>\d+)(?P<p1>)(?P<p2>)')
        
        eth_addrs = re.compile('.*/(ARP|REVARP|OTHER)/(?P<a1>([\dabcdef]+:){5}[\dabcdef]+)/(?P<a2>([\dabcdef]+:){5}[\dabcdef]+)/(?P<tss>\d+)\.(?P<tsus>\d+)(?P<p1>)(?P<p2>)')

        err_dirs = re.compile('.*/IP/(\d+|OTHER)/(?P<err>\d+)/(?P<a>(\d+\.){3}\d+)')

        #
        # Dictionary of directory->data mapping and packet-checking fns
        # keyed by path re match
        #

        if self.npe:
            checks = [(err_dirs, base_errs, check_errs)]
        else:
            checks = [
                (ip_addrs_ports, base_ip_addrs_ports, check_ip_addrs_ports),
                (ip_addrs, base_ip_addrs, check_ip_addrs),
                (ip_frags, base_ip_frags, check_ip_frags),
                (eth_addrs, base_eth_addrs, check_eth_addrs)
                ]

        for root, dirs, files in walk(self.odir):

            for f in files:

                if f == 'log':
                    continue

                ppc = pcap.py_pcap()
                next = ppc.next_o_tts
                nconns += 1
                nthis = 0
                p = os.path.join(root, f)
                
                m = None

                #
                # find the path-matching re, path translation and check
                # functions
                #
                for match, base, check in checks:
                    m = match.match(p)
                    if m:
                        break
                    
                if not m:
                    print 'File %s does not meet naming convention - not checked' % (p)
                    nerrors += 1
                    continue

                stuff = base(m)
                if self.npe:
                    fts = 0
                else:
                    fts = stuff[2]
                
                try:
                    ppc.open_offline(p)
                except (IOError, TypeError), s:
                    print
                    print p
                    self.exception(str(s))

                try:
                    ppc.set_filter("")
                except TypeError, s:
                    print
                    self.exception(str(s))

                np = -1

                #
                # Loop through packets and check'm
                #
                while 1:
                    try:
                        pkt = next()
                        ts = pkt[3]
                        npkts += 1
                        np += 1
                        nthis += 1
                    except EOFError:
                        break
                    except TypeError, s:
                        print
                        self.exception(str(s))

                    ts = (ts[0]*1000000 + ts[1])
                    nerrors += check(stuff, pkt, ts, np)
                    try:
                        assert ts >= fts
                        assert ts <= fts + self.timeout
                    except AssertionError:
                        #
                        # Collecting packet filter may have misordered packets
                        # in the dump - check this from the log before
                        # identifying an error
                        #
                        lines = get_log(p, quiet=quiet)
                        found = 0
                        for l in lines:
                            m2 = re.match('Unordered time stamp.*?No\. (?P<pno>\d+)', l)
                            if m2 and int(m2.group('pno')) == np:
                                found += 1
                                break
                        if not found:
                            # fallback - check on hash of pkt contents
                            if self.oots.has_key(hash(tuple(pkt[2]))):
                                found = 1
                        if not found:
                            tbs = traceback.format_list(traceback.extract_tb(sys.exc_info()[2]))
                            print 'Check failed  - time stamp ordering (current %u.%06u, last %u.%06u) file %s pkt. No. %d\n%s' % (ts/1000000, ts%1000000, fts/1000000, fts%1000000, p, np, tbs[0])
                            nerrors += 1

                    try:
                        assert ts <= fts + self.timeout
                    except AssertionError:
                        if is_frag(data):
                            frag_nto.append((p, np))
                        else:
                            tbs = traceback.format_list(traceback.extract_tb(sys.exc_info()[2]))
                            print 'Check failed  - timeout missed (current %u.%06u, last %u.%06u) file %s pkt. No. %d\n%s' % (ts/1000000, ts%1000000, fts/1000000, fts%1000000, p, np, tbs[0])
                            nerrors += 1
                        
                    fts = ts
                    

        try:
            assert npkts == self.ipkts - self.nosyn -self.truncated
            assert nconns == self.nconns + self.resumed
        except AssertionError:
            nerrors += 1
            print 'Check failed  - inconsistent No. of packets/flows (%d/%d)/(%d/%d)' % (npkts, self.ipkts, nconns, self.nconns + self.resumed)

        if not nerrors:
            print 'Check passed'
        else:
            print '%d fails' % (nerrors)

        if frag_nto:
            print 'Fragment timeouts not regarded as errors:'
            for f, pn in frag_nto:
                print f, 'packet No.', pn
            
            
        
############################################################################
############################################################################
