From neto at caltech.edu  Fri Aug  1 17:24:28 2014
From: neto at caltech.edu (Naoki Eto)
Date: Fri, 1 Aug 2014 17:24:28 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Changed the key struct in
	Connection.cc and Connection.hh to be less convoluted. (d0e18fe)
Message-ID: <201408020024.s720OSlo017357@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit d0e18fe34d077ea6c420f471e8d237e1b21aab7f
Author: NaokiEto <neto at caltech.edu>
Date:   Fri Aug 1 20:12:00 2014 -0400

    Changed the key struct in Connection.cc and Connection.hh to be less convoluted.


>---------------------------------------------------------------

d0e18fe34d077ea6c420f471e8d237e1b21aab7f
 src/Connection.cc | 159 +++++++++++++++++++++++++++++-------------------------
 src/Connection.hh |  70 +++++++++++++++++-------
 src/H3.h          |   2 +-
 3 files changed, 139 insertions(+), 92 deletions(-)

diff --git a/src/Connection.cc b/src/Connection.cc
index 71ddad0..74a344a 100644
--- a/src/Connection.cc
+++ b/src/Connection.cc
@@ -66,7 +66,7 @@ inline bool addr6_port_canon_lt(const unsigned char s6_ip[], const unsigned char
 
 
 
-void ConnectionID4::init(proto_t proto,
+void ConnectionID4::init(proto_t proto4,
 						 uint32_t s_ip, uint32_t d_ip,
 						 uint16_t s_port, uint16_t d_port) {
     // DEBUG DEBUG DEBUG
@@ -76,7 +76,7 @@ void ConnectionID4::init(proto_t proto,
     //in4_addr ipv4_s_address;
 
     v6.version = 4;
-	v6.proto=proto;
+	v6.proto=proto4;
 
     static uint8_t v4_mapped_prefix[12]; // top 96 bits of v4-mapped-addr
 
@@ -88,11 +88,14 @@ void ConnectionID4::init(proto_t proto,
 		//    v.is_canonified=true;
 
         // setting v6.ip1 to be  destination address and v6.ip2 to be source address
-        memcpy(key.ip1.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip1.s6_addr[12], &d_ip, sizeof(d_ip));
+        //key.ip1.s6_tm_addr = &v4_mapped_prefix[0];
+        //char *
 
-        memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip2.s6_addr[12], &s_ip, sizeof(s_ip));
+        memcpy(key.ip1, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip1[12], &d_ip, sizeof(d_ip));
+
+        memcpy(key.ip2, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip2[12], &s_ip, sizeof(s_ip));
 
 
 
@@ -132,11 +135,11 @@ void ConnectionID4::init(proto_t proto,
 
         // setting v6.ip1 to be source address and v6.ip2 to be destination address
 
-        memcpy(key.ip1.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip1.s6_addr[12], &s_ip, sizeof(s_ip));
+        memcpy(key.ip1, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip1[12], &s_ip, sizeof(s_ip));
 
-        memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip2.s6_addr[12], &d_ip, sizeof(d_ip));
+        memcpy(key.ip2, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip2[12], &d_ip, sizeof(d_ip));
 
 
         /*
@@ -190,14 +193,14 @@ void ConnectionID4::init(proto_t proto,
     //free_hash_function();
 }
 
-void ConnectionID4::init6(proto_t proto,
+void ConnectionID4::init6(proto_t proto6,
                   unsigned char s_ip[], unsigned char d_ip[],
                   uint16_t s_port, uint16_t d_port) {
     // DEBUG DEBUG DEBUG
 	//tmlog(TM_LOG_NOTE, "connection 4: Connection.cc, ~line 48", "connection 4 for ipv6 initialized");
 
     v6.version = 6;
-	v6.proto=proto;
+	v6.proto=proto6;
 
 	// Lookup up connection based on canonical ordering, which is
 	// the smaller of <src addr, src port> and <dst addr, dst port>
@@ -205,8 +208,8 @@ void ConnectionID4::init6(proto_t proto,
 	if (addr6_port_canon_lt(s_ip,d_ip,s_port,d_port)) {
 		//    v6.is_canonified=true;
         // memcpy(destination, source, size)
-        memcpy(key.ip1.s6_addr, d_ip, 16);
-        memcpy(key.ip2.s6_addr, s_ip, 16);
+        memcpy(key.ip1, d_ip, 16);
+        memcpy(key.ip2, s_ip, 16);
 		//v6.ip1=d_ip;
 		//v6.ip2=s_ip;
 		//v6.port1=d_port;
@@ -226,8 +229,8 @@ void ConnectionID4::init6(proto_t proto,
 
 	} else {
 		//    v6.is_canonified=false;
-        memcpy(key.ip1.s6_addr, s_ip, 16);
-        memcpy(key.ip2.s6_addr, d_ip, 16);
+        memcpy(key.ip1, s_ip, 16);
+        memcpy(key.ip2, d_ip, 16);
 		//v6.ip1=s_ip;
 		//v6.ip2=d_ip;
 		//v6.port1=s_port;
@@ -266,7 +269,7 @@ void ConnectionID4::init6(proto_t proto,
     //free_hash_function();
 }
 
-void ConnectionID3::init(proto_t proto,
+void ConnectionID3::init(proto_t proto4,
 						 uint32_t ip1, uint32_t ip2,
 						 uint16_t port2) {
 
@@ -278,7 +281,7 @@ void ConnectionID3::init(proto_t proto,
 
     v6.version = 4;
 
-	v6.proto=proto;
+	v6.proto=proto4;
 
     static uint8_t v4_mapped_prefix[12]; // top 96 bits of v4-mapped-addr
     
@@ -296,11 +299,11 @@ void ConnectionID3::init(proto_t proto,
 
     */
 
-    memcpy(key.ip1.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-    memcpy(&key.ip1.s6_addr[12], &ip1, sizeof(ip1));
+    memcpy(key.ip1, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+    memcpy(&key.ip1[12], &ip1, sizeof(ip1));
 
-    memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-    memcpy(&key.ip2.s6_addr[12], &ip2, sizeof(ip2));
+    memcpy(key.ip2, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+    memcpy(&key.ip2[12], &ip2, sizeof(ip2));
 
 
 	//key.port2=port2;
@@ -361,21 +364,21 @@ void ConnectionID3::init(proto_t proto,
     //free_hash_function();
 }
 
-void ConnectionID3::init6(proto_t proto,
+void ConnectionID3::init6(proto_t proto6,
 						 unsigned char ip1[], unsigned char ip2[],
 						 uint16_t port2) {
 
     // DEBUG DEBUG DEBUG
 	//tmlog(TM_LOG_DEBUG, "connection 3: Connection.cc, ~line 71", "connection 3 initialized");
 
-	v6.proto=proto;
+	v6.proto=proto6;
 
     v6.version = 6;
 
     // memcpy(destination, source, size)
     // setting v6.ip1 to be source address and v6.ip2 to be destination address 
-    memcpy(key.ip1.s6_addr, ip1, 16);
-    memcpy(key.ip2.s6_addr, ip2, 16);
+    memcpy(key.ip1, ip1, 16);
+    memcpy(key.ip2, ip2, 16);
 
 	//v.ip1=ip1;
 	//v.ip2=ip2;
@@ -471,11 +474,11 @@ void ConnectionID2::init( uint32_t s_ip, uint32_t d_ip) {
 
         // setting v6.ip1 to dest address and v6.ip2 to source address
 
-        memcpy(key.ip1.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip1.s6_addr[12], &d_ip, sizeof(d_ip));
+        memcpy(key.ip1, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip1[12], &d_ip, sizeof(d_ip));
 
-        memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip2.s6_addr[12], &s_ip, sizeof(s_ip));
+        memcpy(key.ip2, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip2[12], &s_ip, sizeof(s_ip));
 
         // this is for the hash key
         //in6_addr s6_ip;
@@ -503,11 +506,11 @@ void ConnectionID2::init( uint32_t s_ip, uint32_t d_ip) {
 
         // setting v6.ip1 to be source address and v6.ip2 to be dest address
 
-        memcpy(key.ip1.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip1.s6_addr[12], &s_ip, sizeof(s_ip));
+        memcpy(key.ip1, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip1[12], &s_ip, sizeof(s_ip));
 
-        memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip2.s6_addr[12], &d_ip, sizeof(d_ip));
+        memcpy(key.ip2, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip2[12], &d_ip, sizeof(d_ip));
 
 
         // this is for the hash key
@@ -548,8 +551,8 @@ void ConnectionID2::init6( unsigned char s_ip[], unsigned char d_ip[]) {
 	if (addr6_port_canon_lt(s_ip,d_ip,0,0)) {
 		//    v.is_canonified=true;
         // memcpy(destination, source, size)
-        memcpy(key.ip1.s6_addr, d_ip, 16);
-        memcpy(key.ip2.s6_addr, s_ip, 16);
+        memcpy(key.ip1, d_ip, 16);
+        memcpy(key.ip2, s_ip, 16);
 
 		//v6.ip1=d_ip;
 		//v6.ip2=s_ip;
@@ -568,8 +571,8 @@ void ConnectionID2::init6( unsigned char s_ip[], unsigned char d_ip[]) {
 
 	} else {
 		//    v.is_canonified=false;
-        memcpy(key.ip1.s6_addr, s_ip, 16);
-        memcpy(key.ip2.s6_addr, d_ip, 16);
+        memcpy(key.ip1, s_ip, 16);
+        memcpy(key.ip2, d_ip, 16);
 		//v.ip1=s_ip;
 		//v.ip2=d_ip;
 
@@ -786,9 +789,16 @@ ConnectionID2::ConnectionID2(const u_char* packet) {
 //TODO: MAke this inline (i.e. move to Connection.hh so that it is
 //consistent with ConnectionID4
 bool ConnectionID3::operator==(const ConnectionID& other) const {
+/*
+        return equal(key.ip1.s6_addr, ((ConnectionID3*)&other)->key.ip2.s6_addr)
+               && equal(key.ip2.s6_addr, ((ConnectionID3*)&other)->key.ip2.s6_addr)
+               && (key.port2 == ((ConnectionID3*)&other)->key.port2)
+               && (v6.proto == ((ConnectionID3*)&other)->v6.proto);
+*/
 
-        return (!memcmp(&key.ip1, &((ConnectionID3*)&other)->key.ip1, sizeof(in6_addr)))
-               && (!memcmp(&key.ip2, &((ConnectionID3*)&other)->key.ip2, sizeof(in6_addr)))
+
+        return (!memcmp(&key.ip1, &((ConnectionID3*)&other)->key.ip1, 16))
+               && (!memcmp(&key.ip2, &((ConnectionID3*)&other)->key.ip2, 16))
                && (key.port2 == ((ConnectionID3*)&other)->key.port2)
                && (v6.proto == ((ConnectionID3*)&other)->v6.proto);
 
@@ -797,9 +807,12 @@ bool ConnectionID3::operator==(const ConnectionID& other) const {
 //TODO: MAke this inline (i.e. move to Connection.hh so that it is
 //consistent with ConnectionID4
 bool ConnectionID2::operator==(const ConnectionID& other) const {
-
-        return (!memcmp(&key.ip1, &((ConnectionID2*)&other)->key.ip1, sizeof(in6_addr)))
-               && (!memcmp(&key.ip2, &((ConnectionID2*)&other)->key.ip2, sizeof(in6_addr)));
+/*
+        return equal(key.ip1.s6_addr, ((ConnectionID2*)&other)->key.ip2.s6_addr)
+               && equal(key.ip2.s6_addr, ((ConnectionID2*)&other)->key.ip2.s6_addr);
+*/
+       return (!memcmp(&key.ip1, &((ConnectionID2*)&other)->key.ip1, 16))
+               && (!memcmp(&key.ip2, &((ConnectionID2*)&other)->key.ip2, 16));
 
 }
 
@@ -875,17 +888,17 @@ std::string ConnectionID4::getStr() const {
 	    << (UCP(d_ip)[3] & 0xff)
 	    << ":"
 */
-        << (UCP(key.ip1.s6_addr)[0] & 0xff) << "."
-        << (UCP(key.ip1.s6_addr)[1] & 0xff) << "."
-        << (UCP(key.ip1.s6_addr)[2] & 0xff) << "."
-        << (UCP(key.ip1.s6_addr)[3] & 0xff)
+        << (UCP(key.ip1)[0] & 0xff) << "."
+        << (UCP(key.ip1)[1] & 0xff) << "."
+        << (UCP(key.ip1)[2] & 0xff) << "."
+        << (UCP(key.ip1)[3] & 0xff)
         << ":"
         << ntohs(get_port1())
         << " - "
-        << (UCP(key.ip2.s6_addr)[0] & 0xff) << "."
-        << (UCP(key.ip2.s6_addr)[1] & 0xff) << "."
-        << (UCP(key.ip2.s6_addr)[2] & 0xff) << "."
-        << (UCP(key.ip2.s6_addr)[3] & 0xff)
+        << (UCP(key.ip2)[0] & 0xff) << "."
+        << (UCP(key.ip2)[1] & 0xff) << "."
+        << (UCP(key.ip2)[2] & 0xff) << "."
+        << (UCP(key.ip2)[3] & 0xff)
         << ":"
 
 
@@ -908,11 +921,11 @@ std::string ConnectionID4::getStr() const {
 
         char str1[INET6_ADDRSTRLEN];
 
-        bro_inet_ntop(AF_INET6, &(key.ip1.s6_addr), str1, INET6_ADDRSTRLEN);
+        bro_inet_ntop(AF_INET6, key.ip1, str1, INET6_ADDRSTRLEN);
 
         char str2[INET6_ADDRSTRLEN];
 
-        bro_inet_ntop(AF_INET6, &(key.ip2.s6_addr), str2, INET6_ADDRSTRLEN);
+        bro_inet_ntop(AF_INET6, key.ip2, str2, INET6_ADDRSTRLEN);
 
 
         ss << " ConnectionID4 for IPv6"
@@ -953,15 +966,15 @@ std::string ConnectionID3::getStr() const {
         //memcpy(d_ip, key.ip2.s6_addr, 16);
 
 	    ss << " ConnectionID3 "
-	    << (UCP(key.ip1.s6_addr)[0] & 0xff) << "."
-	    << (UCP(key.ip1.s6_addr)[1] & 0xff) << "."
-	    << (UCP(key.ip1.s6_addr)[2] & 0xff) << "."
-	    << (UCP(key.ip1.s6_addr)[3] & 0xff)
+	    << (UCP(key.ip1)[0] & 0xff) << "."
+	    << (UCP(key.ip1)[1] & 0xff) << "."
+	    << (UCP(key.ip1)[2] & 0xff) << "."
+	    << (UCP(key.ip1)[3] & 0xff)
 	    << " - "
-	    << (UCP(key.ip2.s6_addr)[0] & 0xff) << "."
-	    << (UCP(key.ip2.s6_addr)[1] & 0xff) << "."
-	    << (UCP(key.ip2.s6_addr)[2] & 0xff) << "."
-	    << (UCP(key.ip2.s6_addr)[3] & 0xff)
+	    << (UCP(key.ip2)[0] & 0xff) << "."
+	    << (UCP(key.ip2)[1] & 0xff) << "."
+	    << (UCP(key.ip2)[2] & 0xff) << "."
+	    << (UCP(key.ip2)[3] & 0xff)
 	    << ":"
 	    << get_port();
 	    return ss.str();
@@ -983,11 +996,11 @@ std::string ConnectionID3::getStr() const {
 
         char str1[INET6_ADDRSTRLEN];
 
-        bro_inet_ntop(AF_INET6, &(key.ip1.s6_addr), str1, INET6_ADDRSTRLEN);
+        bro_inet_ntop(AF_INET6, &(key.ip1), str1, INET6_ADDRSTRLEN);
 
         char str2[INET6_ADDRSTRLEN];
 
-        bro_inet_ntop(AF_INET6, &(key.ip2.s6_addr), str2, INET6_ADDRSTRLEN);
+        bro_inet_ntop(AF_INET6, &(key.ip2), str2, INET6_ADDRSTRLEN);
 
         ss << " ConnectionID3 for IPv6"
 
@@ -1047,15 +1060,15 @@ std::string ConnectionID2::getStr() const {
         //memcpy(d_ip, key.ip2.s6_addr, 16);
 
 	    ss << " ConnectionID2 "
-	    << (UCP(key.ip1.s6_addr)[0] & 0xff) << "."
-	    << (UCP(key.ip1.s6_addr)[1] & 0xff) << "."
-	    << (UCP(key.ip1.s6_addr)[2] & 0xff) << "."
-	    << (UCP(key.ip1.s6_addr)[3] & 0xff)
+	    << (UCP(key.ip1)[0] & 0xff) << "."
+	    << (UCP(key.ip1)[1] & 0xff) << "."
+	    << (UCP(key.ip1)[2] & 0xff) << "."
+	    << (UCP(key.ip1)[3] & 0xff)
 	    << " - "
-	    << (UCP(key.ip2.s6_addr)[0] & 0xff) << "."
-	    << (UCP(key.ip2.s6_addr)[1] & 0xff) << "."
-	    << (UCP(key.ip2.s6_addr)[2] & 0xff) << "."
-	    << (UCP(key.ip2.s6_addr)[3] & 0xff);
+	    << (UCP(key.ip2)[0] & 0xff) << "."
+	    << (UCP(key.ip2)[1] & 0xff) << "."
+	    << (UCP(key.ip2)[2] & 0xff) << "."
+	    << (UCP(key.ip2)[3] & 0xff);
 	    return ss.str();
     }
 
@@ -1076,11 +1089,11 @@ std::string ConnectionID2::getStr() const {
 
         char str1[INET6_ADDRSTRLEN];
 
-        bro_inet_ntop(AF_INET6, &(key.ip1.s6_addr), str1, INET6_ADDRSTRLEN);
+        bro_inet_ntop(AF_INET6, &(key.ip1), str1, INET6_ADDRSTRLEN);
 
         char str2[INET6_ADDRSTRLEN];
 
-        bro_inet_ntop(AF_INET6, &(key.ip2.s6_addr), str2, INET6_ADDRSTRLEN);
+        bro_inet_ntop(AF_INET6, &(key.ip2), str2, INET6_ADDRSTRLEN);
 
         ss << " ConnectionID2 for IPv6"
 
diff --git a/src/Connection.hh b/src/Connection.hh
index cea84b9..3a29f22 100644
--- a/src/Connection.hh
+++ b/src/Connection.hh
@@ -24,7 +24,17 @@ public:
 	//  virtual const void* getConstVPtr() const = 0;
 	virtual void getStr(char* s, int maxsize) const = 0;
 	virtual std::string getStr() const = 0;
-
+/*
+        int equal(const unsigned char* a, const unsigned char* b) const
+        {
+            for (int i = 0; i < 16; i++)
+            {
+                if (a[i] != b[i])
+                    return 0;
+            }
+            return 1;
+        }
+*/
     //virtual HashKey* hash() const = 0;
 
  /**
@@ -46,8 +56,8 @@ public:
         init6(proto, s_ip, d_ip, s_port, d_port);
     }
 	ConnectionID4(ConnectionID4 *c_id) {
-        memcpy(key.ip1.s6_addr, c_id->key.ip1.s6_addr, 16);
-        memcpy(key.ip2.s6_addr, c_id->key.ip2.s6_addr, 16);
+        memcpy(key.ip1, c_id->key.ip1, 16);
+        memcpy(key.ip2, c_id->key.ip2, 16);
 		key.port1 = c_id->key.port1;
 		key.port2 = c_id->key.port2;
 		v6.proto = c_id->v6.proto;
@@ -94,11 +104,18 @@ public:
                         && (v6.proto == ((ConnectionID4*)&other)->v6.proto);
                */
                  
-		return (!memcmp(&key.ip1, &((ConnectionID4*)&other)->key.ip1, sizeof(in6_addr)))
-			   && (!memcmp(&key.ip2, &((ConnectionID4*)&other)->key.ip2, sizeof(in6_addr)))
+		return (!memcmp(&key.ip1, &((ConnectionID4*)&other)->key.ip1, 16))
+			   && (!memcmp(&key.ip2, &((ConnectionID4*)&other)->key.ip2, 16))
 			   && (key.port1 == ((ConnectionID4*)&other)->key.port1)
 			   && (key.port2 == ((ConnectionID4*)&other)->key.port2)
 			   && (v6.proto == ((ConnectionID4*)&other)->v6.proto);
+                /*
+                return equal(key.ip1.s6_addr, ((ConnectionID4*)&other)->key.ip2.s6_addr)
+                           && equal(key.ip2.s6_addr, ((ConnectionID4*)&other)->key.ip2.s6_addr)
+                           && (key.port1 == ((ConnectionID4*)&other)->key.port1)
+                           && (key.port2 == ((ConnectionID4*)&other)->key.port2)
+                           && (v6.proto == ((ConnectionID4*)&other)->v6.proto);
+                 */
 
 	}
 
@@ -125,10 +142,10 @@ public:
 		return v6.proto;
 	}
 	const unsigned char* get_ip1() const {
-		return key.ip1.s6_addr;
+		return key.ip1;
 	}
 	const unsigned char* get_ip2() const {
-		return key.ip2.s6_addr;
+		return key.ip2;
 	}
 	uint16_t get_port1() const {
 		return key.port1;
@@ -164,6 +181,13 @@ public:
 	__attribute__((packed)) v_t;
     */
 
+/*
+    struct in6_tm_addr
+    {
+        unsigned char *s6_tm_addr;
+    };
+*/
+    
 	typedef struct {
 		//  time locality
 		//    uint32_t ts;
@@ -171,18 +195,21 @@ public:
 		//unsigned char ip2[12];
 		//uint16_t port1;
 		//uint16_t port2;
-		proto_t proto;
         int version;
+		proto_t proto;
 		//    bool is_canonified;
 	}
     // have the structure fields align on one-byte boundaries
 	__attribute__((packed)) v6_t;
     
+    //proto_t proto;
+
     typedef struct {
-        in6_addr ip1;
-        in6_addr ip2;
+        unsigned char ip1[16];
+        unsigned char ip2[16];
         uint16 port1;
         uint16 port2;
+        //int version;
     } 
     __attribute__((packed)) key_t;
 
@@ -294,10 +321,10 @@ public:
 		return v6.proto;
 	}
 	const unsigned char* get_ip1() const {
-		return key.ip1.s6_addr;
+		return key.ip1;
 	}
 	const unsigned char* get_ip2() const {
-		return key.ip2.s6_addr;
+		return key.ip2;
 	}
 	uint16_t get_port() const {
 		return key.port2;
@@ -322,6 +349,7 @@ public:
 	__attribute__((packed)) v_t;
     */
 
+    
 	typedef struct {
 		//  time locality
 		//    uint32_t ts;
@@ -334,11 +362,15 @@ public:
 	}
 	__attribute__((packed)) v6_t;
     
+
+    //proto_t proto;
+
     typedef struct {
-        in6_addr ip1;
-        in6_addr ip2;
+        unsigned char ip1[16];
+        unsigned char ip2[16];
         uint16 port1;
         uint16 port2;
+        //int version;
     }
     __attribute__((packed)) key_t;
 
@@ -460,10 +492,10 @@ public:
 */
 
 	const unsigned char* get_ip1() const {
-		return key.ip1.s6_addr;
+		return key.ip1;
 	}
 	const unsigned char* get_ip2() const {
-		return key.ip2.s6_addr;
+		return key.ip2;
 	}
 
 	/*
@@ -494,11 +526,13 @@ public:
 	}
 	__attribute__((packed)) v6_t;
     
+
     typedef struct {
-        in6_addr ip1;
-        in6_addr ip2;
+        unsigned char ip1[16];
+        unsigned char ip2[16];
         uint16 port1;
         uint16 port2;
+        //int version;
     }
     __attribute__((packed)) key_t;
 
diff --git a/src/H3.h b/src/H3.h
index e8b4324..f70c198 100644
--- a/src/H3.h
+++ b/src/H3.h
@@ -79,7 +79,7 @@ public:
 
 	void Init(bool have_seed, T seed)
 		{
-        tmlog(TM_LOG_NOTE, "H3 has function", "Just making sure that initialization was attempted with seed %lu\n", seed);
+        //tmlog(TM_LOG_NOTE, "H3 has function", "Just making sure that initialization was attempted with seed %lu\n", seed);
 		T bit_lookup[N * CHAR_BIT];
 
 		for ( size_t bit = 0; bit < N * CHAR_BIT; bit++ )


From neto at caltech.edu  Mon Aug  4 17:49:42 2014
From: neto at caltech.edu (Naoki Eto)
Date: Mon, 4 Aug 2014 17:49:42 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Fixed a small bug in
	IndexField.hh (had to do with memcpy),
	and tried to add some support for Mac OS (00557be)
Message-ID: <201408050049.s750ngco010898@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit 00557bec9866372972257a29a6df3b9a0151f5ba
Author: NaokiEto <neto at caltech.edu>
Date:   Mon Aug 4 20:31:49 2014 -0400

    Fixed a small bug in IndexField.hh (had to do with memcpy), and tried
    to add some support for Mac OS


>---------------------------------------------------------------

00557bec9866372972257a29a6df3b9a0151f5ba
 src/IndexField.hh | 2 +-
 src/util.h        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/IndexField.hh b/src/IndexField.hh
index fa45197..dad372e 100644
--- a/src/IndexField.hh
+++ b/src/IndexField.hh
@@ -99,7 +99,7 @@ public:
 
         //hash_key = newHashKey->Hash();
 
-        hash_key = HashKey::HashBytes(key, sizeof(key));
+        hash_key = HashKey::HashBytes(key, sizeof(ip));
  
         //delete newHashKey;
 
diff --git a/src/util.h b/src/util.h
index 341f0a7..112c33b 100644
--- a/src/util.h
+++ b/src/util.h
@@ -72,7 +72,7 @@ typedef uint64 bro_uint_t;
 // pointer size. They can be cast safely to a pointer, e.g. in Lists,
 // which represent their entities as void* pointers.
 //
-#ifdef linux
+#if defined(linux) || defined(__APPLE__)
 #if SIZEOF_VOID_P == 8
 
 typedef uint64 ptr_compat_uint;


From neto at caltech.edu  Mon Aug 11 10:55:24 2014
From: neto at caltech.edu (Naoki Eto)
Date: Mon, 11 Aug 2014 10:55:24 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Removed the many instances of
	gettimeofday and used a counter instead to avoid system calls
	(Aashish's idea) (6d6db31)
Message-ID: <201408111755.s7BHtOdV021528@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit 6d6db31c2f3c63fddd2b982060c860a2a9a2d5bb
Author: NaokiEto <neto at caltech.edu>
Date:   Mon Aug 11 13:37:52 2014 -0400

    Removed the many instances of gettimeofday and used a counter instead to avoid system calls (Aashish's idea)
    
    Removed many unnecessary files
    
    Introduced some clock_gettime() calls to replace the gettimeofday calls
    
    Added MLPS support
    
    Increased min sleep time between aggregations to 15, increased max size of queue to 1000000
    
    Changed the Hash table sizes to always be prime number to help avoid inefficient clustering


>---------------------------------------------------------------

6d6db31c2f3c63fddd2b982060c860a2a9a2d5bb
 src/DiskIndex.cc    |  35 ++++++---
 src/DynClass.cc     |  23 +++++-
 src/HashIterator.hh | 124 ------------------------------
 src/Index.cc        | 112 +++++++++++++++++++++------
 src/Index.hh        |  97 ++++++++++++++++++++---
 src/IndexHash.cc    |  95 +++++++++++++++++++++--
 src/IndexHash.hh    |   8 +-
 src/LogFile.cc      |  22 ++++++
 src/Net.h           | 116 ----------------------------
 src/Queue.cc        | 131 -------------------------------
 src/Queue.hh        |  85 --------------------
 src/Reporter.h      | 146 -----------------------------------
 src/Storage.cc      |  89 ++++++++++++++++-----
 src/conf_parser.yy  |  18 ++++-
 src/main.cc.in      | 217 ++++++++++++++++++++++++++++++++++++++++++++++++----
 src/types.h         |  10 +++
 16 files changed, 636 insertions(+), 692 deletions(-)

Diff suppressed because of size. To see it, use:

    git diff-tree --root --patch-with-stat --no-color --find-copies-harder --ignore-space-at-eol --cc 6d6db31c2f3c63fddd2b982060c860a2a9a2d5bb

From neto at caltech.edu  Mon Aug 18 09:09:15 2014
From: neto at caltech.edu (Naoki Eto)
Date: Mon, 18 Aug 2014 09:09:15 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Changed the format of the
	configuration file to allow the class files to be placed in
	different directories based on their bucket type,
	a suggestion made by Partha Also fixed querying for IPv4 for conn2,
	conn3, and conn4. Please note that querying for IPv6 for conn2,
	conn3, conn4 is currently not working. (74b8a32)
Message-ID: <201408181609.s7IG9F0s002066@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit 74b8a329d43d931d2e576b3b19a935f1d12c1188
Author: NaokiEto <neto at caltech.edu>
Date:   Mon Aug 18 11:55:40 2014 -0400

    Changed the format of the configuration file to allow the class files to be placed in different directories based on their bucket type, a suggestion made by Partha
    Also fixed querying for IPv4 for conn2, conn3, and conn4. Please note that querying for IPv6 for conn2, conn3, conn4 is currently not working.


>---------------------------------------------------------------

74b8a329d43d931d2e576b3b19a935f1d12c1188
 src/CMakeLists.txt           |   4 +-
 src/Connection.cc            |  48 ++++-----
 src/Connection.hh            |  12 +++
 src/DiskIndex.cc             |  25 +++++
 src/Fifo.cc                  |  30 +++++-
 src/Fifo.hh                  |   9 +-
 src/FifoDisk.cc              |  74 +++++++++++--
 src/FifoDisk.hh              |   6 +-
 src/Index.cc                 |   6 ++
 src/IndexField.cc            | 220 +++++++++++++++++++++++++++----------
 src/Query.cc                 |  16 ++-
 src/cmd_parser.yy            |   4 +-
 src/conf.h                   |   1 +
 src/conf_parser.yy           |   7 ++
 src/conf_scanner.ll          |   1 +
 src/main.cc.in               |  20 +++-
 src/pcapnav/pcapnav.c        | 251 ++++++++++++++++++++++++++++++++++++++++---
 src/pcapnav/pcapnav.h        |   6 +-
 src/pcapnav/pcapnav_append.c |  12 +--
 src/pcapnav/pcapnav_append.h |   4 +-
 src/tm.h                     |   2 +-
 21 files changed, 627 insertions(+), 131 deletions(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 173cba4..2e9054b 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -82,7 +82,9 @@ set(timemachine_SRCS
 
 set(CMAKE_CXX_FLAGS "-g -Wall")
 
-set (EXTRA_LIBS ${EXTRA_LIBS} -lssl -lcrypto)
+set(CMAKE_C_FLAGS "-DPCAPNAV_DEBUG")
+
+set (EXTRA_LIBS ${EXTRA_LIBS} -lssl -lcrypto -DPCAPNAV_DEBUG)
 
 set(tm_query_benchmark_SRCS
     tm_query_benchmark.cc
diff --git a/src/Connection.cc b/src/Connection.cc
index 74a344a..00661be 100644
--- a/src/Connection.cc
+++ b/src/Connection.cc
@@ -888,17 +888,17 @@ std::string ConnectionID4::getStr() const {
 	    << (UCP(d_ip)[3] & 0xff)
 	    << ":"
 */
-        << (UCP(key.ip1)[0] & 0xff) << "."
-        << (UCP(key.ip1)[1] & 0xff) << "."
-        << (UCP(key.ip1)[2] & 0xff) << "."
-        << (UCP(key.ip1)[3] & 0xff)
+        << (UCP(key.ip1)[12] & 0xff) << "."
+        << (UCP(key.ip1)[13] & 0xff) << "."
+        << (UCP(key.ip1)[14] & 0xff) << "."
+        << (UCP(key.ip1)[15] & 0xff)
         << ":"
         << ntohs(get_port1())
         << " - "
-        << (UCP(key.ip2)[0] & 0xff) << "."
-        << (UCP(key.ip2)[1] & 0xff) << "."
-        << (UCP(key.ip2)[2] & 0xff) << "."
-        << (UCP(key.ip2)[3] & 0xff)
+        << (UCP(key.ip2)[12] & 0xff) << "."
+        << (UCP(key.ip2)[13] & 0xff) << "."
+        << (UCP(key.ip2)[14] & 0xff) << "."
+        << (UCP(key.ip2)[15] & 0xff)
         << ":"
 
 
@@ -966,15 +966,15 @@ std::string ConnectionID3::getStr() const {
         //memcpy(d_ip, key.ip2.s6_addr, 16);
 
 	    ss << " ConnectionID3 "
-	    << (UCP(key.ip1)[0] & 0xff) << "."
-	    << (UCP(key.ip1)[1] & 0xff) << "."
-	    << (UCP(key.ip1)[2] & 0xff) << "."
-	    << (UCP(key.ip1)[3] & 0xff)
+	    << (UCP(key.ip1)[12] & 0xff) << "."
+	    << (UCP(key.ip1)[13] & 0xff) << "."
+	    << (UCP(key.ip1)[14] & 0xff) << "."
+	    << (UCP(key.ip1)[15] & 0xff)
 	    << " - "
-	    << (UCP(key.ip2)[0] & 0xff) << "."
-	    << (UCP(key.ip2)[1] & 0xff) << "."
-	    << (UCP(key.ip2)[2] & 0xff) << "."
-	    << (UCP(key.ip2)[3] & 0xff)
+	    << (UCP(key.ip2)[12] & 0xff) << "."
+	    << (UCP(key.ip2)[13] & 0xff) << "."
+	    << (UCP(key.ip2)[14] & 0xff) << "."
+	    << (UCP(key.ip2)[15] & 0xff)
 	    << ":"
 	    << get_port();
 	    return ss.str();
@@ -1060,15 +1060,15 @@ std::string ConnectionID2::getStr() const {
         //memcpy(d_ip, key.ip2.s6_addr, 16);
 
 	    ss << " ConnectionID2 "
-	    << (UCP(key.ip1)[0] & 0xff) << "."
-	    << (UCP(key.ip1)[1] & 0xff) << "."
-	    << (UCP(key.ip1)[2] & 0xff) << "."
-	    << (UCP(key.ip1)[3] & 0xff)
+	    << (UCP(key.ip1)[12] & 0xff) << "."
+	    << (UCP(key.ip1)[13] & 0xff) << "."
+	    << (UCP(key.ip1)[14] & 0xff) << "."
+	    << (UCP(key.ip1)[15] & 0xff)
 	    << " - "
-	    << (UCP(key.ip2)[0] & 0xff) << "."
-	    << (UCP(key.ip2)[1] & 0xff) << "."
-	    << (UCP(key.ip2)[2] & 0xff) << "."
-	    << (UCP(key.ip2)[3] & 0xff);
+	    << (UCP(key.ip2)[12] & 0xff) << "."
+	    << (UCP(key.ip2)[13] & 0xff) << "."
+	    << (UCP(key.ip2)[14] & 0xff) << "."
+	    << (UCP(key.ip2)[15] & 0xff);
 	    return ss.str();
     }
 
diff --git a/src/Connection.hh b/src/Connection.hh
index 3a29f22..4a0eef0 100644
--- a/src/Connection.hh
+++ b/src/Connection.hh
@@ -153,6 +153,9 @@ public:
 	uint16_t get_port2() const {
 		return key.port2;
 	}
+    int get_version() const {
+        return v6.version;
+    }
 
 	//  bool get_is_canonified() const { return v.is_canonified; }
 	/*
@@ -329,6 +332,11 @@ public:
 	uint16_t get_port() const {
 		return key.port2;
 	}
+
+    int get_version() const {
+        return v6.version;
+    }
+
 	/*
 	bool get_is_canonified() const { return v.is_canonified; }
 	uint32_t get_s_ip() const {
@@ -497,6 +505,10 @@ public:
 	const unsigned char* get_ip2() const {
 		return key.ip2;
 	}
+    int get_version() const {
+        return v6.version;
+    }
+
 
 	/*
 	bool get_is_canonified() const { return v.is_canonified; }
diff --git a/src/DiskIndex.cc b/src/DiskIndex.cc
index 9f34b65..e63c9f1 100644
--- a/src/DiskIndex.cc
+++ b/src/DiskIndex.cc
@@ -47,6 +47,12 @@ my_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream)
  * class IndexFilesReader
  */
 inline IndexFileReader::IndexFileReader(char *fn) : fp(NULL), fname(fn), eof(false) {
+
+        if (chdir(conf_main_workdir)) {
+            fprintf(stderr, "cannot chdir to %s\n", conf_main_workdir);
+            //return(1);
+        }
+
 	fp = fopen(fn, "rb"); // read as binary 
 	if (fp == NULL) {
 		//TODO: Decent error handling
@@ -245,6 +251,14 @@ void IndexFiles<T>::writeIndex( IndexHash *ih) {
     // Object type that identifies a stream and contains the information needed to control 
     // it, including a pointer to its buffer, its position indicator and all its state indicators.
     // It will be used to write index files
+
+    /*
+    if (chdir(conf_main_workdir)) {
+        fprintf(stderr, "cannot chdir to %s\n", conf_main_workdir);
+        //return(1);
+    }
+    */
+
 	FILE *fp;
 	char *new_file_name;
 	IndexEntry *ie; // Hash table is made up of IndexEntry's, an IndexEntry is the data object containing 
@@ -262,6 +276,12 @@ void IndexFiles<T>::writeIndex( IndexHash *ih) {
 	unlock_file_numbers();
     // open the file as a binary file containing that file name with writing privileges
     // w = writing, b = open as binary file
+    if (chdir(conf_main_workdir)) {
+        fprintf(stderr, "cannot chdir to %s\n", conf_main_workdir);
+        //return(1);
+    }
+
+
 	fp = fopen(new_file_name, "wb");
     // if the file could not be opened, log error and return
 	if (fp == NULL) {
@@ -406,6 +426,11 @@ void IndexFiles<T>::aggregate(tm_time_t oldestTimestampDisk) {
  */
 template <class T>
 void IndexFiles<T>::aggregate_internal(int level) {
+    if (chdir(conf_main_workdir)) {
+            fprintf(stderr, "cannot chdir to %s\n", conf_main_workdir);
+            //return(1);
+    }    
+
     // vector of IndexFileReader pointers
 	std::vector<IndexFileReader *> ifr_vec;
 	std::vector<IndexFileReader *>::iterator it;
diff --git a/src/Fifo.cc b/src/Fifo.cc
index 1227ee4..ac4c984 100644
--- a/src/Fifo.cc
+++ b/src/Fifo.cc
@@ -30,18 +30,20 @@ Fifo::~Fifo() {
 }
 
 
-Fifo::Fifo(const std::string& classname, uint64_t fifo_mem_sz, uint64_t fifo_disk_sz, pcap_t* pcap_handle) {
+Fifo::Fifo(const std::string& classname, uint64_t fifo_mem_sz, uint64_t fifo_disk_sz, pcap_t* pcap_handle, const char* classdir) {
 	init();
 	this->classname=classname;
 	this->fifo_mem_sz=fifo_mem_sz;
 	this->fifo_disk_sz=fifo_disk_sz;
 	this->ph=pcap_handle;
+    this->classdir = classdir;
 	this->started=false;
 }
 
 
 void Fifo::init() {
 	classname="default";
+        classdir=conf_main_workdir;
 	filter="";
 	fifo_mem_sz=5000000;
 	fifo_disk_sz=50000000;
@@ -72,7 +74,7 @@ void Fifo::start() {
     // fifo_disk_sz is the size of buffer block in bytes
     // fifo_disk_filesz is the size of the file
     // ph is the handler
-	fd=new FifoDisk(classname, fifo_disk_sz, fifo_disk_filesz, ph);
+	fd=new FifoDisk(classname, fifo_disk_sz, fifo_disk_filesz, ph, classdir);
     
     // setting eviction handler for FifoMem object
 	fm->setEvictionHandler(this);
@@ -250,6 +252,15 @@ const FifoDisk* Fifo::getFd() {
 uint64_t Fifo::query(QueryRequest *qreq, QueryResult *qres,
 				 IntervalSet *interval_set) {
 	uint64_t matches = 0;
+    /*
+        if (chdir(classdir)) {
+            fprintf(stderr, "cannot class(Fifo:query) chdir to %s\n", classdir);
+            //return;
+        }
+    */ 
+
+    printf("The class name is: %s\n", classname.c_str());
+    printf("The directory the classes are in is: %s\n", classdir); 
 	FifoDiskFile *cur_file;
 
 	if (!qreq->isMemOnly()) {
@@ -270,7 +281,20 @@ uint64_t Fifo::query(QueryRequest *qreq, QueryResult *qres,
 					cur_file->getFilename().c_str());
 			if ( (qreq->getT1()+1e-3 >= cur_file->getOldestTimestamp()) &&
 					(qreq->getT0()-1e-3 <= cur_file->getNewestTimestamp()) ) {
-				matches+= cur_file->query(qreq, qres, interval_set);
+ 
+                if (chdir(classdir)) {
+                    fprintf(stderr, "cannot class(Fifo:query) chdir to %s\n", classdir);
+                    //return;
+                }
+
+                char path[70];
+
+                char errbufnav[PCAP_ERRBUF_SIZE];
+
+                printf("The directory for Fifo that we are in is %s\n", getcwd(path, 70));
+
+
+				matches+= cur_file->query(qreq, qres, interval_set, classdir);
 			}
 		}
 		fd->decQueryInProgress();
diff --git a/src/Fifo.hh b/src/Fifo.hh
index 8d06153..4d339ad 100644
--- a/src/Fifo.hh
+++ b/src/Fifo.hh
@@ -37,7 +37,7 @@ class Fifo: FifoMemEvictionHandler {
 public:
 	Fifo();
 	Fifo(const std::string& classname, uint64_t fifo_mem_sz,
-		 uint64_t fifo_disk_sz, pcap_t*);
+		 uint64_t fifo_disk_sz, pcap_t*, const char* classdir);
 	void start();
 	virtual ~Fifo();
 	uint64_t pktEviction();
@@ -66,6 +66,12 @@ public:
 	std::string getClassname() {
 		return classname;
 	}
+    void setClassdir(const char* s) {
+        classdir=s;
+    }
+    const char* getClassdir() {
+        return classdir;
+    }
 	void setFifoMemSz(uint64_t s) {
 		fifo_mem_sz=s;
 	}
@@ -117,6 +123,7 @@ protected:
 	uint64_t cutoff;
 	std::string classname;
 	std::string filter;
+    const char* classdir; 
 	uint64_t fifo_mem_sz;
 	uint64_t fifo_disk_sz;
 	uint64_t fifo_disk_filesz;
diff --git a/src/FifoDisk.cc b/src/FifoDisk.cc
index e694765..159b4a5 100644
--- a/src/FifoDisk.cc
+++ b/src/FifoDisk.cc
@@ -45,8 +45,8 @@
  */
 
 FifoDisk::FifoDisk(const std::string& classname, uint64_t size,
-				   uint64_t file_size, pcap_t* pcap_handle):
-		classname(classname), size(size), file_size(file_size),
+				   uint64_t file_size, pcap_t* pcap_handle, const char* classdir):
+		classname(classname), classdir(classdir), size(size), file_size(file_size),
 		tot_bytes(0), tot_pkts(0),
 		file_number(0), pcap_handle(pcap_handle),
 held_bytes(0), held_pkts(0), oldestTimestamp(0), newestTimestamp(0), queries(0) {
@@ -120,6 +120,13 @@ void FifoDisk::addPkt(const pkt_ptr p) {
                 // do a safe sprintf to create new_file_name
 				snprintf(new_file_name, strsz, "%s_%.6f",
 						 classname.c_str(), newestTimestamp);
+
+                if (chdir(classdir)) {
+                    fprintf(stderr, "cannot class chdir to %s\n", classdir);
+                    //return;
+                }
+
+
                 // push back the newest disk file into the list of files
 				files.push_back(new FifoDiskFile(new_file_name, pcap_handle));
 
@@ -246,7 +253,7 @@ void FifoDiskFile::addPkt(pkt_ptr p) {
 		   p+sizeof(struct pcap_pkthdr));  // packet
 }
 
-uint64_t FifoDiskFile::query( QueryRequest *qreq, QueryResult *qres, IntervalSet *set) {
+uint64_t FifoDiskFile::query( QueryRequest *qreq, QueryResult *qres, IntervalSet *set, const char* classdirectory) {
 	uint64_t matches = 0;
 	uint64_t scanned_packets=0;
 	ConnectionID4 *c_id;
@@ -255,19 +262,68 @@ uint64_t FifoDiskFile::query( QueryRequest *qreq, QueryResult *qres, IntervalSet
 	int res;
 	int intcnt=0;
 	int first_pkt_for_this_int;
+    //pcapnav_t *ph;
 
+
+    /*
+        if (chdir(conf_main_workdir)) {
+                fprintf(stderr, "cannot class chdir to %s\n", conf_main_workdir);
+                return(1);
+        }
+    */
 	// FIXME: Protect the pcap_dumper_handle from capture thread!!
 	if (is_open)
 		flush();
 
-	//char errbuf[PCAP_ERRBUF_SIZE];
+	char errbuf[PCAP_ERRBUF_SIZE];
+
+    printf("The file name we are querying in is %s\n", filename.c_str());
+
+    if (chdir(classdirectory)) {
+        fprintf(stderr, "cannot class(Fifo:query) chdir to %s\n", classdirectory);
+        //return;
+    }
+
+    char path[70];
+
+    char errbufnav[PCAP_ERRBUF_SIZE];
+
+    printf("The directory that we are in is %s\n", getcwd(path, 70));
+/*
+    if (chdir(classdirectory)) {
+        fprintf(stderr, "cannot class(Fifo:query) chdir to %s\n", classdirectory);
+        //return;
+    }
+*/
+
+    /*
+
+    pcap_t *ph_debug = pcap_open_offline(filename.c_str(), errbuf);
+
+    if (ph_debug == NULL) {
+        fprintf(stderr, "Couldn't open file %s: %s\n", filename.c_str(), errbuf);
+        //exit(EXIT_FAILURE);
+    }
+
+    pcap_close(ph_debug);
+    */
+
+	//ph->pcap=pcap_open_offline(filename.c_str(), errbufnav);
+
+    pcapnav_t *ph = pcapnav_open_offline_tm(filename.c_str(), classdirectory);    
 
-	pcapnav_t *ph=pcapnav_open_offline(filename.c_str());
 	if (!ph) {
+        /*
 		char *pcap_errstr = pcapnav_geterr(ph);
 		tmlog(TM_LOG_ERROR, "query", "%d FifoDiskFile::query: could not open file %s: %s",
 				qres->getQueryID(), filename.c_str(), pcap_errstr);
+        */
+        
+        tmlog(TM_LOG_ERROR, "query", "%d FifoDiskFile::query: could not open file %s",
+                qres->getQueryID(), filename.c_str());
+
 	} else {
+    
 		struct pcap_pkthdr hdr;
 		const u_char *pkt;
 
@@ -373,10 +429,12 @@ uint64_t FifoDiskFile::query( QueryRequest *qreq, QueryResult *qres, IntervalSet
                     tmlog(TM_LOG_ERROR, "Bad IP address: %s", s2);
                     }
                 */
-                //tmlog(TM_LOG_NOTE, "FifoDisk.cc: query", "the query packet has source ip address: %s and dst ip address %s and header time stamp %lu and %lu", \
+
+                
+                //tmlog(TM_LOG_ERROR, "FifoDisk.cc: query", "the query packet has source ip address: %s and dst ip address %s and header time stamp %lu and %lu", \
                 str1, str2, hdr.ts.tv_sec, hdr.ts.tv_usec);
-                //tmlog(TM_LOG_NOTE, "FifoDisk.cc:query", "the query parameters are that it has a time interval from %f to %f, a hash of %lu, a timestamp of %f, and a form of %s", \
-                //qreq->getT0(), qreq->getT1(), qreq->getField()->hash(), qreq->getField()->ts, qreq->getField()->getStr().c_str());
+                //tmlog(TM_LOG_ERROR, "FifoDisk.cc:query", "the query parameters are that it has a time interval from %f to %f, a hash of %lu, a timestamp of %f, and a form of %s", \
+                qreq->getT0(), qreq->getT1(), qreq->getField()->hash(), qreq->getField()->ts, qreq->getField()->getStr().c_str());
 				if (qreq->matchPkt(&hdr, pkt))  {
 					matches++;
 					qres->sendPkt(&hdr, pkt);
diff --git a/src/FifoDisk.hh b/src/FifoDisk.hh
index 413185e..e81d03d 100644
--- a/src/FifoDisk.hh
+++ b/src/FifoDisk.hh
@@ -20,7 +20,7 @@ class FifoDiskFile;
 class FifoDisk {
 public:
 	FifoDisk(const std::string& classname, uint64_t size,
-			 uint64_t file_size, pcap_t*);
+			 uint64_t file_size, pcap_t*, const char* classdir);
 	~FifoDisk();
 	//  void addPkt(const struct pcap_pkthdr *header, const unsigned char *packet);
 	void addPkt(const pkt_ptr p);
@@ -69,6 +69,7 @@ public:
 
 protected:
 	std::string classname;
+        const char* classdir;
 	std::list <FifoDiskFile*> files;
 	uint64_t size;
 	uint64_t file_size;
@@ -117,9 +118,10 @@ public:
 	}
 	/* iterator will be increased up to the first interval completeley
 	   not in file */
-	uint64_t query( QueryRequest*, QueryResult*, IntervalSet*);
+	uint64_t query( QueryRequest*, QueryResult*, IntervalSet*, const char* classdirectory);
 protected:
 	std::string filename;
+        const char* classdir;
 	bool is_open;
 	pcap_dumper_t *pcap_dumper_handle;
 	int64_t cur_file_size;
diff --git a/src/Index.cc b/src/Index.cc
index b100bc1..fd97b3f 100644
--- a/src/Index.cc
+++ b/src/Index.cc
@@ -83,6 +83,12 @@ Index<T>::Index(tm_time_t d_t, int hash_size_index, bool do_disk_index, Storage
         */
 	    cur = new IndexHash(hash_size_index);
 	    old = new IndexHash(hash_size_index);
+        
+        if (chdir(conf_main_workdir)) {
+            fprintf(stderr, "cannot chdir to %s\n", conf_main_workdir);
+            //return(1);
+        }
+        
 	    if (do_disk_index)
 		    disk_index = new IndexFiles<T>((std::string)conf_main_indexdir, "index_"+T::getIndexNameStatic());
 	    else
diff --git a/src/IndexField.cc b/src/IndexField.cc
index 5227c22..e7cc5f9 100644
--- a/src/IndexField.cc
+++ b/src/IndexField.cc
@@ -226,7 +226,7 @@ std::string IPAddress::getStr() const
     
 	if ( GetFamily() == IPv4 )
 		{
-        tmlog(TM_LOG_NOTE, "IPAddress: getStr()", "IPAddress, IPv4");
+        tmlog(TM_LOG_ERROR, "IPAddress: getStr()", "IPAddress, IPv4");
 		char s[INET_ADDRSTRLEN];
 
 		if ( ! bro_inet_ntop(AF_INET, &ipv6_address.s6_addr[12], s, INET_ADDRSTRLEN) ) //bro_inet_ntop(AF_INET, &ipv6_address.s6_addr[12], s, INET_ADDRSTRLEN) )
@@ -237,7 +237,7 @@ std::string IPAddress::getStr() const
 	else
 		{
         
-        tmlog(TM_LOG_NOTE, "IPAddress: getStr()", "IPAddress, IPv6");
+        tmlog(TM_LOG_ERROR, "IPAddress: getStr()", "IPAddress, IPv6");
 		char s[INET6_ADDRSTRLEN];
 
 		if ( ! bro_inet_ntop(AF_INET6, ipv6_address.s6_addr, s, INET6_ADDRSTRLEN) ) //bro_inet_ntop(AF_INET6, ipv6_address.s6_addr, s, INET6_ADDRSTRLEN) )
@@ -654,44 +654,88 @@ IndexField* ConnectionIF4::parseQuery(const char *query) {
 
 void ConnectionIF4:: ip_to_str(const unsigned char* ip, char *str, int len) const {
 //#define UCP(x) ((unsigned char *)&(x))
-	str[0] = '\0';
+        str[0] = '\0';
     /*
-	snprintf(str, len, "%d.%d.%d.%d",
-			 UCP(ip)[0] & 0xff,
-			 UCP(ip)[1] & 0xff,
-			 UCP(ip)[2] & 0xff,
-			 UCP(ip)[3] & 0xff);
+        snprintf(str, len, "%d.%d.%d.%d",
+                         UCP(ip)[0] & 0xff,
+                         UCP(ip)[1] & 0xff,
+                         UCP(ip)[2] & 0xff,
+                         UCP(ip)[3] & 0xff);
     */
+    if (c_id.get_version() == 4)
+    {
+        snprintf(str, len, "%d.%d.%d.%d", ip[12] & 0xff, ip[13] & 0xff, ip[14] & 0xff, ip[15] & 0xff);
+        //printf("The string is %s\n", str);
+        //printf("All of the elements of ip are: %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d\n", ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], ip[8], ip[9], ip[10], ip[11], ip[12], ip[13], ip[14], ip[15]);
+    }
+
+    else
+    {
+        snprintf(str, len, "%s", ip);
+    }
 
-    snprintf(str, len, "%s", ip);
 }
 
 void ConnectionIF4::getBPFStr(char *str, int max_str_len) const {
 
-	char s_ip_str[TM_IP_STR_SIZE];
-	char d_ip_str[TM_IP_STR_SIZE];
-	uint32_t s_port;
-	uint32_t d_port;
-	/*
-	if (c_id.get_is_canonified()) {
-	  s_ip=c_id.get_ip2();
-	  d_ip=c_id.get_ip1();
-	  s_port=c_id.get_port2();
-	  d_port=c_id.get_port1();
-	} else {
-	*/
-	ip_to_str(c_id.get_ip1(), s_ip_str, sizeof(s_ip_str));
-	ip_to_str(c_id.get_ip2(), d_ip_str, sizeof(d_ip_str));
-	s_port=c_id.get_port1();
-	d_port=c_id.get_port2();
-	/*  }  */
+    if (c_id.get_version() == 4)
+    {
+	    char s_ip_str[TM_IP_STR_SIZE];
+	    char d_ip_str[TM_IP_STR_SIZE];
+	    uint32_t s_port;
+	    uint32_t d_port;
+	    /*
+	    if (c_id.get_is_canonified()) {
+	      s_ip=c_id.get_ip2();
+	      d_ip=c_id.get_ip1();
+	      s_port=c_id.get_port2();
+	      d_port=c_id.get_port1();
+	      } else {
+	      */
+	    ip_to_str(c_id.get_ip1(), s_ip_str, sizeof(s_ip_str));
+	    ip_to_str(c_id.get_ip2(), d_ip_str, sizeof(d_ip_str));
+	    s_port=c_id.get_port1();
+	    d_port=c_id.get_port2();
+	      /*  }  */
+
+        //printf("The host %s and pord %d and host %s and port %d", s_ip_str, ntohs(s_port), d_ip_str, ntohs(d_port)); 
+
+	    snprintf(str, max_str_len,
+		      	 "host %s and port %d and host %s and port %d",
+			     s_ip_str, 
+			     ntohs(s_port),
+			     d_ip_str,
+			     ntohs(d_port));
+    }
+
+    else
+    {
+        char s_ip6_str[TM_IP_STR_SIZE];
+        char d_ip6_str[TM_IP_STR_SIZE];
+        uint32_t s6_port;
+        uint32_t d6_port;
+        /*
+        if (c_id.get_is_canonified()) {
+          s_ip=c_id.get_ip2();
+          d_ip=c_id.get_ip1();
+          s_port=c_id.get_port2();
+          d_port=c_id.get_port1();
+          } else {
+          */
+        ip_to_str(c_id.get_ip1(), s_ip6_str, sizeof(s_ip6_str));
+        ip_to_str(c_id.get_ip2(), d_ip6_str, sizeof(d_ip6_str));
+        s6_port=c_id.get_port1();
+        d6_port=c_id.get_port2();
+          /*  }  */
+
+        snprintf(str, max_str_len,
+                 "host %s and port %d and host %s and port %d",
+                 s_ip6_str,
+                 ntohs(s6_port),
+                 d_ip6_str,
+                 ntohs(d6_port));
 
-	snprintf(str, max_str_len,
-			 "host %s and port %d and host %s and port %d",
-			 s_ip_str, 
-			 ntohs(s_port),
-			 d_ip_str,
-			 ntohs(d_port));
+    }
 }
 
 
@@ -762,32 +806,63 @@ IndexField* ConnectionIF3::parseQuery(const char *query) {
 
 void ConnectionIF3:: ip_to_str(const unsigned char* ip, char *str, int len) const {
 //#define UCP(x) ((unsigned char *)&(x))
-	str[0] = '\0';
+        str[0] = '\0';
     /*
-	snprintf(str, len, "%d.%d.%d.%d",
-			 UCP(ip)[0] & 0xff,
-			 UCP(ip)[1] & 0xff,
-			 UCP(ip)[2] & 0xff,
-			 UCP(ip)[3] & 0xff);
+        snprintf(str, len, "%d.%d.%d.%d",
+                         UCP(ip)[0] & 0xff,
+                         UCP(ip)[1] & 0xff,
+                         UCP(ip)[2] & 0xff,
+                         UCP(ip)[3] & 0xff);
     */
+    if (c_id.get_version() == 4)
+    {
+        snprintf(str, len, "%d.%d.%d.%d", ip[12] & 0xff, ip[13] & 0xff, ip[14] & 0xff, ip[15] & 0xff);
+        //printf("The string is %s\n", str);
+        //printf("All of the elements of ip are: %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d\n", ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], ip[8], ip[9], ip[10], ip[11], ip[12], ip[13], ip[14], ip[15]);
+    }
 
-    snprintf(str, len, "%s", ip);
+    else
+    {
+        snprintf(str, len, "%s", ip);
+    }
 }
 
-
 void ConnectionIF3::getBPFStr(char *str, int max_str_len) const {
 
-	char ip1_str[TM_IP_STR_SIZE];
-	char ip2_str[TM_IP_STR_SIZE];
+    if (c_id.get_version() == 4)
+    {
+	    char ip1_str[TM_IP_STR_SIZE];
+	    char ip2_str[TM_IP_STR_SIZE];
+
+	    ip_to_str(c_id.get_ip1(), ip1_str, sizeof(ip1_str));
+	    ip_to_str(c_id.get_ip2(), ip2_str, sizeof(ip2_str));
+
+        //printf("The src host %s and dst host %s and dst port %d or dst host %s and src host %s and src port %d", ip1_str, ip2_str, ntohs(c_id.get_port()), ip1_str, ip2_str, ntohs(c_id.get_port()));
+
+	    snprintf(str, max_str_len,
+		     	 "(src host %s and dst host %s and dst port %d) or "
+			     "(dst host %s and src host %s and src port %d)",
+			     ip1_str, ip2_str, ntohs(c_id.get_port()),
+			     ip1_str, ip2_str, ntohs(c_id.get_port()));
+    }
+
+    else
+    {
+        char ip61_str[TM_IP_STR_SIZE];
+        char ip62_str[TM_IP_STR_SIZE];
+
+        ip_to_str(c_id.get_ip1(), ip61_str, sizeof(ip61_str));
+        ip_to_str(c_id.get_ip2(), ip62_str, sizeof(ip62_str));
+
+        //printf("The src host %s and dst host %s and dst port %d or dst host %s and src host %s and src port %d", ip61_str, ip62_str, ntohs(c_id.get_port()), ip61_str, ip62_str, ntohs(c_id.get_port()));
 
-	ip_to_str(c_id.get_ip1(), ip1_str, sizeof(ip1_str));
-	ip_to_str(c_id.get_ip2(), ip2_str, sizeof(ip2_str));
+        snprintf(str, max_str_len,
+                 "(src host %s and dst host %s and dst port %d) or "
+                 "(dst host %s and src host %s and src port %d)",
+                 ip61_str, ip62_str, ntohs(c_id.get_port()),
+                 ip61_str, ip62_str, ntohs(c_id.get_port()));
 
-	snprintf(str, max_str_len,
-			 "(src host %s and dst host %s and dst port %d) or "
-			 "(dst host %s and src host %s and src port %d)",
-			 ip1_str, ip2_str, ntohs(c_id.get_port()),
-			 ip1_str, ip2_str, ntohs(c_id.get_port()));
+    }
 }
 
 
@@ -856,23 +931,54 @@ void ConnectionIF2:: ip_to_str(const unsigned char* ip, char *str, int len) cons
 			 UCP(ip)[2] & 0xff,
 			 UCP(ip)[3] & 0xff);
     */
+    if (c_id.get_version() == 4)
+    {
+        snprintf(str, len, "%d.%d.%d.%d", ip[12] & 0xff, ip[13] & 0xff, ip[14] & 0xff, ip[15] & 0xff);
+        //printf("The string is %s\n", str);
+        //printf("All of the elements of ip are: %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d\n", ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], ip[8], ip[9], ip[10], ip[11], ip[12], ip[13], ip[14], ip[15]);
+    }
 
-    snprintf(str, len, "%s", ip);
+    else
+    {
+        snprintf(str, len, "%s", ip);
+    }
 }
 
 
 void ConnectionIF2::getBPFStr(char *str, int max_str_len) const {
 
-	char s_ip_str[TM_IP_STR_SIZE];
-	char d_ip_str[TM_IP_STR_SIZE];
 
-	ip_to_str(c_id.get_ip1(), s_ip_str, sizeof(s_ip_str));
-	ip_to_str(c_id.get_ip2(), d_ip_str, sizeof(d_ip_str));
+    if (c_id.get_version() == 4)
+    {
+        char s_ip_str[TM_IP_STR_SIZE];
+        char d_ip_str[TM_IP_STR_SIZE];
+
+        ip_to_str(c_id.get_ip1(), s_ip_str, 16);
+        ip_to_str(c_id.get_ip2(), d_ip_str, 16);
+
 
+        tmlog(TM_LOG_ERROR, "ConnectionIF2::getBPFStr()", "we are in connectionIF2::getBPFStr()");
 
-	snprintf(str, max_str_len,
-			 "host %s and host %s",
-			 s_ip_str, d_ip_str);
+        snprintf(str, max_str_len,
+                 "host %s and host %s",
+                 s_ip_str, d_ip_str);
+
+    }
+    else
+    {
+	    char s_ip6_str[TM_IP_STR_SIZE];
+	    char d_ip6_str[TM_IP_STR_SIZE];
+
+	    ip_to_str(c_id.get_ip1(), s_ip6_str, 16);
+	    ip_to_str(c_id.get_ip2(), d_ip6_str, 16);
+
+
+        tmlog(TM_LOG_ERROR, "ConnectionIF2::getBPFStr()", "we are in connectionIF2::getBPFStr()");
+
+	    snprintf(str, max_str_len,
+		    	 "host %s and host %s",
+			     s_ip6_str, d_ip6_str);
+    }
 }
 
 
diff --git a/src/Query.cc b/src/Query.cc
index 0d87bec..60d41b4 100644
--- a/src/Query.cc
+++ b/src/Query.cc
@@ -41,9 +41,13 @@ bool QueryRequest::matchPkt(const pkt_ptr p) {
 }
 
 bool QueryRequest::matchPkt(struct pcap_pkthdr *hdr, const u_char *pkt) {
-	tmlog(TM_LOG_NOTE, "QueryRequest: matchPkt(struct pcap_pkthdr, u_char pkt)", "determine if a packet matches");
+	//tmlog(TM_LOG_ERROR, "QueryRequest: matchPkt(struct pcap_pkthdr, u_char pkt)", "determine if a packet matches");
+        //printf("QueryRequest:matchPkt, determine if a packet matches");
 	if (!have_bpf) compileBPF();
 
+
+        //printf("QueryRequest:matchPkt, determine if a packet matchesi after compileBPF()\n");
+
     uint16_t ether_type=ntohs(ETHERNET(pkt)->ether_type);
     // ETHERTYPE_IP is EtherType 0x800, for IPv4 addresses
     // EtherType 0x8100 is for VLAN header
@@ -110,15 +114,16 @@ bool QueryRequest::matchPkt(struct pcap_pkthdr *hdr, const u_char *pkt) {
 		}
     */
 
+    //printf("QueryRequest: matchPkt, the value of the bpf_filter in query request for packet %lu and %lu is%lu", hdr->ts.tv_sec, hdr->ts.tv_usec, bpf_filter(fp.bf_insns, (unsigned char*)(pkt), hdr->len, hdr->caplen));
+    //tmlog(TM_LOG_ERROR, "QueryRequest: matchPkt(2 args)", "The value of the bpf_filter in query request  for packet %lu and %lu is %lu", hdr->ts.tv_sec, hdr->ts.tv_usec, bpf_filter(fp.bf_insns, (unsigned char*)(pkt), hdr->len, hdr->caplen));
     
-    tmlog(TM_LOG_NOTE, "QueryRequest: matchPkt(2 args)", "The value of the bpf_filter in query request  for packet %lu and %lu is %lu", hdr->ts.tv_sec, hdr->ts.tv_usec, bpf_filter(fp.bf_insns, (unsigned char*)(pkt), hdr->len, hdr->caplen));
 	return bpf_filter(fp.bf_insns, (u_char *)(pkt),
 					  hdr->len,
 					  hdr->caplen);
 }
 
 void QueryRequest::compileBPF() {
-#define MAX_BPF_STR_LEN 2048 
+#define MAX_BPF_STR_LEN 2048 //8192 
 	/*
 	how long?
 	 
@@ -142,6 +147,11 @@ void QueryRequest::compileBPF() {
 QueryResultFile::QueryResultFile(int queryID, const std::string& filename, int linktype, int snaplen) 
 	: QueryResult(queryID)
 {
+       if (chdir(conf_main_workdir)) {
+           fprintf(stderr, "cannot chdir to %s\n", conf_main_workdir);
+           //return(1);
+        }
+
 	ph = pcap_open_dead(linktype, snaplen);
 	f = new FifoDiskFile(conf_main_queryfiledir+std::string("/")+filename, ph);
 }
diff --git a/src/cmd_parser.yy b/src/cmd_parser.yy
index 26e30fe..bc11ead 100644
--- a/src/cmd_parser.yy
+++ b/src/cmd_parser.yy
@@ -348,11 +348,11 @@ queryspec_flag:
 
 queryspec_key:
 	TOK_INDEX TOK_ID TOK_STRING {
-		//fprintf(stderr, "INDEX QUERY: <%s>, string <%s>\n", $2, $3);
+		fprintf(stderr, "INDEX QUERY: <%s>, string <%s>\n", $2, $3);
 		IndexType *idx=cmd_parser_storage->getIndexes()->getIndexByName($2);
 		if (idx) {
 			IndexField *ifp;
-			//fprintf(stderr, "found the index\n");
+			fprintf(stderr, "found the index\n");
 			ifp = idx->parseQuery($3);
 			//DEBUG fprintf(stderr, ">>>>> %s\n", ifp->getStr().c_str());
 			if (ifp==NULL) 
diff --git a/src/conf.h b/src/conf.h
index ef76e01..2cb8eda 100644
--- a/src/conf.h
+++ b/src/conf.h
@@ -13,6 +13,7 @@ extern int conf_main_log_interval;
 extern int conf_main_log_level;
 extern const char* conf_main_workdir;
 extern const char* conf_main_indexdir;
+//extern const char* conf_classdir;
 extern const char* conf_main_logfile_name;
 extern const char* conf_main_bro_connect_str;
 extern int conf_main_console;
diff --git a/src/conf_parser.yy b/src/conf_parser.yy
index 576b97c..14bd65c 100644
--- a/src/conf_parser.yy
+++ b/src/conf_parser.yy
@@ -67,6 +67,7 @@
 %token <s> TOK_ID
 %token <ipaddr> TOK_IPADDRESS;
 %token TOK_CLASS TOK_FILTER TOK_MAIN TOK_LOG_INTERVAL TOK_LOG_LEVEL TOK_DEVICE
+%token TOK_CLASSDIR
 %token TOK_LOGFILE TOK_WORKDIR TOK_QUERYFILEDIR TOK_INDEXDIR
 %token TOK_READ_TRACEFILE TOK_BRO_CONNECT_STR
 %token TOK_MEM TOK_DISK TOK_K TOK_M TOK_G TOK_CUTOFF TOK_PRECEDENCE
@@ -186,6 +187,12 @@ classoption:
 		newclass->setDynTimeout($2);
 		$$=newclass;
 	}
+    | TOK_CLASSDIR TOK_STRING {
+		new_class();
+        //conf_classdir = ($2);
+        newclass->setClassdir($2);
+        $$=newclass;
+	}
 	;
 size:
 	TOK_INTEGER { $$=$1; }
diff --git a/src/conf_scanner.ll b/src/conf_scanner.ll
index 2176765..f9a83af 100644
--- a/src/conf_scanner.ll
+++ b/src/conf_scanner.ll
@@ -38,6 +38,7 @@ NEWLINE \n
 "]"		 return RBRACK;
 ";"		 return SEMICOLON;
 "class"		 return TOK_CLASS;
+"classdir"	 return TOK_CLASSDIR;
 "mem"		 return TOK_MEM;
 "disk"		 return TOK_DISK;
 "filesize"	 return TOK_FILESIZE;
diff --git a/src/main.cc.in b/src/main.cc.in
index 31dc80b..67d4b88 100644
--- a/src/main.cc.in
+++ b/src/main.cc.in
@@ -28,7 +28,7 @@
 #define USE_MALLINFO
 #endif
 #include <sys/ioctl.h>
-#include <gperftools/profiler.h>
+//#include <gperftools/profiler.h>
 
 
 #ifdef USE_BROCCOLI
@@ -117,6 +117,7 @@ int conf_main_log_interval=60;
 int conf_main_log_level=20;
 const char* conf_main_workdir="./";
 const char* conf_main_indexdir="./";
+//const char* conf_classdir = "./";
 const char* conf_main_queryfiledir="./";
 const char* conf_main_logfile_name="timemachine.log";
 const char* conf_main_bro_connect_str=NULL;
@@ -673,6 +674,12 @@ void *statisticslog_thread(void* arg)
 		fifos = storage->getFifos();
 		for(std::list<Fifo*>::iterator i=fifos.begin(); i!=fifos.end(); i++) 
         {
+            /*
+            if (chdir((*i)->getClassdir())) {
+                fprintf(stderr, "cannot class chdir to %s\n", (*i)->getClassdir());
+                //return;
+            }
+            */
             
 			snprintf(s, LOG_STRING_SIZE, "%.2lf %s %s", to_tm_time(&now),  (*i)->getClassname().c_str(), (*i)->getStatsStr().c_str());
             /*
@@ -685,7 +692,12 @@ void *statisticslog_thread(void* arg)
             */
 			classes_log_file->logPlain(s);
 		}
-
+        /*
+        if (chdir(conf_main_workdir)) {
+                fprintf(stderr, "cannot chdir to %s\n", conf_main_workdir);
+                //return(1);
+        }
+        */
 		indexes = storage->getIndexes();
 		for (std::list<IndexType*>::iterator i=indexes->begin(); i!=indexes->end(); i++) 
         {
@@ -808,7 +820,7 @@ void usage() {
 int
 main(int argc, char** argv) {
 
-        ProfilerStart("/home/neto/development-tmV2-naoki/timemachine_results/profile/blah.prof");
+        //ProfilerStart("/home/neto/data/profile/blah.prof");
 	const char *conffile="@BRO_ETC_INSTALL_DIR@/timemachine.cfg";
 	struct sigaction exit_action;
 	int i;
@@ -1046,7 +1058,7 @@ main(int argc, char** argv) {
 				pause();
 		}
 
-        ProfilerStop();
+        //ProfilerStop();
         delete stats_log_file;
         delete classes_log_file;
         delete index_log_file; 
diff --git a/src/pcapnav/pcapnav.c b/src/pcapnav/pcapnav.c
index c3cb442..bea63b1 100644
--- a/src/pcapnav/pcapnav.c
+++ b/src/pcapnav/pcapnav.c
@@ -71,7 +71,7 @@ pcapnav_init(void)
 
 
 pcapnav_t      *
-pcapnav_open_offline(const char *fname)
+pcapnav_open_offline_tm(const char *fname, const char* classdirectory)
 {
   pcapnav_t               *pn;
   u_int32_t                magic;
@@ -79,11 +79,70 @@ pcapnav_open_offline(const char *fname)
   FILE                    *fp;
   struct stat              st;
 
+  FILE *fp_log;
+
+  char filepath[100];
+
+  strcpy(filepath, classdirectory);
+  strcat(filepath, "/");
+  strcat(filepath, fname);
+
+  char logpath[] = "/home/neto/data/pcapLog.txt";
+
+  if (chdir(classdirectory)) {
+      fprintf(stderr, "cannot class(Fifo:query) chdir to %s\n", classdirectory);
+      //return;
+  }
+  /*  
+  fp_log = fopen(logpath, "a");
+
+  if (fp_log == NULL) {
+    fprintf(stderr, "Can't open the log file pcapLog.txt\n");
+  }
+  else
+  {
+      char pcappath[70];
+
+      fprintf(fp_log, "The directory that we are in while in the pcapnav.c file is %s for filename %s\n", getcwd(pcappath, 70), filepath);
+      fclose(fp_log);
+  }
+  */
+  /*
+  fp_log = fopen("/home/lakers/pcapLog.txt", "a");
+
+  if (fp_log == NULL) {
+    fprintf(stderr, "Can't open the log file pcapLog.txt\n");
+  }
+  else
+  {
+      fprintf(fp_log, "come on, work with filename %s!\n", fname);
+      fclose(fp_log);
+  }
+  */
+  //fclose(fp_log);
+
+  //fp_log = fopen("/home/lakers/pcapLog.txt", "w");
+
   D_ENTER;
 
-  if (!fname || fname[0] == '\0')
+  if (!filepath || filepath[0] == '\0')
     {
+      fp_log = fopen(logpath, "a");
+
+      if (fp_log == NULL) {
+         fprintf(stderr, "Can't open the log file pcapLog.txt\n");
+      }
+
+      else
+      {
+          fprintf(fp_log, "Invalid filename: %s\n", fname);
+          fclose(fp_log);
+      }      
+
       D(("Invalid filename: %s\n", fname));
+
+      //fprintf(stderr, "Invalid filename: %s\n", filepath); //fname);
+
       errno = ENOENT;
       D_RETURN_(NULL);
     }
@@ -92,41 +151,156 @@ pcapnav_open_offline(const char *fname)
   
   if (! (pn = NEW(pcapnav_t)))
     {
+      fp_log = fopen(logpath, "a");
+
+      if (fp_log == NULL) {
+         fprintf(stderr, "Can't open the log file pcapLog.txt\n");
+      }
+
+      else
+      {
+          fprintf(fp_log, "Out of memory\n");
+          fclose(fp_log);
+      }
+      //fclose(fp_log);
+
       D(("Out of memory.\n"));
+
+      //fprintf(stderr, "Out of memory.\n");
+
       errno = ENOMEM;
       D_RETURN_(NULL);
     }
+  /*
+  if (chdir(classdirectory)) {
+      fprintf(stderr, "cannot class(Fifo:query) chdir to %s\n", classdirectory);
+      //return;
+  }
 
-  if (lstat(fname, &st) < 0)
+  fp_log = fopen("/home/lakers/pcapLog.txt", "a");
+
+  if (fp_log == NULL) {
+    fprintf(stderr, "Can't open the log file pcapLog.txt\n");
+  }
+  else
+  {
+      char pcappath[70];
+
+      fprintf(fp_log, "The directory that we are in while in the pcapnav.c file is %s for filename %s\n", getcwd(pcappath, 70), fname);
+      fclose(fp_log);
+  }  
+  */
+  
+  //if (lstat(fname, &st) < 0)
+  if (lstat(filepath, &st) < 0)
     {
+      fp_log = fopen(logpath, "a");
+
+      if (fp_log == NULL) {
+         fprintf(stderr, "Can't open the log file pcapLog.txt\n");
+      }
+
+      else
+      {
+          char path[70];
+          fprintf(fp_log, "lstat failed for file %s and error %s, we are in the path %s\n", filepath, strerror(errno), getcwd(path, 70));
+
+          fclose(fp_log);
+      }
+      //fclose(fp_log);
+
       D(("lstat failed: %s\n", strerror(errno)));
+
+      //fprintf("lstat failed: %s for file %s\n", strerror(errno), fname);
+
       goto free_return;
     }
   
   pn->size = st.st_size;
 
   /* Allocate pcap handle */
-  if (! (pn->pcap = pcap_open_offline(fname, pcap_errbuf)))
+  if (! (pn->pcap = pcap_open_offline(filepath, pcap_errbuf)))
     {
-      D(("%s (from pcap, re. %s)\n", pcap_errbuf, fname));
+        if (pn->pcap == NULL) {
+            fp_log = fopen(logpath, "a");
+
+            if (fp_log == NULL) {
+                fprintf(stderr, "Can't open the log file pcapLog.txt\n");
+            }
+
+            else
+            {
+                char path[70];
+                fprintf(fp_log, "Could not open the file %s and error %s, the path is %s\n", filepath, pcap_errbuf, getcwd(path, 70));
+                fclose(fp_log);
+            }
+            //fclose(fp_log);
+
+            //fprintf(stderr, "Couldn't open the file: %s\n", pcap_errbuf);
+            //exit(EXIT_FAILURE);
+        }
+
+      D(("%s (from pcap, re. %s)\n", pcap_errbuf, filepath)); //fname));
       /* Let's hope errno is meaningful now ... */
       goto free_return;
     }
 
+    /*
+    if (pn->pcap == NULL) {
+        fprintf(stderr, "Couldn't open the file: %s\n", pcap_errbuf);
+        //exit(EXIT_FAILURE);
+    }
+    */
+  
   /* Hook pcap's file stream into our own structure: */
   pn->fp = pcap_file(pn->pcap);
 
-  if ((fp = fopen(fname, "r")) == NULL)
+  if (chdir(classdirectory)) {
+      fprintf(stderr, "cannot class(Fifo:query) chdir to %s\n", classdirectory);
+      //return;
+  }
+  
+  if ((fp = fopen(filepath, "r")) == NULL)
     {
-      D(("Could not open trace file %s for reading.\n", fname));
-      /* errno set already */
+      fp_log = fopen(logpath, "a");
+      if (fp_log == NULL) {
+         fprintf(stderr, "Can't open the log file pcapLog.txt\n");
+      }
+
+      else
+      {
+          char path[70];
+          fprintf(fp_log, "Could not open trace file %s for reading. The path is %s\n", filepath, getcwd(path, 70));
+          fclose(fp_log);
+      }
+      //fclose(fp_log);
+
+      //fprintf(stderr, "Couldn't open the trace file for reading: %s\n", fname);
+
+      D(("Could not open trace file %s for reading.\n", filepath));
+      // errno set already
       goto free_return;
     }
   
   if (fread((char *)&pn->trace.filehdr, sizeof(struct pcap_file_header), 1, fp) != 1)
     {
-      D(("Could not read trace file header from %s\n", fname));
-      /* errno set already */
+
+      fp_log = fopen(logpath, "a");
+
+      if (fp_log == NULL) {
+         fprintf(stderr, "Can't open the log file pcapLog.txt\n");
+      }
+      else
+      {
+         fprintf(fp_log, "Could not read trace file header from %s\n", filepath); //fname);
+         fclose(fp_log);
+      }
+      //fclose(fp_log);
+
+      //fprintf(stderr, "Couldn't open the trace file header for reading: %s\n", fname);
+
+      D(("Could not read trace file header from %s\n", filepath)); //fname));
+      //errno set already
       goto cleanup_return;
     }
   
@@ -141,7 +315,22 @@ pcapnav_open_offline(const char *fname)
 
       if (magic != TCPDUMP_MAGIC && magic != PATCHED_TCPDUMP_MAGIC)
 	{
-	  D(("Invalid trace file %s -- didn't recognize file magic.\n", fname));
+	  D(("Invalid trace file %s -- didn't recognize file magic.\n", filepath)); //fname));
+
+          fp_log = fopen(logpath, "a");
+
+          if (fp_log == NULL) {
+               fprintf(stderr, "Can't open the log file pcapLog.txt\n");
+           }
+          else
+          { 
+              fprintf(fp_log, "Invalid trace file %s, did not recognize file magic.\n", filepath); //fname);
+              fclose(fp_log);
+          }
+          //fclose(fp_log);
+
+          //fprintf(stderr, "Invalid trace file %s -- didn't recognize file magic \n", fname);
+
 	  goto cleanup_return;
 	}
 
@@ -182,21 +371,55 @@ pcapnav_open_offline(const char *fname)
   /* Get length of file: */
   if (fseek(fp, 0, SEEK_END) != 0)
     {
+      fp_log = fopen(logpath, "a");
+
+      if (fp_log == NULL) {
+         fprintf(stderr, "Can't open the log file pcapLog.txt\n");
+      }
+
+      else
+      {
+          fprintf(fp_log, "Could not determine file length, fseek failed: %s with error %s\n", filepath, strerror(errno)); //fname, strerror(errno));
+          fclose(fp_log);
+      }
+      //fclose(fp_log);
+
       D(("Couldn't determine file length, fseek() failed: %s\n", strerror(errno)));
+
+      //fprintf(stderr, "Couldn't determine the file length, fseek failed: %s for file %s", strerror(errno), fname);
+
       goto cleanup_return;
     }
 
   if ((pn->trace.length = ftell(fp)) < 0)
     {
+      fp_log = fopen(logpath, "a");
+
+      if (fp_log == NULL) {
+         fprintf(stderr, "Can't open the log file pcapLog.txt\n");
+      }
+      else
+      {
+          fprintf(fp_log, "Could not determine file length, ftell failed: %s with error %s\n", filepath, strerror(errno)); //fname, strerror(errno));
+          fclose(fp_log);
+      }
+      //fclose(fp_log);
+
       D(("Couldn't determine file length, ftell() failed: %s\n", strerror(errno)));
+
+      //fprintf(stderr, "Coudln't determine file length, ftell failed: %s for file %s\n", strerror(errno), fname);
+
       goto cleanup_return;
     }
 
+  //fclose(fp_log);
+
   fclose(fp);  
   D_RETURN_(pn);
   
  cleanup_return:
   fclose(fp);
+  //fclose(fp_log);
   
  free_return:
   FREE(pn);
@@ -683,7 +906,7 @@ pcapnav_geterr(pcapnav_t *pn)
 
 
 pcap_dumper_t *
-pcapnav_dump_open(pcap_t *pcap, const char *filename, pcapnav_dumpmode_t mode)
+pcapnav_dump_open_tm(pcap_t *pcap, const char *filename, pcapnav_dumpmode_t mode, const char* classdirectory)
 {
   if (!pcap)
     {
@@ -703,10 +926,10 @@ pcapnav_dump_open(pcap_t *pcap, const char *filename, pcapnav_dumpmode_t mode)
   switch (mode)
     {
     case PCAPNAV_DUMP_APPEND_FAST:
-      return pcapnav_append_fast(pcap, filename);
+      return pcapnav_append_fast(pcap, filename, classdirectory);
       
     case PCAPNAV_DUMP_APPEND_SAFE:
-      return pcapnav_append_safe(pcap, filename);
+      return pcapnav_append_safe(pcap, filename, classdirectory);
 
     case PCAPNAV_DUMP_TRUNC:
     default:      
diff --git a/src/pcapnav/pcapnav.h b/src/pcapnav/pcapnav.h
index 4ab4919..6293eed 100644
--- a/src/pcapnav/pcapnav.h
+++ b/src/pcapnav/pcapnav.h
@@ -117,7 +117,7 @@ void               pcapnav_init(void);
  * file not readable etc). In that case check errno to
  * see what went wrong.
  */
-pcapnav_t         *pcapnav_open_offline(const char *filename);
+pcapnav_t         *pcapnav_open_offline_tm(const char *filename, const char* classdirectory);
 
 
 /**
@@ -509,8 +509,8 @@ char              *pcapnav_geterr(pcapnav_t *pn);
  * Returns: a pcap dumper, or %NULL if something went wrong, in which
  * case you'll find the reason in the error buffer in the @pcap handle.
  */
-pcap_dumper_t     *pcapnav_dump_open(pcap_t *pcap, const char *filename,
-				     pcapnav_dumpmode_t mode);
+pcap_dumper_t     *pcapnav_dump_open_tm(pcap_t *pcap, const char *filename,
+				     pcapnav_dumpmode_t mode, const char* classdirectory);
 
 #ifdef __cplusplus
 }
diff --git a/src/pcapnav/pcapnav_append.c b/src/pcapnav/pcapnav_append.c
index 9da1ef4..fda5052 100644
--- a/src/pcapnav/pcapnav_append.c
+++ b/src/pcapnav/pcapnav_append.c
@@ -128,7 +128,7 @@ append_fix_trunc_packet(pcapnav_t *pn, FILE *result)
 
 
 static pcap_dumper_t *
-append_impl(pcap_t *pcap, const char *filename, pcapnav_dumpmode_t mode)
+append_impl(pcap_t *pcap, const char *filename, pcapnav_dumpmode_t mode, const char* classdirectory)
 {
   pcapnav_t *pn = NULL;
   FILE *result = NULL;
@@ -148,7 +148,7 @@ append_impl(pcap_t *pcap, const char *filename, pcapnav_dumpmode_t mode)
        */
     }
   
-  if (! (pn = pcapnav_open_offline(filename)))
+  if (! (pn = pcapnav_open_offline_tm(filename, classdirectory)))
     {
       D(("Error opening '%s'\n", filename));
       return NULL;
@@ -223,15 +223,15 @@ append_impl(pcap_t *pcap, const char *filename, pcapnav_dumpmode_t mode)
 
 
 pcap_dumper_t *
-pcapnav_append_fast(pcap_t *pcap, const char *filename)
+pcapnav_append_fast(pcap_t *pcap, const char *filename, const char* classdirectory)
 {
-  return append_impl(pcap, filename, PCAPNAV_DUMP_APPEND_FAST);
+  return append_impl(pcap, filename, PCAPNAV_DUMP_APPEND_FAST, classdirectory);
 }
 
 
 pcap_dumper_t *   
-pcapnav_append_safe(pcap_t *pcap, const char *filename)
+pcapnav_append_safe(pcap_t *pcap, const char *filename, const char* classdirectory)
 {
-  return append_impl(pcap, filename, PCAPNAV_DUMP_APPEND_SAFE);
+  return append_impl(pcap, filename, PCAPNAV_DUMP_APPEND_SAFE, classdirectory);
 }
 
diff --git a/src/pcapnav/pcapnav_append.h b/src/pcapnav/pcapnav_append.h
index c62a37d..478378f 100644
--- a/src/pcapnav/pcapnav_append.h
+++ b/src/pcapnav/pcapnav_append.h
@@ -27,7 +27,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 #include "pcapnav.h"
 
-pcap_dumper_t *   pcapnav_append_fast(pcap_t *pcap, const char *filename);
-pcap_dumper_t *   pcapnav_append_safe(pcap_t *pcap, const char *filename);
+pcap_dumper_t *   pcapnav_append_fast(pcap_t *pcap, const char *filename, const char* classdirectory);
+pcap_dumper_t *   pcapnav_append_safe(pcap_t *pcap, const char *filename, const char* classdirectory);
 
 #endif
diff --git a/src/tm.h b/src/tm.h
index fe9e590..3102fe6 100644
--- a/src/tm.h
+++ b/src/tm.h
@@ -7,7 +7,7 @@
 
 #include "types.h"
 
-// #define QUERY_RACE_PROTECT
+//#define QUERY_RACE_PROTECT 1
 
 
 /* If TM_HEAVY_DEBUG is defined, quite some expensive code with asserts,


From neto at caltech.edu  Tue Aug 19 16:50:17 2014
From: neto at caltech.edu (Naoki Eto)
Date: Tue, 19 Aug 2014 16:50:17 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Fixed the implementation for
	querying for IPv6 for conn2/conn3/conn4 Fixed the regular
	expression so that it is more code friendly (less matching
	arguments needed). (fdf52f7)
Message-ID: <201408192350.s7JNoHcB014890@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit fdf52f71bcc0bd08b62a1b361ae55c311f49a399
Author: NaokiEto <neto at caltech.edu>
Date:   Tue Aug 19 19:37:50 2014 -0400

    Fixed the implementation for querying for IPv6 for conn2/conn3/conn4
    Fixed the regular expression so that it is more code friendly (less matching arguments needed).


>---------------------------------------------------------------

fdf52f71bcc0bd08b62a1b361ae55c311f49a399
 src/Connection.cc           | 106 ++++++++++++-------
 src/Connection.hh           |  54 +++++++---
 src/DiskIndex.cc            |  13 +++
 src/IndexField.cc           | 253 ++++++++++++++++++++++++++++++++++++--------
 src/IndexField.hh           |   4 +-
 src/re2/variadic_function.h | 142 +++++++++++++++++++++++++
 6 files changed, 468 insertions(+), 104 deletions(-)

diff --git a/src/Connection.cc b/src/Connection.cc
index 00661be..ec031fa 100644
--- a/src/Connection.cc
+++ b/src/Connection.cc
@@ -22,10 +22,13 @@
 static std::string pattern_ip ("(\\d+\\.\\d+\\.\\d+\\.\\d+)");
 static std::string pattern_ipport ("(\\d+\\.\\d+\\.\\d+\\.\\d+):(\\d+)");
 
-static std::string pattern_ip6 ("\\[(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]");
+static std::string pattern_ip6 ("\\[([0-9a-fA-F]{1,4}:{7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]");
 // stolen from stackoverflow http://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
 
-static std::string pattern_ip6port ("\\[(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]:(\\d+)");
+//static std::string pattern_ip6 ("\\[([0-9a-fA-F]{1,4}:{7,7}[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,7}:|[0-9a-fA-F]{1,4}:{1,6}:[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,5}:[0-9a-fA-F]{1,4}{1,2}|[0-9a-fA-F]{1,4}:{1,4}:[0-9a-fA-F]{1,4}{1,3}|[0-9a-fA-F]{1,4}:{1,3}:[0-9a-fA-F]{1,4}{1,4}|[0-9a-fA-F]{1,4}:{1,2}:[0-9a-fA-F]{1,4}{1,5}|[0-9a-fA-F]{1,4}::[0-9a-fA-F]{1,4}{1,6}|::[0-9a-fA-F]{1,4}{1,7}|:|fe80::[0-9a-fA-F]{0,4}{0,4}%[0-9a-zA-Z]{1,}|::ffff:0{1,4}{0,1}:{0,1}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9]|[0-9a-fA-F]{1,4}:{1,4}:25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9])\\]");
+
+static std::string pattern_ip6port ("\\[([0-9a-fA-F]{1,4}:{7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]:(\\d+)");
+//static std::string pattern_ip6port ("\\[([0-9a-fA-F]{1,4}:{7,7}[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,7}:|[0-9a-fA-F]{1,4}:{1,6}:[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,5}:[0-9a-fA-F]{1,4}{1,2}|[0-9a-fA-F]{1,4}:{1,4}:[0-9a-fA-F]{1,4}{1,3}|[0-9a-fA-F]{1,4}:{1,3}:[0-9a-fA-F]{1,4}{1,4}|[0-9a-fA-F]{1,4}:{1,2}:[0-9a-fA-F]{1,4}{1,5}|[0-9a-fA-F]{1,4}::[0-9a-fA-F]{1,4}{1,6}|::[0-9a-fA-F]{1,4}{1,7}|:|fe80::[0-9a-fA-F]{0,4}{0,4}%[0-9a-zA-Z]{1,}|::ffff:0{1,4}{0,1}:{0,1}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9]|[0-9a-fA-F]{1,4}:{1,4}:25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9])\\]:(\\d+)");
 // stolen from stackoverflow http://stackoverflow.com/questions/53497/regular-expression-that-
 
 /*
@@ -91,11 +94,11 @@ void ConnectionID4::init(proto_t proto4,
         //key.ip1.s6_tm_addr = &v4_mapped_prefix[0];
         //char *
 
-        memcpy(key.ip1, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip1[12], &d_ip, sizeof(d_ip));
+        memcpy(key.ip1.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip1.s6_addr[12], &d_ip, sizeof(d_ip));
 
-        memcpy(key.ip2, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip2[12], &s_ip, sizeof(s_ip));
+        memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip2.s6_addr[12], &s_ip, sizeof(s_ip));
 
 
 
@@ -135,11 +138,11 @@ void ConnectionID4::init(proto_t proto4,
 
         // setting v6.ip1 to be source address and v6.ip2 to be destination address
 
-        memcpy(key.ip1, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip1[12], &s_ip, sizeof(s_ip));
+        memcpy(key.ip1.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip1.s6_addr[12], &s_ip, sizeof(s_ip));
 
-        memcpy(key.ip2, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip2[12], &d_ip, sizeof(d_ip));
+        memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip2.s6_addr[12], &d_ip, sizeof(d_ip));
 
 
         /*
@@ -208,8 +211,8 @@ void ConnectionID4::init6(proto_t proto6,
 	if (addr6_port_canon_lt(s_ip,d_ip,s_port,d_port)) {
 		//    v6.is_canonified=true;
         // memcpy(destination, source, size)
-        memcpy(key.ip1, d_ip, 16);
-        memcpy(key.ip2, s_ip, 16);
+        memcpy(key.ip1.s6_addr, d_ip, 16);
+        memcpy(key.ip2.s6_addr, s_ip, 16);
 		//v6.ip1=d_ip;
 		//v6.ip2=s_ip;
 		//v6.port1=d_port;
@@ -229,8 +232,8 @@ void ConnectionID4::init6(proto_t proto6,
 
 	} else {
 		//    v6.is_canonified=false;
-        memcpy(key.ip1, s_ip, 16);
-        memcpy(key.ip2, d_ip, 16);
+        memcpy(key.ip1.s6_addr, s_ip, 16);
+        memcpy(key.ip2.s6_addr, d_ip, 16);
 		//v6.ip1=s_ip;
 		//v6.ip2=d_ip;
 		//v6.port1=s_port;
@@ -299,11 +302,11 @@ void ConnectionID3::init(proto_t proto4,
 
     */
 
-    memcpy(key.ip1, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-    memcpy(&key.ip1[12], &ip1, sizeof(ip1));
+    memcpy(key.ip1.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+    memcpy(&key.ip1.s6_addr[12], &ip1, sizeof(ip1));
 
-    memcpy(key.ip2, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-    memcpy(&key.ip2[12], &ip2, sizeof(ip2));
+    memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+    memcpy(&key.ip2.s6_addr[12], &ip2, sizeof(ip2));
 
 
 	//key.port2=port2;
@@ -377,8 +380,8 @@ void ConnectionID3::init6(proto_t proto6,
 
     // memcpy(destination, source, size)
     // setting v6.ip1 to be source address and v6.ip2 to be destination address 
-    memcpy(key.ip1, ip1, 16);
-    memcpy(key.ip2, ip2, 16);
+    memcpy(key.ip1.s6_addr, ip1, 16);
+    memcpy(key.ip2.s6_addr, ip2, 16);
 
 	//v.ip1=ip1;
 	//v.ip2=ip2;
@@ -474,11 +477,11 @@ void ConnectionID2::init( uint32_t s_ip, uint32_t d_ip) {
 
         // setting v6.ip1 to dest address and v6.ip2 to source address
 
-        memcpy(key.ip1, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip1[12], &d_ip, sizeof(d_ip));
+        memcpy(key.ip1.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip1.s6_addr[12], &d_ip, sizeof(d_ip));
 
-        memcpy(key.ip2, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip2[12], &s_ip, sizeof(s_ip));
+        memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip2.s6_addr[12], &s_ip, sizeof(s_ip));
 
         // this is for the hash key
         //in6_addr s6_ip;
@@ -506,11 +509,11 @@ void ConnectionID2::init( uint32_t s_ip, uint32_t d_ip) {
 
         // setting v6.ip1 to be source address and v6.ip2 to be dest address
 
-        memcpy(key.ip1, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip1[12], &s_ip, sizeof(s_ip));
+        memcpy(key.ip1.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip1.s6_addr[12], &s_ip, sizeof(s_ip));
 
-        memcpy(key.ip2, v4_mapped_prefix, sizeof(v4_mapped_prefix));
-        memcpy(&key.ip2[12], &d_ip, sizeof(d_ip));
+        memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
+        memcpy(&key.ip2.s6_addr[12], &d_ip, sizeof(d_ip));
 
 
         // this is for the hash key
@@ -551,8 +554,8 @@ void ConnectionID2::init6( unsigned char s_ip[], unsigned char d_ip[]) {
 	if (addr6_port_canon_lt(s_ip,d_ip,0,0)) {
 		//    v.is_canonified=true;
         // memcpy(destination, source, size)
-        memcpy(key.ip1, d_ip, 16);
-        memcpy(key.ip2, s_ip, 16);
+        memcpy(key.ip1.s6_addr, d_ip, 16);
+        memcpy(key.ip2.s6_addr, s_ip, 16);
 
 		//v6.ip1=d_ip;
 		//v6.ip2=s_ip;
@@ -571,8 +574,8 @@ void ConnectionID2::init6( unsigned char s_ip[], unsigned char d_ip[]) {
 
 	} else {
 		//    v.is_canonified=false;
-        memcpy(key.ip1, s_ip, 16);
-        memcpy(key.ip2, d_ip, 16);
+        memcpy(key.ip1.s6_addr, s_ip, 16);
+        memcpy(key.ip2.s6_addr, d_ip, 16);
 		//v.ip1=s_ip;
 		//v.ip2=d_ip;
 
@@ -589,6 +592,17 @@ void ConnectionID2::init6( unsigned char s_ip[], unsigned char d_ip[]) {
 	    key.port2 = 0;
 	}
 
+    //char str_test1[INET6_ADDRSTRLEN];
+    //char str_test2[INET6_ADDRSTRLEN];
+
+    // now get it back and print it
+    //inet_ntop(AF_INET6, &(key.ip1), str_test1, INET6_ADDRSTRLEN);
+
+    //inet_ntop(AF_INET6, &(key.ip2), str_test2, INET6_ADDRSTRLEN);
+
+    //printf("testing 1 ... %s\n", str_test1); // prints "2001:db8:8714:3a90::12"
+    //printf("testing 2 ... %s\n", str_test2);
+
     //init_hash_function();
 
     //HashKey* newHashKey = new HashKey(&key, sizeof(key));
@@ -781,6 +795,7 @@ ConnectionID2::ConnectionID2(const u_char* packet) {
     else
     {
         //tmlog(TM_LOG_NOTE, "Connection.cc: ConnectionID3", "IPv6");
+        //printf("the source address is %s, and the dest address is %s\n", IP6(packet)->ip6_src.s6_addr, IP6(packet)->ip6_dst.s6_addr);
     	init6(IP6(packet)->ip6_src.s6_addr, IP6(packet)->ip6_dst.s6_addr);
     }
 }
@@ -808,11 +823,11 @@ bool ConnectionID3::operator==(const ConnectionID& other) const {
 //consistent with ConnectionID4
 bool ConnectionID2::operator==(const ConnectionID& other) const {
 /*
-        return equal(key.ip1.s6_addr, ((ConnectionID2*)&other)->key.ip2.s6_addr)
-               && equal(key.ip2.s6_addr, ((ConnectionID2*)&other)->key.ip2.s6_addr);
+        //return equal(key.ip1.s6_addr, ((ConnectionID2*)&other)->key.ip2.s6_addr)
+               //&& equal(key.ip2.s6_addr, ((ConnectionID2*)&other)->key.ip2.s6_addr);
 */
-       return (!memcmp(&key.ip1, &((ConnectionID2*)&other)->key.ip1, 16))
-               && (!memcmp(&key.ip2, &((ConnectionID2*)&other)->key.ip2, 16));
+       return (!memcmp(&key.ip1, &((ConnectionID2*)&other)->key.ip1, sizeof(in6_addr)))
+               && (!memcmp(&key.ip2, &((ConnectionID2*)&other)->key.ip2, sizeof(in6_addr)));
 
 }
 
@@ -921,11 +936,11 @@ std::string ConnectionID4::getStr() const {
 
         char str1[INET6_ADDRSTRLEN];
 
-        bro_inet_ntop(AF_INET6, key.ip1, str1, INET6_ADDRSTRLEN);
+        inet_ntop(AF_INET6, &key.ip1, str1, INET6_ADDRSTRLEN);
 
         char str2[INET6_ADDRSTRLEN];
 
-        bro_inet_ntop(AF_INET6, key.ip2, str2, INET6_ADDRSTRLEN);
+        inet_ntop(AF_INET6, &key.ip2, str2, INET6_ADDRSTRLEN);
 
 
         ss << " ConnectionID4 for IPv6"
@@ -1086,14 +1101,25 @@ std::string ConnectionID2::getStr() const {
 
         // I already put v6.ip1 and v6.ip2 in there
 
+        //struct sockaddr_in6 sa;
+        //char str_test[INET6_ADDRSTRLEN];
+
+        // store this IP address in sa:
+        //inet_pton(AF_INET6, "fe80::213:faff:fe03:c11e", &(sa.sin6_addr));
+
+        // now get it back and print it
+        //inet_ntop(AF_INET6, &(sa.sin6_addr), str_test, INET6_ADDRSTRLEN);
+
+        //printf("testing ... %s\n", str_test); // prints "2001:db8:8714:3a90::12"
+
 
         char str1[INET6_ADDRSTRLEN];
 
-        bro_inet_ntop(AF_INET6, &(key.ip1), str1, INET6_ADDRSTRLEN);
+        inet_ntop(AF_INET6, &(key.ip1), str1, INET6_ADDRSTRLEN);
 
         char str2[INET6_ADDRSTRLEN];
 
-        bro_inet_ntop(AF_INET6, &(key.ip2), str2, INET6_ADDRSTRLEN);
+        inet_ntop(AF_INET6, &(key.ip2), str2, INET6_ADDRSTRLEN);
 
         ss << " ConnectionID2 for IPv6"
 
diff --git a/src/Connection.hh b/src/Connection.hh
index 4a0eef0..9101ab5 100644
--- a/src/Connection.hh
+++ b/src/Connection.hh
@@ -56,8 +56,8 @@ public:
         init6(proto, s_ip, d_ip, s_port, d_port);
     }
 	ConnectionID4(ConnectionID4 *c_id) {
-        memcpy(key.ip1, c_id->key.ip1, 16);
-        memcpy(key.ip2, c_id->key.ip2, 16);
+        memcpy(key.ip1.s6_addr, c_id->key.ip1.s6_addr, 16);
+        memcpy(key.ip2.s6_addr, c_id->key.ip2.s6_addr, 16);
 		key.port1 = c_id->key.port1;
 		key.port2 = c_id->key.port2;
 		v6.proto = c_id->v6.proto;
@@ -104,8 +104,8 @@ public:
                         && (v6.proto == ((ConnectionID4*)&other)->v6.proto);
                */
                  
-		return (!memcmp(&key.ip1, &((ConnectionID4*)&other)->key.ip1, 16))
-			   && (!memcmp(&key.ip2, &((ConnectionID4*)&other)->key.ip2, 16))
+		return (!memcmp(&key.ip1.s6_addr, &((ConnectionID4*)&other)->key.ip1.s6_addr, 16))
+			   && (!memcmp(&key.ip2.s6_addr, &((ConnectionID4*)&other)->key.ip2.s6_addr, 16))
 			   && (key.port1 == ((ConnectionID4*)&other)->key.port1)
 			   && (key.port2 == ((ConnectionID4*)&other)->key.port2)
 			   && (v6.proto == ((ConnectionID4*)&other)->v6.proto);
@@ -142,11 +142,17 @@ public:
 		return v6.proto;
 	}
 	const unsigned char* get_ip1() const {
-		return key.ip1;
+		return key.ip1.s6_addr;
 	}
+    const in6_addr* get_ip1_addr() const {
+        return &(key.ip1);
+    }
 	const unsigned char* get_ip2() const {
-		return key.ip2;
+		return key.ip2.s6_addr;
 	}
+    const in6_addr* get_ip2_addr() const {
+        return &(key.ip2);
+    }
 	uint16_t get_port1() const {
 		return key.port1;
 	}
@@ -208,8 +214,10 @@ public:
     //proto_t proto;
 
     typedef struct {
-        unsigned char ip1[16];
-        unsigned char ip2[16];
+        in6_addr ip1;
+        in6_addr ip2;
+        //unsigned char ip1[16];
+        //unsigned char ip2[16];
         uint16 port1;
         uint16 port2;
         //int version;
@@ -324,11 +332,17 @@ public:
 		return v6.proto;
 	}
 	const unsigned char* get_ip1() const {
-		return key.ip1;
+		return key.ip1.s6_addr;
 	}
+    const in6_addr* get_ip1_addr() const {
+        return &(key.ip1);
+    }
 	const unsigned char* get_ip2() const {
-		return key.ip2;
+		return key.ip2.s6_addr;
 	}
+    const in6_addr* get_ip2_addr() const {
+        return &(key.ip2);
+    }
 	uint16_t get_port() const {
 		return key.port2;
 	}
@@ -374,8 +388,8 @@ public:
     //proto_t proto;
 
     typedef struct {
-        unsigned char ip1[16];
-        unsigned char ip2[16];
+        in6_addr ip1;
+        in6_addr ip2;
         uint16 port1;
         uint16 port2;
         //int version;
@@ -500,11 +514,17 @@ public:
 */
 
 	const unsigned char* get_ip1() const {
-		return key.ip1;
+		return key.ip1.s6_addr;
 	}
+    const in6_addr* get_ip1_addr() const {
+        return &(key.ip1);
+    }
 	const unsigned char* get_ip2() const {
-		return key.ip2;
+		return key.ip2.s6_addr;
 	}
+    const in6_addr* get_ip2_addr() const {
+        return &(key.ip2);
+    }
     int get_version() const {
         return v6.version;
     }
@@ -540,8 +560,10 @@ public:
     
 
     typedef struct {
-        unsigned char ip1[16];
-        unsigned char ip2[16];
+        in6_addr ip1;
+        in6_addr ip2;
+        //unsigned char ip1[16];
+        //unsigned char ip2[16];
         uint16 port1;
         uint16 port2;
         //int version;
diff --git a/src/DiskIndex.cc b/src/DiskIndex.cc
index e63c9f1..7a3cbe0 100644
--- a/src/DiskIndex.cc
+++ b/src/DiskIndex.cc
@@ -48,6 +48,13 @@ my_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream)
  */
 inline IndexFileReader::IndexFileReader(char *fn) : fp(NULL), fname(fn), eof(false) {
 
+        char filepath[100];
+
+        strcpy(filepath, conf_main_workdir);
+        strcat(filepath, "/");
+        strcat(filepath, fname);
+        //printf("The index file name is %s\n", filepath);
+
         if (chdir(conf_main_workdir)) {
             fprintf(stderr, "cannot chdir to %s\n", conf_main_workdir);
             //return(1);
@@ -57,6 +64,12 @@ inline IndexFileReader::IndexFileReader(char *fn) : fp(NULL), fname(fn), eof(fal
 	if (fp == NULL) {
 		//TODO: Decent error handling
 		tmlog(TM_LOG_ERROR, "IFR", "Could not open index file \"%s\" for reading.\n", fname);
+                char path[70];
+
+                //char errbufnav[PCAP_ERRBUF_SIZE];
+
+                printf("The directory for Fifo that we are in is %s\n", getcwd(path, 70));
+
 	}
 	my_fread(&first, sizeof(tm_time_t), 1, fp);
 	my_fread(&last, sizeof(tm_time_t), 1, fp);
diff --git a/src/IndexField.cc b/src/IndexField.cc
index e7cc5f9..7967c5c 100644
--- a/src/IndexField.cc
+++ b/src/IndexField.cc
@@ -25,21 +25,40 @@ static std::string pattern_ip ("(\\d+\\.\\d+\\.\\d+\\.\\d+)"); // TODO: figure o
 
 //static std::string pattern_ip6 ("(\\w+:\\w+:\\w+:\\w+:\\w+:\\w+:\\w+:\\w+)"); // I am using word from re2, perl regular expression for the alphanumeric part
 
-static std::string pattern_ip6 ("\\[(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]");
+//static std::string alphanum14 ("[0-9a-fA-F]{1,4}");
+
+static std::string pattern_ip6 ("\\[([0-9a-fA-F]{1,4}:{7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]");
 // stolen from stackoverflow http://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
 
+//static std::string pattern_ip6 ("\\[([0-9a-fA-F]{1,4}:{7,7}[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,7}:|[0-9a-fA-F]{1,4}:{1,6}:[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,5}:[0-9a-fA-F]{1,4}{1,2}|[0-9a-fA-F]{1,4}:{1,4}:[0-9a-fA-F]{1,4}{1,3}|[0-9a-fA-F]{1,4}:{1,3}:[0-9a-fA-F]{1,4}{1,4}|[0-9a-fA-F]{1,4}:{1,2}:[0-9a-fA-F]{1,4}{1,5}|[0-9a-fA-F]{1,4}::[0-9a-fA-F]{1,4}{1,6}|::[0-9a-fA-F]{1,4}{1,7}|:|fe80::[0-9a-fA-F]{0,4}{0,4}%[0-9a-zA-Z]{1,}|::ffff:0{1,4}{0,1}:{0,1}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9]|[0-9a-fA-F]{1,4}:{1,4}:25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9])\\]");
+
+
+//static std::string pattern_ip6 ("(/^(?>(?>([a-f0-9]{1,4})(?>:(?1)){7}|(?!(?:.*[a-f0-9](?>:|$)){8,})((?1)(?>:(?1)){0,6})?::(?2)?)|(?>(?>(?1)(?>:(?1)){5}:|(?!(?:.*[a-f0-9]:){6,})(?3)?::(?>((?1)(?>:(?1)){0,4}):)?)?(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?>\.(?4)){3}))$/iD)");
+
+//static std::string pattern_ip6 ("("["({HEX}:){7}{HEX}"]")|("["0x{HEX}({HEX}|:)*"::"({HEX}|:)*"]")|("["({HEX}|:)*"::"({HEX}|:)*"]")|("["({HEX}|:)*"::"({HEX}|:)])");
+
+//static std::string pattern_ip6 ("(\\[\\w+\\])");
+
+//static std::string pattern_ip6_first ("(\\[\\w+::\\w+:\\w+:\\w+:\\w+\\])");
+//static std::string pattern_ip6_second ("(\\[\\w+::\\w+\\])");
+
 static std::string pattern_ipport ("(\\d+\\.\\d+\\.\\d+\\.\\d+):(\\d+)");
 
 // brackets are necessary when specifying a port number of IPv6
 //static std::string pattern_ip6port ("([\\w+:\\w+:\\w+:\\w+:\\w+:\\w+:\\w+:\\w+]):(\\d+)"); // IPv6 addresses that have a port are of the form []:#
 
-static std::string pattern_ip6port ("\\[(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]:(\\d+)");
-// stolen from stackoverflow http://stackoverflow.com/questions/53497/regular-expression-that-
+//static std::string pattern_ip6port ("(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])):(\\d+)");
+
+//static std::string pattern_ip6port ("\\[([0-9a-fA-F]{1,4}:{7,7}[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,7}:|[0-9a-fA-F]{1,4}:{1,6}:[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,5}:[0-9a-fA-F]{1,4}{1,2}|[0-9a-fA-F]{1,4}:{1,4}:[0-9a-fA-F]{1,4}{1,3}|[0-9a-fA-F]{1,4}:{1,3}:[0-9a-fA-F]{1,4}{1,4}|[0-9a-fA-F]{1,4}:{1,2}:[0-9a-fA-F]{1,4}{1,5}|[0-9a-fA-F]{1,4}::[0-9a-fA-F]{1,4}{1,6}|::[0-9a-fA-F]{1,4}{1,7}|:|fe80::[0-9a-fA-F]{0,4}{0,4}%[0-9a-zA-Z]{1,}|::ffff:0{1,4}{0,1}:{0,1}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9]|[0-9a-fA-F]{1,4}:{1,4}:25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9])\\]:(\\d+)");
+
+static std::string pattern_ip6port ("\\[([0-9a-fA-F]{1,4}:{7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]:(\\d+)");
+// stolen from stackoverflow http://stacikoverflow.com/questions/53497/regular-expression-that-
 
 
 /* size of an ip addr in dottet decimal as string: 4x3digits, 
   3 dots, terminating nul byte */
 #define TM_IP_STR_SIZE 16
+#define TM_IP6_STR_SIZE 40
 /*
 static void ip_to_str(const unsigned char* ip, char *str, int len) {
 //#define UCP(x) ((unsigned char *)&(x))
@@ -608,12 +627,13 @@ std::list<ConnectionIF4*> ConnectionIF4::genKeys(const u_char* packet) {
 
 //FIXME: merge this somehow with ConnectionID4::parse() !!!!
 IndexField* ConnectionIF4::parseQuery(const char *query) {
-	std::string protostr, src_ip, dst_ip;
+	//std::string protostr, src_ip, dst_ip;
+    std::string protostr, src_ip, dst_ip;
 	unsigned src_port, dst_port;
 	proto_t proto;
 
 
-	if (!RE2::FullMatch(query, re, &protostr, &src_ip, &src_port, &dst_ip, &dst_port) && !RE2::FullMatch(query, re6, &protostr, &src_ip, &src_port, &dst_ip, &dst_port))
+	if (!RE2::FullMatch(query, re) && !RE2::FullMatch(query, re6))
 		return NULL;
 
 	/*
@@ -631,6 +651,31 @@ IndexField* ConnectionIF4::parseQuery(const char *query) {
 	    return new ConnectionIF4(proto, inet_addr(src_ip.c_str()), htons(src_port),
 			    inet_addr(dst_ip.c_str()), htons(dst_port));
     }
+    else if (RE2::FullMatch(query, re6, &protostr, &src_ip, &src_port, &dst_ip, &dst_port))
+    {
+        struct sockaddr_in6 src_ip6;
+        struct sockaddr_in6 dst_ip6;
+        //char str[INET6_ADDRSTRLEN];
+
+        printf("entering FullMatch for conn3, ipv6\n");
+
+        if (protostr == std::string("tcp"))
+            proto = IPPROTO_TCP;
+        else
+            proto = IPPROTO_UDP;
+
+
+        printf("the query in parsequery is %s\n", query);
+
+        printf("for IPv6, the protocol is <%s> and source ip is <%s> and the dest ip is <%s>i and the source port is <%u> and the destination port is <%u>\n", protostr.c_str(), src_ip.c_str(), dst_ip.c_str(), src_port, dst_port);
+
+        //intf("for IPv6, The source ip is <%s> and the dest ip is <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s>\n", src_ip.c_str(), 
+        // store this IP address in sa:
+        inet_pton(AF_INET6, src_ip.c_str(), &(src_ip6.sin6_addr));
+        inet_pton(AF_INET6, dst_ip.c_str(), &(dst_ip6.sin6_addr));
+        return new ConnectionIF4(proto, src_ip6.sin6_addr.s6_addr, htons(src_port), dst_ip6.sin6_addr.s6_addr, htons(dst_port));
+
+    }
  
     else
     {
@@ -710,10 +755,31 @@ void ConnectionIF4::getBPFStr(char *str, int max_str_len) const {
 
     else
     {
+        printf("the query is for %s\n", c_id.getStr().c_str());
+
+        printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
+        printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
+
+        //ip_to_str(c_id.get_ip1_addr(), s_ip6_str, 16); //sizeof(s_ip6_str));
+        //ip_to_str(c_id.get_ip2_addr(), d_ip6_str, 16); //sizeof(d_ip6_str));
+
+
+        char s_ip6_str[INET6_ADDRSTRLEN];
+        char d_ip6_str[INET6_ADDRSTRLEN];
+
+        inet_ntop(AF_INET6, c_id.get_ip1_addr(), s_ip6_str, INET6_ADDRSTRLEN);
+        inet_ntop(AF_INET6, c_id.get_ip2_addr(), d_ip6_str, INET6_ADDRSTRLEN);
+
+        //tmlog(TM_LOG_ERROR, "ConnectionIF2::getBPFStr()", "we are in connectionIF2::getBPFStr()");
+
+        printf("host %s and host %s\n", s_ip6_str, d_ip6_str);
+
+        /*
         char s_ip6_str[TM_IP_STR_SIZE];
         char d_ip6_str[TM_IP_STR_SIZE];
         uint32_t s6_port;
         uint32_t d6_port;
+        */
         /*
         if (c_id.get_is_canonified()) {
           s_ip=c_id.get_ip2();
@@ -722,18 +788,20 @@ void ConnectionIF4::getBPFStr(char *str, int max_str_len) const {
           d_port=c_id.get_port1();
           } else {
           */
+        /*
         ip_to_str(c_id.get_ip1(), s_ip6_str, sizeof(s_ip6_str));
         ip_to_str(c_id.get_ip2(), d_ip6_str, sizeof(d_ip6_str));
         s6_port=c_id.get_port1();
         d6_port=c_id.get_port2();
+        */
           /*  }  */
 
         snprintf(str, max_str_len,
                  "host %s and port %d and host %s and port %d",
                  s_ip6_str,
-                 ntohs(s6_port),
+                 ntohs(c_id.get_port1()),
                  d_ip6_str,
-                 ntohs(d6_port));
+                 ntohs(c_id.get_port2()));
 
     }
 }
@@ -770,26 +838,64 @@ IndexField* ConnectionIF3::parseQuery(const char *query) {
 	unsigned port;
 	proto_t proto;
 
-	if (!RE2::FullMatch(query, re, &protostr, &src_ip, &dst_ip, &port) && !RE2::FullMatch(query, re6, &protostr, &src_ip, &dst_ip, &port))
-		return NULL;
+    printf("Entering parQuery method\n");
+	
+	if (!RE2::FullMatch(query, re) && !RE2::FullMatch(query, re6))
+    {
+        tmlog(TM_LOG_ERROR,"parseQuery", "Cannot do full match!");
+
+        return NULL;
+    }
 
 	/*
 	fprintf(stderr, "%s\nConnectionIF3::parseQuery:  %s ===> <%s> <%s> <%s> <%u>\n", 
 				pattern_connection3.c_str(), query, protostr.c_str(), 
 				src_ip.c_str(), dst_ip.c_str(), port);
 	*/
+    /*
 	if (protostr == std::string("tcp"))
 		proto = IPPROTO_TCP;
 	else 
 		proto = IPPROTO_UDP;
-
+    */
     if (RE2::FullMatch(query, re, &protostr, &src_ip, &dst_ip, &port))
     {
+        if (protostr == std::string("tcp"))
+            proto = IPPROTO_TCP;
+        else
+            proto = IPPROTO_UDP;
+
 		return new ConnectionIF3(proto, inet_addr(src_ip.c_str()), 
 			    inet_addr(dst_ip.c_str()), htons(port));
     }
+    else if (RE2::FullMatch(query, re6, &protostr, &src_ip, &dst_ip, &port))
+    {
+        struct sockaddr_in6 src_ip6;
+        struct sockaddr_in6 dst_ip6;
+        //char str[INET6_ADDRSTRLEN];
+
+        printf("entering FullMatch for conn3, ipv6\n");
+
+        if (protostr == std::string("tcp"))
+            proto = IPPROTO_TCP;
+        else
+            proto = IPPROTO_UDP;
+
+
+        printf("the query in parsequery is %s\n", query);
+
+        printf("for IPv6, the protocol is <%s> and source ip is <%s> and the dest ip is <%s>i and the port is <%u>\n", protostr.c_str(), src_ip.c_str(), dst_ip.c_str(), port);
+
+        //intf("for IPv6, The source ip is <%s> and the dest ip is <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s>\n", src_ip.c_str(), 
+        // store this IP address in sa:
+        inet_pton(AF_INET6, src_ip.c_str(), &(src_ip6.sin6_addr));
+        inet_pton(AF_INET6, dst_ip.c_str(), &(dst_ip6.sin6_addr));
+        return new ConnectionIF3(proto, src_ip6.sin6_addr.s6_addr, dst_ip6.sin6_addr.s6_addr, htons(port));
+        
+    }
     else
     {
+        printf("wut we are not suppose to be here!\n");
         //unsigned char src_ip6[16];
         //unsigned char dst_ip6[16];
 
@@ -804,7 +910,7 @@ IndexField* ConnectionIF3::parseQuery(const char *query) {
     }
 }
 
-void ConnectionIF3:: ip_to_str(const unsigned char* ip, char *str, int len) const {
+void ConnectionIF3:: ip4_to_str(const unsigned char* ip, char *str, int len) const {
 //#define UCP(x) ((unsigned char *)&(x))
         str[0] = '\0';
     /*
@@ -814,17 +920,17 @@ void ConnectionIF3:: ip_to_str(const unsigned char* ip, char *str, int len) cons
                          UCP(ip)[2] & 0xff,
                          UCP(ip)[3] & 0xff);
     */
-    if (c_id.get_version() == 4)
-    {
+    //if (c_id.get_version() == 4)
+    //{
         snprintf(str, len, "%d.%d.%d.%d", ip[12] & 0xff, ip[13] & 0xff, ip[14] & 0xff, ip[15] & 0xff);
         //printf("The string is %s\n", str);
         //printf("All of the elements of ip are: %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d\n", ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], ip[8], ip[9], ip[10], ip[11], ip[12], ip[13], ip[14], ip[15]);
-    }
+    //}
 
-    else
-    {
-        snprintf(str, len, "%s", ip);
-    }
+    //else
+    //{
+        //snprintf(str, len, "%s", ip);
+    //}
 }
 
 void ConnectionIF3::getBPFStr(char *str, int max_str_len) const {
@@ -834,8 +940,8 @@ void ConnectionIF3::getBPFStr(char *str, int max_str_len) const {
 	    char ip1_str[TM_IP_STR_SIZE];
 	    char ip2_str[TM_IP_STR_SIZE];
 
-	    ip_to_str(c_id.get_ip1(), ip1_str, sizeof(ip1_str));
-	    ip_to_str(c_id.get_ip2(), ip2_str, sizeof(ip2_str));
+	    ip4_to_str(c_id.get_ip1(), ip1_str, sizeof(ip1_str));
+	    ip4_to_str(c_id.get_ip2(), ip2_str, sizeof(ip2_str));
 
         //printf("The src host %s and dst host %s and dst port %d or dst host %s and src host %s and src port %d", ip1_str, ip2_str, ntohs(c_id.get_port()), ip1_str, ip2_str, ntohs(c_id.get_port()));
 
@@ -848,6 +954,26 @@ void ConnectionIF3::getBPFStr(char *str, int max_str_len) const {
 
     else
     {
+        printf("the query is for %s\n", c_id.getStr().c_str());
+
+        printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
+        printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
+
+        //ip_to_str(c_id.get_ip1_addr(), s_ip6_str, 16); //sizeof(s_ip6_str));
+        //ip_to_str(c_id.get_ip2_addr(), d_ip6_str, 16); //sizeof(d_ip6_str));
+
+
+        char s_ip6_str[INET6_ADDRSTRLEN];
+        char d_ip6_str[INET6_ADDRSTRLEN];
+
+        inet_ntop(AF_INET6, c_id.get_ip1_addr(), s_ip6_str, INET6_ADDRSTRLEN);
+        inet_ntop(AF_INET6, c_id.get_ip2_addr(), d_ip6_str, INET6_ADDRSTRLEN);
+
+        //tmlog(TM_LOG_ERROR, "ConnectionIF2::getBPFStr()", "we are in connectionIF2::getBPFStr()");
+
+        printf("host %s and host %s\n", s_ip6_str, d_ip6_str);
+
+        /*
         char ip61_str[TM_IP_STR_SIZE];
         char ip62_str[TM_IP_STR_SIZE];
 
@@ -855,12 +981,12 @@ void ConnectionIF3::getBPFStr(char *str, int max_str_len) const {
         ip_to_str(c_id.get_ip2(), ip62_str, sizeof(ip62_str));
 
         //printf("The src host %s and dst host %s and dst port %d or dst host %s and src host %s and src port %d", ip61_str, ip62_str, ntohs(c_id.get_port()), ip61_str, ip62_str, ntohs(c_id.get_port()));
-
+        */
         snprintf(str, max_str_len,
                  "(src host %s and dst host %s and dst port %d) or "
                  "(dst host %s and src host %s and src port %d)",
-                 ip61_str, ip62_str, ntohs(c_id.get_port()),
-                 ip61_str, ip62_str, ntohs(c_id.get_port()));
+                 s_ip6_str, d_ip6_str, ntohs(c_id.get_port()),
+                 d_ip6_str, s_ip6_str, ntohs(c_id.get_port()));
 
     }
 }
@@ -893,18 +1019,42 @@ ConnectionIF2::genKeys(const u_char* packet) {
 IndexField* ConnectionIF2::parseQuery(const char *query) {
 	std::string src_ip, dst_ip;
 	
-	if (!RE2::FullMatch(query, re, &src_ip, &dst_ip) && !RE2::FullMatch(query, re6, &src_ip, &dst_ip))
+	if (!RE2::FullMatch(query, re) && !RE2::FullMatch(query, re6))
+    {
+        tmlog(TM_LOG_ERROR,"parseQuery", "Cannot do full match!");
+
 		return NULL;
+    }
 	
-	/*
 	fprintf(stderr, "%s\nConnectionIF22:parseQuery:  %s ===> <%s> <%s>\n", 
-				pattern_connection2.c_str(), query, src_ip.c_str(), dst_ip.c_str());
-	*/
+				pattern6_connection2.c_str(), query, src_ip.c_str(), dst_ip.c_str());
+	
+
     
     if (RE2::FullMatch(query, re, &src_ip, &dst_ip))
     {
+        printf("for IPv6, The source ip is %s and the dest ip is %s\n", src_ip.c_str(), dst_ip.c_str());
+
     	return new ConnectionIF2(inet_addr(src_ip.c_str()), inet_addr(dst_ip.c_str()));
     }
+    else if (RE2::FullMatch(query, re6, &src_ip, &dst_ip))
+    {
+        struct sockaddr_in6 src_ip6;
+        struct sockaddr_in6 dst_ip6;
+        //char str[INET6_ADDRSTRLEN];
+
+        printf("the query in parsequery is %s\n", query);
+
+        printf("for IPv6, the source ip is <%s> and the dest ip is <%s>\n", src_ip.c_str(), dst_ip.c_str());
+
+        //intf("for IPv6, The source ip is <%s> and the dest ip is <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s>\n", src_ip.c_str(), 
+        // store this IP address in sa:
+        inet_pton(AF_INET6, src_ip.c_str(), &(src_ip6.sin6_addr));
+        inet_pton(AF_INET6, dst_ip.c_str(), &(dst_ip6.sin6_addr));
+        return new ConnectionIF2(src_ip6.sin6_addr.s6_addr, dst_ip6.sin6_addr.s6_addr);
+
+        //return new ConnectionIF2(src_ip.c_str(), dst_ip.c_str());
+    }
     else
     {
         //unsigned char src_ip6[16];
@@ -921,9 +1071,9 @@ IndexField* ConnectionIF2::parseQuery(const char *query) {
     }
 }
 
-void ConnectionIF2:: ip_to_str(const unsigned char* ip, char *str, int len) const {
+void ConnectionIF2:: ip4_to_str(const unsigned char* ip, char *str, int len) const {
 //#define UCP(x) ((unsigned char *)&(x))
-	str[0] = '\0';
+	//str[0] = '\0';
     /*
 	snprintf(str, len, "%d.%d.%d.%d",
 			 UCP(ip)[0] & 0xff,
@@ -931,17 +1081,10 @@ void ConnectionIF2:: ip_to_str(const unsigned char* ip, char *str, int len) cons
 			 UCP(ip)[2] & 0xff,
 			 UCP(ip)[3] & 0xff);
     */
-    if (c_id.get_version() == 4)
-    {
-        snprintf(str, len, "%d.%d.%d.%d", ip[12] & 0xff, ip[13] & 0xff, ip[14] & 0xff, ip[15] & 0xff);
+    str[0] = '\0';
+    snprintf(str, len, "%d.%d.%d.%d", ip[12] & 0xff, ip[13] & 0xff, ip[14] & 0xff, ip[15] & 0xff);
         //printf("The string is %s\n", str);
         //printf("All of the elements of ip are: %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d\n", ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], ip[8], ip[9], ip[10], ip[11], ip[12], ip[13], ip[14], ip[15]);
-    }
-
-    else
-    {
-        snprintf(str, len, "%s", ip);
-    }
 }
 
 
@@ -953,11 +1096,16 @@ void ConnectionIF2::getBPFStr(char *str, int max_str_len) const {
         char s_ip_str[TM_IP_STR_SIZE];
         char d_ip_str[TM_IP_STR_SIZE];
 
-        ip_to_str(c_id.get_ip1(), s_ip_str, 16);
-        ip_to_str(c_id.get_ip2(), d_ip_str, 16);
+        printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
+        printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
+
+        ip4_to_str(c_id.get_ip1(), s_ip_str, 16);
+        ip4_to_str(c_id.get_ip2(), d_ip_str, 16);
 
 
-        tmlog(TM_LOG_ERROR, "ConnectionIF2::getBPFStr()", "we are in connectionIF2::getBPFStr()");
+        //tmlog(TM_LOG_ERROR, "ConnectionIF2::getBPFStr()", "we are in connectionIF2::getBPFStr()");
+
+        printf("ip4: host %s and host %s\n", s_ip_str, d_ip_str);
 
         snprintf(str, max_str_len,
                  "host %s and host %s",
@@ -966,14 +1114,27 @@ void ConnectionIF2::getBPFStr(char *str, int max_str_len) const {
     }
     else
     {
-	    char s_ip6_str[TM_IP_STR_SIZE];
-	    char d_ip6_str[TM_IP_STR_SIZE];
+	    //char s_ip6_str[TM_IP_STR_SIZE];
+	    //char d_ip6_str[TM_IP_STR_SIZE];
+
+        printf("the query is for %s\n", c_id.getStr().c_str());
+
+        printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
+        printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
+
+	    //ip_to_str(c_id.get_ip1_addr(), s_ip6_str, 16); //sizeof(s_ip6_str));
+	    //ip_to_str(c_id.get_ip2_addr(), d_ip6_str, 16); //sizeof(d_ip6_str));
+
+
+        char s_ip6_str[INET6_ADDRSTRLEN];
+        char d_ip6_str[INET6_ADDRSTRLEN];
 
-	    ip_to_str(c_id.get_ip1(), s_ip6_str, 16);
-	    ip_to_str(c_id.get_ip2(), d_ip6_str, 16);
+        inet_ntop(AF_INET6, c_id.get_ip1_addr(), s_ip6_str, INET6_ADDRSTRLEN);
+        inet_ntop(AF_INET6, c_id.get_ip2_addr(), d_ip6_str, INET6_ADDRSTRLEN);
 
+        //tmlog(TM_LOG_ERROR, "ConnectionIF2::getBPFStr()", "we are in connectionIF2::getBPFStr()");
 
-        tmlog(TM_LOG_ERROR, "ConnectionIF2::getBPFStr()", "we are in connectionIF2::getBPFStr()");
+        printf("host %s and host %s\n", s_ip6_str, d_ip6_str);
 
 	    snprintf(str, max_str_len,
 		    	 "host %s and host %s",
diff --git a/src/IndexField.hh b/src/IndexField.hh
index dad372e..70a5d54 100644
--- a/src/IndexField.hh
+++ b/src/IndexField.hh
@@ -698,7 +698,7 @@ public:
 		return &c_id;
 	}
 
-    void ip_to_str(const unsigned char* ip, char *str, int len) const;
+    void ip4_to_str(const unsigned char* ip, char *str, int len) const;
 
 	bool operator==(const IndexField& other) const {
 		return c_id==((ConnectionIF3*)&other)->c_id;
@@ -779,7 +779,7 @@ public:
     */
 	void getBPFStr(char *, int) const;
 
-    void ip_to_str(const unsigned char* ip, char *str, int len) const;
+    void ip4_to_str(const unsigned char* ip, char *str, int len) const;
 
 	ConnectionID2 *getCID() {
 		return &c_id;
diff --git a/src/re2/variadic_function.h b/src/re2/variadic_function.h
index 7c7d6d5..3c8b671 100644
--- a/src/re2/variadic_function.h
+++ b/src/re2/variadic_function.h
@@ -337,6 +337,148 @@ class VariadicFunction2 {
         &a21, &a22, &a23, &a24, &a25, &a26, &a27, &a28, &a29, &a30, &a31 };
     return Func(p0, p1, args, 32);
   }
+  Result operator()(Param0 p0, Param1 p1, const Arg& a0, const Arg& a1,
+      const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5,
+      const Arg& a6, const Arg& a7, const Arg& a8, const Arg& a9,
+      const Arg& a10, const Arg& a11, const Arg& a12, const Arg& a13,
+      const Arg& a14, const Arg& a15, const Arg& a16, const Arg& a17,
+      const Arg& a18, const Arg& a19, const Arg& a20, const Arg& a21,
+      const Arg& a22, const Arg& a23, const Arg& a24, const Arg& a25,
+      const Arg& a26, const Arg& a27, const Arg& a28, const Arg& a29,
+      const Arg& a30, const Arg& a31, const Arg& a32) const {
+    const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8,
+        &a9, &a10, &a11, &a12, &a13, &a14, &a15, &a16, &a17, &a18, &a19, &a20,
+        &a21, &a22, &a23, &a24, &a25, &a26, &a27, &a28, &a29, &a30, &a31, &a32 };
+    return Func(p0, p1, args, 33);
+  }
+  Result operator()(Param0 p0, Param1 p1, const Arg& a0, const Arg& a1,
+      const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5,
+      const Arg& a6, const Arg& a7, const Arg& a8, const Arg& a9,
+      const Arg& a10, const Arg& a11, const Arg& a12, const Arg& a13,
+      const Arg& a14, const Arg& a15, const Arg& a16, const Arg& a17,
+      const Arg& a18, const Arg& a19, const Arg& a20, const Arg& a21,
+      const Arg& a22, const Arg& a23, const Arg& a24, const Arg& a25,
+      const Arg& a26, const Arg& a27, const Arg& a28, const Arg& a29,
+      const Arg& a30, const Arg& a31, const Arg& a32, const Arg& a33) const {
+    const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8,
+        &a9, &a10, &a11, &a12, &a13, &a14, &a15, &a16, &a17, &a18, &a19, &a20,
+        &a21, &a22, &a23, &a24, &a25, &a26, &a27, &a28, &a29, &a30, &a31, &a32,
+        &a33 };
+    return Func(p0, p1, args, 34);
+  }
+
+  Result operator()(Param0 p0, Param1 p1, const Arg& a0, const Arg& a1,
+      const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5,
+      const Arg& a6, const Arg& a7, const Arg& a8, const Arg& a9,
+      const Arg& a10, const Arg& a11, const Arg& a12, const Arg& a13,
+      const Arg& a14, const Arg& a15, const Arg& a16, const Arg& a17,
+      const Arg& a18, const Arg& a19, const Arg& a20, const Arg& a21,
+      const Arg& a22, const Arg& a23, const Arg& a24, const Arg& a25,
+      const Arg& a26, const Arg& a27, const Arg& a28, const Arg& a29,
+      const Arg& a30, const Arg& a31, const Arg& a32, const Arg& a33,
+      const Arg& a34) const {
+    const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8,
+        &a9, &a10, &a11, &a12, &a13, &a14, &a15, &a16, &a17, &a18, &a19, &a20,
+        &a21, &a22, &a23, &a24, &a25, &a26, &a27, &a28, &a29, &a30, &a31, &a32, 
+        &a33, &a34 };
+    return Func(p0, p1, args, 35);
+  }
+
+  Result operator()(Param0 p0, Param1 p1, const Arg& a0, const Arg& a1,
+      const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5,
+      const Arg& a6, const Arg& a7, const Arg& a8, const Arg& a9,
+      const Arg& a10, const Arg& a11, const Arg& a12, const Arg& a13,
+      const Arg& a14, const Arg& a15, const Arg& a16, const Arg& a17,
+      const Arg& a18, const Arg& a19, const Arg& a20, const Arg& a21,
+      const Arg& a22, const Arg& a23, const Arg& a24, const Arg& a25,
+      const Arg& a26, const Arg& a27, const Arg& a28, const Arg& a29,
+      const Arg& a30, const Arg& a31, const Arg& a32, const Arg& a33,
+      const Arg& a34, const Arg& a35) const {
+    const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8,
+        &a9, &a10, &a11, &a12, &a13, &a14, &a15, &a16, &a17, &a18, &a19, &a20,
+        &a21, &a22, &a23, &a24, &a25, &a26, &a27, &a28, &a29, &a30, &a31, &a32,
+        &a33, &a34, &a35 };
+    return Func(p0, p1, args, 36);
+  }
+
+  Result operator()(Param0 p0, Param1 p1, const Arg& a0, const Arg& a1,
+      const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5,
+      const Arg& a6, const Arg& a7, const Arg& a8, const Arg& a9,
+      const Arg& a10, const Arg& a11, const Arg& a12, const Arg& a13,
+      const Arg& a14, const Arg& a15, const Arg& a16, const Arg& a17,
+      const Arg& a18, const Arg& a19, const Arg& a20, const Arg& a21,
+      const Arg& a22, const Arg& a23, const Arg& a24, const Arg& a25,
+      const Arg& a26, const Arg& a27, const Arg& a28, const Arg& a29,
+      const Arg& a30, const Arg& a31, const Arg& a32, const Arg& a33,
+      const Arg& a34, const Arg& a35, const Arg& a36) const {
+    const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8,
+        &a9, &a10, &a11, &a12, &a13, &a14, &a15, &a16, &a17, &a18, &a19, &a20,
+        &a21, &a22, &a23, &a24, &a25, &a26, &a27, &a28, &a29, &a30, &a31, &a32,
+        &a33, &a34, &a35, &a36 };
+    return Func(p0, p1, args, 37);
+  }
+  Result operator()(Param0 p0, Param1 p1, const Arg& a0, const Arg& a1,
+      const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5,
+      const Arg& a6, const Arg& a7, const Arg& a8, const Arg& a9,
+      const Arg& a10, const Arg& a11, const Arg& a12, const Arg& a13,
+      const Arg& a14, const Arg& a15, const Arg& a16, const Arg& a17,
+      const Arg& a18, const Arg& a19, const Arg& a20, const Arg& a21,
+      const Arg& a22, const Arg& a23, const Arg& a24, const Arg& a25,
+      const Arg& a26, const Arg& a27, const Arg& a28, const Arg& a29,
+      const Arg& a30, const Arg& a31, const Arg& a32, const Arg& a33,
+      const Arg& a34, const Arg& a35, const Arg& a36, const Arg& a37,
+      const Arg& a38, const Arg& a39, const Arg& a40, const Arg& a41) const {
+    const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8,
+        &a9, &a10, &a11, &a12, &a13, &a14, &a15, &a16, &a17, &a18, &a19, &a20,
+        &a21, &a22, &a23, &a24, &a25, &a26, &a27, &a28, &a29, &a30, &a31, &a32,
+        &a33, &a34, &a35, &a36, &a37, &a38, &a39, &a40, &a41};
+    return Func(p0, p1, args, 42);
+  }
+
+  Result operator()(Param0 p0, Param1 p1, const Arg& a0, const Arg& a1,
+      const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5,
+      const Arg& a6, const Arg& a7, const Arg& a8, const Arg& a9,
+      const Arg& a10, const Arg& a11, const Arg& a12, const Arg& a13,
+      const Arg& a14, const Arg& a15, const Arg& a16, const Arg& a17,
+      const Arg& a18, const Arg& a19, const Arg& a20, const Arg& a21,
+      const Arg& a22, const Arg& a23, const Arg& a24, const Arg& a25,
+      const Arg& a26, const Arg& a27, const Arg& a28, const Arg& a29,
+      const Arg& a30, const Arg& a31, const Arg& a32, const Arg& a33,
+      const Arg& a34, const Arg& a35, const Arg& a36, const Arg& a37,
+      const Arg& a38, const Arg& a39, const Arg& a40, const Arg& a41,
+      const Arg& a42) const {
+    const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8,
+        &a9, &a10, &a11, &a12, &a13, &a14, &a15, &a16, &a17, &a18, &a19, &a20,
+        &a21, &a22, &a23, &a24, &a25, &a26, &a27, &a28, &a29, &a30, &a31, &a32,
+        &a33, &a34, &a35, &a36, &a37, &a38, &a39, &a40, &a41, &a42};
+    return Func(p0, p1, args, 43);
+  }
+  Result operator()(Param0 p0, Param1 p1, const Arg& a0, const Arg& a1,
+      const Arg& a2, const Arg& a3, const Arg& a4, const Arg& a5,
+      const Arg& a6, const Arg& a7, const Arg& a8, const Arg& a9,
+      const Arg& a10, const Arg& a11, const Arg& a12, const Arg& a13,
+      const Arg& a14, const Arg& a15, const Arg& a16, const Arg& a17,
+      const Arg& a18, const Arg& a19, const Arg& a20, const Arg& a21,
+      const Arg& a22, const Arg& a23, const Arg& a24, const Arg& a25,
+      const Arg& a26, const Arg& a27, const Arg& a28, const Arg& a29,
+      const Arg& a30, const Arg& a31, const Arg& a32, const Arg& a33,
+      const Arg& a34, const Arg& a35, const Arg& a36, const Arg& a37,
+      const Arg& a38, const Arg& a39, const Arg& a40, const Arg& a41,
+      const Arg& a42, const Arg& a43, const Arg& a44, const Arg& a45,
+      const Arg& a46, const Arg& a47, const Arg& a48, const Arg& a49,
+      const Arg& a50, const Arg& a51, const Arg& a52, const Arg& a53,
+      const Arg& a54, const Arg& a55, const Arg& a56, const Arg& a57,
+      const Arg& a58, const Arg& a59, const Arg& a60) const {
+    const Arg* const args[] = { &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, &a8,
+        &a9, &a10, &a11, &a12, &a13, &a14, &a15, &a16, &a17, &a18, &a19, &a20,
+        &a21, &a22, &a23, &a24, &a25, &a26, &a27, &a28, &a29, &a30, &a31, &a32,
+        &a33, &a34, &a35, &a36, &a37, &a38, &a39, &a40, &a41, &a42, &a43, &a44,
+        &a45, &a46, &a47, &a48, &a49, &a50, &a51, &a52, &a53, &a54, &a55, &a56,
+        &a57, &a58, &a59, &a60};
+    return Func(p0, p1, args, 61);
+  }
+
+
 };
 
 }  // namespace re2


From neto at caltech.edu  Tue Aug 19 19:18:10 2014
From: neto at caltech.edu (Naoki Eto)
Date: Tue, 19 Aug 2014 19:18:10 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Fixed a bug in querying for
	longer IPv6 addresses. The regular expression for IPv6
	addresses has been corrected (40329f8)
Message-ID: <201408200218.s7K2IApP015608@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit 40329f83b36fd034fa35b15f8002993babcd099f
Author: NaokiEto <neto at caltech.edu>
Date:   Tue Aug 19 22:05:44 2014 -0400

    Fixed a bug in querying for longer IPv6 addresses. The regular expression for IPv6 addresses has been corrected


>---------------------------------------------------------------

40329f83b36fd034fa35b15f8002993babcd099f
 src/Connection.cc | 4 ++--
 src/IndexField.cc | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Connection.cc b/src/Connection.cc
index ec031fa..f16e5b3 100644
--- a/src/Connection.cc
+++ b/src/Connection.cc
@@ -22,12 +22,12 @@
 static std::string pattern_ip ("(\\d+\\.\\d+\\.\\d+\\.\\d+)");
 static std::string pattern_ipport ("(\\d+\\.\\d+\\.\\d+\\.\\d+):(\\d+)");
 
-static std::string pattern_ip6 ("\\[([0-9a-fA-F]{1,4}:{7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]");
+static std::string pattern_ip6 ("\\[((?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]");
 // stolen from stackoverflow http://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
 
 //static std::string pattern_ip6 ("\\[([0-9a-fA-F]{1,4}:{7,7}[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,7}:|[0-9a-fA-F]{1,4}:{1,6}:[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,5}:[0-9a-fA-F]{1,4}{1,2}|[0-9a-fA-F]{1,4}:{1,4}:[0-9a-fA-F]{1,4}{1,3}|[0-9a-fA-F]{1,4}:{1,3}:[0-9a-fA-F]{1,4}{1,4}|[0-9a-fA-F]{1,4}:{1,2}:[0-9a-fA-F]{1,4}{1,5}|[0-9a-fA-F]{1,4}::[0-9a-fA-F]{1,4}{1,6}|::[0-9a-fA-F]{1,4}{1,7}|:|fe80::[0-9a-fA-F]{0,4}{0,4}%[0-9a-zA-Z]{1,}|::ffff:0{1,4}{0,1}:{0,1}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9]|[0-9a-fA-F]{1,4}:{1,4}:25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9])\\]");
 
-static std::string pattern_ip6port ("\\[([0-9a-fA-F]{1,4}:{7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]:(\\d+)");
+static std::string pattern_ip6port ("\\[((?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]:(\\d+)");
 //static std::string pattern_ip6port ("\\[([0-9a-fA-F]{1,4}:{7,7}[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,7}:|[0-9a-fA-F]{1,4}:{1,6}:[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,5}:[0-9a-fA-F]{1,4}{1,2}|[0-9a-fA-F]{1,4}:{1,4}:[0-9a-fA-F]{1,4}{1,3}|[0-9a-fA-F]{1,4}:{1,3}:[0-9a-fA-F]{1,4}{1,4}|[0-9a-fA-F]{1,4}:{1,2}:[0-9a-fA-F]{1,4}{1,5}|[0-9a-fA-F]{1,4}::[0-9a-fA-F]{1,4}{1,6}|::[0-9a-fA-F]{1,4}{1,7}|:|fe80::[0-9a-fA-F]{0,4}{0,4}%[0-9a-zA-Z]{1,}|::ffff:0{1,4}{0,1}:{0,1}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9]|[0-9a-fA-F]{1,4}:{1,4}:25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9])\\]:(\\d+)");
 // stolen from stackoverflow http://stackoverflow.com/questions/53497/regular-expression-that-
 
diff --git a/src/IndexField.cc b/src/IndexField.cc
index 7967c5c..96340b8 100644
--- a/src/IndexField.cc
+++ b/src/IndexField.cc
@@ -27,7 +27,7 @@ static std::string pattern_ip ("(\\d+\\.\\d+\\.\\d+\\.\\d+)"); // TODO: figure o
 
 //static std::string alphanum14 ("[0-9a-fA-F]{1,4}");
 
-static std::string pattern_ip6 ("\\[([0-9a-fA-F]{1,4}:{7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]");
+static std::string pattern_ip6 ("\\[((?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]");
 // stolen from stackoverflow http://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
 
 //static std::string pattern_ip6 ("\\[([0-9a-fA-F]{1,4}:{7,7}[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,7}:|[0-9a-fA-F]{1,4}:{1,6}:[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,5}:[0-9a-fA-F]{1,4}{1,2}|[0-9a-fA-F]{1,4}:{1,4}:[0-9a-fA-F]{1,4}{1,3}|[0-9a-fA-F]{1,4}:{1,3}:[0-9a-fA-F]{1,4}{1,4}|[0-9a-fA-F]{1,4}:{1,2}:[0-9a-fA-F]{1,4}{1,5}|[0-9a-fA-F]{1,4}::[0-9a-fA-F]{1,4}{1,6}|::[0-9a-fA-F]{1,4}{1,7}|:|fe80::[0-9a-fA-F]{0,4}{0,4}%[0-9a-zA-Z]{1,}|::ffff:0{1,4}{0,1}:{0,1}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9]|[0-9a-fA-F]{1,4}:{1,4}:25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9])\\]");
@@ -51,7 +51,7 @@ static std::string pattern_ipport ("(\\d+\\.\\d+\\.\\d+\\.\\d+):(\\d+)");
 
 //static std::string pattern_ip6port ("\\[([0-9a-fA-F]{1,4}:{7,7}[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,7}:|[0-9a-fA-F]{1,4}:{1,6}:[0-9a-fA-F]{1,4}|[0-9a-fA-F]{1,4}:{1,5}:[0-9a-fA-F]{1,4}{1,2}|[0-9a-fA-F]{1,4}:{1,4}:[0-9a-fA-F]{1,4}{1,3}|[0-9a-fA-F]{1,4}:{1,3}:[0-9a-fA-F]{1,4}{1,4}|[0-9a-fA-F]{1,4}:{1,2}:[0-9a-fA-F]{1,4}{1,5}|[0-9a-fA-F]{1,4}::[0-9a-fA-F]{1,4}{1,6}|::[0-9a-fA-F]{1,4}{1,7}|:|fe80::[0-9a-fA-F]{0,4}{0,4}%[0-9a-zA-Z]{1,}|::ffff:0{1,4}{0,1}:{0,1}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9]|[0-9a-fA-F]{1,4}:{1,4}:25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9].{3,3}25[0-5]|2[0-4]|1{0,1}[0-9]{0,1}[0-9])\\]:(\\d+)");
 
-static std::string pattern_ip6port ("\\[([0-9a-fA-F]{1,4}:{7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]:(\\d+)");
+static std::string pattern_ip6port ("\\[((?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]:(\\d+)");
 // stolen from stackoverflow http://stacikoverflow.com/questions/53497/regular-expression-that-
 
 


From neto at caltech.edu  Wed Aug 20 16:22:08 2014
From: neto at caltech.edu (Naoki Eto)
Date: Wed, 20 Aug 2014 16:22:08 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Fixed getStr() method for IPv4
	(6fe29c1)
Message-ID: <201408202322.s7KNM8DX023403@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit 6fe29c1d24f0dda46475f387a60aeb1a8a34ac7c
Author: NaokiEto <neto at caltech.edu>
Date:   Wed Aug 20 19:10:10 2014 -0400

    Fixed getStr() method for IPv4


>---------------------------------------------------------------

6fe29c1d24f0dda46475f387a60aeb1a8a34ac7c
 src/Connection.cc | 48 ++++++++++++++++++++++++------------------------
 src/IndexField.cc |  7 +++++++
 2 files changed, 31 insertions(+), 24 deletions(-)

diff --git a/src/Connection.cc b/src/Connection.cc
index f16e5b3..a01927d 100644
--- a/src/Connection.cc
+++ b/src/Connection.cc
@@ -903,17 +903,17 @@ std::string ConnectionID4::getStr() const {
 	    << (UCP(d_ip)[3] & 0xff)
 	    << ":"
 */
-        << (UCP(key.ip1)[12] & 0xff) << "."
-        << (UCP(key.ip1)[13] & 0xff) << "."
-        << (UCP(key.ip1)[14] & 0xff) << "."
-        << (UCP(key.ip1)[15] & 0xff)
+        << (UCP(key.ip1.s6_addr)[12] & 0xff) << "."
+        << (UCP(key.ip1.s6_addr)[13] & 0xff) << "."
+        << (UCP(key.ip1.s6_addr)[14] & 0xff) << "."
+        << (UCP(key.ip1.s6_addr)[15] & 0xff)
         << ":"
         << ntohs(get_port1())
         << " - "
-        << (UCP(key.ip2)[12] & 0xff) << "."
-        << (UCP(key.ip2)[13] & 0xff) << "."
-        << (UCP(key.ip2)[14] & 0xff) << "."
-        << (UCP(key.ip2)[15] & 0xff)
+        << (UCP(key.ip2.s6_addr)[12] & 0xff) << "."
+        << (UCP(key.ip2.s6_addr)[13] & 0xff) << "."
+        << (UCP(key.ip2.s6_addr)[14] & 0xff) << "."
+        << (UCP(key.ip2.s6_addr)[15] & 0xff)
         << ":"
 
 
@@ -981,15 +981,15 @@ std::string ConnectionID3::getStr() const {
         //memcpy(d_ip, key.ip2.s6_addr, 16);
 
 	    ss << " ConnectionID3 "
-	    << (UCP(key.ip1)[12] & 0xff) << "."
-	    << (UCP(key.ip1)[13] & 0xff) << "."
-	    << (UCP(key.ip1)[14] & 0xff) << "."
-	    << (UCP(key.ip1)[15] & 0xff)
+	    << (UCP(key.ip1.s6_addr)[12] & 0xff) << "."
+	    << (UCP(key.ip1.s6_addr)[13] & 0xff) << "."
+	    << (UCP(key.ip1.s6_addr)[14] & 0xff) << "."
+	    << (UCP(key.ip1.s6_addr)[15] & 0xff)
 	    << " - "
-	    << (UCP(key.ip2)[12] & 0xff) << "."
-	    << (UCP(key.ip2)[13] & 0xff) << "."
-	    << (UCP(key.ip2)[14] & 0xff) << "."
-	    << (UCP(key.ip2)[15] & 0xff)
+	    << (UCP(key.ip2.s6_addr)[12] & 0xff) << "."
+	    << (UCP(key.ip2.s6_addr)[13] & 0xff) << "."
+	    << (UCP(key.ip2.s6_addr)[14] & 0xff) << "."
+	    << (UCP(key.ip2.s6_addr)[15] & 0xff)
 	    << ":"
 	    << get_port();
 	    return ss.str();
@@ -1075,15 +1075,15 @@ std::string ConnectionID2::getStr() const {
         //memcpy(d_ip, key.ip2.s6_addr, 16);
 
 	    ss << " ConnectionID2 "
-	    << (UCP(key.ip1)[12] & 0xff) << "."
-	    << (UCP(key.ip1)[13] & 0xff) << "."
-	    << (UCP(key.ip1)[14] & 0xff) << "."
-	    << (UCP(key.ip1)[15] & 0xff)
+	    << (UCP(key.ip1.s6_addr)[12] & 0xff) << "."
+	    << (UCP(key.ip1.s6_addr)[13] & 0xff) << "."
+	    << (UCP(key.ip1.s6_addr)[14] & 0xff) << "."
+	    << (UCP(key.ip1.s6_addr)[15] & 0xff)
 	    << " - "
-	    << (UCP(key.ip2)[12] & 0xff) << "."
-	    << (UCP(key.ip2)[13] & 0xff) << "."
-	    << (UCP(key.ip2)[14] & 0xff) << "."
-	    << (UCP(key.ip2)[15] & 0xff);
+	    << (UCP(key.ip2.s6_addr)[12] & 0xff) << "."
+	    << (UCP(key.ip2.s6_addr)[13] & 0xff) << "."
+	    << (UCP(key.ip2.s6_addr)[14] & 0xff) << "."
+	    << (UCP(key.ip2.s6_addr)[15] & 0xff);
 	    return ss.str();
     }
 
diff --git a/src/IndexField.cc b/src/IndexField.cc
index 96340b8..40d1326 100644
--- a/src/IndexField.cc
+++ b/src/IndexField.cc
@@ -729,6 +729,9 @@ void ConnectionIF4::getBPFStr(char *str, int max_str_len) const {
 	    char d_ip_str[TM_IP_STR_SIZE];
 	    uint32_t s_port;
 	    uint32_t d_port;
+
+        printf("the query is for %s\n", c_id.getStr().c_str());
+
 	    /*
 	    if (c_id.get_is_canonified()) {
 	      s_ip=c_id.get_ip2();
@@ -940,6 +943,8 @@ void ConnectionIF3::getBPFStr(char *str, int max_str_len) const {
 	    char ip1_str[TM_IP_STR_SIZE];
 	    char ip2_str[TM_IP_STR_SIZE];
 
+        printf("the query is for %s\n", c_id.getStr().c_str());
+
 	    ip4_to_str(c_id.get_ip1(), ip1_str, sizeof(ip1_str));
 	    ip4_to_str(c_id.get_ip2(), ip2_str, sizeof(ip2_str));
 
@@ -1096,6 +1101,8 @@ void ConnectionIF2::getBPFStr(char *str, int max_str_len) const {
         char s_ip_str[TM_IP_STR_SIZE];
         char d_ip_str[TM_IP_STR_SIZE];
 
+        printf("the query is for %s\n", c_id.getStr().c_str());
+
         printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
         printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
 


From neto at caltech.edu  Thu Aug 21 10:40:46 2014
From: neto at caltech.edu (Naoki Eto)
Date: Thu, 21 Aug 2014 10:40:46 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Changed data structure for
	treating IPv4 addresses in Connection.hh for conn4 only. This
	change was made to help compete with original TM's simple
	comparisons for IPv4 addresses I will change it for conn3 and
	conn2 next. Also commented out unnecessary debug statements
	which take some CPU usage. (0175f2b)
Message-ID: <201408211740.s7LHekK0002447@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit 0175f2b9f3d7a6237d57ba20ecc10233b6e0fb96
Author: NaokiEto <neto at caltech.edu>
Date:   Thu Aug 21 13:26:27 2014 -0400

    Changed data structure for treating IPv4 addresses in Connection.hh for conn4 only.
    This change was made to help compete with original TM's simple comparisons for IPv4 addresses
    I will change it for conn3 and conn2 next.
    Also commented out unnecessary debug statements which take some CPU usage.


>---------------------------------------------------------------

0175f2b9f3d7a6237d57ba20ecc10233b6e0fb96
 src/Connection.cc |  7 +++++--
 src/Connection.hh | 19 +++++++++++++++++++
 src/Fifo.cc       |  6 +++---
 src/FifoDisk.cc   |  6 +++---
 src/IndexField.cc | 38 +++++++++++++++++++-------------------
 src/IndexField.hh |  8 ++++----
 src/IndexHash.cc  | 22 +++++++++++-----------
 src/Storage.cc    |  2 +-
 8 files changed, 65 insertions(+), 43 deletions(-)

diff --git a/src/Connection.cc b/src/Connection.cc
index a01927d..504f337 100644
--- a/src/Connection.cc
+++ b/src/Connection.cc
@@ -100,7 +100,8 @@ void ConnectionID4::init(proto_t proto4,
         memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
         memcpy(&key.ip2.s6_addr[12], &s_ip, sizeof(s_ip));
 
-
+        v6.ip1 = d_ip;
+        v6.ip2 = s_ip;
 
         /*
         IPAddr(ipv4_d_address);
@@ -144,6 +145,8 @@ void ConnectionID4::init(proto_t proto4,
         memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
         memcpy(&key.ip2.s6_addr[12], &d_ip, sizeof(d_ip));
 
+        v6.ip1 = s_ip;
+        v6.ip2 = d_ip;
 
         /*
         IPAddr(ipv4_s_address);
@@ -1187,7 +1190,7 @@ ConnectionID4* ConnectionID4::parse(const char *str) {
 	proto_t proto;
 
 	if (!RE2::FullMatch(str, re, &protostr, &src_ip, &src_port, &dst_ip, &dst_port) && !RE2::FullMatch(str, re6, &protostr, &src_ip, &src_port, &dst_ip, &dst_port)) {
-        tmlog(TM_LOG_ERROR, "ConnectionID4", "No match found");
+        //tmlog(TM_LOG_ERROR, "ConnectionID4", "No match found");
 		return NULL;
 	}
 	if (protostr == std::string("tcp"))
diff --git a/src/Connection.hh b/src/Connection.hh
index 9101ab5..ddddf89 100644
--- a/src/Connection.hh
+++ b/src/Connection.hh
@@ -62,6 +62,8 @@ public:
 		key.port2 = c_id->key.port2;
 		v6.proto = c_id->v6.proto;
         v6.version = c_id->v6.version;
+        v6.ip1 = c_id->v6.ip1;
+        v6.ip2 = c_id->v6.ip2;
 	}
 	ConnectionID4(const u_char* packet);
 	ConnectionID4() {};
@@ -103,12 +105,27 @@ public:
                return (!memcmp(key, ((ConnectionID4*)&other)->key, sizeof(key_t)))
                         && (v6.proto == ((ConnectionID4*)&other)->v6.proto);
                */
+                if (v6.version == 4 && ((ConnectionID4*)&other)->v6.version == 4)
+                {
+                //return (!memcmp(&key.ip1.s6_addr + 12, &((ConnectionID4*)&other)->key.ip1.s6_addr + 12, 4))
+                //           && (!memcmp(&key.ip2.s6_addr + 12, &((ConnectionID4*)&other)->key.ip2.s6_addr + 12, 4))
+                return (v6.ip1 == ((ConnectionID4*)&other)->v6.ip1)
+                           && (v6.ip2 == ((ConnectionID4*)&other)->v6.ip2)
+                           && (key.port1 == ((ConnectionID4*)&other)->key.port1)
+                           && (key.port2 == ((ConnectionID4*)&other)->key.port2)
+                           && (v6.proto == ((ConnectionID4*)&other)->v6.proto);
 
+                }                 
+                else if (v6.version == 6 && ((ConnectionID4*)&other)->v6.version == 6)
+                {
 		return (!memcmp(&key.ip1.s6_addr, &((ConnectionID4*)&other)->key.ip1.s6_addr, 16))
 			   && (!memcmp(&key.ip2.s6_addr, &((ConnectionID4*)&other)->key.ip2.s6_addr, 16))
 			   && (key.port1 == ((ConnectionID4*)&other)->key.port1)
 			   && (key.port2 == ((ConnectionID4*)&other)->key.port2)
 			   && (v6.proto == ((ConnectionID4*)&other)->v6.proto);
+                }
+                else
+                    return false;
                 /*
                 return equal(key.ip1.s6_addr, ((ConnectionID4*)&other)->key.ip2.s6_addr)
                            && equal(key.ip2.s6_addr, ((ConnectionID4*)&other)->key.ip2.s6_addr)
@@ -206,6 +223,8 @@ public:
 		//uint16_t port2;
         int version;
 		proto_t proto;
+                int ip1;
+                int ip2;
 		//    bool is_canonified;
 	}
     // have the structure fields align on one-byte boundaries
diff --git a/src/Fifo.cc b/src/Fifo.cc
index ac4c984..0c84705 100644
--- a/src/Fifo.cc
+++ b/src/Fifo.cc
@@ -259,8 +259,8 @@ uint64_t Fifo::query(QueryRequest *qreq, QueryResult *qres,
         }
     */ 
 
-    printf("The class name is: %s\n", classname.c_str());
-    printf("The directory the classes are in is: %s\n", classdir); 
+    //printf("The class name is: %s\n", classname.c_str());
+    //printf("The directory the classes are in is: %s\n", classdir); 
 	FifoDiskFile *cur_file;
 
 	if (!qreq->isMemOnly()) {
@@ -291,7 +291,7 @@ uint64_t Fifo::query(QueryRequest *qreq, QueryResult *qres,
 
                 char errbufnav[PCAP_ERRBUF_SIZE];
 
-                printf("The directory for Fifo that we are in is %s\n", getcwd(path, 70));
+                //printf("The directory for Fifo that we are in is %s\n", getcwd(path, 70));
 
 
 				matches+= cur_file->query(qreq, qres, interval_set, classdir);
diff --git a/src/FifoDisk.cc b/src/FifoDisk.cc
index 159b4a5..b3b9731 100644
--- a/src/FifoDisk.cc
+++ b/src/FifoDisk.cc
@@ -130,7 +130,7 @@ void FifoDisk::addPkt(const pkt_ptr p) {
                 // push back the newest disk file into the list of files
 				files.push_back(new FifoDiskFile(new_file_name, pcap_handle));
 
-                tmlog(TM_LOG_NOTE, "FifoDisk: addPkt", "the new file name is: %s", new_file_name);
+                //tmlog(TM_LOG_NOTE, "FifoDisk: addPkt", "the new file name is: %s", new_file_name);
                 // free new_file_name since we malloced it and don't need it anymore
 				free(new_file_name);
 			}
@@ -277,7 +277,7 @@ uint64_t FifoDiskFile::query( QueryRequest *qreq, QueryResult *qres, IntervalSet
 
 	char errbuf[PCAP_ERRBUF_SIZE];
 
-    printf("The file name we are querying in is %s\n", filename.c_str());
+    //printf("The file name we are querying in is %s\n", filename.c_str());
 
     if (chdir(classdirectory)) {
         fprintf(stderr, "cannot class(Fifo:query) chdir to %s\n", classdirectory);
@@ -288,7 +288,7 @@ uint64_t FifoDiskFile::query( QueryRequest *qreq, QueryResult *qres, IntervalSet
 
     char errbufnav[PCAP_ERRBUF_SIZE];
 
-    printf("The directory that we are in is %s\n", getcwd(path, 70));
+    //printf("The directory that we are in is %s\n", getcwd(path, 70));
 /*
     if (chdir(classdirectory)) {
         fprintf(stderr, "cannot class(Fifo:query) chdir to %s\n", classdirectory);
diff --git a/src/IndexField.cc b/src/IndexField.cc
index 40d1326..67fef91 100644
--- a/src/IndexField.cc
+++ b/src/IndexField.cc
@@ -58,7 +58,7 @@ static std::string pattern_ip6port ("\\[((?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4
 /* size of an ip addr in dottet decimal as string: 4x3digits, 
   3 dots, terminating nul byte */
 #define TM_IP_STR_SIZE 16
-#define TM_IP6_STR_SIZE 40
+//#define TM_IP6_STR_SIZE 40
 /*
 static void ip_to_str(const unsigned char* ip, char *str, int len) {
 //#define UCP(x) ((unsigned char *)&(x))
@@ -96,7 +96,7 @@ IndexField* IPAddress::parseQuery(const char *query) {
 
 	if (!RE2::FullMatch(query, re, &ip) && !RE2::FullMatch(query, re6, &ip))
     {
-        tmlog(TM_LOG_ERROR,"parseQuery", "Cannot do full match!");
+        //tmlog(TM_LOG_ERROR,"parseQuery", "Cannot do full match!");
         //tmlog(TM_LOG_ERROR, "parseQuery", ip);
 		return NULL;
     }
@@ -113,7 +113,7 @@ IndexField* IPAddress::parseQuery(const char *query) {
     // INET6_ADDRSTRLEN is 46 and INET_ADDRSTRLEN is 16
     //char strIP46[INET6_ADDRSTRLEN];
 
-    tmlog(TM_LOG_NOTE, "parseQuery", "the argument we pass to IPAddress is %s", ip.c_str());
+    //tmlog(TM_LOG_NOTE, "parseQuery", "the argument we pass to IPAddress is %s", ip.c_str());
 
 	return new IPAddress(ip.c_str());//, strIP46);
 }
@@ -138,21 +138,21 @@ void IPAddress::getStr(char* s, int maxsize) const {
 
 	if ( GetFamily() == IPv4 )
 		{
-        tmlog(TM_LOG_NOTE, "IPAddress", "IPAddress, IPv4");
+        //tmlog(TM_LOG_NOTE, "IPAddress", "IPAddress, IPv4");
 		char ucp[INET_ADDRSTRLEN];
 
 		if ( ! bro_inet_ntop(AF_INET, &ipv6_address.s6_addr[12], ucp, INET_ADDRSTRLEN) )
-			tmlog(TM_LOG_ERROR, "IPAddress", "<bad IPv4 address conversion");
+		        tmlog(TM_LOG_ERROR, "IPAddress", "<bad IPv4 address conversion");
 		else
 			snprintf(s, maxsize, "%s", ucp);
 		}
 	else
 		{
-        tmlog(TM_LOG_NOTE, "IPAddress", "IPAddress, IPv6");
+        //tmlog(TM_LOG_NOTE, "IPAddress", "IPAddress, IPv6");
 		char ucp[INET6_ADDRSTRLEN];
 
 		if ( ! bro_inet_ntop(AF_INET6, ipv6_address.s6_addr, ucp, INET6_ADDRSTRLEN) )
-			tmlog(TM_LOG_ERROR, "IPAddress", "<bad IPv6 address conversion");
+		        tmlog(TM_LOG_ERROR, "IPAddress", "<bad IPv6 address conversion");
 		else
 			snprintf(s, maxsize, "%s", ucp);
 		}
@@ -186,7 +186,7 @@ void IPAddress::Init(const std::string& s)
     // if it could not find :, then it is equal to npos and so IPv4
 	if ( s.find(':') == std::string::npos ) // IPv4.
 		{
-        tmlog(TM_LOG_NOTE, "IPAddress:Init", "initializing IPAddress, IPv4");
+        //tmlog(TM_LOG_NOTE, "IPAddress:Init", "initializing IPAddress, IPv4");
 		memcpy(ipv6_address.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
 
 		// Parse the address directly instead of using inet_pton since
@@ -220,7 +220,7 @@ void IPAddress::Init(const std::string& s)
 
 	else
 		{
-        tmlog(TM_LOG_NOTE, "IPAddress:Init", "initializing IPAddress, IPv6");
+        //tmlog(TM_LOG_NOTE, "IPAddress:Init", "initializing IPAddress, IPv6");
 		if ( inet_pton(AF_INET6, s.c_str(), ipv6_address.s6_addr) <=0 )
 			{
             tmlog(TM_LOG_ERROR, "Bad IP address: %s", s.c_str());
@@ -228,7 +228,7 @@ void IPAddress::Init(const std::string& s)
 			memset(ipv6_address.s6_addr, 0, sizeof(ipv6_address.s6_addr));
 			}
 		}
-        tmlog(TM_LOG_NOTE, "IPAddress::Init", "good IP Address %s", s.c_str());
+        //tmlog(TM_LOG_NOTE, "IPAddress::Init", "good IP Address %s", s.c_str());
 
         //init_hash_function();
         HashKey* newHashKey = new HashKey((void*)ipv6_address.s6_addr, sizeof(ipv6_address.s6_addr));
@@ -245,7 +245,7 @@ std::string IPAddress::getStr() const
     
 	if ( GetFamily() == IPv4 )
 		{
-        tmlog(TM_LOG_ERROR, "IPAddress: getStr()", "IPAddress, IPv4");
+        //tmlog(TM_LOG_ERROR, "IPAddress: getStr()", "IPAddress, IPv4");
 		char s[INET_ADDRSTRLEN];
 
 		if ( ! bro_inet_ntop(AF_INET, &ipv6_address.s6_addr[12], s, INET_ADDRSTRLEN) ) //bro_inet_ntop(AF_INET, &ipv6_address.s6_addr[12], s, INET_ADDRSTRLEN) )
@@ -256,7 +256,7 @@ std::string IPAddress::getStr() const
 	else
 		{
         
-        tmlog(TM_LOG_ERROR, "IPAddress: getStr()", "IPAddress, IPv6");
+        //tmlog(TM_LOG_ERROR, "IPAddress: getStr()", "IPAddress, IPv6");
 		char s[INET6_ADDRSTRLEN];
 
 		if ( ! bro_inet_ntop(AF_INET6, ipv6_address.s6_addr, s, INET6_ADDRSTRLEN) ) //bro_inet_ntop(AF_INET6, ipv6_address.s6_addr, s, INET6_ADDRSTRLEN) )
@@ -289,7 +289,7 @@ std::string IPAddress::getStrPkt(const u_char* packet) const
 	else
 		{   
 
-        tmlog(TM_LOG_NOTE, "IPAddress: getStr(u_char*)", "IPAddress, IPv6");
+        //tmlog(TM_LOG_NOTE, "IPAddress: getStr(u_char*)", "IPAddress, IPv6");
 		char str[INET6_ADDRSTRLEN];
 
 		if ( ! bro_inet_ntop(AF_INET6, ipv6_address.s6_addr, str, INET6_ADDRSTRLEN) )
@@ -308,7 +308,7 @@ std::string IPAddress::getStrPkt(const u_char* packet) const
 void IPAddress::getBPFStr(char *str, int max_str_len) const {
 	int rc = snprintf(str, max_str_len, "host %s", getStr().c_str());
 	if ( rc >= max_str_len )
-		tmlog(TM_LOG_ERROR, "query",  "IPAddress::getBPFStr: %s truncated by %d characters",
+	    tmlog(TM_LOG_ERROR, "query",  "IPAddress::getBPFStr: %s truncated by %d characters",
 				str, rc-max_str_len);
 }
 
@@ -317,14 +317,14 @@ SrcIPAddress::SrcIPAddress(const u_char* packet)
 {
     if (IP(packet)->ip_v == 4)
     {
-        tmlog(TM_LOG_NOTE, "IndexField.cc: SrcIPAddress", "IPv4 initialization");
+       //tmlog(TM_LOG_NOTE, "IndexField.cc: SrcIPAddress", "IPv4 initialization");
 	    new IPAddress(IP(packet)->ip_src.s_addr);
 
     }
 
     else
     {
-        tmlog(TM_LOG_NOTE, "IndexField.cc: SrcIPAddress", "IPv6 initialization");
+        //tmlog(TM_LOG_NOTE, "IndexField.cc: SrcIPAddress", "IPv6 initialization");
         new IPAddress(IP6(packet)->ip6_src.s6_addr);  
     } 
 }
@@ -415,7 +415,7 @@ std::string DstIPAddress::getStrPkt(const u_char* packet) const
 	else
 		{   
 
-        tmlog(TM_LOG_NOTE, "DstIPAddress: getStr(u_char*)", "IPAddress, IPv6");
+        //tmlog(TM_LOG_NOTE, "DstIPAddress: getStr(u_char*)", "IPAddress, IPv6");
 		char str[INET6_ADDRSTRLEN];
 
 		if ( ! bro_inet_ntop(AF_INET6, IP6(packet)->ip6_dst.s6_addr, str, INET6_ADDRSTRLEN) )
@@ -845,7 +845,7 @@ IndexField* ConnectionIF3::parseQuery(const char *query) {
 	
 	if (!RE2::FullMatch(query, re) && !RE2::FullMatch(query, re6))
     {
-        tmlog(TM_LOG_ERROR,"parseQuery", "Cannot do full match!");
+        //tmlog(TM_LOG_ERROR,"parseQuery", "Cannot do full match!");
 
         return NULL;
     }
@@ -1026,7 +1026,7 @@ IndexField* ConnectionIF2::parseQuery(const char *query) {
 	
 	if (!RE2::FullMatch(query, re) && !RE2::FullMatch(query, re6))
     {
-        tmlog(TM_LOG_ERROR,"parseQuery", "Cannot do full match!");
+        //tmlog(TM_LOG_ERROR,"parseQuery", "Cannot do full match!");
 
 		return NULL;
     }
diff --git a/src/IndexField.hh b/src/IndexField.hh
index 70a5d54..ae5816a 100644
--- a/src/IndexField.hh
+++ b/src/IndexField.hh
@@ -172,7 +172,7 @@ public:
         //free_hash_function();
 	}
 	virtual ~IPAddress() {
-        tmlog(TM_LOG_NOTE, "IPAddress", "deleting an ipaddress type");
+        //tmlog(TM_LOG_NOTE, "IPAddress", "deleting an ipaddress type");
         //delete [] ip6_address;
         //delete [] strIP;
     };
@@ -185,7 +185,7 @@ public:
 	Family GetFamily() const
 		{
         
-		if ( memcmp(ipv6_address.s6_addr, v4_mapped_prefix, 12) == 0 )
+		if (!memcmp(ipv6_address.s6_addr, v4_mapped_prefix, 12))
 			return IPv4;
 		else
 			return IPv6;
@@ -313,14 +313,14 @@ public:
             if(IP(packet)->ip_v == 4)
             {
                 
-                tmlog(TM_LOG_NOTE, "SrcIPAddress: genKey", "get key for IPv4 address");
+                //tmlog(TM_LOG_NOTE, "SrcIPAddress: genKey", "get key for IPv4 address");
                 return new SrcIPAddress(IP(packet)->ip_src.s_addr);
             }
 
             else
             {
                 //tmlog(TM_LOG_NOTE, "SrcIPAddress:genkey tester", "the version for Ipv6 is: %d", IP6(packet)->ip6_ctlun.ip6_un1.ip6_un1_flow);
-                tmlog(TM_LOG_NOTE, "SrcIPAddress: genKey", "get key for IPv6 address");
+                //tmlog(TM_LOG_NOTE, "SrcIPAddress: genKey", "get key for IPv6 address");
                 return new SrcIPAddress(IP6(packet)->ip6_src.s6_addr);
             }
         }
diff --git a/src/IndexHash.cc b/src/IndexHash.cc
index c50ff48..fd66e72 100644
--- a/src/IndexHash.cc
+++ b/src/IndexHash.cc
@@ -95,7 +95,7 @@ IndexEntry* IndexHash::lookup( IndexField* key) {
 /*
     //IndexEntry *cur;
 
-    tmlog(TM_LOG_NOTE, "idxhash", "checking that there is not a similar key for this timestamp: %f and info: %s", key->ts, key->getStr().c_str());
+    //tmlog(TM_LOG_NOTE, "idxhash", "checking that there is not a similar key for this timestamp: %f and info: %s", key->ts, key->getStr().c_str());
     //printf("This key has the following form: " + key->getStr() + "\n");
 
     //std::cout << "This key has the following form: " << key->getStr() << std::endl;
@@ -104,9 +104,9 @@ IndexEntry* IndexHash::lookup( IndexField* key) {
 
     //cur = htable[key->getInt()%numBuckets];
 
-    tmlog(TM_LOG_NOTE, "idxhash", "the hash is: %u for this timestamp %f and form %s", key->hash(), key->ts, key->getStr().c_str());
-    tmlog(TM_LOG_NOTE, "idxhash", "the index is: %u for this timestamp %f and form %s", key->hash()%numBuckets, key->ts, key->getStr().c_str());
-    tmlog(TM_LOG_NOTE, "idxhash", "the number of buckets is %d for this timestampe %f and form %s", numBuckets, key->ts, key->getStr().c_str());
+    //tmlog(TM_LOG_NOTE, "idxhash", "the hash is: %u for this timestamp %f and form %s", key->hash(), key->ts, key->getStr().c_str());
+    //tmlog(TM_LOG_NOTE, "idxhash", "the index is: %u for this timestamp %f and form %s", key->hash()%numBuckets, key->ts, key->getStr().c_str());
+    //tmlog(TM_LOG_NOTE, "idxhash", "the number of buckets is %d for this timestampe %f and form %s", numBuckets, key->ts, key->getStr().c_str());
 
     // testing out the other method in add method to determine if this would help
     IndexEntry *curalt;
@@ -118,9 +118,9 @@ IndexEntry* IndexHash::lookup( IndexField* key) {
         assert(troot->parent == NULL);
 #endif
 
-    tmlog(TM_LOG_NOTE, "idx_hash: lookup", "the entry to add 'lookup' has key: %d", *(key->getConstKeyPtr()));
+    //tmlog(TM_LOG_NOTE, "idx_hash: lookup", "the entry to add 'lookup' has key: %d", *(key->getConstKeyPtr()));
 
-    tmlog(TM_LOG_NOTE, "idx_hash: lookup", "the entry to add 'lookup' has timestamp: %f", key->ts);
+    //tmlog(TM_LOG_NOTE, "idx_hash: lookup", "the entry to add 'lookup' has timestamp: %f", key->ts);
 
     while (curalt) {
 #ifdef TM_HEAVY_DEBUG
@@ -132,8 +132,8 @@ IndexEntry* IndexHash::lookup( IndexField* key) {
         else if (cmp < 0)
             curalt = curalt->right;
         else {
-            tmlog(TM_LOG_NOTE, "idx_hash: lookup", "this is in lookup using the add method checker. the already existing entry is: %d\n", *(curalt->getKey()->getConstKeyPtr()));
-            tmlog(TM_LOG_NOTE, "idx_hash: lookup",  "this is in lookup using the add method checker. tried to insert an already existing entry into the tree. numEntries=%d\n",
+            //tmlog(TM_LOG_NOTE, "idx_hash: lookup", "this is in lookup using the add method checker. the already existing entry is: %d\n", *(curalt->getKey()->getConstKeyPtr()));
+            //tmlog(TM_LOG_NOTE, "idx_hash: lookup",  "this is in lookup using the add method checker. tried to insert an already existing entry into the tree. numEntries=%d\n",
                     getNumEntries());
             break;
             //h->add_or_update(key, ie);
@@ -141,7 +141,7 @@ IndexEntry* IndexHash::lookup( IndexField* key) {
     }
    
     if (curalt == NULL)
-        tmlog(TM_LOG_NOTE, "idxhash", "cur is NULL, which means that this entry is allegedly unique");
+        //tmlog(TM_LOG_NOTE, "idxhash", "cur is NULL, which means that this entry is allegedly unique");
     
     return curalt;
 */
@@ -151,12 +151,12 @@ IndexEntry* IndexHash::lookup( IndexField* key) {
     cur = htable[key->hash()%numBuckets];
 
     /*
-    tmlog(TM_LOG_NOTE, "IndexHash:Lookup", "huh, number of buckets is shit %d and the key is %d", numBuckets, key->hash());
+    //tmlog(TM_LOG_NOTE, "IndexHash:Lookup", "huh, number of buckets is shit %d and the key is %d", numBuckets, key->hash());
 
     if (numBuckets > 0)
     {
 
-        tmlog(TM_LOG_NOTE, "IndexHash:Lookup", "huh, can we do modular shit, %d", key->hash()%numBuckets);
+        //tmlog(TM_LOG_NOTE, "IndexHash:Lookup", "huh, can we do modular shit, %d", key->hash()%numBuckets);
 
         cur = htable[key->hash()%numBuckets];
 
diff --git a/src/Storage.cc b/src/Storage.cc
index 32d43ee..5d5caec 100644
--- a/src/Storage.cc
+++ b/src/Storage.cc
@@ -172,7 +172,7 @@ Storage::Storage(StorageConfig& conf):
     // I added this in to help understand the code
     else
     {
-        tmlog(TM_LOG_NOTE, "storage", "hmmm, filter is empty!");
+        //tmlog(TM_LOG_NOTE, "storage", "hmmm, filter is empty!");
     }
     */
 	for (std::list<Fifo*>::iterator it=conf.fifos.begin(); it!=conf.fifos.end(); it++) {


From neto at caltech.edu  Thu Aug 21 12:00:01 2014
From: neto at caltech.edu (Naoki Eto)
Date: Thu, 21 Aug 2014 12:00:01 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Added the data structure for
	IPv4 for conn2 and conn3 to help compete with the original
	TM's method of comparing ip addresses (47cb41d)
Message-ID: <201408211900.s7LJ01DE003032@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit 47cb41d1a0df29e12fff2e7a42dc27987014d5a5
Author: NaokiEto <neto at caltech.edu>
Date:   Thu Aug 21 14:47:57 2014 -0400

    Added the data structure for IPv4 for conn2 and conn3 to help compete with
    the original TM's method of comparing ip addresses


>---------------------------------------------------------------

47cb41d1a0df29e12fff2e7a42dc27987014d5a5
 src/Connection.cc | 35 ++++++++++++++++++++++++++++++-----
 src/Connection.hh | 23 ++++++++++++++---------
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/src/Connection.cc b/src/Connection.cc
index 504f337..1f95746 100644
--- a/src/Connection.cc
+++ b/src/Connection.cc
@@ -311,6 +311,8 @@ void ConnectionID3::init(proto_t proto4,
     memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
     memcpy(&key.ip2.s6_addr[12], &ip2, sizeof(ip2));
 
+    v6.ip1 = ip1;
+    v6.ip2 = ip2;
 
 	//key.port2=port2;
 
@@ -486,6 +488,9 @@ void ConnectionID2::init( uint32_t s_ip, uint32_t d_ip) {
         memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
         memcpy(&key.ip2.s6_addr[12], &s_ip, sizeof(s_ip));
 
+        v6.ip1 = d_ip;
+        v6.ip2 = s_ip;
+
         // this is for the hash key
         //in6_addr s6_ip;
         //in6_addr d6_ip;
@@ -518,6 +523,8 @@ void ConnectionID2::init( uint32_t s_ip, uint32_t d_ip) {
         memcpy(key.ip2.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
         memcpy(&key.ip2.s6_addr[12], &d_ip, sizeof(d_ip));
 
+        v6.ip1 = s_ip;
+        v6.ip2 = d_ip;
 
         // this is for the hash key
         //in6_addr s6_ip;
@@ -813,13 +820,22 @@ bool ConnectionID3::operator==(const ConnectionID& other) const {
                && (key.port2 == ((ConnectionID3*)&other)->key.port2)
                && (v6.proto == ((ConnectionID3*)&other)->v6.proto);
 */
-
-
+    if (v6.version == 4 && ((ConnectionID3*)&other)->v6.version == 4)
+    {
+        return (v6.ip1 == ((ConnectionID3*)&other)->v6.ip1)
+               && (v6.ip2 == ((ConnectionID3*)&other)->v6.ip2)
+               && (key.port2 == ((ConnectionID3*)&other)->key.port2)
+               && (v6.proto == ((ConnectionID3*)&other)->v6.proto);
+    }
+    else if (v6.version == 6 && ((ConnectionID3*)&other)->v6.version == 6)
+    {
         return (!memcmp(&key.ip1, &((ConnectionID3*)&other)->key.ip1, 16))
                && (!memcmp(&key.ip2, &((ConnectionID3*)&other)->key.ip2, 16))
                && (key.port2 == ((ConnectionID3*)&other)->key.port2)
                && (v6.proto == ((ConnectionID3*)&other)->v6.proto);
-
+    }
+    else
+        return false;
 }
 
 //TODO: MAke this inline (i.e. move to Connection.hh so that it is
@@ -829,9 +845,18 @@ bool ConnectionID2::operator==(const ConnectionID& other) const {
         //return equal(key.ip1.s6_addr, ((ConnectionID2*)&other)->key.ip2.s6_addr)
                //&& equal(key.ip2.s6_addr, ((ConnectionID2*)&other)->key.ip2.s6_addr);
 */
-       return (!memcmp(&key.ip1, &((ConnectionID2*)&other)->key.ip1, sizeof(in6_addr)))
+    if (v6.version == 4 && ((ConnectionID3*)&other)->v6.version == 4)
+    {
+        return (v6.ip1 == ((ConnectionID3*)&other)->v6.ip1)
+               && (v6.ip2 == ((ConnectionID3*)&other)->v6.ip2);   
+    }
+    else if (v6.version == 6 && ((ConnectionID3*)&other)->v6.version == 6)
+    {
+        return (!memcmp(&key.ip1, &((ConnectionID2*)&other)->key.ip1, sizeof(in6_addr)))
                && (!memcmp(&key.ip2, &((ConnectionID2*)&other)->key.ip2, sizeof(in6_addr)));
-
+    }
+    else
+        return false;
 }
 
 void ConnectionID4::getStr(char* s, int maxsize) const {
diff --git a/src/Connection.hh b/src/Connection.hh
index ddddf89..9e53448 100644
--- a/src/Connection.hh
+++ b/src/Connection.hh
@@ -109,7 +109,7 @@ public:
                 {
                 //return (!memcmp(&key.ip1.s6_addr + 12, &((ConnectionID4*)&other)->key.ip1.s6_addr + 12, 4))
                 //           && (!memcmp(&key.ip2.s6_addr + 12, &((ConnectionID4*)&other)->key.ip2.s6_addr + 12, 4))
-                return (v6.ip1 == ((ConnectionID4*)&other)->v6.ip1)
+                    return (v6.ip1 == ((ConnectionID4*)&other)->v6.ip1)
                            && (v6.ip2 == ((ConnectionID4*)&other)->v6.ip2)
                            && (key.port1 == ((ConnectionID4*)&other)->key.port1)
                            && (key.port2 == ((ConnectionID4*)&other)->key.port2)
@@ -118,11 +118,11 @@ public:
                 }                 
                 else if (v6.version == 6 && ((ConnectionID4*)&other)->v6.version == 6)
                 {
-		return (!memcmp(&key.ip1.s6_addr, &((ConnectionID4*)&other)->key.ip1.s6_addr, 16))
-			   && (!memcmp(&key.ip2.s6_addr, &((ConnectionID4*)&other)->key.ip2.s6_addr, 16))
-			   && (key.port1 == ((ConnectionID4*)&other)->key.port1)
-			   && (key.port2 == ((ConnectionID4*)&other)->key.port2)
-			   && (v6.proto == ((ConnectionID4*)&other)->v6.proto);
+                	return (!memcmp(&key.ip1.s6_addr, &((ConnectionID4*)&other)->key.ip1.s6_addr, 16))
+			            && (!memcmp(&key.ip2.s6_addr, &((ConnectionID4*)&other)->key.ip2.s6_addr, 16))
+			            && (key.port1 == ((ConnectionID4*)&other)->key.port1)
+			            && (key.port2 == ((ConnectionID4*)&other)->key.port2)
+			            && (v6.proto == ((ConnectionID4*)&other)->v6.proto);
                 }
                 else
                     return false;
@@ -397,6 +397,8 @@ public:
 		//unsigned char ip1[16];
 		//unsigned char ip2[16];
 		//uint16_t port2;
+        int ip1;
+        int ip2;
 		proto_t proto;
         int version;
 		//    bool is_canonified;
@@ -445,12 +447,13 @@ public:
     hash_t hash() const;
 
     hash_t hash_key; 
+    v6_t v6; 
 protected:
 	void init(proto_t proto, uint32_t s_ip, uint32_t d_ip, uint16_t port);
     void init6(proto_t proto, unsigned char s_ip[], unsigned char d_ip[], 
                uint16_t port);
 	//v_t v;
-    v6_t v6;
+    //v6_t v6;
     key_t key;
 
 
@@ -572,6 +575,8 @@ public:
 		//    uint32_t ts;
 		//unsigned char ip1[16];
 		//unsigned char ip2[16];
+        int ip1;
+        int ip2;
         int version;
 		//    bool is_canonified;
 	}
@@ -615,12 +620,12 @@ public:
     hash_t hash() const;
 
     hash_t hash_key;
-
+    v6_t v6;
 protected:
 	void init(uint32_t s_ip, uint32_t d_ip);
     void init6( unsigned char s_ip[], unsigned char d_ip[]);
 	//v_t v;
-    v6_t v6;
+    //v6_t v6;
     key_t key;
 
 //private:


From neto at caltech.edu  Thu Aug 21 12:08:24 2014
From: neto at caltech.edu (Naoki Eto)
Date: Thu, 21 Aug 2014 12:08:24 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Got rid of some unnecessary
	variables and warnings about multi-line comments (c41a7a1)
Message-ID: <201408211908.s7LJ8OIA003410@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit c41a7a14487857761598d26eb7b84d1a1dc9e728
Author: NaokiEto <neto at caltech.edu>
Date:   Thu Aug 21 14:56:25 2014 -0400

    Got rid of some unnecessary variables and warnings about multi-line comments


>---------------------------------------------------------------

c41a7a14487857761598d26eb7b84d1a1dc9e728
 src/Fifo.cc     | 4 ++--
 src/FifoDisk.cc | 9 +++++----
 src/FifoMem.cc  | 3 ++-
 src/Index.cc    | 3 ++-
 src/Index.hh    | 3 ++-
 5 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/Fifo.cc b/src/Fifo.cc
index 0c84705..bcfa52c 100644
--- a/src/Fifo.cc
+++ b/src/Fifo.cc
@@ -287,9 +287,9 @@ uint64_t Fifo::query(QueryRequest *qreq, QueryResult *qres,
                     //return;
                 }
 
-                char path[70];
+                //char path[70];
 
-                char errbufnav[PCAP_ERRBUF_SIZE];
+                //char errbufnav[PCAP_ERRBUF_SIZE];
 
                 //printf("The directory for Fifo that we are in is %s\n", getcwd(path, 70));
 
diff --git a/src/FifoDisk.cc b/src/FifoDisk.cc
index b3b9731..fe67790 100644
--- a/src/FifoDisk.cc
+++ b/src/FifoDisk.cc
@@ -275,7 +275,7 @@ uint64_t FifoDiskFile::query( QueryRequest *qreq, QueryResult *qres, IntervalSet
 	if (is_open)
 		flush();
 
-	char errbuf[PCAP_ERRBUF_SIZE];
+	//char errbuf[PCAP_ERRBUF_SIZE];
 
     //printf("The file name we are querying in is %s\n", filename.c_str());
 
@@ -284,9 +284,9 @@ uint64_t FifoDiskFile::query( QueryRequest *qreq, QueryResult *qres, IntervalSet
         //return;
     }
 
-    char path[70];
+    //char path[70];
 
-    char errbufnav[PCAP_ERRBUF_SIZE];
+    //char errbufnav[PCAP_ERRBUF_SIZE];
 
     //printf("The directory that we are in is %s\n", getcwd(path, 70));
 /*
@@ -430,11 +430,12 @@ uint64_t FifoDiskFile::query( QueryRequest *qreq, QueryResult *qres, IntervalSet
                     }
                 */
 
-                
+                /* 
                 //tmlog(TM_LOG_ERROR, "FifoDisk.cc: query", "the query packet has source ip address: %s and dst ip address %s and header time stamp %lu and %lu", \
                 str1, str2, hdr.ts.tv_sec, hdr.ts.tv_usec);
                 //tmlog(TM_LOG_ERROR, "FifoDisk.cc:query", "the query parameters are that it has a time interval from %f to %f, a hash of %lu, a timestamp of %f, and a form of %s", \
                 qreq->getT0(), qreq->getT1(), qreq->getField()->hash(), qreq->getField()->ts, qreq->getField()->getStr().c_str());
+                */
 				if (qreq->matchPkt(&hdr, pkt))  {
 					matches++;
 					qres->sendPkt(&hdr, pkt);
diff --git a/src/FifoMem.cc b/src/FifoMem.cc
index edab252..d55e683 100644
--- a/src/FifoMem.cc
+++ b/src/FifoMem.cc
@@ -432,10 +432,11 @@ uint64_t FifoMem::query(QueryRequest *qreq, QueryResult *qres,
                 //char s2[INET6_ADDRSTRLEN];
 
                 //inet_pton(AF_INET6, s2, str2);
-
+                /*
                 //tmlog(TM_LOG_NOTE, "FifoMem.cc: query", "the query packet has source ip address: %s and dst ip address %s", str1, str2);
                 //tmlog(TM_LOG_NOTE, "FifoMem.cc:query", "the query parameters in mem are that it has a time interval from %f to %f, a hash of %lu, a timestamp of %f, and a form of %s", \
                 qreq->getT0(), qreq->getT1(), qreq->getField()->hash(), qreq->getField()->ts, qreq->getField()->getStr().c_str());
+                */
 				if (qreq->matchPkt(p) && last_match_ts < pkt_t(p))  {
 					qres->sendPkt(p);
 					if (qreq->isSubscribe()) {
diff --git a/src/Index.cc b/src/Index.cc
index fd97b3f..12efdd9 100644
--- a/src/Index.cc
+++ b/src/Index.cc
@@ -159,10 +159,11 @@ void Index<T>::addPkt(const pcap_pkthdr* header, const u_char* packet) {
         // push this IndexField pointer entry to the front of the input queue, which is of type MyQueue
         // (Index.hh)
 		input_q.push_front(curentry);
-
+        /*
         //tmlog(TM_LOG_NOTE, "addPkt for indexfields", "we are pushing in the front an indexfield to the input queue with timestamp %f and form %s and type %s", \
         curentry->ts, curentry->getStrPkt(packet).c_str(), curentry->getIndexName().c_str());
         //tmlog(TM_LOG_NOTE, "addPkt: size of input q", "The size of the input queue in the for loop is %d", input_q.size());
+        */
 	}
 
     //tmlog(TM_LOG_NOTE, "addPkt: size of input q", "The size of the input queue is %d", input_q.size());
diff --git a/src/Index.hh b/src/Index.hh
index f9aafa1..1befe08 100644
--- a/src/Index.hh
+++ b/src/Index.hh
@@ -285,11 +285,12 @@ protected:
 
 	int qlen;
 
+    /*
     //uint64_t primes[35]; // = {1, 2, 3, 7, 13, 29, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289, 24593, 49157, \
                              98317, 196613, 393241, 786433, 1572869, 3145739, 6291469, \
                              12582917, 25165843, 50331653, 100663319, 201326611, 402653189, \
                              805306457, 1610612741, 3221225479, 6442450967, 12884901947};
-
+    */
 };
 
 


From neto at caltech.edu  Fri Aug 22 08:19:27 2014
From: neto at caltech.edu (Naoki Eto)
Date: Fri, 22 Aug 2014 08:19:27 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Commented out some printf
	debugging statements, and updated version number. (01d751e)
Message-ID: <201408221519.s7MFJRZu027493@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit 01d751e997654929117c9eabfe4851492512f16e
Author: NaokiEto <neto at caltech.edu>
Date:   Fri Aug 22 11:07:34 2014 -0400

    Commented out some printf debugging statements, and updated version number.


>---------------------------------------------------------------

01d751e997654929117c9eabfe4851492512f16e
 VERSION            |  2 +-
 src/CMakeLists.txt |  4 +++-
 src/DiskIndex.cc   |  4 ++--
 src/IndexField.cc  | 62 +++++++++++++++++++++++++++---------------------------
 src/cmd_parser.yy  |  4 ++--
 5 files changed, 39 insertions(+), 37 deletions(-)

diff --git a/VERSION b/VERSION
index 653402d..736779c 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.1-4
+0.1-5
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 2e9054b..107c842 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,3 +1,4 @@
+enable_language(C ASM)
 bison_target(CMDParser cmd_parser.yy ${CMAKE_CURRENT_BINARY_DIR}/cmd_parser.cc
              HEADER ${CMAKE_CURRENT_BINARY_DIR}/cmd_parser.h
              VERBOSE ${CMAKE_CURRENT_BINARY_DIR}/cmd_parser.output
@@ -102,7 +103,8 @@ if (USE_BROCCOLI)
 endif()
 
 add_executable(timemachine ${timemachine_SRCS})
-target_link_libraries(timemachine ${CMAKE_THREAD_LIBS_INIT} ${PCAP_LIBRARY} ${Readline_LIBRARY} ${OPTLIBS} ${EXTRA_LIBS} re2Static pcapnavStatic)
+#set_target_properties(timemachine PROPERTIES COMPILE_DEFINITIONS "-save-temps")
+target_link_libraries(timemachine ${CMAKE_THREAD_LIBS_INIT} ${PCAP_LIBRARY} ${Readline_LIBRARY} ${OPTLIBS} ${EXTRA_LIBS} re2Static pcapnavStatic) # "-save-temps")
 
 if (NOT TimeMachine_SKIP_INSTALL)
     install(TARGETS timemachine DESTINATION bin)
diff --git a/src/DiskIndex.cc b/src/DiskIndex.cc
index 7a3cbe0..a90fdba 100644
--- a/src/DiskIndex.cc
+++ b/src/DiskIndex.cc
@@ -64,11 +64,11 @@ inline IndexFileReader::IndexFileReader(char *fn) : fp(NULL), fname(fn), eof(fal
 	if (fp == NULL) {
 		//TODO: Decent error handling
 		tmlog(TM_LOG_ERROR, "IFR", "Could not open index file \"%s\" for reading.\n", fname);
-                char path[70];
+                //char path[70];
 
                 //char errbufnav[PCAP_ERRBUF_SIZE];
 
-                printf("The directory for Fifo that we are in is %s\n", getcwd(path, 70));
+                //printf("The directory for Fifo that we are in is %s\n", getcwd(path, 70));
 
 	}
 	my_fread(&first, sizeof(tm_time_t), 1, fp);
diff --git a/src/IndexField.cc b/src/IndexField.cc
index 67fef91..9cb597a 100644
--- a/src/IndexField.cc
+++ b/src/IndexField.cc
@@ -657,7 +657,7 @@ IndexField* ConnectionIF4::parseQuery(const char *query) {
         struct sockaddr_in6 dst_ip6;
         //char str[INET6_ADDRSTRLEN];
 
-        printf("entering FullMatch for conn3, ipv6\n");
+        //printf("entering FullMatch for conn3, ipv6\n");
 
         if (protostr == std::string("tcp"))
             proto = IPPROTO_TCP;
@@ -665,9 +665,9 @@ IndexField* ConnectionIF4::parseQuery(const char *query) {
             proto = IPPROTO_UDP;
 
 
-        printf("the query in parsequery is %s\n", query);
+        //printf("the query in parsequery is %s\n", query);
 
-        printf("for IPv6, the protocol is <%s> and source ip is <%s> and the dest ip is <%s>i and the source port is <%u> and the destination port is <%u>\n", protostr.c_str(), src_ip.c_str(), dst_ip.c_str(), src_port, dst_port);
+        //printf("for IPv6, the protocol is <%s> and source ip is <%s> and the dest ip is <%s>i and the source port is <%u> and the destination port is <%u>\n", protostr.c_str(), src_ip.c_str(), dst_ip.c_str(), src_port, dst_port);
 
         //intf("for IPv6, The source ip is <%s> and the dest ip is <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s>\n", src_ip.c_str(), 
         // store this IP address in sa:
@@ -730,7 +730,7 @@ void ConnectionIF4::getBPFStr(char *str, int max_str_len) const {
 	    uint32_t s_port;
 	    uint32_t d_port;
 
-        printf("the query is for %s\n", c_id.getStr().c_str());
+        //printf("the query is for %s\n", c_id.getStr().c_str());
 
 	    /*
 	    if (c_id.get_is_canonified()) {
@@ -758,10 +758,10 @@ void ConnectionIF4::getBPFStr(char *str, int max_str_len) const {
 
     else
     {
-        printf("the query is for %s\n", c_id.getStr().c_str());
+        //printf("the query is for %s\n", c_id.getStr().c_str());
 
-        printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
-        printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
+        //printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
+        //printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
 
         //ip_to_str(c_id.get_ip1_addr(), s_ip6_str, 16); //sizeof(s_ip6_str));
         //ip_to_str(c_id.get_ip2_addr(), d_ip6_str, 16); //sizeof(d_ip6_str));
@@ -775,7 +775,7 @@ void ConnectionIF4::getBPFStr(char *str, int max_str_len) const {
 
         //tmlog(TM_LOG_ERROR, "ConnectionIF2::getBPFStr()", "we are in connectionIF2::getBPFStr()");
 
-        printf("host %s and host %s\n", s_ip6_str, d_ip6_str);
+        //printf("host %s and host %s\n", s_ip6_str, d_ip6_str);
 
         /*
         char s_ip6_str[TM_IP_STR_SIZE];
@@ -841,7 +841,7 @@ IndexField* ConnectionIF3::parseQuery(const char *query) {
 	unsigned port;
 	proto_t proto;
 
-    printf("Entering parQuery method\n");
+    //printf("Entering parQuery method\n");
 	
 	if (!RE2::FullMatch(query, re) && !RE2::FullMatch(query, re6))
     {
@@ -877,7 +877,7 @@ IndexField* ConnectionIF3::parseQuery(const char *query) {
         struct sockaddr_in6 dst_ip6;
         //char str[INET6_ADDRSTRLEN];
 
-        printf("entering FullMatch for conn3, ipv6\n");
+        //printf("entering FullMatch for conn3, ipv6\n");
 
         if (protostr == std::string("tcp"))
             proto = IPPROTO_TCP;
@@ -885,9 +885,9 @@ IndexField* ConnectionIF3::parseQuery(const char *query) {
             proto = IPPROTO_UDP;
 
 
-        printf("the query in parsequery is %s\n", query);
+        //printf("the query in parsequery is %s\n", query);
 
-        printf("for IPv6, the protocol is <%s> and source ip is <%s> and the dest ip is <%s>i and the port is <%u>\n", protostr.c_str(), src_ip.c_str(), dst_ip.c_str(), port);
+        //printf("for IPv6, the protocol is <%s> and source ip is <%s> and the dest ip is <%s>i and the port is <%u>\n", protostr.c_str(), src_ip.c_str(), dst_ip.c_str(), port);
 
         //intf("for IPv6, The source ip is <%s> and the dest ip is <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s>\n", src_ip.c_str(), 
         // store this IP address in sa:
@@ -898,7 +898,7 @@ IndexField* ConnectionIF3::parseQuery(const char *query) {
     }
     else
     {
-        printf("wut we are not suppose to be here!\n");
+        //printf("wut we are not suppose to be here!\n");
         //unsigned char src_ip6[16];
         //unsigned char dst_ip6[16];
 
@@ -943,7 +943,7 @@ void ConnectionIF3::getBPFStr(char *str, int max_str_len) const {
 	    char ip1_str[TM_IP_STR_SIZE];
 	    char ip2_str[TM_IP_STR_SIZE];
 
-        printf("the query is for %s\n", c_id.getStr().c_str());
+        //printf("the query is for %s\n", c_id.getStr().c_str());
 
 	    ip4_to_str(c_id.get_ip1(), ip1_str, sizeof(ip1_str));
 	    ip4_to_str(c_id.get_ip2(), ip2_str, sizeof(ip2_str));
@@ -959,10 +959,10 @@ void ConnectionIF3::getBPFStr(char *str, int max_str_len) const {
 
     else
     {
-        printf("the query is for %s\n", c_id.getStr().c_str());
+        //printf("the query is for %s\n", c_id.getStr().c_str());
 
-        printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
-        printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
+        //printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
+        //printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
 
         //ip_to_str(c_id.get_ip1_addr(), s_ip6_str, 16); //sizeof(s_ip6_str));
         //ip_to_str(c_id.get_ip2_addr(), d_ip6_str, 16); //sizeof(d_ip6_str));
@@ -976,7 +976,7 @@ void ConnectionIF3::getBPFStr(char *str, int max_str_len) const {
 
         //tmlog(TM_LOG_ERROR, "ConnectionIF2::getBPFStr()", "we are in connectionIF2::getBPFStr()");
 
-        printf("host %s and host %s\n", s_ip6_str, d_ip6_str);
+        //printf("host %s and host %s\n", s_ip6_str, d_ip6_str);
 
         /*
         char ip61_str[TM_IP_STR_SIZE];
@@ -1031,14 +1031,14 @@ IndexField* ConnectionIF2::parseQuery(const char *query) {
 		return NULL;
     }
 	
-	fprintf(stderr, "%s\nConnectionIF22:parseQuery:  %s ===> <%s> <%s>\n", 
-				pattern6_connection2.c_str(), query, src_ip.c_str(), dst_ip.c_str());
+	//fprintf(stderr, "%s\nConnectionIF22:parseQuery:  %s ===> <%s> <%s>\n", 
+	//			pattern6_connection2.c_str(), query, src_ip.c_str(), dst_ip.c_str());
 	
 
     
     if (RE2::FullMatch(query, re, &src_ip, &dst_ip))
     {
-        printf("for IPv6, The source ip is %s and the dest ip is %s\n", src_ip.c_str(), dst_ip.c_str());
+        //printf("for IPv6, The source ip is %s and the dest ip is %s\n", src_ip.c_str(), dst_ip.c_str());
 
     	return new ConnectionIF2(inet_addr(src_ip.c_str()), inet_addr(dst_ip.c_str()));
     }
@@ -1048,9 +1048,9 @@ IndexField* ConnectionIF2::parseQuery(const char *query) {
         struct sockaddr_in6 dst_ip6;
         //char str[INET6_ADDRSTRLEN];
 
-        printf("the query in parsequery is %s\n", query);
+        //printf("the query in parsequery is %s\n", query);
 
-        printf("for IPv6, the source ip is <%s> and the dest ip is <%s>\n", src_ip.c_str(), dst_ip.c_str());
+        //printf("for IPv6, the source ip is <%s> and the dest ip is <%s>\n", src_ip.c_str(), dst_ip.c_str());
 
         //intf("for IPv6, The source ip is <%s> and the dest ip is <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s> and <%s>\n", src_ip.c_str(), 
         // store this IP address in sa:
@@ -1101,10 +1101,10 @@ void ConnectionIF2::getBPFStr(char *str, int max_str_len) const {
         char s_ip_str[TM_IP_STR_SIZE];
         char d_ip_str[TM_IP_STR_SIZE];
 
-        printf("the query is for %s\n", c_id.getStr().c_str());
+        //printf("the query is for %s\n", c_id.getStr().c_str());
 
-        printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
-        printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
+        //printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
+        //printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
 
         ip4_to_str(c_id.get_ip1(), s_ip_str, 16);
         ip4_to_str(c_id.get_ip2(), d_ip_str, 16);
@@ -1112,7 +1112,7 @@ void ConnectionIF2::getBPFStr(char *str, int max_str_len) const {
 
         //tmlog(TM_LOG_ERROR, "ConnectionIF2::getBPFStr()", "we are in connectionIF2::getBPFStr()");
 
-        printf("ip4: host %s and host %s\n", s_ip_str, d_ip_str);
+        //printf("ip4: host %s and host %s\n", s_ip_str, d_ip_str);
 
         snprintf(str, max_str_len,
                  "host %s and host %s",
@@ -1124,10 +1124,10 @@ void ConnectionIF2::getBPFStr(char *str, int max_str_len) const {
 	    //char s_ip6_str[TM_IP_STR_SIZE];
 	    //char d_ip6_str[TM_IP_STR_SIZE];
 
-        printf("the query is for %s\n", c_id.getStr().c_str());
+        //printf("the query is for %s\n", c_id.getStr().c_str());
 
-        printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
-        printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
+        //printf("The size of the first ip address is %lu\n", sizeof(c_id.get_ip1()));
+        //printf("The size of the second ip address is %lu\n", sizeof(c_id.get_ip2()));
 
 	    //ip_to_str(c_id.get_ip1_addr(), s_ip6_str, 16); //sizeof(s_ip6_str));
 	    //ip_to_str(c_id.get_ip2_addr(), d_ip6_str, 16); //sizeof(d_ip6_str));
@@ -1141,7 +1141,7 @@ void ConnectionIF2::getBPFStr(char *str, int max_str_len) const {
 
         //tmlog(TM_LOG_ERROR, "ConnectionIF2::getBPFStr()", "we are in connectionIF2::getBPFStr()");
 
-        printf("host %s and host %s\n", s_ip6_str, d_ip6_str);
+        //printf("host %s and host %s\n", s_ip6_str, d_ip6_str);
 
 	    snprintf(str, max_str_len,
 		    	 "host %s and host %s",
diff --git a/src/cmd_parser.yy b/src/cmd_parser.yy
index bc11ead..26e30fe 100644
--- a/src/cmd_parser.yy
+++ b/src/cmd_parser.yy
@@ -348,11 +348,11 @@ queryspec_flag:
 
 queryspec_key:
 	TOK_INDEX TOK_ID TOK_STRING {
-		fprintf(stderr, "INDEX QUERY: <%s>, string <%s>\n", $2, $3);
+		//fprintf(stderr, "INDEX QUERY: <%s>, string <%s>\n", $2, $3);
 		IndexType *idx=cmd_parser_storage->getIndexes()->getIndexByName($2);
 		if (idx) {
 			IndexField *ifp;
-			fprintf(stderr, "found the index\n");
+			//fprintf(stderr, "found the index\n");
 			ifp = idx->parseQuery($3);
 			//DEBUG fprintf(stderr, ">>>>> %s\n", ifp->getStr().c_str());
 			if (ifp==NULL) 


From neto at caltech.edu  Fri Aug 22 13:42:14 2014
From: neto at caltech.edu (Naoki Eto)
Date: Fri, 22 Aug 2014 13:42:14 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Implemented Jim's Precedence. It
	improves CPU usage slightly. Basically,
	the buckets are sorted by precedence after the config file is parsed.
	Then,
	the packets go through the buckets and break when it meets a match.
	(418a79a)
Message-ID: <201408222042.s7MKgELQ012056@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit 418a79a73c14d459222625f566a36a9c65b995fd
Author: NaokiEto <neto at caltech.edu>
Date:   Fri Aug 22 16:29:48 2014 -0400

    Implemented Jim's Precedence. It improves CPU usage slightly. Basically,
    the buckets are sorted by precedence after the config file is parsed. Then,
    the packets go through the buckets and break when it meets a match.


>---------------------------------------------------------------

418a79a73c14d459222625f566a36a9c65b995fd
 src/Storage.cc | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/Storage.cc b/src/Storage.cc
index 5d5caec..d734e68 100644
--- a/src/Storage.cc
+++ b/src/Storage.cc
@@ -76,6 +76,11 @@ void *capture_thread(void *arg) {
 	return NULL;
 }
 
+bool compare_precedence(Fifo* first, Fifo* second)
+{
+    return (first->getPrecedence() > second->getPrecedence());
+}
+
 // Abstracts the configuration for a Storage instance,
 // initializing some of member variables
 StorageConfig::StorageConfig() : 
@@ -185,6 +190,7 @@ Storage::Storage(StorageConfig& conf):
     // clear the list of Fifo* from the Storage configuration
 	conf.fifos.clear();
 
+    fifos.sort(compare_precedence);
 
     // go through the list of Fifo* for Storage
 	for (std::list<Fifo*>::iterator i=fifos.begin(); i!=fifos.end(); i++)
@@ -500,6 +506,7 @@ void Storage::addPkt(const struct pcap_pkthdr *header,
 		 * Now evaluate BPF expressions defined for all classes and pick
 		 * the appropriate class
 		*/
+        /*
 		int max_precedence=INT_MAX;
         // go through all the possible classes
 		for (std::list<Fifo*>::iterator i=fifos.begin(); i!=fifos.end(); i++) {
@@ -518,6 +525,18 @@ void Storage::addPkt(const struct pcap_pkthdr *header,
 				max_precedence=f->getPrecedence();
 			}
 		}
+        */
+
+        std::list<Fifo*>::iterator i = fifos.begin();
+        while (i != fifos.end())
+        {
+            if ((*i)->matchPkt(header, packet))
+            {
+                f = *i;
+                break;
+            }
+            i++;
+        }
         // if class is assigned
 		if (f)
             // set cache to which class this connection belongs


From neto at caltech.edu  Fri Aug 22 15:18:37 2014
From: neto at caltech.edu (Naoki Eto)
Date: Fri, 22 Aug 2014 15:18:37 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Implemented Partha's indexdir
	and queryfiledir changes. Basically,
	if you forget to create the index and query directories,
	but they are on the config file, they will be created automatically,
	with a message that lets you know. (a6f204b)
Message-ID: <201408222218.s7MMIbFZ020325@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit a6f204b03ff49cd2c4daf1782f27e144ed07b360
Author: NaokiEto <neto at caltech.edu>
Date:   Fri Aug 22 18:05:43 2014 -0400

    Implemented Partha's indexdir and queryfiledir changes. Basically,
    if you forget to create the index and query directories, but they
    are on the config file, they will be created automatically, with a
    message that lets you know.


>---------------------------------------------------------------

a6f204b03ff49cd2c4daf1782f27e144ed07b360
 src/Index.cc | 9 +++++++++
 src/Query.cc | 9 +++++++++
 2 files changed, 18 insertions(+)

diff --git a/src/Index.cc b/src/Index.cc
index 12efdd9..aeb5717 100644
--- a/src/Index.cc
+++ b/src/Index.cc
@@ -11,6 +11,7 @@
 
 #include <fstream>
 #include <queue>
+#include <string>
 //#include <gperftools/profiler.h> 
 
 #include "tm.h"
@@ -89,6 +90,14 @@ Index<T>::Index(tm_time_t d_t, int hash_size_index, bool do_disk_index, Storage
             //return(1);
         }
 
+        struct stat st;
+
+        if (stat(conf_main_indexdir, &st) != 0)
+        {
+            printf("The index directory %s did not exist. Creating the directory ...\n", conf_main_indexdir);
+            mkdir(conf_main_indexdir, 0755);
+        } 
+        
 	    if (do_disk_index)
 		    disk_index = new IndexFiles<T>((std::string)conf_main_indexdir, "index_"+T::getIndexNameStatic());
 	    else
diff --git a/src/Query.cc b/src/Query.cc
index 60d41b4..6298a8e 100644
--- a/src/Query.cc
+++ b/src/Query.cc
@@ -152,6 +152,15 @@ QueryResultFile::QueryResultFile(int queryID, const std::string& filename, int l
            //return(1);
         }
 
+        struct stat st;
+
+        if (stat(conf_main_queryfiledir, &st) != 0)
+        {
+            printf("The index directory %s did not exist. Creating the directory ...\n", conf_main_queryfiledir);
+            mkdir(conf_main_queryfiledir, 0755);
+        }
+
+
 	ph = pcap_open_dead(linktype, snaplen);
 	f = new FifoDiskFile(conf_main_queryfiledir+std::string("/")+filename, ph);
 }


From robin at icir.org  Fri Aug 22 15:27:14 2014
From: robin at icir.org (Robin Sommer)
Date: Fri, 22 Aug 2014 15:27:14 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Implemented
	Partha's	indexdir and queryfiledir changes. Basically,
	if you forget to	create the index and query directories,
	but they are on the config	file, they will be created automatically,
	with a message that lets	you know. (a6f204b)
In-Reply-To: <201408222218.s7MMIbFZ020325@bro-ids.icir.org>
References: <201408222218.s7MMIbFZ020325@bro-ids.icir.org>
Message-ID: <20140822222714.GK32752@icir.org>



On Fri, Aug 22, 2014 at 15:18 -0700, Naoki Eto wrote:

>     Implemented Partha's indexdir and queryfiledir changes. Basically,
>     if you forget to create the index and query directories, but they
>     are on the config file, they will be created automatically, with a
>     message that lets you know.

I'm wondering if this should have a check if they are indeed relative
to the work dir, as otherwise one might accidentally create
directories at unexpected places (the actual chance for problems is
probably small, but still).

Robin

-- 
Robin Sommer * Phone +1 (510) 722-6541 *     robin at icir.org
ICSI/LBNL    * Fax   +1 (510) 666-2956 * www.icir.org/robin

From neto at caltech.edu  Fri Aug 22 15:49:14 2014
From: neto at caltech.edu (Naoki Eto)
Date: Fri, 22 Aug 2014 15:49:14 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Created Aashish's -v command
	line parameter which outputs the version number of Time
	Machine, and also updated the version number to 2-0 (ed402cd)
Message-ID: <201408222249.s7MMnEJx022243@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit ed402cd9300bb2852f85e85b84134bcbbb915c1a
Author: NaokiEto <neto at caltech.edu>
Date:   Fri Aug 22 18:36:32 2014 -0400

    Created Aashish's -v command line parameter which outputs the version
    number of Time Machine, and also updated the version number to 2-0


>---------------------------------------------------------------

ed402cd9300bb2852f85e85b84134bcbbb915c1a
 VERSION        |  2 +-
 src/main.cc.in | 23 +++++++++++++++++++----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/VERSION b/VERSION
index 736779c..d8801fb 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.1-5
+0.2-0
diff --git a/src/main.cc.in b/src/main.cc.in
index 67d4b88..3c67efa 100644
--- a/src/main.cc.in
+++ b/src/main.cc.in
@@ -811,6 +811,12 @@ void usage() {
 	exit(1);
 }
 
+void print_version()
+{
+    printf("TimeMachine version %s\n", VERSION);
+    exit(1); 
+}
+
 
 
 /***************************************************************************
@@ -836,7 +842,13 @@ main(int argc, char** argv) {
 	 */
 
 	int opt;
-	while ((opt=getopt(argc, argv, "i:r:f:c:h")) != -1) {
+    /*
+    if (getopt(argc, argv, "v") != -1)
+    {
+        print_version();
+    }
+    */
+	while ((opt=getopt(argc, argv, "i:r:f:c:v:h")) != -1) {
         //std::cout << "the command line arg you put in is: " << opt << std::endl;
         //std::cout << "the second command line arg you put in is: " << argv[1] << std::endl;
 		switch(opt) {
@@ -852,6 +864,9 @@ main(int argc, char** argv) {
 		case 'c':
 			conffile=strdup(optarg);
 			break;
+        case 'v':
+            print_version();
+            break;
 		case 'h':
 		default:
 			usage();
@@ -942,7 +957,7 @@ main(int argc, char** argv) {
 		delete log_file;
 		exit(1);
 	}
-	tmlog(TM_LOG_NOTE, "main: main.cc.in, ~line 714", "capture started, capture thread come on");
+	//tmlog(TM_LOG_NOTE, "main: main.cc.in, ~line 714", "capture started, capture thread come on");
 
 	//tmlog(TM_LOG_NOTE, "main", "capture started, capture thread WORK WORK WORK WORK WORK WORK WORK WORK WORK WORK");
 
@@ -956,7 +971,7 @@ main(int argc, char** argv) {
 	i = pthread_create(&index_aggregation_thread_tid, NULL, index_aggregation_thread, NULL);
 
     // NOTE NOTE NOTE
-    tmlog(TM_LOG_NOTE, "index aggregation thread: main.cc.in, ~line 728", "creating index aggregation thread");
+    //tmlog(TM_LOG_NOTE, "index aggregation thread: main.cc.in, ~line 728", "creating index aggregation thread");
 	if (i) {
 		tmlog(TM_LOG_ERROR, "main", "Could not start index aggregation thread.\n");
 		exit(1);
@@ -1016,7 +1031,7 @@ main(int argc, char** argv) {
 						 statisticslog_thread, &conf_main_log_interval);
 
         // NOTE NOTE NOTE
-        tmlog(TM_LOG_NOTE, "statistics log thread: main.cc.in, ~line 788", "creating statistics log thread");
+       // tmlog(TM_LOG_NOTE, "statistics log thread: main.cc.in, ~line 788", "creating statistics log thread");
 
 		if (i) {
 			tmlog(TM_LOG_ERROR, "main", "Could not start statistics log thread.\n");


From neto at caltech.edu  Fri Aug 22 15:45:48 2014
From: neto at caltech.edu (neto at caltech.edu)
Date: Fri, 22 Aug 2014 15:45:48 -0700 (PDT)
Subject: [TM] [git/tm] topic/naokieto/ipv6: Implemented Partha's
 indexdir and queryfiledir changes. Basically,
 if you forget to create the index and query directories,
 but they are on the config file, they will be created automatically,
 with a message that lets you know. (a6f204b)
In-Reply-To: <20140822222714.GK32752@icir.org>
References: <201408222218.s7MMIbFZ020325@bro-ids.icir.org>   
	<20140822222714.GK32752@icir.org>
Message-ID: <40859.128.3.10.211.1408747548.squirrel@webmail.caltech.edu>

Hi Robin,

Do you mean like if the index and query file directories paths are
absolute rather than relative to workdir? If not, could you give an
example/clarify?

Best,

Naoki

>
>
> On Fri, Aug 22, 2014 at 15:18 -0700, Naoki Eto wrote:
>
>>     Implemented Partha's indexdir and queryfiledir changes. Basically,
if you forget to create the index and query directories, but they
are on the config file, they will be created automatically, with a
message that lets you know.
>
> I'm wondering if this should have a check if they are indeed relative to
the work dir, as otherwise one might accidentally create
> directories at unexpected places (the actual chance for problems is
probably small, but still).
>
> Robin
>
> --
> Robin Sommer * Phone +1 (510) 722-6541 *     robin at icir.org
> ICSI/LBNL    * Fax   +1 (510) 666-2956 * www.icir.org/robin
>





From robin at icir.org  Fri Aug 22 16:52:00 2014
From: robin at icir.org (Robin Sommer)
Date: Fri, 22 Aug 2014 16:52:00 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Implemented
	Partha's	indexdir and queryfiledir changes. Basically,
	if you forget to	create the index and query directories,
	but they are on the config	file, they will be created automatically,
	with a message that lets	you know. (a6f204b)
In-Reply-To: <40859.128.3.10.211.1408747548.squirrel@webmail.caltech.edu>
References: <201408222218.s7MMIbFZ020325@bro-ids.icir.org>
	<20140822222714.GK32752@icir.org>
	<40859.128.3.10.211.1408747548.squirrel@webmail.caltech.edu>
Message-ID: <20140822235200.GA91071@icir.org>



On Fri, Aug 22, 2014 at 15:45 -0700, you wrote:

> Do you mean like if the index and query file directories paths are
> absolute rather than relative to workdir?

Yes, that's what I meant.

Robin

-- 
Robin Sommer * Phone +1 (510) 722-6541 *     robin at icir.org
ICSI/LBNL    * Fax   +1 (510) 666-2956 * www.icir.org/robin

From vallentin at icir.org  Fri Aug 22 21:32:48 2014
From: vallentin at icir.org (Matthias Vallentin)
Date: Fri, 22 Aug 2014 21:32:48 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Created Aashish's -v command
 line parameter which outputs the version number of Time Machine,
 and also updated the version number to 2-0 (ed402cd)
In-Reply-To: <201408222249.s7MMnEJx022243@bro-ids.icir.org>
References: <201408222249.s7MMnEJx022243@bro-ids.icir.org>
Message-ID: <20140823043248.GH4237@icir.org>

>     Created Aashish's -v command line parameter which outputs the version
>     number of Time Machine, and also updated the version number to 2-0

What was the rationale for bumping the major version number? Did you
introduce breaking changes with 2.0? Or the sheer number of new features
and code changes?

     Matthias

From vallentin at icir.org  Fri Aug 22 22:07:55 2014
From: vallentin at icir.org (Matthias Vallentin)
Date: Fri, 22 Aug 2014 22:07:55 -0700
Subject: [TM] Fwd: Re: [git/tm] topic/naokieto/ipv6: Created Aashish's -v
 command line parameter which outputs the version number of Time Machine,
 and also updated the version number to 2-0 (ed402cd)
Message-ID: <20140823050755.GJ4237@icir.org>

(Including the rest of the team in the discussion.)

So then I'm relaying this question to you, Aashish :-).

     Matthias

----- Forwarded message from neto at caltech.edu -----

Date: Fri, 22 Aug 2014 21:44:15 -0700 (PDT)
From: neto at caltech.edu
To: Matthias Vallentin <vallentin at icir.org>
Subject: Re: [TM] [git/tm] topic/naokieto/ipv6: Created Aashish's -v      command line parameter which outputs the version number of Time      Machine, and also updated the version number to 2-0 (ed402cd)
User-Agent: SquirrelMail/1.4.8-5.el5_7.13

Hi Matthias,

I originally changed it to only 1-5, but my mentor Aashish recommended
that I change to 2-0.

Best,
Naoki

>>     Created Aashish's -v command line parameter which outputs the
>> version
>>     number of Time Machine, and also updated the version number to 2-0
>
> What was the rationale for bumping the major version number? Did you
> introduce breaking changes with 2.0? Or the sheer number of new features
> and code changes?
>
>      Matthias
>



----- End forwarded message -----

From asharma at lbl.gov  Fri Aug 22 23:41:14 2014
From: asharma at lbl.gov (Aashish Sharma)
Date: Fri, 22 Aug 2014 23:41:14 -0700
Subject: [TM] Fwd: Re: [git/tm] topic/naokieto/ipv6: Created Aashish's
 -v command line parameter which outputs the version number of Time Machine,
 and also updated the version number to 2-0 (ed402cd)
In-Reply-To: <20140823050755.GJ4237@icir.org>
References: <20140823050755.GJ4237@icir.org>
Message-ID: <20140823064012.GA10075@yaksha.lbl.gov>

I  think just the sheer number of changes in the code makes me think to bump to 2 instead of incremental 1.5

However, Its not hard and fast. If we choose to accept code eventually, we can still put version 1.5 - no problem. Important was that timemachine has functionality to spit out a version with -V. 

Aashish 

On Fri, Aug 22, 2014 at 10:07:55PM -0700, Matthias Vallentin wrote:
> (Including the rest of the team in the discussion.)
> 
> So then I'm relaying this question to you, Aashish :-).
> 
>      Matthias
> 
> ----- Forwarded message from neto at caltech.edu -----
> 
> Date: Fri, 22 Aug 2014 21:44:15 -0700 (PDT)
> From: neto at caltech.edu
> To: Matthias Vallentin <vallentin at icir.org>
> Subject: Re: [TM] [git/tm] topic/naokieto/ipv6: Created Aashish's -v      command line parameter which outputs the version number of Time      Machine, and also updated the version number to 2-0 (ed402cd)
> User-Agent: SquirrelMail/1.4.8-5.el5_7.13
> 
> Hi Matthias,
> 
> I originally changed it to only 1-5, but my mentor Aashish recommended
> that I change to 2-0.
> 
> Best,
> Naoki
> 
> >>     Created Aashish's -v command line parameter which outputs the
> >> version
> >>     number of Time Machine, and also updated the version number to 2-0
> >
> > What was the rationale for bumping the major version number? Did you
> > introduce breaking changes with 2.0? Or the sheer number of new features
> > and code changes?
> >
> >      Matthias
> >
> 
> 
> 
> ----- End forwarded message -----

-- 
Aashish Sharma	(asharma at lbl.gov) 				 
Cyber Security, 
Lawrence Berkeley National Laboratory  
http://go.lbl.gov/pgp-aashish 
Office: (510)-495-2680  Cell: (510)-612-7971

From vallentin at icir.org  Sun Aug 24 08:55:04 2014
From: vallentin at icir.org (Matthias Vallentin)
Date: Sun, 24 Aug 2014 08:55:04 -0700
Subject: [TM] Fwd: Re: [git/tm] topic/naokieto/ipv6: Created Aashish's
 -v command line parameter which outputs the version number of Time Machine,
 and also updated the version number to 2-0 (ed402cd)
In-Reply-To: <20140823064012.GA10075@yaksha.lbl.gov>
References: <20140823050755.GJ4237@icir.org>
	<20140823064012.GA10075@yaksha.lbl.gov>
Message-ID: <20140824155504.GL4237@icir.org>

> I  think just the sheer number of changes in the code makes me think
> to bump to 2 instead of incremental 1.5

Ah, I see. Often software changes the major version with breaking API
changes [1], so I was just curious what those breaking changes were.

     Matthias

[1] http://semver.org

From asharma at lbl.gov  Sun Aug 24 09:34:03 2014
From: asharma at lbl.gov (Aashish Sharma)
Date: Sun, 24 Aug 2014 09:34:03 -0700
Subject: [TM] Fwd: Re: [git/tm] topic/naokieto/ipv6: Created Aashish's
 -v command line parameter which outputs the version number of Time Machine,
 and also updated the version number to 2-0 (ed402cd)
In-Reply-To: <20140824155504.GL4237@icir.org>
References: <20140823050755.GJ4237@icir.org>
	<20140823064012.GA10075@yaksha.lbl.gov>
	<20140824155504.GL4237@icir.org>
Message-ID: <20140824163154.GA22809@yaksha.lbl.gov>

Matthias: 

I see!! I Don't think any of the changes/features added to TM qualify as breaking changes. 

Major Changes are pretty much internal - most drastic being various data structues need to accomodate size of IPv6 instead of IPv4. A lot of code optimizations and some new features: IPv6 support in class, indexes, queries; introducing classdir etc. 

Aashish 


On Sun, Aug 24, 2014 at 08:55:04AM -0700, Matthias Vallentin wrote:
> > I  think just the sheer number of changes in the code makes me think
> > to bump to 2 instead of incremental 1.5
> 
> Ah, I see. Often software changes the major version with breaking API
> changes [1], so I was just curious what those breaking changes were.
> 
>      Matthias
> 
> [1] http://semver.org

-- 
Aashish Sharma	(asharma at lbl.gov) 				 
Cyber Security, 
Lawrence Berkeley National Laboratory  
http://go.lbl.gov/pgp-aashish 
Office: (510)-495-2680  Cell: (510)-612-7971
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
Url : http://mailman.ICSI.Berkeley.EDU/pipermail/time-machine/attachments/20140824/2178c8e3/attachment.bin 

From asharma at lbl.gov  Sun Aug 24 09:49:00 2014
From: asharma at lbl.gov (Aashish Sharma)
Date: Sun, 24 Aug 2014 09:49:00 -0700
Subject: [TM] Fwd: Re: [git/tm] topic/naokieto/ipv6: Created Aashish's
 -v command line parameter which outputs the version number of Time Machine,
 and also updated the version number to 2-0 (ed402cd)
In-Reply-To: <20140824155504.GL4237@icir.org>
References: <20140823050755.GJ4237@icir.org>
	<20140823064012.GA10075@yaksha.lbl.gov>
	<20140824155504.GL4237@icir.org>
Message-ID: <20140824164854.GB22809@yaksha.lbl.gov>

As I mentioned, This commit is more useful so that we can have timemachine spit out version with -V. 

I am not really invested in what's going to be the number (as long as something prints out, its fine :))

Aashish 


On Sun, Aug 24, 2014 at 08:55:04AM -0700, Matthias Vallentin wrote:
> > I  think just the sheer number of changes in the code makes me think
> > to bump to 2 instead of incremental 1.5
> 
> Ah, I see. Often software changes the major version with breaking API
> changes [1], so I was just curious what those breaking changes were.
> 
>      Matthias
> 
> [1] http://semver.org

-- 
Aashish Sharma	(asharma at lbl.gov) 				 
Cyber Security, 
Lawrence Berkeley National Laboratory  
http://go.lbl.gov/pgp-aashish 
Office: (510)-495-2680  Cell: (510)-612-7971
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
Url : http://mailman.ICSI.Berkeley.EDU/pipermail/time-machine/attachments/20140824/faf10537/attachment.bin 

From vallentin at icir.org  Sun Aug 24 09:51:20 2014
From: vallentin at icir.org (Matthias Vallentin)
Date: Sun, 24 Aug 2014 09:51:20 -0700
Subject: [TM] Fwd: Re: [git/tm] topic/naokieto/ipv6: Created Aashish's
 -v command line parameter which outputs the version number of Time Machine,
 and also updated the version number to 2-0 (ed402cd)
In-Reply-To: <20140824163154.GA22809@yaksha.lbl.gov>
References: <20140823050755.GJ4237@icir.org>
	<20140823064012.GA10075@yaksha.lbl.gov>
	<20140824155504.GL4237@icir.org>
	<20140824163154.GA22809@yaksha.lbl.gov>
Message-ID: <20140824165120.GO4237@icir.org>

> most drastic being various data structues need to accomodate size of
> IPv6 instead of IPv4. A lot of code optimizations and some new
> features: IPv6 support in class, indexes, queries; introducing
> classdir etc. 

Good to know. I recall that a major pain point of the current TM was
that it's impossible to reuse the indexes across restarts. Is that
something you're working on as well?

     Matthias

From neto at caltech.edu  Mon Aug 25 12:21:59 2014
From: neto at caltech.edu (neto at caltech.edu)
Date: Mon, 25 Aug 2014 12:21:59 -0700 (PDT)
Subject: [TM] Fwd: Re: [git/tm] topic/naokieto/ipv6: Created Aashish's
 -v command line parameter which outputs the version number of Time Machine,
 and also updated the version number to 2-0 (ed402cd)
Message-ID: <36671.198.128.197.74.1408994519.squirrel@webmail.caltech.edu>

Hi Matthias,

I am not working on re-using the indexes across restarts currently. TM
with indexes enabled takes a noticeably much larger amount of CPU
usage/time than TM with no indexes enabled (about 40-50% larger CPU on
diag3). I don't think indexes are implemented very efficiently, and
probably should be revamped to another method that doesn't use AVL trees.

Also, the data structure of 16 char array that is being used to store IPv6
and IPv4 addresses is not very efficient for making comparisons, and since
there are many lookup calls to the hash tables, this results in large CPU
usage.

I think it would first be better to revamp the method of indexes/querying
before determining a method to reuse indexes/allow for querying on data
before restart, if necessary.

Best,

Naoki

> Good to know. I recall that a major pain point of the current TM was
> that it's impossible to reuse the indexes across restarts. Is that
> something you're working on as well?
>
>     Matthias


From neto at caltech.edu  Tue Aug 26 13:49:20 2014
From: neto at caltech.edu (neto at caltech.edu)
Date: Tue, 26 Aug 2014 13:49:20 -0700 (PDT)
Subject: [TM] [git/tm] topic/naokieto/ipv6: Implemented Partha's
 indexdir and queryfiledir changes. Basically,
 if you forget to create the index and query directories,
 but they are on the config file, they will be created automatically,
 with a message that lets you know. (a6f204b)
In-Reply-To: <20140822235200.GA91071@icir.org>
References: <201408222218.s7MMIbFZ020325@bro-ids.icir.org>      
	<20140822222714.GK32752@icir.org>      
	<40859.128.3.10.211.1408747548.squirrel@webmail.caltech.edu>      
	<20140822235200.GA91071@icir.org>
Message-ID: <36085.198.128.197.74.1409086160.squirrel@webmail.caltech.edu>

Hi Robin,

Sorry about the late response. I understand now what you might have been
referring those days ago.

I was wondering if we should keep the index, query, and profile (for
gperftools cpu profiler) directories all strictly relative. I think
determining whether the directories are relative or absolute can be
ambiguous, especially since we are creating them if they don't exist.

If we are to keep all the directories for the index, query, and profile
strictly relative to the work directory, do we still need the parameters
in the configuration file? I assume that the index, query, and profile
directories can be made 1 subdirectory under the work directory.

Also, as a heads up, I made all the class directories absolute paths that
are not relative to the work directory.

Best,

Naoki

>
>
> On Fri, Aug 22, 2014 at 15:45 -0700, you wrote:
>
>> Do you mean like if the index and query file directories paths are
absolute rather than relative to workdir?
>
> Yes, that's what I meant.
>
> Robin
>
> --
> Robin Sommer * Phone +1 (510) 722-6541 *     robin at icir.org
> ICSI/LBNL    * Fax   +1 (510) 666-2956 * www.icir.org/robin
>







From asharma at lbl.gov  Tue Aug 26 13:59:50 2014
From: asharma at lbl.gov (Aashish Sharma)
Date: Tue, 26 Aug 2014 13:59:50 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Implemented Partha's
 indexdir and queryfiledir changes. Basically,
 if you forget to create the index and query directories,
 but they are on the config file, they will be created automatically,
 with a message that lets you know. (a6f204b)
In-Reply-To: <36085.198.128.197.74.1409086160.squirrel@webmail.caltech.edu>
References: <201408222218.s7MMIbFZ020325@bro-ids.icir.org>
	<20140822222714.GK32752@icir.org>
	<40859.128.3.10.211.1408747548.squirrel@webmail.caltech.edu>
	<20140822235200.GA91071@icir.org>
	<36085.198.128.197.74.1409086160.squirrel@webmail.caltech.edu>
Message-ID: <20140826205947.GA26810@yaksha.lbl.gov>

So the idea is to check for existence of indexes and queries dir in workdir. 
    If not create those inside work dir

If we pursue above, then there is actually not much need to have (indexes, and queries ) path be specified in config files. 

Alternative option is to have indexes and queries dir be absolute path in config file and and make those independent of the workdir. 

By introducing a new feature called classdir option, we are creating class dirs in the specified paths within the class spec or otherwise defaulting class to be stored in to the workdir. 

So, similarly, do we want indexes and queries to be bound to workdir or users can store these in different places in the file system. 

Aashish 


On Tue, Aug 26, 2014 at 01:49:20PM -0700, neto at caltech.edu wrote:
> Hi Robin,
> 
> Sorry about the late response. I understand now what you might have been
> referring those days ago.
> 
> I was wondering if we should keep the index, query, and profile (for
> gperftools cpu profiler) directories all strictly relative. I think
> determining whether the directories are relative or absolute can be
> ambiguous, especially since we are creating them if they don't exist.
> 
> If we are to keep all the directories for the index, query, and profile
> strictly relative to the work directory, do we still need the parameters
> in the configuration file? I assume that the index, query, and profile
> directories can be made 1 subdirectory under the work directory.
> 
> Also, as a heads up, I made all the class directories absolute paths that
> are not relative to the work directory.
> 
> Best,
> 
> Naoki
> 
> >
> >
> > On Fri, Aug 22, 2014 at 15:45 -0700, you wrote:
> >
> >> Do you mean like if the index and query file directories paths are
> absolute rather than relative to workdir?
> >
> > Yes, that's what I meant.
> >
> > Robin
> >
> > --
> > Robin Sommer * Phone +1 (510) 722-6541 *     robin at icir.org
> > ICSI/LBNL    * Fax   +1 (510) 666-2956 * www.icir.org/robin
> >
> 
> 
> 
> 
> 
> 

-- 
Aashish Sharma	(asharma at lbl.gov) 				 
Cyber Security, 
Lawrence Berkeley National Laboratory  
http://go.lbl.gov/pgp-aashish 
Office: (510)-495-2680  Cell: (510)-612-7971
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
Url : http://mailman.ICSI.Berkeley.EDU/pipermail/time-machine/attachments/20140826/5ca83cff/attachment.bin 

From neto at caltech.edu  Tue Aug 26 19:08:48 2014
From: neto at caltech.edu (Naoki Eto)
Date: Tue, 26 Aug 2014 19:08:48 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Added a profilepath option in
	the configuration file to place the gperftools cpu profiler's
	.prof file,
	if wanted Hopefully corrected the code for Apple compilers (I don't
	have an Apple compiler). (b08480e)
Message-ID: <201408270208.s7R28maA006387@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit b08480ef974430567cab5c4bbadca4cf13247324
Author: NaokiEto <neto at caltech.edu>
Date:   Tue Aug 26 21:51:58 2014 -0400

    Added a profilepath option in the configuration file to place the gperftools cpu profiler's .prof file, if wanted
    Hopefully corrected the code for Apple compilers (I don't have an Apple compiler).


>---------------------------------------------------------------

b08480ef974430567cab5c4bbadca4cf13247324
 CHANGES             | 14 +++++++++++++
 VERSION             |  2 +-
 config.h.in         |  6 ++++++
 configure           | 27 ++++++++++++------------
 src/DynClass.cc     |  4 +++-
 src/IndexHash.cc    | 21 +++++++++++++++++++
 src/Storage.cc      | 10 +++++++--
 src/conf.h          |  1 +
 src/conf_parser.yy  |  6 +++++-
 src/conf_scanner.ll |  1 +
 src/main.cc.in      | 60 +++++++++++++++++++++++++++++++++++++++++++++++++----
 src/types.h         |  2 ++
 12 files changed, 132 insertions(+), 22 deletions(-)

diff --git a/CHANGES b/CHANGES
index 11d9011..3879688 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,19 @@
+0.1-5 | 2014-08-26 15:35:00 -0800
+
+    * Changed the hash table sizes to always be prime number, to help avoid clustering in the collisions lists. (Naoki Eto) 
+
+    * Added gperftools CPU profiler, which can be enabled (Naoki Eto)
+
+    * Changed the method for reading the configuration classes so that it is first ordered by precedence and then the highest precedence match is found (Naoki Eto)    
+
+    * Implemented querying for IPv4 and IPv6 ip, conn2, conn3, and conn4 (Naoki Eto)
+
+    * Implemented class directories that can be specified in the configuration file (Naoki Eto)
+
 0.1-4 | 2014-07-18 16:53:50 -0800
     
+    * Implemented IPv6 support for the classes. (Naoki Eto)   
+ 
     * Some querying for IPv6 addresses is enabled. (Naoki Eto)
 
     * VLAN tags are taken into account w/o MPLS labels (Naoki Eto)
diff --git a/VERSION b/VERSION
index d8801fb..736779c 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.2-0
+0.1-5
diff --git a/config.h.in b/config.h.in
index 41de9bd..4ca03b5 100644
--- a/config.h.in
+++ b/config.h.in
@@ -7,6 +7,12 @@
 /* Define if you have the `mallinfo' function. */
 #cmakedefine HAVE_MALLINFO
 
+/* Define if you have gperftools library. */
+#cmakedefine HAVE_PERFTOOLS
+
+/* Define if you have gperftools' cpu profiler */
+#cmakedefine USE_PERFTOOLS_CPU
+
 /* Define if you have the <memory.h> header file. */
 #cmakedefine HAVE_MEMORY_H
 
diff --git a/configure b/configure
index 2ccded7..1b60a02 100755
--- a/configure
+++ b/configure
@@ -17,28 +17,29 @@ usage="\
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
   Build Directory:
-    --builddir=DIR         place build files in directory [build]
+    --builddir=DIR             place build files in directory [build]
 
   Installation Directories:
-    --prefix=PREFIX        installation directory [/usr/local]
-    --conf-files-dir=DIR   config files installation directory [PREFIX/etc]
+    --prefix=PREFIX            installation directory [/usr/local]
+    --conf-files-dir=DIR       config files installation directory [PREFIX/etc]
 
   Optional Features:
-    --enable-debug         compile in debugging mode
+    --enable-debug             compile in debugging mode
+    --enable-gperftools-cpu    enable the use of gperftools' cpu profiler
 
   Required Packages in Non-Standard Locations:
-    --with-broccoli=PATH   path to libbroccoli install root
-    --with-pcap=PATH       path to libpcap install root
-    --with-pcapnav=PATH    path to libpcapnav install root
-    --with-flex=PATH       path to flex executable
-    --with-bison=PATH      path to bison executable
+    --with-broccoli=PATH       path to libbroccoli install root
+    --with-pcap=PATH           path to libpcap install root
+    --with-pcapnav=PATH        path to libpcapnav install root
+    --with-flex=PATH           path to flex executable
+    --with-bison=PATH          path to bison executable
 
   Influential Environment Variables (only on first invocation
   per build directory):
-    CC                     C compiler command
-    CFLAGS                 C compiler flags
-    CXX                    C++ compiler command
-    CXXFLAGS               C++ compiler flags
+    CC                         C compiler command
+    CFLAGS                     C compiler flags
+    CXX                        C++ compiler command
+    CXXFLAGS                   C++ compiler flags
 "
 
 sourcedir="$( cd "$( dirname "$0" )" && pwd )"
diff --git a/src/DynClass.cc b/src/DynClass.cc
index 711b604..4a60906 100644
--- a/src/DynClass.cc
+++ b/src/DynClass.cc
@@ -128,12 +128,14 @@ void DynClassTable::removeOld() {
 	lock();
 	//gettimeofday(&tv, NULL);
 
+    /*
     #ifdef __APPLE__
     struct tvalspec tmptv;
     clock_get_time(CLOCK_MONOTONIC_COARSE, &tmptv)i;
     now = valspec_to_tm(&tmptv);
     #endif
-    #ifdef linux
+    */
+    #if defined(linux) || defined(__APPLE__)
     struct timespec tmptv;
     clock_gettime(CLOCK_MONOTONIC_COARSE, &tmptv);
     now = spec_to_tm(&tmptv);
diff --git a/src/IndexHash.cc b/src/IndexHash.cc
index fd66e72..613fcff 100644
--- a/src/IndexHash.cc
+++ b/src/IndexHash.cc
@@ -1,6 +1,27 @@
 #include "IndexHash.hh"
 #include <iostream>
 
+/*
+ * Pseudo-proof about use of prime number hash table sizes. Please let me
+ * know if you find something wrong in the proof via email.
+ *
+ * The indexes in the hash table are calculated by doing key mod (hash table
+ *  size).
+ * We can note that there exists y / GCF(y, n) distinct instances of m * n (mod y) for all m.
+ * To see why this is true, we can let a = GCF(y, n).
+ * Let n be a particular key, and y = a * x and n = a * b
+ * Let's say we have n mod y, which can be written as (a * b) (mod a * x)
+ * So, note that (a * b) (mod a * x) = a * (b + x) (mod a * x), since a * b + a * x (mod a * x) = a * b (mod a * x)
+ * So, we only have x instances of unique remainders for all numbers with a
+ * factor of a.
+ * Note that y = a * x, which can be written as x = y / a
+ * So, x = y / GCF(y, n)
+ * We want x to be equal to y, since we want to utilize all the buckets in the
+ * hash table.
+ * So, GCF(y, n) = 1, which leads to only y being prime would suffice.
+ * So, we need a prime number hash size to avoid clustering.
+ */
+
 static const uint64_t IndexHash_primes[] = {1, 2, 3, 7, 13, 29, 53, 97, 193, 389, 769, 1543, 3079, 6151, \
                                             12289, 24593, 49517, 98317, 196613, 393241, 786433, 1572869, \
                                             3145739, 6291469, 12582917, 25165843, 50331653, 100663319, \
diff --git a/src/Storage.cc b/src/Storage.cc
index d734e68..84b3bda 100644
--- a/src/Storage.cc
+++ b/src/Storage.cc
@@ -271,6 +271,7 @@ Storage::~Storage() {
 	//tmlog(TM_LOG_DEBUG, "storage: Storage.cc, ~line 210", "Fifos deleted.");
 	delete indexes;
 	//tmlog(TM_LOG_DEBUG, "storage: Storage.cc, ~line 212", "pcap handle closed.");
+        //printf("Pcap handle closed\n");
 	pcap_close(ph);
 }
 
@@ -780,13 +781,15 @@ void Storage::query(QueryRequest *query_req, QueryResult *query_res) {
 		}
 	} /* if (subscription requested) */
 
-
+    /*
     #ifdef __APPLE__
     clock_get_time(CLOCK_MONOTONIC_COARSE, &t_end);
     tmlog(TM_LOG_NOTE, "query", "%d Done. It took %.2lf seconds", query_res->getQueryID(), 
         valspec_to_tm(&t_end)-valspec_to_tm(&t_start));
     #endif
-    #ifdef linux
+    */
+    //#ifdef linux
+    #if defined(linux) || defined(__APPLE__)
     clock_gettime(CLOCK_MONOTONIC_COARSE, &t_end);
     tmlog(TM_LOG_NOTE, "query", "%d Done. It took %.2lf seconds", query_res->getQueryID(),  
         spec_to_tm(&t_end)-spec_to_tm(&t_start));
@@ -853,12 +856,15 @@ bool Storage::setDynClass(IPAddress *ip, int dir, const char *classname) {
 	//gettimeofday(&tv, NULL);
 	//now = to_tm_time(&tv);
 
+        /*
         #ifdef __APPLE__
         struct tvalspec tmptv;
         clock_get_time(CLOCK_MONOTONIC_COARSE, &tmptv)i;
         now = valspec_to_tm(&tmptv);
         #endif
         #ifdef linux
+        */
+        #if defined(linux) || defined(__APPLE__)
         struct timespec tmptv;
         clock_gettime(CLOCK_MONOTONIC_COARSE, &tmptv);
         now = spec_to_tm(&tmptv);
diff --git a/src/conf.h b/src/conf.h
index 2cb8eda..c1b5cbd 100644
--- a/src/conf.h
+++ b/src/conf.h
@@ -13,6 +13,7 @@ extern int conf_main_log_interval;
 extern int conf_main_log_level;
 extern const char* conf_main_workdir;
 extern const char* conf_main_indexdir;
+extern const char* conf_main_profilepath;
 //extern const char* conf_classdir;
 extern const char* conf_main_logfile_name;
 extern const char* conf_main_bro_connect_str;
diff --git a/src/conf_parser.yy b/src/conf_parser.yy
index 14bd65c..7572025 100644
--- a/src/conf_parser.yy
+++ b/src/conf_parser.yy
@@ -68,7 +68,7 @@
 %token <ipaddr> TOK_IPADDRESS;
 %token TOK_CLASS TOK_FILTER TOK_MAIN TOK_LOG_INTERVAL TOK_LOG_LEVEL TOK_DEVICE
 %token TOK_CLASSDIR
-%token TOK_LOGFILE TOK_WORKDIR TOK_QUERYFILEDIR TOK_INDEXDIR
+%token TOK_LOGFILE TOK_WORKDIR TOK_QUERYFILEDIR TOK_INDEXDIR TOK_PROFILEPATH
 %token TOK_READ_TRACEFILE TOK_BRO_CONNECT_STR
 %token TOK_MEM TOK_DISK TOK_K TOK_M TOK_G TOK_CUTOFF TOK_PRECEDENCE
 %token TOK_DYN_TIMEOUT
@@ -245,6 +245,10 @@ main_option:
 	  conf_main_indexdir=strdup($2);
 	  free($2);
 	}
+        | TOK_PROFILEPATH TOK_STRING {
+          conf_main_profilepath=strdup($2);
+          free($2);
+        }
 	| TOK_BRO_CONNECT_STR TOK_STRING {
 	  conf_main_bro_connect_str=strdup($2);
 	  free($2);
diff --git a/src/conf_scanner.ll b/src/conf_scanner.ll
index f9a83af..ccb092b 100644
--- a/src/conf_scanner.ll
+++ b/src/conf_scanner.ll
@@ -58,6 +58,7 @@ NEWLINE \n
 "workdir"	 return TOK_WORKDIR;
 "queryfiledir"	 return TOK_QUERYFILEDIR;
 "indexdir"	 return TOK_INDEXDIR;
+"profilepath"    return TOK_PROFILEPATH;
 "index"      return TOK_INDEX;
 "logfile"	 return TOK_LOGFILE;
 "bro_connect_str" return TOK_BRO_CONNECT_STR;
diff --git a/src/main.cc.in b/src/main.cc.in
index 3c67efa..7e27432 100644
--- a/src/main.cc.in
+++ b/src/main.cc.in
@@ -28,8 +28,9 @@
 #define USE_MALLINFO
 #endif
 #include <sys/ioctl.h>
-//#include <gperftools/profiler.h>
-
+#ifdef USE_PERFTOOLS_CPU
+#include <gperftools/profiler.h>
+#endif
 
 #ifdef USE_BROCCOLI
 #include "BroccoliComm.hh"
@@ -117,6 +118,7 @@ int conf_main_log_interval=60;
 int conf_main_log_level=20;
 const char* conf_main_workdir="./";
 const char* conf_main_indexdir="./";
+const char* conf_main_profilepath="./";
 //const char* conf_classdir = "./";
 const char* conf_main_queryfiledir="./";
 const char* conf_main_logfile_name="timemachine.log";
@@ -826,7 +828,29 @@ void print_version()
 int
 main(int argc, char** argv) {
 
-        //ProfilerStart("/home/neto/data/profile/blah.prof");
+        /*
+        char filepath[100];
+
+        strcpy(filepath, conf_main_workdir);
+        strcat(filepath, "/");
+        strcat(filepath, conf_main_profilepath);
+
+        struct stat st;
+
+        printf("the directory to the profile is: %s\n", filepath);
+
+        if (stat(filepath, &st) != 0)
+        {
+            printf("The index directory %s did not exist. Creating the directory ...\n", conf_main_profilepath);
+            mkdir("home/lakers/timemachine_results/profile", 0755);
+        }
+
+        char path[70];
+
+        printf("The directory for initial main that we are in is %s\n", getcwd(path, 70));       
+ 
+        ProfilerStart(filepath);
+        */
 	const char *conffile="@BRO_ETC_INSTALL_DIR@/timemachine.cfg";
 	struct sigaction exit_action;
 	int i;
@@ -926,9 +950,35 @@ main(int argc, char** argv) {
 		/* don't close stderr yet. wait until startup is finished */
 	}
 
+        char filepath[100];
+
+        strcpy(filepath, conf_main_workdir);
+        strcat(filepath, "/");
+        strcat(filepath, conf_main_profilepath);
 
         
 
+        //struct stat st;
+
+        printf("the directory to the profile is: %s\n", filepath);
+
+        /*
+        if (stat(filepath, &st) != 0)
+        {
+            printf("The index directory %s did not exist. Creating the directory ...\n", conf_main_profilepath);
+            mkdir(filepath, 0755);
+        }
+        */
+
+        //char path[70];
+
+        //printf("The directory for initial main that we are in is %s\n", getcwd(path, 70));
+
+        #ifdef USE_PERFTOOLS_CPU 
+        ProfilerStart(filepath);
+        #endif
+
+
 	// Blocl all signals. This signal mask is then inherited by spawned 
 	// threads. After all threads are spanwed we will unblock again. 
 	sigset_t signalSet;
@@ -1073,7 +1123,9 @@ main(int argc, char** argv) {
 				pause();
 		}
 
-        //ProfilerStop();
+        #ifdef USE_PERFTOOLS_CPU
+        ProfilerStop();
+        #endif
         delete stats_log_file;
         delete classes_log_file;
         delete index_log_file; 
diff --git a/src/types.h b/src/types.h
index 9a03d47..a7b7a16 100644
--- a/src/types.h
+++ b/src/types.h
@@ -22,11 +22,13 @@ inline tm_time_t spec_to_tm(const struct timespec* tv) {
     return (double)tv->tv_sec + (double)tv->tv_nsec/1e9;
 }
 
+/*
 #ifdef __APPLE__
 inline tm_time_t valspec_to_tm(const struct tvalspec* tv) {
     return (double)tv->tv_sec + (double)tv->tv_nsec/1e9;
 }
 #endif
+*/
 
 //typedef enum {tcp, udp, icmp} proto_t;
 typedef uint8_t proto_t;


From neto at caltech.edu  Wed Aug 27 09:36:08 2014
From: neto at caltech.edu (Naoki Eto)
Date: Wed, 27 Aug 2014 09:36:08 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Hopefully fixed the Apple
	compiler issues (I do not have an Apple compiler) Added some
	comments to code and change log (a29e64b)
Message-ID: <201408271636.s7RGa8YQ029171@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit a29e64bbca00ea63cc6a22402d532e0d17d3fb59
Author: NaokiEto <neto at caltech.edu>
Date:   Wed Aug 27 12:24:01 2014 -0400

    Hopefully fixed the Apple compiler issues (I do not have an Apple compiler)
    Added some comments to code and change log


>---------------------------------------------------------------

a29e64bbca00ea63cc6a22402d532e0d17d3fb59
 src/DynClass.cc |  8 +++++++-
 src/Storage.cc  | 28 +++++++++++++++-------------
 src/main.cc.in  |  2 +-
 src/types.h     |  4 ++--
 4 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/src/DynClass.cc b/src/DynClass.cc
index 4a60906..5032db5 100644
--- a/src/DynClass.cc
+++ b/src/DynClass.cc
@@ -135,7 +135,8 @@ void DynClassTable::removeOld() {
     now = valspec_to_tm(&tmptv);
     #endif
     */
-    #if defined(linux) || defined(__APPLE__)
+    /*
+    #ifdef linux
     struct timespec tmptv;
     clock_gettime(CLOCK_MONOTONIC_COARSE, &tmptv);
     now = spec_to_tm(&tmptv);
@@ -145,6 +146,11 @@ void DynClassTable::removeOld() {
     clock_gettime(CLOCK_MONOTONIC_FAST, &tmptv);
     now = spec_to_tm(&tmptv);
     #endif
+    */
+    
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    now = to_tm_time(&tv);
     
 	//now = to_tm_time(&tv);
 
diff --git a/src/Storage.cc b/src/Storage.cc
index 84b3bda..8e57c12 100644
--- a/src/Storage.cc
+++ b/src/Storage.cc
@@ -716,9 +716,9 @@ tm_time_t Storage::getOldestTimestampDisk() {
 
 
 void Storage::query(QueryRequest *query_req, QueryResult *query_res) {
-	//struct timeval t_start, t_end;
-	//gettimeofday(&t_start, NULL);
-
+	struct timeval t_start, t_end;
+	gettimeofday(&t_start, NULL);
+    /*
     #ifdef __APPLE__
     struct tvalspec t_start, t_end;
     clock_get_time(CLOCK_MONOTONIC_COARSE, &t_start);
@@ -731,7 +731,7 @@ void Storage::query(QueryRequest *query_req, QueryResult *query_res) {
     struct timespec t_start, t_end;
     clock_gettime(CLOCK_MONOTONIC_FAST, &t_start);
     #endif
-
+    */
 	//fprintf(stderr, "Query ID: %d\n",  query_res->getQueryID());
 
     // getIndexByName is from Index.hh from class Indexes
@@ -789,6 +789,7 @@ void Storage::query(QueryRequest *query_req, QueryResult *query_res) {
     #endif
     */
     //#ifdef linux
+    /*
     #if defined(linux) || defined(__APPLE__)
     clock_gettime(CLOCK_MONOTONIC_COARSE, &t_end);
     tmlog(TM_LOG_NOTE, "query", "%d Done. It took %.2lf seconds", query_res->getQueryID(),  
@@ -799,11 +800,11 @@ void Storage::query(QueryRequest *query_req, QueryResult *query_res) {
     tmlog(TM_LOG_NOTE, "query", "%d Done. It took %.2lf seconds", query_res->getQueryID(),  
         spec_to_tm(&t_end)-spec_to_tm(&t_start));
     #endif
+    */
 
-
-	//gettimeofday(&t_end, NULL);
-	//tmlog(TM_LOG_NOTE, "query", "%d Done. It took %.2lf seconds", query_res->getQueryID(), 
-		//to_tm_time(&t_end)-to_tm_time(&t_start));
+	gettimeofday(&t_end, NULL);
+	tmlog(TM_LOG_NOTE, "query", "%d Done. It took %.2lf seconds", query_res->getQueryID(), 
+		to_tm_time(&t_end)-to_tm_time(&t_start));
 	if (query_res->getUsage() == 0) {
 		/* haven't passed it on to a subscription, delete it */
 		delete query_res;
@@ -811,7 +812,7 @@ void Storage::query(QueryRequest *query_req, QueryResult *query_res) {
 	delete query_req;
 
 	tot_queries_duration+=(uint64_t) ( (t_end.tv_sec-t_start.tv_sec)*1e6
-										+(t_end.tv_nsec-t_start.tv_nsec)/1000 );
+										+(t_end.tv_usec-t_start.tv_usec)/1000 );
 	tot_num_queries++;
 }
 
@@ -848,13 +849,13 @@ bool Storage::suspendTimeout(ConnectionID4 cid, bool b) {
 }
 
 bool Storage::setDynClass(IPAddress *ip, int dir, const char *classname) {
-	//struct timeval tv;
+	struct timeval tv;
 	tm_time_t now;
 	Fifo *f;
 	bool retval = true;
 
-	//gettimeofday(&tv, NULL);
-	//now = to_tm_time(&tv);
+	gettimeofday(&tv, NULL);
+	now = to_tm_time(&tv);
 
         /*
         #ifdef __APPLE__
@@ -864,6 +865,7 @@ bool Storage::setDynClass(IPAddress *ip, int dir, const char *classname) {
         #endif
         #ifdef linux
         */
+        /*
         #if defined(linux) || defined(__APPLE__)
         struct timespec tmptv;
         clock_gettime(CLOCK_MONOTONIC_COARSE, &tmptv);
@@ -874,7 +876,7 @@ bool Storage::setDynClass(IPAddress *ip, int dir, const char *classname) {
         clock_gettime(CLOCK_MONOTONIC_FAST, &tmptv);
         now = spec_to_tm(&tmptv);
         #endif
-
+        */
 	//tmlog(TM_LOG_DEBUG, "dyn_class", "Setting IP %s to class %s, direction %d",
 			//ip->getStr().c_str(), classname, dir);
 	f = getFifoByName(std::string("class_") + classname);
diff --git a/src/main.cc.in b/src/main.cc.in
index 7e27432..11b27cb 100644
--- a/src/main.cc.in
+++ b/src/main.cc.in
@@ -960,7 +960,7 @@ main(int argc, char** argv) {
 
         //struct stat st;
 
-        printf("the directory to the profile is: %s\n", filepath);
+        //printf("the directory to the profile is: %s\n", filepath);
 
         /*
         if (stat(filepath, &st) != 0)
diff --git a/src/types.h b/src/types.h
index a7b7a16..81c7ad4 100644
--- a/src/types.h
+++ b/src/types.h
@@ -17,11 +17,11 @@ typedef double tm_time_t;
 inline tm_time_t to_tm_time(const struct timeval* tv) {
 	return (double)tv->tv_sec+(double)tv->tv_usec/1e6;
 }
-
+/*
 inline tm_time_t spec_to_tm(const struct timespec* tv) {
     return (double)tv->tv_sec + (double)tv->tv_nsec/1e9;
 }
-
+*/
 /*
 #ifdef __APPLE__
 inline tm_time_t valspec_to_tm(const struct tvalspec* tv) {


From neto at caltech.edu  Wed Aug 27 19:22:25 2014
From: neto at caltech.edu (Naoki Eto)
Date: Wed, 27 Aug 2014 19:22:25 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Updated the Changes Deleted an
	unnecessary file (modp_numtoa.h) Made Aashish's change to
	read a pcap file and then exit immediately. This works only
	in daemon mode,
	since I still feel uneasy about doing it in console mode. Also,
	this change has not been fully tested. (300dba6)
Message-ID: <201408280222.s7S2MP6F005322@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit 300dba6263744b90bee6a83476fc732c05524baf
Author: NaokiEto <neto at caltech.edu>
Date:   Wed Aug 27 22:09:36 2014 -0400

    Updated the Changes
    Deleted an unnecessary file (modp_numtoa.h)
    Made Aashish's change to read a pcap file and then exit immediately.
    This works only in daemon mode, since I still feel uneasy about doing it in
    console mode. Also, this change has not been fully tested.


>---------------------------------------------------------------

300dba6263744b90bee6a83476fc732c05524baf
 CHANGES           |  18 +++++++---
 src/Connection.cc |   8 +++--
 src/Storage.cc    |  50 +++++++++++++++++++++++++-
 src/Storage.hh    |  33 ++++++++++++++++++
 src/main.cc.in    |  32 +++++++++++++++--
 src/modp_numtoa.h | 102 ------------------------------------------------------
 src/util.h        |   4 +--
 7 files changed, 134 insertions(+), 113 deletions(-)

diff --git a/CHANGES b/CHANGES
index 3879688..7b57f67 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,21 +1,31 @@
 0.1-5 | 2014-08-26 15:35:00 -0800
 
+    * Implemented the creation of index and query directories by default if the user did not create the index and query directories. Also,
+      if indexes are not enabled, index directory is not created. If querying is not occuring, query directory is not created. 
+
+    * Implemented a counter instead of the many calls to gettimeofday when determining when to write to disk, which costed a lot of CPU.
+
     * Changed the hash table sizes to always be prime number, to help avoid clustering in the collisions lists. (Naoki Eto) 
 
-    * Added gperftools CPU profiler, which can be enabled (Naoki Eto)
+    * Added gperftools CPU profiler, which can be enabled by using --enable-gperftools-cpu in the ./configure option and
+      adding a name to profilepath in the configuration file (Naoki Eto)
 
     * Changed the method for reading the configuration classes so that it is first ordered by precedence and then the highest precedence match is found (Naoki Eto)    
 
     * Implemented querying for IPv4 and IPv6 ip, conn2, conn3, and conn4 (Naoki Eto)
 
-    * Implemented class directories that can be specified in the configuration file (Naoki Eto)
+    * Implemented class directories that can be specified in the configuration file. Example:
+      ...
+      filesize 2000m;
+      mem 100m;
+      classdir "/home/neto/data_http";
+      }
+      (Naoki Eto)
 
 0.1-4 | 2014-07-18 16:53:50 -0800
     
     * Implemented IPv6 support for the classes. (Naoki Eto)   
  
-    * Some querying for IPv6 addresses is enabled. (Naoki Eto)
-
     * VLAN tags are taken into account w/o MPLS labels (Naoki Eto)
 
 0.1-4 | 2013-02-07 14:37:50 -0800
diff --git a/src/Connection.cc b/src/Connection.cc
index 1f95746..e2f85ee 100644
--- a/src/Connection.cc
+++ b/src/Connection.cc
@@ -22,6 +22,8 @@
 static std::string pattern_ip ("(\\d+\\.\\d+\\.\\d+\\.\\d+)");
 static std::string pattern_ipport ("(\\d+\\.\\d+\\.\\d+\\.\\d+):(\\d+)");
 
+// The () are for subpattern matching, and doing (?: ...) avoids the subpattern matching while still allowing grouping. This is the perl regex for ipv6
+
 static std::string pattern_ip6 ("\\[((?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\]");
 // stolen from stackoverflow http://stackoverflow.com/questions/53497/regular-expression-that-matches-valid-ipv6-addresses
 
@@ -67,8 +69,6 @@ inline bool addr6_port_canon_lt(const unsigned char s6_ip[], const unsigned char
 		return (memcmp(s6_ip, d6_ip, 16) < 0);
 }
 
-
-
 void ConnectionID4::init(proto_t proto4,
 						 uint32_t s_ip, uint32_t d_ip,
 						 uint16_t s_port, uint16_t d_port) {
@@ -94,6 +94,10 @@ void ConnectionID4::init(proto_t proto4,
         //key.ip1.s6_tm_addr = &v4_mapped_prefix[0];
         //char *
 
+        // We are going to place all ip addresses into 16 byte char arrays. For the 32-bit/4 bytes
+        // ipv4 addresses, we put the first 12 slots as 0's and the last 4 as the ipv4 address
+        // This seems like a waste of space, and perhaps a better approach can be used. This is
+        // bro's way of doing it.
         memcpy(key.ip1.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
         memcpy(&key.ip1.s6_addr[12], &d_ip, sizeof(d_ip));
 
diff --git a/src/Storage.cc b/src/Storage.cc
index 8e57c12..1bb4869 100644
--- a/src/Storage.cc
+++ b/src/Storage.cc
@@ -19,6 +19,12 @@
 
 #define SNAPLEN 8192
 
+extern LogFile *stats_log_file;
+extern LogFile *classes_log_file;
+extern LogFile *index_log_file;
+//extern int read_file;
+
+extern void tmexit();
 
 /***************************************************************************
  * callback handler for pcap_loop
@@ -71,8 +77,29 @@ void *capture_thread(void *arg) {
 	pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
 	Storage *storage = (Storage *)arg;
 	//  sleep(15);
+
 	pcap_loop(storage->ph, -1, (pcap_handler)callback, (u_char*)storage);
+        //printf("Done reading pcap file \n");
 	//tmlog(TM_LOG_NOTE, "storage: Storage.cc, ~line 59", "pcap input exhausted");
+
+        storage->set_read_file(0);;
+
+        //tmlog(TM_LOG_ERROR, "Storage:capture_thread", "we are done reading the pcap file..");
+
+        int s = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
+
+        storage->lock_readfile();
+
+        //tmlog(TM_LOG_ERROR, "capture_Thread", "broadcasting for deletion and exit");
+
+        storage->cond_broadcast_readfile();
+
+        //delete stats_log_file;
+        //delete classes_log_file;
+        //delete index_log_file;
+
+        //tmexit();
+
 	return NULL;
 }
 
@@ -251,8 +278,9 @@ Storage::Storage(StorageConfig& conf):
 		}
 	}
 #endif
+        read_file = 1;
 	int i=pthread_create(&capture_thread_tid, &capture_thread_attr, capture_thread, (void *)this);
-
+    pthread_create(&read_file_thread_tid, NULL, read_file_thread, (void *)this);
     //tmlog(TM_LOG_DEBUG, "capture thread: Storage.cc, ~line 195", "attempting to create capture thread");
 	if (i!=0) {
 		pcap_close(ph);
@@ -913,3 +941,23 @@ Fifo* Storage::getFifoByName(std::string search_name) {
 	return r;
 }
 
+void Storage::destroy_read_file()
+{
+    while (1)
+    {
+        //tmlog(TM_LOG_ERROR, "destroy_read_file()", "at the beginning of the while loop");
+        cond_wait_readfile();
+        //tmlog(TM_LOG_ERROR, "destroy_read_file()", "that is the signal to delete everything!");
+        if (read_file == 0)
+        {
+            //tmlog(TM_LOG_ERROR, "destroy_read_file()", "read_file was successfully changed to 0");
+            unlock_readfile();
+
+            delete stats_log_file;
+            delete classes_log_file;
+            delete index_log_file;
+
+            tmexit();
+        }
+    }
+}
diff --git a/src/Storage.hh b/src/Storage.hh
index 864f2d6..31f6ce3 100644
--- a/src/Storage.hh
+++ b/src/Storage.hh
@@ -21,6 +21,8 @@
 //#include "Index.hh"
 class Indexes;
 
+extern "C" { void *read_file_thread(void *instance); }
+
 void *capture_thread(void *arg);
 void callback(u_char *args, const struct pcap_pkthdr *header, const u_char *packet);
 
@@ -101,8 +103,38 @@ public:
 		return dynclasses.getNumEntries();
 	}
 
+        void destroy_read_file();
+	
 	friend void *capture_thread(void *arg);
 
+protected:
+        pthread_cond_t readfile_cond;
+        pthread_mutex_t readfile_lock_mutex;
+        void lock_readfile() {
+                pthread_mutex_lock(&readfile_lock_mutex);
+        }
+        void unlock_readfile() {
+                pthread_mutex_unlock(&readfile_lock_mutex);
+        }
+        /** 
+         * Signal the MaintainerThread, that we added something to the queue.
+         * YOU MUST HOLD THE queue_lock WHEN CALLING THIS */
+        void cond_broadcast_readfile() {
+                pthread_cond_broadcast(&readfile_cond);
+            //tmlog(TM_LOG_DEBUG, "Mantainer IndexThread", "signaling to Maintainer IndexThread that we added something to the queue");
+        }
+        /** 
+         * Wait for signal, that data is availabe in the queue 
+         * YOU MUST HOLD THE queue_lock WHEN CALLING THIS */
+        void cond_wait_readfile() {
+                pthread_cond_wait(&readfile_cond, &readfile_lock_mutex);
+        }
+        int read_file;
+        void set_read_file(int new_num)
+        {
+            read_file = new_num;
+        }
+
 /*
 protected:
     void Close();
@@ -118,6 +150,7 @@ private:
 	int snaplen;
 	pthread_t capture_thread_tid;
 	pthread_attr_t capture_thread_attr;
+        pthread_t read_file_thread_tid;
 
 	std::list<Fifo*> fifos;
 
diff --git a/src/main.cc.in b/src/main.cc.in
index 11b27cb..71edb99 100644
--- a/src/main.cc.in
+++ b/src/main.cc.in
@@ -86,6 +86,7 @@ LogFile *stats_log_file;
 LogFile *classes_log_file;
 LogFile *index_log_file;
 
+//int read_file = 1;
 
 #define HOSTNAME_MAXLEN 32
 char hostname[HOSTNAME_MAXLEN];
@@ -247,6 +248,7 @@ void print_stats(FILE *outfp) {
 
 void
 tmexit() {
+        printf("beginning the process of tmexit()\n");
 	
 	if (conf_main_rmtconsole) {
 		// Cancel rmtconsole_listen thread 
@@ -263,6 +265,9 @@ tmexit() {
 	pthread_join(statisticslog_thread_tid, NULL);
 	tmlog(TM_LOG_DEBUG, "main",  "stats thread i DEAD.");
 
+        //printf("The stats thread has been canceled...\n");
+        //tmlog(TM_LOG_ERROR, "tmexit()", "The stats thread has been canceled...");
+
 	// Cancel aggregation thread 
 	tmlog(TM_LOG_DEBUG, "main",  "Canceling aggreagation thread");
 	pthread_cancel(index_aggregation_thread_tid);
@@ -273,6 +278,8 @@ tmexit() {
 #ifdef USE_BROCCOLI
 	broccoli_exit();
 #endif
+        //printf("The aggregation thread has been canceled\n");
+        //tmlog(TM_LOG_ERROR, "tmexit()", "The aggregation thread has been canceled");
 
 	//TODO: Cancel all query threads 
 	//XXX: Assumption: when a thread waits for lock it is cancel-able
@@ -287,8 +294,12 @@ tmexit() {
 	// thus cancel all index maintaining threads
 	//tmlog(TM_LOG_DEBUG, "main", "deactivating storage... ");
 	storage->cancelThread();
+        //printf("the capture thread has been canceled\n");
+        //tmlog(TM_LOG_ERROR, "tmexit()", "The aggregation thread has been canceled");
 	delete storage;
 
+        //printf("The storage has been deleted\n");
+
 	//tmlog("main", "Storage deactivated");
 
 	/*
@@ -300,6 +311,7 @@ tmexit() {
 
 	delete log_file;
 
+        //printf("tm is almost dead now...\n");
 	cmd_parser_finish();
 	exit(0);
 	// No need to unlock -- we are DEAD by now
@@ -357,7 +369,7 @@ void *cli_console_thread(void *arg) {
             //tmlog(TM_LOG_NOTE, "cli console thread", "CLI console thread active");
 			free(line);
 		}
-	} while (line != NULL);
+	} while (line != NULL);// && read_file == 1);
 
 	printf("CLI console thread exiting\n");
 	tmlog(TM_LOG_NOTE, "main", "CLI console thread exiting");
@@ -920,7 +932,7 @@ main(int argc, char** argv) {
 	}
 
 	log_file=new LogFile(conf_main_logfile_name);
-	tmlog("main: main.cc.in, ~line 655", "TimeMachine version %s", VERSION);
+	//tmlog("main: main.cc.in, ~line 655", "TimeMachine version %s", VERSION);
 
 	if (!conf_main_daemon)
 		printf("TimeMachine version %s\n", VERSION);
@@ -1088,10 +1100,14 @@ main(int argc, char** argv) {
 			exit(1);
 		};
 
+                //printf("Reached here\n");
+
 		// All threads have been spwaned now. Re-enable signal delivery
 		// Only this thread will now receive signals. 
 		pthread_sigmask (SIG_SETMASK, &oldSignalSet, NULL );
 
+
+                //printf("but not here...\n");
 		//   struct sched_param param;
 		//   int policy;
 		//   param.sched_priority=1;
@@ -1110,19 +1126,26 @@ main(int argc, char** argv) {
 
 		/* FIXME: Possible race condition */
 		if (conf_main_daemon) {
+                        //printf("we reach here?\n");
+                        //tmlog(TM_LOG_ERROR, "main.cc.in", "we are in the conf_main_daemon");
 			stderr_is_open = 0;
 			fclose(stderr);
 		}
 
 		if (conf_main_console) {
+                        //printf("or here....?\n");
 			pthread_join(cli_console_thread_tid, NULL);
+                        //printf("pthread did some joining...\n");
 		}
 		else {
+                        //printf("blah here?\n");
 			// XXX: There must be a better way to do this!
 			while (1) 
 				pause();
 		}
 
+        //printf("We are exiting...\n");
+        //tmlog(TM_LOG_ERROR, "main.cc.in", "we are exiting...");
         #ifdef USE_PERFTOOLS_CPU
         ProfilerStop();
         #endif
@@ -1143,4 +1166,9 @@ void *start_index_thread(void *instance) {
 	((IndexType *)(instance))->run();
 	return NULL;
 }
+
+void *read_file_thread(void *instance) {
+        ((Storage *)(instance))->destroy_read_file();
+        return NULL;
+}
 }
diff --git a/src/modp_numtoa.h b/src/modp_numtoa.h
deleted file mode 100644
index b848163..0000000
--- a/src/modp_numtoa.h
+++ /dev/null
@@ -1,102 +0,0 @@
-/* -*- mode: c++; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */
-/* vi: set expandtab shiftwidth=4 tabstop=4: */
-
-/**
- * \file
- *
- * <pre>
- * Copyright &copy; 2007, Nick Galbreath -- nickg [at] modp [dot] com
- * All rights reserved.
- * http://code.google.com/p/stringencoders/
- * Released under the bsd license.
- * </pre>
- *
- * This defines signed/unsigned integer, and 'double' to char buffer
- * converters.  The standard way of doing this is with "sprintf", however
- * these functions are
- *   * guarenteed maximum size output
- *   * 5-20x faster!
- *   * core-dump safe
- *
- *
- */
-
-#ifndef COM_MODP_STRINGENCODERS_NUMTOA_H
-#define COM_MODP_STRINGENCODERS_NUMTOA_H
-
-#ifdef __cplusplus
-#define BEGIN_C extern "C" {
-#define END_C }
-#else
-#define BEGIN_C
-#define END_C
-#endif
-
-BEGIN_C
-
-#include <stdint.h>
-
-/** \brief convert an signed integer to char buffer
- *
- * \param[in] value
- * \param[out] buf the output buffer.  Should be 16 chars or more.
- */
-void modp_itoa10(int32_t value, char* buf);
-
-/** \brief convert an unsigned integer to char buffer
- *
- * \param[in] value
- * \param[out] buf The output buffer, should be 16 chars or more.
- */
-void modp_uitoa10(uint32_t value, char* buf);
-
-/** \brief convert an signed long integer to char buffer
- *
- * \param[in] value
- * \param[out] buf the output buffer.  Should be 24 chars or more.
- */
-void modp_litoa10(int64_t value, char* buf);
-
-/** \brief convert an unsigned long integer to char buffer
- *
- * \param[in] value
- * \param[out] buf The output buffer, should be 24 chars or more.
- */
-void modp_ulitoa10(uint64_t value, char* buf);
-
-/** \brief convert a floating point number to char buffer with
- *         fixed-precision format
- *
- * This is similar to "%.[0-9]f" in the printf style.  It will include
- * trailing zeros
- *
- * If the input value is greater than 1<<31, then the output format
- * will be switched exponential format.
- *
- * \param[in] value
- * \param[out] buf  The allocated output buffer.  Should be 32 chars or more.
- * \param[in] precision  Number of digits to the right of the decimal point.
- *    Can only be 0-9.
- */
-void modp_dtoa(double value, char* buf, int precision);
-
-/** \brief convert a floating point number to char buffer with a
- *         variable-precision format, and no trailing zeros
- *
- * This is similar to "%.[0-9]f" in the printf style, except it will
- * NOT include trailing zeros after the decimal point.  This type
- * of format oddly does not exists with printf.
- *
- * If the input value is greater than 1<<31, then the output format
- * will be switched exponential format.
- *
- * \param[in] value
- * \param[out] buf  The allocated output buffer.  Should be 32 chars or more.
- * \param[in] precision  Number of digits to the right of the decimal point.
- *    Can only be 0-9.
- */
-void modp_dtoa2(double value, char* buf, int precision);
-
-END_C
-
-#endif
diff --git a/src/util.h b/src/util.h
index 112c33b..bfdbb84 100644
--- a/src/util.h
+++ b/src/util.h
@@ -109,12 +109,12 @@ typedef int32 ptr_compat_int;
 # error "Unusual pointer size. Please report to bro at bro.org."
 #endif
 #endif
-
+/*
 extern "C"
 	{
 	#include "modp_numtoa.h"
 	}
-
+*/
 template <class T>
 void delete_each(T* t)
 	{


From neto at caltech.edu  Thu Aug 28 15:20:07 2014
From: neto at caltech.edu (Naoki Eto)
Date: Thu, 28 Aug 2014 15:20:07 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Updated some comments about
	prime number hash sizes and the use of the number in the
	counter when trying to write indxes to disk. (3f08ec8)
Message-ID: <201408282220.s7SMK7ho018150@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit 3f08ec896fb1a5b3b38f99faed2082666af0e633
Author: NaokiEto <neto at caltech.edu>
Date:   Thu Aug 28 18:05:21 2014 -0400

    Updated some comments about prime number hash sizes and the use of the number in the
    counter when trying to write indxes to disk.
    
    Implemented Matthias' CMakeLists code for gperftools


>---------------------------------------------------------------

3f08ec896fb1a5b3b38f99faed2082666af0e633
 CHANGES          | 19 +++++++++++++++----
 CMakeLists.txt   |  4 +++-
 configure        |  2 +-
 src/Index.hh     |  3 +++
 src/IndexHash.cc |  2 ++
 5 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/CHANGES b/CHANGES
index 3879688..de3d341 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,21 +1,31 @@
 0.1-5 | 2014-08-26 15:35:00 -0800
 
+    * Implemented the creation of index and query directories by default if the user did not create the index and query directories. Also,
+      if indexes are not enabled, index directory is not created. If querying is not occuring, query directory is not created.
+
+    * Implemented a counter instead of the many calls to gettimeofday when determining when to write to disk, which costed a lot of CPU.
+
     * Changed the hash table sizes to always be prime number, to help avoid clustering in the collisions lists. (Naoki Eto)
 
-    * Added gperftools CPU profiler, which can be enabled (Naoki Eto)
+    * Added gperftools CPU profiler, which can be enabled by using --enable-gperftools-cpu in the ./configure option and
+      adding a name to profilepath in the configuration file (Naoki Eto)
 
     * Changed the method for reading the configuration classes so that it is first ordered by precedence and then the highest precedence match is found (Naoki Eto)
 
     * Implemented querying for IPv4 and IPv6 ip, conn2, conn3, and conn4 (Naoki Eto)
 
-    * Implemented class directories that can be specified in the configuration file (Naoki Eto)
+    * Implemented class directories that can be specified in the configuration file. Example:
+      ...
+      filesize 2000m;
+      mem 100m;
+      classdir "/home/neto/data_http";
+      }
+      (Naoki Eto)
 
 0.1-4 | 2014-07-18 16:53:50 -0800
 
     * Implemented IPv6 support for the classes. (Naoki Eto)
 
-    * Some querying for IPv6 addresses is enabled. (Naoki Eto)
-
     * VLAN tags are taken into account w/o MPLS labels (Naoki Eto)
 
 0.1-4 | 2013-02-07 14:37:50 -0800
@@ -25,3 +35,4 @@
 0.1-3 | 2013-02-07 14:33:20 -0800
 
   * Starting CHANGES.
+
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e718977..83a4457 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -78,7 +78,9 @@ if (ENABLE_PERFTOOLS_CPU OR ENABLE_PERFTOOLS_DEBUG OR ENABLE_PERFTOOLS)
     # perftools weren't found
 endif ()
 
-set(CMAKE_CXX_FLAGS "-g -Wall")
+set(CMAKE_CXX_FLAGS "-g -Wall -O3")
+set(CMAKE_C_FLAGS "-g -Wall -O3")
+add_definitions("-O3")
 
 #detect 32 or 64 bit compiler
 # Also in config.h.in
diff --git a/configure b/configure
index 1b60a02..377beb1 100755
--- a/configure
+++ b/configure
@@ -25,7 +25,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
 
   Optional Features:
     --enable-debug             compile in debugging mode
-    --enable-gperftools-cpu    enable the use of gperftools' cpu profiler
+    --enable-perftools-cpu    enable the use of gperftools' cpu profiler
 
   Required Packages in Non-Standard Locations:
     --with-broccoli=PATH       path to libbroccoli install root
diff --git a/src/Index.hh b/src/Index.hh
index 1befe08..3cf0397 100644
--- a/src/Index.hh
+++ b/src/Index.hh
@@ -398,6 +398,9 @@ public:
                 */
         // returns 0 if lock was successfully achieved
 
+        // 500000 came from testing. In the above commented out code, you can see that previously, gettimeofday was used.
+        // Basically, I did a counter while it was doing gettimeofday, and found that it did around 500000 entries before trying
+        // disk write lock. It was pretty consistent on two different machines.
         if (num_of_entries < 500000)
         {
             num_of_entries++;
diff --git a/src/IndexHash.cc b/src/IndexHash.cc
index 613fcff..e37e6a5 100644
--- a/src/IndexHash.cc
+++ b/src/IndexHash.cc
@@ -7,6 +7,8 @@
  *
  * The indexes in the hash table are calculated by doing key mod (hash table
  *  size).
+ * The statement to prove is: number of buckets used = hash_table_size / GCF(hash_table_size, factor).
+ *
  * We can note that there exists y / GCF(y, n) distinct instances of m * n (mod y) for all m.
  * To see why this is true, we can let a = GCF(y, n).
  * Let n be a particular key, and y = a * x and n = a * b


From neto at caltech.edu  Thu Aug 28 15:23:09 2014
From: neto at caltech.edu (Naoki Eto)
Date: Thu, 28 Aug 2014 15:23:09 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Updated some comments about
	prime number hash sizes and the use of the number in the
	counter when trying to write indxes to disk. (97e862b)
Message-ID: <201408282223.s7SMN9Zq018232@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit 97e862bab618326dab53ba2668036629848d7b18
Author: NaokiEto <neto at caltech.edu>
Date:   Thu Aug 28 18:05:21 2014 -0400

    Updated some comments about prime number hash sizes and the use of the number in the
    counter when trying to write indxes to disk.
    
    Implemented Matthias' CMakeLists code for optimizations (-O3)


>---------------------------------------------------------------

97e862bab618326dab53ba2668036629848d7b18
 CHANGES          | 19 +++++++++++++++----
 CMakeLists.txt   |  4 +++-
 configure        |  2 +-
 src/Index.hh     |  3 +++
 src/IndexHash.cc |  2 ++
 5 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/CHANGES b/CHANGES
index 3879688..de3d341 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,21 +1,31 @@
 0.1-5 | 2014-08-26 15:35:00 -0800
 
+    * Implemented the creation of index and query directories by default if the user did not create the index and query directories. Also,
+      if indexes are not enabled, index directory is not created. If querying is not occuring, query directory is not created.
+
+    * Implemented a counter instead of the many calls to gettimeofday when determining when to write to disk, which costed a lot of CPU.
+
     * Changed the hash table sizes to always be prime number, to help avoid clustering in the collisions lists. (Naoki Eto)
 
-    * Added gperftools CPU profiler, which can be enabled (Naoki Eto)
+    * Added gperftools CPU profiler, which can be enabled by using --enable-gperftools-cpu in the ./configure option and
+      adding a name to profilepath in the configuration file (Naoki Eto)
 
     * Changed the method for reading the configuration classes so that it is first ordered by precedence and then the highest precedence match is found (Naoki Eto)
 
     * Implemented querying for IPv4 and IPv6 ip, conn2, conn3, and conn4 (Naoki Eto)
 
-    * Implemented class directories that can be specified in the configuration file (Naoki Eto)
+    * Implemented class directories that can be specified in the configuration file. Example:
+      ...
+      filesize 2000m;
+      mem 100m;
+      classdir "/home/neto/data_http";
+      }
+      (Naoki Eto)
 
 0.1-4 | 2014-07-18 16:53:50 -0800
 
     * Implemented IPv6 support for the classes. (Naoki Eto)
 
-    * Some querying for IPv6 addresses is enabled. (Naoki Eto)
-
     * VLAN tags are taken into account w/o MPLS labels (Naoki Eto)
 
 0.1-4 | 2013-02-07 14:37:50 -0800
@@ -25,3 +35,4 @@
 0.1-3 | 2013-02-07 14:33:20 -0800
 
   * Starting CHANGES.
+
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e718977..83a4457 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -78,7 +78,9 @@ if (ENABLE_PERFTOOLS_CPU OR ENABLE_PERFTOOLS_DEBUG OR ENABLE_PERFTOOLS)
     # perftools weren't found
 endif ()
 
-set(CMAKE_CXX_FLAGS "-g -Wall")
+set(CMAKE_CXX_FLAGS "-g -Wall -O3")
+set(CMAKE_C_FLAGS "-g -Wall -O3")
+add_definitions("-O3")
 
 #detect 32 or 64 bit compiler
 # Also in config.h.in
diff --git a/configure b/configure
index 1b60a02..377beb1 100755
--- a/configure
+++ b/configure
@@ -25,7 +25,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
 
   Optional Features:
     --enable-debug             compile in debugging mode
-    --enable-gperftools-cpu    enable the use of gperftools' cpu profiler
+    --enable-perftools-cpu    enable the use of gperftools' cpu profiler
 
   Required Packages in Non-Standard Locations:
     --with-broccoli=PATH       path to libbroccoli install root
diff --git a/src/Index.hh b/src/Index.hh
index 1befe08..3cf0397 100644
--- a/src/Index.hh
+++ b/src/Index.hh
@@ -398,6 +398,9 @@ public:
                 */
         // returns 0 if lock was successfully achieved
 
+        // 500000 came from testing. In the above commented out code, you can see that previously, gettimeofday was used.
+        // Basically, I did a counter while it was doing gettimeofday, and found that it did around 500000 entries before trying
+        // disk write lock. It was pretty consistent on two different machines.
         if (num_of_entries < 500000)
         {
             num_of_entries++;
diff --git a/src/IndexHash.cc b/src/IndexHash.cc
index 613fcff..e37e6a5 100644
--- a/src/IndexHash.cc
+++ b/src/IndexHash.cc
@@ -7,6 +7,8 @@
  *
  * The indexes in the hash table are calculated by doing key mod (hash table
  *  size).
+ * The statement to prove is: number of buckets used = hash_table_size / GCF(hash_table_size, factor).
+ *
  * We can note that there exists y / GCF(y, n) distinct instances of m * n (mod y) for all m.
  * To see why this is true, we can let a = GCF(y, n).
  * Let n be a particular key, and y = a * x and n = a * b


From neto at caltech.edu  Thu Aug 28 15:44:53 2014
From: neto at caltech.edu (Naoki Eto)
Date: Thu, 28 Aug 2014 15:44:53 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Fixed a Memory-Illegal access
	error found by Coverity. This error occurred in Index.cc,
	and was in the original tm-master code (the ipv4 only
	implementation). Basically, iqe had a chance to be deleted,
	and then was to be accessed after that. (f5f4c9d)
Message-ID: <201408282244.s7SMirjx018406@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit f5f4c9d405c192659c21e48c5dcff18ba1798083
Author: NaokiEto <neto at caltech.edu>
Date:   Thu Aug 28 18:31:12 2014 -0400

    Fixed a Memory-Illegal access error found by Coverity. This error occurred in
    Index.cc, and was in the original tm-master code (the ipv4 only implementation).
    Basically, iqe had a chance to be deleted, and then was to be accessed after that.
    
    To find where this occurred, do a grep "delete iqe" .


>---------------------------------------------------------------

f5f4c9d405c192659c21e48c5dcff18ba1798083
 src/Index.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/Index.cc b/src/Index.cc
index aeb5717..c354706 100644
--- a/src/Index.cc
+++ b/src/Index.cc
@@ -376,6 +376,9 @@ void Index<T>::addEntry(IndexField *iqe) {
 				iqe->ts-IDX_PKT_SECURITY_MARGIN*idx_thread_iat, iqe->ts);
 		cur->add(iqe, ie_n);
 
+        // update last_updated time
+        last_updated = iqe->ts;
+
     	//last_updated = iqe->ts;
 
         //ProfilerStop();
@@ -389,7 +392,7 @@ void Index<T>::addEntry(IndexField *iqe) {
 		delete iqe;
 	}
     // update last_updated time
-	last_updated = iqe->ts;
+	//last_updated = iqe->ts;
 
     // Note that old hash table is now the formerly current hash table. So, it is in the memory, and we can do look up on it
     // This must be the table that Aashish says that indexes do not persist, part of of index persistence (other part is


From vallentin at icir.org  Thu Aug 28 16:01:07 2014
From: vallentin at icir.org (Matthias Vallentin)
Date: Thu, 28 Aug 2014 16:01:07 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: Updated some comments about
 prime number hash sizes and the use of the number in the counter when
 trying to write indxes to disk. (97e862b)
In-Reply-To: <201408282223.s7SMN9Zq018232@bro-ids.icir.org>
References: <201408282223.s7SMN9Zq018232@bro-ids.icir.org>
Message-ID: <CADTpMNY9aJBxhg62Ggzih=+tXUV3Fqc3+g8ZJU5h57Sj5roHZw@mail.gmail.com>

FWIW, the CMake stuff was really just an ugly hack to bring -O3 to our test
installation and see if it noticeably changes the performance. The correct
way to integrate this would be to fix the CMake setup to add -O3 in the
release builds. I didn't look at it in detail, but it looks like the build
harness only added C++ flags but the compilation took place with
/usr/bin/CC.

---ph0wn

On Thursday, August 28, 2014, Naoki Eto <neto at caltech.edu> wrote:

> Repository : ssh://git at bro-ids.icir.org/time-machine
>
> On branch  : topic/naokieto/ipv6
>
> >---------------------------------------------------------------
>
> commit 97e862bab618326dab53ba2668036629848d7b18
> Author: NaokiEto <neto at caltech.edu <javascript:;>>
> Date:   Thu Aug 28 18:05:21 2014 -0400
>
>     Updated some comments about prime number hash sizes and the use of the
> number in the
>     counter when trying to write indxes to disk.
>
>     Implemented Matthias' CMakeLists code for optimizations (-O3)
>
>
> >---------------------------------------------------------------
>
> 97e862bab618326dab53ba2668036629848d7b18
>  CHANGES          | 19 +++++++++++++++----
>  CMakeLists.txt   |  4 +++-
>  configure        |  2 +-
>  src/Index.hh     |  3 +++
>  src/IndexHash.cc |  2 ++
>  5 files changed, 24 insertions(+), 6 deletions(-)
>
> diff --git a/CHANGES b/CHANGES
> index 3879688..de3d341 100644
> --- a/CHANGES
> +++ b/CHANGES
> @@ -1,21 +1,31 @@
>  0.1-5 | 2014-08-26 15:35:00 -0800
>
> +    * Implemented the creation of index and query directories by default
> if the user did not create the index and query directories. Also,
> +      if indexes are not enabled, index directory is not created. If
> querying is not occuring, query directory is not created.
> +
> +    * Implemented a counter instead of the many calls to gettimeofday
> when determining when to write to disk, which costed a lot of CPU.
> +
>      * Changed the hash table sizes to always be prime number, to help
> avoid clustering in the collisions lists. (Naoki Eto)
>
> -    * Added gperftools CPU profiler, which can be enabled (Naoki Eto)
> +    * Added gperftools CPU profiler, which can be enabled by using
> --enable-gperftools-cpu in the ./configure option and
> +      adding a name to profilepath in the configuration file (Naoki Eto)
>
>      * Changed the method for reading the configuration classes so that it
> is first ordered by precedence and then the highest precedence match is
> found (Naoki Eto)
>
>      * Implemented querying for IPv4 and IPv6 ip, conn2, conn3, and conn4
> (Naoki Eto)
>
> -    * Implemented class directories that can be specified in the
> configuration file (Naoki Eto)
> +    * Implemented class directories that can be specified in the
> configuration file. Example:
> +      ...
> +      filesize 2000m;
> +      mem 100m;
> +      classdir "/home/neto/data_http";
> +      }
> +      (Naoki Eto)
>
>  0.1-4 | 2014-07-18 16:53:50 -0800
>
>      * Implemented IPv6 support for the classes. (Naoki Eto)
>
> -    * Some querying for IPv6 addresses is enabled. (Naoki Eto)
> -
>      * VLAN tags are taken into account w/o MPLS labels (Naoki Eto)
>
>  0.1-4 | 2013-02-07 14:37:50 -0800
> @@ -25,3 +35,4 @@
>  0.1-3 | 2013-02-07 14:33:20 -0800
>
>    * Starting CHANGES.
> +
> diff --git a/CMakeLists.txt b/CMakeLists.txt
> index e718977..83a4457 100644
> --- a/CMakeLists.txt
> +++ b/CMakeLists.txt
> @@ -78,7 +78,9 @@ if (ENABLE_PERFTOOLS_CPU OR ENABLE_PERFTOOLS_DEBUG OR
> ENABLE_PERFTOOLS)
>      # perftools weren't found
>  endif ()
>
> -set(CMAKE_CXX_FLAGS "-g -Wall")
> +set(CMAKE_CXX_FLAGS "-g -Wall -O3")
> +set(CMAKE_C_FLAGS "-g -Wall -O3")
> +add_definitions("-O3")
>
>  #detect 32 or 64 bit compiler
>  # Also in config.h.in
> diff --git a/configure b/configure
> index 1b60a02..377beb1 100755
> --- a/configure
> +++ b/configure
> @@ -25,7 +25,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
>
>    Optional Features:
>      --enable-debug             compile in debugging mode
> -    --enable-gperftools-cpu    enable the use of gperftools' cpu profiler
> +    --enable-perftools-cpu    enable the use of gperftools' cpu profiler
>
>    Required Packages in Non-Standard Locations:
>      --with-broccoli=PATH       path to libbroccoli install root
> diff --git a/src/Index.hh b/src/Index.hh
> index 1befe08..3cf0397 100644
> --- a/src/Index.hh
> +++ b/src/Index.hh
> @@ -398,6 +398,9 @@ public:
>                  */
>          // returns 0 if lock was successfully achieved
>
> +        // 500000 came from testing. In the above commented out code, you
> can see that previously, gettimeofday was used.
> +        // Basically, I did a counter while it was doing gettimeofday,
> and found that it did around 500000 entries before trying
> +        // disk write lock. It was pretty consistent on two different
> machines.
>          if (num_of_entries < 500000)
>          {
>              num_of_entries++;
> diff --git a/src/IndexHash.cc b/src/IndexHash.cc
> index 613fcff..e37e6a5 100644
> --- a/src/IndexHash.cc
> +++ b/src/IndexHash.cc
> @@ -7,6 +7,8 @@
>   *
>   * The indexes in the hash table are calculated by doing key mod (hash
> table
>   *  size).
> + * The statement to prove is: number of buckets used = hash_table_size /
> GCF(hash_table_size, factor).
> + *
>   * We can note that there exists y / GCF(y, n) distinct instances of m *
> n (mod y) for all m.
>   * To see why this is true, we can let a = GCF(y, n).
>   * Let n be a particular key, and y = a * x and n = a * b
>
> _______________________________________________
> Time-Machine mailing list
> Time-Machine at mailman.ICSI.Berkeley.EDU <javascript:;>
> http://mailman.ICSI.Berkeley.EDU/mailman/listinfo/time-machine
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://mailman.ICSI.Berkeley.EDU/pipermail/time-machine/attachments/20140828/38dd3a17/attachment-0001.html 

From neto at caltech.edu  Thu Aug 28 18:53:18 2014
From: neto at caltech.edu (Naoki Eto)
Date: Thu, 28 Aug 2014 18:53:18 -0700
Subject: [TM] [git/tm] topic/naokieto/ipv6: This is my final commit before
	leaving lab for this year. (f984098)
Message-ID: <201408290153.s7T1rIxY019292@bro-ids.icir.org>

Repository : ssh://git at bro-ids.icir.org/time-machine

On branch  : topic/naokieto/ipv6

>---------------------------------------------------------------

commit f984098ab8aa5cd1f6a84c9cbf9ce56621f45efa
Author: NaokiEto <neto at caltech.edu>
Date:   Thu Aug 28 21:39:07 2014 -0400

    This is my final commit before leaving lab for this year.
    
    I fixed the hash table size changes that coverity found. Basically,
    I was not always guranteed to create a new hash table, so that has been
    fixed. Also, the number 1.82 that is in the code has some explanation, but
    may be refined after more experiments.


>---------------------------------------------------------------

f984098ab8aa5cd1f6a84c9cbf9ce56621f45efa
 src/Index.cc     | 20 +++++++++++++++++---
 src/IndexHash.cc |  5 ++++-
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/Index.cc b/src/Index.cc
index c354706..9b94b16 100644
--- a/src/Index.cc
+++ b/src/Index.cc
@@ -310,8 +310,11 @@ void Index<T>::addEntry(IndexField *iqe) {
 			/* Balance number of hash buckets */
 			/* Hash has twice as many buckets as entries. shrink.
 			 * yes, we want to compare the size of cur with the # entries of old (formerly new hash table) */
-           
-		    if (hash_size > 2.05*old->getNumEntries()) {
+            /* Why 2.24? The biggest ratio between consecutive numbers in the hash table size list was 29/13, which is 2.23. So, if the hash table
+             * size is more than 2.24 times bigger than the number of entries, we can safely shrink the hash table size, to the element before in the
+             * hash table size array. 
+             */
+		    if (hash_size > 2.24*old->getNumEntries()) {
               //if (hash_size < old->getNumEntries()) { 
                 // Note that we delete cur - this means we delete the formerly old hash table, which has been written to disk
                 //tmlog(TM_LOG_NOTE, "Index.cc", "we are about to delete the current (formerly old) hash table");
@@ -321,10 +324,18 @@ void Index<T>::addEntry(IndexField *iqe) {
                     //tmlog(TM_LOG_ERROR, "Index.cc:addEntry", "we are decreasing hash table size to %d and the number of entries in old hash is %d with %d buckets", hash_size_index - 1, old->getNumEntries(), old->getNumBuckets()); 
 				    cur = new IndexHash(hash_size_index - 1);
                 }
+                else
+                    cur = new IndexHash(0);
 			}
             
 			/* Hash has half as many buckets than entries. enlarge */
-            else if (1.95 * hash_size < old->getNumEntries()) {
+            /* UPDATED COMMENT: If 1.82 * hash_size is less than the number of entries, then enlarge the hash table (1.82 was chosen instead of 3/2 = 1.5 because 
+             * 1.82 is for the 53/29, which is the smallest and more likely to happen than 3/2. We enlarge it by two (approx factor of 4) via hash table
+             * size array. We do this because sometimes, the number of entries were observed to increase by a factor of 4 (tested via the numerous tmlogs you see
+             * littering this area of the code). Also, 1.82 worked better than simply 1 or 1.95 in terms of packet drops, it appears. More testing may be needed
+             * to determine the optimal number.
+             */
+            else if (1.82 * hash_size < old->getNumEntries()) {
 	    	//else if (1.9*hash_size < old->getNumEntries()) {
             //else if (hash_size > old->getNumEntries()) {
                 // Note that we delete cur - this means we delete the formerly old hash table, which has been written to disk
@@ -335,6 +346,9 @@ void Index<T>::addEntry(IndexField *iqe) {
                     //tmlog(TM_LOG_ERROR, "Index.cc:addEntry", "we are increasing hash table size to %d and the number of entries is %d with %d buckets ", hash_size_index + 2, old->getNumEntries(), old->getNumBuckets());
 				    cur = new IndexHash(hash_size_index + 2);
                 }
+                // Based on experimentation, it should never go here. Hash table sizes range at around 10,000->100,000 only.
+                else
+                    cur = new IndexHash(old->getNumEntries() + (old->getNumEntries() % 2 + 1));
                 /*
                 else
                 {
diff --git a/src/IndexHash.cc b/src/IndexHash.cc
index e37e6a5..66773f4 100644
--- a/src/IndexHash.cc
+++ b/src/IndexHash.cc
@@ -73,7 +73,10 @@ IndexHash::IndexHash(int size_index) {
     */
 	numEntries = 0;
 	numBucketsIndex = size_index;
-    numBuckets = Primes[size_index];
+    if (size_index < 43)
+        numBuckets = Primes[size_index];
+    else
+        numBuckets = size_index;
 
     htable = new hash_t[numBuckets];
 


From vallentin at icir.org  Fri Aug 29 06:52:44 2014
From: vallentin at icir.org (Matthias Vallentin)
Date: Fri, 29 Aug 2014 06:52:44 -0700
Subject: [TM] Questions about TM cutoff implementation
Message-ID: <20140829135244.GF4237@icir.org>

I'm in the process of adding flow cutoff functionality to VAST, in the
same vein as we have in the TM. Several questions arose during the
implementation, and the TM paper did not help me getting answers:

    - Scope: what exactly is the flow size? Because TM uses the
      connection 5-tuple,, I'd go for the cumulative transport-layer
      bytes. That is, for TCP/UDP the full payload size, and for ICMP
      the variable-size data portion. However, one could also imagine
      counting bytes starting at the network layer.

    - Directionality: is there a single cutoff value for both directions
      of the flow? Or does the cutoff mean N bytes per direction, so
      that the trace ends up with O(2N) bytes?

    - Eviction: how does TM evict old/stale entries? Naturally it makes
      sense to age out old entries upon connection termination. For TCP,
      it would be after seeing FIN or RST, and for ICMP maybe getting the
      response to a corresponding request. For UDP, there seems to be
      need for a timer-based approach.

      Moreover, the connection state table must not grow arbitrarily and
      be capped at some size. What is a typical number of concurrent
      flows at, say, UCB? In the case of scanning, adversarial activity,
      or more general any sudden increase in the flow count due to
      numerous (small) connections, we need to evict existing flow
      counters. Choosing a random entry would likely end up with one of
      the smaller flows being evicted, which seems like a robust
      stateless design. Alternatively, one could come up with a scheme
      that walks the flow table periodically (say every 100k packets)
      after it has reached its maximum size and then evict all entries
      less than a given threshold.

Any thoughts on the above points would be much appreciated.

     Matthias

From asharma at lbl.gov  Fri Aug 29 12:33:23 2014
From: asharma at lbl.gov (Aashish Sharma)
Date: Fri, 29 Aug 2014 12:33:23 -0700
Subject: [TM] Questions about TM cutoff implementation
In-Reply-To: <20140829135244.GF4237@icir.org>
References: <20140829135244.GF4237@icir.org>
Message-ID: <20140829193322.GC5319@yaksha.lbl.gov>

[Forwarding on behlaf of Naoki - to the greater list as well]

-------- Forwarded message ----------

Hi Matthias,

For  at  least  the  first point, the parts of the code to look at are Storage.cc and Fifo.cc . In Storage.cc, search for? c->getSuspendCutoff(), and in Fifo.cc, search for the Fifo::addPkt method. I wrote some comments about those parts of the code a while back in my code base. To answer your first question, it looks like tm does connection cut-off by packet bytes.  Specifically, the caplen parameter of pcap_pkthdr (look at Connection.cc or FifoMem.cc and search for? tot_pktbytes+=header->caplen ), the amount of data available during capture.

Best,
Naoki



On Fri, Aug 29, 2014 at 06:52:44AM -0700, Matthias Vallentin wrote:
> I'm in the process of adding flow cutoff functionality to VAST, in the
> same vein as we have in the TM. Several questions arose during the
> implementation, and the TM paper did not help me getting answers:
> 
>     - Scope: what exactly is the flow size? Because TM uses the
>       connection 5-tuple,, I'd go for the cumulative transport-layer
>       bytes. That is, for TCP/UDP the full payload size, and for ICMP
>       the variable-size data portion. However, one could also imagine
>       counting bytes starting at the network layer.
> 
>     - Directionality: is there a single cutoff value for both directions
>       of the flow? Or does the cutoff mean N bytes per direction, so
>       that the trace ends up with O(2N) bytes?
> 
>     - Eviction: how does TM evict old/stale entries? Naturally it makes
>       sense to age out old entries upon connection termination. For TCP,
>       it would be after seeing FIN or RST, and for ICMP maybe getting the
>       response to a corresponding request. For UDP, there seems to be
>       need for a timer-based approach.
> 
>       Moreover, the connection state table must not grow arbitrarily and
>       be capped at some size. What is a typical number of concurrent
>       flows at, say, UCB? In the case of scanning, adversarial activity,
>       or more general any sudden increase in the flow count due to
>       numerous (small) connections, we need to evict existing flow
>       counters. Choosing a random entry would likely end up with one of
>       the smaller flows being evicted, which seems like a robust
>       stateless design. Alternatively, one could come up with a scheme
>       that walks the flow table periodically (say every 100k packets)
>       after it has reached its maximum size and then evict all entries
>       less than a given threshold.
> 
> Any thoughts on the above points would be much appreciated.
> 
>      Matthias
> _______________________________________________
> Time-Machine mailing list
> Time-Machine at mailman.ICSI.Berkeley.EDU
> http://mailman.ICSI.Berkeley.EDU/mailman/listinfo/time-machine

-- 
Aashish Sharma	(asharma at lbl.gov) 				 
Cyber Security, 
Lawrence Berkeley National Laboratory  
http://go.lbl.gov/pgp-aashish 
Office: (510)-495-2680  Cell: (510)-612-7971

From vallentin at icir.org  Fri Aug 29 21:09:37 2014
From: vallentin at icir.org (Matthias Vallentin)
Date: Fri, 29 Aug 2014 21:09:37 -0700
Subject: [TM] Questions about TM cutoff implementation
In-Reply-To: <20140829193322.GC5319@yaksha.lbl.gov>
References: <20140829135244.GF4237@icir.org>
	<20140829193322.GC5319@yaksha.lbl.gov>
Message-ID: <20140830040937.GG4237@icir.org>

> tot_pktbytes+=header->caplen ), the amount of data available during
> capture.

Thanks for providing the entry points. It looks like the TM definition
of "flow cutoff" is to include the full size (header->caplen) as opposed
to just transport-layer payload bytes. It's a probably a matter of
taste, but I find it more intuitive to think about cutoff as
transport-layer bytes because the cutoff has fundamentally to do with a
connection on not packets. Conversely, artifacts such as TCP
retransmissions make it more complicated to reason about it in this way.

     Matthias

From vern at cs.berkeley.edu  Sat Aug 30 10:36:16 2014
From: vern at cs.berkeley.edu (vern at cs.berkeley.edu)
Date: Sat, 30 Aug 2014 10:36:16 -0700
Subject: [TM] Questions about TM cutoff implementation
In-Reply-To: <20140829135244.GF4237@icir.org> (Fri, 29 Aug 2014 06:52:44 PDT).
Message-ID: <20140830173616.B1BE92C4010@rock.ICSI.Berkeley.EDU>

>     - Scope: what exactly is the flow size? Because TM uses the
>       connection 5-tuple,, I'd go for the cumulative transport-layer
>       bytes. That is, for TCP/UDP the full payload size, and for ICMP
>       the variable-size data portion. However, one could also imagine
>       counting bytes starting at the network layer.

The nice thing about heavy tails is that this sort of thing doesn't tend
to matter.  Just pick a definition that's convenient and you'll get pretty
much the same results as for other (reasonable) definitions.

>     - Directionality: is there a single cutoff value for both directions
>       of the flow? Or does the cutoff mean N bytes per direction, so
>       that the trace ends up with O(2N) bytes?

It would be good to do this per-direction, since it can often happen that
one direction exhibits a heavy tail but the other doesn't.  In that case,
you can retain more information about the connection (in particular,
control messages) by managing the directions separately.

>     - Eviction: how does TM evict old/stale entries? Naturally it makes
>       sense to age out old entries upon connection termination. For TCP,
>       it would be after seeing FIN or RST, and for ICMP maybe getting the
>       response to a corresponding request. For UDP, there seems to be
>       need for a timer-based approach.

You should say a bit more about what's meant by eviction or old/stale
entries.  Is this regarding evicting in-memory state, or on-disk?  For the
former, the heavy-tailed premise behind the Time Machine says it's fine
to flush something that hasn't done anything for a while.

>       Moreover, the connection state table must not grow arbitrarily and
>       be capped at some size. What is a typical number of concurrent
>       flows at, say, UCB?

Here you need to tease apart pending flows from active flows, since
there can be many more of the former due to scanning and backscatter.
That's what motivated the "connection compressor", per
http://www.icir.org/vern/papers/high-volume-ccs04.pdf .
I believe the number of concurrent established/productive flows isn't
very high (since most flows tends to be quite short-lived), but
I don't have an order of magnitude figure handy.

> In the case of scanning, adversarial activity,
>       or more general any sudden increase in the flow count due to
>       numerous (small) connections, we need to evict existing flow
>       counters.

Why do you need to evict these early?  What's the bottleneck stressor
that emerges if you don't?  Is this about managing the flow table itself?

		Vern

From vallentin at icir.org  Sun Aug 31 09:52:52 2014
From: vallentin at icir.org (Matthias Vallentin)
Date: Sun, 31 Aug 2014 09:52:52 -0700
Subject: [TM] Questions about TM cutoff implementation
In-Reply-To: <20140830173616.B1BE92C4010@rock.ICSI.Berkeley.EDU>
References: <20140829135244.GF4237@icir.org>
	<20140830173616.B1BE92C4010@rock.ICSI.Berkeley.EDU>
Message-ID: <20140831165252.GJ4237@icir.org>

> It would be good to do this per-direction, since it can often happen that
> one direction exhibits a heavy tail but the other doesn't. 

Asymmetry is a very good point, I've switched to this notion.

> You should say a bit more about what's meant by eviction or old/stale
> entries.  

Here, I mean the hash table that tracks per-connection state for
deciding when the cutoff has been reached. The state encompasses (i) the
last time a flow has been active and (ii) it's cumulative size. An entry
occupies o(8+8) bytes of state plus the size of the 5-tuple o(8+8+2+2+1)
in total, yielding o(37) bytes per entry modulo padding by the compiler.
I've set the default maximum table size to 1M entries.

> > In the case of scanning, adversarial activity,
> >       or more general any sudden increase in the flow count due to
> >       numerous (small) connections, we need to evict existing flow
> >       counters.
> 
> Why do you need to evict these early?  What's the bottleneck stressor
> that emerges if you don't?  Is this about managing the flow table itself?

The reason why I thought they should be evicted early, is that in the
case of scanning the corresponding flow table entries don't constitute
active connections and just make it reach its maximum size faster. Once
the maximum has been reached, I evict random entries. So I was thinking
it's perhaps worthwhile to reduce the probability of evicting active
flows by evicting inactive ones earlier.

But since you state that the number of active connections is much lower
and pending ones, the approach I currently take already works and
doesn't need any change: randomly evicting flow table elements will bite
the inactive flows with higher probability. (Even if we evicted an
active flow, the worst case is that we'll record twice the specified
cutoff.)

     Matthias