all options moved to demux object

ChangeLog

@@ -1,5 +1,7 @@
 2012-10-26  Simson Garfinkel  <[email protected]>
 
+	* src/flow.cpp (flow::print_usage): shortened filename template usage.
+
 	* src/tcpip.cpp (insert): added this function to insert a space at the beginning of a file. Now tcpflow can handle a flow with missing SYNs and out-of-order packets.
 
 2012-10-25  Simson Garfinkel  <[email protected]>

TODO.txt

@@ -1,8 +1,13 @@
-* discover and create MIME objects.
-* Run a subprocess when each file is opened or closed.
-* Regression testing by randomizing packet order and making sure that the results are the same.
-* update tcpip structure to indicate if a SYN was seen; If packets arrive before the beginning of the connection and a SYN was not seen, insert in the beginning of the file.
-* remove syn_set from store_packet. Make sure that it's called when we know the packet offset.
+Accomplished for 1.4:
+
++ update tcpip structure to indicate if a SYN was seen; If packets arrive before the beginning of the connection and a SYN was not seen, insert in the beginning of the file.
++ remove syn_set from store_packet. Make sure that it's called when we know the packet offset.
+
+================================================================
+
+- discover and create MIME objects.
+- Run a subprocess when each file is opened or closed.
+- Regression testing by randomizing packet order and making sure that the results are the same.
 ================
 Here is my currently plan for the plugin approach:
 

src/Makefile.am

@@ -1,7 +1,14 @@
+# Programs that we compile:
 bin_PROGRAMS = tcpflow
+
+# bulk_extractor 1.3 API:
+
+BE13_API = bulk_extractor_i.h cppmutex.h feature_recorder.h feature_recorder_set.h md5.h sbuf.h sbuf_flex_scanner.h sbuf_private.h sbuf_stream.h utf8.h utf8/checked.h utf8/core.h utf8/unchecked.h xml.h 
+
 tcpflow_SOURCES = datalink.cpp flow.cpp \
 	pcap_fake.h pcap_fake.cpp \
 	main.cpp \
 	tcpip.cpp tcpdemux.h tcpdemux.cpp tcpflow.h xml.h xml.cpp util.cpp \
-	md5.h md5.c iface_pcb.cpp iface_pcb.h pcb_timehistogram.cpp
+	md5.h md5.c iface_pcb.cpp iface_pcb.h pcb_timehistogram.cpp \
+	$(BE13_API)
 

src/flow.cpp

@@ -31,17 +31,12 @@ std::string flow::filename_template("%A.%a-%B.%b%V%v%C%c");
 void flow::print_usage()
 {
     std::cerr << "Filename template format:\n";
-    std::cerr << "  %A - source IP address\n";
-    std::cerr << "  %a - source IP port\n";
-    std::cerr << "  %B - dest IP address\n";
-    std::cerr << "  %b - dest IP port\n";
-    std::cerr << "  %T - Timestamp in ISO8601 format\n";
-    std::cerr << "  %t - Unix time_t\n";
-    std::cerr << "  %V - '--' if VLAN is present\n";
-    std::cerr << "  %v - VLAN number if VLAN is present\n";
+    std::cerr << "  %A/%a - source IP address/port\n";
+    std::cerr << "  %B/%b - dest IP address/port\n";
+    std::cerr << "  %T/%t - Timestamp in ISO8601 format/unix time_t\n";
+    std::cerr << "  %V/%v - VLAN number, '--' if no vlan/'' if no vlan\n";
+    std::cerr << "  %c/%# - connection_count; if >0/always \n";
     std::cerr << "  %C - 'c' if connection_count >0\n";
-    std::cerr << "  %c - connection_count if connection_count >0 \n";
-    std::cerr << "  %# - always output connection count\n";
     std::cerr << "  %% - Output a '%'\n";
     std::cerr << "\n";
     std::cerr << "Default filename template is " << filename_template << "\n";

src/main.cpp

@@ -2,8 +2,8 @@
  * This file is part of tcpflow by Simson Garfinkel <[email protected]>.
  * Originally by Jeremy Elson <[email protected]>.
  *
- * This source code is under the GNU Public License (GPL).  See
- * LICENSE for details.
+ * This source code is under the GNU Public License (GPL) version 3.
+ * See COPYING for details.
  *
  */
 
@@ -12,19 +12,14 @@
 #include "tcpflow.h"
 #include <vector>
 
-#define ENABLE_GZIP 0
-
-int debug_level = DEFAULT_DEBUG_LEVEL;
-int max_flows = 0;
-bool console_output = false;
-int suppress_header = 0;
-int strip_nonprint = 0;
-int use_color = 0;
-int max_seek  = 1024*1024*16;
-bool opt_no_purge = false;
-
+/* System-wide options.
+ * Note that most options are now in the individual demux object
+ * This allows a demux to be a class in another program.
+ */
+
 const char *progname = 0;
 
+int debug_level = DEFAULT_DEBUG_LEVEL;
 extern std::string xml_PRId64;
 extern std::string xml_PRIu64;
 
@@ -63,17 +58,15 @@ void print_usage()
     std::cerr << "        -o outdir   : specify output directory (default '.')\n";
     std::cerr << "        -X filename : DFXML output to filename\n";
     std::cerr << "        -m bytes    : specifies the minimum number of bytes that a stream may\n";
-    std::cerr << "                      skip before starting a new stream (default " << max_seek << ").\n";
+    std::cerr << "                      skip before starting a new stream (default " << (unsigned)tcpdemux::options::MAX_SEEK << ").\n";
     std::cerr << "        -AH : extract HTTP objects and unzip GZIP-compressed HTTP messages\n";
     std::cerr << "        -Fc : append the connection counter to ALL filenames\n";
     std::cerr << "        -Ft : prepend the time_t timestamp to ALL filenames\n";
     std::cerr << "        -FT : prepend the ISO8601 timestamp to ALL filenames\n";
     std::cerr << "        -FX : Do not output any files (other than report files)\n";
     std::cerr << "        -FM : Calculate the MD5 for every flow\n";
     std::cerr << "        -T<template> : specify an arbitrary filename template (default " << flow::filename_template << ")\n";
-#if ENABLE_GZIP
     std::cerr << "        -Z: do not decompress gzip-compressed HTTP transactions\n";
-#endif
     std::cerr << "Depricated: (don't use)\n";
     std::cerr << "        -P: don't purge tcp connections on FIN (could result in lost data)\n";
     std::cerr << "expression: tcpdump-like filtering expression\n";
@@ -157,42 +150,42 @@ int main(int argc, char *argv[])
     while ((arg = getopt(argc, argv, "aA:Bb:cCd:eF:f:hi:L:m:o:PpR:r:sT:VvX:Z")) != EOF) {
 	switch (arg) {
 	case 'a':
-	    demux.opt_after_header = true;
-	    demux.opt_md5 = true;
+	    demux.opt.opt_after_header = true;
+	    demux.opt.opt_md5 = true;
 	    opt_all = true;
 	    continue;
 
 	case 'A': 
 	    for(const char *cc=optarg;*cc;cc++){
 		switch(*cc){
-		case 'H': demux.opt_after_header = true;break;
+		case 'H': demux.opt.opt_after_header = true;break;
 		default:
 		    fprintf(stderr,"-A invalid after processing '%c'\n",*cc);
 		    need_usage=true;
 		}
 	    }
 	    break;
 	case 'b':
-	    if ((demux.max_bytes_per_flow = atoi(optarg)) < 0) {
+	    if ((demux.opt.max_bytes_per_flow = atoi(optarg)) < 0) {
 		DEBUG(1) ("warning: invalid value '%s' used with -b ignored", optarg);
-		demux.max_bytes_per_flow = 0;
+		demux.opt.max_bytes_per_flow = 0;
 	    } else {
 		if(debug_level > 1) {
-		    std::cout << "capturing max of " << demux.max_bytes_per_flow << " bytes per flow." << std::endl;
+		    std::cout << "capturing max of " << demux.opt.max_bytes_per_flow << " bytes per flow." << std::endl;
 		}
 	    }
 	    break;
 	case 'B':
 	    force_binary_output = true; DEBUG(10) ("force binary output");
 	    break;
 	case 'C':
-	    console_output = true;	DEBUG(10) ("printing packets to console only");
-	    suppress_header = 1;	DEBUG(10) ("packet header dump suppressed");
-	    strip_nonprint = 1;		DEBUG(10) ("converting non-printable characters to '.'");
+	    demux.opt.console_output = true;	DEBUG(10) ("printing packets to console only");
+	    demux.opt.suppress_header = 1;	DEBUG(10) ("packet header dump suppressed");
+	    demux.opt.strip_nonprint = 1;		DEBUG(10) ("converting non-printable characters to '.'");
 	    break;
 	case 'c':
-	    console_output = true;	DEBUG(10) ("printing packets to console only");
-	    strip_nonprint = 1;		DEBUG(10) ("converting non-printable characters to '.'");
+	    demux.opt.console_output = true;	DEBUG(10) ("printing packets to console only");
+	    demux.opt.strip_nonprint = 1;		DEBUG(10) ("converting non-printable characters to '.'");
 	    break;
 	case 'd':
 	    if ((debug_level = atoi(optarg)) < 0) {
@@ -206,19 +199,19 @@ int main(int argc, char *argv[])
 		case 'c': replace(flow::filename_template,"%c","%C"); break;
 		case 't': flow::filename_template = "%tT" + flow::filename_template; break;
 		case 'T': flow::filename_template = "%T"  + flow::filename_template; break;
-		case 'X': demux.opt_output_enabled = false;break;
-		case 'M': demux.opt_md5 = true;break;
+		case 'X': demux.opt.opt_output_enabled = false;break;
+		case 'M': demux.opt.opt_md5 = true;break;
 		default:
 		    fprintf(stderr,"-F invalid format specification '%c'\n",*cc);
 		    need_usage = true;
 		}
 	    }
 	    break;
 	case 'f':
-	    if ((demux.max_desired_fds = atoi(optarg)) < (NUM_RESERVED_FDS + 2)) {
+	    if ((demux.opt.max_desired_fds = atoi(optarg)) < (NUM_RESERVED_FDS + 2)) {
 		DEBUG(1) ("warning: -f flag must be used with argument >= %d",
 			  NUM_RESERVED_FDS + 2);
-		demux.max_desired_fds = 0;
+		demux.opt.max_desired_fds = 0;
 	    }
 	    break;
 	case 'h':
@@ -228,13 +221,13 @@ int main(int argc, char *argv[])
 	case 'i': device = optarg; break;
 	case 'L': lockname = optarg; break;
 	case 'm':
-	    max_seek = atoi(optarg);
-	    DEBUG(10) ("max_seek set to %d",max_seek); break;
+	    demux.opt.max_seek = atoi(optarg);
+	    DEBUG(10) ("max_seek set to %d",demux.opt.max_seek); break;
 	case 'o': demux.outdir = optarg; break;
 	case 'P': 
-	    opt_no_purge = true;
+	    demux.opt.opt_no_purge = true;
 	    break;
-	case 'p': demux.opt_no_promisc = true;
+	case 'p': demux.opt.opt_no_promisc = true;
 	    DEBUG(10) ("NOT turning on promiscuous mode");
 	    break;
 	case 'R':
@@ -244,13 +237,13 @@ int main(int argc, char *argv[])
 	    rfiles.push_back(optarg);
 	    break;
 	case 's':
-	    strip_nonprint = 1;		DEBUG(10) ("converting non-printable characters to '.'"); break;
+	    demux.opt.strip_nonprint = 1;		DEBUG(10) ("converting non-printable characters to '.'"); break;
 	case 'T': flow::filename_template = optarg;break;
 	case 'V': std::cout << PACKAGE << " " << PACKAGE_VERSION << "\n"; exit (1);
 	case 'v': debug_level = 10; break;
-	case 'Z': demux.opt_gzip_decompress = 0; break;
+	case 'Z': demux.opt.opt_gzip_decompress = 0; break;
 	case 'e':
-	    use_color  = 1;
+	    demux.opt.use_color  = 1;
 	    DEBUG(10) ("using colors");
 	    break;
 	case 'X': reportfilename = optarg;break;
@@ -291,7 +284,7 @@ int main(int argc, char *argv[])
     }
 
     if(force_binary_output){
-	strip_nonprint = false;
+	demux.opt.strip_nonprint = false;
     }
 
     /* make sure outdir is a directory. If it isn't, try to make it.*/

src/tcpdemux.cpp

@@ -191,9 +191,9 @@ unsigned int tcpdemux::get_max_fds(void)
 #endif
 
     /* this must go here, after rlimit code */
-    if (max_desired_fds) {
-	DEBUG(10) ("using only %d FDs", max_desired_fds);
-	return max_desired_fds;
+    if (opt.max_desired_fds) {
+	DEBUG(10) ("using only %d FDs", opt.max_desired_fds);
+	return opt.max_desired_fds;
     }
 
     DEBUG(10) ("found max FDs to be %d using %s", max_descs, method);
@@ -205,11 +205,8 @@ tcpdemux::tcpdemux():outdir("."),flow_counter(0),packet_counter(0),
 		     xreport(0),max_fds(10),flow_map(),
 		     start_new_connections(false),
 		     openflows(),
-		     opt_output_enabled(true),opt_md5(false),
-		     opt_after_header(false),opt_gzip_decompress(true),
-		     opt_no_promisc(false),
-		     max_bytes_per_flow(),
-		     max_desired_fds()
+		     opt()
+
 {
     /* Find out how many files we can have open safely...subtract 4 for
      * stdin, stdout, stderr, and the packet filter; one for breathing
@@ -324,7 +321,7 @@ void tcpdemux::process_tcp(const struct timeval &ts,const u_char *data, uint32_t
 	 */
 	delta = seq - tcp->nsn;		// notice that signed offset is calculated
 
-	if(abs(delta) > max_seek){
+	if(abs(delta) > opt.max_seek){
 	    connection_count = tcp->myflow.connection_count+1;
 	    remove_flow(this_flow);
 	    tcp = 0;
@@ -385,18 +382,18 @@ void tcpdemux::process_tcp(const struct timeval &ts,const u_char *data, uint32_t
      * since they both have no data by definition.
      */
     if (length>0){
-	if (console_output) {
+	if (opt.console_output) {
 	    tcp->print_packet(data, length);
 	} else {
-	    if (opt_output_enabled){
+	    if (opt.opt_output_enabled){
 		tcp->store_packet(data, length, delta);
 	    }
 	}
     }
 
     /* Finally, if there is a FIN, then kill this TCP connection*/
     if (IS_SET(tcp_header->th_flags, TH_FIN)){
-	if(opt_no_purge==false){
+	if(opt.opt_no_purge==false){
 	    DEBUG(50)("packet is FIN; closing connection");
 	    remove_flow(this_flow);	// take it out of the map
 	}
@@ -604,7 +601,7 @@ void tcpdemux::process_infile(const std::string &expression,const char *device,c
 	}
 
 	/* make sure we can open the device */
-	if ((pd = pcap_open_live(device, SNAPLEN, !opt_no_promisc, 1000, error)) == NULL){
+	if ((pd = pcap_open_live(device, SNAPLEN, !opt.opt_no_promisc, 1000, error)) == NULL){
 	    die("%s", error);
 	}
 #if defined(HAVE_SETUID) && defined(HAVE_GETUID)

src/tcpdemux.h

@@ -34,6 +34,8 @@ typedef unsigned short int sa_family_t;
 #define __BYTE_ORDER __LITTLE_ENDIAN
 #endif
 
+extern int	debug_level;
+
 /*
  * Structure of an internet header, naked of options.
  */
@@ -365,16 +367,41 @@ class tcpdemux {
 	}
     };
     tcpdemux(const tcpdemux &t):outdir("."),flow_counter(),packet_counter(),xreport(),
-				max_fds(),flow_map(),start_new_connections(),openflows(),opt_output_enabled(),
-				opt_md5(),opt_after_header(),opt_gzip_decompress(),
-				opt_no_promisc(),
-				max_bytes_per_flow(),max_desired_fds(){
+				max_fds(),flow_map(),start_new_connections(),openflows(),opt(){
 	throw new not_impl();
     }
     tcpdemux &operator=(const tcpdemux &that){
 	throw new not_impl();
     }
 public:
+    /* The pure options class means we can add new options without having to modify the tcpdemux constructor. */
+
+    class options {
+    public:;
+	enum { MAX_SEEK=1024*1024*16 };
+	options():console_output(false),opt_output_enabled(true),opt_md5(false),
+		  opt_after_header(false),opt_gzip_decompress(true),
+		  opt_no_promisc(false),max_bytes_per_flow(),
+		  max_desired_fds(),max_flows(0),suppress_header(0),
+		  strip_nonprint(),use_color(0),max_seek(MAX_SEEK),
+		  opt_no_purge(false) {
+	}
+	bool	console_output;
+	bool	opt_output_enabled;	// do we output?
+	bool	opt_md5;		// do we calculate MD5 on DFXML output?
+	bool	opt_after_header;	// decode headers after tcp connection closes
+	bool	opt_gzip_decompress;
+	bool	opt_no_promisc;		// do not be promiscious
+	uint64_t max_bytes_per_flow;
+	int	max_desired_fds;
+	int	max_flows;
+	int	suppress_header;
+	int	strip_nonprint;
+	int	use_color;
+	uint32_t max_seek;
+	bool	opt_no_purge;
+    };
+
     typedef std::tr1::unordered_set<class tcpip *> tcpset;
     typedef std::tr1::unordered_map<flow_addr,tcpip *,flow_addr_hash,flow_addr_key_eq> flow_map_t; // should be unordered_map
     std::string outdir;		/* output directory */
@@ -386,13 +413,7 @@ class tcpdemux {
     flow_map_t	flow_map;		// the database
     bool	start_new_connections;	// true if we should start new connections
     tcpset	openflows;		// the tcpip flows with open FPs 
-    bool	opt_output_enabled;	// do we output?
-    bool	opt_md5;		// do we calculate MD5 on DFXML output?
-    bool	opt_after_header;	// decode headers after tcp connection closes
-    bool	opt_gzip_decompress;
-    bool	opt_no_promisc;		// do not be promiscious
-    uint64_t	max_bytes_per_flow;
-    int		max_desired_fds;
+    options	opt;
 
     tcpdemux();
     void write_to_file(std::stringstream &ss,

src/tcpflow.h

@@ -302,12 +302,6 @@ void flow_close_all();
 
 /* main.cpp - CLI */
 extern const char *progname;
-extern bool console_output;
-extern int suppress_header;
-extern int strip_nonprint;
-extern int use_color;
-extern int max_seek;			// int and not u_int because abs() returns an int
-extern bool opt_no_purge;
 
 #ifdef HAVE_PTHREAD
 #include <semaphore.h>