[Bro] Fwd: File Extraction

[al brocino]

Thanks Johanna,

I made your recommended change and am still getting the error, see detail
below:

file-extract.bro script
> global ext_map:table[string] of string = { ["application/x/dosexec"] =
> "exe",
you probably want application/x-dosexec here, not x/dosexec. That might
already be enough to fix this.

Changed:

file-extract.bro
global ext_map: table[string] of string = {
    ["application/x-dosexec"] = "exe",
    ["text/plain"] = "txt",
    ["image/jpeg"] = "jpg",
    ["image/png"] = "png",
    ["text/html"] = "html",
} &default ="";

Uncomment #@load ./file-extract-http-local.bro and #@load
./file-extract-types.bro:

_load_.bro
# File extractions (/application\/.*) -- This has changed significantly in
2.2
@load ./file-extract-http-local.bro
@load ./file-extract-types.bro
@load ./bro-file-extract

 I get this error again:

 manager scripts failed.
   internal warning in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 6: Discarded extraneous
Broxygen comment: Modified from base scripts to extract only from external
hosts
   fatal error in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 7: can't find
base/protocols/http/file-ident
proxy scripts failed.
   internal warning in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 6: Discarded extraneous
Broxygen comment: Modified from base scripts to extract only from external
hosts
   fatal error in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 7: can't find
base/protocols/http/file-ident
enm1-eth1-httpproxy scripts failed.
   internal warning in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 6: Discarded extraneous
Broxygen comment: Modified from base scripts to extract only from external
hosts
   fatal error in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 7: can't find
base/protocols/http/file-ident
enm2-eth2-httpinternal scripts failed.
   internal warning in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 6: Discarded extraneous
Broxygen comment: Modified from base scripts to extract only from external
hosts
   fatal error in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 7: can't find
base/protocols/http/file-ident
enm3-eth3-collector scripts failed.
   internal warning in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 6: Discarded extraneous
Broxygen comment: Modified from base scripts to extract only from external
hosts
   fatal error in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 7: can't find
base/protocols/http/file-ident
enm4-eth5-dns scripts failed.
   internal warning in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 6: Discarded extraneous
Broxygen comment: Modified from base scripts to extract only from external
hosts
   fatal error in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 7: can't find
base/protocols/http/file-ident
enm5-eth6-syslog scripts failed.
   internal warning in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 6: Discarded extraneous
Broxygen comment: Modified from base scripts to extract only from external
hosts
   fatal error in /usr/local/bro/share/bro/site/
./custom/./file-extract-http-local.bro, line 7: can't find
base/protocols/http/file-ident

Here's the script that it's failing on:

file-extract-http-local.bro
@load base/protocols/http/main
@load base/protocols/http/file-ident
@load base/utils/files
 module HTTP;
 export {
                ## Pattern of file mime types to extract from HTTP response
entity bodies.
                const extract_file_types_local = /NO_DEFAULT/ &redef;
                ## The on-disk prefix for files to be extracted from HTTP
entity bodies.
                const extraction_prefix_local = "http-item" &redef;

                redef record Info += {
                                ## On-disk file where the response body was
extracted to.
                                extraction_file_local:  file &log &optional;

                                ## Indicates if the response body is to be
extracted or not.  Must be
                                ## set before or by the first
:bro:id:`http_entity_data` event for the
                                ## content.
                                extract_file_local:     bool &default=F;
                };
}

# Define local sources to ignore file extract
global http_extract_file_ignore: set[subnet] = {
                                                172.16.0.0/12,          #
Internal FRS, trusted destination
                                                10.0.0.0/8,             #
Internal FRS, trusted destination
};


event http_entity_data(c: connection, is_orig: bool, length: count, data:
string) &priority=-5
                {
                # Client body extraction is not currently supported in this
script.
                if ( is_orig )
                                return;

                # We do not want to extract files from internal to internal
hosts
                if ( c$id$resp_h in http_extract_file_ignore )
                                return;

                if ( c$http$first_chunk )
                                {
                                if ( c$http?$mime_type &&
                                     extract_file_types_local in
c$http$mime_type )
                                                {
                                                c$http$extract_file_local =
T;
                                                }

                                if ( c$http$extract_file_local )
                                                {
                                                local suffix =
fmt("%s_%d.dat", is_orig ? "orig" : "resp", c$http_state$current_response);
                                                local fname =
generate_extraction_filename(extraction_prefix_local, c, suffix);


c$http$extraction_file_local = open(fname);
                                                enable_raw_output(c$http$
extraction_file_local);
                                                }
                                }

                if ( c$http?$extraction_file_local )
                                print c$http$extraction_file_local, data;
                }

event http_end_entity(c: connection, is_orig: bool)
                {
                if ( c$http?$extraction_file_local )
                                close(c$http$extraction_file_local);
                }
Ideas? Thanks!
-------------- next part --------------
An HTML attachment was scrubbed...
URL: http://mailman.ICSI.Berkeley.EDU/pipermail/bro/attachments/20160826/d83a6cc9/attachment-0001.html