[Bro] http-ext-identified-files

[Will]

On Wed, May 11, 2011 at 12:54 PM, Aashish SHARMA <aashish043 at gmail.com> wrote:
> Hello:
>
> HTTP_WatchedMIMEType is declared in bro/share/bro/http-identified-files.bro.
>
> I think you can make the code work by doing the following changes in the http-ext-identified-files.bro
>
> 1) Load http-identified-files
> 2) change "const" to "redef" for the following variables: watched_mime_types, ignored_urls, mime_types_extensions, ignored_signatures
> 3) Comment out declaration of HTTP_IncorrectFileType from http-ext-identified-files.bro
>
>
> + @load http-identified-files
>
> -       redef enum Notice += {
> -               # This notice is thrown when the file extension doesn't
> -               # seem to match the file contents.
> -               HTTP_IncorrectFileType,
> -       };
>
> -       const watched_mime_types = /application\/x-dosexec/
> +       redef watched_mime_types = /application\/x-dosexec/
>
>
> -       const ignored_urls = /^http:\/\/(au\.|www\.)?download\.windowsupdate\.com\/msdownload\/update/ &redef;
> +       redef ignored_urls = /^http:\/\/(au\.|www\.)?download\.windowsupdate\.com\/msdownload\/update/ ;
>
>
> -       redef mime_types_extensions: table[string] of pattern = {
> +       const mime_types_extensions: table[string] of pattern = {
>
>
> - const ignored_signatures += /^matchfile-/ &redef;
> + redef ignored_signatures += /^matchfile-/;
>
> Aashish
>
> On May 11, 2011, at 6:18 AM, Seth Hall wrote:
>
>> Sorry for not reply earlier.  I started a response to your email and never finished it. :)
>>
>> On Apr 1, 2011, at 2:20 PM, Will wrote:
>>
>>> 1. The old way of flagging via 'HTTP_WatchedMIMEType' appears to have gone away
>>
>> Hm, I wonder why I removed that?  There will be a solution for this problem in the next release.
>>
>> Did you end up figuring out what was wrong with this?
>>

Yes, pretty close to what Aashish describes to do above. Though I
don't see what changing the ignored_signatures file does, because it
already looks redef'd. Our "whitelist" is larger and slightly more
custom to our environment, but otherwise just as below. The
mis-matched file type is great for when a file is down loaded with a
random string and doesn't have a "watched" mime type, i.e. a php file
named "WJ4JR874".

Here is what we are using and seems to be working seemlessly:

@load global-ext
@load http-ext
@load http-reply
@load http-body
@load signatures
redef signature_files += "http-ext-identified-files.sig";

module HTTP;

export {
        redef enum Notice += {
                # This notice is thrown when the file extension doesn't
                # seem to match the file contents.
                HTTP_IncorrectFileType,

                # Generated when we see a MIME type we flagged for watching.
                HTTP_WatchedMIMEType,
        };

        # MIME types that you'd like this script to identify and log.
        const watched_mime_types = /application\/x-dosexec/
                                | /application\/x-executable/
                                | /application\/octet-stream/
                                | /application\/x-compressed/
                                | /application\/x-msdownload/ &redef;

        # URLs included here are not logged and notices are not thrown.
        # Take care when defining regexes to not be overly broad.
        const ignored_urls =
/^http:\/\/(au\.|www\.)?download\.windowsupdate\.com\/msdownload\/update/
                                | /^http:\/\/.*\.adobe\.com\//
                                | /^http:\/\/.*\.cisco\.com\//
                                | /^http:\/\/.*\.hp\.com\//
                                | /^http:\/\/.*\.macromedia\.com\//
                                | /^http:\/\/.*\.microsoft\.com\//
                                | /^http:\/\/.*\.sun\.com\// &redef;

        # Create regexes that *should* in be in the urls for specifics
mime types.
        # Notices are thrown if the pattern doesn't match the url for
the file type.
        const mime_types_extensions: table[string] of pattern = {
                ["application/x-dosexec"] = /\.([eE][xX][eE]|[dD][lL][lL])/,
        } &redef;
}

# Don't delete the http sessions at the end of the request!
redef watch_reply=T;

# Ignore the signatures used to match files
redef ignored_signatures += /^matchfile-/;

# This script uses the file tagging method to create a separate file.
event bro_init()
        {
        # Add the tag for log file splitting.
        LOG::define_tag("http-ext", "identified-files");
        }

event signature_match(state: signature_state, msg: string, data: string)
        {
        # Only signatures matching file types are dealt with here.
        if ( /^matchfile/ !in state$id ) return;

        # Not much point in any of this if we don't know about the
        # HTTP-ness of the connection.
        if ( state$conn$id !in conn_info ) return;

        local si = conn_info[state$conn$id];
        # Set the mime type seen.
        si$mime_type = msg;
        local defanged_url = gsub(si$url, /\./, "[.]");
        local message = fmt("%s %s", msg, defanged_url);
        if ( ignored_urls !in si$url )
                {
                if ( watched_mime_types in msg )
                        {
                        NOTICE([$note=HTTP_WatchedMIMEType,
$msg=message, $conn=state$conn, $method=si$method, $URL=si$url]);
                        # Add a tag for logging purposes.
                        add si$tags["identified-files"];
                        }

                if ( msg in mime_types_extensions &&
                     mime_types_extensions[msg] !in si$url )
                        {
                        NOTICE([$note=HTTP_IncorrectFileType,
$msg=message, $conn=state$conn, $method=si$method, $URL=si$url]);
                        }

                event file_transferred(state$conn, data, "", msg);
                }
        }

Thanks to both!

-Will

>>  .Seth
>>
>> --
>> Seth Hall
>> International Computer Science Institute
>> (Bro) because everyone has a network
>> http://www.bro-ids.org/
>>
>>
>> _______________________________________________
>> Bro mailing list
>> bro at bro-ids.org
>> http://mailman.ICSI.Berkeley.EDU/mailman/listinfo/bro
>
>