cron/eventStats.py

#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation

import json
import os
import sys
from datetime import datetime
from configlib import getConfig, OptionParser
from time import sleep
import socket

from mozdef_util.utilities.logger import logger
from mozdef_util.utilities.toUTC import toUTC
from mozdef_util.elasticsearch_client import ElasticsearchClient
from mozdef_util.query_models import SearchQuery, Aggregation


def esSearch(es):
    search_query = SearchQuery(minutes=options.aggregationminutes)
    search_query.add_aggregation(Aggregation('category'))
    results = search_query.execute(es)

    mozdefstats = dict(utctimestamp=toUTC(datetime.now()).isoformat())
    mozdefstats['category'] = 'stats'
    mozdefstats['hostname'] = socket.gethostname()
    mozdefstats['mozdefhostname'] = mozdefstats['hostname']
    mozdefstats['type'] = 'mozdefstats'
    mozdefstats['severity'] = 'INFO'
    mozdefstats['source'] = 'mozdef'
    mozdefstats['tags'] = ['mozdef', 'stats']
    mozdefstats['summary'] = 'Aggregated category counts'
    mozdefstats['processid'] = os.getpid()
    mozdefstats['processname'] = sys.argv[0]
    mozdefstats['details'] = dict(counts=list())
    for bucket in results['aggregations']['category']['terms']:
        entry = dict()
        entry[bucket['key']] = bucket['count']
        mozdefstats['details']['counts'].append(entry)
    return mozdefstats


def main():
    '''
    Get aggregated statistics on incoming events
    to use in alerting/notices/queries about event patterns over time
    '''
    logger.debug('starting')
    logger.debug(options)
    es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers)))
    index = options.index
    stats = esSearch(es)
    logger.debug(json.dumps(stats))
    sleepcycles = 0
    try:
        while not es.index_exists(index):
            sleep(3)
            if sleepcycles == 3:
                logger.debug("The index is not created. Terminating eventStats.py cron job.")
                exit(1)
            sleepcycles += 1
        if es.index_exists(index):
            # post to elastic search servers directly without going through
            # message queues in case there is an availability issue
            es.save_event(index=index, body=json.dumps(stats))

    except Exception as e:
        logger.error("Exception %r when gathering statistics " % e)

    logger.debug('finished')


def initConfig():
    # output our log to stdout or syslog
    options.output = getConfig('output', 'stdout', options.configfile)
    # syslog hostname
    options.sysloghostname = getConfig('sysloghostname',
                                       'localhost',
                                       options.configfile)
    # syslog port
    options.syslogport = getConfig('syslogport', 514, options.configfile)

    # elastic search server settings
    options.esservers = list(getConfig('esservers',
                                       'http://localhost:9200',
                                       options.configfile).split(','))

    # field to use as the aggegation point (category, type, etc)
    options.aggregationfield = getConfig('aggregationfield',
                                         'category',
                                         options.configfile)

    # default time period in minutes to look back in time for the aggregation
    options.aggregationminutes = getConfig('aggregationminutes',
                                           15,
                                           options.configfile)
    # configure the index to save events to
    options.index = getConfig('index', 'mozdefstate', options.configfile)


if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option(
        "-c",
        dest='configfile',
        default=sys.argv[0].replace('.py', '.conf'),
        help="configuration file to use")
    (options, args) = parser.parse_args()
    initConfig()
    main()