File size: 5,394 Bytes
7c89ed7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | #!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation
import boto3
import netaddr
import sys
from datetime import datetime
from datetime import timedelta
from configlib import getConfig, OptionParser
from pymongo import MongoClient
from mozdef_util.utilities.logger import logger
def isIPv4(ip):
try:
# netaddr on it's own considers 1 and 0 to be valid_ipv4
# so a little sanity check prior to netaddr.
# Use IPNetwork instead of valid_ipv4 to allow CIDR
if '.' in ip and len(ip.split('.')) == 4:
# some ips are quoted
netaddr.IPNetwork(ip.strip("'").strip('"'))
return True
else:
return False
except:
return False
def isIPv6(ip):
try:
return netaddr.valid_ipv6(ip)
except:
return False
def parse_network_whitelist(network_whitelist_location):
networks = []
with open(network_whitelist_location, "r") as text_file:
for line in text_file:
line = line.strip().strip("'").strip('"')
if isIPv4(line) or isIPv6(line):
networks.append(line)
return networks
def main():
logger.debug('starting')
logger.debug(options)
try:
# connect to mongo
client = MongoClient(options.mongohost, options.mongoport)
mozdefdb = client.meteor
ipblocklist = mozdefdb['ipblocklist']
# ensure indexes
ipblocklist.create_index([('dateExpiring', -1)])
# delete any that expired
ipblocklist.delete_many({'dateExpiring': {"$lte": datetime.utcnow() - timedelta(days=options.expireage)}})
# Export the blocklist
ipCursor = mozdefdb['ipblocklist'].aggregate([
{"$sort": {"dateAdded": -1}},
{"$match": {"address": {"$exists": True}}},
{"$match": {
"$or": [
{"dateExpiring": {"$gte": datetime.utcnow()}},
{"dateExpiring": {"$exists": False}},
]},
},
{"$project": {"address": 1}},
{"$limit": options.iplimit}
])
ips = []
for ip in ipCursor:
ips.append(ip['address'])
uniq_ranges = netaddr.cidr_merge(ips)
# to text
with open(options.outputfile, 'w') as outputfile:
for ip in uniq_ranges:
outputfile.write("{0}\n".format(ip))
outputfile.close()
# to s3?
if len(options.aws_bucket_name) > 0:
s3_upload_file(options.outputfile, options.aws_bucket_name, options.aws_document_key_name)
except ValueError as e:
logger.error("Exception %r generating IP block list" % e)
def initConfig():
# output our log to stdout or syslog
options.output = getConfig('output', 'stdout', options.configfile)
# syslog hostname
options.sysloghostname = getConfig('sysloghostname',
'localhost',
options.configfile)
# syslog port
options.syslogport = getConfig('syslogport', 514, options.configfile)
# mongo instance
options.mongohost = getConfig('mongohost', 'localhost', options.configfile)
options.mongoport = getConfig('mongoport', 3001, options.configfile)
# CIDR whitelist as a line separted list of 8.8.8.0/24 style masks
options.network_list_file = getConfig('network_whitelist_file', '', options.configfile)
options.ipwhitelist = parse_network_whitelist(options.network_list_file)
# Output File Name
options.outputfile = getConfig('outputfile', 'ipblocklist.txt', options.configfile)
# Days after expiration that we purge an ipblocklist entry (from the ui, they don't end up in the export after expiring)
options.expireage = getConfig('expireage', 1, options.configfile)
# Max IPs to emit
options.iplimit = getConfig('iplimit', 1000, options.configfile)
# AWS creds
options.aws_access_key_id = getConfig('aws_access_key_id', '', options.configfile) # aws credentials to use to connect to mozilla_infosec_blocklist
options.aws_secret_access_key = getConfig('aws_secret_access_key', '', options.configfile)
options.aws_bucket_name = getConfig('aws_bucket_name', '', options.configfile)
options.aws_document_key_name = getConfig('aws_document_key_name', '', options.configfile)
def s3_upload_file(file_path, bucket_name, key_name):
"""
Upload a file to the given s3 bucket and return a template url.
"""
s3 = boto3.resource(
's3',
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key
)
s3.meta.client.upload_file(
file_path, bucket_name, key_name, ExtraArgs={'ACL': 'public-read'})
url = "https://s3.amazonaws.com/{}/{}".format(bucket_name, key_name)
print("URL: {}".format(url))
return url
if __name__ == '__main__':
parser = OptionParser()
parser.add_option(
"-c",
dest='configfile',
default=sys.argv[0].replace('.py', '.conf'),
help="configuration file to use")
(options, args) = parser.parse_args()
initConfig()
main()
|