Merge pull request #1 from userjack6880/dev

Dev

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0-α1.2
+- Fixed log regex for monitor script.
+- Added a case for where punishment is issued for a prefix has a ton of bad IPs that do not have their ban expirations timeout.
+- Fixed DB query column typo.
+- When an IP network range is added, on duplicate key it now adds ban expiration.
+- Added variation of `spam` to the keywords monitor looks for.
+- Fixed issue where script couldn't find the config file.
+- Fixed bug where script would die if it encountered a JASON error when doing a BGP Info Query.
+
 ## 0-α1.1
 - Fixed bug where config file cannot be found if script is not run from the directory it's located in.
 

README.md

@@ -1,12 +1,38 @@
 # RBL Updater Suite
 
-This is the RBL Updater Suite version 0 alpha-1.1 (0-α1.1) by John Bradley ([email protected]). The RBL Updater Suite is an Open Source suite of tools to be used in conjunction with rpsamd to help autogenerate a local realtime block list (RBL) not reliant on any external lists, such as spamhaus and the like.
+This is the RBL Updater Suite version 0 alpha-1.2 (0-α1.2) by John Bradley ([email protected]). The RBL Updater Suite is an Open Source suite of tools to be used in conjunction with rpsamd to help autogenerate a local realtime block list (RBL) not reliant on any external lists, such as spamhaus and the like.
 
 This software is extremely experimental and may cause collateral damage on deliverability. USE AT YOUR OWN RISK.
 
+# Suite Components
+
+## `monitor`
+
+This is the script that monitors your mail log for a `NOQUEUE: reject` message or a `milter-reject` message containing additional keywords `BLOCKLIST`, `spam`, or `Spam`. When it does that, it flags the IP address associated with the message, and performs a number of actions outlined under the Principle of Operation section of this Readme.
+
+## `report`
+
+This script is used to manually report an IP address or range. Regardless of previous infractions, it will always issue a 1-day ban based on the current time. This can inadvertantly shorten a ban if you are not careful.
+
+```
+        Usage:
+                ./report [OPTIONS]
+
+        This script add to the database either an IP address or an IP Range.
+
+        Options:
+                -i      [IPv4 Address]  Adds a single IP address
+                -n      [CIDR Notation] Adds a CIDR notation network range
+                -p      Makes either IP address or network range permabanned
+```
+
+## `generate_list`
+
+This script will create a plaintext file with the IP addresses and network ranges, deliminated by newlines, at the location specified in the config file.
+
 # Principle of Operation
 
-The script assumes that you have configured postfix in a way that it blocks misconfigured hosts attempting to connect to your mail server, already is blocking messages, and has rspamd installed and running.
+The `monitor` script assumes that you have configured postfix in a way that it blocks misconfigured hosts attempting to connect to your mail server, already is blocking messages, and has rspamd installed and running.
 
 Whenever an IP address gets blocked in the mail logs, the monitor script will flag the IP and increase the time it is banned. The ban gets more agressive the more the IP is flagged, ultimately ended up in prefix and asn bans as the issue worsens.
 
@@ -23,6 +49,7 @@ For network prefixes, infractions and bans are given based on the number of indi
 - On the third IP permaban, the prefix receives a 1 day ban.
 - Every additional IP permaban after the third results in a 1 week ban.
 - On the twenty-fifth (25th) IP permaban, the prefix is permanently banned.
+- Exception: if more than 5 IP addresses within a prefix have concurrent temporary bans at the same time, the prefix is issued a ban.
 
 Bans are cumulative, and infractions are permanently recorded.
 
@@ -70,12 +97,14 @@ Install anywhere you want. Probably will want to run it as a privleged user, or
 
 # Latest Changes
 
-## 0-α1.1
-- Fixed bug where config file cannot be found if script is not run from the directory it's located in.
-
-## 0-α1
-
-- Created the project.
+## 0-α1.2
+- Fixed log regex for monitor script.
+- Added a case for where punishment is issued for a prefix has a ton of bad IPs that do not have their ban expirations timeout.
+- Fixed DB query column typo.
+- When an IP network range is added, on duplicate key it now adds ban expiration.
+- Added variation of `spam` to the keywords monitor looks for.
+- Fixed issue where script couldn't find the config file.
+- Fixed bug where script would die if it encountered a JSON error when doing a BGP Info Query.
 
 # Planned Features
 

config.conf.pub

@@ -15,4 +15,4 @@ $dbuser  = '';
 $dbpass  = '';
 
 # version number
-$version = '0 alpha-1.1';
+$version = '0 alpha-1.2';

generate_list

@@ -50,12 +50,12 @@ sub usage {
 our ($asnlist,$iplist,$dbname,$dbhost,$dbport,$dbuser,$dbpass,$log,$version);
 my $conffile = 'config.conf';
 
-if (-e $conffile) { $conffile = "./$conffile"; }
-elsif (-e File::Basename::dirname($0)."/$conffile") { $conffile = File::Basename::dirname($0)."/$conffile"; }
-else { usage(); die "Error! Could not read $conffile\n"; }
-
 if (substr($conffile,0,1) ne '/' and substr($conffile,0,1) ne '.') { $conffile = "./$conffile"; }
 
+if (-e File::Basename::dirname($0)."/$conffile") { $conffile = File::Basename::dirname($0)."/$conffile"; }
+unless (-e $conffile) { usage(); die "Error! Could not read $conffile\n"; }
+
+
 my $conftest = do $conffile;
 die "$conffile could not be parsed: $@" if $@;
 die "could not do $conffile: $!" if !defined $conftest;

monitor

@@ -54,12 +54,11 @@ sub usage {
 our ($asnlist,$iplist,$dbname,$dbhost,$dbport,$dbuser,$dbpass,$log,$version);
 my $conffile = 'config.conf';
 
-if (-e $conffile) { $conffile = "./$conffile"; }
-elsif (-e File::Basename::dirname($0)."/$conffile") { $conffile = File::Basename::dirname($0)."/$conffile"; }
-else { usage(); die "Error! Could not read $conffile\n"; }
-
 if (substr($conffile,0,1) ne '/' and substr($conffile,0,1) ne '.') { $conffile = "./$conffile"; }
 
+if (-e File::Basename::dirname($0)."/$conffile") { $conffile = File::Basename::dirname($0)."/$conffile"; }
+unless (-e $conffile) { usage(); die "Error! Could not read $conffile\n"; }
+
 my $conftest = do $conffile;
 die "$conffile could not be parsed: $@" if $@;
 die "could not do $conffile: $!" if !defined $conftest;
@@ -81,6 +80,11 @@ sub query_ip {
 	            ]
 	          );
 	my $res = $ua->request($req);
+	# return error if JSON isn't returned
+	if (substr($res->decoded_content,0,1) eq '<') {
+		$result->{error} = 1;
+		return $result;
+	}
 	my $data = decode_json($res->decoded_content);
 	$result->{prefix} = $data->{data}{prefixes}[0]{prefix};
 	$result->{prefix} = $data->{data}{rir_allocation}{prefix} if not defined($result->{prefix});
@@ -104,6 +108,10 @@ sub query_asn {
 	            ]
 	          );
 	my $res = $ua->request($req);
+	if (substr($res->decoded_content,0,1) eq '<') {
+		$result[0] = 1;
+		return @result;
+	}
 	my $data = decode_json($res->decoded_content);
 	my $itr = 0;
 	foreach my $entry (@{$data->{data}{ipv4_prefixes}}) {
@@ -153,6 +161,10 @@ sub add_asn {
 			print "no ASN info!\n";
 			return;
 		}
+		if ($ranges[0] == 1) {
+			print "JSON error\n";
+			return;
+		}
 		else {
 			print "AS$asn has ".scalar(@ranges)." ranges\n";
 
@@ -186,6 +198,10 @@ sub add_asn {
 				print "no ASN info!\n";
 				return;
 			}
+			if ($ranges[0] == 1) {
+				print "JSON error\n";
+				return;
+			}
 			# we need to increase the infraction count, or permaban if infraction count is exceeded
 			if ($infractions == 0) {
 				print "AS$asn: first infraction, 1 week ban\n";
@@ -257,7 +273,11 @@ sub add_net {
 	my $sth = $dbh->prepare(q{
 	  INSERT INTO ipnet_blocklist (ip4_net, asn, infractions, infractions_type, ban_expiration, permaban)
 		VALUES (?,?,1,1,NOW(),0)
-		ON DUPLICATE KEY UPDATE infractions = infractions = + 1;
+		ON DUPLICATE KEY UPDATE infractions = infractions + 1,
+	                          ban_expiration = CASE
+	                            WHEN ban_expiration < NOW() THEN DATE_ADD(NOW(), INTERVAL 1 HOUR)
+	                            WHEN ban_expiration >= NOW() THEN DATE_ADD(ban_expiration, INTERVAL 1 HOUR)
+	                          END;
 	});
 	$sth->execute($prefix,$asn);
 
@@ -287,7 +307,7 @@ sub add_net {
 			    WHEN ban_expiration < NOW() THEN DATE_ADD(NOW(), INTERVAL 1 DAY)
 			    WHEN ban_expiration >= NOW() THEN DATE_ADD(ban_expiration, INTERVAL 1 DAY)
 			  END
-			  where ip4net = ?;
+			  where ip4_net = ?;
 			});
 			print "$prefix has recieved a 1 day ban\n";
 		}
@@ -298,7 +318,7 @@ sub add_net {
 			    WHEN ban_expiration < NOW() THEN DATE_ADD(NOW(), INTERVAL 1 WEEK)
 			    WHEN ban_expiration >= NOW() THEN DATE_ADD(ban_expiration, INTERVAL 1 WEEK)
 			  END
-			  where ip4net = ?;
+			  where ip4_net = ?;
 			});
 			print "$prefix has recieved a 1 week ban\n";
 		}
@@ -312,6 +332,10 @@ sub add_ip {
 	db_keepalive();
 
 	my $range = query_ip($ip);
+	if (defined $range->{error}) {
+		print "JSON error\n";
+		return;
+	}
 
 	# simple increment, create if it doesn't exist
 	print "FLAGGED: $ip\n";
@@ -327,6 +351,14 @@ sub add_ip {
 
 	print "\t$infractions infractions\n";
 
+	# check to see if there are more than 5 address in the net that are currently banned... if so, then more severely punish the prefix
+	# does not penalize for permabanned IPs
+	if ($dbh->selectrow_array("SELECT COUNT(*) FROM ip_blocklist WHERE ip4_net = '$range->{prefix}' AND ban_expiration > NOW();") > 5) {
+		print "\tUnusually high infraction from net $range->{prefix}. Punishing\n";
+
+		add_net($range);
+	}
+
 	if ($infractions > 3) {
 		print "$ip is now permanently banned!\n";
 		# adds ip to permaban, add infraction to the IP4net
@@ -412,7 +444,8 @@ while (defined(my $line=$file->read)) {
 	if ($line =~ /^.+milter-reject.+/) {
 		# this will only penalize IPs that exit on the current blocklist or legit spam
 		if ($line =~ /^.+BLOCKLIST.+/ || 
-		    $line =~ /^.+spam.+/ ) {
+		    $line =~ /^.+spam.+/ ||
+		    $line =~ /^.+Spam.+/ ) {
 			if ($line =~ m/^.+\[(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).+/) {
 				add_ip($1);
 			}

report

@@ -65,12 +65,11 @@ sub usage {
 our ($asnlist,$iplist,$dbname,$dbhost,$dbport,$dbuser,$dbpass,$log,$version);
 my $conffile = 'config.conf';
 
-if (-e $conffile) { $conffile = "./$conffile"; }
-elsif (-e File::Basename::dirname($0)."/$conffile") { $conffile = File::Basename::dirname($0)."/$conffile"; }
-else { usage(); die "Error! Could not read $conffile\n"; }
-
 if (substr($conffile,0,1) ne '/' and substr($conffile,0,1) ne '.') { $conffile = "./$conffile"; }
 
+if (-e File::Basename::dirname($0)."/$conffile") { $conffile = File::Basename::dirname($0)."/$conffile"; }
+unless (-e $conffile) { usage(); die "Error! Could not read $conffile\n"; }
+
 my $conftest = do $conffile;
 die "$conffile could not be parsed: $@" if $@;
 die "could not do $conffile: $!" if !defined $conftest;