Web Application Firewall country blacklist heuristic. (#8)

* Initial WIP country blacklist structure.
* Add request de-serialization protections.
* Improved hinting, now storing packed IPs.
* Correct missing import, pre-trigger cached de-serialization operations which may fail.
* Additional optional installation USE flags.
* Document addition of WAF extension.
* Add API definition for persistent blacklists.
* Permit persistence of the blacklist and exemptions.
* Deserialization errors are better handled in core at time of collect callback execution.
* Initial WIP country blacklist structure.
* Sorting of example countries.
* Register the WAF heuristics as plugins.
* Use correct code, we are not a client ourselves.
* Pass client IP down to heuristics, prime query string arguments.
* Heuristics are now passed the client IP.
* IP2Location utilization.
* Ban-by-country implementation.
* Hosting combined heuristic default extensions.
* Who needs a temporary variable?
* Additional example country, short and long name for logs.
* Adjustments to logging levels and extras.
* Blacklist serialization and deserialization.
* Correction for escaping within a quoted string, additional example geographic exclusions.

Author: amcgregor

setup.py

@@ -102,5 +102,13 @@
 					'matches = web.security.predicate:ContextMatch',
 					'contains = web.security.predicate:ContextContains',
 				],
+			'web.security.heuristic': [
+					'dns = web.security.waf:ClientDNSHeuristic',
+					'path = web.security.waf:PathHeuristic',
+					'php = web.security.waf:PHPHeuristic',
+					'wordpress = web.security.waf:WordpressHeuristic',
+					'hosting = web.security.waf:HostingCombinedHeuristic',
+					'country = web.security.waf:GeoCountryHeuristic',
+				]
 		},
 )

web/ext/acl.py

@@ -249,7 +249,7 @@ def __init__(self, *_policy, default=None, policy=None):
 	def prepare(self, context):
 		"""Called to prepare the request context by adding an `acl` attribute."""
 
-		if __debug__: log.debug("Populating request context with ACL.", extra=dict(request=id(context)))
+		if __debug__: log.trace("Populating request context with ACL.", extra=context.extra)
 
 		context.acl = ACL(context=context, policy=self.policy)
 
@@ -262,24 +262,24 @@ def dispatch(self, context, crumb):
 		acl = getattr(crumb.handler, '__acl__', ())
 		inherit = getattr(crumb.handler, '__acl_inherit__', True)
 
-		if __debug__: log.debug(f"Handling dispatch event: {crumb.handler!r} {acl!r}", extra=dict(
-				request = id(context),
-				consumed = crumb.path,
-				handler = safe_name(crumb.handler),
-				endpoint = crumb.endpoint,
-				acl = [repr(i) for i in acl],
-				inherit = inherit,
-			))
+		if __debug__: log.trace(f"Handling dispatch event: {crumb.handler!r} {acl!r}", extra={
+				'consumed': crumb.path,
+				'handler': safe_name(crumb.handler),
+				'endpoint': crumb.endpoint,
+				'acl': [repr(i) for i in acl],
+				'inherit': inherit,
+				**context.extra
+			})
 
 		if not inherit:
-			if __debug__: log.info("Clearing collected access control list.")
+			if __debug__: log.warn("Clearing collected access control list.")
 			del context.acl[:]
 
 		context.acl.extend((Path(context.request.path), i, handler) for i in acl)
 
 	def collect(self, context, handler, args, kw):
 		if not context.acl:
-			if __debug__: log.debug("Skipping validation of empty ACL.", extra=dict(request=id(context)))
+			if __debug__: log.debug("Skipping validation of empty ACL.", extra=context.extra)
 			return
 
 		grant = context.acl.is_authorized

web/ext/waf.py

@@ -12,6 +12,7 @@
 
 from abc import ABCMeta, abstractmethod
 from html import escape
+from pathlib import Path
 from re import compile as re
 from socket import inet_aton
 
@@ -34,26 +35,56 @@
 ClientSet = MutableSet[bytes]
 
 class PersistentClientSet(ClientSet, metaclass=ABCMeta):
-	"""A mutable set exposing two methods for persisting and restoring its contents."""
+	"""An ABC describing a mutable set that exposes methods for persisting and restoring its contents."""
 
 	@abstractmethod
 	def persist(self, context:Context) -> None:
-		...
+		"""Persist the state of the set.
+		
+		It is up to the individual implementation to decide how to do this. Typically this would involve serialization
+		on-disk or the use of some form of data store, such as SQLite, PostgreSQL, or MongoDB.
+		"""
+		
+		raise NotImplementedError()
 
 	@abstractmethod
 	def restore(self, context:Context) -> None:
-		...
+		"""Restore the state of the set.
+		
+		It is up to the individual implementation to decide how to do this. Typically this involves deserialization
+		from disk or the use of some form of data store, such as SQLite, PostgreSQL, or MongoDB.
+		"""
+		
+		raise NotImplementedError()
+
+
+class LineSerializedSet(set, PersistentClientSet):
+	location:Path  # The target path to read and write data from/to.
+	
+	def __init__(self, *args, location:Union[str,Path]):
+		self.location = Path(location)
+	
+	def persist(self, context:Context) -> None:
+		with self.location.open('w') as fh:
+			for element in sorted(self):
+				fh.write(str(element) + "\n")
+	
+	def restore(self, context:Context) -> None:
+		self.clear()
+		
+		with self.location.open('r') as fh:
+			for line in fh.readlines():
+				self.add(int(line.strip()))
 
 
 class WebApplicationFirewallExtension:
 	"""A basic rules-based Web Application Firewall implementation."""
 
+	uses:ClassVar[Tags] = {'timing.prefix'}  # We want our execution time to be counted.
 	provides:ClassVar[Tags] = {'waf'}  # A set of keywords usable in `uses` and `needs` declarations.
 	first:ClassVar[bool] = True  # Always try to be first: if truthy, become a dependency for all non-first extensions.
 	extensions:ClassVar[Tags] = {'waf.rule'}  # A set of entry_point namespaces to search for related plugin registrations.
 
-	uses:ClassVar[Tags] = {'timing.prefix'}  # We want our execution time to be counted.
-	
 	heuristics:Iterable[WAFHeuristic]  # The prepared heuristic instances.
 	blacklist:ClientSet  # The current blacklist. Can theoretically be swapped for any mutable set-like object.
 	exempt:ClientSet  # IP addresses exempt from blacklisting.
@@ -73,7 +104,7 @@ def __init__(self, *heuristics, blacklist:Optional[ClientSet]=None, exempt:Optio
 		self.heuristics = heuristics
 
 		# Permit custom backing stores to be passed in; we optimize by storing packed binary values, not strings.
-		self.blacklist = set() if blacklist is None else set(inet_aton(i) for i in blacklist)
+		self.blacklist = set() if blacklist is None else blacklist.__class__(inet_aton(i) for i in blacklist)
 
 		# Permit custom backing stores to be passed in for the exemptions, as well.
 		self.exempt = set() if exempt is None else exempt
@@ -87,6 +118,7 @@ def inner(environ:WSGIEnvironment, start_response:WSGIStartResponse):
 			try:
 				request: Request = Request(environ)  # This will be remembered and re-used as a singleton later.
 				uri: URI = URI(request.url)
+				request.GET  # As will this "attempt to access query string parameters", malformation detection.
 
 			except Exception as e:  # Protect against de-serialization errors.
 				return HTTPBadRequest(f"Encountered error de-serializing the request: {e!r}")(environ, start_response)
@@ -103,7 +135,7 @@ def inner(environ:WSGIEnvironment, start_response:WSGIStartResponse):
 				# Validate the heuristic rules.
 				for heuristic in self.heuristics:
 					try:
-						heuristic(environ, uri)
+						heuristic(environ, uri, client)
 					except HTTPClose as e:
 						log.error(f"{heuristic} {e.args[0].lower()}")
 						raise

web/security/exc.py

@@ -4,6 +4,6 @@
 class HTTPClose(HTTPClientError):
 	"""Indicate to the front-end load balancer (FELB) that it should hang up on the client."""
 
-	code = 499
-	title = "Client Closed Request"
+	code = 444
+	title = "Connection Closed Without Response"
 	explanation = "The server did not accept your request."

web/security/waf.py

@@ -11,9 +11,14 @@
 from .util import DNS
 from .exc import HTTPClose
 
+try:
+	from IP2Location import IP2Location
+except ImportError:
+	IP2Location = None
+
 
 class WAFHeuristic:
-	def __call__(self, environ:WSGIEnvironment, uri:URI) -> Optional[bool]:
+	def __call__(self, environ:WSGIEnvironment, uri:URI, client:str) -> Optional[bool]:
 		"""Perform the heuristic check.
 		
 		May return True to indicate processing should stop, raise an HTTPException to propagate to the client, or may
@@ -78,7 +83,7 @@ def __repr__(self, *extra:str) -> str:
 				*extra
 			)
 
-	def __call__(self, environ:WSGIEnvironment, uri:URI) -> Optional[bool]:
+	def __call__(self, environ:WSGIEnvironment, uri:URI, client:str) -> Optional[bool]:
 		assert check_argument_types()
 
 		addr:str = environ.get(self.origin, '')  # Attempt to retrieve the client IP from the WSGI environment.
@@ -139,7 +144,7 @@ class PathHeuristic(WAFHeuristic):
 	
 	One can also deny any request targeting a PHP script:
 	
-		PathHeuristic(re.compile(r'\.phps?($|/)'))
+		PathHeuristic(re.compile(r'\\.phps?($|/)'))
 	
 	It's important to note that regular expression flags (such as case insensitivity) will be ignored; the search is
 	always case sensitive.  (phpMyAdmin != phpmyadmin; these are legitimately separate resources.)
@@ -170,7 +175,7 @@ def __repr__(self, *extra:str) -> str:
 				*extra
 			)
 
-	def __call__(self, environ:dict, uri:URI) -> None:
+	def __call__(self, environ:dict, uri:URI, client:str) -> None:
 		assert check_argument_types()
 
 		if self.forbidden & set(uri.path.parts):  # This is ~a third faster than the simplest regex use.
@@ -209,16 +214,73 @@ def __init__(self) -> None:
 class HostingCombinedHeuristic(PathHeuristic):
 	"""A combined set of suspicious URI fragments and general patterns matching commonly exploited tools.
 	
-	This is the result of casually browsing through around ten years of error logs on an active hosting service.
+	This is the result of casually browsing through around ten years of error logs on an active hosting service and
+	combines a number of the other PathHeuristic rules into one for convenience. (The WAF already optimizes these down
+	into a single regex for runtime checking; this is an import optimization.)
+	
+	Several filename extensions which ought to be delivered by a front-end load balancer are included in this list;
+	DO NOT INCLUDE THIS HEURISTIC AT DEVELOPMENT TIME if you are delivering static content via an endpoint within your
+	application. A critical message will be emitted if used at development time.
 	"""
 
-	def __init__(self) -> None:
+	def __init__(self, *extensions:str) -> None:
+		"""Prepare a 'combined hosting experience' heuristic.
+		
+		You can pass in additional extensions to block beyond the basic set included as stringy regular expression
+		fragments via positional arguments.
+		"""
+		
+		if __debug__:
+			log.critical("Use of this heuristic if delivering statics from the application at development time will" \
+					"likely blacklist you.")
+		
+		extensions = set(extensions) | {'html?', 'phps?', 'py', 'js', 'css', 'swf', 'txt', 'md'}
+		
 		super().__init__(
-				re(r'\.(html?|swf|phps?)($|/)'),  # Bare HTML files, Adobe Flash, or PHP.
+				re(r'\.(' + '|'.join(sorted(extensions)) + r')($|/)'),  # Forbidden filename extensions.
 				re(r'((web)?mail)|(round|cube|roundcube)((web)?mail)?2?(-[0-9\.]+)?'),  # Webmail service, in general.
 					'wm', 'rc', 'rms', 'mss', 'mss2',  # More common webmail containers.
 				'FlexDataServices', 'amfphp', 'soapCaller.bs',  # Adobe Flex AMF and RPC services.
 				'wordpress', 'wp', 'wp-admin', 'wp-includes', 'wlwmanifest.xml',  # WordPress-related.
 				'admin', 'mysql', 'phpMyAdmin', 'pma', 'dbadmin', 'MyAdmin', 'phppgadmin',  # Common administrative access.
 				'crossdomain.xml', 'README', 'LICENSE', 'webdav', re(r'w00tw00t'),  # Generic probes.
 			)
+
+
+class GeoCountryHeuristic(WAFHeuristic):
+	"""A rule which preemptively blocks attempted access from specific countries of origin.
+	
+	Example usage:
+	
+		GeoCountryHeuristic(
+				'cn', 'kp',  # China, take that, "Great Firewall", and North Korea.
+				'ae', 'ir', 'iq', 'sa', 'tr',  # Middle-eastern nations.
+				'by', 'ru', 'ua',  # Russia and nearby former bloc states.
+				'am', 'az', 'ee', 'ge', 'kg', 'kz', 'lt', 'lv', 'md', 'tj', 'tm', 'uz',  # Additional former states.
+				'af', 'mr', 'ng', 'ph', 'pl', 'sd', 'ye',  # LGBTQ and human rights violators, others included above.
+			)
+	"""
+	
+	countries: Set[str]  # The set of blocked ISO 3166 country codes.
+	resolver: IP2Location
+	
+	def __init__(self, *countries:str, db:str='IP2LOCATION-LITE-DB1.IPV6.BIN') -> None:
+		"""Initialize the country heuristic's geographic database and blacklist."""
+		
+		assert check_argument_types()
+		
+		if IP2Location is None:
+			raise ImportError("You must have the IP2Location library installed.")
+		
+		self.countries = {i.upper() for i in countries}
+		self.resolver = IP2Location(db)
+	
+	def __repr__(self, *extra:str) -> str:
+		countries = "'" + "', '".join(sorted(self.countries)) + "'"
+		return super().__repr__(countries, *extra)
+	
+	def __call__(self, environ:dict, uri:URI, client:str) -> None:
+		assert check_argument_types()
+		
+		if (short := self.resolver.get_country_short(client)) in self.countries:
+			raise HTTPClose(f"Access from {short} ({self.resolver.get_country_long(client)}) forbidden.")