Mini Shell
"""This plugin periodically checks set of rules and ipsets,
and recreates it if needed, process block/unblock messages."""
import asyncio
import json
import logging
import os
import time
from contextlib import suppress
from pathlib import Path
from typing import Set
from defence360agent.internals.global_scope import g
from defence360agent.utils import log_error_and_ignore, timeit
from defence360agent.utils.common import DAY, ServiceBase, rate_limit
from im360.contracts.config import Firewall, NetworkInterface, DOS, EnhancedDOS
from im360.contracts.config import Protector as Config
from im360.contracts.config import Webshield
from im360.internals.core import IPSetNoRedirectPort, RuleSet, ip_versions
from im360.internals.core.ipset.libipset import IPSetError
from im360.internals.core.firewall import get_firewall
from im360.internals.core.firewall.base import (
FirewallBatchCommandError,
FirewallError,
FirewallTemporaryError,
)
from im360.internals.core.ipset.country import IPSetCountry
from im360.internals.core.ipset.ip import (
IPSetStatic,
IPSetStaticRemoteProxy,
)
from im360.internals.core.ipset.port_deny import (
InputPortBlockingDenyModeIPSet,
OutputPortBlockingDenyModeIPSet,
)
from im360.subsys.webshield_mode import Mode as WebshieldMode
from defence360agent.utils.validate import IP, IPVersion
from im360.subsys import smtp_blocking
logger = logging.getLogger(__name__)
RULES_CHECK_IN_PROGRESS = Path("/var/imunify360/.rules_check_in_progress")
class VersionState:
def __init__(self):
self.transient_error_on_create = False
self.errors = 0
self.next_try_time = 0.0
self.running = True
class RulesChecker(ServiceBase):
"""Periodically checks if rules exist and if not, recreate them."""
#: delay in secs between two iptables rule/ipsets checks
RULE_CHECK_INTERVAL = int(
os.environ.get("IMUNIFY360_RULE_CHECK_INTERVAL", 30)
)
CONSECUTIVE_ERROR_LIMIT = 10 # retries until giving up
ERROR_THRESHOLD = 10 # disabled IPv6 support after this many failures
RETRY_INTERVAL = 3600 # interval between attempts to enable IPv6
CAPTURE_CSF_LOCK = True
CSF_LOCK_TIMEOUT = 300 # seconds
IPSETS_CHECK_INTERVAL = DAY # seconds
HOST_IPSETS = {
IP.V4: {"i360.ipv4.whitelist.host_ips"},
IP.V6: {"i360.ipv6.whitelist.host_ips"},
}
def __init__(self, loop):
super().__init__(loop)
self.ruleset = RuleSet()
# Prevent rules create/destroy public method calls
# from overlapping with @recurring_check->create_rules/destroy_rules()
#
# Use this lock to operate on rules (add to ipset)
# while keeping the rules in consistent state (not half created
# or half destroyed).
self.lock_rules_create_destroy = Config.RULE_EDIT_LOCK
self.active_interface_conf = NetworkInterface.get_interface_conf()
self.versions = {ver: VersionState() for ver in ip_versions.all()}
self._ipsets_outdated_events = {
ver: asyncio.Event() for ver in self.versions
}
self.outdated_ipsets = {ver: set() for ver in ip_versions.all()}
self.ipset_lock = asyncio.Lock()
async def recreate_rules_if_needed(self, recreate_any_way=False) -> None:
# re-reading config file
async with self.lock_rules_create_destroy:
new_conf = NetworkInterface.get_interface_conf()
if not new_conf == self.active_interface_conf:
logger.info(
"Target & ignore interfaces config was changed,"
"recreating rules"
)
await self._destroy_rules_and_sets(self.active_interface_conf)
else:
for version, state in self.versions.items():
if self.versions[version].transient_error_on_create:
await self.__check_ipset_consistent(version, state)
await self._ensure_rules_exist_for_all_versions(
new_conf, recreate_any_way
)
self.active_interface_conf = new_conf
@rate_limit(period=IPSETS_CHECK_INTERVAL)
@log_error_and_ignore()
async def check_ipsets_consistent(self): # pragma: no cover
await self._check_ipsets_consistent()
async def _check_ipsets_consistent(self):
async with self.lock_rules_create_destroy:
for version, state in self.versions.items():
await self.__check_ipset_consistent(version, state)
async def __check_ipset_consistent(
self, version: int, state: VersionState
) -> None:
with timeit(f"Checking outdated ipsets {version}", log=logger.info):
if state.running:
self.outdated_ipsets[version] = self.HOST_IPSETS[version]
if outdated := await self.ruleset.get_outdated_ipsets(version):
outdated_ipsets = set(
ip_count.name for ip_count in outdated
)
self.outdated_ipsets[version].update(outdated_ipsets)
logger.warning(
f"{outdated} ipsets are outdated. Recreate them"
)
else:
logger.info(
"Only host ipsets"
f" {', '.join(self.HOST_IPSETS[version])} will"
" be updated"
)
self._ipsets_outdated_events[version].set()
async def _ensure_rules(
self, interface_conf, recreate_any_way=False
) -> bool:
now = time.monotonic()
target_versions = set(
version
for version in ip_versions.all()
if (
ip_versions.is_enabled(version)
or self.versions[version].next_try_time < now
)
)
recreated_any = False
has_failed = False
for version in target_versions:
self.versions[version].running = False
try:
async with self.ipset_lock:
r = await self._ensure_for(
version, interface_conf, recreate_any_way
)
recreated_any = recreated_any or r
except (FirewallTemporaryError, IPSetError) as exc:
has_failed = True
if isinstance(exc, IPSetError):
self.versions[version].transient_error_on_create = True
logger.info(
"Transient error while creating firewall rules for %s: %s",
version,
exc,
)
except FirewallError as exc:
enabled_ip_version = ip_versions.is_enabled(version)
if enabled_ip_version:
has_failed = True
self.versions[version].errors += 1
if self.versions[version].errors >= self.ERROR_THRESHOLD:
if ip_versions.disable(version):
self.versions[version].next_try_time = (
now + self.RETRY_INTERVAL
)
logger.info(
"%s firewall support is disabled due"
" to multiple consecutive errors",
version,
)
else:
self.versions[version].next_try_time = (
now + self.RETRY_INTERVAL
)
logger.warning(
"Failed to recreate firewall rules for %s %s: %s",
"enabled" if enabled_ip_version else "disabled",
version,
exc,
)
else:
self.versions[version].running = True
self.versions[version].errors = 0
self.versions[version].transient_error_on_create = False
if not ip_versions.is_enabled(version):
ip_versions.enable(version)
logger.info("%s firewall support is enabled", version)
# whether recreated rules for any ip version and no failures
# in creating rules for enabled ip versions
return recreated_any and not has_failed
async def _ensure_rules_exist_for_all_versions(
self, interface_conf=None, recreate_any_way=False
):
interface_conf = interface_conf or self.active_interface_conf
if await self._ensure_rules(interface_conf, recreate_any_way):
logger.info(
"Rules and sets successfully recreated for enabled ip versions"
)
async def _recreate_for(
self,
ip_version: IPVersion,
interface_conf: dict,
recreate_ipsets: bool,
outdated_ipsets: Set[str],
missing_ipsets: Set[str],
redundant_ipsets: Set[str],
recreate_rules: bool,
):
we_have_ipsets_to_destroy = self.ruleset.has_ipset_to_destroy(
ip_version, redundant_ipsets
)
if recreate_rules or we_have_ipsets_to_destroy:
with timeit(f"Destroying rules for {ip_version}", log=logger.info):
await self._destroy_rules(interface_conf, ip_version)
if redundant_ipsets:
with timeit(
f"Destroying redundant ipsets: {redundant_ipsets}",
log=logger.info,
):
await self.ruleset.destroy_ipsets(ip_version, redundant_ipsets)
else:
self.ruleset.clean_previously_failed_ipsets(ip_version)
if missing_ipsets:
with timeit(
f"Creating missing ipsets: {missing_ipsets}", log=logger.info
):
await self.ruleset.fill_ipsets(ip_version, missing_ipsets)
if recreate_rules or we_have_ipsets_to_destroy:
with timeit(f"Recreating rules for {ip_version}", log=logger.info):
await self._create_rules(interface_conf, ip_version)
if recreate_ipsets:
with timeit("Recreating ipsets", log=logger.info):
await self.ruleset.recreate_ipsets(ip_version, outdated_ipsets)
async def _ensure_for(
self,
ip_version: IPVersion,
interface_conf: dict,
recreate_any_way=False,
) -> bool:
"""Creates imunify360 ruleset for given IP version in iptables.
If all required ipsets, rules and chains exist, does nothing.
Otherwise recreates everything as required.
Returns True if rules or sets has been (re-)created, False
otherwise."""
existing_ipsets = await self.ruleset.existing_ipsets(ip_version)
required_ipsets = self.ruleset.required_ipsets(ip_version)
missing_ipsets = required_ipsets.copy()
to_refill_ipsets = self.ruleset.ipsets_to_refill(
ip_version, existing_ipsets, required_ipsets
)
ipsets_ok = (
existing_ipsets == required_ipsets
and not self._ipsets_outdated_events[ip_version].is_set()
and not to_refill_ipsets
)
redundant_ipsets = existing_ipsets - required_ipsets
missing_ipsets -= existing_ipsets
# don't log in case when no existing ipsets, it is OK on start
if existing_ipsets and existing_ipsets != required_ipsets:
_log_ipsets_mismatch(
missing_ipsets=missing_ipsets,
redundant_ipsets=redundant_ipsets,
log=logger.warning,
)
rules_ok = await self._rules_ok(interface_conf, ip_version)
# remove redundant ipsets from outdated ipsets, as they will be
# removed anyway
# remove missing ipsets from outdated ipsets, as they will be
# created anyway
outdated_ipsets = (
self.outdated_ipsets[ip_version]
- redundant_ipsets
- missing_ipsets
) | to_refill_ipsets
if g.get("DEBUG"):
logger.info("Required ipsets: %s", required_ipsets)
logger.info("Existing ipsets: %s", existing_ipsets)
logger.info("Missing ipsets: %s", missing_ipsets)
logger.info("Redundant ipsets: %s", redundant_ipsets)
logger.info("Outdated ipsets: %s", outdated_ipsets)
logger.info(
"dbg Rules status for %s [rules: %s], [ipset: %s],"
" [forced: %s]",
ip_version,
"ok" if rules_ok else "bad",
"ok" if ipsets_ok else "bad",
recreate_any_way,
)
if not (rules_ok and ipsets_ok and not recreate_any_way):
if not ipsets_ok: # ipsets will be re-created
self._ipsets_outdated_events[ip_version].clear()
self.outdated_ipsets[ip_version].clear()
logger.info(
"Rules status for %s [rules: %s], [ipset: %s], [forced: %s]",
ip_version,
"ok" if rules_ok else "bad",
"ok" if ipsets_ok else "bad",
recreate_any_way,
)
await self._recreate_for(
ip_version,
interface_conf,
recreate_ipsets=not ipsets_ok,
outdated_ipsets=outdated_ipsets,
redundant_ipsets=redundant_ipsets,
missing_ipsets=missing_ipsets,
recreate_rules=not rules_ok or recreate_any_way,
)
return True
return False
async def _create_rules(
self, interface_conf, ip_version: IPVersion
) -> None:
logger.info("Creating rules for %s", ip_version)
async with await get_firewall(ip_version) as firewall:
batch = await self.ruleset.create_commands(
firewall, interface_conf, ip_version
)
await firewall.commit(batch)
async def _destroy_rules(
self, interface_conf, ip_version: IPVersion
) -> None:
logger.info("Destroying rules for %s", ip_version)
async with await get_firewall(ip_version) as firewall:
for batch in self.ruleset.destroy_commands(
firewall, interface_conf, ip_version
):
with suppress(FirewallBatchCommandError):
# Command may fail and that is ok
await firewall.commit(batch)
async def _rules_ok(self, interface_conf, ip_version: IPVersion) -> bool:
async with await get_firewall(ip_version) as firewall:
actions = await self.ruleset.check_commands(
firewall, interface_conf, ip_version
)
try:
await firewall.commit(actions)
except FirewallBatchCommandError:
return False
else:
return True
async def _destroy_rules_and_sets(
self,
interface_conf,
destroy_rules=True,
destroy_ipsets=True,
force_destroy_ipset=False,
):
errors = []
for ip_version in ip_versions.enabled():
self.versions[ip_version].running = False
try:
if destroy_rules:
await self._destroy_rules(interface_conf, ip_version)
if destroy_ipsets:
await self.ruleset.destroy_ipsets(
ip_version, force=force_destroy_ipset
)
except Exception as e:
errors.append(e)
if not errors:
return
elif len(errors) == 1:
raise errors[0]
elif len(errors) == 2:
# hack: to get "free" readable traceback for both errors
raise errors[1] from errors[0]
else: # pragma: no cover
assert 0, "max 2 ip versions expected"
async def clear_everything(self, interface_conf=None) -> None:
interface_conf = interface_conf or self.active_interface_conf
async with self.lock_rules_create_destroy:
await smtp_blocking.reset_rules_for_all_versions()
await self._destroy_rules_and_sets(
interface_conf, force_destroy_ipset=True
)
async def clear_rules(self, interface_conf=None) -> None:
interface_conf = interface_conf or self.active_interface_conf
async with self.lock_rules_create_destroy:
await smtp_blocking.reset_rules_for_all_versions()
await self._destroy_rules_and_sets(
interface_conf, destroy_ipsets=False
)
async def check_smtp_state_and_reset(self, check_settings=False):
new_settings = smtp_blocking.read_SMTP_settings()
if check_settings and not self._is_smtp_settings_changed(new_settings):
return
if not await smtp_blocking.is_SMTP_blocking_supported():
return
if await smtp_blocking.conflicts_exist():
await smtp_blocking.reset_rules_for_all_versions()
return
await smtp_blocking.sync_rules_for_all_versions(new_settings)
@staticmethod
def _is_smtp_settings_changed(new_settings):
return any(
active_settings != new_settings
for active_settings in smtp_blocking.get_active_settings_list()
)
class RealProtector:
_rules_checker: RulesChecker
def __init__(self):
self._port_blocking_mode = Firewall.port_blocking_mode
self._dos_enabled: bool = DOS.ENABLED or EnhancedDOS.ENABLED
self._port_blocking_deny_mode_values = (
self._get_port_blocking_deny_mode_values()
)
# saving webshield status before updating the rules
# (on _rules_checker.start()), to avoid re-checking
# the rules on the 1st ConfigUpdate (on start-up)
self._webshield_status = (
Webshield.ENABLE,
WebshieldMode.wants_redirect(WebshieldMode.get()),
Webshield.SPLASH_SCREEN,
Webshield.PANEL_PROTECTION,
)
def _get_port_blocking_deny_mode_values(self):
return (
Firewall.TCP_IN_IPV4,
Firewall.TCP_OUT_IPV4,
Firewall.UDP_IN_IPV4,
Firewall.UDP_OUT_IPV4,
)
async def process_global_whitelist_update(self):
# whitelist update will be applied to ipset when rules/ipsets
# are reloaded back from the db
async with self._rules_checker.lock_rules_create_destroy:
logger.info("Applying global white list update")
for ipset in [IPSetStatic(), IPSetStaticRemoteProxy()]:
if ipset.is_enabled():
await ipset.reset()
async def process_country_list_update(self) -> None:
async with self._rules_checker.lock_rules_create_destroy:
logger.info("Updating ipset rules on geo ip update")
for ip_version in ip_versions.enabled():
await IPSetCountry().restore(ip_version)
async def _on_config_update_unlocked(self, message):
recreate = False
current_status = (
Webshield.ENABLE,
WebshieldMode.wants_redirect(WebshieldMode.get()),
Webshield.SPLASH_SCREEN,
Webshield.PANEL_PROTECTION,
)
if current_status != self._webshield_status:
logger.info(
"Webshield status (Webshield.ENABLE, "
"WebshieldMode.wants_redirect, Webshield.SPLASH_SCREEN, "
"Webshield.PANEL_PROTECTION) changed from %s to %s",
self._webshield_status,
current_status,
)
self._webshield_status = current_status
recreate = True
current_port_blocking_mode = Firewall.port_blocking_mode
if self._port_blocking_mode != current_port_blocking_mode:
logger.info(
"Ports blocking mode changed from %s to %s",
self._port_blocking_mode,
current_port_blocking_mode,
)
self._port_blocking_mode = current_port_blocking_mode
recreate = True
current_dos_enabled = DOS.ENABLED or EnhancedDOS.ENABLED
if self._dos_enabled != current_dos_enabled:
logger.info(
"Effective DoS protection status changed "
"from %s to %s. Triggering rules recreation.",
self._dos_enabled,
current_dos_enabled,
)
self._dos_enabled = current_dos_enabled
recreate = True
refill_bp_ipest = self._port_blocking_mode == "DENY"
if recreate: # recreate everything
await self._rules_checker.recreate_rules_if_needed(
recreate_any_way=True
)
logger.info("Firewall rules recreated due to ConfigUpdate")
if refill_bp_ipest and await self._refill_port_blocking_ipsets():
logger.info("Blocked ports ipsets reffiled")
elif (
await self._update_port_blocking_deny_mode_ipsets_if_needed()
): # update just port blocking deny mode
logger.info("Blocked ports deny mode updated on ConfigUpdate")
async def _update_port_blocking_deny_mode_ipsets_if_needed(self):
updated = False
new_ports_blocking_values = self._get_port_blocking_deny_mode_values()
if new_ports_blocking_values != self._port_blocking_deny_mode_values:
logger.info(
"Port blocking deny mode changed from %s to %s",
_format_ports(self._port_blocking_deny_mode_values),
_format_ports(new_ports_blocking_values),
)
self._port_blocking_deny_mode_values = new_ports_blocking_values
updated = await self._refill_port_blocking_ipsets()
return updated
async def _refill_port_blocking_ipsets(self):
updated = True
async with self._rules_checker.lock_rules_create_destroy:
sets = [
InputPortBlockingDenyModeIPSet(),
OutputPortBlockingDenyModeIPSet(),
IPSetNoRedirectPort(),
]
for ip_set in sets:
# FIREWALL.port_blocking_mode supports only ipv4
try:
await ip_set.restore(IP.V4)
except IPSetError as e:
logger.error(
"Failed to update ipset %s: %s",
ip_set.__class__.__name__,
e,
)
updated = False
continue
return updated
def _log_ipsets_mismatch(missing_ipsets, redundant_ipsets, log):
"""Report missing/redundant ipsets."""
assert missing_ipsets or redundant_ipsets
log(
"Detected %s%s%s ipsets while ensuring ipsets/rules%s%s",
"missing" * bool(missing_ipsets),
"/" * bool(missing_ipsets and redundant_ipsets),
"redundant" * bool(redundant_ipsets),
f"; missing ipsets: {missing_ipsets}" * bool(missing_ipsets),
f"; redundant ipsets: {redundant_ipsets}" * bool(redundant_ipsets),
)
def _format_ports(ports):
"""Format ports for logging."""
# note: the order is asserted in unit tests
return json.dumps(
dict(
zip(
"TCP_IN_IPV4 TCP_OUT_IPV4 UDP_IN_IPV4 UDP_OUT_IPV4".split(),
ports,
)
)
)