diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py new file mode 100644 index 00000000..98f0316c --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py @@ -0,0 +1,144 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import ipaddress +import re + +# The prefixes the native code reads the client IP keys under, in the order +# it reads them. +PREFIXES = ("query.", "server.") + +# An IPv6 address in brackets, with or without a port, and an IPv4 address +# with a port. A port is one or more ASCII digits. +BRACKETED = re.compile(r"\[([^\[\]]*)\](?::([0-9]+))?") +WITH_PORT = re.compile(r"([0-9.]+):([0-9]+)") +MAX_PORT = 65535 + + +def client_ip_keys(keys): + + """! + The client IP evidence keys in the order the native code reads them, + every query key in the order given, then every server key in the + order given. + + @type keys: list + @param keys: the engine's evidence keys, in the data file's order + @rtype: list + """ + + return [key for prefix in PREFIXES for key in keys + if key.lower().startswith(prefix)] + + +def read_client_ip(value): + + """! + The address a client IP value holds, or None when the whole value is + not one address, an IPv4 address with a port, or an IPv6 address in + brackets with or without a port. A port is dropped. An IPv4-mapped IPv6 + address is its IPv4 address. A value holding a zone index is not read. + + @rtype: ipaddress.IPv4Address or ipaddress.IPv6Address or None + """ + + text = str(value) + # ipaddress reads a zone index, such as fe80::1%eth0, and the whole + # value must be refused rather than read with the zone dropped. The + # percent sign only ever starts a zone index here, in the plain and the + # %25 encoded form alike. + if "%" in text: + return None + version = None + port = None + match = BRACKETED.fullmatch(text) + if match: + text, port, version = match.group(1), match.group(2), 6 + else: + match = WITH_PORT.fullmatch(text) + if match: + text, port, version = match.group(1), match.group(2), 4 + if port is not None and int(port) > MAX_PORT: + return None + try: + address = ipaddress.ip_address(text) + except ValueError: + return None + if version is not None and address.version != version: + return None + if address.version == 6 and address.ipv4_mapped is not None: + address = address.ipv4_mapped + return address + + +def readable_client_ips(evidence, keys): + + """! + Each client IP value that is read, as (key, address), in the order the + native code reads the keys. A key is matched with the evidence in any + letter case, and keeps the spelling the engine gives it. + + @type evidence: dict + @param evidence: every evidence key and value + @type keys: list + @param keys: the engine's evidence keys, in the data file's order + @rtype: list + """ + + supplied = {} + for key, value in evidence.items(): + supplied.setdefault(str(key).lower(), value) + found = [] + for key in client_ip_keys(keys): + value = supplied.get(key.lower()) + if value is None: + continue + address = read_client_ip(value) + if address is not None: + found.append((key, address)) + return found + + +def client_ips(evidence, keys): + + """! + The address the engine reports in Ip or IpV6, being the first client IP + value that is read, which is the one the native code looks up, as the + Choosing the IP address and Reading the client IP value sections of the + specification say. A value that is not read does not hide a value under + a key that comes later. + + @type evidence: dict + @param evidence: every evidence key and value + @type keys: list + @param keys: the engine's evidence keys, in the data file's order + @rtype: tuple + @returns: (ipv4 or None, ipv6 or None), with at most one set + """ + + found = readable_client_ips(evidence, keys) + if not found: + return (None, None) + address = found[0][1] + if address.version == 4: + return (str(address), None) + return (None, str(address)) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index fc707307..865506ba 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -21,7 +21,6 @@ # ********************************************************************* from __future__ import absolute_import -import ipaddress import multiprocessing import os.path @@ -33,6 +32,7 @@ from .IpIntelligenceEngineModule import * +from .client_ip import client_ip_keys, client_ips, readable_client_ips from .swig_data import SwigData from .ipintelligence_datafile import IpIntelligenceDataFile @@ -257,9 +257,13 @@ def init_engine(self): evidence_keys_list = [] for x in range(evidence_keys.size()): - evidence_keys_list.append(evidence_keys.__getitem__(x).lower()) + evidence_keys_list.append(evidence_keys.__getitem__(x)) - self.evidence_keys_list = evidence_keys_list + # The client IP keys in the order the native code reads them, with + # the engine's own spelling, which the native code matches. + self.client_ip_keys = client_ip_keys(evidence_keys_list) + + self.evidence_keys_list = [key.lower() for key in evidence_keys_list] # Get properties list @@ -373,39 +377,15 @@ def get_client_ips(self, flow_data): """! - Get the IPv4 and IPv6 addresses from the evidence, using the same - priority order as the other IP intelligence engines: - query.client-ip first, then server.client-ip, then any other - parseable value in the evidence accepted by the engine. + The address the engine reports in Ip or IpV6 for the flow data's + evidence, which is the one the native code looks up. @rtype: tuple - @returns: (ipv4 or None, ipv6 or None) + @returns: (ipv4 or None, ipv6 or None), with at most one set """ - evidence = flow_data.evidence.get_all() - - ordered = [] - for key in ["query.client-ip", "server.client-ip"]: - if key in evidence: - ordered.append(evidence[key]) - for key, value in evidence.items(): - if value not in ordered: - ordered.append(value) - - ipv4 = None - ipv6 = None - for value in ordered: - try: - parsed = ipaddress.ip_address(str(value).strip()) - except ValueError: - continue - if parsed.version == 4 and ipv4 is None: - ipv4 = str(parsed) - elif parsed.version == 6 and ipv6 is None: - ipv6 = str(parsed) - - return (ipv4, ipv6) + return client_ips(flow_data.evidence.get_all(), self.client_ip_keys) def process_internal(self, flow_data): @@ -423,14 +403,27 @@ def process_internal(self, flow_data): if getattr(self, "engine", None) is None: raise Exception(ENGINE_CLOSED) + # Only the client IP values that are read reach the native code, + # each as its address alone, so the first of them is the address + # the native code looks up and the one reported in Ip or IpV6. + found = readable_client_ips( + flow_data.evidence.get_all(), self.client_ip_keys) + evidence = EvidenceIpiSwig() - for key, value in flow_data.evidence.get_all().items(): - evidence.__setitem__(key, value) + for key, address in found: + evidence.__setitem__(key, str(address)) result = self.engine.process(evidence) - ipv4, ipv6 = self.get_client_ips(flow_data) + ipv4 = None + ipv6 = None + if found: + address = found[0][1] + if address.version == 4: + ipv4 = str(address) + else: + ipv6 = str(address) data = SwigData(self, result, echo_ip=ipv4, echo_ipv6=ipv6) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py new file mode 100644 index 00000000..2b02fc8d --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py @@ -0,0 +1,154 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The client addresses the engine reports in Ip and IpV6, as the Choosing +the IP address, Reading the client IP value and Address forms sections of +the IP Intelligence on-premise specification say.""" + +import unittest + +from fiftyone_ipintelligence_onpremise.client_ip import client_ips + + +# The client IP keys as an engine might list them, in the data file's order +# with both prefixes mixed together. +KEYS = ["server.true-client-ip-51d", "query.true-client-ip-51d", + "server.client-ip", "query.client-ip"] + + +class ClientIpTests(unittest.TestCase): + + def test_an_ipv4_mapped_address_is_its_ipv4_address(self): + for text in ("::ffff:8.8.8.8", "::ffff:808:808", + "0:0:0:0:0:ffff:808:808"): + self.assertEqual( + ("8.8.8.8", None), + client_ips({"query.client-ip": text}, KEYS), text) + + def test_ipv4_compatible_and_embedded_addresses_stay_ipv6(self): + self.assertEqual( + (None, "::808:808"), + client_ips({"query.client-ip": "::8.8.8.8"}, KEYS)) + self.assertEqual( + (None, "64:ff9b::808:808"), + client_ips({"query.client-ip": "64:ff9b::8.8.8.8"}, KEYS)) + + def test_an_address_with_spaces_is_not_read(self): + self.assertEqual( + (None, None), client_ips({"query.client-ip": " 8.8.8.8 "}, KEYS)) + + def test_a_zone_index_is_not_read(self): + for text in ("fe80::1%eth0", "fe80::1%1", "[fe80::1%eth0]", + "[fe80::1%25eth0]", "::ffff:8.8.8.8%1", + "::ffff:8.8.8.8%eth0"): + self.assertEqual( + (None, None), + client_ips({"query.client-ip": text}, KEYS), text) + + def test_a_zone_index_does_not_hide_a_later_value(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"query.client-ip": "fe80::1%eth0", + "server.client-ip": "1.2.3.4"}, KEYS)) + + def test_the_client_ip_query_comes_first(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"server.client-ip": "5.6.7.8", + "query.client-ip": "1.2.3.4"}, KEYS)) + + def test_a_leading_zero_is_not_read(self): + # The rule is kept by ipaddress, which refuses a leading zero, so + # this catches a Python that reads one. + for text in ("01.2.3.4", "1.02.3.4", "1.2.3.04", "00.0.0.0", + "::ffff:01.2.3.4"): + self.assertEqual( + (None, None), + client_ips({"query.client-ip": text}, KEYS), text) + self.assertEqual( + ("0.0.0.0", None), client_ips({"query.client-ip": "0.0.0.0"}, KEYS)) + + +class ClientIpChoiceTests(unittest.TestCase): + + def test_only_the_client_ip_keys_are_read(self): + self.assertEqual( + (None, None), + client_ips({"header.x-real-ip": "1.2.3.4"}, KEYS)) + + def test_the_query_keys_come_first_in_the_data_files_order(self): + self.assertEqual( + ("9.9.9.9", None), + client_ips({"server.true-client-ip-51d": "5.6.7.8", + "query.client-ip": "1.2.3.4", + "query.true-client-ip-51d": "9.9.9.9"}, KEYS)) + + def test_the_server_keys_are_read_when_no_query_key_has_an_address(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"server.client-ip": "5.6.7.8", + "server.true-client-ip-51d": "1.2.3.4", + "query.client-ip": "unknown"}, KEYS)) + + def test_the_keys_match_in_any_letter_case(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"Server.Client-IP": "5.6.7.8", + "QUERY.CLIENT-IP": "1.2.3.4"}, KEYS)) + + def test_only_the_address_looked_up_is_reported(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"query.client-ip": "1.2.3.4", + "server.client-ip": "2001:db8::1"}, KEYS)) + + def test_a_port_and_brackets_are_read_and_the_port_dropped(self): + for text, expected in ( + ("203.0.113.1:8080", ("203.0.113.1", None)), + ("203.0.113.1:0", ("203.0.113.1", None)), + ("203.0.113.1:65535", ("203.0.113.1", None)), + ("[2001:db8::1]", (None, "2001:db8::1")), + ("[2001:db8::1]:8080", (None, "2001:db8::1")), + ("[::ffff:8.8.8.8]:80", ("8.8.8.8", None))): + self.assertEqual( + expected, client_ips({"query.client-ip": text}, KEYS), text) + + def test_an_ipv6_address_without_brackets_has_no_port(self): + self.assertEqual( + (None, "2001:db8::1:80"), + client_ips({"query.client-ip": "2001:db8::1:80"}, KEYS)) + + def test_the_values_the_specification_does_not_read(self): + for text in ("203.0.113.0/24", "203.0.113.1 x", + "203.0.113.1, 198.51.100.7", "203.0.113.1 ", + "[203.0.113.1]", "[2001:db8::1", "2001:db8::1]:80", + "203.0.113.1:", "203.0.113.1:65536", + "203.0.113.1:８０", "256.0.113.1", + "203.0.113", "2001:db8::1::2", "2001:db8:1:2:3:4:5:6:7", + "2001:db8::12345", "unknown", "", "[fe80::1%eth0]:80"): + self.assertEqual( + (None, None), client_ips({"query.client-ip": text}, KEYS), + text) + + +if __name__ == "__main__": + unittest.main()