From a4808de083b896db88f88fb9e62fa2f04faf3765 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 20:26:14 +0000 Subject: [PATCH 1/9] Read the client addresses in their own module, with no change in what is read --- .../client_ip.py | 62 +++++++++++++++++++ .../ipintelligence_onpremise.py | 32 ++-------- 2 files changed, 66 insertions(+), 28 deletions(-) create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py new file mode 100644 index 00000000..12ae16ce --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py @@ -0,0 +1,62 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import ipaddress + +# The evidence keys read first, in order. +CLIENT_IP_KEYS = ("query.client-ip", "server.client-ip") + + +def client_ips(evidence): + + """! + The IPv4 and IPv6 addresses the engine reports in Ip and IpV6, read from + the evidence: query.client-ip first, then server.client-ip, then any + other value that is an address. + + @type evidence: dict + @param evidence: evidence key to value + @rtype: tuple + @returns: (ipv4 or None, ipv6 or None) + """ + + ordered = [] + for key in CLIENT_IP_KEYS: + if key in evidence: + ordered.append(evidence[key]) + for key, value in evidence.items(): + if value not in ordered: + ordered.append(value) + + ipv4 = None + ipv6 = None + for value in ordered: + try: + parsed = ipaddress.ip_address(str(value).strip()) + except ValueError: + continue + if parsed.version == 4 and ipv4 is None: + ipv4 = str(parsed) + elif parsed.version == 6 and ipv6 is None: + ipv6 = str(parsed) + + return (ipv4, ipv6) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index 9f15784e..3b5f029d 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -21,7 +21,6 @@ # ********************************************************************* from __future__ import absolute_import -import ipaddress import multiprocessing import os.path @@ -33,6 +32,7 @@ from .IpIntelligenceEngineModule import * +from .client_ip import client_ips from .swig_data import SwigData from .ipintelligence_datafile import IpIntelligenceDataFile @@ -348,39 +348,15 @@ def get_client_ips(self, flow_data): """! - Get the IPv4 and IPv6 addresses from the evidence, using the same - priority order as the other IP intelligence engines: - query.client-ip first, then server.client-ip, then any other - parseable value in the evidence accepted by the engine. + Get the IPv4 and IPv6 addresses from the evidence accepted by the + engine, as client_ips reads them. @rtype: tuple @returns: (ipv4 or None, ipv6 or None) """ - evidence = flow_data.evidence.get_all() - - ordered = [] - for key in ["query.client-ip", "server.client-ip"]: - if key in evidence: - ordered.append(evidence[key]) - for key, value in evidence.items(): - if value not in ordered: - ordered.append(value) - - ipv4 = None - ipv6 = None - for value in ordered: - try: - parsed = ipaddress.ip_address(str(value).strip()) - except ValueError: - continue - if parsed.version == 4 and ipv4 is None: - ipv4 = str(parsed) - elif parsed.version == 6 and ipv6 is None: - ipv6 = str(parsed) - - return (ipv4, ipv6) + return client_ips(flow_data.evidence.get_all()) def process_internal(self, flow_data): From ab33fbac007e75be5288e8ee557df9a7e58bc6a5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 20:26:15 +0000 Subject: [PATCH 2/9] Test that an IPv4-mapped address is reported as Ip and an address with spaces is not read --- .../tests/test_client_ip.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py new file mode 100644 index 00000000..d863d8cc --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py @@ -0,0 +1,60 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The client addresses the engine reports in Ip and IpV6, as the Address +forms section of the IP Intelligence on-premise specification says.""" + +import unittest + +from fiftyone_ipintelligence_onpremise.client_ip import client_ips + + +class ClientIpTests(unittest.TestCase): + + def test_an_ipv4_mapped_address_is_its_ipv4_address(self): + for text in ("::ffff:8.8.8.8", "::ffff:808:808", + "0:0:0:0:0:ffff:808:808"): + self.assertEqual( + ("8.8.8.8", None), + client_ips({"query.client-ip": text}), text) + + def test_ipv4_compatible_and_embedded_addresses_stay_ipv6(self): + self.assertEqual( + (None, "::808:808"), + client_ips({"query.client-ip": "::8.8.8.8"})) + self.assertEqual( + (None, "64:ff9b::808:808"), + client_ips({"query.client-ip": "64:ff9b::8.8.8.8"})) + + def test_an_address_with_spaces_is_not_read(self): + self.assertEqual( + (None, None), client_ips({"query.client-ip": " 8.8.8.8 "})) + + def test_the_client_ip_query_comes_first(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"server.client-ip": "5.6.7.8", + "query.client-ip": "1.2.3.4"})) + + +if __name__ == "__main__": + unittest.main() From f87ad74af54d6b7500b9649d9bd79d79e0a19ac6 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 20:26:16 +0000 Subject: [PATCH 3/9] Report an IPv4-mapped address as its IPv4 address, and read an address as given, as the Address forms section says --- .../src/fiftyone_ipintelligence_onpremise/client_ip.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py index 12ae16ce..244f4624 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py @@ -30,8 +30,10 @@ def client_ips(evidence): """! The IPv4 and IPv6 addresses the engine reports in Ip and IpV6, read from - the evidence: query.client-ip first, then server.client-ip, then any - other value that is an address. + the evidence, query.client-ip first, then server.client-ip, then any + other value that is an address as given. An IPv4-mapped IPv6 address, + such as ::ffff:8.8.8.8, is its IPv4 address and is reported as Ip, as + the native code looks it up. @type evidence: dict @param evidence: evidence key to value @@ -51,9 +53,11 @@ def client_ips(evidence): ipv6 = None for value in ordered: try: - parsed = ipaddress.ip_address(str(value).strip()) + parsed = ipaddress.ip_address(str(value)) except ValueError: continue + if parsed.version == 6 and parsed.ipv4_mapped is not None: + parsed = parsed.ipv4_mapped if parsed.version == 4 and ipv4 is None: ipv4 = str(parsed) elif parsed.version == 6 and ipv6 is None: From 30b987e9b1836cbee1f6bac702b4dd23332858da Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 00:26:06 +0000 Subject: [PATCH 4/9] Test that a value holding a zone index is not read --- .../tests/test_client_ip.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py index d863d8cc..18ab4828 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py @@ -49,6 +49,20 @@ def test_an_address_with_spaces_is_not_read(self): self.assertEqual( (None, None), client_ips({"query.client-ip": " 8.8.8.8 "})) + def test_a_zone_index_is_not_read(self): + for text in ("fe80::1%eth0", "fe80::1%1", "[fe80::1%eth0]", + "[fe80::1%25eth0]", "::ffff:8.8.8.8%1", + "::ffff:8.8.8.8%eth0"): + self.assertEqual( + (None, None), + client_ips({"query.client-ip": text}), text) + + def test_a_zone_index_does_not_hide_a_later_value(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"query.client-ip": "fe80::1%eth0", + "server.client-ip": "1.2.3.4"})) + def test_the_client_ip_query_comes_first(self): self.assertEqual( ("1.2.3.4", None), From 7aa71eff3c4ecf78b5d6e31e704ed97570114779 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 00:26:15 +0000 Subject: [PATCH 5/9] Refuse a client IP value that holds a zone index, as the Zone indexes section says --- .../fiftyone_ipintelligence_onpremise/client_ip.py | 14 +++++++++++--- .../ipintelligence_onpremise.py | 4 ++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py index 244f4624..1a7ce4c2 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py @@ -33,10 +33,11 @@ def client_ips(evidence): the evidence, query.client-ip first, then server.client-ip, then any other value that is an address as given. An IPv4-mapped IPv6 address, such as ::ffff:8.8.8.8, is its IPv4 address and is reported as Ip, as - the native code looks it up. + the native code looks it up. A value holding a zone index, such as + fe80::1%eth0, is not read at all. @type evidence: dict - @param evidence: evidence key to value + @param evidence: every evidence key and value @rtype: tuple @returns: (ipv4 or None, ipv6 or None) """ @@ -52,8 +53,15 @@ def client_ips(evidence): ipv4 = None ipv6 = None for value in ordered: + text = str(value) + # ipaddress reads a zone index, such as fe80::1%eth0, and the + # engine must refuse the whole value rather than read the address + # and drop the zone. The percent sign only ever starts a zone + # index here, in the plain and the %25 encoded form alike. + if "%" in text: + continue try: - parsed = ipaddress.ip_address(str(value)) + parsed = ipaddress.ip_address(text) except ValueError: continue if parsed.version == 6 and parsed.ipv4_mapped is not None: diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index 3b5f029d..c288a114 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -348,8 +348,8 @@ def get_client_ips(self, flow_data): """! - Get the IPv4 and IPv6 addresses from the evidence accepted by the - engine, as client_ips reads them. + Get the IPv4 and IPv6 addresses from every evidence key and value + the flow data holds, as client_ips reads them. @rtype: tuple @returns: (ipv4 or None, ipv6 or None) From 848b54f6f14b0ff5b3c646776997b1c640e3bbc2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 08:09:21 +0000 Subject: [PATCH 6/9] Test that an IPv4 address with a leading zero is not read Reading the client IP value says the Engine must not read an IPv4 address with a leading zero in any part, such as 01.2.3.4, and must read 0.0.0.0. ipaddress refuses a leading zero, so the rule holds today, and this test catches a Python whose ipaddress reads one. --- .../tests/test_client_ip.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py index 18ab4828..e492dde8 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py @@ -69,6 +69,17 @@ def test_the_client_ip_query_comes_first(self): client_ips({"server.client-ip": "5.6.7.8", "query.client-ip": "1.2.3.4"})) + def test_a_leading_zero_is_not_read(self): + # The rule is kept by ipaddress, which refuses a leading zero, so + # this catches a Python that reads one. + for text in ("01.2.3.4", "1.02.3.4", "1.2.3.04", "00.0.0.0", + "::ffff:01.2.3.4"): + self.assertEqual( + (None, None), + client_ips({"query.client-ip": text}), text) + self.assertEqual( + ("0.0.0.0", None), client_ips({"query.client-ip": "0.0.0.0"})) + if __name__ == "__main__": unittest.main() From acd4b008f2b8dcf7d52e5ccf9335fcdffa5cdd6a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 08:56:20 +0000 Subject: [PATCH 7/9] Let client_ips be given the engine's client IP keys A structural change only, so the tests that follow can give the keys. Nothing reads the parameter yet. --- .../src/fiftyone_ipintelligence_onpremise/client_ip.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py index 1a7ce4c2..850c3f1e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py @@ -26,7 +26,7 @@ CLIENT_IP_KEYS = ("query.client-ip", "server.client-ip") -def client_ips(evidence): +def client_ips(evidence, keys=None): """! The IPv4 and IPv6 addresses the engine reports in Ip and IpV6, read from @@ -38,6 +38,8 @@ def client_ips(evidence): @type evidence: dict @param evidence: every evidence key and value + @type keys: list + @param keys: the engine's client IP evidence keys, not yet used @rtype: tuple @returns: (ipv4 or None, ipv6 or None) """ From 7e60be858eb1980ca090f64b0fb602647e856830 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 08:57:15 +0000 Subject: [PATCH 8/9] Test that the client address is chosen as the native code chooses it Choosing the IP address says the native code uses one value, from the client IP keys the data file lists, the query keys first in the data file's order and the server keys only when no query key supplied an address, and that the Property values describe that one address. Reading the client IP value reads a port, which is dropped, and an IPv6 address in brackets. evidence.md says keys are case insensitive. The tests give client_ips the keys as an engine lists them and check each of those, and that only the one address looked up is reported. This commit is the tests alone, so six of them fail. --- .../tests/test_client_ip.py | 73 ++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py index e492dde8..c8c6f764 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py @@ -20,8 +20,9 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -"""The client addresses the engine reports in Ip and IpV6, as the Address -forms section of the IP Intelligence on-premise specification says.""" +"""The client addresses the engine reports in Ip and IpV6, as the Choosing +the IP address, Reading the client IP value and Address forms sections of +the IP Intelligence on-premise specification say.""" import unittest @@ -81,5 +82,73 @@ def test_a_leading_zero_is_not_read(self): ("0.0.0.0", None), client_ips({"query.client-ip": "0.0.0.0"})) +# The client IP keys as an engine might list them, in the data file's order +# with both prefixes mixed together. +KEYS = ["server.true-client-ip-51d", "query.true-client-ip-51d", + "server.client-ip", "query.client-ip"] + + +class ClientIpChoiceTests(unittest.TestCase): + + def test_only_the_client_ip_keys_are_read(self): + self.assertEqual( + (None, None), + client_ips({"header.x-real-ip": "1.2.3.4"}, KEYS)) + + def test_the_query_keys_come_first_in_the_data_files_order(self): + self.assertEqual( + ("9.9.9.9", None), + client_ips({"server.true-client-ip-51d": "5.6.7.8", + "query.client-ip": "1.2.3.4", + "query.true-client-ip-51d": "9.9.9.9"}, KEYS)) + + def test_the_server_keys_are_read_when_no_query_key_has_an_address(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"server.client-ip": "5.6.7.8", + "server.true-client-ip-51d": "1.2.3.4", + "query.client-ip": "unknown"}, KEYS)) + + def test_the_keys_match_in_any_letter_case(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"Server.Client-IP": "5.6.7.8", + "QUERY.CLIENT-IP": "1.2.3.4"}, KEYS)) + + def test_only_the_address_looked_up_is_reported(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"query.client-ip": "1.2.3.4", + "server.client-ip": "2001:db8::1"}, KEYS)) + + def test_a_port_and_brackets_are_read_and_the_port_dropped(self): + for text, expected in ( + ("203.0.113.1:8080", ("203.0.113.1", None)), + ("203.0.113.1:0", ("203.0.113.1", None)), + ("203.0.113.1:65535", ("203.0.113.1", None)), + ("[2001:db8::1]", (None, "2001:db8::1")), + ("[2001:db8::1]:8080", (None, "2001:db8::1")), + ("[::ffff:8.8.8.8]:80", ("8.8.8.8", None))): + self.assertEqual( + expected, client_ips({"query.client-ip": text}, KEYS), text) + + def test_an_ipv6_address_without_brackets_has_no_port(self): + self.assertEqual( + (None, "2001:db8::1:80"), + client_ips({"query.client-ip": "2001:db8::1:80"}, KEYS)) + + def test_the_values_the_specification_does_not_read(self): + for text in ("203.0.113.0/24", "203.0.113.1 x", + "203.0.113.1, 198.51.100.7", "203.0.113.1 ", + "[203.0.113.1]", "[2001:db8::1", "2001:db8::1]:80", + "203.0.113.1:", "203.0.113.1:65536", + "203.0.113.1:８０", "256.0.113.1", + "203.0.113", "2001:db8::1::2", "2001:db8:1:2:3:4:5:6:7", + "2001:db8::12345", "unknown", "", "[fe80::1%eth0]:80"): + self.assertEqual( + (None, None), client_ips({"query.client-ip": text}, KEYS), + text) + + if __name__ == "__main__": unittest.main() From beb0d770165c092e05f01be7b945636e2fb6c591 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 08:59:38 +0000 Subject: [PATCH 9/9] Choose the client address as the native code does, and report only that one The engine reported any evidence value that was an address, from a fixed pair of keys first and then from any key at all, matched with letter case, and it could not read a port or brackets. So the Ip or IpV6 it reported could differ from the address the native code looked up, which the other Property values describe. As .NET's IpiOnPremiseEngine does, the engine now takes the client IP keys from the native engine's own list, every query key in the data file's order and then every server key, and reads each from the evidence in any letter case. read_client_ip reads a whole value only, being an address, an IPv4 address with a port, or an IPv6 address in brackets with or without a port, and drops the port, as Reading the client IP value says. Only the values read reach the native code, each as its address alone under the engine's spelling of the key, so the first of them is the address the native code looks up, and that one address is reported, in Ip or in IpV6. --- .../client_ip.py | 148 +++++++++++++----- .../ipintelligence_onpremise.py | 37 +++-- .../tests/test_client_ip.py | 30 ++-- 3 files changed, 150 insertions(+), 65 deletions(-) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py index 850c3f1e..98f0316c 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py @@ -21,56 +21,124 @@ # ********************************************************************* import ipaddress +import re -# The evidence keys read first, in order. -CLIENT_IP_KEYS = ("query.client-ip", "server.client-ip") +# The prefixes the native code reads the client IP keys under, in the order +# it reads them. +PREFIXES = ("query.", "server.") +# An IPv6 address in brackets, with or without a port, and an IPv4 address +# with a port. A port is one or more ASCII digits. +BRACKETED = re.compile(r"\[([^\[\]]*)\](?::([0-9]+))?") +WITH_PORT = re.compile(r"([0-9.]+):([0-9]+)") +MAX_PORT = 65535 -def client_ips(evidence, keys=None): + +def client_ip_keys(keys): """! - The IPv4 and IPv6 addresses the engine reports in Ip and IpV6, read from - the evidence, query.client-ip first, then server.client-ip, then any - other value that is an address as given. An IPv4-mapped IPv6 address, - such as ::ffff:8.8.8.8, is its IPv4 address and is reported as Ip, as - the native code looks it up. A value holding a zone index, such as - fe80::1%eth0, is not read at all. + The client IP evidence keys in the order the native code reads them, + every query key in the order given, then every server key in the + order given. + + @type keys: list + @param keys: the engine's evidence keys, in the data file's order + @rtype: list + """ + + return [key for prefix in PREFIXES for key in keys + if key.lower().startswith(prefix)] + + +def read_client_ip(value): + + """! + The address a client IP value holds, or None when the whole value is + not one address, an IPv4 address with a port, or an IPv6 address in + brackets with or without a port. A port is dropped. An IPv4-mapped IPv6 + address is its IPv4 address. A value holding a zone index is not read. + + @rtype: ipaddress.IPv4Address or ipaddress.IPv6Address or None + """ + + text = str(value) + # ipaddress reads a zone index, such as fe80::1%eth0, and the whole + # value must be refused rather than read with the zone dropped. The + # percent sign only ever starts a zone index here, in the plain and the + # %25 encoded form alike. + if "%" in text: + return None + version = None + port = None + match = BRACKETED.fullmatch(text) + if match: + text, port, version = match.group(1), match.group(2), 6 + else: + match = WITH_PORT.fullmatch(text) + if match: + text, port, version = match.group(1), match.group(2), 4 + if port is not None and int(port) > MAX_PORT: + return None + try: + address = ipaddress.ip_address(text) + except ValueError: + return None + if version is not None and address.version != version: + return None + if address.version == 6 and address.ipv4_mapped is not None: + address = address.ipv4_mapped + return address + + +def readable_client_ips(evidence, keys): + + """! + Each client IP value that is read, as (key, address), in the order the + native code reads the keys. A key is matched with the evidence in any + letter case, and keeps the spelling the engine gives it. @type evidence: dict @param evidence: every evidence key and value @type keys: list - @param keys: the engine's client IP evidence keys, not yet used - @rtype: tuple - @returns: (ipv4 or None, ipv6 or None) + @param keys: the engine's evidence keys, in the data file's order + @rtype: list """ - ordered = [] - for key in CLIENT_IP_KEYS: - if key in evidence: - ordered.append(evidence[key]) + supplied = {} for key, value in evidence.items(): - if value not in ordered: - ordered.append(value) - - ipv4 = None - ipv6 = None - for value in ordered: - text = str(value) - # ipaddress reads a zone index, such as fe80::1%eth0, and the - # engine must refuse the whole value rather than read the address - # and drop the zone. The percent sign only ever starts a zone - # index here, in the plain and the %25 encoded form alike. - if "%" in text: + supplied.setdefault(str(key).lower(), value) + found = [] + for key in client_ip_keys(keys): + value = supplied.get(key.lower()) + if value is None: continue - try: - parsed = ipaddress.ip_address(text) - except ValueError: - continue - if parsed.version == 6 and parsed.ipv4_mapped is not None: - parsed = parsed.ipv4_mapped - if parsed.version == 4 and ipv4 is None: - ipv4 = str(parsed) - elif parsed.version == 6 and ipv6 is None: - ipv6 = str(parsed) - - return (ipv4, ipv6) + address = read_client_ip(value) + if address is not None: + found.append((key, address)) + return found + + +def client_ips(evidence, keys): + + """! + The address the engine reports in Ip or IpV6, being the first client IP + value that is read, which is the one the native code looks up, as the + Choosing the IP address and Reading the client IP value sections of the + specification say. A value that is not read does not hide a value under + a key that comes later. + + @type evidence: dict + @param evidence: every evidence key and value + @type keys: list + @param keys: the engine's evidence keys, in the data file's order + @rtype: tuple + @returns: (ipv4 or None, ipv6 or None), with at most one set + """ + + found = readable_client_ips(evidence, keys) + if not found: + return (None, None) + address = found[0][1] + if address.version == 4: + return (str(address), None) + return (None, str(address)) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index c288a114..fa23e2ee 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -32,7 +32,7 @@ from .IpIntelligenceEngineModule import * -from .client_ip import client_ips +from .client_ip import client_ip_keys, client_ips, readable_client_ips from .swig_data import SwigData from .ipintelligence_datafile import IpIntelligenceDataFile @@ -247,9 +247,13 @@ def init_engine(self): evidence_keys_list = [] for x in range(evidence_keys.size()): - evidence_keys_list.append(evidence_keys.__getitem__(x).lower()) + evidence_keys_list.append(evidence_keys.__getitem__(x)) - self.evidence_keys_list = evidence_keys_list + # The client IP keys in the order the native code reads them, with + # the engine's own spelling, which the native code matches. + self.client_ip_keys = client_ip_keys(evidence_keys_list) + + self.evidence_keys_list = [key.lower() for key in evidence_keys_list] # Get properties list @@ -348,15 +352,15 @@ def get_client_ips(self, flow_data): """! - Get the IPv4 and IPv6 addresses from every evidence key and value - the flow data holds, as client_ips reads them. + The address the engine reports in Ip or IpV6 for the flow data's + evidence, which is the one the native code looks up. @rtype: tuple - @returns: (ipv4 or None, ipv6 or None) + @returns: (ipv4 or None, ipv6 or None), with at most one set """ - return client_ips(flow_data.evidence.get_all()) + return client_ips(flow_data.evidence.get_all(), self.client_ip_keys) def process_internal(self, flow_data): @@ -374,14 +378,27 @@ def process_internal(self, flow_data): if getattr(self, "engine", None) is None: raise Exception(ENGINE_CLOSED) + # Only the client IP values that are read reach the native code, + # each as its address alone, so the first of them is the address + # the native code looks up and the one reported in Ip or IpV6. + found = readable_client_ips( + flow_data.evidence.get_all(), self.client_ip_keys) + evidence = EvidenceIpiSwig() - for key, value in flow_data.evidence.get_all().items(): - evidence.__setitem__(key, value) + for key, address in found: + evidence.__setitem__(key, str(address)) result = self.engine.process(evidence) - ipv4, ipv6 = self.get_client_ips(flow_data) + ipv4 = None + ipv6 = None + if found: + address = found[0][1] + if address.version == 4: + ipv4 = str(address) + else: + ipv6 = str(address) data = SwigData(self, result, echo_ip=ipv4, echo_ipv6=ipv6) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py index c8c6f764..2b02fc8d 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py @@ -29,6 +29,12 @@ from fiftyone_ipintelligence_onpremise.client_ip import client_ips +# The client IP keys as an engine might list them, in the data file's order +# with both prefixes mixed together. +KEYS = ["server.true-client-ip-51d", "query.true-client-ip-51d", + "server.client-ip", "query.client-ip"] + + class ClientIpTests(unittest.TestCase): def test_an_ipv4_mapped_address_is_its_ipv4_address(self): @@ -36,19 +42,19 @@ def test_an_ipv4_mapped_address_is_its_ipv4_address(self): "0:0:0:0:0:ffff:808:808"): self.assertEqual( ("8.8.8.8", None), - client_ips({"query.client-ip": text}), text) + client_ips({"query.client-ip": text}, KEYS), text) def test_ipv4_compatible_and_embedded_addresses_stay_ipv6(self): self.assertEqual( (None, "::808:808"), - client_ips({"query.client-ip": "::8.8.8.8"})) + client_ips({"query.client-ip": "::8.8.8.8"}, KEYS)) self.assertEqual( (None, "64:ff9b::808:808"), - client_ips({"query.client-ip": "64:ff9b::8.8.8.8"})) + client_ips({"query.client-ip": "64:ff9b::8.8.8.8"}, KEYS)) def test_an_address_with_spaces_is_not_read(self): self.assertEqual( - (None, None), client_ips({"query.client-ip": " 8.8.8.8 "})) + (None, None), client_ips({"query.client-ip": " 8.8.8.8 "}, KEYS)) def test_a_zone_index_is_not_read(self): for text in ("fe80::1%eth0", "fe80::1%1", "[fe80::1%eth0]", @@ -56,19 +62,19 @@ def test_a_zone_index_is_not_read(self): "::ffff:8.8.8.8%eth0"): self.assertEqual( (None, None), - client_ips({"query.client-ip": text}), text) + client_ips({"query.client-ip": text}, KEYS), text) def test_a_zone_index_does_not_hide_a_later_value(self): self.assertEqual( ("1.2.3.4", None), client_ips({"query.client-ip": "fe80::1%eth0", - "server.client-ip": "1.2.3.4"})) + "server.client-ip": "1.2.3.4"}, KEYS)) def test_the_client_ip_query_comes_first(self): self.assertEqual( ("1.2.3.4", None), client_ips({"server.client-ip": "5.6.7.8", - "query.client-ip": "1.2.3.4"})) + "query.client-ip": "1.2.3.4"}, KEYS)) def test_a_leading_zero_is_not_read(self): # The rule is kept by ipaddress, which refuses a leading zero, so @@ -77,15 +83,9 @@ def test_a_leading_zero_is_not_read(self): "::ffff:01.2.3.4"): self.assertEqual( (None, None), - client_ips({"query.client-ip": text}), text) + client_ips({"query.client-ip": text}, KEYS), text) self.assertEqual( - ("0.0.0.0", None), client_ips({"query.client-ip": "0.0.0.0"})) - - -# The client IP keys as an engine might list them, in the data file's order -# with both prefixes mixed together. -KEYS = ["server.true-client-ip-51d", "query.true-client-ip-51d", - "server.client-ip", "query.client-ip"] + ("0.0.0.0", None), client_ips({"query.client-ip": "0.0.0.0"}, KEYS)) class ClientIpChoiceTests(unittest.TestCase):