diff --git a/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml index eee03edfd..0e59101da 100644 --- a/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -7,6 +7,11 @@ name: Cloud 51Did Tests # the cloud for a 51Did and fail, saying so, when a key gets an answer with no # identifier in it. That is entitlement working rather than a fault. # +# A run that has a license key also runs the Cloud Request Engine's live test +# on a license key alone (pipeline/pipeline.cloudrequestengine, +# CloudRequestEngineLiveTests), because no other workflow has a license key +# for the cloud service. Without one that test skips. +# # The secrets are named one by one rather than passed as toJSON(secrets). # GitHub marks a workflow that hands every secret to a step as possibly # malicious and holds each run for manual approval, so the nightly never ran. @@ -32,6 +37,7 @@ on: branches: [main, rc] paths: - 'pipeline/pipeline.did/**' + - 'pipeline/pipeline.cloudrequestengine/**' - 'pipeline/owid-java/**' - '.github/workflows/cloud-51did.yml' @@ -73,7 +79,7 @@ jobs: } $failed = $false foreach ($k in $keys) { - Write-Host "::group::DidClientLiveTests with $($k.Name)" + Write-Host "::group::Live tests with $($k.Name)" $env:_51DEGREES_RESOURCE_KEY = $k.Value $licenseName = $k.Name -replace '^_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY' $license = [Environment]::GetEnvironmentVariable($licenseName) @@ -85,8 +91,10 @@ jobs: Remove-Item Env:_51DEGREES_LICENSE_KEY -ErrorAction SilentlyContinue } # Only the Pipeline API tree is built, as in its own repository. - mvn -B -f pipeline/pom.xml -pl pipeline.did -am test ` - -Dtest=DidClientLiveTests -DfailIfNoTests=false + mvn -B -f pipeline/pom.xml ` + -pl pipeline.did,pipeline.cloudrequestengine -am test ` + '-Dtest=DidClientLiveTests,CloudRequestEngineLiveTests#build_Process_LicenseKeyAlone*' ` + -DfailIfNoTests=false if ($LASTEXITCODE -ne 0) { $failed = $true } Write-Host '::endgroup::' } diff --git a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilder.java b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilder.java index 9d1e2ecfc..a063a588d 100644 --- a/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilder.java +++ b/device-detection/device-detection/src/main/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilder.java @@ -93,6 +93,10 @@ public Pipeline build() throws Exception { if (licenseKey != null && licenseKey.isEmpty() == false) { cloudRequestEngineBuilder.setLicenseKey(licenseKey); } + if (requestedProperties.isEmpty() == false) { + cloudRequestEngineBuilder.setRequestedProperties( + requestedProperties); + } if (cloudRequestOrigin != null && cloudRequestOrigin.isEmpty() == false) { cloudRequestEngineBuilder.setCloudRequestOrigin(cloudRequestOrigin); } diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilderTests.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilderTests.java new file mode 100644 index 000000000..2d9e0cbd3 --- /dev/null +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionCloudPipelineBuilderTests.java @@ -0,0 +1,252 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection; + +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * The cloud Pipeline builder passes the keys and the requested properties + * to the Cloud Request Engine, so a Pipeline can authenticate on a license + * key alone. A fake cloud service records each data request. + */ +public class DeviceDetectionCloudPipelineBuilderTests { + + private static final String BASE = "https://cloud.example/api/v4/"; + private static final String LICENSE_KEY = "abcdefghijkl1234"; + private static final String RESOURCE_KEY = "resourcekey"; + private static final String FIRST = "device.ismobile"; + private static final String SECOND = "device.iscrawler"; + private static final String PROPERTIES_RESPONSE = + "{\"Products\":{\"device\":{\"DataTier\":\"t\",\"Properties\":" + + "[{\"Name\":\"IsMobile\",\"Type\":\"Boolean\"," + + "\"Category\":\"c\"}]}}}"; + private static final String DATA_RESPONSE = "{\"device\":{\"ismobile\":true}}"; + + /** Records every data request and answers from fixed responses. */ + private static class FakeCloud implements HttpClient { + final List dataRequests = + Collections.synchronizedList(new ArrayList()); + + @Override + public HttpURLConnection connect(URL url) { + return new HttpURLConnection(url) { + @Override + public void connect() { + } + + @Override + public void disconnect() { + } + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public int getResponseCode() { + return 200; + } + }; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + return answer( + connection.getURL(), + data == null ? "" : + new String(data, StandardCharsets.UTF_8)); + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return answer(connection.getURL(), ""); + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + return answer(connection.getURL(), ""); + } + + private String answer(URL url, String body) { + String path = url.getPath(); + if (path.endsWith("accessibleproperties")) { + return PROPERTIES_RESPONSE; + } + if (path.endsWith("evidencekeys")) { + return "[\"header.user-agent\"]"; + } + dataRequests.add(body); + return DATA_RESPONSE; + } + } + + private FakeCloud cloud; + + @Before + public void setUp() { + cloud = new FakeCloud(); + } + + private DeviceDetectionCloudPipelineBuilder builder(String resourceKey) { + return new DeviceDetectionPipelineBuilder( + LoggerFactory.getILoggerFactory(), cloud) + .useCloud(resourceKey) + .setEndPoint(BASE); + } + + /** Process one request and give the fields of the form it sent. */ + private Map process(DeviceDetectionCloudPipelineBuilder builder) + throws Exception { + try (Pipeline pipeline = builder.build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "iPhone").process(); + assertTrue(data.get(DeviceData.class) + .getIsMobile().getValue()); + } + assertEquals(1, cloud.dataRequests.size()); + return form(cloud.dataRequests.get(0)); + } + + private static Map form(String body) + throws UnsupportedEncodingException { + Map fields = new HashMap<>(); + for (String pair : body.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] parts = pair.split("=", 2); + fields.put( + URLDecoder.decode(parts[0], "UTF-8"), + URLDecoder.decode(parts.length > 1 ? parts[1] : "", "UTF-8")); + } + return fields; + } + + @Test + public void LicenseKeyAlone_RequestedProperties_SentAsValues() + throws Exception { + Map sent = process(builder(null) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList(FIRST, SECOND))); + + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST + "," + SECOND, sent.get("values")); + assertFalse(sent.containsKey("resource")); + } + + @Test + public void LicenseKeyAlone_RequestedPropertiesAsText_SentAsValues() + throws Exception { + Map sent = process(builder("") + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(FIRST + ", " + SECOND)); + + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST + "," + SECOND, sent.get("values")); + assertFalse(sent.containsKey("resource")); + } + + @Test + public void BothKeys_RequestedProperties_SendsAllThree() + throws Exception { + Map sent = process(builder(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Collections.singletonList(FIRST))); + + assertEquals(RESOURCE_KEY, sent.get("resource")); + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST, sent.get("values")); + } + + @Test + public void ResourceKey_NoRequestedProperties_SendsNoValues() + throws Exception { + Map sent = process(builder(RESOURCE_KEY)); + + assertEquals(RESOURCE_KEY, sent.get("resource")); + assertFalse(sent.containsKey("license")); + assertFalse(sent.containsKey("values")); + } + + @Test + public void LicenseKeyAlone_NoRequestedProperties_Refused() + throws Exception { + try { + builder(null).setLicenseKey(LICENSE_KEY).build().close(); + fail("A license key with no requested properties was built."); + } catch (PipelineConfigurationException e) { + assertTrue(e.getMessage(), + e.getMessage().contains("answers 400")); + assertTrue(e.getMessage(), + e.getMessage().contains("setRequestedProperties")); + assertFalse(e.getMessage().contains(LICENSE_KEY)); + } + assertTrue(cloud.dataRequests.isEmpty()); + } + + @Test + public void SetRequestedProperties_Null_Refused() { + try { + builder(RESOURCE_KEY).setRequestedProperties((List) null); + fail("A null list was accepted."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("properties")); + } + try { + builder(RESOURCE_KEY).setRequestedProperties((String) null); + fail("A null string was accepted."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("properties")); + } + } +} diff --git a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilder.java b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilder.java index b19e90a08..1706cc0ee 100644 --- a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilder.java +++ b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilder.java @@ -93,6 +93,10 @@ public Pipeline build() throws Exception { if (licenseKey != null && licenseKey.isEmpty() == false) { cloudRequestEngineBuilder.setLicenseKey(licenseKey); } + if (requestedProperties.isEmpty() == false) { + cloudRequestEngineBuilder.setRequestedProperties( + requestedProperties); + } if (cloudRequestOrigin != null && cloudRequestOrigin.isEmpty() == false) { cloudRequestEngineBuilder.setCloudRequestOrigin(cloudRequestOrigin); } diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilderTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilderTests.java new file mode 100644 index 000000000..7c68c499a --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceCloudPipelineBuilderTests.java @@ -0,0 +1,249 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * The cloud Pipeline builder passes the keys and the requested properties + * to the Cloud Request Engine, so a Pipeline can authenticate on a license + * key alone. A fake cloud service records each data request. + */ +public class IPIntelligenceCloudPipelineBuilderTests { + + private static final String BASE = "https://cloud.example/api/v4/"; + private static final String LICENSE_KEY = "abcdefghijkl1234"; + private static final String RESOURCE_KEY = "resourcekey"; + private static final String FIRST = "ip.registeredname"; + private static final String SECOND = "ip.registeredcountry"; + private static final String PROPERTIES_RESPONSE = + "{\"Products\":{\"ip\":{\"DataTier\":\"t\",\"Properties\":" + + "[{\"Name\":\"RegisteredName\",\"Type\":\"String\"," + + "\"Category\":\"c\"}]}}}"; + private static final String DATA_RESPONSE = "{\"ip\":{}}"; + + /** Records every data request and answers from fixed responses. */ + private static class FakeCloud implements HttpClient { + final List dataRequests = + Collections.synchronizedList(new ArrayList()); + + @Override + public HttpURLConnection connect(URL url) { + return new HttpURLConnection(url) { + @Override + public void connect() { + } + + @Override + public void disconnect() { + } + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public int getResponseCode() { + return 200; + } + }; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + return answer( + connection.getURL(), + data == null ? "" : + new String(data, StandardCharsets.UTF_8)); + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return answer(connection.getURL(), ""); + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + return answer(connection.getURL(), ""); + } + + private String answer(URL url, String body) { + String path = url.getPath(); + if (path.endsWith("accessibleproperties")) { + return PROPERTIES_RESPONSE; + } + if (path.endsWith("evidencekeys")) { + return "[\"query.client-ip\"]"; + } + dataRequests.add(body); + return DATA_RESPONSE; + } + } + + private FakeCloud cloud; + + @Before + public void setUp() { + cloud = new FakeCloud(); + } + + private IPIntelligenceCloudPipelineBuilder builder(String resourceKey) { + return new IPIntelligencePipelineBuilder( + LoggerFactory.getILoggerFactory(), cloud) + .useCloud(resourceKey) + .setEndPoint(BASE); + } + + /** Process one request and give the fields of the form it sent. */ + private Map process(IPIntelligenceCloudPipelineBuilder builder) + throws Exception { + try (Pipeline pipeline = builder.build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.client-ip", "8.8.8.8").process(); + } + assertEquals(1, cloud.dataRequests.size()); + return form(cloud.dataRequests.get(0)); + } + + private static Map form(String body) + throws UnsupportedEncodingException { + Map fields = new HashMap<>(); + for (String pair : body.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] parts = pair.split("=", 2); + fields.put( + URLDecoder.decode(parts[0], "UTF-8"), + URLDecoder.decode(parts.length > 1 ? parts[1] : "", "UTF-8")); + } + return fields; + } + + @Test + public void LicenseKeyAlone_RequestedProperties_SentAsValues() + throws Exception { + Map sent = process(builder(null) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList(FIRST, SECOND))); + + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST + "," + SECOND, sent.get("values")); + assertFalse(sent.containsKey("resource")); + } + + @Test + public void LicenseKeyAlone_RequestedPropertiesAsText_SentAsValues() + throws Exception { + Map sent = process(builder("") + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(FIRST + ", " + SECOND)); + + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST + "," + SECOND, sent.get("values")); + assertFalse(sent.containsKey("resource")); + } + + @Test + public void BothKeys_RequestedProperties_SendsAllThree() + throws Exception { + Map sent = process(builder(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Collections.singletonList(FIRST))); + + assertEquals(RESOURCE_KEY, sent.get("resource")); + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST, sent.get("values")); + } + + @Test + public void ResourceKey_NoRequestedProperties_SendsNoValues() + throws Exception { + Map sent = process(builder(RESOURCE_KEY)); + + assertEquals(RESOURCE_KEY, sent.get("resource")); + assertFalse(sent.containsKey("license")); + assertFalse(sent.containsKey("values")); + } + + @Test + public void LicenseKeyAlone_NoRequestedProperties_Refused() + throws Exception { + try { + builder(null).setLicenseKey(LICENSE_KEY).build().close(); + fail("A license key with no requested properties was built."); + } catch (PipelineConfigurationException e) { + assertTrue(e.getMessage(), + e.getMessage().contains("answers 400")); + assertTrue(e.getMessage(), + e.getMessage().contains("setRequestedProperties")); + assertFalse(e.getMessage().contains(LICENSE_KEY)); + } + assertTrue(cloud.dataRequests.isEmpty()); + } + + @Test + public void SetRequestedProperties_Null_Refused() { + try { + builder(RESOURCE_KEY).setRequestedProperties((List) null); + fail("A null list was accepted."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("properties")); + } + try { + builder(RESOURCE_KEY).setRequestedProperties((String) null); + fail("A null string was accepted."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("properties")); + } + } +} diff --git a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java index 6c8eae436..984fd29de 100644 --- a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java +++ b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java @@ -256,8 +256,10 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { new ArrayList(), true, null, - item.delayExecution, - item.evidenceProperties); + item.delayExecution != null ? + item.delayExecution : false, + item.evidenceProperties != null ? + item.evidenceProperties : new ArrayList()); loaded.add(property); } diff --git a/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilder.java b/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilder.java index 4cece057e..0c2e328bc 100644 --- a/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilder.java +++ b/location/geo-location/src/main/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilder.java @@ -75,6 +75,10 @@ public Pipeline build() throws Exception { if (licenseKey != null && licenseKey.isEmpty() == false) { cloudRequestEngineBuilder.setLicenseKey(licenseKey); } + if (requestedProperties.isEmpty() == false) { + cloudRequestEngineBuilder.setRequestedProperties( + requestedProperties); + } if (cloudRequestOrigin != null && cloudRequestOrigin.isEmpty() == false) { cloudRequestEngineBuilder.setCloudRequestOrigin(cloudRequestOrigin); } diff --git a/location/geo-location/src/test/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilderTests.java b/location/geo-location/src/test/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilderTests.java new file mode 100644 index 000000000..2c2943105 --- /dev/null +++ b/location/geo-location/src/test/java/fiftyone/geolocation/GeoLocationCloudPipelineBuilderTests.java @@ -0,0 +1,252 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation; + +import fiftyone.geolocation.core.Enums; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * The cloud Pipeline builder passes the keys and the requested properties + * to the Cloud Request Engine, so a Pipeline can authenticate on a license + * key alone. A fake cloud service records each data request. + */ +public class GeoLocationCloudPipelineBuilderTests { + + private static final String BASE = "https://cloud.example/api/v4/"; + private static final String LICENSE_KEY = "abcdefghijkl1234"; + private static final String RESOURCE_KEY = "resourcekey"; + private static final String FIRST = "location.country"; + private static final String SECOND = "location.town"; + private static final String PROPERTIES_RESPONSE = + "{\"Products\":{\"location\":{\"DataTier\":\"t\",\"Properties\":" + + "[{\"Name\":\"Country\",\"Type\":\"String\"," + + "\"Category\":\"c\"}]}}}"; + private static final String DATA_RESPONSE = "{\"location\":{}}"; + + /** Records every data request and answers from fixed responses. */ + private static class FakeCloud implements HttpClient { + final List dataRequests = + Collections.synchronizedList(new ArrayList()); + + @Override + public HttpURLConnection connect(URL url) { + return new HttpURLConnection(url) { + @Override + public void connect() { + } + + @Override + public void disconnect() { + } + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public int getResponseCode() { + return 200; + } + }; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + return answer( + connection.getURL(), + data == null ? "" : + new String(data, StandardCharsets.UTF_8)); + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return answer(connection.getURL(), ""); + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + return answer(connection.getURL(), ""); + } + + private String answer(URL url, String body) { + String path = url.getPath(); + if (path.endsWith("accessibleproperties")) { + return PROPERTIES_RESPONSE; + } + if (path.endsWith("evidencekeys")) { + return "[\"query.51D_Pos_latitude\"]"; + } + dataRequests.add(body); + return DATA_RESPONSE; + } + } + + private FakeCloud cloud; + + @Before + public void setUp() { + cloud = new FakeCloud(); + } + + private GeoLocationCloudPipelineBuilder builder(String resourceKey) { + return new GeoLocationPipelineBuilder( + LoggerFactory.getILoggerFactory(), cloud) + .useCloud( + resourceKey, + Enums.GeoLocationProvider.FiftyOneDegrees) + .setEndPoint(BASE); + } + + /** Process one request and give the fields of the form it sent. */ + private Map process(GeoLocationCloudPipelineBuilder builder) + throws Exception { + try (Pipeline pipeline = builder.build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.51D_Pos_latitude", "51.5").process(); + } + assertEquals(1, cloud.dataRequests.size()); + return form(cloud.dataRequests.get(0)); + } + + private static Map form(String body) + throws UnsupportedEncodingException { + Map fields = new HashMap<>(); + for (String pair : body.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] parts = pair.split("=", 2); + fields.put( + URLDecoder.decode(parts[0], "UTF-8"), + URLDecoder.decode(parts.length > 1 ? parts[1] : "", "UTF-8")); + } + return fields; + } + + @Test + public void LicenseKeyAlone_RequestedProperties_SentAsValues() + throws Exception { + Map sent = process(builder(null) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList(FIRST, SECOND))); + + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST + "," + SECOND, sent.get("values")); + assertFalse(sent.containsKey("resource")); + } + + @Test + public void LicenseKeyAlone_RequestedPropertiesAsText_SentAsValues() + throws Exception { + Map sent = process(builder("") + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(FIRST + ", " + SECOND)); + + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST + "," + SECOND, sent.get("values")); + assertFalse(sent.containsKey("resource")); + } + + @Test + public void BothKeys_RequestedProperties_SendsAllThree() + throws Exception { + Map sent = process(builder(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Collections.singletonList(FIRST))); + + assertEquals(RESOURCE_KEY, sent.get("resource")); + assertEquals(LICENSE_KEY, sent.get("license")); + assertEquals(FIRST, sent.get("values")); + } + + @Test + public void ResourceKey_NoRequestedProperties_SendsNoValues() + throws Exception { + Map sent = process(builder(RESOURCE_KEY)); + + assertEquals(RESOURCE_KEY, sent.get("resource")); + assertFalse(sent.containsKey("license")); + assertFalse(sent.containsKey("values")); + } + + @Test + public void LicenseKeyAlone_NoRequestedProperties_Refused() + throws Exception { + try { + builder(null).setLicenseKey(LICENSE_KEY).build().close(); + fail("A license key with no requested properties was built."); + } catch (PipelineConfigurationException e) { + assertTrue(e.getMessage(), + e.getMessage().contains("answers 400")); + assertTrue(e.getMessage(), + e.getMessage().contains("setRequestedProperties")); + assertFalse(e.getMessage().contains(LICENSE_KEY)); + } + assertTrue(cloud.dataRequests.isEmpty()); + } + + @Test + public void SetRequestedProperties_Null_Refused() { + try { + builder(RESOURCE_KEY).setRequestedProperties((List) null); + fail("A null list was accepted."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("properties")); + } + try { + builder(RESOURCE_KEY).setRequestedProperties((String) null); + fail("A null string was accepted."); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("properties")); + } + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java index 20b2e4d87..c354f7e90 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java @@ -30,14 +30,27 @@ import fiftyone.pipeline.engines.services.HttpClientDefault; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.TreeMap; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * Calls the live 51Degrees cloud service through the cloud request @@ -65,6 +78,12 @@ public class CloudRequestEngineLiveTests { "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"; + /** A property every device data file has. */ + private static final String IS_MOBILE = "device.ismobile"; + + /** A name no product has, which the service leaves out of its answer. */ + private static final String NOT_A_PROPERTY = "device.notaproperty51d"; + @Test public void build_Process_ReadsDeviceDataFromTheLiveService() throws Exception { @@ -136,4 +155,142 @@ public void build_Process_ReadsDeviceDataFromTheLiveService() } } } + + /** + * Authenticates on a license key alone, with no resource key, and + * names the properties wanted. The service then answers only those + * properties. One the license does not grant comes back as null with a + * reason, and a name that is not a property is left out without the + * service saying so, which the engine has to report itself, once. + *

+ * Skipped when no license key is set, or the one set grants no cloud + * product, because a resource key cannot stand in for one. No message + * here holds the service's answer, only the names of its fields, so + * that nothing the service says about the key reaches a build log. + */ + @Test + public void build_Process_LicenseKeyAlone_AnswersOnlyWhatWasAsked() + throws Exception { + String licenceKey = + LiveCloudKeys.find(LiveCloudKeys.LICENCE_KEY_NAMES); + assumeTrue(licenceKey != null, + "None of " + String.join(", ", LiveCloudKeys.LICENCE_KEY_NAMES) + + " is set, so there is no license key to authenticate with."); + + // The service answers a request that names no property the license + // grants with an error, so one granted property is asked for too. + // Discovery under the license key lists what the license grants. + String granted; + try (CloudRequestEngine discovery = new CloudRequestEngineBuilder( + LoggerFactory.getILoggerFactory(), new HttpClientDefault()) + .setLicenseKey(licenceKey) + .setRequestedProperties(Collections.singletonList(IS_MOBILE)) + .build()) { + granted = firstProperty(discovery.getPublicProperties()); + } + assumeTrue(granted != null, + "The live cloud service lists no product for this license " + + "key, so it is not a key for the cloud service."); + + Logger logger = mock(Logger.class); + ILoggerFactory loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + List asked = + Arrays.asList(granted, IS_MOBILE, NOT_A_PROPERTY); + CloudRequestEngine engine = new CloudRequestEngineBuilder( + loggerFactory, new HttpClientDefault()) + .setLicenseKey(licenceKey) + .setRequestedProperties(asked) + .build(); + + JSONObject answer = null; + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build()) { + // Twice, because the engine reports a name left out once and + // not on every request. + for (int i = 0; i < 2; i++) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", USER_AGENT) + .process(); + answer = new JSONObject( + data.getFromElement(engine).getJsonResponse()); + } + } + } + + assertFalse(answer.has("errors"), + "The live cloud service answered a license key and a list of " + + "properties with errors. Its answer has the fields " + + answer.keySet() + "."); + JSONObject device = answer.optJSONObject("device"); + assertTrue(device != null && device.has("ismobile"), + "The live cloud service left " + IS_MOBILE + " out of its " + + "answer to a license key that asked for it. Its answer has " + + "the fields " + answer.keySet() + "."); + assertTrue( + device.isNull("ismobile") == false + || device.has("ismobilenullreason"), + "The live cloud service answered " + IS_MOBILE + " as null " + + "with no reason beside it."); + for (String element : answer.keySet()) { + JSONObject values = answer.optJSONObject(element); + if (values == null) { + continue; + } + for (String property : values.keySet()) { + assertTrue( + wasAsked(asked, element, property), + "The live cloud service answered a license key alone " + + "with " + element + "." + property + ", which was " + + "not asked for."); + } + } + + // The name that is not a property is reported once, and the + // property answered as null is not reported. + verify(logger, times(1)).warn(argThat( + (String m) -> m.contains(NOT_A_PROPERTY) + && m.contains(IS_MOBILE) == false)); + } + + /** + * The fully qualified name of the first property of the first product + * the service lists, in name order, or null when it lists none. + */ + private static String firstProperty( + Map products) { + if (products == null) { + return null; + } + for (Map.Entry + product : new TreeMap<>(products).entrySet()) { + if (product.getValue().properties != null + && product.getValue().properties.isEmpty() == false) { + return product.getKey() + "." + + product.getValue().properties.get(0).name; + } + } + return null; + } + + /** + * True if the property of the element, or the property its null reason + * belongs to, is one of the names asked for, in any letter case. + */ + private static boolean wasAsked( + List asked, String element, String property) { + String suffix = "nullreason"; + for (String name : asked) { + if (name.equalsIgnoreCase(element + "." + property)) { + return true; + } + if (property.endsWith(suffix) && name.equalsIgnoreCase( + element + "." + property.substring( + 0, property.length() - suffix.length()))) { + return true; + } + } + return false; + } } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudPipelineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudPipelineBuilderBase.java index fed30283a..ffdf0e361 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudPipelineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/CloudPipelineBuilderBase.java @@ -24,6 +24,10 @@ import org.slf4j.ILoggerFactory; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + /** * Base class for pipeline builders that will produce a pipeline with specific * flow elements. @@ -45,6 +49,8 @@ public abstract class CloudPipelineBuilderBase< protected String licenseKey = ""; + protected final List requestedProperties = new ArrayList<>(); + protected String cloudRequestOrigin = ""; /** @@ -120,7 +126,11 @@ public TBuilder setResourceKey(String key) { } /** - * Set the license key to be used when calling the endpoints. + * Set the license key to be used when calling the endpoints. A license + * key used without a resource key needs the properties wanted, named + * with {@link #setRequestedProperties(List)}. Beside a resource key it + * adds the requested properties the license grants to what the resource + * key carries. * @param key license key * @return this builder */ @@ -130,6 +140,38 @@ public TBuilder setLicenseKey(String key) { return (TBuilder)this; } + /** + * Name the properties each request asks the cloud service for, as fully + * qualified names such as {@code device.ismobile}. Required with a + * license key alone. Beside a resource key the cloud service adds them + * to the properties the resource key carries. + * @param properties the fully qualified property names + * @return this builder + */ + @SuppressWarnings("unchecked") + public TBuilder setRequestedProperties(List properties) { + if (properties == null) { + throw new IllegalArgumentException( + "properties must not be null"); + } + requestedProperties.addAll(properties); + return (TBuilder)this; + } + + /** + * As {@link #setRequestedProperties(List)}, taking the names as one + * comma separated string. + * @param properties comma separated fully qualified property names + * @return this builder + */ + public TBuilder setRequestedProperties(String properties) { + if (properties == null) { + throw new IllegalArgumentException( + "properties must not be null"); + } + return setRequestedProperties(Arrays.asList(properties.split(","))); + } + /** * The value to set for the Origin header when making requests * to the cloud service.