From 575d562e3666cd7e81af88cb10031be9fd09af35 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 21:02:31 +0000 Subject: [PATCH 01/68] Point the documentation build at the example folders and package (#23) * Point the documentation build at the example folders and package List the cloud request engine examples and the device detection examples source folder in EXAMPLE_PATH, so Doxygen finds the files the examples name. Include each client hints example's template from its own folder, and read the location package from src/fiftyone_location. * Keep the cloud web example doc comment unbroken and name its view by folder * Keep the onpremise web example doc comment unbroken and name its view by folder --- device-detection/docs/Doxyfile | 1 + .../cloud/gettingstarted_web/app.py | 4 ++-- .../cloud/useragentclienthints_web/app.py | 2 +- .../onpremise/gettingstarted_web/app.py | 4 ++-- .../onpremise/useragentclienthints_web/app.py | 2 +- location/docs/Doxyfile | 2 +- pipeline/docs/Doxyfile | 1 + 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/device-detection/docs/Doxyfile b/device-detection/docs/Doxyfile index c8478bd7..b97c34ec 100644 --- a/device-detection/docs/Doxyfile +++ b/device-detection/docs/Doxyfile @@ -898,6 +898,7 @@ EXCLUDE_SYMBOLS = # command). EXAMPLE_PATH = ../fiftyone_devicedetection_examples \ + ../fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples \ ../../../documentation/src/snippets # If the value of the EXAMPLE_PATH tag contains directories, you can use the diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py index 340250fb..683efca1 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py @@ -42,7 +42,7 @@ # ```{py} # flowdata.evidence.add_from_dict(webevidence(request)) # ``` - +# # The module can also handling setting response headers (e.g. Accept-CH for User-Agent # Client Hints) and serving requests for client-side JavaScript and JSON resources. # ```{py} @@ -70,7 +70,7 @@ # ``` # # ## View -# @include templates/index.html +# @include cloud/gettingstarted_web/templates/index.html # # ## App diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py index 448e6707..f4d8e73f 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py @@ -39,7 +39,7 @@ # - [flask](https://pypi.org/project/flask/) # # ## View -# @include templates/index.html +# @include cloud/useragentclienthints_web/templates/index.html # # ## App diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index fb130b99..481f6c58 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -42,7 +42,7 @@ # ```{py} # flowdata.evidence.add_from_dict(webevidence(request)) # ``` - +# # The module can also handling setting response headers (e.g. Accept-CH for User-Agent # Client Hints) and serving requests for client-side JavaScript and JSON resources. # ```{py} @@ -70,7 +70,7 @@ # ``` # # ## View -# @include templates/index.html +# @include onpremise/gettingstarted_web/templates/index.html # # ## App diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py index 62f427c5..1adccf41 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py @@ -45,7 +45,7 @@ # - [flask](https://pypi.org/project/flask/) # # ## View -# @include templates/index.html +# @include onpremise/useragentclienthints_web/templates/index.html # # ## App diff --git a/location/docs/Doxyfile b/location/docs/Doxyfile index 19312f87..4d98cfcb 100644 --- a/location/docs/Doxyfile +++ b/location/docs/Doxyfile @@ -818,7 +818,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = ../fiftyone_location \ +INPUT = ../src/fiftyone_location \ ../examples/cloud \ ../examples \ ../readme.md diff --git a/pipeline/docs/Doxyfile b/pipeline/docs/Doxyfile index 9ddb62fb..c62f0df0 100644 --- a/pipeline/docs/Doxyfile +++ b/pipeline/docs/Doxyfile @@ -898,6 +898,7 @@ EXCLUDE_SYMBOLS = EXAMPLE_PATH = ../fiftyone_pipeline_core/examples \ ../fiftyone_pipeline_engines_fiftyone/examples \ + ../fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples \ ../../../documentation/src/snippets # If the value of the EXAMPLE_PATH tag contains directories, you can use the From 2cd9dc8244f1ce54a313b835cd3274afc8f02212 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 21:28:11 +0000 Subject: [PATCH 02/68] Point links to the archived repositories at where the code now lives (#25) * Point links to the archived repositories at where the code now lives The examples, readmes and example web pages linked to the archived Python repositories, and the IP Intelligence readmes linked to the archived IP Intelligence C repository. Each link now points at the same file or folder in 51degrees-python, or at the IP Intelligence readme in 51degrees-cxx, so readers land on code that is still maintained. Links to a file follow the file to its folder in this repository, for example the Device Detection examples now sit under device-detection/fiftyone_devicedetection_examples/src. Links that used the old version/4.5 branch now use main. The match metrics example closed its Markdown link with a square bracket instead of a parenthesis. The link now closes with a parenthesis. The cloud examples readme changes into the cloned folder after cloning, so its cd line now follows the new clone address. The history table in the root README still names the earlier repositories, because their tags and releases stay there. The links to the data repositories are unchanged, because those repositories are still current. * Describe the IP Intelligence performance test gap without old repository names --- .../cloud/gettingstarted_console.py | 2 +- .../cloud/gettingstarted_web/app.py | 2 +- .../cloud/metadata_console.py | 2 +- .../cloud/nativemodellookup_console.py | 2 +- .../cloud/taclookup_console.py | 2 +- .../cloud/useragentclienthints_web/app.py | 2 +- .../onpremise/datafileupdate_console.py | 2 +- .../onpremise/gettingstarted_console.py | 2 +- .../onpremise/gettingstarted_web/app.py | 2 +- .../onpremise/match_metrics.py | 2 +- .../onpremise/metadata_console.py | 2 +- .../onpremise/offlineprocessing.py | 2 +- .../onpremise/performance.py | 2 +- .../onpremise/useragentclienthints_web/app.py | 2 +- device-detection/readme.md | 2 +- ip-intelligence/ci/run-performance-tests.ps1 | 8 ++++---- .../fiftyone_ipintelligence_onpremise/readme.md | 2 +- ip-intelligence/readme.md | 4 ++-- location/examples/cloud/web.py | 2 +- .../examples/readme.md | 8 ++++---- .../cloud/mixed/gettingstarted_console.py | 6 +++--- .../cloud/mixed/gettingstarted_web/app.py | 2 +- .../cloud/mixed/gettingstarted_web/templates/index.html | 4 ++-- .../examples/creator_context_web/page.html | 2 +- .../examples/usagesharing/usagesharing.py | 4 ++-- pipeline/readme.md | 4 ++-- 26 files changed, 38 insertions(+), 38 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py index e6bd2457..caa7c7b8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py @@ -31,7 +31,7 @@ # 2. How to pass input data (evidence) to the Pipeline # 3. How to retrieve the results # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/version/4.5/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py). # # @include{doc} example-require-resourcekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py index 683efca1..ba86890b 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py @@ -25,7 +25,7 @@ # # @include{doc} example-getting-started-web.txt # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py). # # @include{doc} example-require-resourcekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py index dd87b710..74b0df56 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py @@ -38,7 +38,7 @@ # cloud. If you are only using a single product (for example - device detection) then not all # of these keys will be relevant. # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/cloud/metadata_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py). # # @include{doc} example-require-resourcekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py index 2995aa9b..4d11f6f5 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py @@ -29,7 +29,7 @@ # [Android devices](https://developer.android.com/reference/android/os/Build#MODEL) and # [iOS devices](https://gist.github.com/soapyigu/c99e1f45553070726f14c1bb0a54053b#file-machinename-swift) # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py). # # @include{doc} example-require-resourcekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py index cfe005b0..4563bd26 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py @@ -32,7 +32,7 @@ # 2. How to pass a TAC as evidence to the Pipeline # 3. How to retrieve device details associated with the TAC # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/version/4.5/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py). # # @include{doc} example-require-resourcekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py index f4d8e73f..730153df 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py @@ -30,7 +30,7 @@ # setting the Accept-CH header in the response. Selecting the "Make second request" # button reloads the page so those additional headers are included as evidence. # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py). # # @include{doc} example-require-resourcekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index 2cfdd3ed..bf9ff7a3 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -148,7 +148,7 @@ # ``` # # # Location -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py). # # @include{doc} example-require-licensekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index 57210dc7..9c695f43 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -31,7 +31,7 @@ # 2. How to pass input data (evidence) to the Pipeline # 3. How to retrieve the results # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/version/4.5/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py). # # @include{doc} example-require-datafile.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index 481f6c58..c2ec0e2b 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -25,7 +25,7 @@ # # @include{doc} example-getting-started-web.txt # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py). # # @include{doc} example-require-datafile.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py index 1662eabb..f9f254f8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py @@ -36,7 +36,7 @@ # [https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=top] # page. # # Location -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/match_metrics.py]. +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py). from itertools import groupby from pathlib import Path diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py index ea36bca1..1e5e0929 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py @@ -33,7 +33,7 @@ # keys that, when added to the evidence collection in flow data, could have some impact on the # result returned by device detection. # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/metadata_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py). # # @include{doc} example-require-datafile.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py index 1e639138..2487f3e9 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py @@ -42,7 +42,7 @@ # performance and predictive power using Performance Profile, Graph and Difference and Drift # settings. # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py). # # @include{doc} example-require-datafile.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py index 95864e36..665cdabf 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py @@ -35,7 +35,7 @@ # and [hash dataset options](/documentation/4.5/_device_detection__hash.html#DeviceDetection_Hash_DataSetProduction_Performance) # for more information about adjusting performance. # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/performance.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py). # # @include{doc} example-require-datafile.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py index 1adccf41..6d10a1a7 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py @@ -36,7 +36,7 @@ # SetHeaderBrowserAccept-CH, SetHeaderPlatformAccept-CH and SetHeaderHardwareAccept-CH # properties respectively. # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py). # # @include{doc} example-require-datafile.txt # diff --git a/device-detection/readme.md b/device-detection/readme.md index 42ffce09..379b1ed1 100644 --- a/device-detection/readme.md +++ b/device-detection/readme.md @@ -13,7 +13,7 @@ The Pipeline is a generic web request intelligence and data processing solution This git repository uses submodules, to clone the git repository run: ```bash -git clone --recurse-submodules https://github.com/51Degrees/device-detection-python.git +git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git ``` or if already cloned run the following to obtain all sub modules: diff --git a/ip-intelligence/ci/run-performance-tests.ps1 b/ip-intelligence/ci/run-performance-tests.ps1 index fb41eaca..c9a3dd82 100644 --- a/ip-intelligence/ci/run-performance-tests.ps1 +++ b/ip-intelligence/ci/run-performance-tests.ps1 @@ -6,7 +6,7 @@ param ( [boolean]$RunPerformance = $True ) -# Performance tests are not yet implemented for ip-intelligence-python. -# They should mirror the approach used by device-detection-python's -# performance-tests once a stable data file source is available in CI. -Write-Output "Skipping performance tests - not yet implemented for ip-intelligence-python" +# Performance tests are not yet implemented for IP Intelligence. They should +# mirror the device detection performance tests once a stable data file +# source is available in CI. +Write-Output "Skipping performance tests - not yet implemented for IP Intelligence" diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md index 056db37c..b7eb9ac3 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md @@ -10,7 +10,7 @@ This package resolves IP addresses to location, network and registration information using a local `.ipi` data file and the [51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise-readme.md&utm_term=introduction). -The native [ip-intelligence-cxx](https://github.com/51Degrees/ip-intelligence-cxx) +The native [51degrees-cxx](https://github.com/51Degrees/51degrees-cxx/blob/main/docs/ipi/README.md) library is included as a git submodule and wrapped with SWIG. ## Building diff --git a/ip-intelligence/readme.md b/ip-intelligence/readme.md index d01aa594..68e79e2e 100644 --- a/ip-intelligence/readme.md +++ b/ip-intelligence/readme.md @@ -27,14 +27,14 @@ page shows the Python versions that we currently test against. ### Git submodules The on-premise engine wraps the native -[ip-intelligence-cxx](https://github.com/51Degrees/ip-intelligence-cxx) +[51degrees-cxx](https://github.com/51Degrees/51degrees-cxx/blob/main/docs/ipi/README.md) library, and the [ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data) repository (data and evidence files used for testing) is also included, both as git submodules. When cloning, use: ``` -git clone --recurse-submodules https://github.com/51Degrees/ip-intelligence-python.git +git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git ``` or if you have already cloned: diff --git a/location/examples/cloud/web.py b/location/examples/cloud/web.py index ce6bac8f..93c8e3aa 100644 --- a/location/examples/cloud/web.py +++ b/location/examples/cloud/web.py @@ -29,7 +29,7 @@ # This JavaScript is then used to obtain the additional evidence from the client and pass it back to the server. # The Pipeline running on the server then processes this evidence to populate a new JSON result object that is returned to the client. # -# This example is available in full on [GitHub](https://github.com/51Degrees/location-python/blob/main/examples/cloud/web.py) +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/location/examples/cloud/web.py) from fiftyone_location.location_pipelinebuilder import LocationPipelineBuilder from fiftyone_pipeline_core.web import webevidence diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index 01071f38..cd8b5b28 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -62,8 +62,8 @@ virtual environment and install the requirements, which install the Pipeline packages from this checkout. ```sh -git clone --recurse-submodules https://github.com/51Degrees/pipeline-python.git -cd pipeline-python/fiftyone_pipeline_cloudrequestengine/examples +git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git +cd 51degrees-python/pipeline/fiftyone_pipeline_cloudrequestengine/examples python -m venv .venv .venv/bin/python -m pip install -r requirements.txt ``` @@ -106,6 +106,6 @@ which can also be run by hand with `python -m tox -e py` from here. - [IP intelligence](https://51degrees.com/ip-intelligence?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-readme.md&utm_term=find-out-more-ip-intelligence) - [Resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-readme.md&utm_term=find-out-more-resource-keys) - [Pricing](https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-readme.md&utm_term=find-out-more-pricing) -- Pipeline API for Python: https://github.com/51Degrees/pipeline-python -- Device detection for Python: https://github.com/51Degrees/device-detection-python +- Pipeline API for Python: https://github.com/51Degrees/51degrees-python/tree/main/pipeline +- Device detection for Python: https://github.com/51Degrees/51degrees-python/tree/main/device-detection - IP intelligence engine: https://github.com/51Degrees/51degrees-cxx diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py index 03673e18..16f0dd6b 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py @@ -33,7 +33,7 @@ # 3. How to read device and IP intelligence results from one set of flow # data # -# This example is available in full on [GitHub](https://github.com/51Degrees/pipeline-python/blob/main/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py). # # To run this example you need a resource key or a licence key. # @@ -170,9 +170,9 @@ class GettingStartedConsole(): ("Pricing", "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_console.py&utm_term=find-out-more-pricing"), ("Pipeline API for Python", - "https://github.com/51Degrees/pipeline-python"), + "https://github.com/51Degrees/51degrees-python/tree/main/pipeline"), ("Device detection for Python", - "https://github.com/51Degrees/device-detection-python"), + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), ("IP intelligence engine", "https://github.com/51Degrees/51degrees-cxx"), ] diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py index 46a6fbfd..66856b86 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py @@ -31,7 +31,7 @@ # Each time the pipeline processes a request, one request to the cloud # service returns the results of every engine in it. # -# This example is available in full on [GitHub](https://github.com/51Degrees/pipeline-python/blob/main/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py). # # To run this example you need a resource key or a licence key. # diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/templates/index.html b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/templates/index.html index eb93e493..88648aa2 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/templates/index.html +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/templates/index.html @@ -151,8 +151,8 @@

Find out more

  • IP intelligence
  • Resource keys
  • Pricing
    • -
  • Pipeline API for Python: https://github.com/51Degrees/pipeline-python
    • -
  • Device detection for Python: https://github.com/51Degrees/device-detection-python
    • +
  • Pipeline API for Python: https://github.com/51Degrees/51degrees-python/tree/main/pipeline
    • +
  • Device detection for Python: https://github.com/51Degrees/51degrees-python/tree/main/device-detection
  • IP intelligence engine: https://github.com/51Degrees/51degrees-cxx
    • diff --git a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/page.html b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/page.html index 2baec8db..96c55f56 100644 --- a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/page.html +++ b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/page.html @@ -148,7 +148,7 @@

    Find out more

    The source of this demo and of the code it runs on.

    diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py index fb1bee19..ade418a8 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py @@ -25,7 +25,7 @@ # @include{doc} example-usage-sharing-intro.txt # # Usage sharing is enabled by default if using some 51Degrees pipeline -# builders such as the [DeviceDetectionOnPremisePipelineBuilder](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py). +# builders such as the [DeviceDetectionOnPremisePipelineBuilder](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py). # In this example, we show how to specifically add a usage sharing # element to a Pipeline using configuration. # @@ -42,7 +42,7 @@ # For details of what each setting does, see the # [Share Usage reference](/documentation/4.5/_pipeline_api__features__usage_sharing.html) # -# This example is available in full on [GitHub](https://github.com/51Degrees/pipeline-python/blob/main/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py). # # Expected output: # ``` diff --git a/pipeline/readme.md b/pipeline/readme.md index bf4c6da1..a5c12c34 100755 --- a/pipeline/readme.md +++ b/pipeline/readme.md @@ -38,7 +38,7 @@ Generally, you will want to be installing one of the engines such as [device det ### From GitHub -* Clone the repository with its submodules, `git clone --recurse-submodules https://github.com/51Degrees/pipeline-python.git` +* Clone the repository with its submodules, `git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git` * Create and activate a Python virtual environment * Run `pwsh ./setup.ps1` in the root of the folder, which installs the modules from the working copy, apart from `fiftyone_pipeline_did`, together with flask @@ -56,7 +56,7 @@ The tests that call the 51Degrees cloud service read a resource key from the `re ## Examples There are several examples available that demonstrate how to make use of the Pipeline API in isolation. These are described in the table below. -If you want examples that demonstrate how to use 51Degrees products such as device detection, then these are available in the corresponding [repository](https://github.com/51Degrees/device-detection-python) and on our [website](https://51degrees.com/documentation/_examples__device_detection__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=examples). +If you want examples that demonstrate how to use 51Degrees products such as device detection, then these are available in the corresponding [folder](https://github.com/51Degrees/51degrees-python/tree/main/device-detection) and on our [website](https://51degrees.com/documentation/_examples__device_detection__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=examples). | Example | Description | |-----------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------| From 4fc6676a12293bfe79826b69796ac2735110828a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 09:16:14 +0000 Subject: [PATCH 03/68] Generate both SWIG wrappers again for 51degrees-cxx 4.5.102 (#27) * Add a temporary workflow that generates the SWIG wrappers for 51degrees-cxx 4.5.102 * Name the SWIG modules in the temporary workflow instead of reading them * Generate both SWIG wrappers again for 51degrees-cxx 4.5.102 Between 4.5.100 and 4.5.102, 51degrees-cxx removed the two EngineHash process overloads and the three EngineIpi process overloads that take required property indexes. The committed wrappers still called them, so neither on-premise package compiled against 4.5.102, which the nightly submodule update brings in #26. This moves the cxx submodule from 4110d681, the 4.5.100 release, to 1bec7b57, the 4.5.102 release, and generates hash_python_wrap.cxx, DeviceDetectionHashEngineModule.py, ipi_python_wrap.cxx and IpIntelligenceEngineModule.py again with SWIG 4.4.1 built from its v4.4.1 tag. Run against 4110d681, the same command gives files byte for byte the same as the ones committed before, so every difference here comes from cxx. Both engines call only process(evidence), which cxx keeps. * Remove the temporary SWIG wrapper workflow --- cxx | 2 +- .../DeviceDetectionHashEngineModule.py | 120 + .../hash_python_wrap.cxx | 4022 ++++++++++++++++- .../ipi_python_wrap.cxx | 345 +- 4 files changed, 3895 insertions(+), 594 deletions(-) diff --git a/cxx b/cxx index 4110d681..1bec7b57 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit 4110d6813e167a803a06489318b33b83a230fa32 +Subproject commit 1bec7b570575e2e180b52435599bef9b03e35ad8 diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/DeviceDetectionHashEngineModule.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/DeviceDetectionHashEngineModule.py index 1d25c57e..d199e300 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/DeviceDetectionHashEngineModule.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/DeviceDetectionHashEngineModule.py @@ -1138,6 +1138,12 @@ def setUsePredictiveGraph(self, use): def setTraceRoute(self, trace): return _DeviceDetectionHashEngineModule.ConfigHashSwig_setTraceRoute(self, trace) + def setPropertyValueIndex(self, index): + return _DeviceDetectionHashEngineModule.ConfigHashSwig_setPropertyValueIndex(self, index) + + def setIndexedProperties(self, names): + return _DeviceDetectionHashEngineModule.ConfigHashSwig_setIndexedProperties(self, names) + def getStrings(self): return _DeviceDetectionHashEngineModule.ConfigHashSwig_getStrings(self) @@ -1173,10 +1179,121 @@ def getConcurrency(self): def getTraceRoute(self): return _DeviceDetectionHashEngineModule.ConfigHashSwig_getTraceRoute(self) + + def getPropertyValueIndex(self): + return _DeviceDetectionHashEngineModule.ConfigHashSwig_getPropertyValueIndex(self) + + def getIndexedProperties(self): + return _DeviceDetectionHashEngineModule.ConfigHashSwig_getIndexedProperties(self) __swig_destroy__ = _DeviceDetectionHashEngineModule.delete_ConfigHashSwig # Register ConfigHashSwig in _DeviceDetectionHashEngineModule: _DeviceDetectionHashEngineModule.ConfigHashSwig_swigregister(ConfigHashSwig) +class VectorUInt32Swig(object): + thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def iterator(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_iterator(self) + def __iter__(self): + return self.iterator() + + def __nonzero__(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___nonzero__(self) + + def __bool__(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___bool__(self) + + def __len__(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___len__(self) + + def __getslice__(self, i, j): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___getslice__(self, i, j) + + def __setslice__(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___setslice__(self, *args) + + def __delslice__(self, i, j): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___delslice__(self, i, j) + + def __delitem__(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___delitem__(self, *args) + + def __getitem__(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___getitem__(self, *args) + + def __setitem__(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___setitem__(self, *args) + + def pop(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_pop(self) + + def append(self, x): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_append(self, x) + + def empty(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_empty(self) + + def size(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_size(self) + + def swap(self, v): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_swap(self, v) + + def begin(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_begin(self) + + def end(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_end(self) + + def rbegin(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_rbegin(self) + + def rend(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_rend(self) + + def clear(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_clear(self) + + def get_allocator(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_get_allocator(self) + + def pop_back(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_pop_back(self) + + def erase(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_erase(self, *args) + + def __init__(self, *args): + _DeviceDetectionHashEngineModule.VectorUInt32Swig_swiginit(self, _DeviceDetectionHashEngineModule.new_VectorUInt32Swig(*args)) + + def push_back(self, x): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_push_back(self, x) + + def front(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_front(self) + + def back(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_back(self) + + def assign(self, n, x): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_assign(self, n, x) + + def resize(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_resize(self, *args) + + def insert(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_insert(self, *args) + + def reserve(self, n): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_reserve(self, n) + + def capacity(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_capacity(self) + __swig_destroy__ = _DeviceDetectionHashEngineModule.delete_VectorUInt32Swig + +# Register VectorUInt32Swig in _DeviceDetectionHashEngineModule: +_DeviceDetectionHashEngineModule.VectorUInt32Swig_swigregister(VectorUInt32Swig) class EngineHashSwig(EngineDeviceDetectionSwig): thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") __repr__ = _swig_repr @@ -1208,6 +1325,9 @@ def refreshDataFromMemory(self, data): def process(self, *args): return _DeviceDetectionHashEngineModule.EngineHashSwig_process(self, *args) + def getProfileIdsForPropertyValue(self, propertyName, valueName): + return _DeviceDetectionHashEngineModule.EngineHashSwig_getProfileIdsForPropertyValue(self, propertyName, valueName) + def processBase(self, evidence): return _DeviceDetectionHashEngineModule.EngineHashSwig_processBase(self, evidence) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/hash_python_wrap.cxx b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/hash_python_wrap.cxx index 20fe7cba..e973fa1e 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/hash_python_wrap.cxx +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/hash_python_wrap.cxx @@ -3486,19 +3486,21 @@ SWIG_Python_NonDynamicSetAttr(PyObject *obj, PyObject *name, PyObject *value) { #define SWIGTYPE_p_std__allocatorT_std__pairT_std__string_const_std__string_t_t swig_types[43] #define SWIGTYPE_p_std__allocatorT_std__string_t swig_types[44] #define SWIGTYPE_p_std__allocatorT_unsigned_char_t swig_types[45] -#define SWIGTYPE_p_std__invalid_argument swig_types[46] -#define SWIGTYPE_p_std__lessT_std__string_t swig_types[47] -#define SWIGTYPE_p_std__mapT_std__string_std__string_t swig_types[48] -#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[49] -#define SWIGTYPE_p_std__vectorT_uint8_t_t swig_types[50] -#define SWIGTYPE_p_swig__SwigPyIterator swig_types[51] -#define SWIGTYPE_p_unsigned_char swig_types[52] -#define SWIGTYPE_p_unsigned_int swig_types[53] -#define SWIGTYPE_p_unsigned_long_long swig_types[54] -#define SWIGTYPE_p_unsigned_short swig_types[55] -#define SWIGTYPE_p_value_type swig_types[56] -static swig_type_info *swig_types[58]; -static swig_module_info swig_module = {swig_types, 57, 0, 0, 0, 0}; +#define SWIGTYPE_p_std__allocatorT_unsigned_int_t swig_types[46] +#define SWIGTYPE_p_std__invalid_argument swig_types[47] +#define SWIGTYPE_p_std__lessT_std__string_t swig_types[48] +#define SWIGTYPE_p_std__mapT_std__string_std__string_t swig_types[49] +#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[50] +#define SWIGTYPE_p_std__vectorT_uint32_t_t swig_types[51] +#define SWIGTYPE_p_std__vectorT_uint8_t_t swig_types[52] +#define SWIGTYPE_p_swig__SwigPyIterator swig_types[53] +#define SWIGTYPE_p_unsigned_char swig_types[54] +#define SWIGTYPE_p_unsigned_int swig_types[55] +#define SWIGTYPE_p_unsigned_long_long swig_types[56] +#define SWIGTYPE_p_unsigned_short swig_types[57] +#define SWIGTYPE_p_value_type swig_types[58] +static swig_type_info *swig_types[60]; +static swig_module_info swig_module = {swig_types, 59, 0, 0, 0, 0}; #define SWIG_TypeQuery(name) SWIG_TypeQueryModule(&swig_module, &swig_module, name) #define SWIG_MangledTypeQuery(name) SWIG_MangledTypeQueryModule(&swig_module, &swig_module, name) @@ -5961,6 +5963,9 @@ SWIGINTERN std::vector< std::string >::iterator std_vector_Sl_std_string_Sg__era SWIGINTERN std::vector< std::string >::iterator std_vector_Sl_std_string_Sg__insert__SWIG_0(std::vector< std::string > *self,std::vector< std::string >::iterator pos,std::vector< std::string >::value_type const &x){ return self->insert(pos, x); } SWIGINTERN void std_vector_Sl_std_string_Sg__insert__SWIG_1(std::vector< std::string > *self,std::vector< std::string >::iterator pos,std::vector< std::string >::size_type n,std::vector< std::string >::value_type const &x){ self->insert(pos, n, x); } +#include // Use the C99 official header + + @@ -6004,9 +6009,6 @@ SWIGINTERNINLINE PyObject* } -#include // Use the C99 official header - - SWIGINTERN int SWIG_AsVal_unsigned_SS_char (PyObject * obj, unsigned char *val) { @@ -6218,6 +6220,127 @@ SWIG_AsVal_unsigned_SS_short (PyObject * obj, unsigned short *val) return res; } + +namespace swig { + template <> struct traits< unsigned int > { + typedef value_category category; + static const char* type_name() { return"unsigned int"; } + }; + template <> struct traits_asval< unsigned int > { + typedef unsigned int value_type; + static int asval(PyObject *obj, value_type *val) { + return SWIG_AsVal_unsigned_SS_int (obj, val); + } + }; + template <> struct traits_from< unsigned int > { + typedef unsigned int value_type; + static PyObject *from(const value_type& val) { + return SWIG_From_unsigned_SS_int (val); + } + }; +} + + + namespace swig { + template <> struct traits > > { + typedef pointer_category category; + static const char* type_name() { + return "std::vector<" "unsigned int" "," "std::allocator< unsigned int >" " >"; + } + }; + } + +SWIGINTERN swig::SwigPyIterator *std_vector_Sl_uint32_t_Sg__iterator(std::vector< uint32_t > *self,PyObject **PYTHON_SELF){ + return swig::make_output_iterator(self->begin(), self->begin(), self->end(), *PYTHON_SELF); + } +SWIGINTERN bool std_vector_Sl_uint32_t_Sg____nonzero__(std::vector< uint32_t > const *self){ + return !(self->empty()); + } +SWIGINTERN bool std_vector_Sl_uint32_t_Sg____bool__(std::vector< uint32_t > const *self){ + return !(self->empty()); + } +SWIGINTERN std::vector< unsigned int >::size_type std_vector_Sl_uint32_t_Sg____len__(std::vector< uint32_t > const *self){ + return self->size(); + } +SWIGINTERN std::vector< unsigned int,std::allocator< unsigned int > > *std_vector_Sl_uint32_t_Sg____getslice__(std::vector< uint32_t > *self,std::vector< unsigned int >::difference_type i,std::vector< unsigned int >::difference_type j){ + return swig::getslice(self, i, j, 1); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____setslice____SWIG_0(std::vector< uint32_t > *self,std::vector< unsigned int >::difference_type i,std::vector< unsigned int >::difference_type j){ + swig::setslice(self, i, j, 1, std::vector< unsigned int,std::allocator< unsigned int > >()); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____setslice____SWIG_1(std::vector< uint32_t > *self,std::vector< unsigned int >::difference_type i,std::vector< unsigned int >::difference_type j,std::vector< unsigned int,std::allocator< unsigned int > > const &v){ + swig::setslice(self, i, j, 1, v); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____delslice__(std::vector< uint32_t > *self,std::vector< unsigned int >::difference_type i,std::vector< unsigned int >::difference_type j){ + swig::delslice(self, i, j, 1); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____delitem____SWIG_0(std::vector< uint32_t > *self,std::vector< unsigned int >::difference_type i){ + swig::erase(self, swig::getpos(self, i)); + } +SWIGINTERN std::vector< unsigned int,std::allocator< unsigned int > > *std_vector_Sl_uint32_t_Sg____getitem____SWIG_0(std::vector< uint32_t > *self,SWIGPY_SLICEOBJECT *slice){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return NULL; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type id = i; + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type jd = j; + return swig::getslice(self, id, jd, step); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____setitem____SWIG_0(std::vector< uint32_t > *self,SWIGPY_SLICEOBJECT *slice,std::vector< unsigned int,std::allocator< unsigned int > > const &v){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type id = i; + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type jd = j; + swig::setslice(self, id, jd, step, v); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____setitem____SWIG_1(std::vector< uint32_t > *self,SWIGPY_SLICEOBJECT *slice){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type id = i; + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type jd = j; + swig::delslice(self, id, jd, step); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____delitem____SWIG_1(std::vector< uint32_t > *self,SWIGPY_SLICEOBJECT *slice){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type id = i; + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type jd = j; + swig::delslice(self, id, jd, step); + } +SWIGINTERN std::vector< unsigned int >::value_type const &std_vector_Sl_uint32_t_Sg____getitem____SWIG_1(std::vector< uint32_t > const *self,std::vector< unsigned int >::difference_type i){ + return *(swig::cgetpos(self, i)); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____setitem____SWIG_2(std::vector< uint32_t > *self,std::vector< unsigned int >::difference_type i,std::vector< unsigned int >::value_type const &x){ + *(swig::getpos(self,i)) = x; + } +SWIGINTERN std::vector< unsigned int >::value_type std_vector_Sl_uint32_t_Sg__pop(std::vector< uint32_t > *self){ + if (self->size() == 0) + throw std::out_of_range("pop from empty container"); + std::vector< unsigned int,std::allocator< unsigned int > >::value_type x = self->back(); + self->pop_back(); + return x; + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg__append(std::vector< uint32_t > *self,std::vector< unsigned int >::value_type const &x){ + self->push_back(x); + } +SWIGINTERN std::vector< unsigned int >::iterator std_vector_Sl_uint32_t_Sg__erase__SWIG_0(std::vector< uint32_t > *self,std::vector< unsigned int >::iterator pos){ return self->erase(pos); } +SWIGINTERN std::vector< unsigned int >::iterator std_vector_Sl_uint32_t_Sg__erase__SWIG_1(std::vector< uint32_t > *self,std::vector< unsigned int >::iterator first,std::vector< unsigned int >::iterator last){ return self->erase(first, last); } +SWIGINTERN std::vector< unsigned int >::iterator std_vector_Sl_uint32_t_Sg__insert__SWIG_0(std::vector< uint32_t > *self,std::vector< unsigned int >::iterator pos,std::vector< unsigned int >::value_type const &x){ return self->insert(pos, x); } +SWIGINTERN void std_vector_Sl_uint32_t_Sg__insert__SWIG_1(std::vector< uint32_t > *self,std::vector< unsigned int >::iterator pos,std::vector< unsigned int >::size_type n,std::vector< unsigned int >::value_type const &x){ self->insert(pos, n, x); } #ifdef __cplusplus extern "C" { #endif @@ -25527,6 +25650,130 @@ SWIGINTERN PyObject *_wrap_ConfigHashSwig_setTraceRoute(PyObject *self, PyObject } +SWIGINTERN PyObject *_wrap_ConfigHashSwig_setPropertyValueIndex(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ConfigHash *arg1 = 0 ; + bool arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + bool val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ConfigHashSwig_setPropertyValueIndex", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ConfigHash, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ConfigHashSwig_setPropertyValueIndex" "', argument " "1"" of type '" "ConfigHash *""'"); + } + arg1 = reinterpret_cast< ConfigHash * >(argp1); + ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ConfigHashSwig_setPropertyValueIndex" "', argument " "2"" of type '" "bool""'"); + } + arg2 = static_cast< bool >(val2); + { + try { + (arg1)->setPropertyValueIndex(arg2);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ConfigHashSwig_setIndexedProperties(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ConfigHash *arg1 = 0 ; + std::vector< std::string,std::allocator< std::string > > arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ConfigHashSwig_setIndexedProperties", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ConfigHash, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ConfigHashSwig_setIndexedProperties" "', argument " "1"" of type '" "ConfigHash *""'"); + } + arg1 = reinterpret_cast< ConfigHash * >(argp1); + { + std::vector< std::string,std::allocator< std::string > > *ptr = (std::vector< std::string,std::allocator< std::string > > *)0; + int res = swig::asptr(swig_obj[1], &ptr); + if (!SWIG_IsOK(res) || !ptr) { + SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "ConfigHashSwig_setIndexedProperties" "', argument " "2"" of type '" "std::vector< std::string,std::allocator< std::string > >""'"); + } + arg2 = *ptr; + if (SWIG_IsNewObj(res)) delete ptr; + } + { + try { + (arg1)->setIndexedProperties(SWIG_STD_MOVE(arg2));; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + SWIGINTERN PyObject *_wrap_ConfigHashSwig_getStrings(PyObject *self, PyObject *args) { PyObject *resultobj = 0; ConfigHash *arg1 = 0 ; @@ -26187,24 +26434,25 @@ SWIGINTERN PyObject *_wrap_ConfigHashSwig_getTraceRoute(PyObject *self, PyObject } -SWIGINTERN PyObject *_wrap_delete_ConfigHashSwig(PyObject *self, PyObject *args) { +SWIGINTERN PyObject *_wrap_ConfigHashSwig_getPropertyValueIndex(PyObject *self, PyObject *args) { PyObject *resultobj = 0; ConfigHash *arg1 = 0 ; void *argp1 = 0 ; int res1 = 0 ; PyObject *swig_obj[1] ; + bool result; (void)self; if (!args) SWIG_fail; swig_obj[0] = args; - res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ConfigHash, SWIG_POINTER_DISOWN | 0 ); + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ConfigHash, 0 | 0 ); if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ConfigHashSwig" "', argument " "1"" of type '" "ConfigHash *""'"); + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ConfigHashSwig_getPropertyValueIndex" "', argument " "1"" of type '" "ConfigHash *""'"); } arg1 = reinterpret_cast< ConfigHash * >(argp1); { try { - delete arg1;; + result = (bool)(arg1)->getPropertyValueIndex();; } catch(FatalException& e) { SWIG_exception(SWIG_RuntimeError, e.what()); @@ -26234,53 +26482,3401 @@ SWIGINTERN PyObject *_wrap_delete_ConfigHashSwig(PyObject *self, PyObject *args) SWIG_exception(SWIG_ValueError, e.what()); } } - resultobj = SWIG_Py_Void(); + resultobj = SWIG_From_bool(static_cast< bool >(result)); return resultobj; fail: return NULL; } -SWIGINTERN PyObject *ConfigHashSwig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *obj = NULL; - if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; - SWIG_TypeNewClientData(SWIGTYPE_p_ConfigHash, SWIG_NewClientData(obj)); - return SWIG_Py_Void(); -} - -SWIGINTERN PyObject *ConfigHashSwig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - return SWIG_Python_InitShadowInstance(args); -} - -SWIGINTERN PyObject *_wrap_new_EngineHashSwig__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { +SWIGINTERN PyObject *_wrap_ConfigHashSwig_getIndexedProperties(PyObject *self, PyObject *args) { PyObject *resultobj = 0; - std::string *arg1 = 0 ; - ConfigHash *arg2 = 0 ; - RequiredPropertiesConfig *arg3 = 0 ; - int res1 = SWIG_OLDOBJ ; - void *argp2 = 0 ; - int res2 = 0 ; - void *argp3 = 0 ; - int res3 = 0 ; - EngineHash *result = 0 ; + ConfigHash *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< std::string,std::allocator< std::string > > result; (void)self; - if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ConfigHash, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ConfigHashSwig_getIndexedProperties" "', argument " "1"" of type '" "ConfigHash *""'"); + } + arg1 = reinterpret_cast< ConfigHash * >(argp1); { - std::string *ptr = (std::string *)0; - res1 = SWIG_AsPtr_std_string(swig_obj[0], &ptr); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_EngineHashSwig" "', argument " "1"" of type '" "std::string const &""'"); + try { + result = (arg1)->getIndexedProperties();; } - if (!ptr) { - SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "new_EngineHashSwig" "', argument " "1"" of type '" "std::string const &""'"); + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); } - arg1 = ptr; - } - res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_ConfigHash, 0 | 0 ); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "new_EngineHashSwig" "', argument " "2"" of type '" "ConfigHash *""'"); - } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = swig::from(static_cast< std::vector< std::string,std::allocator< std::string > > >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_ConfigHashSwig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ConfigHash *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ConfigHash, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ConfigHashSwig" "', argument " "1"" of type '" "ConfigHash *""'"); + } + arg1 = reinterpret_cast< ConfigHash * >(argp1); + { + try { + delete arg1;; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *ConfigHashSwig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj = NULL; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_ConfigHash, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *ConfigHashSwig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_iterator(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + PyObject **arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + swig::SwigPyIterator *result = 0 ; + + arg2 = &swig_obj[0]; + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_iterator" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (swig::SwigPyIterator *)std_vector_Sl_uint32_t_Sg__iterator(arg1,arg2);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + if (!result) SWIG_fail; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___nonzero__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___nonzero__" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (bool)std_vector_Sl_uint32_t_Sg____nonzero__((std::vector< unsigned int > const *)arg1);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___bool__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___bool__" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (bool)std_vector_Sl_uint32_t_Sg____bool__((std::vector< unsigned int > const *)arg1);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___len__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned int >::size_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___len__" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = std_vector_Sl_uint32_t_Sg____len__((std::vector< unsigned int > const *)arg1);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___getslice__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + std::vector< unsigned int >::difference_type arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + std::vector< unsigned int,std::allocator< unsigned int > > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig___getslice__", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___getslice__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___getslice__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig___getslice__" "', argument " "3"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg3 = static_cast< std::vector< unsigned int >::difference_type >(val3); + { + try { + try { + result = (std::vector< unsigned int,std::allocator< unsigned int > > *)std_vector_Sl_uint32_t_Sg____getslice__(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setslice____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + std::vector< unsigned int >::difference_type arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___setslice__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___setslice__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig___setslice__" "', argument " "3"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg3 = static_cast< std::vector< unsigned int >::difference_type >(val3); + { + try { + try { + std_vector_Sl_uint32_t_Sg____setslice____SWIG_0(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setslice____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + std::vector< unsigned int >::difference_type arg3 ; + std::vector< unsigned int,std::allocator< unsigned int > > *arg4 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + int res4 = SWIG_OLDOBJ ; + + (void)self; + if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___setslice__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___setslice__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig___setslice__" "', argument " "3"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg3 = static_cast< std::vector< unsigned int >::difference_type >(val3); + { + std::vector< unsigned int,std::allocator< unsigned int > > *ptr = (std::vector< unsigned int,std::allocator< unsigned int > > *)0; + res4 = swig::asptr(swig_obj[3], &ptr); + if (!SWIG_IsOK(res4)) { + SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "VectorUInt32Swig___setslice__" "', argument " "4"" of type '" "std::vector< unsigned int,std::allocator< unsigned int > > const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "VectorUInt32Swig___setslice__" "', argument " "4"" of type '" "std::vector< unsigned int,std::allocator< unsigned int > > const &""'"); + } + arg4 = ptr; + } + { + try { + try { + std_vector_Sl_uint32_t_Sg____setslice____SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(std::vector< unsigned int,std::allocator< unsigned int > > const &)*arg4); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res4)) delete arg4; + return resultobj; +fail: + if (SWIG_IsNewObj(res4)) delete arg4; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setslice__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[5] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig___setslice__", 0, 4, argv))) SWIG_fail; + --argc; + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig___setslice____SWIG_0(self, argc, argv); + } + } + } + } + if (argc == 4) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + int res = swig::asptr(argv[3], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_VectorUInt32Swig___setslice____SWIG_1(self, argc, argv); + } + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig___setslice__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::__setslice__(std::vector< unsigned int >::difference_type,std::vector< unsigned int >::difference_type)\n" + " std::vector< uint32_t >::__setslice__(std::vector< unsigned int >::difference_type,std::vector< unsigned int >::difference_type,std::vector< unsigned int,std::allocator< unsigned int > > const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___delslice__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + std::vector< unsigned int >::difference_type arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig___delslice__", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___delslice__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___delslice__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig___delslice__" "', argument " "3"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg3 = static_cast< std::vector< unsigned int >::difference_type >(val3); + { + try { + try { + std_vector_Sl_uint32_t_Sg____delslice__(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___delitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___delitem__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___delitem__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + { + try { + try { + std_vector_Sl_uint32_t_Sg____delitem____SWIG_0(arg1,SWIG_STD_MOVE(arg2)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___getitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SWIGPY_SLICEOBJECT *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::vector< unsigned int,std::allocator< unsigned int > > *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___getitem__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig___getitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + { + try { + try { + result = (std::vector< unsigned int,std::allocator< unsigned int > > *)std_vector_Sl_uint32_t_Sg____getitem____SWIG_0(arg1,arg2); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SWIGPY_SLICEOBJECT *arg2 = 0 ; + std::vector< unsigned int,std::allocator< unsigned int > > *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + int res3 = SWIG_OLDOBJ ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___setitem__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig___setitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + { + std::vector< unsigned int,std::allocator< unsigned int > > *ptr = (std::vector< unsigned int,std::allocator< unsigned int > > *)0; + res3 = swig::asptr(swig_obj[2], &ptr); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "VectorUInt32Swig___setitem__" "', argument " "3"" of type '" "std::vector< unsigned int,std::allocator< unsigned int > > const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "VectorUInt32Swig___setitem__" "', argument " "3"" of type '" "std::vector< unsigned int,std::allocator< unsigned int > > const &""'"); + } + arg3 = ptr; + } + { + try { + try { + std_vector_Sl_uint32_t_Sg____setitem____SWIG_0(arg1,arg2,(std::vector< unsigned int,std::allocator< unsigned int > > const &)*arg3); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res3)) delete arg3; + return resultobj; +fail: + if (SWIG_IsNewObj(res3)) delete arg3; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SWIGPY_SLICEOBJECT *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___setitem__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig___setitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + { + try { + try { + std_vector_Sl_uint32_t_Sg____setitem____SWIG_1(arg1,arg2); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___delitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SWIGPY_SLICEOBJECT *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___delitem__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig___delitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + { + try { + try { + std_vector_Sl_uint32_t_Sg____delitem____SWIG_1(arg1,arg2); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___delitem__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig___delitem__", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + return _wrap_VectorUInt32Swig___delitem____SWIG_1(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig___delitem____SWIG_0(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig___delitem__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::__delitem__(std::vector< unsigned int >::difference_type)\n" + " std::vector< uint32_t >::__delitem__(SWIGPY_SLICEOBJECT *)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___getitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + std::vector< unsigned int >::value_type *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___getitem__" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___getitem__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + { + try { + try { + result = (std::vector< unsigned int >::value_type *) &std_vector_Sl_uint32_t_Sg____getitem____SWIG_1((std::vector< unsigned int > const *)arg1,SWIG_STD_MOVE(arg2)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(*result)); + if (!swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]) && PyErr_Occurred()) { + SWIG_fail; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___getitem__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig___getitem__", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + return _wrap_VectorUInt32Swig___getitem____SWIG_0(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig___getitem____SWIG_1(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig___getitem__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::__getitem__(SWIGPY_SLICEOBJECT *)\n" + " std::vector< uint32_t >::__getitem__(std::vector< unsigned int >::difference_type) const\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setitem____SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + std::vector< unsigned int >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + std::vector< unsigned int >::value_type temp3 ; + unsigned int val3 ; + int ecode3 = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___setitem__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___setitem__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + ecode3 = SWIG_AsVal_unsigned_SS_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig___setitem__" "', argument " "3"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp3 = static_cast< std::vector< unsigned int >::value_type >(val3); + arg3 = &temp3; + { + try { + try { + std_vector_Sl_uint32_t_Sg____setitem____SWIG_2(arg1,SWIG_STD_MOVE(arg2),(unsigned int const &)*arg3); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setitem__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig___setitem__", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + return _wrap_VectorUInt32Swig___setitem____SWIG_1(self, argc, argv); + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + int res = swig::asptr(argv[2], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_VectorUInt32Swig___setitem____SWIG_0(self, argc, argv); + } + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_int(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig___setitem____SWIG_2(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig___setitem__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::__setitem__(SWIGPY_SLICEOBJECT *,std::vector< unsigned int,std::allocator< unsigned int > > const &)\n" + " std::vector< uint32_t >::__setitem__(SWIGPY_SLICEOBJECT *)\n" + " std::vector< uint32_t >::__setitem__(std::vector< unsigned int >::difference_type,std::vector< unsigned int >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_pop(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned int >::value_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_pop" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + try { + result = (std::vector< unsigned int >::value_type)std_vector_Sl_uint32_t_Sg__pop(arg1); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_append(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::value_type *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::vector< unsigned int >::value_type temp2 ; + unsigned int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_append", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_append" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_unsigned_SS_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig_append" "', argument " "2"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp2 = static_cast< std::vector< unsigned int >::value_type >(val2); + arg2 = &temp2; + { + try { + std_vector_Sl_uint32_t_Sg__append(arg1,(unsigned int const &)*arg2);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_VectorUInt32Swig__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) { + PyObject *resultobj = 0; + std::vector< uint32_t > *result = 0 ; + + (void)self; + if ((nobjs < 0) || (nobjs > 0)) SWIG_fail; + { + try { + result = (std::vector< uint32_t > *)new std::vector< uint32_t >();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_VectorUInt32Swig__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + int res1 = SWIG_OLDOBJ ; + std::vector< uint32_t > *result = 0 ; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + { + std::vector< unsigned int,std::allocator< unsigned int > > *ptr = (std::vector< unsigned int,std::allocator< unsigned int > > *)0; + res1 = swig::asptr(swig_obj[0], &ptr); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_VectorUInt32Swig" "', argument " "1"" of type '" "std::vector< uint32_t > const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "new_VectorUInt32Swig" "', argument " "1"" of type '" "std::vector< uint32_t > const &""'"); + } + arg1 = ptr; + } + { + try { + result = (std::vector< uint32_t > *)new std::vector< uint32_t >((std::vector< uint32_t > const &)*arg1);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_NEW | 0 ); + if (SWIG_IsNewObj(res1)) delete arg1; + return resultobj; +fail: + if (SWIG_IsNewObj(res1)) delete arg1; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_empty(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_empty" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (bool)((std::vector< uint32_t > const *)arg1)->empty();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_size(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned int >::size_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_size" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = ((std::vector< uint32_t > const *)arg1)->size();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_swap(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< uint32_t > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_swap", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_swap" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__vectorT_uint32_t_t, 0 ); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "VectorUInt32Swig_swap" "', argument " "2"" of type '" "std::vector< uint32_t > &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "VectorUInt32Swig_swap" "', argument " "2"" of type '" "std::vector< uint32_t > &""'"); + } + arg2 = reinterpret_cast< std::vector< uint32_t > * >(argp2); + { + try { + (arg1)->swap(*arg2);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_begin(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< unsigned int >::iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_begin" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (arg1)->begin();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_end(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< unsigned int >::iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_end" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (arg1)->end();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_rbegin(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< unsigned int >::reverse_iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_rbegin" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (arg1)->rbegin();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::reverse_iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_rend(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< unsigned int >::reverse_iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_rend" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (arg1)->rend();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::reverse_iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_clear(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_clear" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + (arg1)->clear();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_get_allocator(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::allocator< unsigned int > > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_get_allocator" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = ((std::vector< uint32_t > const *)arg1)->get_allocator();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj((new std::vector< unsigned int >::allocator_type(result)), SWIGTYPE_p_std__allocatorT_unsigned_int_t, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_VectorUInt32Swig__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< unsigned int >::size_type arg1 ; + size_t val1 ; + int ecode1 = 0 ; + std::vector< uint32_t > *result = 0 ; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + ecode1 = SWIG_AsVal_size_t(swig_obj[0], &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_VectorUInt32Swig" "', argument " "1"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg1 = static_cast< std::vector< unsigned int >::size_type >(val1); + { + try { + result = (std::vector< uint32_t > *)new std::vector< uint32_t >(SWIG_STD_MOVE(arg1));; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_pop_back(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_pop_back" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + (arg1)->pop_back();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_resize__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::size_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_resize" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig_resize" "', argument " "2"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::size_type >(val2); + { + try { + (arg1)->resize(SWIG_STD_MOVE(arg2));; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_erase__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > result; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_erase" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_erase" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_erase" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } + } + { + try { + result = std_vector_Sl_uint32_t_Sg__erase__SWIG_0(arg1,SWIG_STD_MOVE(arg2));; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_erase__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > arg2 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + swig::SwigPyIterator *iter3 = 0 ; + int res3 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > result; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_erase" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_erase" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_erase" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } + } + res3 = SWIG_ConvertPtr(swig_obj[2], SWIG_as_voidptrptr(&iter3), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res3) || !iter3) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_erase" "', argument " "3"" of type '" "std::vector< unsigned int >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter3); + if (iter_t) { + arg3 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_erase" "', argument " "3"" of type '" "std::vector< unsigned int >::iterator""'"); + } + } + { + try { + result = std_vector_Sl_uint32_t_Sg__erase__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3));; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_erase(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_erase", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + return _wrap_VectorUInt32Swig_erase__SWIG_0(self, argc, argv); + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[2], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + return _wrap_VectorUInt32Swig_erase__SWIG_1(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig_erase'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::erase(std::vector< unsigned int >::iterator)\n" + " std::vector< uint32_t >::erase(std::vector< unsigned int >::iterator,std::vector< unsigned int >::iterator)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_new_VectorUInt32Swig__SWIG_3(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< unsigned int >::size_type arg1 ; + std::vector< unsigned int >::value_type *arg2 = 0 ; + size_t val1 ; + int ecode1 = 0 ; + std::vector< unsigned int >::value_type temp2 ; + unsigned int val2 ; + int ecode2 = 0 ; + std::vector< uint32_t > *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + ecode1 = SWIG_AsVal_size_t(swig_obj[0], &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_VectorUInt32Swig" "', argument " "1"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg1 = static_cast< std::vector< unsigned int >::size_type >(val1); + ecode2 = SWIG_AsVal_unsigned_SS_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_VectorUInt32Swig" "', argument " "2"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp2 = static_cast< std::vector< unsigned int >::value_type >(val2); + arg2 = &temp2; + { + try { + result = (std::vector< uint32_t > *)new std::vector< uint32_t >(SWIG_STD_MOVE(arg1),(std::vector< unsigned int >::value_type const &)*arg2);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_VectorUInt32Swig(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "new_VectorUInt32Swig", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 0) { + return _wrap_new_VectorUInt32Swig__SWIG_0(self, argc, argv); + } + if (argc == 1) { + int _v = 0; + { + int res = SWIG_AsVal_size_t(argv[0], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_new_VectorUInt32Swig__SWIG_2(self, argc, argv); + } + } + if (argc == 1) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_new_VectorUInt32Swig__SWIG_1(self, argc, argv); + } + } + if (argc == 2) { + int _v = 0; + { + int res = SWIG_AsVal_size_t(argv[0], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_int(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_new_VectorUInt32Swig__SWIG_3(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_VectorUInt32Swig'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::vector()\n" + " std::vector< uint32_t >::vector(std::vector< uint32_t > const &)\n" + " std::vector< uint32_t >::vector(std::vector< unsigned int >::size_type)\n" + " std::vector< uint32_t >::vector(std::vector< unsigned int >::size_type,std::vector< unsigned int >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_push_back(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::value_type *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::vector< unsigned int >::value_type temp2 ; + unsigned int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_push_back", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_push_back" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_unsigned_SS_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig_push_back" "', argument " "2"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp2 = static_cast< std::vector< unsigned int >::value_type >(val2); + arg2 = &temp2; + { + try { + (arg1)->push_back((std::vector< unsigned int >::value_type const &)*arg2);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_front(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned int >::value_type *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_front" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (std::vector< unsigned int >::value_type *) &((std::vector< uint32_t > const *)arg1)->front();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(*result)); + if (!swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]) && PyErr_Occurred()) { + SWIG_fail; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_back(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned int >::value_type *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_back" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (std::vector< unsigned int >::value_type *) &((std::vector< uint32_t > const *)arg1)->back();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(*result)); + if (!swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]) && PyErr_Occurred()) { + SWIG_fail; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_assign(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::size_type arg2 ; + std::vector< unsigned int >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + std::vector< unsigned int >::value_type temp3 ; + unsigned int val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_assign", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_assign" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig_assign" "', argument " "2"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::size_type >(val2); + ecode3 = SWIG_AsVal_unsigned_SS_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig_assign" "', argument " "3"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp3 = static_cast< std::vector< unsigned int >::value_type >(val3); + arg3 = &temp3; + { + try { + (arg1)->assign(SWIG_STD_MOVE(arg2),(std::vector< unsigned int >::value_type const &)*arg3);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_resize__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::size_type arg2 ; + std::vector< unsigned int >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + std::vector< unsigned int >::value_type temp3 ; + unsigned int val3 ; + int ecode3 = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_resize" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig_resize" "', argument " "2"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::size_type >(val2); + ecode3 = SWIG_AsVal_unsigned_SS_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig_resize" "', argument " "3"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp3 = static_cast< std::vector< unsigned int >::value_type >(val3); + arg3 = &temp3; + { + try { + (arg1)->resize(SWIG_STD_MOVE(arg2),(std::vector< unsigned int >::value_type const &)*arg3);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_resize(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_resize", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig_resize__SWIG_0(self, argc, argv); + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_int(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig_resize__SWIG_1(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig_resize'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::resize(std::vector< unsigned int >::size_type)\n" + " std::vector< uint32_t >::resize(std::vector< unsigned int >::size_type,std::vector< unsigned int >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_insert__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > arg2 ; + std::vector< unsigned int >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + std::vector< unsigned int >::value_type temp3 ; + unsigned int val3 ; + int ecode3 = 0 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > result; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_insert" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_insert" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_insert" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } + } + ecode3 = SWIG_AsVal_unsigned_SS_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig_insert" "', argument " "3"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp3 = static_cast< std::vector< unsigned int >::value_type >(val3); + arg3 = &temp3; + { + try { + result = std_vector_Sl_uint32_t_Sg__insert__SWIG_0(arg1,SWIG_STD_MOVE(arg2),(unsigned int const &)*arg3);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_insert__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > arg2 ; + std::vector< unsigned int >::size_type arg3 ; + std::vector< unsigned int >::value_type *arg4 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + size_t val3 ; + int ecode3 = 0 ; + std::vector< unsigned int >::value_type temp4 ; + unsigned int val4 ; + int ecode4 = 0 ; + + (void)self; + if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_insert" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_insert" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_insert" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } + } + ecode3 = SWIG_AsVal_size_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig_insert" "', argument " "3"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg3 = static_cast< std::vector< unsigned int >::size_type >(val3); + ecode4 = SWIG_AsVal_unsigned_SS_int(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "VectorUInt32Swig_insert" "', argument " "4"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp4 = static_cast< std::vector< unsigned int >::value_type >(val4); + arg4 = &temp4; + { + try { + std_vector_Sl_uint32_t_Sg__insert__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(unsigned int const &)*arg4);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_insert(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[5] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_insert", 0, 4, argv))) SWIG_fail; + --argc; + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_int(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig_insert__SWIG_0(self, argc, argv); + } + } + } + } + if (argc == 4) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_int(argv[3], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig_insert__SWIG_1(self, argc, argv); + } + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig_insert'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::insert(std::vector< unsigned int >::iterator,std::vector< unsigned int >::value_type const &)\n" + " std::vector< uint32_t >::insert(std::vector< unsigned int >::iterator,std::vector< unsigned int >::size_type,std::vector< unsigned int >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_reserve(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::size_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_reserve", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_reserve" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig_reserve" "', argument " "2"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::size_type >(val2); + { + try { + (arg1)->reserve(SWIG_STD_MOVE(arg2));; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_capacity(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned int >::size_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_capacity" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = ((std::vector< uint32_t > const *)arg1)->capacity();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_VectorUInt32Swig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_VectorUInt32Swig" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + delete arg1;; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *VectorUInt32Swig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj = NULL; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *VectorUInt32Swig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_EngineHashSwig__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::string *arg1 = 0 ; + ConfigHash *arg2 = 0 ; + RequiredPropertiesConfig *arg3 = 0 ; + int res1 = SWIG_OLDOBJ ; + void *argp2 = 0 ; + int res2 = 0 ; + void *argp3 = 0 ; + int res3 = 0 ; + EngineHash *result = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + { + std::string *ptr = (std::string *)0; + res1 = SWIG_AsPtr_std_string(swig_obj[0], &ptr); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_EngineHashSwig" "', argument " "1"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "new_EngineHashSwig" "', argument " "1"" of type '" "std::string const &""'"); + } + arg1 = ptr; + } + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_ConfigHash, 0 | 0 ); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "new_EngineHashSwig" "', argument " "2"" of type '" "ConfigHash *""'"); + } arg2 = reinterpret_cast< ConfigHash * >(argp2); res3 = SWIG_ConvertPtr(swig_obj[2], &argp3,SWIGTYPE_p_RequiredPropertiesConfig, 0 | 0 ); if (!SWIG_IsOK(res3)) { @@ -27030,125 +30626,95 @@ SWIGINTERN PyObject *_wrap_EngineHashSwig_process__SWIG_1(PyObject *self, Py_ssi } -SWIGINTERN PyObject *_wrap_EngineHashSwig_process__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { - PyObject *resultobj = 0; - EngineHash *arg1 = 0 ; - EvidenceDeviceDetection *arg2 = 0 ; - int *arg3 = 0 ; - int arg4 ; - void *argp1 = 0 ; - int res1 = 0 ; - void *argp2 = 0 ; - int res2 = 0 ; - void *argp3 = 0 ; - int res3 = 0 ; - int val4 ; - int ecode4 = 0 ; - ResultsHash *result = 0 ; +SWIGINTERN PyObject *_wrap_EngineHashSwig_process(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; - (void)self; - if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; - res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_EngineHash, 0 | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EngineHashSwig_process" "', argument " "1"" of type '" "EngineHash *""'"); - } - arg1 = reinterpret_cast< EngineHash * >(argp1); - res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_EvidenceDeviceDetection, 0 | 0 ); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EngineHashSwig_process" "', argument " "2"" of type '" "EvidenceDeviceDetection *""'"); - } - arg2 = reinterpret_cast< EvidenceDeviceDetection * >(argp2); - res3 = SWIG_ConvertPtr(swig_obj[2], &argp3,SWIGTYPE_p_int, 0 | 0 ); - if (!SWIG_IsOK(res3)) { - SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "EngineHashSwig_process" "', argument " "3"" of type '" "int const *""'"); - } - arg3 = reinterpret_cast< int * >(argp3); - ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); - if (!SWIG_IsOK(ecode4)) { - SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "EngineHashSwig_process" "', argument " "4"" of type '" "int""'"); - } - arg4 = static_cast< int >(val4); - { - try { - result = (ResultsHash *)(arg1)->process(arg2,(int const *)arg3,arg4);; - } - catch(FatalException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NotImplementedException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(InvalidPropertyException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(EvidenceException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(TooManyValuesException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NoValuesAvailableException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(StatusCodeException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(runtime_error& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); + if (!(argc = SWIG_Python_UnpackTuple(args, "EngineHashSwig_process", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineHash, 0); + _v = SWIG_CheckState(res); + if (_v) { + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_EvidenceDeviceDetection, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_EngineHashSwig_process__SWIG_0(self, argc, argv); + } } - catch(invalid_argument& e) { - SWIG_exception(SWIG_ValueError, e.what()); + } + if (argc == 2) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineHash, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_EngineHashSwig_process__SWIG_1(self, argc, argv); + } } } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ResultsHash, SWIG_POINTER_OWN | 0 ); - return resultobj; + fail: - return NULL; + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'EngineHashSwig_process'.\n" + " Possible C/C++ prototypes are:\n" + " EngineHash::process(EvidenceDeviceDetection *)\n" + " EngineHash::process(char const *)\n"); + return 0; } -SWIGINTERN PyObject *_wrap_EngineHashSwig_process__SWIG_3(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { +SWIGINTERN PyObject *_wrap_EngineHashSwig_getProfileIdsForPropertyValue(PyObject *self, PyObject *args) { PyObject *resultobj = 0; EngineHash *arg1 = 0 ; - char *arg2 = 0 ; - int *arg3 = 0 ; - int arg4 ; + std::string *arg2 = 0 ; + std::string *arg3 = 0 ; void *argp1 = 0 ; int res1 = 0 ; - int res2 ; - char *buf2 = 0 ; - int alloc2 = 0 ; - void *argp3 = 0 ; - int res3 = 0 ; - int val4 ; - int ecode4 = 0 ; - ResultsHash *result = 0 ; + int res2 = SWIG_OLDOBJ ; + int res3 = SWIG_OLDOBJ ; + PyObject *swig_obj[3] ; + std::vector< uint32_t,std::allocator< uint32_t > > result; (void)self; - if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; + if (!SWIG_Python_UnpackTuple(args, "EngineHashSwig_getProfileIdsForPropertyValue", 3, 3, swig_obj)) SWIG_fail; res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_EngineHash, 0 | 0 ); if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EngineHashSwig_process" "', argument " "1"" of type '" "EngineHash *""'"); + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EngineHashSwig_getProfileIdsForPropertyValue" "', argument " "1"" of type '" "EngineHash *""'"); } arg1 = reinterpret_cast< EngineHash * >(argp1); - res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EngineHashSwig_process" "', argument " "2"" of type '" "char const *""'"); + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EngineHashSwig_getProfileIdsForPropertyValue" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "EngineHashSwig_getProfileIdsForPropertyValue" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; } - arg2 = reinterpret_cast< char * >(buf2); - res3 = SWIG_ConvertPtr(swig_obj[2], &argp3,SWIGTYPE_p_int, 0 | 0 ); - if (!SWIG_IsOK(res3)) { - SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "EngineHashSwig_process" "', argument " "3"" of type '" "int const *""'"); + { + std::string *ptr = (std::string *)0; + res3 = SWIG_AsPtr_std_string(swig_obj[2], &ptr); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "EngineHashSwig_getProfileIdsForPropertyValue" "', argument " "3"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "EngineHashSwig_getProfileIdsForPropertyValue" "', argument " "3"" of type '" "std::string const &""'"); + } + arg3 = ptr; } - arg3 = reinterpret_cast< int * >(argp3); - ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); - if (!SWIG_IsOK(ecode4)) { - SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "EngineHashSwig_process" "', argument " "4"" of type '" "int""'"); - } - arg4 = static_cast< int >(val4); { try { - result = (ResultsHash *)(arg1)->process((char const *)arg2,(int const *)arg3,arg4);; + result = (arg1)->getProfileIdsForPropertyValue((std::string const &)*arg2,(std::string const &)*arg3);; } catch(FatalException& e) { SWIG_exception(SWIG_RuntimeError, e.what()); @@ -27178,111 +30744,17 @@ SWIGINTERN PyObject *_wrap_EngineHashSwig_process__SWIG_3(PyObject *self, Py_ssi SWIG_exception(SWIG_ValueError, e.what()); } } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ResultsHash, SWIG_POINTER_OWN | 0 ); - if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + resultobj = swig::from(static_cast< std::vector< unsigned int,std::allocator< unsigned int > > >(result)); + if (SWIG_IsNewObj(res2)) delete arg2; + if (SWIG_IsNewObj(res3)) delete arg3; return resultobj; fail: - if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + if (SWIG_IsNewObj(res2)) delete arg2; + if (SWIG_IsNewObj(res3)) delete arg3; return NULL; } -SWIGINTERN PyObject *_wrap_EngineHashSwig_process(PyObject *self, PyObject *args) { - Py_ssize_t argc; - PyObject *argv[5] = { - 0 - }; - - if (!(argc = SWIG_Python_UnpackTuple(args, "EngineHashSwig_process", 0, 4, argv))) SWIG_fail; - --argc; - if (argc == 2) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineHash, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_EvidenceDeviceDetection, 0); - _v = SWIG_CheckState(res); - if (_v) { - return _wrap_EngineHashSwig_process__SWIG_0(self, argc, argv); - } - } - } - if (argc == 2) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineHash, 0); - _v = SWIG_CheckState(res); - if (_v) { - int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); - _v = SWIG_CheckState(res); - if (_v) { - return _wrap_EngineHashSwig_process__SWIG_1(self, argc, argv); - } - } - } - if (argc == 4) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineHash, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_EvidenceDeviceDetection, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[2], &vptr, SWIGTYPE_p_int, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_int(argv[3], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - return _wrap_EngineHashSwig_process__SWIG_2(self, argc, argv); - } - } - } - } - } - if (argc == 4) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineHash, 0); - _v = SWIG_CheckState(res); - if (_v) { - int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[2], &vptr, SWIGTYPE_p_int, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_int(argv[3], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - return _wrap_EngineHashSwig_process__SWIG_3(self, argc, argv); - } - } - } - } - } - -fail: - SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'EngineHashSwig_process'.\n" - " Possible C/C++ prototypes are:\n" - " EngineHash::process(EvidenceDeviceDetection *)\n" - " EngineHash::process(char const *)\n" - " EngineHash::process(EvidenceDeviceDetection *,int const *,int)\n" - " EngineHash::process(char const *,int const *,int)\n"); - return 0; -} - - SWIGINTERN PyObject *_wrap_EngineHashSwig_processBase(PyObject *self, PyObject *args) { PyObject *resultobj = 0; EngineHash *arg1 = 0 ; @@ -28577,6 +32049,8 @@ static PyMethodDef SwigMethods[] = { { "ConfigHashSwig_setUsePerformanceGraph", _wrap_ConfigHashSwig_setUsePerformanceGraph, METH_VARARGS, NULL}, { "ConfigHashSwig_setUsePredictiveGraph", _wrap_ConfigHashSwig_setUsePredictiveGraph, METH_VARARGS, NULL}, { "ConfigHashSwig_setTraceRoute", _wrap_ConfigHashSwig_setTraceRoute, METH_VARARGS, NULL}, + { "ConfigHashSwig_setPropertyValueIndex", _wrap_ConfigHashSwig_setPropertyValueIndex, METH_VARARGS, NULL}, + { "ConfigHashSwig_setIndexedProperties", _wrap_ConfigHashSwig_setIndexedProperties, METH_VARARGS, NULL}, { "ConfigHashSwig_getStrings", _wrap_ConfigHashSwig_getStrings, METH_O, NULL}, { "ConfigHashSwig_getProperties", _wrap_ConfigHashSwig_getProperties, METH_O, NULL}, { "ConfigHashSwig_getValues", _wrap_ConfigHashSwig_getValues, METH_O, NULL}, @@ -28589,9 +32063,46 @@ static PyMethodDef SwigMethods[] = { { "ConfigHashSwig_getUsePredictiveGraph", _wrap_ConfigHashSwig_getUsePredictiveGraph, METH_O, NULL}, { "ConfigHashSwig_getConcurrency", _wrap_ConfigHashSwig_getConcurrency, METH_O, NULL}, { "ConfigHashSwig_getTraceRoute", _wrap_ConfigHashSwig_getTraceRoute, METH_O, NULL}, + { "ConfigHashSwig_getPropertyValueIndex", _wrap_ConfigHashSwig_getPropertyValueIndex, METH_O, NULL}, + { "ConfigHashSwig_getIndexedProperties", _wrap_ConfigHashSwig_getIndexedProperties, METH_O, NULL}, { "delete_ConfigHashSwig", _wrap_delete_ConfigHashSwig, METH_O, NULL}, { "ConfigHashSwig_swigregister", ConfigHashSwig_swigregister, METH_O, NULL}, { "ConfigHashSwig_swiginit", ConfigHashSwig_swiginit, METH_VARARGS, NULL}, + { "VectorUInt32Swig_iterator", _wrap_VectorUInt32Swig_iterator, METH_O, NULL}, + { "VectorUInt32Swig___nonzero__", _wrap_VectorUInt32Swig___nonzero__, METH_O, NULL}, + { "VectorUInt32Swig___bool__", _wrap_VectorUInt32Swig___bool__, METH_O, NULL}, + { "VectorUInt32Swig___len__", _wrap_VectorUInt32Swig___len__, METH_O, NULL}, + { "VectorUInt32Swig___getslice__", _wrap_VectorUInt32Swig___getslice__, METH_VARARGS, NULL}, + { "VectorUInt32Swig___setslice__", _wrap_VectorUInt32Swig___setslice__, METH_VARARGS, NULL}, + { "VectorUInt32Swig___delslice__", _wrap_VectorUInt32Swig___delslice__, METH_VARARGS, NULL}, + { "VectorUInt32Swig___delitem__", _wrap_VectorUInt32Swig___delitem__, METH_VARARGS, NULL}, + { "VectorUInt32Swig___getitem__", _wrap_VectorUInt32Swig___getitem__, METH_VARARGS, NULL}, + { "VectorUInt32Swig___setitem__", _wrap_VectorUInt32Swig___setitem__, METH_VARARGS, NULL}, + { "VectorUInt32Swig_pop", _wrap_VectorUInt32Swig_pop, METH_O, NULL}, + { "VectorUInt32Swig_append", _wrap_VectorUInt32Swig_append, METH_VARARGS, NULL}, + { "VectorUInt32Swig_empty", _wrap_VectorUInt32Swig_empty, METH_O, NULL}, + { "VectorUInt32Swig_size", _wrap_VectorUInt32Swig_size, METH_O, NULL}, + { "VectorUInt32Swig_swap", _wrap_VectorUInt32Swig_swap, METH_VARARGS, NULL}, + { "VectorUInt32Swig_begin", _wrap_VectorUInt32Swig_begin, METH_O, NULL}, + { "VectorUInt32Swig_end", _wrap_VectorUInt32Swig_end, METH_O, NULL}, + { "VectorUInt32Swig_rbegin", _wrap_VectorUInt32Swig_rbegin, METH_O, NULL}, + { "VectorUInt32Swig_rend", _wrap_VectorUInt32Swig_rend, METH_O, NULL}, + { "VectorUInt32Swig_clear", _wrap_VectorUInt32Swig_clear, METH_O, NULL}, + { "VectorUInt32Swig_get_allocator", _wrap_VectorUInt32Swig_get_allocator, METH_O, NULL}, + { "VectorUInt32Swig_pop_back", _wrap_VectorUInt32Swig_pop_back, METH_O, NULL}, + { "VectorUInt32Swig_erase", _wrap_VectorUInt32Swig_erase, METH_VARARGS, NULL}, + { "new_VectorUInt32Swig", _wrap_new_VectorUInt32Swig, METH_VARARGS, NULL}, + { "VectorUInt32Swig_push_back", _wrap_VectorUInt32Swig_push_back, METH_VARARGS, NULL}, + { "VectorUInt32Swig_front", _wrap_VectorUInt32Swig_front, METH_O, NULL}, + { "VectorUInt32Swig_back", _wrap_VectorUInt32Swig_back, METH_O, NULL}, + { "VectorUInt32Swig_assign", _wrap_VectorUInt32Swig_assign, METH_VARARGS, NULL}, + { "VectorUInt32Swig_resize", _wrap_VectorUInt32Swig_resize, METH_VARARGS, NULL}, + { "VectorUInt32Swig_insert", _wrap_VectorUInt32Swig_insert, METH_VARARGS, NULL}, + { "VectorUInt32Swig_reserve", _wrap_VectorUInt32Swig_reserve, METH_VARARGS, NULL}, + { "VectorUInt32Swig_capacity", _wrap_VectorUInt32Swig_capacity, METH_O, NULL}, + { "delete_VectorUInt32Swig", _wrap_delete_VectorUInt32Swig, METH_O, NULL}, + { "VectorUInt32Swig_swigregister", VectorUInt32Swig_swigregister, METH_O, NULL}, + { "VectorUInt32Swig_swiginit", VectorUInt32Swig_swiginit, METH_VARARGS, NULL}, { "new_EngineHashSwig", _wrap_new_EngineHashSwig, METH_VARARGS, NULL}, { "EngineHashSwig_getPublishedTime", _wrap_EngineHashSwig_getPublishedTime, METH_O, NULL}, { "EngineHashSwig_getUpdateAvailableTime", _wrap_EngineHashSwig_getUpdateAvailableTime, METH_O, NULL}, @@ -28601,6 +32112,7 @@ static PyMethodDef SwigMethods[] = { { "EngineHashSwig_refreshDataFromFile", _wrap_EngineHashSwig_refreshDataFromFile, METH_VARARGS, NULL}, { "EngineHashSwig_refreshDataFromMemory", _wrap_EngineHashSwig_refreshDataFromMemory, METH_VARARGS, NULL}, { "EngineHashSwig_process", _wrap_EngineHashSwig_process, METH_VARARGS, NULL}, + { "EngineHashSwig_getProfileIdsForPropertyValue", _wrap_EngineHashSwig_getProfileIdsForPropertyValue, METH_VARARGS, NULL}, { "EngineHashSwig_processBase", _wrap_EngineHashSwig_processBase, METH_VARARGS, NULL}, { "EngineHashSwig_processDeviceDetection", _wrap_EngineHashSwig_processDeviceDetection, METH_VARARGS, NULL}, { "delete_EngineHashSwig", _wrap_delete_EngineHashSwig, METH_O, NULL}, @@ -28706,10 +32218,12 @@ static swig_type_info _swigt__p_size_type = {"_p_size_type", "size_type *", 0, 0 static swig_type_info _swigt__p_std__allocatorT_std__pairT_std__string_const_std__string_t_t = {"_p_std__allocatorT_std__pairT_std__string_const_std__string_t_t", "std::map< std::string,std::string >::allocator_type *|std::allocator< std::pair< std::string const,std::string > > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__allocatorT_std__string_t = {"_p_std__allocatorT_std__string_t", "std::vector< std::string >::allocator_type *|std::allocator< std::string > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__allocatorT_unsigned_char_t = {"_p_std__allocatorT_unsigned_char_t", "std::vector< unsigned char >::allocator_type *|std::allocator< unsigned char > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__allocatorT_unsigned_int_t = {"_p_std__allocatorT_unsigned_int_t", "std::vector< unsigned int >::allocator_type *|std::allocator< unsigned int > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__invalid_argument = {"_p_std__invalid_argument", "std::invalid_argument *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__lessT_std__string_t = {"_p_std__lessT_std__string_t", "std::less< std::string > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__mapT_std__string_std__string_t = {"_p_std__mapT_std__string_std__string_t", "std::map< std::string,std::string,std::less< std::string >,std::allocator< std::pair< std::string const,std::string > > > *|std::map< std::string,std::string > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__vectorT_std__string_t = {"_p_std__vectorT_std__string_t", "std::vector< std::string,std::allocator< std::string > > *|std::vector< std::string > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__vectorT_uint32_t_t = {"_p_std__vectorT_uint32_t_t", "std::vector< uint32_t > *|std::vector< unsigned int,std::allocator< unsigned int > > *|std::vector< unsigned int > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__vectorT_uint8_t_t = {"_p_std__vectorT_uint8_t_t", "std::vector< uint8_t > *|std::vector< unsigned char,std::allocator< unsigned char > > *|std::vector< unsigned char > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_swig__SwigPyIterator = {"_p_swig__SwigPyIterator", "swig::SwigPyIterator *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_unsigned_char = {"_p_unsigned_char", "uint8_t *|uint_fast8_t *|uint_least8_t *|unsigned char *", 0, 0, (void*)0, 0}; @@ -28765,10 +32279,12 @@ static swig_type_info *swig_type_initial[] = { &_swigt__p_std__allocatorT_std__pairT_std__string_const_std__string_t_t, &_swigt__p_std__allocatorT_std__string_t, &_swigt__p_std__allocatorT_unsigned_char_t, + &_swigt__p_std__allocatorT_unsigned_int_t, &_swigt__p_std__invalid_argument, &_swigt__p_std__lessT_std__string_t, &_swigt__p_std__mapT_std__string_std__string_t, &_swigt__p_std__vectorT_std__string_t, + &_swigt__p_std__vectorT_uint32_t_t, &_swigt__p_std__vectorT_uint8_t_t, &_swigt__p_swig__SwigPyIterator, &_swigt__p_unsigned_char, @@ -28824,10 +32340,12 @@ static swig_cast_info _swigc__p_size_type[] = { {&_swigt__p_size_type, 0, 0, 0} static swig_cast_info _swigc__p_std__allocatorT_std__pairT_std__string_const_std__string_t_t[] = { {&_swigt__p_std__allocatorT_std__pairT_std__string_const_std__string_t_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__allocatorT_std__string_t[] = { {&_swigt__p_std__allocatorT_std__string_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__allocatorT_unsigned_char_t[] = { {&_swigt__p_std__allocatorT_unsigned_char_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__allocatorT_unsigned_int_t[] = { {&_swigt__p_std__allocatorT_unsigned_int_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__invalid_argument[] = { {&_swigt__p_std__invalid_argument, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__lessT_std__string_t[] = { {&_swigt__p_std__lessT_std__string_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__mapT_std__string_std__string_t[] = { {&_swigt__p_std__mapT_std__string_std__string_t, 0, 0, 0}, {&_swigt__p_EvidenceBase, _p_EvidenceBaseTo_p_std__mapT_std__string_std__string_t, 0, 0}, {&_swigt__p_EvidenceDeviceDetection, _p_EvidenceDeviceDetectionTo_p_std__mapT_std__string_std__string_t, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__vectorT_std__string_t[] = { {&_swigt__p_std__vectorT_std__string_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__vectorT_uint32_t_t[] = { {&_swigt__p_std__vectorT_uint32_t_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__vectorT_uint8_t_t[] = { {&_swigt__p_std__vectorT_uint8_t_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_swig__SwigPyIterator[] = { {&_swigt__p_swig__SwigPyIterator, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_unsigned_char[] = { {&_swigt__p_unsigned_char, 0, 0, 0},{0, 0, 0, 0}}; @@ -28883,10 +32401,12 @@ static swig_cast_info *swig_cast_initial[] = { _swigc__p_std__allocatorT_std__pairT_std__string_const_std__string_t_t, _swigc__p_std__allocatorT_std__string_t, _swigc__p_std__allocatorT_unsigned_char_t, + _swigc__p_std__allocatorT_unsigned_int_t, _swigc__p_std__invalid_argument, _swigc__p_std__lessT_std__string_t, _swigc__p_std__mapT_std__string_std__string_t, _swigc__p_std__vectorT_std__string_t, + _swigc__p_std__vectorT_uint32_t_t, _swigc__p_std__vectorT_uint8_t_t, _swigc__p_swig__SwigPyIterator, _swigc__p_unsigned_char, diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx index 61d05328..0d472453 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx @@ -47460,263 +47460,13 @@ SWIGINTERN PyObject *_wrap_EngineIpiSwig_process__SWIG_2(PyObject *self, Py_ssiz } -SWIGINTERN PyObject *_wrap_EngineIpiSwig_process__SWIG_3(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { - PyObject *resultobj = 0; - EngineIpi *arg1 = 0 ; - EvidenceIpi *arg2 = 0 ; - int *arg3 = 0 ; - int arg4 ; - void *argp1 = 0 ; - int res1 = 0 ; - void *argp2 = 0 ; - int res2 = 0 ; - void *argp3 = 0 ; - int res3 = 0 ; - int val4 ; - int ecode4 = 0 ; - ResultsIpi *result = 0 ; - - (void)self; - if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; - res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_EngineIpi, 0 | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EngineIpiSwig_process" "', argument " "1"" of type '" "EngineIpi *""'"); - } - arg1 = reinterpret_cast< EngineIpi * >(argp1); - res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_EvidenceIpi, 0 | 0 ); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EngineIpiSwig_process" "', argument " "2"" of type '" "EvidenceIpi *""'"); - } - arg2 = reinterpret_cast< EvidenceIpi * >(argp2); - res3 = SWIG_ConvertPtr(swig_obj[2], &argp3,SWIGTYPE_p_int, 0 | 0 ); - if (!SWIG_IsOK(res3)) { - SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "EngineIpiSwig_process" "', argument " "3"" of type '" "int const *""'"); - } - arg3 = reinterpret_cast< int * >(argp3); - ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); - if (!SWIG_IsOK(ecode4)) { - SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "EngineIpiSwig_process" "', argument " "4"" of type '" "int""'"); - } - arg4 = static_cast< int >(val4); - { - try { - result = (ResultsIpi *)(arg1)->process(arg2,(int const *)arg3,arg4);; - } - catch(FatalException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NotImplementedException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(InvalidPropertyException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(EvidenceException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(TooManyValuesException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NoValuesAvailableException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(StatusCodeException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(runtime_error& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(invalid_argument& e) { - SWIG_exception(SWIG_ValueError, e.what()); - } - } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ResultsIpi, SWIG_POINTER_OWN | 0 ); - return resultobj; -fail: - return NULL; -} - - -SWIGINTERN PyObject *_wrap_EngineIpiSwig_process__SWIG_4(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { - PyObject *resultobj = 0; - EngineIpi *arg1 = 0 ; - char *arg2 = 0 ; - int *arg3 = 0 ; - int arg4 ; - void *argp1 = 0 ; - int res1 = 0 ; - int res2 ; - char *buf2 = 0 ; - int alloc2 = 0 ; - void *argp3 = 0 ; - int res3 = 0 ; - int val4 ; - int ecode4 = 0 ; - ResultsIpi *result = 0 ; - - (void)self; - if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; - res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_EngineIpi, 0 | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EngineIpiSwig_process" "', argument " "1"" of type '" "EngineIpi *""'"); - } - arg1 = reinterpret_cast< EngineIpi * >(argp1); - res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EngineIpiSwig_process" "', argument " "2"" of type '" "char const *""'"); - } - arg2 = reinterpret_cast< char * >(buf2); - res3 = SWIG_ConvertPtr(swig_obj[2], &argp3,SWIGTYPE_p_int, 0 | 0 ); - if (!SWIG_IsOK(res3)) { - SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "EngineIpiSwig_process" "', argument " "3"" of type '" "int const *""'"); - } - arg3 = reinterpret_cast< int * >(argp3); - ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); - if (!SWIG_IsOK(ecode4)) { - SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "EngineIpiSwig_process" "', argument " "4"" of type '" "int""'"); - } - arg4 = static_cast< int >(val4); - { - try { - result = (ResultsIpi *)(arg1)->process((char const *)arg2,(int const *)arg3,arg4);; - } - catch(FatalException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NotImplementedException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(InvalidPropertyException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(EvidenceException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(TooManyValuesException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NoValuesAvailableException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(StatusCodeException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(runtime_error& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(invalid_argument& e) { - SWIG_exception(SWIG_ValueError, e.what()); - } - } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ResultsIpi, SWIG_POINTER_OWN | 0 ); - if (alloc2 == SWIG_NEWOBJ) delete[] buf2; - return resultobj; -fail: - if (alloc2 == SWIG_NEWOBJ) delete[] buf2; - return NULL; -} - - -SWIGINTERN PyObject *_wrap_EngineIpiSwig_process__SWIG_5(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { - PyObject *resultobj = 0; - EngineIpi *arg1 = 0 ; - unsigned char *arg2 = (unsigned char *)0 ; - long arg3 ; - fiftyoneDegreesIpType arg4 ; - int *arg5 = 0 ; - int arg6 ; - void *argp1 = 0 ; - int res1 = 0 ; - void *argp2 = 0 ; - int res2 = 0 ; - long val3 ; - int ecode3 = 0 ; - int val4 ; - int ecode4 = 0 ; - void *argp5 = 0 ; - int res5 = 0 ; - int val6 ; - int ecode6 = 0 ; - ResultsIpi *result = 0 ; - - (void)self; - if ((nobjs < 6) || (nobjs > 6)) SWIG_fail; - res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_EngineIpi, 0 | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EngineIpiSwig_process" "', argument " "1"" of type '" "EngineIpi *""'"); - } - arg1 = reinterpret_cast< EngineIpi * >(argp1); - res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_unsigned_char, 0 | 0 ); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EngineIpiSwig_process" "', argument " "2"" of type '" "unsigned char []""'"); - } - arg2 = reinterpret_cast< unsigned char * >(argp2); - ecode3 = SWIG_AsVal_long(swig_obj[2], &val3); - if (!SWIG_IsOK(ecode3)) { - SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "EngineIpiSwig_process" "', argument " "3"" of type '" "long""'"); - } - arg3 = static_cast< long >(val3); - ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); - if (!SWIG_IsOK(ecode4)) { - SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "EngineIpiSwig_process" "', argument " "4"" of type '" "fiftyoneDegreesIpType""'"); - } - arg4 = static_cast< fiftyoneDegreesIpType >(val4); - res5 = SWIG_ConvertPtr(swig_obj[4], &argp5,SWIGTYPE_p_int, 0 | 0 ); - if (!SWIG_IsOK(res5)) { - SWIG_exception_fail(SWIG_ArgError(res5), "in method '" "EngineIpiSwig_process" "', argument " "5"" of type '" "int const *""'"); - } - arg5 = reinterpret_cast< int * >(argp5); - ecode6 = SWIG_AsVal_int(swig_obj[5], &val6); - if (!SWIG_IsOK(ecode6)) { - SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "EngineIpiSwig_process" "', argument " "6"" of type '" "int""'"); - } - arg6 = static_cast< int >(val6); - { - try { - result = (ResultsIpi *)(arg1)->process(arg2,arg3,arg4,(int const *)arg5,arg6);; - } - catch(FatalException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NotImplementedException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(InvalidPropertyException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(EvidenceException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(TooManyValuesException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NoValuesAvailableException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(StatusCodeException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(runtime_error& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(invalid_argument& e) { - SWIG_exception(SWIG_ValueError, e.what()); - } - } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ResultsIpi, SWIG_POINTER_OWN | 0 ); - return resultobj; -fail: - return NULL; -} - - SWIGINTERN PyObject *_wrap_EngineIpiSwig_process(PyObject *self, PyObject *args) { Py_ssize_t argc; - PyObject *argv[7] = { + PyObject *argv[5] = { 0 }; - if (!(argc = SWIG_Python_UnpackTuple(args, "EngineIpiSwig_process", 0, 6, argv))) SWIG_fail; + if (!(argc = SWIG_Python_UnpackTuple(args, "EngineIpiSwig_process", 0, 4, argv))) SWIG_fail; --argc; if (argc == 2) { int _v = 0; @@ -47771,102 +47521,13 @@ SWIGINTERN PyObject *_wrap_EngineIpiSwig_process(PyObject *self, PyObject *args) } } } - if (argc == 4) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineIpi, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_EvidenceIpi, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[2], &vptr, SWIGTYPE_p_int, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_int(argv[3], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - return _wrap_EngineIpiSwig_process__SWIG_3(self, argc, argv); - } - } - } - } - } - if (argc == 4) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineIpi, 0); - _v = SWIG_CheckState(res); - if (_v) { - int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[2], &vptr, SWIGTYPE_p_int, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_int(argv[3], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - return _wrap_EngineIpiSwig_process__SWIG_4(self, argc, argv); - } - } - } - } - } - if (argc == 6) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineIpi, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_unsigned_char, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_long(argv[2], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - { - int res = SWIG_AsVal_int(argv[3], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[4], &vptr, SWIGTYPE_p_int, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_int(argv[5], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - return _wrap_EngineIpiSwig_process__SWIG_5(self, argc, argv); - } - } - } - } - } - } - } fail: SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'EngineIpiSwig_process'.\n" " Possible C/C++ prototypes are:\n" " EngineIpi::process(EvidenceIpi *)\n" " EngineIpi::process(char const *)\n" - " EngineIpi::process(unsigned char [],long,fiftyoneDegreesIpType)\n" - " EngineIpi::process(EvidenceIpi *,int const *,int)\n" - " EngineIpi::process(char const *,int const *,int)\n" - " EngineIpi::process(unsigned char [],long,fiftyoneDegreesIpType,int const *,int)\n"); + " EngineIpi::process(unsigned char [],long,fiftyoneDegreesIpType)\n"); return 0; } From 23188d3f241b273476c7ff736ebcc2e471c2cd3a Mon Sep 17 00:00:00 2001 From: Automation51D <60396166+Automation51D@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:56:47 +0100 Subject: [PATCH 04/68] Merged Pull Request '#26 update-submodules/rc->rc: Update submodules' Co-authored-by: Automation51D <51DCI@51Degrees.com> --- ip-intelligence/ip-intelligence-data | 2 +- .../src/fiftyone_pipeline_core/js_templates | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ip-intelligence/ip-intelligence-data b/ip-intelligence/ip-intelligence-data index b0f0b6af..b8b3d7d4 160000 --- a/ip-intelligence/ip-intelligence-data +++ b/ip-intelligence/ip-intelligence-data @@ -1 +1 @@ -Subproject commit b0f0b6afe4fe741bc93a7bd6a7a2ad1058dd6e71 +Subproject commit b8b3d7d4e7747a0412281b9b48e0144b2ef13fcc diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/js_templates b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/js_templates index d3f96049..c773ce22 160000 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/js_templates +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/js_templates @@ -1 +1 @@ -Subproject commit d3f96049bde389951c79638047b8ea80dfdd5cdf +Subproject commit c773ce221ccec6fcb67fbdd5ad2175a66fab11d6 From 6732e18af0789559991b8d0d8f4f37666e77a50b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 17:24:48 +0000 Subject: [PATCH 05/68] Read the data files from the data repository in cxx/data (#28) --- .gitmodules | 3 - AGENTS.md | 18 ++++- README.md | 23 ++++++ ci/packages.json | 2 - cxx | 2 +- device-detection/ci/fetch-assets.ps1 | 12 ++- .../examples/gettingstarted.py | 2 +- .../example_utils.py | 76 +++++++++++++++++-- .../onpremise/datafileupdate_console.py | 5 +- .../onpremise/gettingstarted_console.py | 6 +- .../onpremise/gettingstarted_web/app.py | 5 +- .../gettingstarted_web/templates/index.html | 4 +- .../onpremise/match_metrics.py | 5 +- .../onpremise/metadata_console.py | 5 +- .../onpremise/offlineprocessing.py | 5 +- .../onpremise/performance.py | 2 +- .../onpremise/shareusage.py | 5 +- .../onpremise/useragentclienthints_web/app.py | 5 +- .../tests/test_devicedetection.py | 4 +- .../tests/test_properties.py | 2 +- .../tests/test_swig.py | 2 +- device-detection/readme.md | 19 +++-- ip-intelligence/ci/fetch-assets.ps1 | 8 ++ .../readme.md | 13 ++-- .../example_utils.py | 72 +++++++++++++++--- .../onpremise/compare_console.py | 10 +-- .../onpremise/countries_console.py | 5 +- .../onpremise/gettingstarted_console.py | 13 ++-- .../onpremise/gettingstarted_web/app.py | 8 +- .../onpremise/metadata_console.py | 5 +- .../onpremise/mixed_console.py | 19 +++-- .../onpremise/offlineprocessing.py | 8 +- .../onpremise/performance_console.py | 6 +- .../tests/test_examples.py | 12 +-- .../readme.md | 19 +++-- .../tests/constants.py | 11 +-- .../tests/test_ipintelligence.py | 39 +++++----- .../example_constants.py | 13 ++-- .../.gitignore | 4 +- .../setup.py | 15 ++-- .../resources.py | 15 ++-- .../tests/test_translation.py | 15 ++++ ip-intelligence/ip-intelligence-data | 1 - ip-intelligence/readme.md | 43 ++++++----- 44 files changed, 388 insertions(+), 178 deletions(-) delete mode 160000 ip-intelligence/ip-intelligence-data diff --git a/.gitmodules b/.gitmodules index f0d05919..af600379 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,9 +4,6 @@ [submodule "pipeline/owid-python"] path = pipeline/owid-python url = https://github.com/SWAN-community/owid-python.git -[submodule "ip-intelligence/ip-intelligence-data"] - path = ip-intelligence/ip-intelligence-data - url = ../ip-intelligence-data [submodule "cxx"] path = cxx url = ../51degrees-cxx diff --git a/AGENTS.md b/AGENTS.md index 8dececdf..11855b13 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,10 +39,20 @@ so a pipeline change is tested with every product in the same run. ## Submodules -The C/C++ engines and the data files are submodules, listed in the root -`.gitmodules` at their paths under each product folder. The IP Intelligence -on-premise package must compile its own copy of the common C library with -large data file support; the Device Detection one without it. +The C/C++ engines are the `cxx` submodule, listed in the root `.gitmodules`. +The data files come through cxx's own `data` submodule, at +`cxx/data/device-detection` and `cxx/data/ip-intelligence`, where each free +file is a `.link` that `cxx/data/get-data-files.ps1` or +`cxx/data/get-data-files.sh` downloads. Each product's `ci/fetch-assets.ps1` +gets the files its tests need, and the examples' `example_utils.py` runs the +script for a file that is missing. The IP Intelligence on-premise package must +compile its own copy of the common C library with large data file support; +the Device Detection one without it. + +A package's `ExtraPaths` in `ci/packages.json` cannot be a folder inside +`cxx`, because git reports any change there as a change to `cxx` itself. A +cxx update therefore releases no package, even when it brings new translations +in `cxx/data/ip-intelligence`. ## History diff --git a/README.md b/README.md index 3110be9c..2303b4fe 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,29 @@ Clone with the submodules: git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git ``` +The data files the examples and tests use are not stored in a repository. The +[data](https://github.com/51Degrees/data) repository, which is the `cxx/data` +submodule, holds a `.link` file for each free one, where the file goes, and +two scripts that download them. From the root of this repository, run + +``` +sh cxx/data/get-data-files.sh +``` + +or, in PowerShell, + +``` +./cxx/data/get-data-files.ps1 +``` + +An example whose file is missing runs the script for that file itself. + +Git leaves the folder of a removed submodule on disk. A clone made before the +data files moved to `cxx/data` still has an `ip-intelligence/ip-intelligence-data` +folder, and keeps `cxx/device-detection-data` and `cxx/ip-intelligence-data` +once cxx removes them. Delete those folders, so that nothing finds an old copy +of a data file in them. + Each product's README says what else it needs, such as a C/C++ compiler for the on-premise engines and the data files. diff --git a/ci/packages.json b/ci/packages.json index 873df967..deb06ce9 100644 --- a/ci/packages.json +++ b/ci/packages.json @@ -140,7 +140,6 @@ "Name": "fiftyone_ipintelligence_onpremise", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_onpremise", - "ExtraPaths": ["ip-intelligence/ip-intelligence-data"], "DependsOn": [ "fiftyone_ipintelligence_shared", "fiftyone_pipeline_engines_fiftyone" @@ -153,7 +152,6 @@ "Name": "fiftyone_ipintelligence_translation", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_translation", - "ExtraPaths": ["ip-intelligence/ip-intelligence-data"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_translation" diff --git a/cxx b/cxx index 1bec7b57..e866f6d3 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit 1bec7b570575e2e180b52435599bef9b03e35ad8 +Subproject commit e866f6d3cea936587790f87967eff7863778b30e diff --git a/device-detection/ci/fetch-assets.ps1 b/device-detection/ci/fetch-assets.ps1 index c551f8f9..45cb2875 100644 --- a/device-detection/ci/fetch-assets.ps1 +++ b/device-detection/ci/fetch-assets.ps1 @@ -11,7 +11,8 @@ if ($env:GITHUB_JOB -eq "PreBuild") { exit 0 } -$deviceDetectionData = "$PSScriptRoot/../../cxx/device-detection-data" +$data = "$PSScriptRoot/../../cxx/data" +$deviceDetectionData = "$data/device-detection" $assets = '51Degrees-LiteV4.1.hash', '20000 Evidence Records.yml', '20000 User Agents.csv' if ($DeviceDetection) { @@ -29,3 +30,12 @@ foreach ($asset in $assets) { default {New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/$_" -Path "$deviceDetectionData/$_"} } } + +# Downloads any other file the data repository links to in the folder. The +# script keeps a file that is already there, so the assets linked in above +# are not downloaded again. +$global:LASTEXITCODE = 0 +& "$data/get-data-files.ps1" device-detection +if ($LASTEXITCODE -ne 0) { + throw "Not every data file in $deviceDetectionData was downloaded" +} diff --git a/device-detection/fiftyone_devicedetection/examples/gettingstarted.py b/device-detection/fiftyone_devicedetection/examples/gettingstarted.py index 994d736f..cf3a7b7f 100644 --- a/device-detection/fiftyone_devicedetection/examples/gettingstarted.py +++ b/device-detection/fiftyone_devicedetection/examples/gettingstarted.py @@ -24,7 +24,7 @@ # First create the device detection pipeline with the desired settings. -data_file = "../../cxx/device-detection-data/51Degrees-LiteV4.1.hash" +data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" resource_key = "!!YOUR_RESOURCE_KEY!!" # Uncommment the line below 'On-Premise' and comment the line below 'Cloud' to diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py index d36c35ea..05260f95 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py @@ -23,6 +23,8 @@ from datetime import datetime, timedelta from threading import Event, Thread import os +import shlex +import subprocess from pathlib import Path @@ -51,6 +53,15 @@ class ExampleUtils: # file named in its configuration. DATA_FILE_ENV_VAR = "51DEGREES_DD_PATH" + # The data repository's device detection folder, from the root of a + # checkout of this repository. Each file the data repository links to + # belongs in this folder, beside its .link file. + DATA_FOLDER = os.path.join("cxx", "data", "device-detection") + + # Sent with each download, so the downloads these examples make can be + # counted. + DOWNLOAD_SOURCE = "examples-51degrees-python" + @staticmethod def get_resource_key_from_config(config): key = "" @@ -183,12 +194,29 @@ def __get_env_variable(name): else: return "" - # Find the specified filename within the specified directory. - # If no directory is specified, the working directory is used. - # If the file cannot be found, the algorithm will move to the - # parent directory and repeat the process. + # Find the specified file. A path to a file that exists is returned as + # it is. Next, the data repository's folder is checked in the working + # directory and in each directory above it, and a file the data + # repository links to is downloaded there when it is missing. Otherwise + # the working directory is searched for a file with the same name, then + # each parent directory in turn. None is returned when nothing is found. @staticmethod def find_file(file_name): + path = Path(file_name).absolute() + if path.is_file(): + return str(path.resolve()) + + # The data repository's copy is preferred to a file with the same + # name elsewhere in the tree, which may be out of date or a Git LFS + # pointer. + current = Path(os.getcwd()).absolute() + for root in [current, *current.parents]: + candidate = root / ExampleUtils.DATA_FOLDER / path.name + if candidate.is_file(): + return str(candidate) + if candidate.with_name(candidate.name + ".link").is_file(): + return ExampleUtils.__download(str(candidate)) + stop_event = Event() result = [None] # Start the file system search as a separate task. @@ -235,6 +263,39 @@ def __find_file(file_name, stop_event, result, search_dir=None): if search_dir.parent != search_dir and not stop_event.is_set(): ExampleUtils.__find_file(file_name, stop_event, result, search_dir.parent) + # Download a file the data repository links to, by running the data + # repository's own script for that file, and return its path, or None + # when it could not be downloaded. The data repository is the folder + # above the one that holds the file's .link. + @staticmethod + def __download(path): + name = os.path.basename(path) + data = os.path.dirname(os.path.dirname(path)) + if os.name == "nt": + command = [ + "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", + "-File", os.path.join(data, "get-data-files.ps1"), name, + "-Source", ExampleUtils.DOWNLOAD_SOURCE] + text = subprocess.list2cmdline(command) + else: + command = [ + "sh", os.path.join(data, "get-data-files.sh"), + "-s", ExampleUtils.DOWNLOAD_SOURCE, name] + text = shlex.join(command) + + # Flushed first, because the script writes to the same output. + print(f"{name} is missing, so it is being downloaded by running\n" + f"{text}", flush=True) + try: + subprocess.run(command, check=False) + except OSError: + pass + if os.path.isfile(path): + return path + print(f"{name} could not be downloaded. Run this command to try " + f"again.\n{text}") + return None + @staticmethod def get_data_file_date(engine): date = engine.engine.getPublishedTime() @@ -272,9 +333,10 @@ def check_data_file(pipeline, logger): f"that is more than '{ExampleUtils.DATA_FILE_AGE_WARNING}' days " + "old. A more recent data file may be needed to " + "correctly detect the latest devices, browsers, " + - "etc. The latest lite data file is available from " + - "the device-detection-data repository on GitHub " + - "https://github.com/51Degrees/device-detection-data. " + + "etc. Get the latest lite data file by running " + + "cxx/data/get-data-files.sh -u 51Degrees-LiteV4.1.hash, " + + "or cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash " + + "-Update on Windows. " + "Find out about the Enterprise data file, which " + "includes automatic updates, on our pricing page: " + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=data-file-age-warning") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index bf9ff7a3..d5fd2228 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -53,9 +53,8 @@ # # ## Lite Data File # Lite data files (free-to-use, limited capabilities, no license key required) are created roughly -# once a month and cannot be updated using auto-update, they may be downloaded from -# [Github](https://github.com/51Degrees/device-detection-data) and are included with -# source distributions of this software. +# once a month and cannot be updated using auto-update, they may be downloaded with the +# scripts in the [data](https://github.com/51Degrees/data) repository on GitHub. # # # Update on Start-Up # You can configure the pipeline builder to download an Enterprise data file on start-up. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index 9c695f43..f49c5bc0 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -152,9 +152,9 @@ def main(argv): else: logger.log("error", "Failed to find a device detection " + - "data file. Make sure the device-detection-data " + - "submodule has been updated by running " + - "`git submodule update --recursive`.") + "data file. Make sure the data submodule is there, " + + "by running `git submodule update --init --recursive`, " + + "and run this example again to download the file.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index c2ec0e2b..dee9b99b 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -246,8 +246,9 @@ def build_config(): if foundDataFile == False: raise Exception("Failed to find a device detection data file matching " + f"'{dataFile}'. If using the lite file, then make sure the " + - "device-detection-data submodule has been updated by running " + - "`git submodule update --recursive`. Otherwise, ensure that the filename " + + "data submodule is there, by running " + + "`git submodule update --init --recursive`, and run this " + + "example again to download the file. Otherwise, ensure that the filename " + "is correct in config.json, or set the environment variable " + f"'{ExampleUtils.DATA_FILE_ENV_VAR}' to the path of the data file.") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html index 8106f4cc..d26ab8da 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html @@ -30,8 +30,8 @@
    WARNING: This example is using a data file that is more than {{ utils.DATA_FILE_AGE_WARNING }} days old. A more recent data file may be needed to correctly detect the latest devices, - browsers, etc. The latest lite data file is available from the - device-detection-data + browsers, etc. Get the latest lite data file with the scripts in the + data repository on GitHub. Find out about the Enterprise data file, which includes automatic daily updates, on our pricing page.
    diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py index f9f254f8..5daeb284 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py @@ -189,8 +189,9 @@ def main(argv): else: logger.log("error", "Failed to find a device detection data file. Make sure the " + - "device-detection-data submodule has been updated by running " + - "`git submodule update --recursive`.") + "data submodule is there, by running " + + "`git submodule update --init --recursive`, and run this " + + "example again to download the file.") if __name__ == "__main__": main(sys.argv[1:]) \ No newline at end of file diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py index 1e5e0929..ed974ae6 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py @@ -129,8 +129,9 @@ def main(argv): else: logger.log("error", "Failed to find a device detection data file. Make sure the " + - "device-detection-data submodule has been updated by running " + - "`git submodule update --recursive`.") + "data submodule is there, by running " + + "`git submodule update --init --recursive`, and run this " + + "example again to download the file.") if __name__ == "__main__": main(sys.argv[1:]) \ No newline at end of file diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py index 2487f3e9..80ca344f 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py @@ -211,8 +211,9 @@ def main(argv): else: logger.log("error", "Failed to find a device detection data file. Make sure the " + - "device-detection-data submodule has been updated by running " + - "`git submodule update --recursive`.") + "data submodule is there, by running " + + "`git submodule update --init --recursive`, and run this " + + "example again to download the file.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py index 665cdabf..271ce134 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py @@ -145,7 +145,7 @@ def run(data_file, skip = False): if __name__ == "__main__": ap = argparse.ArgumentParser(description='Run detection benchmark.') ap.add_argument('-d', '--data_file', default='', help='Path to data file') - ap.add_argument('-u', '--user_agents_file', default='../../cxx/device-detection-data/20000 User Agents.csv', help='Path to user agents evidence file') + ap.add_argument('-u', '--user_agents_file', default='../../cxx/data/device-detection/20000 User Agents.csv', help='Path to user agents evidence file') ap.add_argument('-j', '--json_output', default='', help='Output results in JSON format') args = ap.parse_args() if args.data_file == "": diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py index 5d808bf4..6cbbbfd7 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py @@ -184,8 +184,9 @@ def main(argv): ShareUsageExample().run(input) else: print("ERROR: Failed to find the evidence file. Make sure the " - "device-detection-data submodule has been updated by running " - "`git submodule update --recursive`.") + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file.") if __name__ == "__main__": diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py index 6d10a1a7..570ab57e 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py @@ -148,8 +148,9 @@ def build_config(): if foundDataFile == False: raise Exception("Failed to find a device detection data file matching " + f"'{dataFile}'. If using the lite file, then make sure the " + - "device-detection-data submodule has been updated by running " + - "`git submodule update --recursive`. Otherwise, ensure that the filename " + + "data submodule is there, by running " + + "`git submodule update --init --recursive`, and run this " + + "example again to download the file. Otherwise, ensure that the filename " + "is correct in config.json.") return config diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py index 3cd885ce..135cc69d 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py @@ -36,7 +36,7 @@ from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_engines.lru_cache import LRUEngineCache -data_file = "../../cxx/device-detection-data/51Degrees-LiteV4.1.hash" +data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" mobile_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" @@ -46,7 +46,7 @@ # User agents file -with open('../../cxx/device-detection-data/20000 User Agents.csv', newline='') as file: +with open('../../cxx/data/device-detection/20000 User Agents.csv', newline='') as file: reader = csv.reader(file) user_agents = list(reader) no_user_agents = len(user_agents) diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py index def20c1c..0d9b6435 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py @@ -28,7 +28,7 @@ from fiftyone_devicedetection_shared.utils import * from fiftyone_devicedetection_onpremise import constants -data_file = "../../cxx/device-detection-data/51Degrees-LiteV4.1.hash" +data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" mobile_ua = ("Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) " "AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile" "/11D167 Safari/9537.53") diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py index 04b4ee50..0b4bea5a 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py @@ -28,7 +28,7 @@ from fiftyone_devicedetection_onpremise.swig_data import SwigData from fiftyone_devicedetection_shared.utils import * -data_file = "../../cxx/device-detection-data/51Degrees-LiteV4.1.hash" +data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" # Create a simple pipeline to access the engine with and process it with flow data deviceDetectionOnPremiseEngine = DeviceDetectionOnPremise( diff --git a/device-detection/readme.md b/device-detection/readme.md index 379b1ed1..379be178 100644 --- a/device-detection/readme.md +++ b/device-detection/readme.md @@ -43,22 +43,25 @@ to use the Cloud API. You can create resource keys using our #### On-Premise In order to perform device detection on-premise, you will need to use a 51Degrees data file. -This repository includes a free, 'lite' file in the 'device-detection-data' sub-module that has a -significantly reduced set of properties. To obtain a file with a more complete set of device -properties see the [51Degrees website](https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=on-premise). If you want to use the lite -file, you will need to install [GitLFS](https://git-lfs.github.com/): +A free, 'lite' file is available, with a significantly reduced set of properties. To obtain a +file with a more complete set of device properties see the [51Degrees website](https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=on-premise). + +The lite file is not stored in a repository. The [data](https://github.com/51Degrees/data) +repository, which is the `cxx/data` submodule, links to it from +`cxx/data/device-detection`. Download it from the root of this repository by running: ``` -sudo apt-get install git-lfs -git lfs install +sh cxx/data/get-data-files.sh 51Degrees-LiteV4.1.hash ``` -Then, navigate to 'src/fiftyone_devicedetection_onpremise/cxx/device-detection-data' and execute: +or, in PowerShell: ``` -git lfs pull +./cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash ``` +The examples run this for themselves when the file is missing. + ## Folders * `fiftyone_devicedetection` - references both cloud and on-premise packages, contains generic Device-Detection Pipeline Builder. diff --git a/ip-intelligence/ci/fetch-assets.ps1 b/ip-intelligence/ci/fetch-assets.ps1 index 00e2bd91..a26332bc 100644 --- a/ip-intelligence/ci/fetch-assets.ps1 +++ b/ip-intelligence/ci/fetch-assets.ps1 @@ -29,6 +29,14 @@ foreach ($asset in $assets) { } } +# The ASN file is not an asset, so the data repository's script downloads it +# beside its link in cxx/data/ip-intelligence, where the tests look for it. +$global:LASTEXITCODE = 0 +& "$PSScriptRoot/../../cxx/data/get-data-files.ps1" 51Degrees-IPIV4AsnIpiV41.ipi +if ($LASTEXITCODE -ne 0) { + throw "The ASN data file was not downloaded" +} + # If the Enterprise file was downloaded, prefer it for tests by exposing # it through the environment variable used by the test suites. if ($IpIntelligenceUrl) { diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md index af0db8c7..d9542f6a 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md @@ -34,11 +34,12 @@ python -m fiftyone_ipintelligence_examples.cloud.gettingstarted_web ## On-premise examples Require a `.ipi` data file in the format the engine reads (see -[Data files](../readme.md#data-files)). The -`ip-intelligence-data` submodule includes a freely available network-only -file (`51Degrees-IPIV4AsnIpiV41.ipi`) that the examples fall back to; it -only contains the ASN/network properties, so location and registration -properties are reported as "not present in the data file". +[Data files](../readme.md#data-files)). The data repository, at +`cxx/data`, links to a freely available network-only file +(`51Degrees-IPIV4AsnIpiV41.ipi`) that the examples fall back to, and an +example downloads it when it is missing. It only contains the ASN/network +properties, so location and registration properties are reported as "not +present in the data file". For the full set of properties (location, registration, etc.), a more complete data file is required - to obtain one for testing please @@ -50,7 +51,7 @@ the first command line argument. - `onpremise/metadata_console.py` - list the properties available from the data file. - `onpremise/offlineprocessing.py` - batch process the evidence file - from the ip-intelligence-data repository. + from the data repository. - `onpremise/performance_console.py` - benchmark the number of IP lookups performed per second. - `onpremise/compare_console.py` - compare the location returned for each diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py index 56e3d783..81eb5829 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py @@ -22,6 +22,8 @@ from datetime import datetime, timedelta import os +import shlex +import subprocess from pathlib import Path @@ -43,6 +45,15 @@ class ExampleUtils: # to use when running examples. DATA_FILE_ENV_VAR = "IPINTELLIGENCEDATAFILE" + # The data repository's IP intelligence folder, from the root of a + # checkout of this repository. Each file the data repository links to + # belongs in this folder, beside its .link file. + DATA_FOLDER = os.path.join("cxx", "data", "ip-intelligence") + + # Sent with each download, so the downloads these examples make can be + # counted. + DOWNLOAD_SOURCE = "examples-51degrees-python" + @staticmethod def get_human_readable(ip_data, property): try: @@ -112,8 +123,9 @@ def find_file(file_name): """! Find the specified file. First checks the data file environment variable, then searches up the directory tree from the current - working directory looking in 'assets' and 'ip-intelligence-data' - folders. + working directory looking in 'assets' and the data repository's + folder. A file the data repository links to is downloaded there + when it is missing. """ env = os.environ.get(ExampleUtils.DATA_FILE_ENV_VAR) @@ -123,15 +135,54 @@ def find_file(file_name): current = Path(os.getcwd()).resolve() roots = [current, *current.parents] for root in roots[:8]: - for sub_dir in [ - ".", - "assets", - "ip-intelligence-data", - os.path.join("cxx", "ip-intelligence-data")]: + for sub_dir in [".", "assets", ExampleUtils.DATA_FOLDER]: candidate = os.path.join(str(root), sub_dir, file_name) if os.path.isfile(candidate): return candidate + for root in roots[:8]: + candidate = os.path.join( + str(root), ExampleUtils.DATA_FOLDER, file_name) + if os.path.isfile(candidate + ".link"): + return ExampleUtils.__download(candidate) + + return None + + @staticmethod + def __download(path): + + """! + Download a file the data repository links to, by running the data + repository's own script for that file, and return its path, or None + when it could not be downloaded. The data repository is the folder + above the one that holds the file's .link. + """ + + name = os.path.basename(path) + data = os.path.dirname(os.path.dirname(path)) + if os.name == "nt": + command = [ + "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", + "-File", os.path.join(data, "get-data-files.ps1"), name, + "-Source", ExampleUtils.DOWNLOAD_SOURCE] + text = subprocess.list2cmdline(command) + else: + command = [ + "sh", os.path.join(data, "get-data-files.sh"), + "-s", ExampleUtils.DOWNLOAD_SOURCE, name] + text = shlex.join(command) + + # Flushed first, because the script writes to the same output. + print(f"{name} is missing, so it is being downloaded by running\n" + f"{text}", flush=True) + try: + subprocess.run(command, check=False) + except OSError: + pass + if os.path.isfile(path): + return path + print(f"{name} could not be downloaded. Run this command to try " + f"again.\n{text}") return None @staticmethod @@ -167,9 +218,10 @@ def check_data_file(pipeline, logger): f"This data file is more than " f"{ExampleUtils.DATA_FILE_AGE_WARNING} days old. A more " f"recent data file may be needed to correctly resolve " - f"the latest IP addresses. The latest data file is " - f"available from the ip-intelligence-data repository on " - f"GitHub https://github.com/51Degrees/ip-intelligence-data.") + f"the latest IP addresses. Get the latest free data files " + f"by running cxx/data/get-data-files.sh -u ip-intelligence, " + f"or cxx/data/get-data-files.ps1 ip-intelligence -Update " + f"on Windows.") if product == "Lite": logger.log("warning", diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py index d0479c2e..062e24da 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py @@ -30,8 +30,8 @@ # an IP-to-location service compare to real world information, especially # when evaluating different solutions. # -# The input CSV (geoip_comparison_evidence.csv in the ip-intelligence-data -# repository) provides these fields: +# The input CSV (geoip_comparison_evidence.csv in the data repository's +# ip-intelligence folder) provides these fields: # DateTimeUtc, Latitude, Longitude, Ip, AddressFamily, Continent, Country # # For each record the engine is used to obtain the latitude and longitude @@ -56,7 +56,7 @@ from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME -# Name of the truth CSV file in the ip-intelligence-data repository. +# Name of the truth CSV file in the data repository. GEOIP_COMPARISON_EVIDENCE_FILE_NAME = "geoip_comparison_evidence.csv" # Mean radius of the earth in kilometres, used for the distance calculation. @@ -184,8 +184,8 @@ def main(argv): else: logger.log("error", "Failed to find a data file or truth file. Make sure the " - "ip-intelligence-data submodule has been updated by running " - "`git submodule update --recursive`. To obtain an enterprise " + "data submodule is there, by running " + "`git submodule update --init --recursive`. To obtain an enterprise " "data file for testing, please contact us: " "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-compare_console.py&utm_term=data-file-required") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py index 35266657..6d26ce73 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py @@ -150,8 +150,9 @@ def main(argv): else: logger.log("error", "Failed to find an IP intelligence data file. Make sure the " - "ip-intelligence-data submodule has been updated by running " - "`git submodule update --recursive`, or supply the path to a " + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file, or supply the path to a " "data file as the first command line argument.") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py index 0f211919..f5ad8cdc 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py @@ -32,10 +32,10 @@ # 2. How to pass input data (evidence) to the Pipeline # 3. How to retrieve the results # -# This example uses a local IP intelligence data file (.ipi). The -# ip-intelligence-data submodule provides a freely available data file; -# a more complete data file can be supplied as the first command line -# argument. +# This example uses a local IP intelligence data file (.ipi). The data +# repository links to a freely available data file, which the example +# downloads when it is missing. A more complete data file can be supplied +# as the first command line argument. # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -170,8 +170,9 @@ def main(argv): else: logger.log("error", "Failed to find an IP intelligence data file. Make sure the " - "ip-intelligence-data submodule has been updated by running " - "`git submodule update --recursive`, or supply the path to a " + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file, or supply the path to a " "data file as the first command line argument.") if __name__ == "__main__": diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py index 49bc8b72..debc4a1b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py @@ -133,9 +133,11 @@ def main(data_file = None): data_file = ExampleUtils.find_file(LITE_DATAFILE_NAME) if not data_file: raise Exception( - "Failed to find an IP intelligence data file. Download the " - "Lite file from https://51ddatafiles.blob.core.windows.net/" - "enterpriseipi/51Degrees-LiteIpiV41.ipi.gz") + "Failed to find an IP intelligence data file. Make sure the " + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file, or supply the path to a " + "data file as the first command line argument.") build_pipeline(data_file) app.run(debug=False) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py index e10df504..ec2be8a1 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py @@ -66,9 +66,8 @@ def main(argv): MetadataConsole().run(data_file, logger, print) else: logger.log("error", - "Failed to find an IP intelligence data file. Make sure the " - "ip-intelligence-data submodule has been updated by running " - "`git submodule update --recursive`. To obtain an enterprise " + "Failed to find an IP intelligence data file. Supply its path " + "as the first command line argument. To obtain an enterprise " "data file for testing, please contact us: " "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-metadata_console.py&utm_term=data-file-required") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py index 11805592..07839ab1 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py @@ -48,8 +48,8 @@ from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME -# Default name of the device detection data file (the lite file included in -# the device-detection-data repository). +# Default name of the device detection data file (the lite file the data +# repository links to). HASH_DATAFILE_NAME = "51Degrees-LiteV4.1.hash" # Combined evidence: a User-Agent for device detection and an IP address for @@ -199,14 +199,19 @@ def main(argv): else: if not device_file: logger.log("error", - "Failed to find a device detection data file. Make sure the " - "device-detection-data submodule has been updated, or supply " - "the path as the first command line argument.") + "Failed to find a device detection data file. Supply its " + "path as the first command line argument. Get the free Lite " + "file by running " + "cxx/data/get-data-files.sh 51Degrees-LiteV4.1.hash, or " + "cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash on " + "Windows, which saves it in cxx/data/device-detection.") if not ip_file: logger.log("error", "Failed to find an IP intelligence data file. Make sure the " - "ip-intelligence-data submodule has been updated, or supply " - "the path as the second command line argument.") + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file, or supply the path as " + "the second command line argument.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py index a32c11cb..9c7109cc 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py @@ -23,8 +23,8 @@ ## @example onpremise/offlineprocessing.py # Provides an example of processing a YAML file containing evidence for # IP intelligence. There are 20,000 IP addresses in the supplied file of -# evidence records ('evidence.yml' in the ip-intelligence-data -# repository). +# evidence records ('evidence.yml' in the data repository's +# ip-intelligence folder). # # We create a pipeline using the on-premise engine, deserialize the # YAML evidence records, pass each entry to the pipeline and write the @@ -120,8 +120,8 @@ def main(argv): else: logger.log("error", "Failed to find a data file or evidence file. Make sure the " - "ip-intelligence-data submodule has been updated by running " - "`git submodule update --recursive`.") + "data submodule is there, by running " + "`git submodule update --init --recursive`.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py index 7ca71715..310bd727 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py @@ -25,7 +25,7 @@ # # This example measures the performance of the on-premise IP intelligence # engine. It processes the IP addresses from the evidence file supplied in -# the ip-intelligence-data repository and reports the number of IP lookups +# the data repository and reports the number of IP lookups # performed per second. # # Requesting fewer properties reduces processing time. If you do not specify @@ -225,8 +225,8 @@ def main(argv): else: logger.log("error", "Failed to find a data file or evidence file. Make sure the " - "ip-intelligence-data submodule has been updated by running " - "`git submodule update --recursive`. To obtain an enterprise " + "data submodule is there, by running " + "`git submodule update --init --recursive`. To obtain an enterprise " "data file for testing, please contact us: " "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-performance_console.py&utm_term=data-file-required") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py index b0e359a3..a2d17c30 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py @@ -29,10 +29,10 @@ from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_FILE_NAME -# The tests run against the freely available data file committed to the -# ip-intelligence-data submodule, so they only rely on properties that the -# free data file provides (location properties such as Latitude, Longitude -# and Areas are not present in it). +# The tests run against the freely available ASN data file the data +# repository links to, so they only rely on properties that the free data +# file provides (location properties such as Latitude, Longitude and Areas +# are not present in it). data_file = ExampleUtils.find_file(ASN_DATAFILE_NAME) evidence_file = ExampleUtils.find_file(EVIDENCE_FILE_NAME) truth_file = ExampleUtils.find_file("geoip_comparison_evidence.csv") @@ -61,8 +61,8 @@ class ExampleTests(unittest.TestCase): @unittest.skipUnless( bool(data_file), "No IP intelligence data file available") def test_onpremise_datafile_loads(self): - # Verify the freely available data file committed to the - # ip-intelligence-data submodule loads and processes correctly. + # Verify the freely available ASN data file loads and processes + # correctly. from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder pipeline = IpIntelligencePipelineBuilder( data_file_path=data_file, diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md index b7eb9ac3..65b11d8e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md @@ -28,12 +28,17 @@ python setup.py build_clib build_ext ## Data files -The on-premise engine requires a `.ipi` data file. A Lite data file can be -downloaded from -[51Degrees](https://51ddatafiles.blob.core.windows.net/enterpriseipi/51Degrees-LiteIpiV41.ipi.gz), -see also the [ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data) -repository which is included as a nested submodule and contains evidence -files for testing. +The on-premise engine requires a `.ipi` data file. The free Lite data file, +`51Degrees-LiteV41.ipi`, is linked from the +[data](https://github.com/51Degrees/data) repository, which is included as +the nested submodule `cxx/data` and contains evidence files for testing. From +the root of a checkout of +[51degrees-python](https://github.com/51Degrees/51degrees-python), download it +by running `sh cxx/data/get-data-files.sh 51Degrees-LiteV41.ipi`, or +`./cxx/data/get-data-files.ps1 51Degrees-LiteV41.ipi` in PowerShell. Without a +checkout, download it from +https://distributor.51degrees.com/api/v2/free/ipi-lite-v41, which sends it +gzip compressed. ## Usage @@ -41,7 +46,7 @@ files for testing. from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise_pipelinebuilder import IpIntelligenceOnPremisePipelineBuilder pipeline = IpIntelligenceOnPremisePipelineBuilder( - data_file_path="51Degrees-LiteIpiV41.ipi", + data_file_path="51Degrees-LiteV41.ipi", licence_keys="", usage_sharing=False).build() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py index 6f9123f3..2c5180f8 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py @@ -42,11 +42,12 @@ def get_data_file_path(): os.path.dirname(__file__), "..", "..", "assets", "51Degrees-LiteIpiV41.ipi") - # The ASN data file committed to the repository's ip-intelligence-data - # submodule. Unlike the Lite and Enterprise files it needs no download - # or secret, so it is always present wherever the engine can be built. + # The free ASN data file, which ci/fetch-assets.ps1 downloads with the + # data repository's script, beside its link in cxx/data/ip-intelligence. + # Unlike the Enterprise file it needs no secret. @staticmethod - def get_committed_data_file_path(): + def get_asn_data_file_path(): return os.path.join( os.path.dirname(__file__), - "..", "..", "ip-intelligence-data", ASN_DATAFILE_NAME) + "..", "..", "..", "cxx", "data", "ip-intelligence", + ASN_DATAFILE_NAME) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py index 13f5d042..df9f835e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py @@ -171,42 +171,41 @@ def test_missing_licence_key(self): self.assertIn("licence key is required", str(context.exception)) -class CommittedDataFileTests(unittest.TestCase): +class AsnDataFileTests(unittest.TestCase): """! - The native engine accepts exactly one data file format version, so the - cxx (51degrees-cxx) pin and the ip-intelligence-data pin must agree. - They once did not: the engine moved to format 4.6 while the data pin - still held the 4.5 ASN file. These tests open the ASN file committed to - the data submodule, which needs no download or secret, so pins that - disagree fail here on any machine. - - They do not prove the engine reads the data files 51Degrees publishes. - That is IpIntelligenceOnPremiseTests above, against the Lite or - Enterprise file CI downloads; it skips when neither is present. + The native engine accepts exactly one data file format version. These + tests open the free ASN data file. The Distributor always serves the + latest one published, and it needs no secret, so an engine that cannot + read the format 51Degrees publishes fails here on any machine. + + IpIntelligenceOnPremiseTests above runs against the Lite or Enterprise + file CI downloads, and skips when neither is present. test_onpremise_datafile_loads in the examples package also opens the ASN file, but it skips silently when the file is not found and it runs after the examples build, so it is not relied on here. """ - def test_committed_data_file_present(self): + def test_asn_data_file_present(self): """! - The committed data file exists. If it does not, the - ip-intelligence-data submodule was not checked out, and the test + The ASN data file has been downloaded. If it has not, the test below would prove nothing. """ self.assertTrue( - os.path.isfile(TestConstants.get_committed_data_file_path()), - "Run 'git submodule update --init --recursive'.") + os.path.isfile(TestConstants.get_asn_data_file_path()), + "Download it from the repository root by running " + "'sh cxx/data/get-data-files.sh 51Degrees-IPIV4AsnIpiV41.ipi', " + "or './cxx/data/get-data-files.ps1 51Degrees-IPIV4AsnIpiV41.ipi' " + "in PowerShell.") - def test_committed_data_file_opens(self): + def test_asn_data_file_opens(self): """! - The engine opens the committed data file, so it reads that file's - format version, and returns its properties for an IPv4 address. + The engine opens the ASN data file, so it reads that file's format + version, and returns its properties for an IPv4 address. """ engine = IpIntelligenceOnPremise( - data_file_path=TestConstants.get_committed_data_file_path(), + data_file_path=TestConstants.get_asn_data_file_path(), licence_keys="") pipeline = PipelineBuilder().add(engine).build() diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/example_constants.py b/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/example_constants.py index e11dd421..35945298 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/example_constants.py +++ b/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/example_constants.py @@ -38,12 +38,13 @@ # Note: the .NET examples' Constants.cs still uses the legacy name # "51Degrees-EnterpriseIpiV41.ipi", but the published enterprise/ASN data # files use the "IPIV4" infix (cf. 51Degrees-IPIV4AsnIpiV41.ipi in the -# ip-intelligence-data repo), so the current name is used here. +# data repository), so the current name is used here. ENTERPRISE_DATAFILE_NAME = '51Degrees-IPIV4EnterpriseIpiV41.ipi' -LITE_DATAFILE_NAME = '51Degrees-LiteIpiV41.ipi' -# The ASN data file is the freely available data file committed -# to the ip-intelligence-data submodule. It only contains the network -# (ASN) properties, but is useful for testing the on-premise engine against -# the current data format without an enterprise data file. +# The free Lite data file, named as the data repository links to it. +LITE_DATAFILE_NAME = '51Degrees-LiteV41.ipi' +# The ASN data file is a freely available data file the data repository +# links to. It only contains the network (ASN) properties, but is useful for +# testing the on-premise engine against the current data format without an +# enterprise data file. ASN_DATAFILE_NAME = '51Degrees-IPIV4AsnIpiV41.ipi' EVIDENCE_FILE_NAME = 'evidence.yml' diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/.gitignore b/ip-intelligence/fiftyone_ipintelligence_translation/.gitignore index 36ed5ddc..c39739ff 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/.gitignore +++ b/ip-intelligence/fiftyone_ipintelligence_translation/.gitignore @@ -1,3 +1,3 @@ -# Translation YAML staged from the ip-intelligence-data submodule at build -# time (see setup.py stage_translations). Not committed; generated on build. +# Translation YAML staged from cxx/data/ip-intelligence at build time (see +# setup.py stage_translations). Not committed; generated on build. src/fiftyone_ipintelligence_translation/translations/ diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/setup.py b/ip-intelligence/fiftyone_ipintelligence_translation/setup.py index a29e4097..3f06c4a4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_translation/setup.py @@ -40,14 +40,17 @@ def read(file_name): def stage_translations(): - """Copy the translation YAML files shipped in the ip-intelligence-data - submodule into the package so they are included in the built distribution. - This mirrors how the .NET project embeds the same files as resources. If the - submodule is not present the package falls back to reading it at run time, - so any failure here is ignored.""" + """Copy the translation YAML files from the data repository, the + cxx/data submodule, into the package so they are included in the built + distribution. This mirrors how the .NET project embeds the same files as + resources. A build from a source distribution has no cxx folder, but its + package already holds the files, so a missing folder is skipped. Any + failure here is ignored, and a package run from a source checkout reads + the folder at run time instead.""" here = os.path.dirname(os.path.abspath(__file__)) - source = os.path.join(here, "..", "ip-intelligence-data", "Translations") + source = os.path.join( + here, "..", "..", "cxx", "data", "ip-intelligence", "Translations") dest = os.path.join( here, "src", "fiftyone_ipintelligence_translation", "translations") if not os.path.isdir(source): diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/src/fiftyone_ipintelligence_translation/resources.py b/ip-intelligence/fiftyone_ipintelligence_translation/src/fiftyone_ipintelligence_translation/resources.py index eb9aff55..4c0ffc4c 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/src/fiftyone_ipintelligence_translation/resources.py +++ b/ip-intelligence/fiftyone_ipintelligence_translation/src/fiftyone_ipintelligence_translation/resources.py @@ -30,14 +30,15 @@ def _find_submodule_translations(): - """Walk up from this package looking for the ip-intelligence-data - submodule's Translations directory, used when the package is run from a - source checkout rather than an installed distribution.""" + """Walk up from this package looking for the data repository's + Translations directory, cxx/data/ip-intelligence/Translations, used when + the package is run from a source checkout rather than an installed + distribution.""" current = _PACKAGE_DIR for _ in range(8): candidate = os.path.join( - current, "ip-intelligence-data", "Translations") + current, "cxx", "data", "ip-intelligence", "Translations") if os.path.isdir(candidate): return candidate parent = os.path.dirname(current) @@ -50,9 +51,9 @@ def _find_submodule_translations(): def _translation_files(): """Return a dict of translation file name -> absolute path. The packaged - translations directory is preferred; if it is empty the ip-intelligence-data - submodule is used as a fallback (countrycodes at the root, countries under - OSM).""" + translations directory is preferred; if it is empty the data repository's + Translations directory is used as a fallback (countrycodes at the root, + countries under OSM).""" files = {} diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_translation.py b/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_translation.py index 4decac97..199ca3b6 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_translation.py +++ b/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_translation.py @@ -20,6 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +import os import unittest import yaml @@ -34,6 +35,7 @@ from fiftyone_ipintelligence_translation.countries_translation_engine_builder import ( CountriesTranslationEngineBuilder) from fiftyone_ipintelligence_translation.comparer import make_comparer +from fiftyone_ipintelligence_translation import resources from fiftyone_ipintelligence_translation.resources import ( get_all_countries, get_country_resources) @@ -448,5 +450,18 @@ def test_population_translated_names(self): self.assertNotEqual(geo_codes[0], pop_codes[0]) +class PackagedTranslationsTests(unittest.TestCase): + + def test_translations_are_packaged(self): + # The tests run against the built package. A build that cannot find + # the data repository's Translations folder still succeeds, without + # the YAML files, and the other tests can still pass by reading the + # folder from the checkout, so this checks the package itself. + folder = resources._PACKAGED_TRANSLATIONS + names = os.listdir(folder) if os.path.isdir(folder) else [] + self.assertIn("countrycodes.en_GB.yml", names) + self.assertIn("countries.de_DE.yml", names) + + if __name__ == "__main__": unittest.main() diff --git a/ip-intelligence/ip-intelligence-data b/ip-intelligence/ip-intelligence-data deleted file mode 160000 index b8b3d7d4..00000000 --- a/ip-intelligence/ip-intelligence-data +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b8b3d7d4e7747a0412281b9b48e0144b2ef13fcc diff --git a/ip-intelligence/readme.md b/ip-intelligence/readme.md index 68e79e2e..18d82f7e 100644 --- a/ip-intelligence/readme.md +++ b/ip-intelligence/readme.md @@ -28,10 +28,9 @@ page shows the Python versions that we currently test against. The on-premise engine wraps the native [51degrees-cxx](https://github.com/51Degrees/51degrees-cxx/blob/main/docs/ipi/README.md) -library, and the -[ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data) -repository (data and evidence files used for testing) is also included, both -as git submodules. When cloning, use: +library, which is included as the `cxx` git submodule. It has the +[data](https://github.com/51Degrees/data) repository (data and evidence files +used for testing) as its own submodule, at `cxx/data`. When cloning, use: ``` git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git @@ -45,13 +44,25 @@ git submodule update --init --recursive ### Data files -The on-premise engine requires a `.ipi` data file. The -[ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data) -submodule provides a freely available network-only data file +The on-premise engine requires a `.ipi` data file. The data repository's +`cxx/data/ip-intelligence` folder holds the evidence files used for testing, +and links to a freely available network-only data file (`51Degrees-IPIV4AsnIpiV41.ipi`), which the examples tests and -`CommittedDataFileTests` use, along with the evidence files used for -testing. The on-premise unit tests default to -`assets/51Degrees-LiteIpiV41.ipi`. +`AsnDataFileTests` use. The linked files are not stored in a repository. +Download them from the root of this repository by running: + +``` +sh cxx/data/get-data-files.sh ip-intelligence +``` + +or, in PowerShell: + +``` +./cxx/data/get-data-files.ps1 ip-intelligence +``` + +An example whose file is missing runs the script for that file itself. The +on-premise unit tests default to `assets/51Degrees-LiteIpiV41.ipi`. For the full set of properties (location, registration, etc.) an enterprise data file is required. To obtain one for testing, please @@ -59,12 +70,12 @@ data file is required. To obtain one for testing, please line argument to the examples, or place it within the project space. Note: the engine reads exactly one data file format version, set by -`FIFTYONE_DEGREES_IPI_TARGET_VERSION_MAJOR` and `_MINOR` in the -`ip-intelligence-cxx` submodule's `src/ipi.c`, and refuses any other as -"an unsupported version". Move the `ip-intelligence-cxx` and -`ip-intelligence-data` pins together. `CommittedDataFileTests` in +`FIFTYONE_DEGREES_IPI_TARGET_VERSION_MAJOR` and `_MINOR` in the `cxx` +submodule's `src/ipi/ipi.c`, and refuses any other as "an unsupported +version". The ASN data file is downloaded from the Distributor, which always +serves the latest one published, so `AsnDataFileTests` in [test_ipintelligence.py](fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py) -fails when the two disagree. +fails when the engine cannot read the format 51Degrees publishes. ## Folders @@ -77,8 +88,6 @@ fails when the two disagree. wraps ip-intelligence-cxx via SWIG. - `fiftyone_ipintelligence_shared` - code shared between the engines. - `fiftyone_ipintelligence_examples` - usage examples for both engines. -- `ip-intelligence-data` - submodule containing the test data file and - evidence files used by the examples and tests. ## Installation From 9c11656ae10c68ee96fdeaf169942e4a46f30cad Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 17:25:12 +0000 Subject: [PATCH 06/68] Say in AGENTS.md how to get the data, build and run the examples (#34) --- AGENTS.md | 258 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 257 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 11855b13..deb8846b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,256 @@ # Working in this repository -Notes on what is not obvious from the code. Keep them short and current. The +If you have been asked to get an example running, follow +[Run the examples](#run-the-examples). It gives every step in order, from a +fresh clone to the output. The sections after it are notes on what is not +obvious from the code, for changing it. Keep them short and current. The layout and how to get the code are in the [README](README.md). +## Run the examples + +Device Detection's on-premise `gettingstarted_console` is the quickest +example to show, because its data file is about 35 MB. IP Intelligence's +on-premise `gettingstarted_console` runs on the free Asn data file, which is +about 6 MB, and shows registration values once it is given the Lite file, +which is a download of about 230 MB. Run each command from the repository root +unless the step says otherwise. + +The on-premise engines are C and C++ code that is compiled when the package is +installed. PyPI has only source distributions of +`fiftyone-devicedetection-onpremise` and `fiftyone-ipintelligence-onpremise`, +so a compiler is needed whether the packages come from PyPI or from this +checkout. The steps below build them from this checkout, as the product +READMEs do. + +### Prerequisites + +- Git. +- Python 3.10 or later, as the README says. CI tests 3.10 to 3.14 on Ubuntu, + Windows and macOS (`ci/options.json`). +- A C11 and C++17 compiler. On Windows that is Visual Studio 2019 or later, or + the [C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/), + with Platform Toolset `v142` and Windows SDK `10.0.18362.0` or later. On + Linux the Device Detection README gives + `sudo apt-get install g++ make libatomic1`. +- For the data files, `curl`, `gzip` and `md5sum` (`md5` on macOS) on Linux + and macOS, where the examples download with the shell script, and Windows + PowerShell 5.1 or PowerShell 7 on Windows, where they use the PowerShell + script. +- Network access to GitHub, to PyPI for the packages the examples depend on, + and to `distributor.51degrees.com` for the data files. + +Clone the repository with its submodules. + +``` +git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git +cd 51degrees-python +``` + +In a clone made without `--recurse-submodules`, fetch the submodules. + +``` +git submodule update --init --recursive +``` + +### Data files + +The examples use the free data files, which need no licence key. They are not +stored in this repository. The `cxx` submodule holds the +[data](https://github.com/51Degrees/data) repository as `cxx/data`, which has a +link to each file, and a script downloads them into `cxx/data/device-detection` +and `cxx/data/ip-intelligence`. + +On Linux and macOS, run the shell script. + +``` +sh cxx/data/get-data-files.sh +``` + +On Windows, run the PowerShell script. Windows PowerShell can refuse to run a +script file under the policy Windows sets by default, so run it the way the +examples do. + +``` +powershell -NoProfile -ExecutionPolicy Bypass -File cxx/data/get-data-files.ps1 +``` + +Where PowerShell 7 is installed, `pwsh cxx/data/get-data-files.ps1` does the +same on any system. Either script downloads every free file, the Asn file +included, and keeps a file that is already there. To download one file, give +its name, as in `sh cxx/data/get-data-files.sh 51Degrees-LiteV41.ipi`. An +example run with no argument downloads the file it needs itself, when the file +is missing. + +#### Check the files + +- `cxx/data/device-detection/51Degrees-LiteV4.1.hash` is more than 10 MB. +- `cxx/data/ip-intelligence/51Degrees-IPIV4AsnIpiV41.ipi` is more than 1 MB. +- `cxx/data/ip-intelligence/51Degrees-LiteV41.ipi` is more than 100 MB. + +### Build + +Install into a virtual environment, as CI does for the examples +(`device-detection/ci/run-integration-tests.ps1`). On Linux and macOS, make and +activate one with these commands. + +``` +python3 -m venv .venv +. .venv/bin/activate +``` + +On Windows, in PowerShell, run `python -m venv .venv` and then +`.venv\Scripts\Activate.ps1`. + +Then install the Device Detection packages from source, with the commands the +Device Detection README gives, and the examples package. `setuptools` is +added because `python setup.py` needs it and Python 3.12 and later no longer +install it (`ip-intelligence/ci/setup-environment.ps1`). + +``` +cd device-detection +python -m pip install -r requirements.txt setuptools +python -m pip install -e fiftyone_devicedetection_shared/ +python -m pip install -e fiftyone_devicedetection_cloud/ +cd fiftyone_devicedetection_onpremise/ +python setup.py build_clib build_ext +cd ../ +python -m pip install -e fiftyone_devicedetection_onpremise/ +python -m pip install -e fiftyone_devicedetection/ +python -m pip install -e fiftyone_devicedetection_examples/ +cd .. +``` + +Then install the IP Intelligence packages, with the commands the IP +Intelligence README gives. + +``` +cd ip-intelligence +pip install -e fiftyone_ipintelligence_shared +pip install -e fiftyone_ipintelligence_cloud +pip install -e fiftyone_ipintelligence_onpremise +pip install -e fiftyone_ipintelligence +pip install -e fiftyone_ipintelligence_examples +cd .. +``` + +The packages these need from elsewhere, such as the Pipeline API packages, +come from PyPI. CI installs the ones in `pipeline/` over them +(`ci/setup-environment.ps1`), which running an example does not need. + +### Run + +Run the Device Detection example from the repository root with no argument. +It looks for `cxx/data/device-detection/51Degrees-LiteV4.1.hash` in the folder +it runs in and each folder above it, and runs the download script when the +file is missing, with Windows PowerShell on Windows and `sh` on Linux and +macOS (`find_file` in +`device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py`). + +``` +python -m fiftyone_devicedetection_examples.onpremise.gettingstarted_console +``` + +The IP Intelligence example with no argument looks for the Asn file in the +folder it runs in and in its `assets` and `cxx/data/ip-intelligence` folders, +then does the same in each of the seven folders above it, and downloads the +file into `cxx/data/ip-intelligence` when it is missing. To see registration +values, download the Lite file first, as "Data files" says, and give its path +as the first argument. + +``` +python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console +python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console cxx/data/ip-intelligence/51Degrees-LiteV41.ipi +``` + +Either example uses a path given as the first argument as it is, so a file +named that way is never downloaded. + +### Correct output + +When an example downloads its data file, it first prints +` is missing, so it is being downloaded by running` and, on the next +line, the command, then the script's `Downloading` and `Saved` lines. + +The pipeline logger writes through Python's `logging`, so its lines go to +standard error, each as `INFO:root:` or `WARNING:root:` followed by a +dictionary that holds the time, the level and the message. + +The Device Detection example logs `Using a 'Lite' data file created '' +from location ''`, a warning that it is using the Lite file, and possibly +a warning that the file is more than 30 days old. Both warnings are expected. +Then for each of five sets of evidence it prints `Input values:` and the +evidence, then `Results:` with `Mobile Device`, `Platform Name`, +`Platform Version`, `Browser Name` and `Browser Version`. The first set is a +User-Agent from a Samsung Android phone. + +The IP Intelligence example logs `Using a '' data file created +''`, then for each of four IP addresses, starting with `116.154.188.222`, +prints `Input values:` with `query.client-ip`, then `Results:` with a line for +each of 16 properties, starting with `RegisteredName`, `RegisteredOwner` and +`RegisteredCountry`. A property the data file lacks reads +`This property is not present in the data file.` With the Asn file that is +every registration and location property, as the examples' README says. The +Lite file has RegisteredName, RegisteredOwner and RegisteredCountry, so with it +those three lines are read from the file. + +The paid data files have many more properties, as the +[pricing page](https://51degrees.com/pricing?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=correct-output) +describes. + +### Cloud examples + +The cloud examples need a resource key, which a person creates in the +configurator, so ask for one if none is set. A key with the properties the +Device Detection cloud example uses can be made +[here](https://configure.51degrees.com/1QWJwHxl?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples), +which is the configuration the example's own message names, and a key for IP +Intelligence in the +[configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples). + +The Device Detection cloud example takes the key as its first argument, or +from `_51DEGREES_RESOURCE_KEY`, or from the older name `resource_key`, which +the Device Detection README uses. The IP Intelligence cloud example takes it +as its first argument or from `resource_key`. + +``` +resource_key= python -m fiftyone_devicedetection_examples.cloud.gettingstarted_console +resource_key= python -m fiftyone_ipintelligence_examples.cloud.gettingstarted_console +``` + +In PowerShell, set `$env:resource_key = ''` first. Without a key, each +example logs an error that names the variable and the configurator. + +### Pitfalls + +- **A data file that is not there.** `cxx/data` holds a `.link` for each file + rather than the file. An example run with no argument downloads a missing + file, so it needs the network and the tools under "Prerequisites". Check + the file sizes as above. +- **Running outside the repository, or without the submodules.** The Device + Detection example then finds no `cxx/data`, searches for up to 5 seconds + and, when it finds nothing, logs `Failed to find a device detection data + file` with the advice to run `git submodule update --init --recursive`. Run + it from the repository root, in a clone made with Git, because a zip + download of the repository has no submodules. +- **No compiler, or no `setuptools`.** `python setup.py build_clib build_ext` + and the on-premise installs compile C and C++, and on Python 3.12 and later + `python setup.py` stops with `No module named 'setuptools'` until it is + installed. +- **Long paths on Windows.** CI sets `TEMP` to `C:\tmp` on Windows to keep + within the MSVC path length limit + (`device-detection/ci/run-integration-tests.ps1`), so build from a short + folder path. +- **Scripts blocked in Windows PowerShell.** Running `Activate.ps1`, like the + Device Detection README's `setup.ps1`, needs PowerShell to allow scripts, + which that README does with `Set-ExecutionPolicy -Scope CurrentUser Bypass`. +- **Pipenv.** The Device Detection README suggests `pipenv install`, but there + is no `Pipfile` in the repository. Use the virtual environment above. +- **What CI does not test.** CI runs the examples' tests, which call each + example's code rather than its `main`. The Device Detection tests use the + Lite file that `device-detection/ci/fetch-assets.ps1` links in from CI's + assets, and the IP Intelligence example tests use the Asn file and throw + their output away. + ## Products keep their own CI scripts Each product folder still has the `ci/` scripts it had as its own repository, @@ -54,6 +302,14 @@ A package's `ExtraPaths` in `ci/packages.json` cannot be a folder inside cxx update therefore releases no package, even when it brings new translations in `cxx/data/ip-intelligence`. +## Releases + +Pull requests go to `rc`, not `main`, and stay drafts until they have been +reviewed, because the nightly merges any pull request into `rc` that is not a +draft and has no review requested. When a release is wanted, one pull request +merges `rc` into `main`, and the "Only rc into main" check fails a pull request +into `main` from any other branch. + ## History The four repositories were brought in with `git subtree add`, so commit SHAs From 7364fe9a84dec2ec6ff31205e034117ee1fb0c23 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 17:25:36 +0000 Subject: [PATCH 07/68] Point package metadata and package READMEs at the right pages (#35) --- README.md | 2 +- device-detection/ci/README.md | 2 +- .../fiftyone_devicedetection/readme.md | 12 ++++++++++-- .../fiftyone_devicedetection/setup.py | 6 +++++- .../fiftyone_devicedetection_cloud/readme.md | 14 ++++++++++---- .../fiftyone_devicedetection_cloud/setup.py | 6 +++++- .../fiftyone_devicedetection_examples/setup.py | 6 +++++- .../cloud/configurator_console.py | 4 ++-- .../cloud/gettingstarted_console.py | 4 ++-- .../cloud/gettingstarted_web/app.py | 4 ++-- .../cloud/metadata_console.py | 2 +- .../cloud/nativemodellookup_console.py | 4 ++-- .../cloud/taclookup_console.py | 4 ++-- .../cloud/useragentclienthints_web/app.py | 2 +- .../onpremise/gettingstarted_console.py | 6 +++--- .../onpremise/gettingstarted_web/app.py | 2 +- .../onpremise/metadata_console.py | 6 +++--- .../onpremise/offlineprocessing.py | 8 ++++---- .../readme.md | 15 ++++++++++++--- .../fiftyone_devicedetection_onpremise/setup.py | 5 +++++ .../fiftyone_devicedetection_shared/readme.md | 12 ++++++++++-- .../fiftyone_devicedetection_shared/setup.py | 6 +++++- device-detection/readme.md | 6 +++--- ip-intelligence/ci/README.md | 2 +- .../fiftyone_ipintelligence/readme.md | 14 +++++++++++--- .../fiftyone_ipintelligence/setup.py | 6 +++++- .../fiftyone_ipintelligence_cloud/readme.md | 16 ++++++++++++---- .../fiftyone_ipintelligence_cloud/setup.py | 6 +++++- .../fiftyone_ipintelligence_examples/readme.md | 2 +- .../fiftyone_ipintelligence_examples/setup.py | 6 +++++- .../cloud/getallproperties_console.py | 2 +- .../cloud/gettingstarted_console.py | 4 ++-- .../cloud/metadata_console.py | 2 +- .../onpremise/gettingstarted_console.py | 6 +++--- .../fiftyone_ipintelligence_onpremise/readme.md | 15 ++++++++++++--- .../fiftyone_ipintelligence_onpremise/setup.py | 5 +++++ .../fiftyone_ipintelligence_shared/readme.md | 14 +++++++++++--- .../fiftyone_ipintelligence_shared/setup.py | 6 +++++- .../readme.md | 12 ++++++++++-- .../setup.py | 6 +++++- ip-intelligence/readme.md | 4 ++-- location/ci/README.md | 2 +- location/readme.md | 17 ++++++++++++----- location/setup.py | 6 +++++- pipeline/ci/README.md | 2 +- .../readme.md | 14 +++++++++++--- .../setup.py | 6 +++++- .../clouddata.py | 2 +- .../cloudengine.py | 4 ++-- pipeline/fiftyone_pipeline_core/readme.md | 13 ++++++++++--- pipeline/fiftyone_pipeline_core/setup.py | 6 +++++- pipeline/fiftyone_pipeline_did/readme.md | 7 +++++++ pipeline/fiftyone_pipeline_did/setup.py | 6 +++++- pipeline/fiftyone_pipeline_engines/readme.md | 10 ++++++++-- pipeline/fiftyone_pipeline_engines/setup.py | 6 +++++- .../readme.md | 11 +++++++++-- .../fiftyone_pipeline_engines_fiftyone/setup.py | 6 +++++- .../fiftyone_pipeline_translation/readme.md | 11 +++++++++-- pipeline/fiftyone_pipeline_translation/setup.py | 6 +++++- pipeline/readme.md | 4 ++-- 60 files changed, 300 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index 2303b4fe..04fb4d05 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ the on-premise engines and the data files. ## Supported Python versions The packages support Python 3.10 and later. The -[tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=tested-versions) +[tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=tested-versions) page lists the versions each release is tested against. ## Versions diff --git a/device-detection/ci/README.md b/device-detection/ci/README.md index 0db0f74d..76a328d6 100644 --- a/device-detection/ci/README.md +++ b/device-detection/ci/README.md @@ -4,7 +4,7 @@ This API complies with the `common-ci` approach. The following secrets are required: * `ACCESS_TOKEN` - GitHub [access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#about-personal-access-tokens) for cloning repos, creating PRs, etc. * Example: `github_pat_l0ng_r4nd0m_s7r1ng` -* `SUPER_RESOURCE_KEY` - [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for integration tests +* `SUPER_RESOURCE_KEY` - [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for integration tests * Example: `R4nd0m-S7r1ng` * `DEVICE_DETECTION_KEY` - [license key](https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for downloading assets (TAC hashes file and TAC CSV data file) * Example: `V3RYL0NGR4ND0M57R1NG` diff --git a/device-detection/fiftyone_devicedetection/readme.md b/device-detection/fiftyone_devicedetection/readme.md index 7cd26f83..72ffdfb4 100644 --- a/device-detection/fiftyone_devicedetection/readme.md +++ b/device-detection/fiftyone_devicedetection/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - Device Detection -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-readme.md&utm_term=51degrees-device-detection-engines-device-detection "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=51degrees-device-detection-engines-device-detection "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-readme.md&utm_term=51degrees-device-detection-engines-device-detection "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-readme.md&utm_term=51degrees-device-detection-engines-device-detection "View all available properties and values") +[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=51degrees-device-detection-engines-device-detection "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=51degrees-device-detection-engines-device-detection "View all available properties and values") ## Introduction @@ -25,3 +25,11 @@ The Pipeline is a generic web request intelligence and data processing solution To run the tests use: `python -m unittest discover -s tests -p test*.py -b` + +## Find out more + +- [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-device-detection) +- [Device Detection documentation](https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/device-detection/fiftyone_devicedetection/setup.py b/device-detection/fiftyone_devicedetection/setup.py index 99b1c73d..e3bd7af0 100644 --- a/device-detection/fiftyone_devicedetection/setup.py +++ b/device-detection/fiftyone_devicedetection/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("Device Detection and User-Agent parser for Python that detects the device, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints, on-premise with a data file or in the cloud. An alternative to UAParser.js, DeviceAtlas, and WURFL."), keywords=[ "51degrees", diff --git a/device-detection/fiftyone_devicedetection_cloud/readme.md b/device-detection/fiftyone_devicedetection_cloud/readme.md index dead810b..10fe2fe4 100644 --- a/device-detection/fiftyone_devicedetection_cloud/readme.md +++ b/device-detection/fiftyone_devicedetection_cloud/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - Cloud -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-readme.md&utm_term=51degrees-device-detection-engines-cloud "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=51degrees-device-detection-engines-cloud "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-readme.md&utm_term=51degrees-device-detection-engines-cloud "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-readme.md&utm_term=51degrees-device-detection-engines-cloud "View all available properties and values") +[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=51degrees-device-detection-engines-cloud "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=51degrees-device-detection-engines-cloud "View all available properties and values") ## Introduction @@ -12,7 +12,7 @@ You can confirm this is working with the following micro-example. -* Create a resource key for free with the 51Degrees [configurator](https://configure.51degrees.com/np5M4nlF?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-readme.md&utm_term=from-pypi). This defines the properties you want to access. +* Create a resource key for free with the 51Degrees [configurator](https://configure.51degrees.com/np5M4nlF?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=from-pypi). This defines the properties you want to access. * On the 'implement' page of the configurator, copy the resource key and replace YOUR_RESOURCE_KEY in the example below. Save this as exampledd.py * Run the example with `python exampledd.py` * Feel free to try different user-agents and property values. @@ -26,7 +26,7 @@ fd.process() print(fd.device.ismobile.value()) ``` -For more in-depth examples, check out the [examples](https://51degrees.com/device-detection-python/examples.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-readme.md&utm_term=from-pypi) page in the documentation. +For more in-depth examples, check out the [examples](https://51degrees.com/device-detection-python/examples.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=from-pypi) page in the documentation. ### From GitHub @@ -40,4 +40,10 @@ To run the tests use: `python -m unittest discover -s tests -p test*.py -b` +## Find out more +- [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-device-detection) +- [Device Detection documentation](https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/device-detection/fiftyone_devicedetection_cloud/setup.py b/device-detection/fiftyone_devicedetection_cloud/setup.py index 0d721623..7318548d 100644 --- a/device-detection/fiftyone_devicedetection_cloud/setup.py +++ b/device-detection/fiftyone_devicedetection_cloud/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("Device Detection and User-Agent parser for Python that detects the device, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints using the 51Degrees cloud service. An alternative to UAParser.js, DeviceAtlas, and WURFL."), keywords=[ "51degrees", diff --git a/device-detection/fiftyone_devicedetection_examples/setup.py b/device-detection/fiftyone_devicedetection_examples/setup.py index 7f6dd0e9..15fa344c 100644 --- a/device-detection/fiftyone_devicedetection_examples/setup.py +++ b/device-detection/fiftyone_devicedetection_examples/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("The 51Degrees Pipeline API is a generic web request intelligence and data processing solution with the ability to add a range of 51Degrees and/or custom plug ins (Engines). " "This project contains examples of using 51Degrees Device Detection engines with the Pipeline API"), long_description=read("readme.md"), diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py index b985afe9..9e6d0b5f 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py @@ -31,7 +31,7 @@ # It shows how to call the cloud with the newly created key and how to access the values # of the selected properties. # -# See [Getting Started](https://51degrees.com/documentation/_examples__device_detection__getting_started__console__cloud.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=top) +# See [Getting Started](https://51degrees.com/documentation/_device_detection__examples__getting_started__console__cloud.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=top) # for a fuller example. # # Required PyPi Dependencies: @@ -96,7 +96,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees " + "cloud service is accessed using a 'ResourceKey'. " + "For more detail see " + - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + "https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py index caa7c7b8..3a92915b 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py @@ -50,7 +50,7 @@ def run(self, config, logger, output): # In this example, we use the PipelineBuilder and configure it from a file. # For more information about builders in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=run # Create the pipeline using the service provider and the configured options. pipeline = PipelineBuilder().add_logger(logger).build_from_configuration(config) @@ -174,7 +174,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " + "service is accessed using a 'ResourceKey'. For more information " + "see " + - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + "A resource key with the properties required by this example can be " + "created for free at https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + "Once complete, populate the config file or environment variable " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py index ba86890b..8260065b 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py @@ -58,7 +58,7 @@ # ``` # # Results can also be accessed in client-side code by using the `fod` object. See the -# [JavaScriptBuilderElementBuilder](https://51degrees.com/pipeline-python/4.3/classpipeline-python_1_1fiftyone__pipeline__core_1_1fiftyone__pipeline__core_1_1javascriptbuilde778a9036818b19ab55d981a40be4a4d7.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=top) +# [JavascriptBuilderElement](https://51degrees.com/documentation/apis/pipeline-python/classfiftyone__pipeline__core_1_1javascriptbuilder_1_1_javascript_builder_element.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=top) # for details on available settings such as changing the `fod` name. # ```{js} # window.onload = function () { @@ -230,7 +230,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees " + "cloud service is accessed using a 'ResourceKey'. " + "For more detail see " + - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + "https://configure.51degrees.com/g3gMZdPY?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py index 74b0df56..d8b06656 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py @@ -120,7 +120,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees " + "cloud service is accessed using a 'ResourceKey'. " + "For more detail see " + - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + "https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py index 4d11f6f5..98b7212e 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py @@ -61,7 +61,7 @@ def run(self, resource_key, logger, output, cloudEndPoint = ""): # This example creates the pipeline and engines in code. For a demonstration # of how to do this using a configuration file instead, see the TacLookup example. # For more information about builders in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=run cloudRequestEngineSettings = { "resource_key": resource_key } # If a cloud endpoint has been provided then set the @@ -128,7 +128,7 @@ def main(argv): f"environment variable '{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. " + "The 51Degrees cloud service is accessed using a 'ResourceKey'. " + "For more information " + - "see https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. " + + "see https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. " + "Native model lookup is not available as a free service. This means that " + "you will first need a license key, which can be purchased from our " + "pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. Once this is done, a resource " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py index 4563bd26..63355916 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py @@ -62,7 +62,7 @@ def run(self, config, logger, output): # For a demonstration of how to do this in code instead, see the # NativeModelLookup example. # For more information about builders in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=run # Create the pipeline using the service provider and the configured options. pipeline = PipelineBuilder().add_logger(logger).build_from_configuration(config) @@ -129,7 +129,7 @@ def main(argv): f"the environment variable '{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. " + "The 51Degrees cloud service is accessed using a 'ResourceKey'. " + "For more information see " + - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=resource-key-required. " + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=resource-key-required. " + "TAC lookup is not available as a free service. This means " + "that you will first need a license key, which can be purchased " + "from our pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=resource-key-required. Once this is " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py index 730153df..728f0594 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py @@ -136,7 +136,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees " + "cloud service is accessed using a 'ResourceKey'. " + "For more detail see " + - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=resource-key-required. " + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=resource-key-required. " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index f49c5bc0..946f06bd 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -51,15 +51,15 @@ def run(self, data_file, logger, output): # In this example, we use the DeviceDetectionPipelineBuilder # and configure it in code. For more information about # pipelines in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run pipeline = DeviceDetectionPipelineBuilder( data_file_path = data_file, # We use the low memory profile as its performance is # sufficient for this example. See the documentation for # more detail on this and other configuration options: # https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run - # https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run - # https://51degrees.com/documentation/_features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run performance_profile = "MaxPerformance", # inhibit sharing usage for this test, usually this # should be set "true" diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index dee9b99b..d382e6f2 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -58,7 +58,7 @@ # ``` # # Results can also be accessed in client-side code by using the `fod` object. See the -# [JavaScriptBuilderElementBuilder](https://51degrees.com/pipeline-python/4.3/classpipeline-python_1_1fiftyone__pipeline__core_1_1fiftyone__pipeline__core_1_1javascriptbuilde778a9036818b19ab55d981a40be4a4d7.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_web-app.py&utm_term=top) +# [JavascriptBuilderElement](https://51degrees.com/documentation/apis/pipeline-python/classfiftyone__pipeline__core_1_1javascriptbuilder_1_1_javascript_builder_element.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_web-app.py&utm_term=top) # for details on available settings such as changing the `fod` name. # ```{js} # window.onload = function () { diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py index ed974ae6..2df9d104 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py @@ -60,15 +60,15 @@ def run(self, data_file, logger, output): # In this example, we use the DeviceDetectionPipelineBuilder # and configure it in code. For more information about # pipelines in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run pipeline = DeviceDetectionPipelineBuilder( data_file_path = data_file, # We use the low memory profile as its performance is # sufficient for this example. See the documentation for # more detail on this and other configuration options: # https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run - # https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run - # https://51degrees.com/documentation/_features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run performance_profile = "LowMemory", # inhibit sharing usage for this test, usually this # should be set "true" diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py index 80ca344f..7a90c401 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py @@ -73,15 +73,15 @@ def run(self, data_file, evidence_yaml, logger, output): # In this example, we use the DeviceDetectionPipelineBuilder # and configure it in code. For more information about # pipelines in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run pipeline = DeviceDetectionPipelineBuilder( data_file_path = data_file, # We use the low memory profile as its performance is # sufficient for this example. See the documentation for # more detail on this and other configuration options: # https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run - # https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run - # https://51degrees.com/documentation/_features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run performance_profile = "LowMemory", # inhibit sharing usage for this test, usually this # should be set "true" @@ -92,7 +92,7 @@ def run(self, data_file, evidence_yaml, logger, output): # in order to help us improve detection of new devices/browsers/etc, then # this additional data will need to be collected and included as evidence # to the Pipeline. See - # https://51degrees.com/documentation/_features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run#Low_Level_Usage_Sharing + # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run#Low_Level_Usage_Sharing # for more details on this. usage_sharing = False, # Inhibit auto-update of the data file for this example diff --git a/device-detection/fiftyone_devicedetection_onpremise/readme.md b/device-detection/fiftyone_devicedetection_onpremise/readme.md index 284077ce..5a83973d 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/readme.md +++ b/device-detection/fiftyone_devicedetection_onpremise/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - On-Premise -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise-readme.md&utm_term=51degrees-device-detection-engines-on-premise "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=51degrees-device-detection-engines-on-premise "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise-readme.md&utm_term=51degrees-device-detection-engines-on-premise "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise-readme.md&utm_term=51degrees-device-detection-engines-on-premise "View all available properties and values") +[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=51degrees-device-detection-engines-on-premise "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=51degrees-device-detection-engines-on-premise "View all available properties and values") ## Introduction @@ -56,7 +56,7 @@ fd.process() print(fd.device.ismobile.value()) ``` -For more in-depth examples, check out the [examples](https://51degrees.com/device-detection-python/examples.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise-readme.md&utm_term=from-pypi) page in the documentation. +For more in-depth examples, check out the [examples](https://51degrees.com/device-detection-python/examples.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=from-pypi) page in the documentation. ### From GitHub @@ -77,3 +77,12 @@ If you've cloned the GitHub repository, you will be able to run the examples in To run the tests use: `python -m unittest discover -s tests -p test*.py -b` + +## Find out more + +- [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-device-detection) +- [Device Detection documentation](https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python +- https://github.com/51Degrees/51degrees-cxx diff --git a/device-detection/fiftyone_devicedetection_onpremise/setup.py b/device-detection/fiftyone_devicedetection_onpremise/setup.py index d4f60fbc..0af5631d 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/setup.py +++ b/device-detection/fiftyone_devicedetection_onpremise/setup.py @@ -232,6 +232,11 @@ def stage_native_source(): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", + project_urls={ + "Homepage": "https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description = """Device Detection and User-Agent parser for Python that detects the device, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints by reading a data file on your server. An alternative to UAParser.js, DeviceAtlas, and WURFL.""", keywords=[ "51degrees", diff --git a/device-detection/fiftyone_devicedetection_shared/readme.md b/device-detection/fiftyone_devicedetection_shared/readme.md index bbff13b6..1a5deb11 100644 --- a/device-detection/fiftyone_devicedetection_shared/readme.md +++ b/device-detection/fiftyone_devicedetection_shared/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - Shared -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared-readme.md&utm_term=51degrees-device-detection-engines-shared "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=51degrees-device-detection-engines-shared "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared-readme.md&utm_term=51degrees-device-detection-engines-shared "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared-readme.md&utm_term=51degrees-device-detection-engines-shared "View all available properties and values") +[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=51degrees-device-detection-engines-shared "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=51degrees-device-detection-engines-shared "View all available properties and values") ## Introduction @@ -23,3 +23,11 @@ The Pipeline is a generic web request intelligence and data processing solution To run the tests use: `python -m unittest discover -s tests -p test*.py -b` + +## Find out more + +- [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-device-detection) +- [Device Detection documentation](https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/device-detection/fiftyone_devicedetection_shared/setup.py b/device-detection/fiftyone_devicedetection_shared/setup.py index e367b7d1..07b89cdf 100644 --- a/device-detection/fiftyone_devicedetection_shared/setup.py +++ b/device-detection/fiftyone_devicedetection_shared/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author_email="engineering@51degrees.com", author="51Degrees Engineering", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("The 51Degrees Pipeline API is a generic web request intelligence and data processing solution with the ability to add a range of 51Degrees and/or custom plug ins (Engines). This project contains 51Degrees Device Detection engines that can be used with the Pipeline API."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/device-detection/readme.md b/device-detection/readme.md index 379be178..809d47e4 100644 --- a/device-detection/readme.md +++ b/device-detection/readme.md @@ -25,7 +25,7 @@ git submodule update --init --recursive ## Dependencies For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page. -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. @@ -35,10 +35,10 @@ The API can either use our cloud service to get its data or it can use a local ( #### Cloud -You will require a [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud) +You will require a [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud) to use the Cloud API. You can create resource keys using our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud), see our -[documentation](https://51degrees.com/documentation/_concepts__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud) on how to use this. +[documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud) on how to use this. #### On-Premise diff --git a/ip-intelligence/ci/README.md b/ip-intelligence/ci/README.md index a5089246..7f4b5fa1 100644 --- a/ip-intelligence/ci/README.md +++ b/ip-intelligence/ci/README.md @@ -6,7 +6,7 @@ The secrets each workflow passes are named in [.github/workflows](../.github/wor The following secrets are required: * `ACCESS_TOKEN` - GitHub [access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#about-personal-access-tokens) for cloning repos, creating PRs, etc. * Example: `github_pat_l0ng_r4nd0m_s7r1ng` -* `_51DEGREES_RESOURCE_KEY_BESPOKE` - [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for the cloud tests. It must include the IP Intelligence (`ip`) product, or every cloud test fails as if the service were broken. +* `_51DEGREES_RESOURCE_KEY_BESPOKE` - [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for the cloud tests. It must include the IP Intelligence (`ip`) product, or every cloud test fails as if the service were broken. * Example: `R4nd0m-S7r1ng` * `PYPI_TOKEN` - [PyPI token](https://pypi.org/help/#apitoken) for publishing packages * Example: `pypi-sUp3r-l0nG_r4nd0m-s7r1Ng` diff --git a/ip-intelligence/fiftyone_ipintelligence/readme.md b/ip-intelligence/fiftyone_ipintelligence/readme.md index da4cdcb3..1b1c9eec 100644 --- a/ip-intelligence/fiftyone_ipintelligence/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence/readme.md @@ -1,13 +1,13 @@ # 51Degrees IP Intelligence Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence-readme.md&utm_term=51degrees-ip-intelligence-engines "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=51degrees-ip-intelligence-engines "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence-readme.md&utm_term=51degrees-ip-intelligence-engines "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=51degrees-ip-intelligence-engines "developer documentation") ## Introduction This package aggregates the cloud and on-premise 51Degrees IP intelligence -engines for the [51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence-readme.md&utm_term=introduction) +engines for the [51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=introduction) and provides the `IpIntelligencePipelineBuilder` which builds a pipeline with either engine depending on the settings provided: @@ -34,3 +34,11 @@ flowdata.evidence.add("query.client-ip", "185.28.167.78") flowdata.process() print(flowdata.ip.registeredname.value()) ``` + +## Find out more + +- [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-ip-intelligence) +- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/fiftyone_ipintelligence/setup.py b/ip-intelligence/fiftyone_ipintelligence/setup.py index ef684aa9..a079441e 100644 --- a/ip-intelligence/fiftyone_ipintelligence/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence/setup.py @@ -40,7 +40,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("IP Intelligence and IP geolocation for Python that finds an IP address's likely location (country, region, town, and coordinates), connection type, and network, on-premise with a data file or in the cloud. An alternative to MaxMind and IPinfo."), keywords=[ "51degrees", diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md b/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md index dba487eb..83d568f9 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md @@ -1,17 +1,17 @@ # 51Degrees IP Intelligence Engines - Cloud -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud-readme.md&utm_term=51degrees-ip-intelligence-engines-cloud "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=51degrees-ip-intelligence-engines-cloud "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud-readme.md&utm_term=51degrees-ip-intelligence-engines-cloud "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=51degrees-ip-intelligence-engines-cloud "developer documentation") ## Introduction This package uses the 51Degrees cloud service to resolve IP addresses to location, network and registration information using the -[51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud-readme.md&utm_term=introduction). +[51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=introduction). To use the cloud service you will need a resource key. You can obtain one -from the [51Degrees Configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud-readme.md&utm_term=introduction). +from the [51Degrees Configurator](https://configure.51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=introduction). ## Usage @@ -34,3 +34,11 @@ print(flowdata.ip.registeredname.value()) - `tests` - unit tests. These mock the cloud service so no resource key is required; set the `resource_key` environment variable to also run the live cloud integration test. + +## Find out more + +- [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-ip-intelligence) +- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/setup.py b/ip-intelligence/fiftyone_ipintelligence_cloud/setup.py index 8eca633e..a5cd5e2f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/setup.py @@ -40,7 +40,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("IP Intelligence and IP geolocation for Python that finds an IP address's likely location (country, region, town, and coordinates), connection type, and network using the 51Degrees cloud service. An alternative to MaxMind and IPinfo."), keywords=[ "51degrees", diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md index d9542f6a..c41c0630 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md @@ -2,7 +2,7 @@ ![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=51degrees-ip-intelligence-engines-examples "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=51degrees-ip-intelligence-engines-examples "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=51degrees-ip-intelligence-engines-examples "developer documentation") ## Introduction diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/setup.py b/ip-intelligence/fiftyone_ipintelligence_examples/setup.py index 0ae1c723..2e1def33 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/setup.py @@ -40,7 +40,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("Examples for the 51Degrees IP intelligence engines."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py index 0c4eee08..75c8c9d9 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py @@ -105,7 +105,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " "service is accessed using a 'ResourceKey'. For more information " "see " - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-getallproperties_console.py&utm_term=resource-key-required. " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-getallproperties_console.py&utm_term=resource-key-required. " "A resource key with the properties required by this example " "can be created for free at https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-getallproperties_console.py&utm_term=resource-key-required.") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py index 882a8b03..7118db82 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py @@ -81,7 +81,7 @@ def run(self, resource_key, logger, output, cloud_endpoint = None): # In this example, we use the IpIntelligencePipelineBuilder and # configure it in code. For more information about pipelines in # general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=run settings = { # Obtain a resource key from https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=run "resource_key": resource_key @@ -172,7 +172,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " "service is accessed using a 'ResourceKey'. For more " "information see " - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " "A resource key with the properties required by this example " "can be created for free at https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " "Once complete, populate the environment variable mentioned at " diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py index c4d26b6e..60db3582 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py @@ -104,7 +104,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " "service is accessed using a 'ResourceKey'. For more information " "see " - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-metadata_console.py&utm_term=resource-key-required. " "A resource key with the properties required by this example " "can be created for free at https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-metadata_console.py&utm_term=resource-key-required.") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py index f5ad8cdc..8e82567b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py @@ -75,14 +75,14 @@ def run(self, data_file, logger, output): # In this example, we use the IpIntelligencePipelineBuilder # and configure it in code. For more information about # pipelines in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=run pipeline = IpIntelligencePipelineBuilder( data_file_path = data_file, # We use the low memory profile as its performance is # sufficient for this example. See the documentation for # more detail on this and other configuration options: - # https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=run - # https://51degrees.com/documentation/_features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=run performance_profile = "LowMemory", # inhibit sharing usage for this example, usually this # should be set "true" diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md index 65b11d8e..569058a1 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md @@ -1,14 +1,14 @@ # 51Degrees IP Intelligence Engines - OnPremise -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise-readme.md&utm_term=51degrees-ip-intelligence-engines-onpremise "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=51degrees-ip-intelligence-engines-onpremise "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise-readme.md&utm_term=51degrees-ip-intelligence-engines-onpremise "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=51degrees-ip-intelligence-engines-onpremise "developer documentation") ## Introduction This package resolves IP addresses to location, network and registration information using a local `.ipi` data file and the -[51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise-readme.md&utm_term=introduction). +[51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=introduction). The native [51degrees-cxx](https://github.com/51Degrees/51degrees-cxx/blob/main/docs/ipi/README.md) library is included as a git submodule and wrapped with SWIG. @@ -55,3 +55,12 @@ flowdata.evidence.add("query.client-ip", "185.28.167.78") flowdata.process() print(flowdata.ip.registeredname.value()) ``` + +## Find out more + +- [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-ip-intelligence) +- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python +- https://github.com/51Degrees/51degrees-cxx diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py index 6c2d0349..b15a227a 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py @@ -250,6 +250,11 @@ def stage_native_source(): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", + project_urls={ + "Homepage": "https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description = """IP Intelligence and IP geolocation for Python that finds an IP address's likely location (country, region, town, and coordinates), connection type, and network by reading a data file on your server. An alternative to MaxMind and IPinfo.""", keywords=[ "51degrees", diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/readme.md b/ip-intelligence/fiftyone_ipintelligence_shared/readme.md index 7980c868..ced1fe62 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_shared/readme.md @@ -1,14 +1,14 @@ # 51Degrees IP Intelligence Engines - Shared -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared-readme.md&utm_term=51degrees-ip-intelligence-engines-shared "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=51degrees-ip-intelligence-engines-shared "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared-readme.md&utm_term=51degrees-ip-intelligence-engines-shared "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=51degrees-ip-intelligence-engines-shared "developer documentation") ## Introduction This package contains shared code used by both the cloud and on-premise variants of the 51Degrees IP intelligence engines for the -[51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared-readme.md&utm_term=introduction). +[51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=introduction). It is not intended to be used directly. See [fiftyone_ipintelligence](https://pypi.org/project/fiftyone-ipintelligence/) @@ -26,3 +26,11 @@ instead. * The `fiftyone_pipeline_core`, `fiftyone_pipeline_engines`, `fiftyone_pipeline_cloudrequestengine` and `fiftyone_pipeline_engines_fiftyone` packages + +## Find out more + +- [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-ip-intelligence) +- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/setup.py b/ip-intelligence/fiftyone_ipintelligence_shared/setup.py index c4480d4e..7f89e1d0 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_shared/setup.py @@ -40,7 +40,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("Shared code used by the cloud and on-premise variants of the 51Degrees IP Intelligence engines for the 51Degrees Pipeline API."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/readme.md b/ip-intelligence/fiftyone_ipintelligence_translation/readme.md index ad37b2fc..7f12b228 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_translation/readme.md @@ -1,8 +1,8 @@ # IP Intelligence Country Translation -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation-readme.md&utm_term=ip-intelligence-country-translation "Data rewards the curious") **Python IP Intelligence Country Translation** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=ip-intelligence-country-translation "Data rewards the curious") **Python IP Intelligence Country Translation** -[Developer Documentation](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation-readme.md&utm_term=ip-intelligence-country-translation "Developer Documentation") +[Developer Documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=ip-intelligence-country-translation "Developer Documentation") ## Introduction @@ -33,3 +33,11 @@ Built on top of: * [fiftyone_pipeline_core](https://pypi.org/project/fiftyone-pipeline-core/) * [fiftyone_pipeline_translation](https://pypi.org/project/fiftyone-pipeline-translation/) + +## Find out more + +- [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-ip-intelligence) +- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/setup.py b/ip-intelligence/fiftyone_ipintelligence_translation/setup.py index 3f06c4a4..91ca1dd4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_translation/setup.py @@ -80,7 +80,11 @@ def copy_yaml(folder): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("Country translation engines for 51Degrees on-premise IP Intelligence. Turns the weighted ISO country codes returned by the IP Intelligence engine into localized country names and complete ordered country lists."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/ip-intelligence/readme.md b/ip-intelligence/readme.md index 18d82f7e..7a1be44e 100644 --- a/ip-intelligence/readme.md +++ b/ip-intelligence/readme.md @@ -2,7 +2,7 @@ ![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines "developer documentation") ## Introduction @@ -21,7 +21,7 @@ design of this API. For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) -page. The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) +page. The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. ### Git submodules diff --git a/location/ci/README.md b/location/ci/README.md index 8e1cb35c..399dcd50 100644 --- a/location/ci/README.md +++ b/location/ci/README.md @@ -4,7 +4,7 @@ This API complies with the `common-ci` approach. The following secrets are required: * `ACCESS_TOKEN` - GitHub [access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#about-personal-access-tokens) for cloning repos, creating PRs, etc. * Example: `github_pat_l0ng_r4nd0m_s7r1ng` -* `SUPER_RESOURCE_KEY` - [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for integration tests +* `SUPER_RESOURCE_KEY` - [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for integration tests * Example: `R4nd0m-S7r1ng` * `PYPI_TOKEN` - [PyPI token](https://pypi.org/help/#apitoken) for publishing packages * Example: `pypi-sUp3r-l0nG_r4nd0m-s7r1Ng` diff --git a/location/readme.md b/location/readme.md index 5077f856..e3c7ee9d 100644 --- a/location/readme.md +++ b/location/readme.md @@ -1,9 +1,9 @@ # 51Degrees Geo-Location Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-geo-location-engines) +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=51degrees-geo-location-engines) **v4 Location Python** -[Developer Documentation](https://51degrees.com/location-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-geo-location-engines "Developer documentation") +[Developer Documentation](https://51degrees.com/location-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=51degrees-geo-location-engines "Developer documentation") ## Introduction @@ -13,8 +13,8 @@ The Pipeline is a generic web request intelligence and data processing solution ## Dependencies -For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page. -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. +For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=dependencies) page. +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. ## Installation and Examples @@ -24,7 +24,7 @@ The [tested versions](https://51degrees.com/documentation/_info__tested_versions You can confirm this is working with the following micro-example. -* Create a resource key for free with the 51Degrees [configurator](https://configure.51degrees.com/6CTsmbPx?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=from-pypi). This defines the properties you want to access. +* Create a resource key for free with the 51Degrees [configurator](https://configure.51degrees.com/6CTsmbPx?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=from-pypi). This defines the properties you want to access. * On the 'implement' page of the configurator, copy the resource key and replace YOUR_RESOURCE_KEY in the example below. Save this as examplelocation.py * Run the example with `python examplelocation.py` * Feel free to try different locations and property values. @@ -61,3 +61,10 @@ Execute `export FLASK_APP=web` where `web` is the example file, and start your a Execute `$env:FLASK_APP = "web"` where `web` is the example file, and start your application via `flask run`. +## Find out more + +- [Geolocation](https://51degrees.com/geolocation?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-geolocation) +- [Reverse geocoding documentation](https://51degrees.com/documentation/_reverse_geocoding__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_reverse_geocoding__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/location/setup.py b/location/setup.py index 9c8df34b..b7e00ecd 100644 --- a/location/setup.py +++ b/location/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/geolocation?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_reverse_geocoding__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("51Degrees Geolocation performs reverse geocoding (using longitude and latitude information) to populate postal addresses."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/ci/README.md b/pipeline/ci/README.md index 8e1cb35c..399dcd50 100644 --- a/pipeline/ci/README.md +++ b/pipeline/ci/README.md @@ -4,7 +4,7 @@ This API complies with the `common-ci` approach. The following secrets are required: * `ACCESS_TOKEN` - GitHub [access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#about-personal-access-tokens) for cloning repos, creating PRs, etc. * Example: `github_pat_l0ng_r4nd0m_s7r1ng` -* `SUPER_RESOURCE_KEY` - [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for integration tests +* `SUPER_RESOURCE_KEY` - [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for integration tests * Example: `R4nd0m-S7r1ng` * `PYPI_TOKEN` - [PyPI token](https://pypi.org/help/#apitoken) for publishing packages * Example: `pypi-sUp3r-l0nG_r4nd0m-s7r1Ng` diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md index 5d82e16d..5979c0e8 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md @@ -1,8 +1,8 @@ # Cloud Request Engine -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-readme.md&utm_term=cloud-request-engine "Data rewards the curious") **Python Pipeline Cloud Request Engine** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=cloud-request-engine "Data rewards the curious") **Python Pipeline Cloud Request Engine** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-readme.md&utm_term=cloud-request-engine "Developer Documentation") +[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=cloud-request-engine "Developer Documentation") ## Introduction @@ -16,7 +16,7 @@ The Pipeline is a generic web request intelligence and data processing solution This package uses the `engines` class created by the `fiftyone-pipeline-engines`. It makes available: -* A `Cloud Request Engine` which calls the 51Degrees cloud service to fetch properties and metadata about them based on a provided resource key. Get a resource key at https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-readme.md&utm_term=this-package-fiftyone_pipeline_cloudrequestengine +* A `Cloud Request Engine` which calls the 51Degrees cloud service to fetch properties and metadata about them based on a provided resource key. Get a resource key at https://configure.51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=this-package-fiftyone_pipeline_cloudrequestengine * A `Cloud Engine` template which reads data from the Cloud Request Engine. ## Credentials @@ -79,3 +79,11 @@ It is used by the cloud versions of the following 51Degrees engines: - [**fiftyone_devicedetection**](https://pypi.org/project/fiftyone-devicedetection/) - Get details about the devices accessing your web page - [**fiftyone_location**](https://pypi.org/project/fiftyone-location/) - Get postal address details from the location of devices accessing your web page + +## Find out more + +- [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-build-your-own) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-documentation) +- [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-examples) +- [Cloud service documentation](https://51degrees.com/documentation/_services__cloud__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-cloud-service) +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py b/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py index 434c0a25..223d23d8 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py @@ -42,7 +42,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("This package contains a shared engine that is used to make requests to the 51Degrees cloud service on behalf of other data services such as device detection or location."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py index d28a9c86..4f4f8c88 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py @@ -42,7 +42,7 @@ def check(self, key, flow_element): + flow_element.datakey + " are " + ", ".join(list(flow_element.get_properties().keys())) + ". For more details on resource keys, see our explainer: " - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-clouddata.py&utm_term=property-not-found") + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-clouddata.py&utm_term=property-not-found") class CloudData(AspectDataDictionary): diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py index 2d872d16..9f285f75 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py @@ -73,7 +73,7 @@ def on_registration(self, pipeline): # key in use, a resource key or a licence key, so a section missing # from them means that key does not carry the product at all. if not self.datakey in pipeline.flow_elements_list["cloud"].flow_element_properties: - raise Exception("The key in use does not include access to any properties under the engine with key " + self.datakey + " that was added to the pipeline. For more details on resource keys, see our explainer: https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-cloudengine.py&utm_term=engine-properties-not-accessible " + "Available engine data keys are: " + str([e for e in pipeline.flow_elements_list["cloud"].flow_element_properties])) + raise Exception("The key in use does not include access to any properties under the engine with key " + self.datakey + " that was added to the pipeline. For more details on resource keys, see our explainer: https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-cloudengine.py&utm_term=engine-properties-not-accessible " + "Available engine data keys are: " + str([e for e in pipeline.flow_elements_list["cloud"].flow_element_properties])) self.properties = pipeline.flow_elements_list["cloud"].flow_element_properties[self.datakey] @@ -95,7 +95,7 @@ def pipeline_element_not_found(self, element, flowdata): """ - raise Exception("The key in use does not include access to any properties under " + element + ". For more details on resource keys, see our explainer: https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-cloudengine.py&utm_term=element-properties-not-accessible " + "Available element data keys are: " + str([e for e in flowdata.pipeline.flow_elements_display_list])) + raise Exception("The key in use does not include access to any properties under " + element + ". For more details on resource keys, see our explainer: https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-cloudengine.py&utm_term=element-properties-not-accessible " + "Available element data keys are: " + str([e for e in flowdata.pipeline.flow_elements_display_list])) def process_internal(self, flowdata): diff --git a/pipeline/fiftyone_pipeline_core/readme.md b/pipeline/fiftyone_pipeline_core/readme.md index 078967db..ad3470f1 100644 --- a/pipeline/fiftyone_pipeline_core/readme.md +++ b/pipeline/fiftyone_pipeline_core/readme.md @@ -1,8 +1,8 @@ # Pipeline Core -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core-readme.md&utm_term=pipeline-core "Data rewards the curious") **Python Pipeline Core** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=pipeline-core "Data rewards the curious") **Python Pipeline Core** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core-readme.md&utm_term=pipeline-core "Developer Documentation") +[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=pipeline-core "Developer Documentation") ## Introduction @@ -24,4 +24,11 @@ Engines created by 51Degrees: ## Requirements * Python 3.8+ -* The `flask` python library to run the web examples +* The `flask` python library to run the web examples + +## Find out more + +- [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=find-out-more-build-your-own) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=find-out-more-documentation) +- [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=find-out-more-examples) +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_core/setup.py b/pipeline/fiftyone_pipeline_core/setup.py index a79339f4..1da2b014 100644 --- a/pipeline/fiftyone_pipeline_core/setup.py +++ b/pipeline/fiftyone_pipeline_core/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("This package defines the essential components of the 51Degrees Pipeline API such as flow elements, flow data and evidence. It also packages together JavaScript served by a pipeline and allows for client side requests for additional data populated by evidence from the client side."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/fiftyone_pipeline_did/readme.md b/pipeline/fiftyone_pipeline_did/readme.md index 837353b0..fe51f236 100644 --- a/pipeline/fiftyone_pipeline_did/readme.md +++ b/pipeline/fiftyone_pipeline_did/readme.md @@ -713,3 +713,10 @@ is refreshed by common-ci's `update-example-assets` step. not a format limit. - **No creation of new 51Dids.** This is a parser; new 51Dids are issued by the 51Degrees cloud / on-premise hashing engines. + +## Find out more + +- [51Did](https://51degrees.com/developers/51did?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did&utm_term=find-out-more-51did) +- [51Did documentation](https://51degrees.com/documentation/_identifiers_51_did.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did&utm_term=find-out-more-documentation) +- [51Did inspector](https://51degrees.com/developers/51did-inspector?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did&utm_term=find-out-more-51did-inspector) +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_did/setup.py b/pipeline/fiftyone_pipeline_did/setup.py index 0c0fb67b..7d5e581b 100644 --- a/pipeline/fiftyone_pipeline_did/setup.py +++ b/pipeline/fiftyone_pipeline_did/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/developers/51did?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_identifiers_51_did.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("Strongly typed reader and cloud client for the 51Did (51Degrees Identifier) value returned by the 51Degrees Cloud service. Parses the OWID envelope in either base64 alphabet and exposes the Flags, License Id and match key plus the identifier type, and verifies a 51Did's signature offline or through the cloud and redeems a sealed creator context result on the server. Compare match keys, never envelopes."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/fiftyone_pipeline_engines/readme.md b/pipeline/fiftyone_pipeline_engines/readme.md index 95eebd5d..b9993940 100644 --- a/pipeline/fiftyone_pipeline_engines/readme.md +++ b/pipeline/fiftyone_pipeline_engines/readme.md @@ -1,8 +1,8 @@ # Pipeline Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-readme.md&utm_term=pipeline-engines "Data rewards the curious") **Python Pipeline Engines** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=pipeline-engines "Data rewards the curious") **Python Pipeline Engines** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-readme.md&utm_term=pipeline-engines "Developer Documentation") +[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=pipeline-engines "Developer Documentation") ## Introduction @@ -30,3 +30,9 @@ Engines created by 51Degrees: * Python 3.8+ * The gitversion lib using `python -m pip install gitversion` +## Find out more + +- [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=find-out-more-build-your-own) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=find-out-more-documentation) +- [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=find-out-more-examples) +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_engines/setup.py b/pipeline/fiftyone_pipeline_engines/setup.py index b3b09ee2..d98d758e 100644 --- a/pipeline/fiftyone_pipeline_engines/setup.py +++ b/pipeline/fiftyone_pipeline_engines/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("This package extends the flow element class created by the fiftyone-pipeline-core package into a specialized type of flow element called an engine."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/readme.md b/pipeline/fiftyone_pipeline_engines_fiftyone/readme.md index 49d04854..8dba8ae2 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/readme.md +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/readme.md @@ -1,8 +1,8 @@ # Pipeline Engines Fiftyone -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone-readme.md&utm_term=pipeline-engines-fiftyone "Data rewards the curious") **Python Pipeline Engines** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=pipeline-engines-fiftyone "Data rewards the curious") **Python Pipeline Engines** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone-readme.md&utm_term=pipeline-engines-fiftyone "Developer Documentation") +[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=pipeline-engines-fiftyone "Developer Documentation") ## Introduction @@ -15,3 +15,10 @@ It includes a ShareUsage engine that sends usage data to 51Degrees in zipped bat ## Requirements * Python 3.8+ + +## Find out more + +- [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=find-out-more-build-your-own) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=find-out-more-documentation) +- [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=find-out-more-examples) +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/setup.py b/pipeline/fiftyone_pipeline_engines_fiftyone/setup.py index a4c89ef1..b9445460 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/setup.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/setup.py @@ -40,7 +40,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("The 51Degrees Pipeline API is a generic web request intelligence and data processing solution with the ability to add a range of 51Degrees and/or custom plug ins (Engines). It includes a ShareUsage engine that sends usage data to 51Degrees in zipped batches."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/fiftyone_pipeline_translation/readme.md b/pipeline/fiftyone_pipeline_translation/readme.md index e38c02dd..4c0450eb 100644 --- a/pipeline/fiftyone_pipeline_translation/readme.md +++ b/pipeline/fiftyone_pipeline_translation/readme.md @@ -1,8 +1,8 @@ # Pipeline Translation -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation-readme.md&utm_term=pipeline-translation "Data rewards the curious") **Python Pipeline Translation** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=pipeline-translation "Data rewards the curious") **Python Pipeline Translation** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation-readme.md&utm_term=pipeline-translation "Developer Documentation") +[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=pipeline-translation "Developer Documentation") ## Introduction @@ -24,3 +24,10 @@ header). This package is built on top of: * [fiftyone_pipeline_core](https://pypi.org/project/fiftyone-pipeline-core/) + +## Find out more + +- [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=find-out-more-build-your-own) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=find-out-more-documentation) +- [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=find-out-more-examples) +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_translation/setup.py b/pipeline/fiftyone_pipeline_translation/setup.py index fa2a9f4d..6460ddb6 100644 --- a/pipeline/fiftyone_pipeline_translation/setup.py +++ b/pipeline/fiftyone_pipeline_translation/setup.py @@ -43,7 +43,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("This package adds a generic translation flow element to the fiftyone-pipeline, which translates values from a source element into another language using YAML translation files."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/readme.md b/pipeline/readme.md index a5c12c34..ea7419ec 100755 --- a/pipeline/readme.md +++ b/pipeline/readme.md @@ -24,7 +24,7 @@ The examples are described under Examples below. The `owid-python` folder is a g ## Dependencies For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page. -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. ## Installation @@ -56,7 +56,7 @@ The tests that call the 51Degrees cloud service read a resource key from the `re ## Examples There are several examples available that demonstrate how to make use of the Pipeline API in isolation. These are described in the table below. -If you want examples that demonstrate how to use 51Degrees products such as device detection, then these are available in the corresponding [folder](https://github.com/51Degrees/51degrees-python/tree/main/device-detection) and on our [website](https://51degrees.com/documentation/_examples__device_detection__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=examples). +If you want examples that demonstrate how to use 51Degrees products such as device detection, then these are available in the corresponding [folder](https://github.com/51Degrees/51degrees-python/tree/main/device-detection) and on our [website](https://51degrees.com/documentation/_device_detection__examples__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=examples). | Example | Description | |-----------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------| From 47cc252b8e26678f547e725983ee8a8d9d2c9b97 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 17:25:57 +0000 Subject: [PATCH 08/68] Ask the country property for its no value message in the location example (#32) --- location/examples/cloud/gettingstarted.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/location/examples/cloud/gettingstarted.py b/location/examples/cloud/gettingstarted.py index f5a4041c..a0052c3f 100644 --- a/location/examples/cloud/gettingstarted.py +++ b/location/examples/cloud/gettingstarted.py @@ -77,4 +77,4 @@ if flow_data.location.country.has_value(): print(flow_data.location.country.value()) else: - print(flow_data.location.no_value_message()) + print(flow_data.location.country.no_value_message()) From 653e5521aed4c311fe463487cf7f97fddbbf018a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 17:26:19 +0000 Subject: [PATCH 09/68] Describe the rc approach to releases in AGENTS.md (#29) --- AGENTS.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index deb8846b..539b95af 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -310,6 +310,14 @@ draft and has no review requested. When a release is wanted, one pull request merges `rc` into `main`, and the "Only rc into main" check fails a pull request into `main` from any other branch. +## Releases + +Pull requests go to `rc`, not `main`, and stay drafts until they have been +reviewed, because the nightly merges any pull request into `rc` that is not a +draft and has no review requested. When a release is wanted, one pull request +merges `rc` into `main`, and the "Only rc into main" check fails a pull request +into `main` from any other branch. + ## History The four repositories were brought in with `git subtree add`, so commit SHAs From b3e0cd757870808c17fdb52833544a1aead9aab4 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 19:53:17 +0000 Subject: [PATCH 10/68] Check the callback URL before the JavaScript builder writes it (#31) --- .../fiftyone_pipeline_core/callback_url.py | 274 ++++++++++++++++++ .../javascriptbuilder.py | 44 +-- .../tests/test_callback_url.py | 200 +++++++++++++ .../tests/test_javascriptbuilder_variables.py | 84 ++++++ 4 files changed, 581 insertions(+), 21 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_callback_url.py diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py new file mode 100644 index 00000000..afafa240 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py @@ -0,0 +1,274 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +The callback URL the client script posts to for updated JSON. The protocol +and host usually come from the request, and the script holds the URL in a +single-quoted string, so the URL is checked before it is written into the +script. A URL that is not valid is refused. In the user information, in +anything after the host and port, and in the endpoint, the characters a URL +cannot hold or that could end the string or the script are percent-encoded. + +See https://github.com/51Degrees/specifications/blob/main/pipeline-specification/pipeline-elements/javascript-builder.md#callback-url +""" + +import re +from collections import namedtuple + +# Whether the URL can be written into the script, and the URL. That is the +# URL to write into the script when it is valid, and otherwise the URL as it +# was built, for the log. +CallbackUrl = namedtuple("CallbackUrl", ["valid", "url"]) + +# The largest port number. +MAX_PORT = 65535 + +# The most digits a port can have once its leading zeros are dropped. +MAX_PORT_DIGITS = 5 + +# The printable ASCII characters that are percent-encoded, besides the +# control characters, the space and everything outside ASCII. +ENCODED_CHARACTERS = frozenset(b"\"'<>\\^`{|}") + +HEX_DIGITS = frozenset(b"0123456789ABCDEFabcdef") + +# Text made only of the Unicode White_Space characters, which counts as a +# part that is not set. The first range is tab, line feed, vertical tab, +# form feed and carriage return. +BLANK_PATTERN = re.compile( + r"[\t-\r \u0085\u00a0\u1680\u2000-\u200a" + r"\u2028\u2029\u202f\u205f\u3000]*") + +# A scheme as RFC 3986 section 3.1 defines one, being a letter followed by +# letters, digits, '+', '-' and '.'. +SCHEME_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*") + +# Where the authority ends, which is at the first '/', '?' or '#' of the +# host. +AUTHORITY_END_PATTERN = re.compile(r"[/?#]") + +# A label of a host name, being ASCII letters, digits, '-' and '_'. +LABEL_PATTERN = re.compile(r"[A-Za-z0-9_-]+") + +# An IPv6 zone identifier, being unreserved characters as RFC 3986 section +# 2.3 defines them. +ZONE_PATTERN = re.compile(r"[A-Za-z0-9._~-]+") + +# A port as it follows the host, being ':' and any number of digits. +PORT_PATTERN = re.compile(r":[0-9]*") + +# One group of an IPv6 address. +HEX_GROUP_PATTERN = re.compile(r"[0-9A-Fa-f]{1,4}") + +# An IPv4 address as RFC 3986 section 3.2.2 defines one, being four numbers +# from 0 to 255 with no leading zeros. +_DEC_OCTET = r"(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])" +IPV4_PATTERN = re.compile(_DEC_OCTET + r"(\." + _DEC_OCTET + r"){3}") + +# A lone surrogate, which has no UTF-8 encoding, so it is written as U+FFFD, +# as a browser writes it. +SURROGATE_PATTERN = re.compile(r"[\ud800-\udfff]") + + +def is_blank(text): + + """! + Whether a part of the URL is not set. + + @type text: object + @param text: The part + @rtype: bool + @return: True if the part is not a string, or is empty or only + whitespace + """ + + return (not isinstance(text, str) + or BLANK_PATTERN.fullmatch(text) is not None) + + +def callback_url(protocol, host, endpoint): + + """! + Build the callback URL from its parts and check it. One slash separates + the host and the endpoint, being added when neither has one and removed + when both do. The protocol and the host are written in lower case and + the rest as given, apart from the percent-encoding. + + @type protocol: str + @param protocol: The protocol, such as https + @type host: str + @param host: The host, which can start with user information and end + with a port, a path, a query or a fragment + @type endpoint: str + @param endpoint: The endpoint + @rtype: CallbackUrl + @return: The callback URL, or None when the protocol, host or endpoint + is not set + """ + + if is_blank(protocol) or is_blank(host) or is_blank(endpoint): + return None + host_has_slash = host.endswith("/") + endpoint_has_slash = endpoint.startswith("/") + if not host_has_slash and not endpoint_has_slash: + endpoint = "/" + endpoint + elif host_has_slash and endpoint_has_slash: + endpoint = endpoint[1:] + built = protocol + "://" + host + endpoint + if SCHEME_PATTERN.fullmatch(protocol) is None: + return CallbackUrl(False, built) + match = AUTHORITY_END_PATTERN.search(host) + end = len(host) if match is None else match.start() + authority = _checked_authority(host[:end]) + if authority is None: + return CallbackUrl(False, built) + return CallbackUrl( + True, + protocol.lower() + "://" + authority + + _encoded(host[end:]) + _encoded(endpoint)) + + +def _checked_authority(authority): + + """! + The user information, host and port as they are written into the URL, + or None when they are not valid. The user information ends at the first + '@', so a second '@' is part of the host and refused. + """ + + user, at, host_port = authority.partition("@") + if not at: + return _checked_host_port(authority) + host_port = _checked_host_port(host_port) + if host_port is None: + return None + return _encoded(user) + "@" + host_port + + +def _checked_host_port(host_port): + + """! + The host and optional port in lower case, or None when they are not + valid. The host is an IPv6 address in brackets with an optional zone + identifier, or a name of labels separated by dots with a dot allowed at + the end. The port is ':' followed by digits up to 65535, or by nothing. + """ + + if host_port.startswith("["): + close = host_port.find("]") + if close == -1: + return None + address, separator, zone = host_port[1:close].partition("%25") + if separator and ZONE_PATTERN.fullmatch(zone) is None: + return None + if not _is_ipv6(address): + return None + host = host_port[:close + 1] + port = host_port[close + 1:] + else: + host, colon, port = host_port.partition(":") + port = colon + port + name = host[:-1] if host.endswith(".") else host + for label in name.split("."): + if LABEL_PATTERN.fullmatch(label) is None: + return None + if port and not _is_port(port): + return None + return host.lower() + port + + +def _is_port(text): + + """! + Whether the text is ':' followed by digits whose value is no more than + 65535, or by nothing. + """ + + if PORT_PATTERN.fullmatch(text) is None: + return False + digits = text[1:].lstrip("0") + return len(digits) <= MAX_PORT_DIGITS and ( + not digits or int(digits) <= MAX_PORT) + + +def _is_ipv6(text): + + """! + Whether the text is an IPv6 address as RFC 3986 section 3.2.2 defines + one. That is eight groups of one to four hexadecimal digits separated by + ':', where one '::' can stand for one or more groups of zeros and the + last two groups can be written as an IPv4 address. + """ + + gap = text.find("::") + if gap == -1: + return _groups(text, True) == 8 + if text.find("::", gap + 1) != -1: + return False + head = _groups(text[:gap], False) + tail = _groups(text[gap + 2:], True) + return head >= 0 and tail >= 0 and head + tail <= 7 + + +def _groups(run, ipv4_at_end): + + """! + The number of 16 bit groups in a run of groups separated by ':', or -1 + when the run is not valid. An empty run has none, and an IPv4 address at + the end of the run, where one is allowed, counts as two. + """ + + if not run: + return 0 + parts = run.split(":") + count = 0 + for i, part in enumerate(parts): + if HEX_GROUP_PATTERN.fullmatch(part): + count += 1 + elif (ipv4_at_end and i == len(parts) - 1 + and IPV4_PATTERN.fullmatch(part)): + count += 2 + else: + return -1 + return count + + +def _encoded(text): + + """! + The text with each byte of its UTF-8 encoding percent-encoded where the + byte is a control character, a space, outside ASCII or one of + ENCODED_CHARACTERS, and with a '%' encoded where two hexadecimal digits + do not follow it. + """ + + data = SURROGATE_PATTERN.sub("\ufffd", text).encode("utf-8") + out = [] + for i, b in enumerate(data): + if b == 0x25: + keep = (i + 2 < len(data) + and data[i + 1] in HEX_DIGITS + and data[i + 2] in HEX_DIGITS) + else: + keep = 0x20 < b < 0x7F and b not in ENCODED_CHARACTERS + out.append(chr(b) if keep else "%{:02X}".format(b)) + return "".join(out) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py index 1f9e206b..31d063a7 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py @@ -38,6 +38,7 @@ from .evidence_keyfilter import EvidenceKeyFilter from .elementdata_dictionary import ElementDataDictionary from .constants import Constants +from .callback_url import callback_url # Evidence keys the rendered script is not configured with. Both are @@ -295,6 +296,10 @@ def __init__(self, settings = {} ): self.exclude_from_messages = True + # Set once a callback URL that is not valid has been logged, because + # a misconfigured element would otherwise log on every request. + self._invalid_url_logged = False + # Load template file contents into memory template = Path(__file__).absolute().parent / "js_templates" / "JavaScriptResource.mustache" @@ -413,27 +418,24 @@ def process_internal(self, flowdata): variables["_parameters"] = json.dumps( variables["_parameters"], separators=(",", ":")) - if variables["_host"] and variables["_protocol"] and variables["_endpoint"]: - - # The callback URL is the protocol, the host and the endpoint, - # with exactly one slash between the host and the endpoint, as - # in the .NET builder. No query values are added to it. They - # reach the callback in the request body through the parameters - # above, and the script appends the session id and the sequence - # itself. Adding every evidence value here, as this builder used - # to, sent those two twice and, because the evidence reaching - # this element is not narrowed to query values, also wrote the - # end user's User-Agent, IP address and host headers into the - # served script's URL. - host = variables["_host"] - endpoint = variables["_endpoint"] - if not endpoint.startswith("/") and not host.endswith("/"): - endpoint = "/" + endpoint - elif endpoint.startswith("/") and host.endswith("/"): - endpoint = endpoint[1:] - - variables["_url"] = variables["_protocol"] + "://" + host + endpoint - + # The callback URL is the protocol, the host and the endpoint, as in + # the .NET builder, and callback_url checks it. No query values are + # added to it, because they reach the callback in the request body + # through the parameters above, and the script appends the session + # id and the sequence itself. + url = callback_url( + variables["_protocol"], variables["_host"], variables["_endpoint"]) + if url is not None and not url.valid and not self._invalid_url_logged: + self._invalid_url_logged = True + self._log( + "error", + "The generated URL " + repr(url.url) + " is not valid, so " + "the JavaScript has no callback URL. Check the JavaScript " + "builder's protocol, host and endpoint, and the protocol and " + "host evidence. This is reported once.") + + if url is not None and url.valid: + variables["_url"] = url.url variables["_updateEnabled"] = True else: variables["_updateEnabled"] = False diff --git a/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py new file mode 100644 index 00000000..9048401d --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py @@ -0,0 +1,200 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Checks the callback URL against the cases every implementation of the +JavaScript builder shares, in the same order, so that each one refuses and +writes the same URLs. +""" + +import unittest + +from parameterized import parameterized + +from fiftyone_pipeline_core.callback_url import CallbackUrl, callback_url + +ENDPOINT = "/51dpipeline/json" + + +def _url(protocol, host): + return callback_url(protocol, host, ENDPOINT) + + +def _valid(text): + return CallbackUrl(True, text) + + +class CallbackUrlTests(unittest.TestCase): + + def test_adds_a_single_slash(self): + self.assertEqual( + _valid("https://example.com/51dpipeline/json"), + callback_url("https", "example.com", "51dpipeline/json")) + + def test_keeps_a_single_slash_on_the_endpoint(self): + self.assertEqual( + _valid("https://example.com/51dpipeline/json"), + callback_url("https", "example.com", "/51dpipeline/json")) + + def test_collapses_a_double_slash(self): + self.assertEqual( + _valid("http://example.com/json"), + callback_url("http", "example.com/", "/json")) + + def test_missing_when_a_part_is_empty(self): + self.assertIsNone(callback_url("https", "", "/json")) + self.assertIsNone(callback_url("", "example.com", "/json")) + self.assertIsNone(callback_url("https", "example.com", "")) + self.assertIsNone(callback_url("https", " ", "/json")) + + @parameterized.expand([ + ("EXAMPLE.com", "https://example.com/51dpipeline/json"), + ("example.com:8080", "https://example.com:8080/51dpipeline/json"), + ("example.com:", "https://example.com:/51dpipeline/json"), + ("example.com:65535", "https://example.com:65535/51dpipeline/json"), + ("a_b.com", "https://a_b.com/51dpipeline/json"), + ("-ab.com", "https://-ab.com/51dpipeline/json"), + ("127.0.0.1:80", "https://127.0.0.1:80/51dpipeline/json"), + ("[::1]", "https://[::1]/51dpipeline/json"), + ("[::1]:8080", "https://[::1]:8080/51dpipeline/json"), + ("[fe80::1%25eth0]", "https://[fe80::1%25eth0]/51dpipeline/json"), + ("user:pw@example.com", + "https://user:pw@example.com/51dpipeline/json"), + ("u x@example.com", "https://u%20x@example.com/51dpipeline/json"), + ("u\tx@example.com", "https://u%09x@example.com/51dpipeline/json"), + ("example.com.", "https://example.com./51dpipeline/json"), + ("example.com/path", "https://example.com/path/51dpipeline/json"), + ("example.com?x", "https://example.com?x/51dpipeline/json"), + ("example.com/a b", "https://example.com/a%20b/51dpipeline/json"), + ("example.com/a%zzb", + "https://example.com/a%25zzb/51dpipeline/json"), + ("example.com/a%41b", "https://example.com/a%41b/51dpipeline/json"), + ("example.com/\u00fc", + "https://example.com/%C3%BC/51dpipeline/json"), + ("example.com/a\nb", "https://example.com/a%0Ab/51dpipeline/json"), + ("example.com/a\u2028b", + "https://example.com/a%E2%80%A8b/51dpipeline/json"), + ]) + def test_accepts_the_host(self, host, expected): + self.assertEqual(_valid(expected), _url("https", host)) + + @parameterized.expand([ + ("example.com:65536",), + ("example.com:99999",), + ("example.com:abc",), + ("exa mple.com",), + ("example.com');alert(1);//",), + ("a\"b.com",), + ("a'b.com",), + ("a",), + ]) + def test_nothing_given_can_end_the_string(self, host): + """A quote, a backslash or a line break never reaches the script, + wherever in the URL it was given.""" + + result = _url("https", host) + self.assertTrue(result.valid, repr(host)) + for c in "'\"\\<>\n\r\u2028\u2029": + self.assertNotIn(c, result.url, repr(host)) + + def test_the_endpoint_is_encoded(self): + self.assertEqual( + _valid("https://example.com/json%27x"), + callback_url("https", "example.com", "/json'x")) + + @parameterized.expand([ + ("b\u00fccher.de",), + ("a\u00a0b.com",), + ("a\u200bb.com",), + ("a\u2028b.com",), + ]) + def test_refuses_the_host_outside_ascii(self, host): + """A host must be ASCII, so an internationalized name is given as + A-labels, and anything else outside ASCII is refused.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + def test_writes_a_lone_surrogate_as_the_replacement_character(self): + """Not one of the shared cases, because only some languages' strings + can hold a lone surrogate. It has no UTF-8 encoding, so it is written + as U+FFFD, as a browser writes it.""" + + self.assertEqual( + _valid("https://example.com/%EF%BF%BD/51dpipeline/json"), + _url("https", "example.com/\ud800")) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py index d2c13a41..4c9302f4 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py @@ -27,6 +27,40 @@ from fiftyone_pipeline_core import javascriptbuilder from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from .classes.memorylogger import MemoryLogger + + +class _CapturedAll: + """Records the variables of every render and renders nothing.""" + + def __init__(self): + self.variables = [] + + def render(self, template, variables): + self.variables.append(dict(variables)) + return "" + + +def _render_with_hosts(hosts): + """Renders once for each host given, all through the same element, with + the host taken from the request's evidence. Returns the variables of + each render and the errors the pipeline logged.""" + + logger = MemoryLogger("error") + pipeline = PipelineBuilder({"javascript_builder_settings": { + "minify": False, "endpoint": "/json"}}).add_logger(logger).build() + captured = _CapturedAll() + with mock.patch.object( + javascriptbuilder.chevron, "render", captured.render): + for host in hosts: + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.host", host) + flowdata.process() + errors = [ + entry["message"] for entry in logger.memory_log + if entry["level"] == "error"] + return captured.variables, errors + class _Captured: """Records the variables of the last render and renders nothing.""" @@ -147,6 +181,56 @@ def test_url_keeps_a_query_string_given_in_the_endpoint(self): self.assertEqual( "https://example.com/json?tenant=1", variables["_url"]) + def test_refused_url_is_left_out_and_logged_once(self): + """A host the check refuses gives a script with no callback, and the + refusal is logged as an error the first time only, whatever the + requests in between.""" + + rendered, errors = _render_with_hosts( + ["exa mple.com", "exa mple.com", "example.com", "exa mple.com"]) + + for i, variables in enumerate(rendered): + if i == 2: + self.assertEqual( + "https://example.com/json", variables["_url"]) + self.assertTrue(variables["_updateEnabled"]) + else: + self.assertNotIn("_url", variables) + self.assertFalse(variables["_updateEnabled"]) + self.assertEqual(1, len(errors), errors) + self.assertIn("'https://exa mple.com/json'", errors[0]) + + def test_refused_url_is_logged_on_one_line(self): + """The URL is quoted and escaped in the log, so a line break in it + cannot start a new log line.""" + + _, errors = _render_with_hosts(["a\nb.com\r\u2028"]) + + self.assertEqual(1, len(errors), errors) + for c in "\n\r\u2028": + self.assertNotIn(c, errors[0]) + self.assertIn("a\\nb.com\\r\\u2028", errors[0]) + + def test_blank_host_gives_no_callback_and_is_not_logged(self): + """A host made only of whitespace is not set, so the script has no + callback and nothing is logged.""" + + rendered, errors = _render_with_hosts([" "]) + + self.assertNotIn("_url", rendered[0]) + self.assertFalse(rendered[0]["_updateEnabled"]) + self.assertEqual([], errors) + + def test_host_from_the_request_is_written_as_checked(self): + """A host from the request reaches the script as the check writes + it, in lower case and with anything that could end the string the + script holds it in percent-encoded.""" + + rendered, _ = _render_with_hosts(["EXAMPLE.com/');alert(1);//"]) + + self.assertEqual( + "https://example.com/%27);alert(1);//json", rendered[0]["_url"]) + if __name__ == "__main__": unittest.main() From 85c7a5cd363f85c3669d713f4818afa2b7c01adf Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 21:00:37 +0000 Subject: [PATCH 11/68] Let FlowData and pipelines release native memory, and close them in the tests and examples (#36) --- .gitignore | 4 + AGENTS.md | 9 + ci/setup-environment.ps1 | 4 + ci/write-package-versions.ps1 | 28 +++ device-detection/ci/run-integration-tests.ps1 | 7 +- .../examples/gettingstarted.py | 61 +++--- .../performance-tests/test_performance.py | 31 +-- .../setup.py | 4 +- .../onpremise/datafileupdate_console.py | 92 ++++----- .../onpremise/failuretomatch.py | 61 +++--- .../onpremise/gettingstarted_console.py | 82 ++++---- .../onpremise/gettingstarted_web/app.py | 95 ++++----- .../onpremise/match_metrics.py | 148 +++++++------- .../onpremise/metadata_console.py | 10 +- .../onpremise/offlineprocessing.py | 141 +++++++------- .../onpremise/performance.py | 65 ++++--- .../onpremise/shareusage.py | 85 ++++---- .../onpremise/useragentclienthints_web/app.py | 49 ++--- .../tests/conftest.py | 119 ++++++++++++ .../tests/test_onpremisegettingstartedweb.py | 7 + .../fiftyone_devicedetection_examples/tox.ini | 6 + .../setup.py | 9 +- .../constants.py | 7 + .../devicedetection_onpremise.py | 17 ++ .../swig_data.py | 29 +++ .../tests/conftest.py | 119 ++++++++++++ .../tests/test_automatic_data_file_updates.py | 47 +++-- .../tests/test_devicedetection.py | 132 ++++++++++++- .../tests/test_properties.py | 7 + .../tests/test_swig.py | 4 + .../tox.ini | 6 + .../examples/gettingstarted.py | 59 +++--- .../fiftyone_ipintelligence_examples/setup.py | 2 + .../onpremise/compare_console.py | 117 +++++------ .../onpremise/countries_console.py | 69 ++++--- .../onpremise/datafileupdate_console.py | 62 +++--- .../onpremise/gettingstarted_console.py | 64 +++--- .../onpremise/gettingstarted_web/app.py | 84 ++++---- .../onpremise/metadata_console.py | 17 +- .../onpremise/mixed_console.py | 51 ++--- .../onpremise/offlineprocessing.py | 60 +++--- .../onpremise/performance_console.py | 25 ++- .../tests/conftest.py | 119 ++++++++++++ .../tests/test_examples.py | 2 + .../fiftyone_ipintelligence_examples/tox.ini | 7 + .../setup.py | 9 +- .../constants.py | 7 + .../ipintelligence_onpremise.py | 17 ++ .../swig_data.py | 29 +++ .../tests/conftest.py | 119 ++++++++++++ .../tests/test_ipintelligence.py | 53 +++++ .../fiftyone_ipintelligence_onpremise/tox.ini | 6 + .../src/fiftyone_pipeline_core/flowdata.py | 28 +++ .../src/fiftyone_pipeline_core/flowelement.py | 11 ++ .../src/fiftyone_pipeline_core/pipeline.py | 48 ++++- .../fiftyone_pipeline_core/pipelinebuilder.py | 63 ++++-- .../tests/test_close.py | 182 ++++++++++++++++++ .../datafile_update_service.py | 48 ++++- .../src/fiftyone_pipeline_engines/engine.py | 16 ++ .../tests/test_datafile_update_service.py | 39 ++++ .../share_usage.py | 11 ++ 61 files changed, 2148 insertions(+), 761 deletions(-) create mode 100644 ci/write-package-versions.ps1 create mode 100644 device-detection/fiftyone_devicedetection_examples/tests/conftest.py create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_close.py diff --git a/.gitignore b/.gitignore index 7c20e4e1..21cc57bb 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,7 @@ __pycache__/ .venv/ build/ dist/ + +# The version each package's setup.py reads, written by the publish and by +# ci/write-package-versions.ps1. +version.txt diff --git a/AGENTS.md b/AGENTS.md index 539b95af..81f3a40b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -285,6 +285,15 @@ installed the pipeline packages from PyPI. In CI the root `ci/setup-environment.ps1` installs them from `pipeline/` over those, so a pipeline change is tested with every product in the same run. +A package's `setup.py` reads its version from a `version.txt`, which only +`pipeline/fiftyone_pipeline_did` commits, so any other package installed from +a checkout is 0.0.0. That is below the minimum versions of the pipeline +packages that the on-premise and examples packages require, so +`ci/write-package-versions.ps1` gives each published package the version this +commit would release it at. The root `ci/setup-environment.ps1` runs it +first. Run `pwsh ci/write-package-versions.ps1` before running those +packages' tests from a checkout. + ## Submodules The C/C++ engines are the `cxx` submodule, listed in the root `.gitmodules`. diff --git a/ci/setup-environment.ps1 b/ci/setup-environment.ps1 index 9994e2ef..932f5d81 100644 --- a/ci/setup-environment.ps1 +++ b/ci/setup-environment.ps1 @@ -12,6 +12,10 @@ if ($env:GITHUB_JOB -eq "PreBuild") { exit 0 } +# Before anything installs a package from this checkout, so that each one +# installs at the version this commit would release it at. +& (Join-Path $PSScriptRoot 'write-package-versions.ps1') -RepoName $RepoName + Invoke-ProductStep 'setup-environment' $RepoName $PSBoundParameters # The device detection, IP intelligence and location scripts install the diff --git a/ci/write-package-versions.ps1 b/ci/write-package-versions.ps1 new file mode 100644 index 00000000..44b23f25 --- /dev/null +++ b/ci/write-package-versions.ps1 @@ -0,0 +1,28 @@ +param ( + [string]$RepoName = '.' +) +$ErrorActionPreference = "Stop" +. (Join-Path $PSScriptRoot 'packages.ps1') + +# Writes the version.txt that each published package's setup.py reads. Without +# one a package installed from this checkout is 0.0.0, which is below the +# versions the packages here require of each other, so pip refuses it, or +# replaces it with a release from PyPI. Each package gets the version this +# commit would release it at, as the publish gives it when it builds the +# package, or its last release when this commit would not release it. +$plan = @(Get-ReleasePlan $RepoName) +$published = @(Get-Packages $RepoName | Where-Object { $_.Publish -eq $true }) +foreach ($package in $published) { + # A version.txt committed with the package is left as it is. + if (git -C $RepoName ls-files -- "$($package.Path)/version.txt") { + continue + } + $release = $plan | Where-Object { $_.Name -eq $package.Name } + if ($null -ne $release) { + $version = $release.Version + } else { + $version = "$((Get-ReleasedVersions $RepoName $package.Name)[-1])" + } + $version | Out-File (Join-Path $RepoName $package.Path 'version.txt') + Write-Output "$($package.Name) $version" +} diff --git a/device-detection/ci/run-integration-tests.ps1 b/device-detection/ci/run-integration-tests.ps1 index 9aa2cf2f..f1483528 100644 --- a/device-detection/ci/run-integration-tests.ps1 +++ b/device-detection/ci/run-integration-tests.ps1 @@ -67,7 +67,12 @@ Push-Location $examplesDir try { python3 -m venv .virtualenv if ($LASTEXITCODE -ne 0) { throw "failed to create the examples virtual environment" } - & $py -m pip install -e . + # The pipeline packages come from this checkout rather than a package + # index, so the examples run with the pipeline code they are released with. + $pipeline = 'fiftyone_pipeline_core', 'fiftyone_pipeline_engines', + 'fiftyone_pipeline_engines_fiftyone', 'fiftyone_pipeline_cloudrequestengine' + $local = $pipeline | ForEach-Object { '-e'; "../../pipeline/$_" } + & $py -m pip install -e . @local if ($LASTEXITCODE -ne 0) { throw "failed to install the examples package" } } finally { Pop-Location } diff --git a/device-detection/fiftyone_devicedetection/examples/gettingstarted.py b/device-detection/fiftyone_devicedetection/examples/gettingstarted.py index cf3a7b7f..b439c3d3 100644 --- a/device-detection/fiftyone_devicedetection/examples/gettingstarted.py +++ b/device-detection/fiftyone_devicedetection/examples/gettingstarted.py @@ -35,46 +35,49 @@ # Cloud #pipeline = DeviceDetectionPipelineBuilder({"resource_key": resource_key}).build() -# We create a FlowData object from the pipeline -# this is used to add evidence to and then process +# The pipeline is closed at the end of the with block, which releases the +# native data it holds. +with pipeline: + # We create a FlowData object from the pipeline + # this is used to add evidence to and then process. It is closed at the + # end of its with block, which releases the results of the detection. + with pipeline.create_flowdata() as flowdata1: -flowdata1 = pipeline.create_flowdata() + # Here we add a User-Agent of a desktop as evidence -# Here we add a User-Agent of a desktop as evidence + desktop_ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" -desktop_ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" + flowdata1.evidence.add("header.user-agent", desktop_ua) -flowdata1.evidence.add("header.user-agent", desktop_ua) + # Now we process the FlowData -# Now we process the FlowData + flowdata1.process() -flowdata1.process() + # To check whether the User-Agent is a mobile device we look at the ismobile property + # inside the Device Detection Engine -# To check whether the User-Agent is a mobile device we look at the ismobile property -# inside the Device Detection Engine + # first we check if this has a meaningful result -# first we check if this has a meaningful result + print("Is User-Agent " + desktop_ua + " a mobile? ") + if flowdata1.device.ismobile.has_value(): + print(flowdata1.device.ismobile.value()) + else: + # Output why the value isn't meaningful + print(flowdata1.device.ismobile.no_value_message()) -print("Is User-Agent " + desktop_ua + " a mobile? ") -if flowdata1.device.ismobile.has_value(): - print(flowdata1.device.ismobile.value()) -else: - # Output why the value isn't meaningful - print(flowdata1.device.ismobile.no_value_message()) + # Now we do the same with a new User-Agent, this time from an iphone -# Now we do the same with a new User-Agent, this time from an iphone + with pipeline.create_flowdata() as flowdata2: -flowdata2 = pipeline.create_flowdata() + iphone_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" -iphone_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" + flowdata2.evidence.add("header.user-agent", iphone_ua) -flowdata2.evidence.add("header.user-agent", iphone_ua) + flowdata2.process() -flowdata2.process() - -print("Is User-Agent " + iphone_ua + " a mobile device?: ") -if flowdata2.device.ismobile.has_value(): - print(flowdata2.device.ismobile.value()) -else: - # Output why the value isn't meaningful - print(flowdata2.device.ismobile.no_value_message()) + print("Is User-Agent " + iphone_ua + " a mobile device?: ") + if flowdata2.device.ismobile.has_value(): + print(flowdata2.device.ismobile.value()) + else: + # Output why the value isn't meaningful + print(flowdata2.device.ismobile.no_value_message()) diff --git a/device-detection/fiftyone_devicedetection_examples/performance-tests/test_performance.py b/device-detection/fiftyone_devicedetection_examples/performance-tests/test_performance.py index b7064b14..a8de1b36 100644 --- a/device-detection/fiftyone_devicedetection_examples/performance-tests/test_performance.py +++ b/device-detection/fiftyone_devicedetection_examples/performance-tests/test_performance.py @@ -48,16 +48,18 @@ def detect(app, pipeline, user_agents): # in flowdata.process(). This mock-request doesn't set it, so we set it manually request.remote_addr = "127.0.0.1" - flowdata = pipeline.create_flowdata() + # The FlowData is closed when the function returns, which releases + # the results of the detection straight away. + with pipeline.create_flowdata() as flowdata: - # Add any information from the request (headers, cookies and additional - # client side provided information) - flowdata.evidence.add_from_dict(webevidence(request)) + # Add any information from the request (headers, cookies and additional + # client side provided information) + flowdata.evidence.add_from_dict(webevidence(request)) - # Process the flowdata - flowdata.process() + # Process the flowdata + flowdata.process() - return getValueHelper(flowdata, "device", "ismobile") + return getValueHelper(flowdata, "device", "ismobile") def benchmark(data_file, user_agents_file, output=False): app = Flask(__name__) @@ -73,13 +75,16 @@ def benchmark(data_file, user_agents_file, output=False): settings = { "add_javascript_builder": False }, ).build() - with open(user_agents_file) as f: - user_agents_list = [ua.rstrip() for ua in f] - user_agents = iter(user_agents_list) + # The pipeline is closed once the detections have been timed, which + # releases the native data it holds. + with pipeline: + with open(user_agents_file) as f: + user_agents_list = [ua.rstrip() for ua in f] + user_agents = iter(user_agents_list) - print(f"Benchmarking with {len(user_agents_list)} user agents", file=sys.stderr) - environment = {"detect": detect, "app": app, "pipeline": pipeline, "user_agents": user_agents} - time = timeit("detect(app, pipeline, user_agents)", globals=environment, number=len(user_agents_list)) + print(f"Benchmarking with {len(user_agents_list)} user agents", file=sys.stderr) + environment = {"detect": detect, "app": app, "pipeline": pipeline, "user_agents": user_agents} + time = timeit("detect(app, pipeline, user_agents)", globals=environment, number=len(user_agents_list)) if output: json.dump({ diff --git a/device-detection/fiftyone_devicedetection_examples/setup.py b/device-detection/fiftyone_devicedetection_examples/setup.py index 15fa344c..e5ab4dbf 100644 --- a/device-detection/fiftyone_devicedetection_examples/setup.py +++ b/device-detection/fiftyone_devicedetection_examples/setup.py @@ -65,7 +65,9 @@ def read(file_name): # named here so that is visible to anyone reading them. # fiftyone_devicedetection is still required because the on-premise # examples in this same package use the on-premise engine. - install_requires=["fiftyone_devicedetection_cloud", "fiftyone_devicedetection", "flask", "flask-unittest", "json5", "ruamel.yaml"], + # fiftyone_pipeline_core 4.5.42 is the first version with close(), which + # the examples use. + install_requires=["fiftyone_devicedetection_cloud", "fiftyone_devicedetection", "fiftyone_pipeline_core>=4.5.42", "flask", "flask-unittest", "json5", "ruamel.yaml"], license="EUPL-1.2", classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index d5fd2228..4fd126fc 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -234,15 +234,18 @@ def run(self, data_file, license_key, interactive, logger, output): auto_update = False, licence_keys = "").add_logger(logger).build() - # and output the results - ExampleUtils.check_data_file(pipeline, logger) - if (ExampleUtils.get_data_file_tier(pipeline.get_element("device")) == "Lite"): - logger.log("error", - "Will not download an 'Enterprise' data file over the top of " - "a 'Lite' data file, please supply another location.") - raise Exception("File supplied has wrong data tier") - logger.log("info", "Existing data file will be replaced with downloaded data file") - logger.log("info", f"Existing data file will be copied to {copy_data_file_name}") + # The pipeline is closed at the end of the with block, which + # releases the native data it holds before the file is replaced. + with pipeline: + # and output the results + ExampleUtils.check_data_file(pipeline, logger) + if (ExampleUtils.get_data_file_tier(pipeline.get_element("device")) == "Lite"): + logger.log("error", + "Will not download an 'Enterprise' data file over the top of " + "a 'Lite' data file, please supply another location.") + raise Exception("File supplied has wrong data tier") + logger.log("info", "Existing data file will be replaced with downloaded data file") + logger.log("info", f"Existing data file will be copied to {copy_data_file_name}") # do we really want to do this if (interactive): @@ -295,41 +298,44 @@ def run(self, data_file, license_key, interactive, logger, output): data_update_product_type="V4Enterprise", ).add_logger(logger).build() - - # thread blocks till update checking is complete - or if there is an - # exception we don't get this far - update_event.wait() - output(f"Update on start-up complete - status - {update_event.status}") - - if update_event.status == UpdateStatus.AUTO_UPDATE_SUCCESS: - - output("Modifying downloaded file to trigger reload - please wait for that" - " to complete") - - # wait for the dataUpdateService to notify us that it has updated - update_event.clear() - - # it's the same file but changing the file metadata will trigger reload, - # demonstrating that if you download a new file and replace the - # existing one, then it will be loaded - now = datetime.now().timestamp() - try: - os.utime(data_file, (now, now)) - except: - raise Exception("Could not modify file time, abandoning " - "example") - - if update_event.wait(120): - output(f"Update on file modification complete, status: {update_event.status}") - else: - output("Update on file modification timed out") - else: - logger.log("error", "Auto update was not successful, abandoning example") - error_message = f"Auto update failed: {update_event.status}" - if update_event.status == UpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS: - output(error_message) + # The pipeline is closed at the end of the with block, which stops + # the automatic updates and the file system watcher, and releases + # the native data it holds. + with pipeline: + # thread blocks till update checking is complete - or if there is an + # exception we don't get this far + update_event.wait() + output(f"Update on start-up complete - status - {update_event.status}") + + if update_event.status == UpdateStatus.AUTO_UPDATE_SUCCESS: + + output("Modifying downloaded file to trigger reload - please wait for that" + " to complete") + + # wait for the dataUpdateService to notify us that it has updated + update_event.clear() + + # it's the same file but changing the file metadata will trigger reload, + # demonstrating that if you download a new file and replace the + # existing one, then it will be loaded + now = datetime.now().timestamp() + try: + os.utime(data_file, (now, now)) + except: + raise Exception("Could not modify file time, abandoning " + "example") + + if update_event.wait(120): + output(f"Update on file modification complete, status: {update_event.status}") + else: + output("Update on file modification timed out") else: - raise Exception(error_message) + logger.log("error", "Auto update was not successful, abandoning example") + error_message = f"Auto update failed: {update_event.status}" + if update_event.status == UpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS: + output(error_message) + else: + raise Exception(error_message) output("Finished Example") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/failuretomatch.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/failuretomatch.py index f640b1d8..9db313bd 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/failuretomatch.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/failuretomatch.py @@ -33,46 +33,49 @@ performance_profile = 'MaxPerformance', auto_update=False).build() -# We create a FlowData object from the pipeline -# this is used to add evidence to and then process +# The pipeline is closed at the end of the with block, which releases the +# native data it holds. +with pipeline: + # We create a FlowData object from the pipeline + # this is used to add evidence to and then process. It is closed at the + # end of its with block, which releases the results of the detection. + with pipeline.create_flowdata() as flowdata1: -flowdata1 = pipeline.create_flowdata() + # Here we add a User-Agent of an iphone as evidence -# Here we add a User-Agent of an iphone as evidence + iphone_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" -iphone_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" + flowdata1.evidence.add("header.user-agent", iphone_ua) -flowdata1.evidence.add("header.user-agent", iphone_ua) + # Now we process the FlowData -# Now we process the FlowData + flowdata1.process() -flowdata1.process() + # To check whether the User-Agent is a mobile device we look at the ismobile property + # inside the Device Detection Engine -# To check whether the User-Agent is a mobile device we look at the ismobile property -# inside the Device Detection Engine + # first we check if this has a meaningful result -# first we check if this has a meaningful result + print("Is User-Agent " + iphone_ua + " a mobile device?: ") + if flowdata1.device.ismobile.has_value(): + print(flowdata1.device.ismobile.value()) + else: + # Output why the value isn't meaningful + print(flowdata1.device.ismobile.no_value_message()) -print("Is User-Agent " + iphone_ua + " a mobile device?: ") -if flowdata1.device.ismobile.has_value(): - print(flowdata1.device.ismobile.value()) -else: - # Output why the value isn't meaningful - print(flowdata1.device.ismobile.no_value_message()) + # Now we do the same with a new User-Agent, this time a corrupted one -# Now we do the same with a new User-Agent, this time a corrupted one + badUA = "--" -badUA = "--" + with pipeline.create_flowdata() as flowdata2: -flowdata2 = pipeline.create_flowdata() + flowdata2.evidence.add("header.user-agent", badUA) -flowdata2.evidence.add("header.user-agent", badUA) + flowdata2.process() -flowdata2.process() - -print("Is User-Agent " + badUA + " a mobile device?: ") -if flowdata2.device.ismobile.has_value(): - print(flowdata2.device.ismobile.value()) -else: - # Output why the value isn't meaningful - print(flowdata2.device.ismobile.no_value_message()) + print("Is User-Agent " + badUA + " a mobile device?: ") + if flowdata2.device.ismobile.has_value(): + print(flowdata2.device.ismobile.value()) + else: + # Output why the value isn't meaningful + print(flowdata2.device.ismobile.no_value_message()) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index 946f06bd..3675fee2 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -68,11 +68,16 @@ def run(self, data_file, logger, output): auto_update = False, licence_keys = "").add_logger(logger).build() - ExampleUtils.check_data_file(pipeline, logger) + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. An application normally builds + # one pipeline when it starts, uses it for every detection, and + # closes it when it stops. + with pipeline: + ExampleUtils.check_data_file(pipeline, logger) - # carry out some sample detections - for values in EVIDENCE_VALUES: - self.analyseEvidence(values, pipeline, output) + # carry out some sample detections + for values in EVIDENCE_VALUES: + self.analyseEvidence(values, pipeline, output) def analyseEvidence(self, evidence, pipeline, output): @@ -83,42 +88,45 @@ def analyseEvidence(self, evidence, pipeline, output): # and usually consists of a number of HTTP Header field # values, in this case represented by a dictionary of header # name/value entries. - data = pipeline.create_flowdata() + # Closing the FlowData, which the with statement does at the end of + # the block, releases the results of the detection straight away, + # rather than when the garbage collector next runs. + with pipeline.create_flowdata() as data: - message = [] + message = [] - # List the evidence - message.append("Input values:\n") - for key in evidence: - message.append(f"\t{key}: {evidence[key]}\n") + # List the evidence + message.append("Input values:\n") + for key in evidence: + message.append(f"\t{key}: {evidence[key]}\n") - output("".join(message)) - - # Add the evidence values to the flow data - data.evidence.add_from_dict(evidence) - - # Process the flow data. - data.process() - - message = [] - message.append("Results:\n") - - # Now that it's been processed, the flow data will have - # been populated with the result. In this case, we want - # information about the device, which we can get by - # asking for a result matching named "device" - device = data.device - - # Display the results of the detection, which are called - # device properties. See the property dictionary at - # https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=analyseevidence - # for details of all available properties. - self.outputValue("Mobile Device", device.ismobile, message) - self.outputValue("Platform Name", device.platformname, message) - self.outputValue("Platform Version", device.platformversion, message) - self.outputValue("Browser Name", device.browsername, message) - self.outputValue("Browser Version", device.browserversion, message) - output("".join(message)) + output("".join(message)) + + # Add the evidence values to the flow data + data.evidence.add_from_dict(evidence) + + # Process the flow data. + data.process() + + message = [] + message.append("Results:\n") + + # Now that it's been processed, the flow data will have + # been populated with the result. In this case, we want + # information about the device, which we can get by + # asking for a result matching named "device" + device = data.device + + # Display the results of the detection, which are called + # device properties. See the property dictionary at + # https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=analyseevidence + # for details of all available properties. + self.outputValue("Mobile Device", device.ismobile, message) + self.outputValue("Platform Name", device.platformname, message) + self.outputValue("Platform Version", device.platformversion, message) + self.outputValue("Browser Name", device.browsername, message) + self.outputValue("Browser Version", device.browserversion, message) + output("".join(message)) def outputValue(self, name, value, message): # Individual result values have a wrapper called diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index d382e6f2..a8e1e3ce 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -102,7 +102,10 @@ def run(self): # The port can be set with the PORT environment variable, as it can # for the cloud example, so the example can run next to other services. - GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5001))) + # The pipeline lives as long as the app, and is closed when the server + # stops, which releases the native data it holds. + with GettingStartedWeb.pipeline: + GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5001))) # First we make a JSON route that will be called from the client side and will return # a JSON encoded property database using any additional evidence provided by the client @@ -111,21 +114,23 @@ def run(self): @app.route('/json', methods=['POST']) def jsonroute(): - # Create the flowdata object for the JSON route - flowdata = GettingStartedWeb.pipeline.create_flowdata() + # Create the flowdata object for the JSON route. It is closed when the + # route returns, which releases the results of the detection straight + # away rather than when the garbage collector next runs. + with GettingStartedWeb.pipeline.create_flowdata() as flowdata: - # Add any information from the request (headers, cookies and additional - # client side provided information) + # Add any information from the request (headers, cookies and additional + # client side provided information) - flowdata.evidence.add_from_dict(webevidence(request)) + flowdata.evidence.add_from_dict(webevidence(request)) - # Process the flowdata + # Process the flowdata - flowdata.process() + flowdata.process() - # Return the JSON from the JSONBundler engine + # Return the JSON from the JSONBundler engine - return json.dumps(flowdata.jsonbundler.json) + return json.dumps(flowdata.jsonbundler.json) # Next we serve the client-side JavaScript from its own route. The other # Pipeline APIs ship a web integration that intercepts '/51Degrees.core.js' @@ -136,26 +141,27 @@ def jsonroute(): @app.route('/51Degrees.core.js') def core_js(): - # Create the flowdata object for the JavaScript route - flowdata = GettingStartedWeb.pipeline.create_flowdata() + # Create the flowdata object for the JavaScript route. It is closed + # when the route returns. + with GettingStartedWeb.pipeline.create_flowdata() as flowdata: - # Add any information from the request (headers, cookies and additional - # client side provided information). Query parameters are included, so - # the per-request 'fod-js-enable-cookies' override is picked up here and - # applied by the JavaScriptBuilder engine. + # Add any information from the request (headers, cookies and additional + # client side provided information). Query parameters are included, so + # the per-request 'fod-js-enable-cookies' override is picked up here and + # applied by the JavaScriptBuilder engine. - flowdata.evidence.add_from_dict(webevidence(request)) + flowdata.evidence.add_from_dict(webevidence(request)) - # Process the flowdata + # Process the flowdata - flowdata.process() + flowdata.process() - # Return the JavaScript from the JavaScriptBuilder engine + # Return the JavaScript from the JavaScriptBuilder engine - response = make_response(flowdata.javascriptbuilder.javascript) - response.headers["Content-Type"] = "application/x-javascript" + response = make_response(flowdata.javascriptbuilder.javascript) + response.headers["Content-Type"] = "application/x-javascript" - return response + return response # In the main route we dynamically update the screen's device property display # using the above JSON route @@ -164,36 +170,37 @@ def core_js(): @app.route('/') def server(): - # Create the flowdata object for the JSON route - flowdata = GettingStartedWeb.pipeline.create_flowdata() + # Create the flowdata object for the page. It is closed when the route + # returns, once the page has been rendered from it. + with GettingStartedWeb.pipeline.create_flowdata() as flowdata: - # Add any information from the request (headers, cookies and additional - # client side provided information) + # Add any information from the request (headers, cookies and additional + # client side provided information) - flowdata.evidence.add_from_dict(webevidence(request)) + flowdata.evidence.add_from_dict(webevidence(request)) - # Process the flowdata + # Process the flowdata - flowdata.process() + flowdata.process() - response = make_response() + response = make_response() - # Some browsers require that extra HTTP headers are explicitly - # requested. So set whatever headers are required by the browser in - # order to return the evidence needed by the pipeline. - # More info on this can be found at - # https://51degrees.com/blog/user-agent-client-hints?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_web-app.py&utm_term=server + # Some browsers require that extra HTTP headers are explicitly + # requested. So set whatever headers are required by the browser in + # order to return the evidence needed by the pipeline. + # More info on this can be found at + # https://51degrees.com/blog/user-agent-client-hints?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_web-app.py&utm_term=server - set_response_header(flowdata, response) + set_response_header(flowdata, response) - # Generate the HTML - response.set_data(render_template( - 'index.html', - data=flowdata, - utils=ExampleUtils, - response=response)) + # Generate the HTML + response.set_data(render_template( + 'index.html', + data=flowdata, + utils=ExampleUtils, + response=response)) - return response + return response # Typically, something like this will not be necessary. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py index 5daeb284..6a223d00 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py @@ -88,83 +88,89 @@ def run(self, data_file, show_descs, logger, output): # We want to show the matching evidence characters as part of this example, so we have to set # this flag to true. update_matched_useragent = True).add_logger(logger).build() + + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + ExampleUtils.check_data_file(pipeline, logger) + + # Closing the FlowData, which the with statement does at the end + # of the block, releases the results of the detection straight + # away, rather than when the garbage collector next runs. + with pipeline.create_flowdata() as data: + + # Process a single evidence to retrieve the values + # associated with the user-agent and other evidence such as sec-ch-* for the + # selected properties. + data.evidence.add_from_dict(self.Evidence) + data.process() + + device = data.device + + output("--- Compare evidence with what was matched ---\n") + output("Evidence") + # output the evidence in reverse value length order + for entry in sorted(self.Evidence.items(), key=lambda item: len(item[1]), reverse=True): + output(f" {entry[0]}: {entry[1]}") + # Obtain the matched User-Agents: the matched substrings in the + # User-Agents are separated with underscores - output in forward length order. + output("Matches") + for entry in sorted(device.useragents.value(), key=lambda item: len(item)): + output(f" Matched User-Agent: {entry}") + + output("") + + + output("--- Listing all available properties, by component, by property name ---") + output("For a discussion of what the match properties mean, see: https://51degrees.com/documentation/_device_detection__hash.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=match-properties-discussion#DeviceDetection_Hash_DataSetProduction_Performance\n") + + # get the properties available from the DeviceDetection engine + # which has the key "device". For the sake of illustration we will + # retrieve it indirectly. + hashEngineElementKey = pipeline.get_element("device").datakey + + # retrieve the available properties from the hash engine. The properties + # available depends on + # a) the use of restricted_properties in the builder (see above) + # which controls which properties will be extracted, and also affects + # the performance of extraction + # b) the tier of data file being used. The Lite data file contains fewer + # than 20 of the >200 available properties + availableProperties = dict(pipeline.get_properties()[hashEngineElementKey]) - ExampleUtils.check_data_file(pipeline, logger) - - data = pipeline.create_flowdata() - - # Process a single evidence to retrieve the values - # associated with the user-agent and other evidence such as sec-ch-* for the - # selected properties. - data.evidence.add_from_dict(self.Evidence) - data.process() - - device = data.device - - output("--- Compare evidence with what was matched ---\n") - output("Evidence") - # output the evidence in reverse value length order - for entry in sorted(self.Evidence.items(), key=lambda item: len(item[1]), reverse=True): - output(f" {entry[0]}: {entry[1]}") - # Obtain the matched User-Agents: the matched substrings in the - # User-Agents are separated with underscores - output in forward length order. - output("Matches") - for entry in sorted(device.useragents.value(), key=lambda item: len(item)): - output(f" Matched User-Agent: {entry}") - - output("") - - - output("--- Listing all available properties, by component, by property name ---") - output("For a discussion of what the match properties mean, see: https://51degrees.com/documentation/_device_detection__hash.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=match-properties-discussion#DeviceDetection_Hash_DataSetProduction_Performance\n") - - # get the properties available from the DeviceDetection engine - # which has the key "device". For the sake of illustration we will - # retrieve it indirectly. - hashEngineElementKey = pipeline.get_element("device").datakey - - # retrieve the available properties from the hash engine. The properties - # available depends on - # a) the use of restricted_properties in the builder (see above) - # which controls which properties will be extracted, and also affects - # the performance of extraction - # b) the tier of data file being used. The Lite data file contains fewer - # than 20 of the >200 available properties - availableProperties = dict(pipeline.get_properties()[hashEngineElementKey]) - - # create a Map keyed on the component name of the properties available - # components being hardware, browser, OS and Crawler. - def get_component(property): - return property[1]["component"] - - availableProperties = dict(sorted(availableProperties.items(),key=lambda item: get_component(item))) - categoryMap = groupby(availableProperties.items(), get_component) - # iterate the map created above - for component, properties in categoryMap: - output(component) - for propertyKey, property in properties: - propertyName = property["name"] - propertyDescription = property["description"] + # create a Map keyed on the component name of the properties available + # components being hardware, browser, OS and Crawler. + def get_component(property): + return property[1]["component"] + + availableProperties = dict(sorted(availableProperties.items(),key=lambda item: get_component(item))) + categoryMap = groupby(availableProperties.items(), get_component) + # iterate the map created above + for component, properties in categoryMap: + output(component) + for propertyKey, property in properties: + propertyName = property["name"] + propertyDescription = property["description"] - # while we get the available properties and their metadata from the - # pipeline we get the values for the last detection from flowData - value = device[propertyKey] + # while we get the available properties and their metadata from the + # pipeline we get the values for the last detection from flowData + value = device[propertyKey] - # output property names, values and descriptions - # some property values are lists. - if value.has_value() and isinstance(value.value(), list): - output(f" {propertyName}: {len(value.value())} Values") - for item in value.value(): - output(f" : {item}") + # output property names, values and descriptions + # some property values are lists. + if value.has_value() and isinstance(value.value(), list): + output(f" {propertyName}: {len(value.value())} Values") + for item in value.value(): + output(f" : {item}") - else: - output(f" {propertyName}: {value.value()}") + else: + output(f" {propertyName}: {value.value()}") - if (show_descs == True): - output(f" {propertyDescription}") + if (show_descs == True): + output(f" {propertyDescription}") - output() - logger.log("info", "Finished Match Metrics Example") + output() + logger.log("info", "Finished Match Metrics Example") # Evidence values from a windows 11 device using a browser # that supports User-Agent Client Hints. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py index 2df9d104..5e605158 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py @@ -76,10 +76,14 @@ def run(self, data_file, logger, output): # Inhibit auto-update of the data file for this example auto_update = False, licence_keys = "").add_logger(logger).build() - self.outputProperties(pipeline.get_element("device"), output) - self.outputEvidenceKeyDetails(pipeline.get_element("device"), output) - ExampleUtils.check_data_file(pipeline, logger) + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + self.outputProperties(pipeline.get_element("device"), output) + self.outputEvidenceKeyDetails(pipeline.get_element("device"), output) + + ExampleUtils.check_data_file(pipeline, logger) def outputEvidenceKeyDetails(self, engine, output): diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py index 7a90c401..500fdba7 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py @@ -99,40 +99,43 @@ def run(self, data_file, evidence_yaml, logger, output): auto_update = False, licence_keys = "").add_logger(logger).build() - records = 0 - yaml = YAML() - # Replace tab characters as they are not valid in unquoted YAML values - # and some evidence files contain them in user-agent strings. - content = evidence_yaml.read().replace('\t', ' ') - yaml_data = yaml.load_all(content) + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + records = 0 + yaml = YAML() + # Replace tab characters as they are not valid in unquoted YAML values + # and some evidence files contain them in user-agent strings. + content = evidence_yaml.read().replace('\t', ' ') + yaml_data = yaml.load_all(content) - try: - # Keep going as long as we have more document records. - for evidence in yaml_data: - # Output progress. - records = records + 1 - if (records % 100 == 0): - logger.log("info", f"Processed {records} records") + try: + # Keep going as long as we have more document records. + for evidence in yaml_data: + # Output progress. + records = records + 1 + if (records % 100 == 0): + logger.log("info", f"Processed {records} records") - # write the yaml document separator - print("---", file = output) - # Pass the record to the pipeline as evidence so that it can be analyzed - headers = {} - for key in evidence: - # Convert to string - YAML may parse values like ?1 as booleans - # Keys already have header. prefix in the YAML file - headers[key] = str(evidence[key]) + # write the yaml document separator + print("---", file = output) + # Pass the record to the pipeline as evidence so that it can be analyzed + headers = {} + for key in evidence: + # Convert to string - YAML may parse values like ?1 as booleans + # Keys already have header. prefix in the YAML file + headers[key] = str(evidence[key]) - self.analyseEvidence(headers, pipeline, output, yaml) - except BaseException as err: - # We can't read the evidence values, so cant write them to the output. Will just - # have to skip this entry. - logger.log("error", err) + self.analyseEvidence(headers, pipeline, output, yaml) + except BaseException as err: + # We can't read the evidence values, so cant write them to the output. Will just + # have to skip this entry. + logger.log("error", err) - # write the yaml document end marker - print("...", file = output) + # write the yaml document end marker + print("...", file = output) - ExampleUtils.check_data_file(pipeline, logger) + ExampleUtils.check_data_file(pipeline, logger) def analyseEvidence(self, evidence, pipeline, output, yaml): # FlowData is a data structure that is used to convey information required for @@ -140,46 +143,50 @@ def analyseEvidence(self, evidence, pipeline, output, yaml): # Information required for detection is called "evidence" and usually consists # of a number of HTTP Header field values, in this case represented by a # dictionary of header name/value entries. - data = pipeline.create_flowdata() - # Add the evidence values to the flow data - data.evidence.add_from_dict(evidence) - # Process the flow data. - data.process() + # Closing the FlowData, which the with statement does at the end of + # the block, releases the results of the detection straight away. + # Without it each record's results would stay in memory until the + # garbage collector next runs. + with pipeline.create_flowdata() as data: + # Add the evidence values to the flow data + data.evidence.add_from_dict(evidence) + # Process the flow data. + data.process() - device = data.device + device = data.device - values = {} - # Add the evidence values to the output - for key in evidence: - values[key] = evidence[key] - # Now add the values that we want to store against the record. - values["device.ismobile"] = device.ismobile.value() if device.ismobile.has_value() else "Unknown" - values["device.platformname"] = ExampleUtils.get_human_readable(device, "platformname") - values["device.platformversion"] = ExampleUtils.get_human_readable(device, "platformversion") - values["device.browsername"] = ExampleUtils.get_human_readable(device, "browsername") - values["device.browserversion"] = ExampleUtils.get_human_readable(device, "browserversion") - values["device.tac"] = ExampleUtils.get_human_readable(device, "tac") - values["device.devicetype"] = ExampleUtils.get_human_readable(device, "devicetype") - values["device.hardwarevendor"] = ExampleUtils.get_human_readable(device, "hardwarevendor") - values["device.hardwarename"] = ExampleUtils.get_human_readable(device, "hardwarename") - values["device.hardwaremodel"] = ExampleUtils.get_human_readable(device, "hardwaremodel") - values["device.nativedevice"] = ExampleUtils.get_human_readable(device, "nativedevice") - values["device.nativename"] = ExampleUtils.get_human_readable(device, "nativename") - values["device.nativemodel"] = ExampleUtils.get_human_readable(device, "nativemodel") - # DeviceId is a unique identifier for the combination of hardware, operating - # system, browser and crawler that has been detected. - # Our device detection solution uses machine learning to find the optimal - # way to identify devices based on the real-world evidence values that we - # observe each day. - # As this changes over time, the result of detection can potentially change - # as well. By storing the device id, we can use this as a lookup in future - # rather than performing detection with the original evidence again. - # Do this by passing an evidence entry with: - # key = query.51D_ProfileIds - # value = [the device id] - # This is much faster and avoids the potential for getting a different - # result. - values["device.deviceid"] = ExampleUtils.get_human_readable(device, "deviceid") + values = {} + # Add the evidence values to the output + for key in evidence: + values[key] = evidence[key] + # Now add the values that we want to store against the record. + values["device.ismobile"] = device.ismobile.value() if device.ismobile.has_value() else "Unknown" + values["device.platformname"] = ExampleUtils.get_human_readable(device, "platformname") + values["device.platformversion"] = ExampleUtils.get_human_readable(device, "platformversion") + values["device.browsername"] = ExampleUtils.get_human_readable(device, "browsername") + values["device.browserversion"] = ExampleUtils.get_human_readable(device, "browserversion") + values["device.tac"] = ExampleUtils.get_human_readable(device, "tac") + values["device.devicetype"] = ExampleUtils.get_human_readable(device, "devicetype") + values["device.hardwarevendor"] = ExampleUtils.get_human_readable(device, "hardwarevendor") + values["device.hardwarename"] = ExampleUtils.get_human_readable(device, "hardwarename") + values["device.hardwaremodel"] = ExampleUtils.get_human_readable(device, "hardwaremodel") + values["device.nativedevice"] = ExampleUtils.get_human_readable(device, "nativedevice") + values["device.nativename"] = ExampleUtils.get_human_readable(device, "nativename") + values["device.nativemodel"] = ExampleUtils.get_human_readable(device, "nativemodel") + # DeviceId is a unique identifier for the combination of hardware, operating + # system, browser and crawler that has been detected. + # Our device detection solution uses machine learning to find the optimal + # way to identify devices based on the real-world evidence values that we + # observe each day. + # As this changes over time, the result of detection can potentially change + # as well. By storing the device id, we can use this as a lookup in future + # rather than performing detection with the original evidence again. + # Do this by passing an evidence entry with: + # key = query.51D_ProfileIds + # value = [the device id] + # This is much faster and avoids the potential for getting a different + # result. + values["device.deviceid"] = ExampleUtils.get_human_readable(device, "deviceid") yaml.dump(values, output) def main(argv): diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py index 271ce134..d795b548 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py @@ -62,24 +62,26 @@ def process_user_agent(user_agent): - # First we create the flowdata using the global pipeline - flowdata = pipeline.create_flowdata() # pylint: disable=used-before-assignment + # First we create the flowdata using the global pipeline. Closing it, + # which the with statement does when the function returns, releases the + # results of the detection straight away. + with pipeline.create_flowdata() as flowdata: # pylint: disable=used-before-assignment - # Here we add the user agent as evidence - flowdata.evidence.add("header.user-agent", user_agent) + # Here we add the user agent as evidence + flowdata.evidence.add("header.user-agent", user_agent) - # We process the flowdata to get the results - flowdata.process() + # We process the flowdata to get the results + flowdata.process() - # To check whether the User-Agent is a mobile device we look at the ismobile - # property inside the Device Detection Engine + # To check whether the User-Agent is a mobile device we look at the ismobile + # property inside the Device Detection Engine - # first we check if this has a meaningful result + # first we check if this has a meaningful result - if flowdata.device.ismobile.has_value(): - return flowdata.device.ismobile.value() - else: - return None + if flowdata.device.ismobile.has_value(): + return flowdata.device.ismobile.value() + else: + return None def process_user_agent_list(data_file, user_agent_list, list_number, output, skip=False): global pipeline @@ -92,23 +94,26 @@ def process_user_agent_list(data_file, user_agent_list, list_number, output, ski usage_sharing=False, auto_update=False).build() - results = { - "mobile": 0, - "notmobile": 0, - "unknown": 0 - } - for user_agent in user_agent_list: - if skip: - break - result = process_user_agent(user_agent[0]) - if(result == None): - results["unknown"] += 1 - if(result == True): - results["mobile"] += 1 - if(result == False): - results["notmobile"] += 1 - - output.put(results, list_number) + # The pipeline is closed at the end of the with block, which releases + # the native data it holds. + with pipeline: + results = { + "mobile": 0, + "notmobile": 0, + "unknown": 0 + } + for user_agent in user_agent_list: + if skip: + break + result = process_user_agent(user_agent[0]) + if(result == None): + results["unknown"] += 1 + if(result == True): + results["mobile"] += 1 + if(result == False): + results["notmobile"] += 1 + + output.put(results, list_number) # Run the process def run(data_file, skip = False): diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py index 6cbbbfd7..51bcff0c 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py @@ -130,46 +130,51 @@ def run(self, evidence_yaml): # Build a pipeline with only the ShareUsage element pipeline = PipelineBuilder().add(share_usage_element).build() - print("Pipeline built successfully with ShareUsage element") - - # Load YAML evidence - yaml = YAML() - yaml_data = yaml.load_all(evidence_yaml) - - # Process evidence records - count = 0 - for evidence in yaml_data: - if count >= RECORDS_TO_PROCESS: - break - - # Filter to only header.* entries - data = filter_evidence( - {k: str(v) for k, v in evidence.items()}, - "header.") - - # Add a client IP address to the evidence for deduplication purposes - # (see class-level note). In a real web application, this would come - # from the request (e.g., REMOTE_ADDR or X-Forwarded-For header) - data[EVIDENCE_CLIENTIP_KEY] = generate_random_ip() - - # Add the usage-from identifier so 51Degrees knows the source of this data. - # This appears as
    YourCompanyName
    in the XML. - data[EVIDENCE_USAGE_FROM_KEY] = USAGE_FROM_VALUE - - # Create flow data and process - flow_data = pipeline.create_flowdata() - flow_data.evidence.add_from_dict(data) - flow_data.process() - - count += 1 - if count % 10 == 0: - print(f"Processed {count} records") - - print(f"Finished processing {count} records") - print("Waiting for share usage to complete sending...") - - # Give the background thread time to send any remaining data - time.sleep(5) + # The pipeline is closed at the end of the with block, which waits + # for any usage data being sent and then stops the threads that send + # it. + with pipeline: + print("Pipeline built successfully with ShareUsage element") + + # Load YAML evidence + yaml = YAML() + yaml_data = yaml.load_all(evidence_yaml) + + # Process evidence records + count = 0 + for evidence in yaml_data: + if count >= RECORDS_TO_PROCESS: + break + + # Filter to only header.* entries + data = filter_evidence( + {k: str(v) for k, v in evidence.items()}, + "header.") + + # Add a client IP address to the evidence for deduplication purposes + # (see class-level note). In a real web application, this would come + # from the request (e.g., REMOTE_ADDR or X-Forwarded-For header) + data[EVIDENCE_CLIENTIP_KEY] = generate_random_ip() + + # Add the usage-from identifier so 51Degrees knows the source of this data. + # This appears as
    YourCompanyName
    in the XML. + data[EVIDENCE_USAGE_FROM_KEY] = USAGE_FROM_VALUE + + # Create flow data and process. The with statement closes it + # once it has been processed. + with pipeline.create_flowdata() as flow_data: + flow_data.evidence.add_from_dict(data) + flow_data.process() + + count += 1 + if count % 10 == 0: + print(f"Processed {count} records") + + print(f"Finished processing {count} records") + print("Waiting for share usage to complete sending...") + + # Give the background thread time to send any remaining data + time.sleep(5) print("Done!") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py index 570ab57e..789c2d59 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py @@ -73,7 +73,10 @@ def build(self, config, logger): return self def run(self): - UserAgentClientHintsWeb.app.run(port=5001) + # The pipeline lives as long as the app, and is closed when the server + # stops, which releases the native data it holds. + with UserAgentClientHintsWeb.pipeline: + UserAgentClientHintsWeb.app.run(port=5001) # The main route performs device detection and renders the results. On the # second request, any additional client hints headers requested via the @@ -83,33 +86,35 @@ def run(self): @app.route('/') def server(): - # Create the flowdata object for the device detection - flowdata = UserAgentClientHintsWeb.pipeline.create_flowdata() + # Create the flowdata object for the device detection. It is closed + # when the route returns, once the page has been rendered from it, + # which releases the results of the detection straight away. + with UserAgentClientHintsWeb.pipeline.create_flowdata() as flowdata: - # Add any information from the request (headers, cookies and any other - # additional information) - flowdata.evidence.add_from_dict(webevidence(request)) + # Add any information from the request (headers, cookies and any other + # additional information) + flowdata.evidence.add_from_dict(webevidence(request)) - # Process the flowdata - flowdata.process() + # Process the flowdata + flowdata.process() - response = make_response() + response = make_response() - # Some browsers require that extra HTTP headers are explicitly - # requested. So set whatever headers are required by the browser in - # order to return the evidence needed by the pipeline. - # More info on this can be found at - # https://51degrees.com/blog/user-agent-client-hints?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-useragentclienthints_web-app.py&utm_term=server - set_response_header(flowdata, response) + # Some browsers require that extra HTTP headers are explicitly + # requested. So set whatever headers are required by the browser in + # order to return the evidence needed by the pipeline. + # More info on this can be found at + # https://51degrees.com/blog/user-agent-client-hints?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-useragentclienthints_web-app.py&utm_term=server + set_response_header(flowdata, response) - # Generate the HTML - response.set_data(render_template( - 'index.html', - data=flowdata, - utils=ExampleUtils, - response=response)) + # Generate the HTML + response.set_data(render_template( + 'index.html', + data=flowdata, + utils=ExampleUtils, + response=response)) - return response + return response # Typically, something like this will not be necessary. # The device detection API will accept an absolute or relative path for the data file. diff --git a/device-detection/fiftyone_devicedetection_examples/tests/conftest.py b/device-detection/fiftyone_devicedetection_examples/tests/conftest.py new file mode 100644 index 00000000..fc53a6d0 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/conftest.py @@ -0,0 +1,119 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Fails a test that leaves the native results or engine of an on-premise +# engine alive, because they hold the engine's data in memory. The same +# file is in each test folder that runs on-premise engines. +# +# Reference counting frees a native object as soon as nothing holds it. A +# FlowData refers to itself through its evidence, and a pipeline through +# its elements, so when they are not closed only the cycle collector frees +# what they hold. The collector is off while each test runs, so whatever +# the test leaves open is still alive when it ends. Objects that existed +# before the test, such as a pipeline built in setUpClass, are checked once +# every test has run. + +import gc +import importlib + +import pytest + +# The SWIG modules of the on-premise engines, whichever are installed. +NATIVE_MODULES = [ + "fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule", + "fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule", +] + + +def _native_types(): + types = [] + for name in NATIVE_MODULES: + try: + module = importlib.import_module(name) + except ImportError: + continue + types.append(module.ResultsBaseSwig) + types.append(module.EngineBaseSwig) + return tuple(types) + + +NATIVE_TYPES = _native_types() + +# A failed test is not reported again, because its traceback can hold the +# objects it would have closed. +_failed_tests = set() + +# Objects created when the tests were collected are checked at the end only +# when every test ran, because the teardown that closes them runs with the +# tests. +_some_tests_deselected = False + + +def _alive(): + # The real type is used, because isinstance also asks the object for + # its __class__, which some proxies cannot answer. + return [o for o in gc.get_objects() + if issubclass(type(o), NATIVE_TYPES)] + + +def _describe(names): + return (f"{len(names)} native object(s) still alive " + f"({', '.join(sorted(names))}). Close every FlowData and " + "pipeline the test creates, for example with a with " + "statement, so their native data is released straight away.") + + +def pytest_runtest_logreport(report): + if report.failed: + _failed_tests.add(report.nodeid) + + +def pytest_deselected(items): + global _some_tests_deselected + _some_tests_deselected = True + + +@pytest.fixture(autouse=True) +def native_objects_released(request): + gc.collect() + before = {id(o) for o in _alive()} + gc.disable() + try: + yield + left = [type(o).__name__ for o in _alive() if id(o) not in before] + finally: + gc.enable() + gc.collect() + if left and request.node.nodeid not in _failed_tests: + pytest.fail(_describe(left), pytrace=False) + + +def pytest_sessionfinish(session, exitstatus): + gc.collect() + left = [type(o).__name__ for o in _alive()] + if (left and exitstatus == pytest.ExitCode.OK and + not _some_tests_deselected): + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line( + "After the last test, " + _describe(left), red=True) + session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py b/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py index 70374c6a..a997f5c7 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py @@ -39,6 +39,13 @@ class OnPremiseGettingStartedWebTests(flask_unittest.ClientTestCase): config = GettingStartedWeb.build_config() app = GettingStartedWeb().build(config, logger).app + # The example's pipeline lives as long as its app, so it is closed once + # the tests are finished with the app. + @classmethod + def tearDownClass(cls): + GettingStartedWeb.pipeline.close() + super().tearDownClass() + def test_onpremise_getting_started_web(self, client): response = client.get('/') self.assertEqual(200, response.status_code) diff --git a/device-detection/fiftyone_devicedetection_examples/tox.ini b/device-detection/fiftyone_devicedetection_examples/tox.ini index 6b005879..0b6222c0 100644 --- a/device-detection/fiftyone_devicedetection_examples/tox.ini +++ b/device-detection/fiftyone_devicedetection_examples/tox.ini @@ -22,6 +22,12 @@ deps = -e../fiftyone_devicedetection_cloud/ -e../fiftyone_devicedetection_onpremise/ -e../fiftyone_devicedetection_shared/ + # The pipeline packages come from this checkout rather than a package + # index, so the tests run with the pipeline code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pwsh -ExecutionPolicy Bypass -File ../ci/build-onpremise-c-module.ps1 {[common]commands} diff --git a/device-detection/fiftyone_devicedetection_onpremise/setup.py b/device-detection/fiftyone_devicedetection_onpremise/setup.py index 0af5631d..83b58226 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/setup.py +++ b/device-detection/fiftyone_devicedetection_onpremise/setup.py @@ -261,5 +261,12 @@ def stage_native_source(): py_modules=["fiftyone_devicedetection_onpremise"], packages=["fiftyone_devicedetection_onpremise"], package_dir={"": "src"}, - install_requires=["fiftyone_devicedetection_shared", "fiftyone_pipeline_engines_fiftyone"], + install_requires=[ + "fiftyone_devicedetection_shared", + "fiftyone_pipeline_engines_fiftyone", + # The first versions with close(), which releases the native memory + # this package holds. + "fiftyone_pipeline_core>=4.5.42", + "fiftyone_pipeline_engines>=4.5.42", + ], ) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py index bb8b3b64..db622b40 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py @@ -41,3 +41,10 @@ 'on-premise device detection engine. Its results ' 'read from native memory, which is not safe to ' 'share between requests.') + +ENGINE_CLOSED = ('The on-premise device detection engine has been closed, ' + 'so it cannot process evidence. Build a new pipeline.') + +RESULTS_CLOSED = ('These device detection results have been closed, so ' + 'their properties cannot be read. Read the values needed ' + 'before closing the FlowData.') diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index d8de7ec7..632751ae 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -403,6 +403,20 @@ def on_registration(self, pipeline): super(DeviceDetectionOnPremise, self).on_registration(pipeline) self.init_engine() + def close(self): + + """! + + Stop the data file updates and release the native engine. Its data + is freed once no results read from it are still held, so close + every FlowData the engine processed as well. Closing the pipeline + closes the engine. + + """ + + super(DeviceDetectionOnPremise, self).close() + self.engine = None + def process_internal(self, flow_data): """! @@ -414,6 +428,9 @@ def process_internal(self, flow_data): """ + if getattr(self, "engine", None) is None: + raise Exception(ENGINE_CLOSED) + evidence = EvidenceDeviceDetectionSwig() for key, value in flow_data.evidence.get_all().items(): diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index b1b4e8dd..17987a59 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -25,6 +25,8 @@ from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService +from .constants import RESULTS_CLOSED + class OnPremiseMissingPropertyService(MissingPropertyService): def check(self, key, flow_element): @@ -59,6 +61,33 @@ def __init__(self, flow_element, swig_results): self.missing_property_service = OnPremiseMissingPropertyService() + def close(self): + + """! + Release the native results, which keep the engine's data in memory + while they are held, even after the engine is closed. Values already + read are unaffected, and reading a property afterwards raises an + exception. Closing the FlowData closes its results. + """ + + self.swig_results = None + + def get(self, key): + + """! + Get a property value, once the results are known to be open + @type key: string + @param key: The key of the property to retrieve + @returns mixed + """ + + # Checked here, because AspectData.get turns an exception from + # get_internal into a missing property message. + if self.swig_results is None: + raise Exception(RESULTS_CLOSED) + + return super(SwigData, self).get(key) + def get_internal(self, key): """! diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py b/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py new file mode 100644 index 00000000..fc53a6d0 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py @@ -0,0 +1,119 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Fails a test that leaves the native results or engine of an on-premise +# engine alive, because they hold the engine's data in memory. The same +# file is in each test folder that runs on-premise engines. +# +# Reference counting frees a native object as soon as nothing holds it. A +# FlowData refers to itself through its evidence, and a pipeline through +# its elements, so when they are not closed only the cycle collector frees +# what they hold. The collector is off while each test runs, so whatever +# the test leaves open is still alive when it ends. Objects that existed +# before the test, such as a pipeline built in setUpClass, are checked once +# every test has run. + +import gc +import importlib + +import pytest + +# The SWIG modules of the on-premise engines, whichever are installed. +NATIVE_MODULES = [ + "fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule", + "fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule", +] + + +def _native_types(): + types = [] + for name in NATIVE_MODULES: + try: + module = importlib.import_module(name) + except ImportError: + continue + types.append(module.ResultsBaseSwig) + types.append(module.EngineBaseSwig) + return tuple(types) + + +NATIVE_TYPES = _native_types() + +# A failed test is not reported again, because its traceback can hold the +# objects it would have closed. +_failed_tests = set() + +# Objects created when the tests were collected are checked at the end only +# when every test ran, because the teardown that closes them runs with the +# tests. +_some_tests_deselected = False + + +def _alive(): + # The real type is used, because isinstance also asks the object for + # its __class__, which some proxies cannot answer. + return [o for o in gc.get_objects() + if issubclass(type(o), NATIVE_TYPES)] + + +def _describe(names): + return (f"{len(names)} native object(s) still alive " + f"({', '.join(sorted(names))}). Close every FlowData and " + "pipeline the test creates, for example with a with " + "statement, so their native data is released straight away.") + + +def pytest_runtest_logreport(report): + if report.failed: + _failed_tests.add(report.nodeid) + + +def pytest_deselected(items): + global _some_tests_deselected + _some_tests_deselected = True + + +@pytest.fixture(autouse=True) +def native_objects_released(request): + gc.collect() + before = {id(o) for o in _alive()} + gc.disable() + try: + yield + left = [type(o).__name__ for o in _alive() if id(o) not in before] + finally: + gc.enable() + gc.collect() + if left and request.node.nodeid not in _failed_tests: + pytest.fail(_describe(left), pytrace=False) + + +def pytest_sessionfinish(session, exitstatus): + gc.collect() + left = [type(o).__name__ for o in _alive()] + if (left and exitstatus == pytest.ExitCode.OK and + not _some_tests_deselected): + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line( + "After the last test, " + _describe(left), red=True) + session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py index ea5a031f..e1ac9944 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py @@ -68,8 +68,10 @@ def test_on_premise_data_update_url(self): data_update_verify_md5=False, data_update_url=f"http://127.0.0.1:{self.port}" ).build() + self.addCleanup(pipeline.close) - pipeline.create_flowdata().process() + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -85,8 +87,10 @@ def test_on_premise_data_old_property_data_file_update_base_url(self): data_update_verify_md5=False, data_file_update_base_url=f"http://127.0.0.1:{self.port}" ).build() + self.addCleanup(pipeline.close) - pipeline.create_flowdata().process() + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -106,7 +110,9 @@ def test_on_premise_data_update_url_config(self): ) pipeline = PipelineBuilder().build_from_configuration(config) - pipeline.create_flowdata().process() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -126,7 +132,9 @@ def test_on_premise_data_config_old_property_data_file_update_base_url(self): ) pipeline = PipelineBuilder().build_from_configuration(config) - pipeline.create_flowdata().process() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -142,8 +150,10 @@ def test_on_premise_data_update_verify_md5(self): data_update_verify_md5=True, data_update_url=f"http://127.0.0.1:{self.port}/md5" ).build() + self.addCleanup(pipeline.close) - pipeline.create_flowdata().process() + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -159,8 +169,10 @@ def test_on_premise_data_update_verify_md5_old_property(self): verify_md5=True, data_update_url=f"http://127.0.0.1:{self.port}/md5" ).build() + self.addCleanup(pipeline.close) - pipeline.create_flowdata().process() + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -180,7 +192,9 @@ def test_on_premise_data_update_verify_md5_config(self): ) pipeline = PipelineBuilder().build_from_configuration(config) - pipeline.create_flowdata().process() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -200,7 +214,9 @@ def test_on_premise_data_update_verify_md5_config_old_property(self): ) pipeline = PipelineBuilder().build_from_configuration(config) - pipeline.create_flowdata().process() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -217,8 +233,10 @@ def test_on_premise_data_update_use_url_formatter(self): data_update_url=f"http://127.0.0.1:{self.port}/", data_update_use_url_formatter=False ).build() + self.addCleanup(pipeline.close) - pipeline.create_flowdata().process() + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -238,7 +256,9 @@ def test_on_premise_data_update_use_url_formatter_config(self): ) pipeline = PipelineBuilder().build_from_configuration(config) - pipeline.create_flowdata().process() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -253,8 +273,10 @@ def test_on_premise_data_update_status_rate_limit(self): verify_md5=True, data_update_url=f"http://127.0.0.1:{self.port}/status/429" ).build() + self.addCleanup(pipeline.close) - pipeline.create_flowdata().process() + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -325,7 +347,8 @@ def _init_hash_file_archive(): with gzip.open(archive_file_path, "wb", compresslevel=1) as f_out: shutil.copyfileobj(f_in, f_out) - return hashlib.md5(open(archive_file_path, "rb").read()).hexdigest() + with open(archive_file_path, "rb") as archive: + return hashlib.md5(archive.read()).hexdigest() @staticmethod def clear(): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py index 135cc69d..11ed21f6 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py @@ -61,6 +61,7 @@ def test_pipeline_builder_shareusage_init(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, licence_keys="").build() + self.addCleanup(pipeline.close) self.assertTrue(pipeline.flow_elements[0].datakey == "shareusage") @@ -71,6 +72,7 @@ def test_on_premise_engine_datafile(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="test", auto_update=True).build() + self.addCleanup(pipeline.close) self.assertTrue(isinstance( pipeline.flow_elements[0].data_file, DeviceDetectionDataFile)) @@ -83,6 +85,7 @@ def test_properties_onpremise(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) properties = pipeline.flow_elements[0].get_properties() @@ -95,6 +98,7 @@ def test_evidencekey_filter_onpremise(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( ) @@ -110,8 +114,10 @@ def test_basic_get_onpremise(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", mobile_ua) @@ -126,8 +132,10 @@ def test_process_with_no_evidence(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.process() @@ -140,8 +148,10 @@ def test_process_with_empty_user_agent(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", "") @@ -156,8 +166,10 @@ def test_process_with_no_headers(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("irrelevant.evidence", "some-evidence") @@ -172,8 +184,10 @@ def test_process_with_no_useful_headers(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.irrelevant", "some-evidence") @@ -188,8 +202,10 @@ def test_case_insensitive_evidence_keys(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.User-Agent", mobile_ua) @@ -205,8 +221,10 @@ def test_missing_property_service_not_found_anywhere(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.User-Agent", mobile_ua) @@ -230,8 +248,10 @@ def test_missing_property_service_not_found_in_current(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.User-Agent", mobile_ua) @@ -262,8 +282,10 @@ def test_excluded_property(self): data_file_path=data_file, licence_keys="", restricted_properties=["ismobile"]).build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.User-Agent", mobile_ua) @@ -286,6 +308,7 @@ def test_evidencekey_filter_contains_user_agent(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( ) @@ -300,6 +323,7 @@ def test_evidencekey_filter_contains_device_stock_ua(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( ) @@ -314,6 +338,7 @@ def test_evidencekey_filter_contains_query_params(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( ) @@ -330,6 +355,7 @@ def test_evidencekey_filter_case_insensitive_keys(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( ) @@ -350,6 +376,7 @@ def test_evidencekey_filter_overrides(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( ) @@ -366,8 +393,10 @@ def test_profile_overrides(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("query.header.user-agent", "Some user agent") @@ -384,8 +413,10 @@ def test_profile_overrides_no_headers(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("query.51D_ProfileIds", "12280|17779|17470|18092") @@ -400,8 +431,10 @@ def test_profile_overrides_deviceid(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("query.51D_ProfileIds", "12280-17779-17470-18092") @@ -416,8 +449,10 @@ def test_device_id(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.User-Agent", mobile_ua) @@ -443,8 +478,10 @@ def test_build_from_config(self): } pipeline = PipelineBuilder().build_from_configuration(config) + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.User-Agent", mobile_ua) @@ -459,8 +496,10 @@ def test_no_element_exists(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.process() @@ -481,8 +520,10 @@ def test_has_value_false(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", "?£$%^&*") @@ -497,8 +538,10 @@ def test_matched_user_agents(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", mobile_ua) @@ -519,8 +562,10 @@ def test_value_types(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", mobile_ua) @@ -539,8 +584,10 @@ def test_available_properties(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", mobile_ua) @@ -561,8 +608,10 @@ def test_engine_init_performance(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", mobile_ua) @@ -586,8 +635,10 @@ def test_validate_data_true(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", mobile_ua) @@ -613,8 +664,10 @@ def test_validate_data_false(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", "") @@ -638,30 +691,89 @@ def test_engine_reload(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.process() fd.pipeline.get_element("device").engine.refreshData() + def test_flowdata_close_releases_results(self): + """! + Closing the FlowData releases its native results, and reading a + property afterwards raises an exception + """ + + pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) + + with pipeline.create_flowdata() as fd: + fd.evidence.add("header.user-agent", mobile_ua) + fd.process() + ismobile = fd.device.ismobile.value() + + self.assertTrue(ismobile) + self.assertIsNone(fd.device.swig_results) + with self.assertRaises(Exception) as context: + fd.device.ismobile + self.assertIn("closed", str(context.exception)) + + def test_pipeline_close_releases_engine(self): + """! + Closing the pipeline releases the native engine, and processing + afterwards raises an exception + """ + + pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=data_file, usage_sharing=False, licence_keys="").build() + engine = pipeline.get_element("device") + + pipeline.close() + + self.assertIsNone(engine.engine) + fd = pipeline.create_flowdata() + self.addCleanup(fd.close) + fd.evidence.add("header.user-agent", mobile_ua) + with self.assertRaises(Exception) as context: + fd.process() + self.assertIn("closed", str(context.exception)) + + def test_pipeline_close_stops_updates(self): + """! + Closing the pipeline cancels the engine's next automatic update, + whose timer would otherwise keep the engine alive + """ + + pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=data_file, usage_sharing=False, licence_keys="test", + auto_update=True).build() + timer = pipeline.get_element("device").data_file.update_timer + + pipeline.close() + + timer.join(10) + self.assertFalse(timer.is_alive()) + def process_user_agent_list(self, pipeline): results = {"mobile": 0, "not_mobile": 0, "unknown": 0} for user_agent in user_agents: user_agent = user_agent[0] - flowdata = pipeline.create_flowdata() - flowdata.evidence.add("header.user-agent", user_agent) - flowdata.process() - - if flowdata.device.ismobile.has_value(): - if(flowdata.device.ismobile.value()): - results["mobile"] += 1 + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", user_agent) + flowdata.process() + + if flowdata.device.ismobile.has_value(): + if(flowdata.device.ismobile.value()): + results["mobile"] += 1 + else: + results["not_mobile"] += 1 else: - results["not_mobile"] += 1 - else: - results["unknown"] += 1 + results["unknown"] += 1 return results diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py index 0d9b6435..df503004 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py @@ -42,6 +42,10 @@ .add(deviceDetectionOnPremiseEngine) \ .build() +# Releases the native engine once every test in the module has run. +def tearDownModule(): + pipeline.close() + class PropertyTests(unittest.TestCase): def test_available_properties(self): @@ -50,6 +54,7 @@ def test_available_properties(self): """ flowData = pipeline.create_flowdata() + self.addCleanup(flowData.close) flowData.evidence.add("header.user-agent", mobile_ua) flowData.process() device = flowData.device @@ -75,6 +80,7 @@ def test_match_metrics_description(self): """ flowData = pipeline.create_flowdata() + self.addCleanup(flowData.close) flowData.evidence.add("header.user-agent", mobile_ua) flowData.process() @@ -110,6 +116,7 @@ def test_value_types(self): """ flowData = pipeline.create_flowdata() + self.addCleanup(flowData.close) flowData.evidence.add("header.user-agent", mobile_ua) flowData.process() device = flowData.device diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py index 0b4bea5a..67879404 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py @@ -39,6 +39,10 @@ .add(deviceDetectionOnPremiseEngine) \ .build() +# Releases the native engine once every test in the module has run. +def tearDownModule(): + pipeline.close() + class SwigTests(unittest.TestCase): # Mock the native code parts so we are only testing the wrapper. diff --git a/device-detection/fiftyone_devicedetection_onpremise/tox.ini b/device-detection/fiftyone_devicedetection_onpremise/tox.ini index 4600cd37..5cdeeca3 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tox.ini +++ b/device-detection/fiftyone_devicedetection_onpremise/tox.ini @@ -22,6 +22,12 @@ deps = -e../fiftyone_devicedetection_cloud/ -e../fiftyone_devicedetection_shared/ -e../fiftyone_devicedetection_examples/ + # The pipeline packages come from this checkout rather than a package + # index, so the tests run with the pipeline code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = python setup.py build_clib build_ext {[common]commands} diff --git a/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py b/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py index c7ea9615..a2511324 100644 --- a/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py +++ b/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py @@ -35,45 +35,48 @@ # Cloud #pipeline = IpIntelligencePipelineBuilder(resource_key = resource_key).build() -# We create a FlowData object from the pipeline -# this is used to add evidence to and then process +# The pipeline is closed at the end of the with block, which releases the +# native data it holds. +with pipeline: + # We create a FlowData object from the pipeline + # this is used to add evidence to and then process. It is closed at the + # end of its with block, which releases the results of the lookup. + with pipeline.create_flowdata() as flowdata1: -flowdata1 = pipeline.create_flowdata() + # Here we add an IPv4 address as evidence -# Here we add an IPv4 address as evidence + ipv4_address = "185.28.167.78" -ipv4_address = "185.28.167.78" + flowdata1.evidence.add("query.client-ip", ipv4_address) -flowdata1.evidence.add("query.client-ip", ipv4_address) + # Now we process the FlowData -# Now we process the FlowData + flowdata1.process() -flowdata1.process() + # To find the name the IP range is registered under we look at the + # registeredname property inside the IP Intelligence Engine -# To find the name the IP range is registered under we look at the -# registeredname property inside the IP Intelligence Engine + # first we check if this has a meaningful result -# first we check if this has a meaningful result + print("What is IP address " + ipv4_address + " registered as? ") + if flowdata1.ip.registeredname.has_value(): + print(flowdata1.ip.registeredname.value()) + else: + # Output why the value isn't meaningful + print(flowdata1.ip.registeredname.no_value_message()) -print("What is IP address " + ipv4_address + " registered as? ") -if flowdata1.ip.registeredname.has_value(): - print(flowdata1.ip.registeredname.value()) -else: - # Output why the value isn't meaningful - print(flowdata1.ip.registeredname.no_value_message()) + # Now we do the same with a new IP address, this time an IPv6 one -# Now we do the same with a new IP address, this time an IPv6 one + with pipeline.create_flowdata() as flowdata2: -flowdata2 = pipeline.create_flowdata() + ipv6_address = "2001:4860:4860::8888" -ipv6_address = "2001:4860:4860::8888" + flowdata2.evidence.add("query.client-ip", ipv6_address) -flowdata2.evidence.add("query.client-ip", ipv6_address) + flowdata2.process() -flowdata2.process() - -print("What is IP address " + ipv6_address + " registered as? ") -if flowdata2.ip.registeredname.has_value(): - print(flowdata2.ip.registeredname.value()) -else: - print(flowdata2.ip.registeredname.no_value_message()) + print("What is IP address " + ipv6_address + " registered as? ") + if flowdata2.ip.registeredname.has_value(): + print(flowdata2.ip.registeredname.value()) + else: + print(flowdata2.ip.registeredname.no_value_message()) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/setup.py b/ip-intelligence/fiftyone_ipintelligence_examples/setup.py index 2e1def33..cc127a01 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/setup.py @@ -54,6 +54,8 @@ def read(file_name): install_requires=[ "fiftyone_ipintelligence", "fiftyone_ipintelligence_translation", + # The first version with close(), which the examples use. + "fiftyone_pipeline_core>=4.5.42", "flask", "ruamel.yaml" ], diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py index 062e24da..e418f786 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py @@ -96,65 +96,70 @@ def run(self, data_file, truth_file, logger, output, record_limit=RECORD_LIMIT): auto_update = False, licence_keys = "").add_logger(logger).build() - ExampleUtils.check_data_file(pipeline, logger) - - writer = csv.DictWriter(output, fieldnames=OUTPUT_FIELDS) - writer.writeheader() - - processed = 0 - found = 0 - with open(truth_file, "r", newline="", encoding="utf-8") as fin: - for truth in csv.DictReader(fin): - if processed >= record_limit: - break - processed += 1 - result = self.compare(pipeline, truth) - if result["Found"]: - found += 1 - writer.writerow(result) - - logger.log("info", - f"Compared {processed} records; {found} resolved to a location") + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + ExampleUtils.check_data_file(pipeline, logger) + + writer = csv.DictWriter(output, fieldnames=OUTPUT_FIELDS) + writer.writeheader() + + processed = 0 + found = 0 + with open(truth_file, "r", newline="", encoding="utf-8") as fin: + for truth in csv.DictReader(fin): + if processed >= record_limit: + break + processed += 1 + result = self.compare(pipeline, truth) + if result["Found"]: + found += 1 + writer.writerow(result) + + logger.log("info", + f"Compared {processed} records; {found} resolved to a location") def compare(self, pipeline, truth): - data = pipeline.create_flowdata() - data.evidence.add_from_dict({"query.client-ip": truth["Ip"]}) - data.process() - ip_data = data.ip - - row = { - "DateTimeUtc": truth.get("DateTimeUtc", ""), - "Ip": truth.get("Ip", ""), - "AddressFamily": truth.get("AddressFamily", ""), - "TruthLatitude": truth.get("Latitude", ""), - "TruthLongitude": truth.get("Longitude", ""), - "TruthCountry": truth.get("Country", ""), - "ResultLatitude": "", - "ResultLongitude": "", - "ResultCountry": ExampleUtils.get_human_readable(ip_data, "country"), - "Confidence": ExampleUtils.get_human_readable(ip_data, "locationconfidence"), - "Found": False, - "DistanceKms": "", - } - - latitude = ExampleUtils.get_value(ip_data, "latitude") - longitude = ExampleUtils.get_value(ip_data, "longitude") - if latitude is not None and latitude.has_value() and \ - longitude is not None and longitude.has_value(): - row["ResultLatitude"] = latitude.value() - row["ResultLongitude"] = longitude.value() - row["Found"] = True - try: - row["DistanceKms"] = round(haversine_km( - float(truth["Latitude"]), float(truth["Longitude"]), - float(latitude.value()), float(longitude.value())), 3) - except (ValueError, TypeError): - # The truth record did not provide a usable latitude or - # longitude, so the distance cannot be calculated. - row["DistanceKms"] = "" - - return row + # The FlowData is closed when the method returns, which releases the + # results of the lookup straight away. + with pipeline.create_flowdata() as data: + data.evidence.add_from_dict({"query.client-ip": truth["Ip"]}) + data.process() + ip_data = data.ip + + row = { + "DateTimeUtc": truth.get("DateTimeUtc", ""), + "Ip": truth.get("Ip", ""), + "AddressFamily": truth.get("AddressFamily", ""), + "TruthLatitude": truth.get("Latitude", ""), + "TruthLongitude": truth.get("Longitude", ""), + "TruthCountry": truth.get("Country", ""), + "ResultLatitude": "", + "ResultLongitude": "", + "ResultCountry": ExampleUtils.get_human_readable(ip_data, "country"), + "Confidence": ExampleUtils.get_human_readable(ip_data, "locationconfidence"), + "Found": False, + "DistanceKms": "", + } + + latitude = ExampleUtils.get_value(ip_data, "latitude") + longitude = ExampleUtils.get_value(ip_data, "longitude") + if latitude is not None and latitude.has_value() and \ + longitude is not None and longitude.has_value(): + row["ResultLatitude"] = latitude.value() + row["ResultLongitude"] = longitude.value() + row["Found"] = True + try: + row["DistanceKms"] = round(haversine_km( + float(truth["Latitude"]), float(truth["Longitude"]), + float(latitude.value()), float(longitude.value())), 3) + except (ValueError, TypeError): + # The truth record did not provide a usable latitude or + # longitude, so the distance cannot be calculated. + row["DistanceKms"] = "" + + return row def main(argv): diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py index 6d26ce73..eb239813 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py @@ -71,42 +71,47 @@ def run(self, data_file, logger, output): add_country_translation(builder) pipeline = builder.add_logger(logger).build() - ExampleUtils.check_data_file(pipeline, logger) + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + ExampleUtils.check_data_file(pipeline, logger) - for values in EVIDENCE_VALUES: - self.analyseEvidence(values, pipeline, output) + for values in EVIDENCE_VALUES: + self.analyseEvidence(values, pipeline, output) def analyseEvidence(self, evidence, pipeline, output): - data = pipeline.create_flowdata() - - message = [] - message.append("Input values:\n") - for key in evidence: - message.append(f"\t{key}: {evidence[key]}\n") - if TRANSLATION: - message.append(f"\tquery.translation: {TRANSLATION}\n") - output("".join(message)) - - # Add the IP address evidence, and the language to translate to. - data.evidence.add_from_dict(evidence) - if TRANSLATION: - data.evidence.add("query.translation", TRANSLATION) - - data.process() - - translated = data.get("countrynamestranslated") - - message = [] - message.append("Results:\n") - self.outputList( - translated, "CountryCodesGeographicalAll", - "Geographical countries (most probable first)", message) - self.outputList( - translated, "CountryCodesPopulationAll", - "Population countries (most probable first)", message) - self.outputDropdown(translated, message) - output("".join(message)) + # The FlowData is closed at the end of the with block, which releases + # the results of the lookup straight away. + with pipeline.create_flowdata() as data: + + message = [] + message.append("Input values:\n") + for key in evidence: + message.append(f"\t{key}: {evidence[key]}\n") + if TRANSLATION: + message.append(f"\tquery.translation: {TRANSLATION}\n") + output("".join(message)) + + # Add the IP address evidence, and the language to translate to. + data.evidence.add_from_dict(evidence) + if TRANSLATION: + data.evidence.add("query.translation", TRANSLATION) + + data.process() + + translated = data.get("countrynamestranslated") + + message = [] + message.append("Results:\n") + self.outputList( + translated, "CountryCodesGeographicalAll", + "Geographical countries (most probable first)", message) + self.outputList( + translated, "CountryCodesPopulationAll", + "Population countries (most probable first)", message) + self.outputDropdown(translated, message) + output("".join(message)) def outputList(self, translated, property_name, label, message): value = translated.get(property_name) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py index 1236a3b1..37200afd 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py @@ -155,7 +155,7 @@ def run(self, data_file, license_key, interactive, logger, output): update_service.on_complete(lambda status, file: update_event.set(status)) # Build the pipeline, configuring it for automatic updates. - IpIntelligencePipelineBuilder( + pipeline = IpIntelligencePipelineBuilder( # The file need not exist when using update on start-up, but its # directory must. Any file present is overwritten, so the engine # is configured to use a temporary copy. @@ -175,35 +175,39 @@ def run(self, data_file, license_key, interactive, logger, output): data_update_product_type = DATA_UPDATE_PRODUCT_TYPE, usage_sharing = False).add_logger(logger).build() - # Block until the start-up update check is complete. - update_event.wait() - output(f"Update on start-up complete - status - {update_event.status}") - - if update_event.status == UpdateStatus.AUTO_UPDATE_SUCCESS: - output("Modifying downloaded file to trigger reload - please wait " - "for that to complete") - update_event.clear() - - # Touching the file metadata triggers the filesystem watcher, - # demonstrating reload when a new file is dropped into place. - now = datetime.now().timestamp() - try: - os.utime(data_file, (now, now)) - except OSError: - raise Exception("Could not modify file time, abandoning example") - - if update_event.wait(120): - output(f"Update on file modification complete, status: " - f"{update_event.status}") + # The pipeline is closed at the end of the with block, which stops the + # automatic updates and the file system watcher, and releases the + # native data it holds. + with pipeline: + # Block until the start-up update check is complete. + update_event.wait() + output(f"Update on start-up complete - status - {update_event.status}") + + if update_event.status == UpdateStatus.AUTO_UPDATE_SUCCESS: + output("Modifying downloaded file to trigger reload - please wait " + "for that to complete") + update_event.clear() + + # Touching the file metadata triggers the filesystem watcher, + # demonstrating reload when a new file is dropped into place. + now = datetime.now().timestamp() + try: + os.utime(data_file, (now, now)) + except OSError: + raise Exception("Could not modify file time, abandoning example") + + if update_event.wait(120): + output(f"Update on file modification complete, status: " + f"{update_event.status}") + else: + output("Update on file modification timed out") else: - output("Update on file modification timed out") - else: - logger.log("error", "Auto update was not successful, abandoning example") - error_message = f"Auto update failed: {update_event.status}" - if update_event.status == UpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS: - output(error_message) - else: - raise Exception(error_message) + logger.log("error", "Auto update was not successful, abandoning example") + error_message = f"Auto update failed: {update_event.status}" + if update_event.status == UpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS: + output(error_message) + else: + raise Exception(error_message) output("Finished Example") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py index 8e82567b..4988da7d 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py @@ -91,11 +91,16 @@ def run(self, data_file, logger, output): auto_update = False, licence_keys = "").add_logger(logger).build() - ExampleUtils.check_data_file(pipeline, logger) + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. An application normally builds + # one pipeline when it starts, uses it for every lookup, and closes + # it when it stops. + with pipeline: + ExampleUtils.check_data_file(pipeline, logger) - # carry out some sample lookups - for values in EVIDENCE_VALUES: - self.analyseEvidence(values, pipeline, output) + # carry out some sample lookups + for values in EVIDENCE_VALUES: + self.analyseEvidence(values, pipeline, output) def analyseEvidence(self, evidence, pipeline, output): @@ -104,38 +109,41 @@ def analyseEvidence(self, evidence, pipeline, output): # through the pipeline. # Information required is called "evidence", in this case an # IP address supplied under the "query.client-ip" key. - data = pipeline.create_flowdata() + # Closing the FlowData, which the with statement does at the end of + # the block, releases the results of the lookup straight away, + # rather than when the garbage collector next runs. + with pipeline.create_flowdata() as data: - message = [] + message = [] - # List the evidence - message.append("Input values:\n") - for key in evidence: - message.append(f"\t{key}: {evidence[key]}\n") + # List the evidence + message.append("Input values:\n") + for key in evidence: + message.append(f"\t{key}: {evidence[key]}\n") - output("".join(message)) + output("".join(message)) - # Add the evidence values to the flow data - data.evidence.add_from_dict(evidence) + # Add the evidence values to the flow data + data.evidence.add_from_dict(evidence) - # Process the flow data. - data.process() + # Process the flow data. + data.process() - message = [] - message.append("Results:\n") + message = [] + message.append("Results:\n") - # Now that it's been processed, the flow data will have - # been populated with the result. In this case, we want - # information about the IP address, which we can get by - # asking for a result matching the key "ip" - ip_data = data.ip + # Now that it's been processed, the flow data will have + # been populated with the result. In this case, we want + # information about the IP address, which we can get by + # asking for a result matching the key "ip" + ip_data = data.ip - # Display the results of the lookup. See the property - # dictionary at https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=analyseevidence - # for details of all available properties. - for name in OUTPUT_PROPERTIES: - self.outputValue(ip_data, name, message) - output("".join(message)) + # Display the results of the lookup. See the property + # dictionary at https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=analyseevidence + # for details of all available properties. + for name in OUTPUT_PROPERTIES: + self.outputValue(ip_data, name, message) + output("".join(message)) def outputValue(self, ip_data, name, message): # Individual result values have a wrapper called diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py index debc4a1b..56ea9799 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py @@ -88,44 +88,46 @@ def country_dropdown(translated): @app.route("/") def index(): - # Create the flowdata object for this request. - flowdata = pipeline.create_flowdata() - - # Use the IP address supplied as a query parameter, or fall back - # to the address of the requester. - client_ip = request.args.get("client-ip", request.remote_addr) - flowdata.evidence.add("query.client-ip", client_ip) - - # Supply the browser language so the country names are localized. The - # Accept-Language header is used by default, overridable with a - # "translation" query parameter. - accept_language = request.headers.get("Accept-Language") - if accept_language: - flowdata.evidence.add("header.accept-language", accept_language) - translation = request.args.get("translation") - if translation: - flowdata.evidence.add("query.translation", translation) - - # Process the flowdata - flowdata.process() - - ip_data = flowdata.ip - translated = flowdata.get("countrynamestranslated") - - return f""" -

    51Degrees IP Intelligence - Getting Started Web (On-Premise)

    -

    IP address: {client_ip}

    -
      -
    • Registered Name: {get_value(ip_data, "registeredname")}
      • -
    • Registered Owner: {get_value(ip_data, "registeredowner")}
      • -
    • Registered Country: {get_value(ip_data, "registeredcountry")}
      • -
    -

    Country

    -

    Most probable country first, then every other country alphabetically.

    - {country_dropdown(translated)} -

    Add ?client-ip=x.x.x.x to the url to lookup a different address, and - &translation=fr_FR to choose a language.

    - """ + # Create the flowdata object for this request. It is closed when the + # route returns, once the page has been built from it, which releases + # the results of the lookup straight away. + with pipeline.create_flowdata() as flowdata: + + # Use the IP address supplied as a query parameter, or fall back + # to the address of the requester. + client_ip = request.args.get("client-ip", request.remote_addr) + flowdata.evidence.add("query.client-ip", client_ip) + + # Supply the browser language so the country names are localized. The + # Accept-Language header is used by default, overridable with a + # "translation" query parameter. + accept_language = request.headers.get("Accept-Language") + if accept_language: + flowdata.evidence.add("header.accept-language", accept_language) + translation = request.args.get("translation") + if translation: + flowdata.evidence.add("query.translation", translation) + + # Process the flowdata + flowdata.process() + + ip_data = flowdata.ip + translated = flowdata.get("countrynamestranslated") + + return f""" +

    51Degrees IP Intelligence - Getting Started Web (On-Premise)

    +

    IP address: {client_ip}

    +
      +
    • Registered Name: {get_value(ip_data, "registeredname")}
      • +
    • Registered Owner: {get_value(ip_data, "registeredowner")}
      • +
    • Registered Country: {get_value(ip_data, "registeredcountry")}
      • +
    +

    Country

    +

    Most probable country first, then every other country alphabetically.

    + {country_dropdown(translated)} +

    Add ?client-ip=x.x.x.x to the url to lookup a different address, and + &translation=fr_FR to choose a language.

    + """ def main(data_file = None): @@ -138,8 +140,10 @@ def main(data_file = None): "`git submodule update --init --recursive`, and run this " "example again to download the file, or supply the path to a " "data file as the first command line argument.") - build_pipeline(data_file) - app.run(debug=False) + # The pipeline lives as long as the app, and is closed when the server + # stops, which releases the native data it holds. + with build_pipeline(data_file): + app.run(debug=False) if __name__ == "__main__": diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py index ec2be8a1..3819776c 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py @@ -47,14 +47,17 @@ def run(self, data_file, logger, output): auto_update = False, licence_keys = "").add_logger(logger).build() - engine = pipeline.flow_elements_list["ip"] + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + engine = pipeline.flow_elements_list["ip"] - output("IP Intelligence properties:") - for property_metadata in engine.properties.values(): - output(f" {property_metadata['name']} ({property_metadata['type']}) - " - f"category: {property_metadata['category']}") - if property_metadata.get("description"): - output(f" {property_metadata['description']}") + output("IP Intelligence properties:") + for property_metadata in engine.properties.values(): + output(f" {property_metadata['name']} ({property_metadata['type']}) - " + f"category: {property_metadata['category']}") + if property_metadata.get("description"): + output(f" {property_metadata['description']}") def main(argv): diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py index 07839ab1..2f4f5e18 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py @@ -140,38 +140,43 @@ def run(self, device_file, ip_file, logger, output): .add_logger(logger) \ .build() - for evidence in COMBINED_EVIDENCE: - self.analyse_evidence(evidence, pipeline, output) + # The pipeline is closed at the end of the with block, which closes + # both engines and releases the native data they hold. + with pipeline: + for evidence in COMBINED_EVIDENCE: + self.analyse_evidence(evidence, pipeline, output) def analyse_evidence(self, evidence, pipeline, output): - data = pipeline.create_flowdata() + # The FlowData is closed at the end of the with block, which releases + # the results of both engines straight away. + with pipeline.create_flowdata() as data: - message = [] - message.append("=" * 70 + "\n") - message.append("Input values:\n") - for key in evidence: - message.append(f"\t{key}: {evidence[key]}\n") - output("".join(message)) + message = [] + message.append("=" * 70 + "\n") + message.append("Input values:\n") + for key in evidence: + message.append(f"\t{key}: {evidence[key]}\n") + output("".join(message)) - # Add the evidence values to the flow data and process it. Both - # engines run against the same flow data. - data.evidence.add_from_dict(evidence) - data.process() + # Add the evidence values to the flow data and process it. Both + # engines run against the same flow data. + data.evidence.add_from_dict(evidence) + data.process() - message = [] + message = [] - # Device detection results, keyed under "device". - message.append("Device Detection Results:\n") - for name in DEVICE_PROPERTIES: - self.outputValue(data.device, name, message) + # Device detection results, keyed under "device". + message.append("Device Detection Results:\n") + for name in DEVICE_PROPERTIES: + self.outputValue(data.device, name, message) - # IP intelligence results, keyed under "ip". - message.append("\nIP Intelligence Results:\n") - for name in IP_PROPERTIES: - self.outputValue(data.ip, name, message) + # IP intelligence results, keyed under "ip". + message.append("\nIP Intelligence Results:\n") + for name in IP_PROPERTIES: + self.outputValue(data.ip, name, message) - output("".join(message)) + output("".join(message)) def outputValue(self, aspect_data, name, message): # `get_value` returns None when the property is not present in the diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py index 9c7109cc..4785249e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py @@ -65,41 +65,47 @@ def run(self, data_file, evidence_yaml, logger, output): auto_update = False, licence_keys = "").add_logger(logger).build() - ExampleUtils.check_data_file(pipeline, logger) - - records = 0 - with open(evidence_yaml, "r") as fin: - for evidence in yaml.load_all(fin): - if records >= RECORD_LIMIT: - break - - # write the yaml document separator - output("---") - self.analyse_evidence(evidence, pipeline, output, yaml) - records += 1 - - # write the yaml document end marker - output("...") - logger.log("info", f"Processed {records} evidence records") + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + ExampleUtils.check_data_file(pipeline, logger) + + records = 0 + with open(evidence_yaml, "r") as fin: + for evidence in yaml.load_all(fin): + if records >= RECORD_LIMIT: + break + + # write the yaml document separator + output("---") + self.analyse_evidence(evidence, pipeline, output, yaml) + records += 1 + + # write the yaml document end marker + output("...") + logger.log("info", f"Processed {records} evidence records") def analyse_evidence(self, evidence, pipeline, output, yaml): - data = pipeline.create_flowdata() + # The FlowData is closed at the end of the with block, once the + # values needed have been read, which releases the results of the + # lookup straight away. + with pipeline.create_flowdata() as data: - # Add the evidence values to the flow data. The YAML loader - # returns a dict subclass so convert to a plain dict first. - data.evidence.add_from_dict(dict(evidence)) + # Add the evidence values to the flow data. The YAML loader + # returns a dict subclass so convert to a plain dict first. + data.evidence.add_from_dict(dict(evidence)) - # Process the flow data. - data.process() + # Process the flow data. + data.process() - ip_data = data.ip + ip_data = data.ip - result = dict(evidence) + result = dict(evidence) - # Add the values that we want to store against the record. - result["ip.registeredname"] = ExampleUtils.get_human_readable(ip_data, "registeredname") - result["ip.registeredcountry"] = ExampleUtils.get_human_readable(ip_data, "registeredcountry") + # Add the values that we want to store against the record. + result["ip.registeredname"] = ExampleUtils.get_human_readable(ip_data, "registeredname") + result["ip.registeredcountry"] = ExampleUtils.get_human_readable(ip_data, "registeredcountry") buffer = io.BytesIO() yaml.dump(result, buffer) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py index 310bd727..1d910d81 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py @@ -84,11 +84,13 @@ def _build_pipeline(data_file, performance_profile, logger=None): def _process_record(pipeline, record): - data = pipeline.create_flowdata() - data.evidence.add_from_dict(record) - data.process() - value = ExampleUtils.get_value(data.ip, PERFORMANCE_PROPERTY) - return value is not None and value.has_value() + # The FlowData is closed when the function returns, which releases the + # results of the lookup straight away. + with pipeline.create_flowdata() as data: + data.evidence.add_from_dict(record) + data.process() + value = ExampleUtils.get_value(data.ip, PERFORMANCE_PROPERTY) + return value is not None and value.has_value() def _time_lookups(pipeline, records): @@ -115,8 +117,10 @@ def _worker(data_file, performance_profile, records, queue): count and elapsed seconds back to the parent. """ - pipeline = _build_pipeline(data_file, performance_profile) - elapsed = _time_lookups(pipeline, records) + # The pipeline is closed at the end of the with block, which releases + # the native data it holds. + with _build_pipeline(data_file, performance_profile) as pipeline: + elapsed = _time_lookups(pipeline, records) queue.put((len(records), elapsed)) @@ -156,9 +160,10 @@ def run(self, data_file, evidence, logger, output, workers=None, # wall time of the run. elapsed = max(elapsed for _records, elapsed in results) else: - pipeline = _build_pipeline(data_file, performance_profile, logger) - ExampleUtils.check_data_file(pipeline, logger) - elapsed = _time_lookups(pipeline, evidence) + with _build_pipeline( + data_file, performance_profile, logger) as pipeline: + ExampleUtils.check_data_file(pipeline, logger) + elapsed = _time_lookups(pipeline, evidence) per_second = count / elapsed if elapsed > 0 else 0 ms_per_lookup = (elapsed / count) * 1000 if count > 0 else 0 diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py new file mode 100644 index 00000000..fc53a6d0 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py @@ -0,0 +1,119 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Fails a test that leaves the native results or engine of an on-premise +# engine alive, because they hold the engine's data in memory. The same +# file is in each test folder that runs on-premise engines. +# +# Reference counting frees a native object as soon as nothing holds it. A +# FlowData refers to itself through its evidence, and a pipeline through +# its elements, so when they are not closed only the cycle collector frees +# what they hold. The collector is off while each test runs, so whatever +# the test leaves open is still alive when it ends. Objects that existed +# before the test, such as a pipeline built in setUpClass, are checked once +# every test has run. + +import gc +import importlib + +import pytest + +# The SWIG modules of the on-premise engines, whichever are installed. +NATIVE_MODULES = [ + "fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule", + "fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule", +] + + +def _native_types(): + types = [] + for name in NATIVE_MODULES: + try: + module = importlib.import_module(name) + except ImportError: + continue + types.append(module.ResultsBaseSwig) + types.append(module.EngineBaseSwig) + return tuple(types) + + +NATIVE_TYPES = _native_types() + +# A failed test is not reported again, because its traceback can hold the +# objects it would have closed. +_failed_tests = set() + +# Objects created when the tests were collected are checked at the end only +# when every test ran, because the teardown that closes them runs with the +# tests. +_some_tests_deselected = False + + +def _alive(): + # The real type is used, because isinstance also asks the object for + # its __class__, which some proxies cannot answer. + return [o for o in gc.get_objects() + if issubclass(type(o), NATIVE_TYPES)] + + +def _describe(names): + return (f"{len(names)} native object(s) still alive " + f"({', '.join(sorted(names))}). Close every FlowData and " + "pipeline the test creates, for example with a with " + "statement, so their native data is released straight away.") + + +def pytest_runtest_logreport(report): + if report.failed: + _failed_tests.add(report.nodeid) + + +def pytest_deselected(items): + global _some_tests_deselected + _some_tests_deselected = True + + +@pytest.fixture(autouse=True) +def native_objects_released(request): + gc.collect() + before = {id(o) for o in _alive()} + gc.disable() + try: + yield + left = [type(o).__name__ for o in _alive() if id(o) not in before] + finally: + gc.enable() + gc.collect() + if left and request.node.nodeid not in _failed_tests: + pytest.fail(_describe(left), pytrace=False) + + +def pytest_sessionfinish(session, exitstatus): + gc.collect() + left = [type(o).__name__ for o in _alive()] + if (left and exitstatus == pytest.ExitCode.OK and + not _some_tests_deselected): + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line( + "After the last test, " + _describe(left), red=True) + session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py index a2d17c30..0ada1f94 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py @@ -70,7 +70,9 @@ def test_onpremise_datafile_loads(self): usage_sharing=False, auto_update=False, licence_keys="").build() + self.addCleanup(pipeline.close) data = pipeline.create_flowdata() + self.addCleanup(data.close) data.evidence.add_from_dict({"query.client-ip": "8.8.8.8"}) data.process() # The supplied IP address is echoed back in the results. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini index 0008718b..3bc4fbd3 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini @@ -24,6 +24,13 @@ deps = # these packages is on a package index yet, so it comes from the # working copy as the others do. -e../fiftyone_ipintelligence_translation/ + # The pipeline packages come from this checkout rather than a package + # index, so the tests run with the pipeline code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ + -e../../pipeline/fiftyone_pipeline_translation/ # --cov is needed even though coverage of the examples is not the point: # common-ci's run-integration-tests.ps1 moves the .coverage file every # package writes, and throws when there is none, after the tests pass. diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py index b15a227a..2d850cf7 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py @@ -277,5 +277,12 @@ def stage_native_source(): py_modules=["fiftyone_ipintelligence_onpremise"], packages=["fiftyone_ipintelligence_onpremise"], package_dir={"": "src"}, - install_requires=["fiftyone_ipintelligence_shared", "fiftyone_pipeline_engines_fiftyone"], + install_requires=[ + "fiftyone_ipintelligence_shared", + "fiftyone_pipeline_engines_fiftyone", + # The first versions with close(), which releases the native memory + # this package holds. + "fiftyone_pipeline_core>=4.5.42", + "fiftyone_pipeline_engines>=4.5.42", + ], ) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/constants.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/constants.py index 3e129185..deea6331 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/constants.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/constants.py @@ -29,3 +29,10 @@ 'on-premise IP intelligence engine. Its results ' 'read from native memory, which is not safe to ' 'share between requests.') + +ENGINE_CLOSED = ('The on-premise IP intelligence engine has been closed, ' + 'so it cannot process evidence. Build a new pipeline.') + +RESULTS_CLOSED = ('These IP intelligence results have been closed, so ' + 'their properties cannot be read. Read the values needed ' + 'before closing the FlowData.') diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index c724d64d..37b45904 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -328,6 +328,20 @@ def on_registration(self, pipeline): super(IpIntelligenceOnPremise, self).on_registration(pipeline) self.init_engine() + def close(self): + + """! + + Stop the data file updates and release the native engine. Its data + is freed once no results read from it are still held, so close + every FlowData the engine processed as well. Closing the pipeline + closes the engine. + + """ + + super(IpIntelligenceOnPremise, self).close() + self.engine = None + def get_client_ips(self, flow_data): """! @@ -379,6 +393,9 @@ def process_internal(self, flow_data): """ + if getattr(self, "engine", None) is None: + raise Exception(ENGINE_CLOSED) + evidence = EvidenceIpiSwig() for key, value in flow_data.evidence.get_all().items(): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index ea285c84..40413cf3 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -25,6 +25,8 @@ from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService +from .constants import RESULTS_CLOSED + # Number of decimal places used when converting WKB geometry # properties (e.g. Areas) to their WKT string form. WKT_DECIMAL_PLACES = 16 @@ -71,6 +73,33 @@ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): self.missing_property_service = OnPremiseMissingPropertyService() + def close(self): + + """! + Release the native results, which keep the engine's data in memory + while they are held, even after the engine is closed. Values already + read are unaffected, and reading a property afterwards raises an + exception. Closing the FlowData closes its results. + """ + + self.swig_results = None + + def get(self, key): + + """! + Get a property value, once the results are known to be open + @type key: string + @param key: The key of the property to retrieve + @returns mixed + """ + + # Checked here, because AspectData.get turns an exception from + # get_internal into a missing property message. + if self.swig_results is None: + raise Exception(RESULTS_CLOSED) + + return super(SwigData, self).get(key) + @staticmethod def utf8_string_to_str(utf8_string): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py new file mode 100644 index 00000000..fc53a6d0 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py @@ -0,0 +1,119 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Fails a test that leaves the native results or engine of an on-premise +# engine alive, because they hold the engine's data in memory. The same +# file is in each test folder that runs on-premise engines. +# +# Reference counting frees a native object as soon as nothing holds it. A +# FlowData refers to itself through its evidence, and a pipeline through +# its elements, so when they are not closed only the cycle collector frees +# what they hold. The collector is off while each test runs, so whatever +# the test leaves open is still alive when it ends. Objects that existed +# before the test, such as a pipeline built in setUpClass, are checked once +# every test has run. + +import gc +import importlib + +import pytest + +# The SWIG modules of the on-premise engines, whichever are installed. +NATIVE_MODULES = [ + "fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule", + "fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule", +] + + +def _native_types(): + types = [] + for name in NATIVE_MODULES: + try: + module = importlib.import_module(name) + except ImportError: + continue + types.append(module.ResultsBaseSwig) + types.append(module.EngineBaseSwig) + return tuple(types) + + +NATIVE_TYPES = _native_types() + +# A failed test is not reported again, because its traceback can hold the +# objects it would have closed. +_failed_tests = set() + +# Objects created when the tests were collected are checked at the end only +# when every test ran, because the teardown that closes them runs with the +# tests. +_some_tests_deselected = False + + +def _alive(): + # The real type is used, because isinstance also asks the object for + # its __class__, which some proxies cannot answer. + return [o for o in gc.get_objects() + if issubclass(type(o), NATIVE_TYPES)] + + +def _describe(names): + return (f"{len(names)} native object(s) still alive " + f"({', '.join(sorted(names))}). Close every FlowData and " + "pipeline the test creates, for example with a with " + "statement, so their native data is released straight away.") + + +def pytest_runtest_logreport(report): + if report.failed: + _failed_tests.add(report.nodeid) + + +def pytest_deselected(items): + global _some_tests_deselected + _some_tests_deselected = True + + +@pytest.fixture(autouse=True) +def native_objects_released(request): + gc.collect() + before = {id(o) for o in _alive()} + gc.disable() + try: + yield + left = [type(o).__name__ for o in _alive() if id(o) not in before] + finally: + gc.enable() + gc.collect() + if left and request.node.nodeid not in _failed_tests: + pytest.fail(_describe(left), pytrace=False) + + +def pytest_sessionfinish(session, exitstatus): + gc.collect() + left = [type(o).__name__ for o in _alive()] + if (left and exitstatus == pytest.ExitCode.OK and + not _some_tests_deselected): + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line( + "After the last test, " + _describe(left), red=True) + session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py index df9f835e..7b14231c 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py @@ -49,8 +49,13 @@ def setUpClass(cls): .add(cls.engine) \ .build() + @classmethod + def tearDownClass(cls): + cls.pipeline.close() + def process(self, key, value): flowdata = self.pipeline.create_flowdata() + self.addCleanup(flowdata.close) flowdata.evidence.add(key, value) flowdata.process() return flowdata @@ -133,6 +138,24 @@ def test_process_ipv6(self): ipv6 = flowdata.ip.get("ipv6") self.assertTrue(ipv6.has_value()) + def test_flowdata_close_releases_results(self): + """! + Closing the FlowData releases its native results, and reading a + property afterwards raises an exception. + """ + + with self.pipeline.create_flowdata() as flowdata: + flowdata.evidence.add( + "query.client-ip", TestConstants.IPV4_ADDRESS) + flowdata.process() + registered_name = flowdata.ip.get("registeredname") + + self.assertTrue(registered_name.has_value()) + self.assertIsNone(flowdata.ip.swig_results) + with self.assertRaises(Exception) as context: + flowdata.ip.get("registeredname") + self.assertIn("closed", str(context.exception)) + def test_missing_property(self): """! Accessing a property that does not exist raises an exception @@ -208,10 +231,12 @@ def test_asn_data_file_opens(self): data_file_path=TestConstants.get_asn_data_file_path(), licence_keys="") pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) self.assertIn("asnname", engine.properties) flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) flowdata.evidence.add("query.client-ip", TestConstants.IPV4_ADDRESS) flowdata.process() @@ -262,3 +287,31 @@ def test_pipeline_builder_refuses_cache(self): cache=LRUEngineCache()).build() self.assertIn("results cache", str(context.exception)) + + +@unittest.skipUnless( + os.path.isfile(data_file), + "No IP intelligence data file found at '%s'. Run ci/fetch-assets.ps1 " + "or set the IPINTELLIGENCEDATAFILE environment variable." % data_file) +class IpIntelligenceCloseTests(unittest.TestCase): + + def test_pipeline_close_releases_engine(self): + """! + Closing the pipeline releases the native engine, and processing + afterwards raises an exception. + """ + + engine = IpIntelligenceOnPremise( + data_file_path=data_file, + licence_keys="") + pipeline = PipelineBuilder().add(engine).build() + + pipeline.close() + + self.assertIsNone(engine.engine) + flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) + flowdata.evidence.add("query.client-ip", TestConstants.IPV4_ADDRESS) + with self.assertRaises(Exception) as context: + flowdata.process() + self.assertIn("closed", str(context.exception)) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini b/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini index 3dc3b4c9..211e4571 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini @@ -24,6 +24,12 @@ deps = # of it in beside the wheel tox builds and made examples and on-premise # depend on each other. -e../fiftyone_ipintelligence_shared/ + # The pipeline packages come from this checkout rather than a package + # index, so the tests run with the pipeline code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = python setup.py build_clib build_ext {[common]commands} diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py index daa6b387..c6f83c7c 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py @@ -94,6 +94,34 @@ def process(self): return self + def close(self): + + """! + Release the native resources held by the element data in this + FlowData, such as the results of an on-premise engine, which keep + the engine's data in memory until they are released. Use a with + statement so that this happens as soon as the FlowData is finished + with, rather than when the garbage collector next runs. + + Only element data with a close method of its own is closed, so + element data that needs no cleanup, such as an entry in a results + cache, is left alone. Values already read are unaffected. + + """ + + for element_data in list(self.data.values()): + # Looked up on the class, because ElementData treats an + # attribute it does not have as the name of a property. + close = getattr(type(element_data), "close", None) + if callable(close): + close(element_data) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + def get_from_element(self, flow_element): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py index 87ee2800..73e6922d 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py @@ -77,6 +77,17 @@ def on_registration(self, pipeline): pass + def close(self): + + """! + Release the resources the element holds. The pipeline calls this + when it is closed. This element holds none, and elements that hold + native data or threads override it. + + """ + + pass + def get_evidence_key_filter(self): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py index cac6aea1..4ee68108 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py @@ -33,7 +33,7 @@ class Pipeline: """ - def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = False, data_file_update_service = None): + def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = False, data_file_update_service = None, auto_close_elements = True): """! Pipeline constructor. @@ -49,6 +49,12 @@ def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = exceptions added to FlowData.errors otherwise will throw the exception occurred during the processing of first element. + @type auto_close_elements: boolean + @param auto_close_elements: If true, the default, closing the + pipeline closes its flow elements. Set it to false when the elements + are shared with another pipeline, and close each element once every + pipeline using it is finished with. + @rtype: Pipeline @return: Returns a Pipeline @@ -63,6 +69,8 @@ def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = self.flow_elements_display_list = [] self.suppress_process_exceptions = suppress_process_exceptions + + self.auto_close_elements = auto_close_elements if data_file_update_service != None: self.data_file_update_service = data_file_update_service @@ -92,6 +100,44 @@ def create_flowdata(self): return FlowData(self) + def close(self): + + """! + Close the flow elements, which releases the native data they hold + and stops the updates of their data files, unless the pipeline was + built with auto_close_elements set to false. Use a with statement + so that this happens when the pipeline is finished with. + + FlowData created by the pipeline are not closed by it. Close each + one as well, because the results it holds keep the engine's data in + memory until they are released. + + """ + + if not self.auto_close_elements: + return + + # Every element is closed even if one fails, and the first failure + # is raised once they all have been. + failure = None + for flow_element in self.flow_elements: + close = getattr(flow_element, "close", None) + if close is None: + continue + try: + close() + except Exception as e: + if failure is None: + failure = e + if failure is not None: + raise failure + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + def log(self, level, message): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py index 72f89752..c3fa52e0 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py @@ -52,6 +52,9 @@ def __init__(self, settings= {}): while processing is re-raised from flow_data.process(); if True, errors are stored on flow_data.errors and process() returns normally. Recommended True for web apps so a processing failure degrades gracefully instead of failing the request. + `auto_close_elements (Bool)` - Whether closing the pipeline closes its flow elements + (default true). Set to False when the elements are shared with another pipeline, + and close each element once every pipeline using it is finished with. @return: Returns a Pipeline Builder """ @@ -82,6 +85,11 @@ def __init__(self, settings= {}): else: self.suppress_process_exceptions = False + if "auto_close_elements" in settings: + self.auto_close_elements = settings["auto_close_elements"] + else: + self.auto_close_elements = True + def get_javascript_elements(self): """ Adds the JavaScriptBuilder, JSONBundler and SequenceElement to the pipeline if @@ -148,7 +156,7 @@ def build(self): self.flow_elements.extend(self.get_javascript_elements()) self.flow_elements.extend(self.get_setheader_elements()) - return Pipeline(self.flow_elements, logger=self.logger, suppress_process_exceptions=self.suppress_process_exceptions, data_file_update_service=self.data_file_update_service) + return Pipeline(self.flow_elements, logger=self.logger, suppress_process_exceptions=self.suppress_process_exceptions, data_file_update_service=self.data_file_update_service, auto_close_elements=self.auto_close_elements) def add_logger(self, logger): """ @@ -176,27 +184,46 @@ def build_from_configuration(self, config): @rtype: Pipeline @return: Returns built pipeline + If an element fails to build, the elements already built are closed + and the error is raised. + """ flow_elements = [] - for element in config["PipelineOptions"]["Elements"]: - flow_element = importlib.import_module(element["elementPath"]).__getattribute__(element["elementName"]) - if("elementParameters" in element): - arguments = self._map_properties_names( - { - "verify_md5": "data_update_verify_md5", - "data_file_update_base_url": "data_update_url" - }, - element["elementParameters"], - ) - - flow_element = flow_element(**arguments) - else: - flow_element = flow_element() - flow_elements.append(flow_element) - - return Pipeline(flow_elements, suppress_process_exceptions=self._get_suppress_process_exceptions(config)) + try: + for element in config["PipelineOptions"]["Elements"]: + flow_element = importlib.import_module(element["elementPath"]).__getattribute__(element["elementName"]) + if("elementParameters" in element): + arguments = self._map_properties_names( + { + "verify_md5": "data_update_verify_md5", + "data_file_update_base_url": "data_update_url" + }, + element["elementParameters"], + ) + + flow_element = flow_element(**arguments) + else: + flow_element = flow_element() + flow_elements.append(flow_element) + + return Pipeline(flow_elements, suppress_process_exceptions=self._get_suppress_process_exceptions(config), auto_close_elements=self.auto_close_elements) + except BaseException: + # A failure to close is logged rather than raised, so the caller + # sees the error that stopped the build. + for flow_element in flow_elements: + close = getattr(flow_element, "close", None) + if close is None: + continue + try: + close() + except Exception as e: + self.logger.log( + "error", + "Could not close {} after the pipeline failed to " + "build: {}".format(type(flow_element).__name__, e)) + raise def _get_suppress_process_exceptions(self, config): """ diff --git a/pipeline/fiftyone_pipeline_core/tests/test_close.py b/pipeline/fiftyone_pipeline_core/tests/test_close.py new file mode 100644 index 00000000..e99583d5 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_close.py @@ -0,0 +1,182 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_pipeline_core.elementdata import ElementData +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + + +class ClosingData(ElementData): + """Element data with a close method of its own, as the results of an + on-premise engine have.""" + + def __init__(self, flow_element): + super(ClosingData, self).__init__(flow_element) + self.closed = False + + def close(self): + self.closed = True + + +class ClosingElement(FlowElement): + """Adds ClosingData to each FlowData, and counts its own closes.""" + + def __init__(self): + super(ClosingElement, self).__init__() + self.datakey = "closing" + self.closes = 0 + + def process_internal(self, flowdata): + flowdata.set_element_data(ClosingData(self)) + + def close(self): + self.closes += 1 + + +class PlainElement(FlowElement): + """Adds element data that needs no cleanup.""" + + def __init__(self): + super(PlainElement, self).__init__() + self.datakey = "plain" + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary(self, {"close": 1})) + + +class FailingElement(FlowElement): + """Fails to close.""" + + def __init__(self): + super(FailingElement, self).__init__() + self.datakey = "failing" + + def close(self): + raise Exception("close failed") + + +class RecordedElement(FlowElement): + """Records each instance that is closed, for a test whose pipeline + failed to build and so cannot be reached.""" + + closed = [] + + def __init__(self): + super(RecordedElement, self).__init__() + self.datakey = "recorded" + + def close(self): + RecordedElement.closed.append(self) + + +class BrokenElement(FlowElement): + """Fails to build.""" + + def __init__(self): + raise ValueError("element failed to build") + + +def build(*elements, **settings): + settings["add_javascript_builder"] = False + settings["use_setheader_properties"] = False + builder = PipelineBuilder(settings) + for element in elements: + builder.add(element) + return builder.build() + + +class CloseTests(unittest.TestCase): + + # Element data with a close method is closed with the FlowData, and + # element data without one is left alone, even with a property called + # close. + def test_flowdata_closes_element_data(self): + with build(ClosingElement(), PlainElement()) as pipeline: + with pipeline.create_flowdata() as flowdata: + flowdata.process() + closing = flowdata.get("closing") + self.assertFalse(closing.closed) + self.assertTrue(closing.closed) + self.assertEqual(1, flowdata.get("plain").get("close")) + + # The with statement closes the FlowData when an exception is raised. + def test_flowdata_closed_on_exception(self): + with build(ClosingElement()) as pipeline: + with self.assertRaises(ValueError): + with pipeline.create_flowdata() as flowdata: + flowdata.process() + closing = flowdata.get("closing") + raise ValueError() + self.assertTrue(closing.closed) + + # A FlowData that was never processed holds nothing to close. + def test_unprocessed_flowdata_closes(self): + with build(ClosingElement()) as pipeline: + pipeline.create_flowdata().close() + + def test_pipeline_closes_elements(self): + element = ClosingElement() + with build(element): + self.assertEqual(0, element.closes) + self.assertEqual(1, element.closes) + + # Elements shared with another pipeline are left for the caller to + # close. + def test_pipeline_leaves_elements_open(self): + element = ClosingElement() + with build(element, auto_close_elements=False): + pass + self.assertEqual(0, element.closes) + + # Every element is closed even when one fails, and the failure is + # raised afterwards. + def test_pipeline_closes_every_element(self): + element = ClosingElement() + pipeline = build(FailingElement(), element) + with self.assertRaises(Exception) as context: + pipeline.close() + self.assertEqual("close failed", str(context.exception)) + self.assertEqual(1, element.closes) + + # When an element in a configuration fails to build, the elements built + # before it are closed, and the error that stopped the build is raised. + def test_failed_configuration_closes_built_elements(self): + RecordedElement.closed = [] + config = { + "PipelineOptions": { + "Elements": [ + {"elementName": "RecordedElement", "elementPath": __name__}, + {"elementName": "BrokenElement", "elementPath": __name__} + ] + } + } + with self.assertRaises(ValueError) as context: + PipelineBuilder().build_from_configuration(config) + self.assertEqual("element failed to build", str(context.exception)) + self.assertEqual(1, len(RecordedElement.closed)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py index 59168577..c4c6b047 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py @@ -90,6 +90,10 @@ def update_data_file(self, data_file): @param data_file: Datafile to update """ + # A data file whose engine has been closed is not updated. + if not self.__is_registered(data_file): + return False + # exit if already updating if data_file.updating: @@ -230,6 +234,9 @@ def check_next_update(self, data_file, run_after_error = False): @param data_file: Datafile to check updates for """ + if not self.__is_registered(data_file): + return + if data_file.update_on_start and not data_file.updated_on_start and not run_after_error: self.update_data_file(data_file) @@ -241,6 +248,7 @@ def check_next_update(self, data_file, run_after_error = False): t = threading.Timer(interval, self.async_update_data_file, args=[data_file]) t.daemon = True t.name = "AutoUpdate - Scheduled" + data_file.update_timer = t t.start() def register_pipeline(self, pipeline): @@ -265,6 +273,26 @@ def register_data_file(self, data_file): data_file.last_changed_time = os.stat(data_file.path).st_mtime self.track_local_data_file_changes(data_file) + def unregister_data_file(self, data_file): + """! + Stop the checks and updates of a datafile, for example when its + engine is closed. Checks already scheduled are cancelled, and a + check already running schedules no other, so nothing is left + holding the data file or its engine. + @type data_file: DataFile + @param data_file: Datafile to stop checking + """ + + data_file.registered = False + for name in ("update_timer", "watcher_timer"): + timer = getattr(data_file, name, None) + if timer is not None: + timer.cancel() + + def __is_registered(self, data_file): + # A data file checked without being registered first is checked as + # before. Only one that has been unregistered is left alone. + return getattr(data_file, "registered", True) def track_local_data_file_changes(self, data_file): """! @@ -272,10 +300,12 @@ def track_local_data_file_changes(self, data_file): the file system watcher. A refresh that fails is logged, and reported with AUTO_UPDATE_REFRESH_FAILED as on the update route. The next check is scheduled whatever happens, so that a later file - is still picked up. + is still picked up, until the data file is unregistered. @type data_file: DataFile @param data_file: Datafile to check updates for """ + if not self.__is_registered(data_file): + return try: changed_time = os.stat(data_file.path).st_mtime except OSError: @@ -296,13 +326,15 @@ def track_local_data_file_changes(self, data_file): "' failed. " + str(e)) self.__emit(UpdateStatus.AUTO_UPDATE_REFRESH_FAILED, data_file) finally: - t = threading.Timer( - self.FILE_SYSTEM_WATCHER_INTERVAL_SECONDS, - self.track_local_data_file_changes, - args=[data_file]) - t.daemon = True - t.name = "AutoUpdate - FileWatcher" - t.start() + if self.__is_registered(data_file): + t = threading.Timer( + self.FILE_SYSTEM_WATCHER_INTERVAL_SECONDS, + self.track_local_data_file_changes, + args=[data_file]) + t.daemon = True + t.name = "AutoUpdate - FileWatcher" + data_file.watcher_timer = t + t.start() def on_complete(self, on_complete_method): self.on_complete_methods.append(on_complete_method) diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py index f22f654c..5ef3556a 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py @@ -150,6 +150,22 @@ def on_registration(self, pipeline): pipeline.data_file_update_service = DataFileUpdateService(pipeline) pipeline.data_file_update_service.register_data_file(self.data_file) + def close(self): + + """! + + Stop the checks and updates of the engine's data file, whose timers + would otherwise keep the engine alive. Engines that hold native data + release it as well. + + """ + + if self.data_file: + for pipeline in self.pipelines: + service = getattr(pipeline, "data_file_update_service", None) + if service is not None: + service.unregister_data_file(self.data_file) + def register_data_file(self, data_file): """! diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py index af290377..5fd9c872 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py @@ -73,10 +73,16 @@ def __init__(self, pending, function, args): self.pending = pending self.function = function self.args = args + self.cancelled = False def start(self): self.pending.append(self) + def cancel(self): + self.cancelled = True + if self in self.pending: + self.pending.remove(self) + class FileSystemWatcherTests(unittest.TestCase): @@ -149,6 +155,39 @@ def testFileSystemWatcherKeepsWatchingWhileFileIsMissing(self): self.assertNotIn( UpdateStatus.AUTO_UPDATE_REFRESH_FAILED, self.statuses) + # Unregistering, as closing the engine does, cancels the next check, and + # a check already running neither refreshes nor schedules another, so + # nothing is left holding the data file. + def testUnregisterStopsFileSystemWatcher(self): + data_file = StandInDataFile(self.path) + self.service.register_data_file(data_file) + check = self.pending[0] + + self.service.unregister_data_file(data_file) + self.assertTrue(check.cancelled) + self.assertEqual(0, len(self.pending)) + + self.write("version 2") + check.function(*check.args) + self.assertEqual("version 1", data_file.data) + self.assertEqual(0, len(self.pending)) + + # Unregistering cancels the next automatic update, and an update that + # starts afterwards does nothing. + def testUnregisterStopsAutomaticUpdates(self): + data_file = StandInDataFile(self.path) + data_file.file_system_watcher = False + data_file.auto_update = True + data_file.polling_interval = 30 + data_file.update_time_maximum_randomisation = 0 + self.service.register_data_file(data_file) + check = self.pending[0] + + self.service.unregister_data_file(data_file) + self.assertTrue(check.cancelled) + self.assertEqual(0, len(self.pending)) + self.assertFalse(self.service.update_data_file(data_file)) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 55ef341f..3173318e 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -122,6 +122,17 @@ def __init__( def __del__(self): self.thread_pool.shutdown(wait=True) + def close(self): + + """! + Wait for any usage data already being sent, then stop the threads + that send it. The pipeline calls this when it is closed. + + """ + + super(ShareUsage, self).close() + self.thread_pool.shutdown(wait=True) + def get_constant_xml(self): if self.constant_xml is None: coreVersion = version("fiftyone_pipeline_core") From dfa67572d385c799e985d6f9dfbc12413e95e6d3 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 23:04:16 +0000 Subject: [PATCH 12/68] Look for the resource keys link the cloud engine now gives in the cloud tests (#38) --- .../fiftyone_devicedetection_cloud/tests/test_cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py index 3151778e..c5369df1 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py @@ -121,7 +121,7 @@ def test_missing_property_service_element_not_found(self): result = str(raised.exception) self.assertIn("notpresent", result) self.assertIn("['device']", result) - self.assertIn("https://51degrees.com/documentation/_info__resource_keys.html", result) + self.assertIn("https://51degrees.com/documentation/_services__cloud__resource_keys.html", result) def test_engine_init_performance(self): """! @@ -172,7 +172,7 @@ def test_missing_property_service_not_found_anywhere(self): self.assertIn("notpresent", result) self.assertIn("device", result) self.assertIn(', '.join(pipeline.get_element("device").get_properties().keys()), result) - self.assertIn("https://51degrees.com/documentation/_info__resource_keys.html", result) + self.assertIn("https://51degrees.com/documentation/_services__cloud__resource_keys.html", result) def test_cloud_request_origin(self): """! From 8b03b74394e3f9e9e5a6784005b130cd96328c24 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 5 Nov 2026 06:58:21 +0000 Subject: [PATCH 13/68] Allow only http and https in the callback URL, and test the zone and port rules (#40) --- .../fiftyone_pipeline_core/callback_url.py | 54 ++++++++++--- .../javascriptbuilder.py | 18 ++++- .../tests/test_callback_url.py | 77 ++++++++++++++++++- .../tests/test_javascriptbuilder_variables.py | 49 ++++++++++++ 4 files changed, 182 insertions(+), 16 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py index afafa240..b18fbb13 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py @@ -24,9 +24,10 @@ The callback URL the client script posts to for updated JSON. The protocol and host usually come from the request, and the script holds the URL in a single-quoted string, so the URL is checked before it is written into the -script. A URL that is not valid is refused. In the user information, in -anything after the host and port, and in the endpoint, the characters a URL -cannot hold or that could end the string or the script are percent-encoded. +script. A URL that is not valid, including one whose protocol is not http or +https, is refused. In the user information, in anything after the host and +port, and in the endpoint, the characters a URL cannot hold or that could end +the string or the script are percent-encoded. See https://github.com/51Degrees/specifications/blob/main/pipeline-specification/pipeline-elements/javascript-builder.md#callback-url """ @@ -58,9 +59,13 @@ r"[\t-\r \u0085\u00a0\u1680\u2000-\u200a" r"\u2028\u2029\u202f\u205f\u3000]*") -# A scheme as RFC 3986 section 3.1 defines one, being a letter followed by -# letters, digits, '+', '-' and '.'. -SCHEME_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*") +# The schemes the script can send its request with. It uses the URL only with +# fetch and XMLHttpRequest, which reach a server with no other scheme. +SCHEMES = ("http", "https") + +# Lower-cases the ASCII letters A to Z and nothing else. +_ASCII_LOWER = str.maketrans( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") # Where the authority ends, which is at the first '/', '?' or '#' of the # host. @@ -105,6 +110,37 @@ def is_blank(text): or BLANK_PATTERN.fullmatch(text) is not None) +def ascii_lower(text): + + """! + The text with the ASCII letters A to Z in lower case and every other + character as it is, so that no letter outside ASCII can stand in for an + ASCII one, as str.casefold would let the long s stand in for "s". + + @type text: str + @param text: The text + @rtype: str + @return: The text with A to Z in lower case + """ + + return text.translate(_ASCII_LOWER) + + +def is_scheme(protocol): + + """! + Whether the protocol is http or https, ignoring the case of its ASCII + letters only. + + @type protocol: str + @param protocol: The protocol + @rtype: bool + @return: True if the protocol is http or https + """ + + return ascii_lower(protocol) in SCHEMES + + def callback_url(protocol, host, endpoint): """! @@ -114,7 +150,7 @@ def callback_url(protocol, host, endpoint): the rest as given, apart from the percent-encoding. @type protocol: str - @param protocol: The protocol, such as https + @param protocol: The protocol, http or https in any case @type host: str @param host: The host, which can start with user information and end with a port, a path, a query or a fragment @@ -134,7 +170,7 @@ def callback_url(protocol, host, endpoint): elif host_has_slash and endpoint_has_slash: endpoint = endpoint[1:] built = protocol + "://" + host + endpoint - if SCHEME_PATTERN.fullmatch(protocol) is None: + if not is_scheme(protocol): return CallbackUrl(False, built) match = AUTHORITY_END_PATTERN.search(host) end = len(host) if match is None else match.start() @@ -143,7 +179,7 @@ def callback_url(protocol, host, endpoint): return CallbackUrl(False, built) return CallbackUrl( True, - protocol.lower() + "://" + authority + ascii_lower(protocol) + "://" + authority + _encoded(host[end:]) + _encoded(endpoint)) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py index 31d063a7..bdb85c50 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py @@ -38,7 +38,7 @@ from .evidence_keyfilter import EvidenceKeyFilter from .elementdata_dictionary import ElementDataDictionary from .constants import Constants -from .callback_url import callback_url +from .callback_url import callback_url, is_scheme # Evidence keys the rendered script is not configured with. Both are @@ -255,9 +255,10 @@ def __init__(self, settings = {} ): * starting with a digit) and not a reserved word. An invalid * configured name raises a ValueError, and an invalid requested name * is ignored with a warning logged. - * @param {string} options.protocol The protocol ("http" or "https") - * used by the client side callback url. - * This can be overriden with header.protocol evidence + * @param {string} options.protocol The protocol of the client side + * callback url, http or https in any case of its ASCII letters. Any + * other configured protocol raises a ValueError. Left out, the + * header.protocol evidence is used, or https when there is none. * @param {string} options.host The host of the client side * callback url. This can be overriden with header.host evidence. * @param {string} options.endpoint The endpoint of the client side @@ -286,6 +287,15 @@ def __init__(self, settings = {} ): "JavaScript identifier (ASCII letters, digits, '_' and '$', " "not starting with a digit) and not a reserved word.") self.settings['_protocol'] = settings["protocol"] if "protocol" in settings else None + + # A protocol given here is used for every request, so one the script + # cannot send its request with is refused now rather than logged on + # the first request. + protocol = self.settings['_protocol'] + if protocol and not (isinstance(protocol, str) and is_scheme(protocol)): + raise ValueError( + "The JavaScript builder's protocol setting must be http or " + "https.") self.settings['_host'] = settings["host"] if "host" in settings else None self.settings['_endpoint'] = settings["endpoint"] if "endpoint" in settings else '' self.settings['_enableCookies'] = settings["enable_cookies"] if "enable_cookies" in settings else True diff --git a/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py index 9048401d..129fee0d 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py @@ -137,19 +137,90 @@ def test_accepts_the_host(self, host, expected): def test_refuses_the_host(self, host): self.assertFalse(_url("https", host).valid, repr(host)) + @parameterized.expand([ + ("[fe80::1%25]",), + ("[fe80::1%25'+alert(1)+']",), + ("[fe80::1%eth0]",), + ("[fe80::1%25eth%0]",), + ("[::1]x",), + ("[::1]80",), + ("[::1]:80x",), + ("[::1]]",), + ]) + def test_refuses_the_zone_or_the_text_after_the_bracket(self, host): + """Inside the brackets only an IPv6 address, optionally followed by + '%25' and a zone of unreserved characters, and after them only a + port. The zone check is what keeps a quote out of the script for a + bracketed host.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + def test_accepts_a_zone_followed_by_a_port(self): + self.assertEqual( + _valid("https://[fe80::1%25eth0]:8080/51dpipeline/json"), + _url("https", "[fe80::1%25eth0]:8080")) + + def test_checks_the_port_by_its_value(self): + """A port goes by its value, so leading zeros are allowed, and a + port too long to read is refused rather than read as small enough.""" + + self.assertEqual( + _valid("https://example.com:00080/51dpipeline/json"), + _url("https", "example.com:00080")) + self.assertEqual( + _valid("https://example.com:0065535/51dpipeline/json"), + _url("https", "example.com:0065535")) + for host in [ + "example.com:0065536", + "example.com:123456", + "example.com:4294967296", + "example.com:99999999999999999999"]: + self.assertFalse(_url("https", host).valid, host) + + def test_encodes_the_characters_rule_8_lists(self): + """Each character rule 8 lists, in the user information, after the + host and in the endpoint, and a '%' without two hexadecimal digits + after it, even at the end.""" + + self.assertEqual( + _valid("https://u%22%5E%60%7B%7C%7D@example.com/%22%5E%60%7B%7C%7D" + "/j%22%5E%60%7B%7C%7D"), + callback_url( + "https", 'u"^`{|}@example.com/"^`{|}', '/j"^`{|}')) + self.assertEqual( + _valid("https://example.com/a%25/51dpipeline/json"), + _url("https", "example.com/a%")) + self.assertEqual( + _valid("https://example.com/a%254/51dpipeline/json"), + _url("https", "example.com/a%4")) + self.assertEqual( + _valid("https://example.com/a%41/51dpipeline/json"), + _url("https", "example.com/a%41")) + self.assertEqual( + _valid("https://example.com/json%25"), + callback_url("https", "example.com", "/json%")) + def test_the_invalid_url_is_the_one_built(self): self.assertEqual( CallbackUrl(False, "https://exa mple.com/51dpipeline/json"), _url("https", "exa mple.com")) def test_checks_the_scheme(self): - for protocol in ["https", "http", "ftp", "h+t-p.s"]: + """Only http and https, ignoring the case of ASCII letters only, so + a letter outside ASCII that a case-insensitive comparison can match + to an ASCII one, such as the long s, is refused.""" + + for protocol in ["https", "http", "HtTp"]: self.assertTrue(_url(protocol, "example.com").valid, protocol) self.assertEqual( _valid("https://example.com/51dpipeline/json"), _url("HTTPS", "example.com")) - for protocol in ["1http", "h t", "x');alert(1);//", "a://b"]: - self.assertFalse(_url(protocol, "example.com").valid, protocol) + for protocol in [ + "ftp", "h+t-p.s", "ws", "javascript", "httpx", "https ", + "1http", "h t", "x');alert(1);//", "a://b", "http\u017f", + "HTTP\u017f"]: + self.assertFalse( + _url(protocol, "example.com").valid, repr(protocol)) @parameterized.expand([ ("example.com/');alert(1);//",), diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py index 4c9302f4..7b601e61 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py @@ -24,6 +24,8 @@ import unittest from unittest import mock +from parameterized import parameterized + from fiftyone_pipeline_core import javascriptbuilder from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder @@ -211,6 +213,33 @@ def test_refused_url_is_logged_on_one_line(self): self.assertNotIn(c, errors[0]) self.assertIn("a\\nb.com\\r\\u2028", errors[0]) + def test_refused_protocol_from_the_request_is_left_out_and_logged_once( + self): + """A protocol from the request that is neither http nor https gives + a script with no callback, and is logged once like any other URL + that is not valid.""" + + logger = MemoryLogger("error") + pipeline = PipelineBuilder({"javascript_builder_settings": { + "minify": False, "endpoint": "/json"}}).add_logger(logger).build() + captured = _CapturedAll() + with mock.patch.object( + javascriptbuilder.chevron, "render", captured.render): + for _ in range(2): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.host", "example.com") + flowdata.evidence.add("header.protocol", "ftp") + flowdata.process() + errors = [ + entry["message"] for entry in logger.memory_log + if entry["level"] == "error"] + + for variables in captured.variables: + self.assertNotIn("_url", variables) + self.assertFalse(variables["_updateEnabled"]) + self.assertEqual(1, len(errors), errors) + self.assertIn("'ftp://example.com/json'", errors[0]) + def test_blank_host_gives_no_callback_and_is_not_logged(self): """A host made only of whitespace is not set, so the script has no callback and nothing is logged.""" @@ -232,5 +261,25 @@ def test_host_from_the_request_is_written_as_checked(self): "https://example.com/%27);alert(1);//json", rendered[0]["_url"]) +class JavaScriptBuilderProtocolSettingTests(unittest.TestCase): + + @parameterized.expand([ + ("ftp",), ("ws",), ("javascript",), ("https ",), ("http\u017f",), + ]) + def test_invalid_configured_protocol_is_refused(self, protocol): + """A protocol in the settings is used for every request, so one the + script cannot send its request with is refused when the builder is + created.""" + + with self.assertRaises(ValueError): + javascriptbuilder.JavascriptBuilderElement({"protocol": protocol}) + + @parameterized.expand([("http",), ("https",), ("HTTPS",)]) + def test_valid_configured_protocol_is_accepted(self, protocol): + element = javascriptbuilder.JavascriptBuilderElement( + {"protocol": protocol}) + self.assertEqual(protocol, element.settings["_protocol"]) + + if __name__ == "__main__": unittest.main() From 081386da8930066d81e2c78942f1499335e6cb45 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 5 Nov 2026 08:29:31 +0000 Subject: [PATCH 14/68] Test every package against the packages it depends on in this checkout (#41) --- ci/packages.Tests.ps1 | 5 +++ ci/packages.ps1 | 45 +++++++++++++++++++ ci/run-unit-tests.ps1 | 10 +++++ .../fiftyone_devicedetection_cloud/tox.ini | 6 +++ .../fiftyone_ipintelligence_cloud/tox.ini | 6 +++ .../fiftyone_ipintelligence_shared/tox.ini | 6 +++ .../tox.ini | 4 ++ location/tox.ini | 5 +++ 8 files changed, 87 insertions(+) diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 77ecd47e..2a2887d9 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -285,6 +285,11 @@ Describe 'the real packages.json' { $ipi.Count | Should -Be 5 $ipi.FirstVersion | Select-Object -Unique | Should -Be '4.5.0' } + + It 'has every tox.ini install the packages its package depends on' { + Get-ToxGaps (Split-Path $PSScriptRoot) $script:Real | + Should -BeNullOrEmpty + } } } diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 5043d0aa..70e6af0c 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -30,6 +30,51 @@ function Test-PathUnder([string]$Path, [string]$Folder) { return $Path -eq $Folder -or $Path.StartsWith("$Folder/") } +# Each tox.ini that does not install every package its package depends on, +# directly or through another, from this checkout, as a line naming what is +# missing. Taken from a package index instead, a dependency is tested at its +# last release rather than with the code it is released with. +function Get-ToxGaps([string]$RepoPath, $Packages = $null) { + if ($null -eq $Packages) { + $Packages = Get-Packages $RepoPath + } + $byName = @{} + foreach ($package in $Packages) { + $byName[$package.Name] = $package + } + $gaps = @() + foreach ($package in $Packages) { + $tox = Join-Path $RepoPath $package.Path 'tox.ini' + if ((Test-Path $tox) -eq $false) { + continue + } + $text = Get-Content -Raw $tox + $needed = [Collections.Generic.HashSet[string]]::new() + $queue = [Collections.Generic.Queue[string]]::new() + foreach ($name in $package.DependsOn) { + $queue.Enqueue($name) + } + while ($queue.Count -gt 0) { + $name = $queue.Dequeue() + if ($needed.Add($name) -eq $true) { + foreach ($next in $byName[$name].DependsOn) { + $queue.Enqueue($next) + } + } + } + foreach ($name in $needed) { + $relative = [IO.Path]::GetRelativePath( + (Join-Path $RepoPath $package.Path), + (Join-Path $RepoPath $byName[$name].Path)).Replace('\', '/') + $pattern = '(?m)^\s*-e\s*' + [regex]::Escape($relative) + '/?\s*$' + if ($text -notmatch $pattern) { + $gaps += "$($package.Path)/tox.ini does not install $name from $relative" + } + } + } + return $gaps +} + # The names of the given packages and of every package that depends on one of # them, directly or through another. function Get-DependentClosure($Packages, [string[]]$Names) { diff --git a/ci/run-unit-tests.ps1 b/ci/run-unit-tests.ps1 index 77ad8cad..e74a60ff 100644 --- a/ci/run-unit-tests.ps1 +++ b/ci/run-unit-tests.ps1 @@ -3,5 +3,15 @@ param ( ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') +. (Join-Path $PSScriptRoot 'packages.ps1') + +# Each package's tests take the packages it depends on from this checkout, so +# they test the code those packages are released with. A tox.ini that takes +# one from PyPI instead stops the tests before they run. +$gaps = @(Get-ToxGaps (Split-Path $PSScriptRoot)) +if ($gaps.Count -gt 0) { + throw "A tox.ini takes a package from PyPI that this checkout has:`n" + + ($gaps -join "`n") +} Invoke-ProductStep 'run-unit-tests' $RepoName $PSBoundParameters diff --git a/device-detection/fiftyone_devicedetection_cloud/tox.ini b/device-detection/fiftyone_devicedetection_cloud/tox.ini index 14fad039..a0d62c6b 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tox.ini +++ b/device-detection/fiftyone_devicedetection_cloud/tox.ini @@ -12,7 +12,13 @@ package = wheel wheel_build_env = .pkg deps = {[common]deps} + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. -e../fiftyone_devicedetection_shared/ + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_devicedetection_cloud {tty:--color=yes} {posargs} pass_env = diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini b/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini index 4aa63411..668932c9 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini @@ -12,7 +12,13 @@ package = wheel wheel_build_env = .pkg deps = {[common]deps} + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. -e../fiftyone_ipintelligence_shared/ + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_cloud {tty:--color=yes} {posargs} pass_env = diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini b/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini index 2bf335a2..98972fd0 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini @@ -12,6 +12,12 @@ package = wheel wheel_build_env = .pkg deps = {[common]deps} + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_shared {tty:--color=yes} {posargs} pass_env = diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini b/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini index 11283b9e..b055ee03 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini @@ -12,6 +12,10 @@ package = wheel wheel_build_env = .pkg deps = {[common]deps} + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_translation/ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_translation {tty:--color=yes} {posargs} pass_env = diff --git a/location/tox.ini b/location/tox.ini index 3cd28ff4..504de885 100644 --- a/location/tox.ini +++ b/location/tox.ini @@ -8,6 +8,11 @@ wheel_build_env = .pkg deps = pytest>=6 pytest-cov + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. + -e../pipeline/fiftyone_pipeline_core/ + -e../pipeline/fiftyone_pipeline_engines/ + -e../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_location {tty:--color=yes} {posargs} pass_env = From e14dcda99ecaa220d4f1c0e37bfa8046ef50b20c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 00:23:38 +0000 Subject: [PATCH 15/68] Use a name in the object name test that a session id cannot contain (#44) --- .../tests/test_javascriptbuilder_object_name.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py index 92f95380..8342461f 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py @@ -151,7 +151,9 @@ def test_valid_name_from_evidence_minified(self): @parameterized.expand([ ["statement", "a;b", True], - ["leading_digit", "9bad", True], + # The name has a letter that is not a hex digit, so the random + # session id the script also carries can never contain it. + ["leading_digit", "9zz", True], ["quote", 'x"y', True], ["empty", "", False], # A reserved word is also an ordinary word in the script's comments, From c7c45b5186d2b65ceafa9b5ab5b46eae14676c11 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 01:08:14 +0000 Subject: [PATCH 16/68] Run every example on the free data files, and show what needs the Enterprise data file (#43) --- .../readme.md | 19 ++ .../example_utils.py | 129 +++++++++--- .../onpremise/csv_profile_metadata.py | 15 +- .../onpremise/datafileupdate_console.py | 47 ++++- .../onpremise/gettingstarted_console.py | 35 +++- .../gettingstarted_web/templates/index.html | 26 +-- .../onpremise/match_metrics.py | 17 +- .../onpremise/metadata_console.py | 11 +- .../onpremise/offlineprocessing.py | 62 ++++-- .../onpremise/performance.py | 22 +- .../onpremise/profile_metadata_console.py | 15 +- .../templates/index.html | 18 +- .../tests/test_examples_without_settings.py | 140 +++++++++++++ ip-intelligence/ci/fetch-assets.ps1 | 5 + .../examples/gettingstarted.py | 2 +- .../readme.md | 36 ++-- .../example_utils.py | 192 +++++++++++++++--- .../onpremise/compare_console.py | 26 ++- .../onpremise/countries_console.py | 28 ++- .../onpremise/datafileupdate_console.py | 51 +++-- .../onpremise/gettingstarted_console.py | 53 +++-- .../onpremise/gettingstarted_web/app.py | 10 + .../onpremise/metadata_console.py | 24 ++- .../onpremise/mixed_console.py | 51 +++-- .../onpremise/offlineprocessing.py | 29 ++- .../onpremise/performance_console.py | 13 +- .../tests/test_examples_without_settings.py | 133 ++++++++++++ 27 files changed, 1002 insertions(+), 207 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py diff --git a/device-detection/fiftyone_devicedetection_examples/readme.md b/device-detection/fiftyone_devicedetection_examples/readme.md index c668d1d5..d25b881e 100644 --- a/device-detection/fiftyone_devicedetection_examples/readme.md +++ b/device-detection/fiftyone_devicedetection_examples/readme.md @@ -16,12 +16,31 @@ getting started example: `python -m fiftyone_devicedetection_examples.cloud.gettingstarted_console` +The on-premise examples use the free Lite data file by default. The data +repository, at `cxx/data`, links to it, and an example downloads it the +first time it is missing, so the examples run from a fresh clone with no +settings. Where an example shows a property that only the Enterprise data +file has, such as HardwareVendor, HardwareName or DeviceType, it shows the +property as needing the Enterprise data file and gives a link to +[contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-readme.md&utm_term=enterprise-data-file) +about a copy. Supply the path to an Enterprise data file as the first +command line argument to see every property. + +The data file update example needs a licence key, and the hardware profile +examples need the TAC data file or its CSV export, so without them each one +says what it needs and where to ask for it, and stops. + ## Tests To run the tests use: `python -m unittest discover -s tests -p test*.py -b` +The tests in `tests/test_examples_without_settings.py` run the on-premise +examples as they run from a fresh clone, with no licence key, and check that +each one finishes on the free Lite data file and gives the link to ask for +the Enterprise data file. + Cloud tests will only run with a valid 51Degrees resource key (see above) set as a resource_key operating system environment variable. For example, use following command to set resource_key on Linux: diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py index 05260f95..2a50c478 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py @@ -24,6 +24,7 @@ from threading import Event, Thread import os import shlex +import shutil import subprocess from pathlib import Path @@ -59,9 +60,16 @@ class ExampleUtils: DATA_FOLDER = os.path.join("cxx", "data", "device-detection") # Sent with each download, so the downloads these examples make can be - # counted. + # counted. It is left out inside GitHub Actions, where the data + # repository's script names the build instead, so that test runs are + # not counted as people using the examples. DOWNLOAD_SOURCE = "examples-51degrees-python" + # Shown in place of a value for a property that the data file in use + # does not have. Every property the examples ask for is in the + # Enterprise data file. + NEEDS_ENTERPRISE = "needs the Enterprise data file" + @staticmethod def get_resource_key_from_config(config): key = "" @@ -246,14 +254,16 @@ def __find_file(file_name, stop_event, result, search_dir=None): if search_dir is None: search_dir = Path(os.getcwd()).absolute() - # Search for the file in the current directory - files = list(search_dir.rglob(_file.name)) - - # If the file is found in the current directory - # then set the result and return - if len(files): - result[0] = str(files[0].absolute()) - return + # Search the current directory and every folder below it. The stop + # event is checked in each folder, so the search ends soon after it + # times out, and a folder that cannot be read, such as one under + # /proc, is skipped. + for folder, _, names in os.walk(search_dir): + if stop_event.is_set(): + return + if _file.name in names: + result[0] = str(Path(folder, _file.name).absolute()) + return # If the file is not found in the current directory # then move to the parent directory and repeat the process @@ -271,23 +281,34 @@ def __find_file(file_name, stop_event, result, search_dir=None): def __download(path): name = os.path.basename(path) data = os.path.dirname(os.path.dirname(path)) + in_actions = os.environ.get("GITHUB_ACTIONS") == "true" + environment = None if os.name == "nt": + # PowerShell 7 when it is on the path, otherwise Windows + # PowerShell, which, started from PowerShell 7, inherits its + # module path and then cannot load its own commands. + power_shell = "pwsh" if shutil.which("pwsh") else "powershell" command = [ - "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", - "-File", os.path.join(data, "get-data-files.ps1"), name, - "-Source", ExampleUtils.DOWNLOAD_SOURCE] + power_shell, "-NoProfile", "-ExecutionPolicy", "Bypass", + "-File", os.path.join(data, "get-data-files.ps1"), name] + if not in_actions: + command += ["-Source", ExampleUtils.DOWNLOAD_SOURCE] + if power_shell == "powershell": + environment = dict(os.environ) + environment.pop("PSModulePath", None) text = subprocess.list2cmdline(command) else: - command = [ - "sh", os.path.join(data, "get-data-files.sh"), - "-s", ExampleUtils.DOWNLOAD_SOURCE, name] + command = ["sh", os.path.join(data, "get-data-files.sh")] + if not in_actions: + command += ["-s", ExampleUtils.DOWNLOAD_SOURCE] + command.append(name) text = shlex.join(command) # Flushed first, because the script writes to the same output. print(f"{name} is missing, so it is being downloaded by running\n" f"{text}", flush=True) try: - subprocess.run(command, check=False) + subprocess.run(command, check=False, env=environment) except OSError: pass if os.path.isfile(path): @@ -310,8 +331,70 @@ def data_file_is_old(engine): def get_data_file_tier(engine): return engine.engine.getProduct() + @staticmethod + def is_lite_data_file(engine): + """! + True when the engine uses the free Lite data file. + """ + return ExampleUtils.get_data_file_tier(engine) == "Lite" + + @staticmethod + def has_property(engine, name): + """! + True when the engine's data file has the property. The Lite data + file leaves out the properties only the Enterprise data file has, + such as HardwareVendor, HardwareName and DeviceType. The data + file's own list is read, because the engine's properties are only + those it was built to give. + """ + properties = engine.engine.getMetaData().getProperties() + name = name.lower() + for index in range(properties.getSize()): + if properties.getByIndex(index).getName().lower() == name: + return True + return False + + @staticmethod + def get_enterprise_value(engine, device, property): + """! + Format a property for display, or say that it needs the Enterprise + data file when the engine's data file does not have it. The value + is only read when the data file has the property. + """ + if not ExampleUtils.has_property(engine, property): + return ExampleUtils.NEEDS_ENTERPRISE + return ExampleUtils.get_human_readable(device, property) + + @staticmethod + def get_enterprise_note(contact_us): + """! + Where to ask for a copy of the Enterprise data file, written after an + example has shown properties that the data file in use does not + have. The link is tagged with the example's file, so that an enquiry + can be traced to the example. + """ + return ("Some of these properties need the Enterprise data file. " + f"Contact us for a copy at {contact_us}") + + @staticmethod + def get_enterprise_offer(contact_us): + """! + What the Enterprise data file adds and where to ask for a copy, + written after an example that shows no properties has run on the + free Lite data file. + """ + return ("This example used the free Lite data file. The Enterprise " + "data file adds many more properties, such as " + "HardwareVendor, HardwareName and DeviceType. Contact us for " + f"a copy at {contact_us}") + @staticmethod def check_data_file(pipeline, logger): + """! + Log the data file in use, and a warning if it is old. An example run + on the free Lite data file says what the Enterprise data file adds + in its own output. + """ # Get the 'engine' element within the pipeline that # performs device detection. We can use this to get # details about the data file as well as meta-data @@ -337,14 +420,6 @@ def check_data_file(pipeline, logger): "cxx/data/get-data-files.sh -u 51Degrees-LiteV4.1.hash, " + "or cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash " + "-Update on Windows. " + - "Find out about the Enterprise data file, which " + - "includes automatic updates, on our pricing page: " + - "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=data-file-age-warning") - - if ExampleUtils.get_data_file_tier(engine) == "Lite": - logger.log("warning", - "This example is using the 'Lite' " + - "data file. This is used for illustration, and " + - "has limited accuracy and capabilities. Find " + - "out about the Enterprise data file on our " + - "pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=lite-data-file") + "Contact us about the Enterprise data file, which " + + "includes automatic updates, at " + + "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=data-file-age-warning") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/csv_profile_metadata.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/csv_profile_metadata.py index da30fdbe..c4c78c15 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/csv_profile_metadata.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/csv_profile_metadata.py @@ -40,7 +40,6 @@ import sys from fiftyone_devicedetection_examples.example_utils import ExampleUtils -from fiftyone_pipeline_core.logger import Logger # CSV file name to search for CSV_FILE_NAME = "51Degrees-Tac-HardwarePlatform.csv" @@ -52,6 +51,10 @@ # Columns whose values are pipe-delimited lists MULTI_VALUE_COLUMNS = ["TAC", "HardwareName", "NativeName", "NativeModel"] +# Where to ask for a copy of the hardware profile export, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-csv_profile_metadata.py&utm_term=enterprise-data-file" + def flush_print(*args, **kwargs): print(*args, **kwargs, flush=True) @@ -110,15 +113,17 @@ def main(argv): output_file = None if output_arg == "-" else output_arg limit = int(argv[2]) if len(argv) > 2 else None - logger = Logger(min_level="info") output_func = flush_print if output_file else stderr_print if csv_file is not None: run(csv_file, output_file, output_func, limit) else: - logger.log("error", - "Failed to find the CSV data file. " - "Please provide the path to 51Degrees-Tac-HardwarePlatform.csv as the first argument.") + # The export is not one of the free data files, so the example says + # what it needs and where to ask for it, and stops. + print(f"Listing the hardware profiles needs the {CSV_FILE_NAME} " + "export, which is not one of the free data files. Give its " + "path as the first command line argument, or contact us for a " + f"copy at {CONTACT_US}") if __name__ == "__main__": diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index 4fd126fc..c6d0e6a5 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -32,9 +32,9 @@ # - Daily auto-update # # ## License Key -# In order to test this example you will need a 51Degrees Enterprise license which can be -# purchased from our [pricing page](https://51degrees.com/pricing/annual?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=top). Look for our "Bigger" or -# "Biggest" options. +# Updating the data file needs a 51Degrees licence key for the Enterprise data file. +# Without one, the example shows the free Lite data file and says where to ask for a key. +# Contact us for a key at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=top # # # Data Files # You can find out more about data files, licenses etc. at our [FAQ page](https://51degrees.com/resources/faqs?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=top) @@ -166,10 +166,14 @@ from fiftyone_devicedetection_shared.key_utils import KeyUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_devicedetection_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME from fiftyone_pipeline_engines.datafile_update_service import DataFileUpdateService from fiftyone_pipeline_engines.datafile_update_service import UpdateStatus UPDATE_EXAMPLE_LICENSE_KEY_NAME = "license_key" +# Where to ask for a licence key and a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=licence-key" DEFAULT_DATA_FILENAME = os.path.expanduser("~") + os.path.sep + ENTERPRISE_DATAFILE_NAME @@ -187,6 +191,32 @@ def clear(self): super().clear() class DataFileUpdateConsole(): + def explain_license_key(self, data_file, output): + # Without a licence key the example cannot download a data file, so + # it shows the data file it would start from, the free Lite file + # unless another is given, and says what an update needs. + if not data_file: + data_file = ExampleUtils.find_file(LITE_DATAFILE_NAME) + if data_file and os.path.isfile(data_file): + pipeline = DeviceDetectionPipelineBuilder( + data_file_path = data_file, + performance_profile = "LowMemory", + usage_sharing = False, + auto_update = False, + licence_keys = "").build() + with pipeline: + engine = pipeline.get_element("device") + output(f"The data file in use is the " + f"'{ExampleUtils.get_data_file_tier(engine)}' file published on " + f"{ExampleUtils.get_data_file_date(engine):%Y-%m-%d}, at " + f"'{data_file}'.") + output("") + output("Updating the data file downloads the Enterprise data file, " + "which needs a licence key. Give the key as the first command line " + f"argument, or in the environment variable " + f"'{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'. Contact us for a key and a " + f"copy at {CONTACT_US}") + def run(self, data_file, license_key, interactive, logger, output): logger.log("info", "Starting example") @@ -195,13 +225,8 @@ def run(self, data_file, license_key, interactive, logger, output): license_key = KeyUtils.get_named_key(UPDATE_EXAMPLE_LICENSE_KEY_NAME) if not license_key: - logger.log("error", - "In order to test this example you will need a 51Degrees Enterprise " - "license which can be obtained on a trial basis or purchased from our\n" - "pricing page https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=license-key-required. You must supply the license " - "key as an argument to this program, or as an environment or system variable " - f"named '{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'") - raise Exception("No license key available") + self.explain_license_key(data_file, output) + return # work out where the downloaded file will be put, directory must exist if (data_file != None): @@ -283,7 +308,7 @@ def run(self, data_file, license_key, interactive, logger, output): # to notify when update complete data_file_update_service = update_service, # For automatic updates to work you will need to provide a license key. - # A license key can be obtained with a subscription from https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=run + # Contact us for a licence key at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=run licence_keys = license_key, # Enable update on startup, the auto update system # will be used to check for an update before the diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index 3675fee2..12c4cff6 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -45,6 +45,17 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_VALUES from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=enterprise-data-file" + +# The properties only the Enterprise data file has, with the names they are +# shown with. The free Lite data file has the ones the example shows first. +ENTERPRISE_PROPERTIES = [ + ("Hardware Vendor", "hardwarevendor"), + ("Hardware Name", "hardwarename"), + ("Device Type", "devicetype")] + class GettingStartedConsole(): def run(self, data_file, logger, output): @@ -76,8 +87,12 @@ def run(self, data_file, logger, output): ExampleUtils.check_data_file(pipeline, logger) # carry out some sample detections + needs_enterprise = False for values in EVIDENCE_VALUES: - self.analyseEvidence(values, pipeline, output) + needs_enterprise |= self.analyseEvidence( + values, pipeline, output) + if needs_enterprise: + output("\n" + ExampleUtils.get_enterprise_note(CONTACT_US)) def analyseEvidence(self, evidence, pipeline, output): @@ -126,7 +141,18 @@ def analyseEvidence(self, evidence, pipeline, output): self.outputValue("Platform Version", device.platformversion, message) self.outputValue("Browser Name", device.browsername, message) self.outputValue("Browser Version", device.browserversion, message) + + # The engine says which properties its data file has, and the + # value of a property it does not have is not read. + engine = pipeline.get_element("device") + needs_enterprise = False + for name, property in ENTERPRISE_PROPERTIES: + value = ExampleUtils.get_enterprise_value( + engine, device, property) + needs_enterprise |= value == ExampleUtils.NEEDS_ENTERPRISE + message.append(f"\t{name}: {value}\n") output("".join(message)) + return needs_enterprise def outputValue(self, name, value, message): # Individual result values have a wrapper called @@ -147,9 +173,10 @@ def main(argv): # a command line parameter. # # Note that the Lite data file is only used for illustration, and has - # limited accuracy and capabilities. - # Find out about the Enterprise data file on our pricing page: - # https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=main + # limited accuracy and capabilities. The example shows the properties + # only the Enterprise data file has as needing it. Contact us for a copy + # of the Enterprise data file: + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html index d26ab8da..a1352ee8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html @@ -32,8 +32,8 @@ days old. A more recent data file may be needed to correctly detect the latest devices, browsers, etc. Get the latest lite data file with the scripts in the data - repository on GitHub. Find out about the Enterprise data file, which includes automatic - daily updates, on our pricing page. + repository on GitHub. Contact us + about the Enterprise data file, which includes automatic daily updates. {% endif %}

    Web integration example

    @@ -130,17 +130,17 @@

    Device data

    - Hardware Vendor:{{ utils.get_human_readable(data.device, "hardwarevendor") }} - Hardware Name:{{ utils.get_human_readable(data.device, "hardwarename") }} - Device Type:{{ utils.get_human_readable(data.device, "devicetype") }} - Platform Vendor:{{ utils.get_human_readable(data.device, "platformvendor") }} - Platform Name:{{ utils.get_human_readable(data.device, "platformname") }} - Platform Version:{{ utils.get_human_readable(data.device, "platformversion") }} - Browser Vendor:{{ utils.get_human_readable(data.device, "browservendor") }} - Browser Name:{{ utils.get_human_readable(data.device, "browsername") }} - Browser Version:{{ utils.get_human_readable(data.device, "browserversion") }} - Screen width (pixels):{{ utils.get_human_readable(data.device, "screenpixelswidth") }} - Screen height (pixels):{{ utils.get_human_readable(data.device, "screenpixelsheight") }} + Hardware Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "hardwarevendor") }} + Hardware Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "hardwarename") }} + Device Type:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "devicetype") }} + Platform Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformvendor") }} + Platform Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformname") }} + Platform Version:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformversion") }} + Browser Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browservendor") }} + Browser Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browsername") }} + Browser Version:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browserversion") }} + Screen width (pixels):{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "screenpixelswidth") }} + Screen height (pixels):{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "screenpixelsheight") }} {# The device id identifies the matched hardware, platform and browser profiles, so it is the compact form of everything else in this table. #} Device Id:{{ utils.get_human_readable(data.device, "deviceid") }} diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py index 6a223d00..dd53ed68 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py @@ -48,6 +48,10 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_VALUES from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=enterprise-data-file" + class MatchMetricsConsole(): def run(self, data_file, show_descs, logger, output): @@ -76,8 +80,9 @@ def run(self, data_file, show_descs, logger, output): # Uncomment BrowserName to include Browser component profile ID # in the device ID value. #restricted_properties=["ismobile", "hardwarename", "browsername"], - # If using the full on-premise data file this property will be - # present in the data file. See https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=run + # Only the Enterprise data file has this property, and the free + # Lite data file leaves it out. Contact us for a copy at + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=run # Only use the predictive graph to better handle variances # between the training data and the target User-Agent string. # For a more detailed description of the differences between @@ -169,7 +174,9 @@ def get_component(property): if (show_descs == True): output(f" {propertyDescription}") - output() + output("") + if ExampleUtils.is_lite_data_file(pipeline.get_element("device")): + output(ExampleUtils.get_enterprise_offer(CONTACT_US)) logger.log("info", "Finished Match Metrics Example") # Evidence values from a windows 11 device using a browser @@ -183,8 +190,8 @@ def main(argv): # # Note that the Lite data file is only used for illustration, and has # limited accuracy and capabilities. - # Find out about the Enterprise data file on our pricing page: - # https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=main + # Contact us for a copy of the Enterprise data file: + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py index 5e605158..e3141e06 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py @@ -50,6 +50,10 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_VALUES from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=enterprise-data-file" + bgRed = "\u001b[41m" fgWhite = "\u001b[37;1m" colReset = "\u001b[0m" @@ -84,6 +88,9 @@ def run(self, data_file, logger, output): self.outputEvidenceKeyDetails(pipeline.get_element("device"), output) ExampleUtils.check_data_file(pipeline, logger) + if ExampleUtils.is_lite_data_file(pipeline.get_element("device")): + output("") + output(ExampleUtils.get_enterprise_offer(CONTACT_US)) def outputEvidenceKeyDetails(self, engine, output): @@ -121,8 +128,8 @@ def main(argv): # # Note that the Lite data file is only used for illustration, and has # limited accuracy and capabilities. - # Find out about the Enterprise data file on our pricing page: - # https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=main + # Contact us for a copy of the Enterprise data file: + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py index 500fdba7..c4b437f5 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py @@ -59,6 +59,22 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_FILE_NAME from ruamel.yaml import YAML +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=enterprise-data-file" + +# The properties only the Enterprise data file has, which are recorded as +# needing it when the data file in use does not have them. +ENTERPRISE_PROPERTIES = [ + "tac", + "devicetype", + "hardwarevendor", + "hardwarename", + "hardwaremodel", + "nativedevice", + "nativename", + "nativemodel"] + class OfflineProcessing(): def run(self, data_file, evidence_yaml, logger, output): """! @@ -68,6 +84,8 @@ def run(self, data_file, evidence_yaml, logger, output): @param evidence_yaml: File containing the yaml representation of the evidence to process @param logger: Logger to use within the pipeline @param output: Output file to write results to + @return: True when the data file lacks properties that only the + Enterprise data file has """ # In this example, we use the DeviceDetectionPipelineBuilder @@ -108,7 +126,12 @@ def run(self, data_file, evidence_yaml, logger, output): # and some evidence files contain them in user-agent strings. content = evidence_yaml.read().replace('\t', ' ') yaml_data = yaml.load_all(content) - + # The properties only the Enterprise data file has that the data + # file in use lacks, found once rather than for every record. + engine = pipeline.get_element("device") + missing = [name for name in ENTERPRISE_PROPERTIES + if not ExampleUtils.has_property(engine, name)] + try: # Keep going as long as we have more document records. for evidence in yaml_data: @@ -126,18 +149,25 @@ def run(self, data_file, evidence_yaml, logger, output): # Keys already have header. prefix in the YAML file headers[key] = str(evidence[key]) - self.analyseEvidence(headers, pipeline, output, yaml) + self.analyseEvidence( + headers, pipeline, output, yaml, missing) except BaseException as err: # We can't read the evidence values, so cant write them to the output. Will just # have to skip this entry. logger.log("error", err) + if missing: + # A comment keeps the output a valid YAML stream. + print("# " + ExampleUtils.get_enterprise_note(CONTACT_US), + file = output) + # write the yaml document end marker print("...", file = output) ExampleUtils.check_data_file(pipeline, logger) + return len(missing) > 0 - def analyseEvidence(self, evidence, pipeline, output, yaml): + def analyseEvidence(self, evidence, pipeline, output, yaml, missing): # FlowData is a data structure that is used to convey information required for # detection and the results of the detection through the pipeline. # Information required for detection is called "evidence" and usually consists @@ -165,14 +195,12 @@ def analyseEvidence(self, evidence, pipeline, output, yaml): values["device.platformversion"] = ExampleUtils.get_human_readable(device, "platformversion") values["device.browsername"] = ExampleUtils.get_human_readable(device, "browsername") values["device.browserversion"] = ExampleUtils.get_human_readable(device, "browserversion") - values["device.tac"] = ExampleUtils.get_human_readable(device, "tac") - values["device.devicetype"] = ExampleUtils.get_human_readable(device, "devicetype") - values["device.hardwarevendor"] = ExampleUtils.get_human_readable(device, "hardwarevendor") - values["device.hardwarename"] = ExampleUtils.get_human_readable(device, "hardwarename") - values["device.hardwaremodel"] = ExampleUtils.get_human_readable(device, "hardwaremodel") - values["device.nativedevice"] = ExampleUtils.get_human_readable(device, "nativedevice") - values["device.nativename"] = ExampleUtils.get_human_readable(device, "nativename") - values["device.nativemodel"] = ExampleUtils.get_human_readable(device, "nativemodel") + # The value of a property the data file does not have is not + # read. + for property in ENTERPRISE_PROPERTIES: + values["device." + property] = ( + ExampleUtils.NEEDS_ENTERPRISE if property in missing + else ExampleUtils.get_human_readable(device, property)) # DeviceId is a unique identifier for the combination of hardware, operating # system, browser and crawler that has been detected. # Our device detection solution uses machine learning to find the optimal @@ -196,8 +224,8 @@ def main(argv): # # Note that the Lite data file is only used for illustration, and has # limited accuracy and capabilities. - # Find out about the Enterprise data file on our pricing page: - # https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=main + # Contact us for a copy of the Enterprise data file: + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # This file contains the 20,000 most commonly seen combinations of header values # that are relevant to device detection. For example, User-Agent and UA-CH headers. @@ -212,9 +240,11 @@ def main(argv): if (data_file != None): with open(output_file, "w") as output: with open(evidence_file, "r") as input: - OfflineProcessing().run(data_file, input, logger, output) - logger.log("info", - f"Processing complete. See results in: '{output_file}'") + needs_enterprise = OfflineProcessing().run( + data_file, input, logger, output) + print(f"Processing complete. See results in: '{output_file}'") + if needs_enterprise: + print(ExampleUtils.get_enterprise_note(CONTACT_US)) else: logger.log("error", "Failed to find a device detection data file. Make sure the " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py index d795b548..000186b4 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py @@ -51,6 +51,11 @@ from fiftyone_devicedetection_examples.example_utils import ExampleUtils # This example goes through a CSV of 20000 user agents and processes them, returning the time and information about the matches from fiftyone_devicedetection_onpremise.devicedetection_onpremise_pipelinebuilder import DeviceDetectionOnPremisePipelineBuilder +from fiftyone_devicedetection_shared.example_constants import UA_FILE_NAME + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-performance.py&utm_term=enterprise-data-file" # Here we make a function that processes a user agent # And returns if it is a mobile device @@ -150,11 +155,13 @@ def run(data_file, skip = False): if __name__ == "__main__": ap = argparse.ArgumentParser(description='Run detection benchmark.') ap.add_argument('-d', '--data_file', default='', help='Path to data file') - ap.add_argument('-u', '--user_agents_file', default='../../cxx/data/device-detection/20000 User Agents.csv', help='Path to user agents evidence file') + ap.add_argument('-u', '--user_agents_file', default='', help='Path to user agents evidence file') ap.add_argument('-j', '--json_output', default='', help='Output results in JSON format') args = ap.parse_args() if args.data_file == "": args.data_file = ExampleUtils.find_file("51Degrees-LiteV4.1.hash") + if args.user_agents_file == "": + args.user_agents_file = ExampleUtils.find_file(UA_FILE_NAME) # First we read the contents of the 20000 user agents file as a list with open(args.user_agents_file, newline='') as file: @@ -206,3 +213,16 @@ def run(data_file, skip = False): final_result["notmobile"] += result["notmobile"] print("Results", final_result) + + # Say what the Enterprise data file adds when the benchmark ran on the + # free Lite data file. The data file is opened with the low memory + # profile, which reads little of it. + with DeviceDetectionOnPremisePipelineBuilder( + data_file_path=args.data_file, + licence_keys="", + performance_profile='LowMemory', + add_javascript_builder=False, + usage_sharing=False, + auto_update=False).build() as check: + if ExampleUtils.is_lite_data_file(check.get_element("device")): + print(ExampleUtils.get_enterprise_offer(CONTACT_US)) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/profile_metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/profile_metadata_console.py index 60fe8f6d..015c35d3 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/profile_metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/profile_metadata_console.py @@ -61,6 +61,11 @@ def stderr_print(*args, **kwargs): print(*args, **kwargs, file=sys.stderr, flush=True) +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-profile_metadata_console.py&utm_term=enterprise-data-file" + + class ProfileMetadataConsole(): # Properties we want to extract from each hardware profile # Note: TAC, HardwareName, NativeName, NativeModel are multi-value properties (string[]) @@ -199,9 +204,13 @@ def main(argv): if data_file is not None: ProfileMetadataConsole().run(data_file, output_file, logger, output_func, limit) else: - logger.log("error", - "Failed to find a device detection data file. " + - "Please provide the path to a TAC-HashV41.hash file as the first argument.") + # Only the Enterprise data file has the properties the example lists, + # so without one it says so and where to ask for one, and stops. + print("Listing the hardware profiles needs the TAC, HardwareVendor " + "and HardwareName properties, which only the Enterprise data " + "file has. Give the path to an Enterprise data file as the " + "first command line argument, or contact us for a copy at " + f"{CONTACT_US}") if __name__ == "__main__": diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/templates/index.html b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/templates/index.html index a8c19cd5..31a88034 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/templates/index.html +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/templates/index.html @@ -85,15 +85,15 @@

    Detection results

    - Hardware Vendor:{{ utils.get_human_readable(data.device, "hardwarevendor") }} - Hardware Name:{{ utils.get_human_readable(data.device, "hardwarename") }} - Device Type:{{ utils.get_human_readable(data.device, "devicetype") }} - Platform Vendor:{{ utils.get_human_readable(data.device, "platformvendor") }} - Platform Name:{{ utils.get_human_readable(data.device, "platformname") }} - Platform Version:{{ utils.get_human_readable(data.device, "platformversion") }} - Browser Vendor:{{ utils.get_human_readable(data.device, "browservendor") }} - Browser Name:{{ utils.get_human_readable(data.device, "browsername") }} - Browser Version:{{ utils.get_human_readable(data.device, "browserversion") }} + Hardware Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "hardwarevendor") }} + Hardware Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "hardwarename") }} + Device Type:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "devicetype") }} + Platform Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformvendor") }} + Platform Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformname") }} + Platform Version:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformversion") }} + Browser Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browservendor") }} + Browser Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browsername") }} + Browser Version:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browserversion") }} diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py b/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py new file mode 100644 index 00000000..c035294d --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py @@ -0,0 +1,140 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Runs the on-premise examples as they run from a fresh clone, with no +# arguments and no licence key, so that they use the free Lite data file. +# Each must finish, show the properties only the Enterprise data file has +# as needing it, and give the link to ask for a copy. + +import io +import os +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from fiftyone_pipeline_core.logger import Logger + +from fiftyone_devicedetection_examples.example_utils import ExampleUtils +from fiftyone_devicedetection_examples.onpremise import csv_profile_metadata +from fiftyone_devicedetection_examples.onpremise import datafileupdate_console +from fiftyone_devicedetection_examples.onpremise import gettingstarted_console +from fiftyone_devicedetection_examples.onpremise import match_metrics +from fiftyone_devicedetection_examples.onpremise import metadata_console +from fiftyone_devicedetection_examples.onpremise import offlineprocessing +from fiftyone_devicedetection_examples.onpremise import profile_metadata_console +from fiftyone_devicedetection_shared.example_constants import EVIDENCE_FILE_NAME +from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME + +# The settings the examples read, which a fresh clone does not have. +SETTINGS = ["license_key", "LICENSE_KEY"] + +# The evidence records the offline processing test reads. The example's own +# file has 20,000, which take minutes. +OFFLINE_RECORDS = 20 + + +def run_without_settings(main, argv=None): + """Runs an example's main with the settings taken out of the + environment, and returns what it printed.""" + printed = io.StringIO() + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in SETTINGS: + os.environ.pop(name, None) + main(argv or []) + return printed.getvalue() + + +def first_records(path, count): + """The first records of a YAML evidence file.""" + lines = [] + with open(path, encoding="utf-8") as evidence: + for line in evidence: + if line.rstrip() == "---": + count -= 1 + if count < 0: + break + lines.append(line) + return io.StringIO("".join(lines)) + + +class ExamplesWithoutSettingsTests(unittest.TestCase): + + def test_gettingstarted_console(self): + printed = run_without_settings(gettingstarted_console.main) + self.assertIn( + "Hardware Vendor: " + ExampleUtils.NEEDS_ENTERPRISE, printed) + self.assertIn( + ExampleUtils.get_enterprise_note(gettingstarted_console.CONTACT_US), + printed) + + def test_match_metrics(self): + printed = run_without_settings(match_metrics.main) + self.assertIn( + ExampleUtils.get_enterprise_offer(match_metrics.CONTACT_US), + printed) + + def test_metadata_console(self): + printed = run_without_settings(metadata_console.main) + self.assertIn( + ExampleUtils.get_enterprise_offer(metadata_console.CONTACT_US), + printed) + + def test_offline_processing(self): + output = io.StringIO() + lacks_properties = offlineprocessing.OfflineProcessing().run( + ExampleUtils.find_file(LITE_DATAFILE_NAME), + first_records( + ExampleUtils.find_file(EVIDENCE_FILE_NAME), OFFLINE_RECORDS), + Logger(min_level="error"), + output) + self.assertTrue(lacks_properties) + self.assertIn( + "device.hardwarevendor: " + ExampleUtils.NEEDS_ENTERPRISE, + output.getvalue()) + self.assertIn( + "# " + ExampleUtils.get_enterprise_note( + offlineprocessing.CONTACT_US), + output.getvalue()) + + def test_datafileupdate_console(self): + printed = run_without_settings(datafileupdate_console.main) + self.assertIn("The data file in use is the 'Lite' file", printed) + self.assertIn(datafileupdate_console.CONTACT_US, printed) + + # The two examples below need files that are not free, so in a fresh + # clone the search for them finds nothing, which the tests stand in for. + + def test_csv_profile_metadata(self): + with mock.patch.object(ExampleUtils, "find_file", return_value=None): + printed = run_without_settings(csv_profile_metadata.main) + self.assertIn(csv_profile_metadata.CSV_FILE_NAME, printed) + self.assertIn(csv_profile_metadata.CONTACT_US, printed) + + def test_profile_metadata_console(self): + with mock.patch.object(ExampleUtils, "find_file", return_value=None): + printed = run_without_settings(profile_metadata_console.main) + self.assertIn("only the Enterprise data file has", printed) + self.assertIn(profile_metadata_console.CONTACT_US, printed) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/ci/fetch-assets.ps1 b/ip-intelligence/ci/fetch-assets.ps1 index a26332bc..f4653aa8 100644 --- a/ip-intelligence/ci/fetch-assets.ps1 +++ b/ip-intelligence/ci/fetch-assets.ps1 @@ -31,6 +31,11 @@ foreach ($asset in $assets) { # The ASN file is not an asset, so the data repository's script downloads it # beside its link in cxx/data/ip-intelligence, where the tests look for it. +# The examples look for the Lite data file in the data repository, under the +# name its script saves it as, so the asset is linked there too rather than +# downloaded again. +New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/51Degrees-LiteIpiV41.ipi" -Path "$PSScriptRoot/../../cxx/data/ip-intelligence/51Degrees-LiteV41.ipi" + $global:LASTEXITCODE = 0 & "$PSScriptRoot/../../cxx/data/get-data-files.ps1" 51Degrees-IPIV4AsnIpiV41.ipi if ($LASTEXITCODE -ne 0) { diff --git a/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py b/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py index a2511324..0450b9eb 100644 --- a/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py +++ b/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py @@ -24,7 +24,7 @@ # First create the IP intelligence pipeline with the desired settings. -data_file = "../assets/51Degrees-LiteIpiV41.ipi" +data_file = "../../cxx/data/ip-intelligence/51Degrees-LiteV41.ipi" resource_key = "!!YOUR_RESOURCE_KEY!!" # Uncommment the line below 'On-Premise' and comment the line below 'Cloud' to diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md index c41c0630..dd0f3423 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md @@ -33,23 +33,32 @@ python -m fiftyone_ipintelligence_examples.cloud.gettingstarted_web ## On-premise examples -Require a `.ipi` data file in the format the engine reads (see +Use a `.ipi` data file in the format the engine reads (see [Data files](../readme.md#data-files)). The data repository, at -`cxx/data`, links to a freely available network-only file -(`51Degrees-IPIV4AsnIpiV41.ipi`) that the examples fall back to, and an -example downloads it when it is missing. It only contains the ASN/network -properties, so location and registration properties are reported as "not -present in the data file". - -For the full set of properties (location, registration, etc.), a more -complete data file is required - to obtain one for testing please -[contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=on-premise-examples) - and supply its path as -the first command line argument. +`cxx/data`, links to the two free data files, and an example downloads the +one it uses the first time it is missing, so the examples run from a fresh +clone with no settings. The free Lite data file (`51Degrees-LiteV41.ipi`) +has the registration properties, such as RegisteredName and +RegisteredCountry, and most examples use it. The free ASN data file +(`51Degrees-IPIV4AsnIpiV41.ipi`) has the ASN properties and is far smaller, +so the performance and data file update examples use it. + +The location, range and connection properties are only in the Enterprise +data file. An example run on a free data file shows each property it lacks +as needing the Enterprise data file, and gives a link to +[contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=on-premise-examples) about a copy. +The countries and comparison examples need those properties throughout, so +on a free data file they say which they need and stop. Supply the path to +an Enterprise data file as the first command line argument, or with `-d`, +to see every property. - `onpremise/gettingstarted_console.py` - resolve IP addresses from the console using a local data file. - `onpremise/metadata_console.py` - list the properties available from the data file. +- `onpremise/countries_console.py` - list the countries an IP address is + most likely in, with their names translated (needs the Enterprise data + file). - `onpremise/offlineprocessing.py` - batch process the evidence file from the data repository. - `onpremise/performance_console.py` - benchmark the number of IP lookups @@ -57,7 +66,7 @@ the first command line argument. - `onpremise/compare_console.py` - compare the location returned for each IP address against a CSV of known-true locations. - `onpremise/datafileupdate_console.py` - demonstrate automatic data file - updates (requires a license key). + updates (needs a licence key, and without one says where to ask for one). - `onpremise/mixed_console.py` - combine device detection and IP intelligence in a single on-premise pipeline (requires the device detection package and a `.hash` data file). @@ -67,10 +76,11 @@ the first command line argument. ``` python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console [data-file] python -m fiftyone_ipintelligence_examples.onpremise.metadata_console [data-file] +python -m fiftyone_ipintelligence_examples.onpremise.countries_console [data-file] python -m fiftyone_ipintelligence_examples.onpremise.offlineprocessing [data-file [evidence-file]] python -m fiftyone_ipintelligence_examples.onpremise.performance_console [-d data-file] [-e evidence-file] python -m fiftyone_ipintelligence_examples.onpremise.compare_console [-d data-file] [-t truth-file] -python -m fiftyone_ipintelligence_examples.onpremise.datafileupdate_console [data-file] +python -m fiftyone_ipintelligence_examples.onpremise.datafileupdate_console [license-key [data-file]] python -m fiftyone_ipintelligence_examples.onpremise.mixed_console [hash-file [ipi-file]] python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_web ``` diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py index 81eb5829..3ffd5499 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py @@ -23,6 +23,7 @@ from datetime import datetime, timedelta import os import shlex +import shutil import subprocess from pathlib import Path @@ -50,10 +51,21 @@ class ExampleUtils: # belongs in this folder, beside its .link file. DATA_FOLDER = os.path.join("cxx", "data", "ip-intelligence") + # The data repository's device detection folder, for the examples that + # use both products. + DEVICE_DETECTION_DATA_FOLDER = os.path.join("cxx", "data", "device-detection") + # Sent with each download, so the downloads these examples make can be - # counted. + # counted. It is left out inside GitHub Actions, where the data + # repository's script names the build instead, so that test runs are + # not counted as people using the examples. DOWNLOAD_SOURCE = "examples-51degrees-python" + # Shown in place of a value for a property that the data file in use + # does not have. Every property the examples ask for is in the + # Enterprise data file. + NEEDS_ENTERPRISE = "needs the Enterprise data file" + @staticmethod def get_human_readable(ip_data, property): try: @@ -124,7 +136,7 @@ def find_file(file_name): Find the specified file. First checks the data file environment variable, then searches up the directory tree from the current working directory looking in 'assets' and the data repository's - folder. A file the data repository links to is downloaded there + folders. A file the data repository links to is downloaded there when it is missing. """ @@ -134,17 +146,20 @@ def find_file(file_name): current = Path(os.getcwd()).resolve() roots = [current, *current.parents] + folders = [ + ExampleUtils.DATA_FOLDER, + ExampleUtils.DEVICE_DETECTION_DATA_FOLDER] for root in roots[:8]: - for sub_dir in [".", "assets", ExampleUtils.DATA_FOLDER]: + for sub_dir in [".", "assets", *folders]: candidate = os.path.join(str(root), sub_dir, file_name) if os.path.isfile(candidate): return candidate for root in roots[:8]: - candidate = os.path.join( - str(root), ExampleUtils.DATA_FOLDER, file_name) - if os.path.isfile(candidate + ".link"): - return ExampleUtils.__download(candidate) + for folder in folders: + candidate = os.path.join(str(root), folder, file_name) + if os.path.isfile(candidate + ".link"): + return ExampleUtils.__download(candidate) return None @@ -160,23 +175,34 @@ def __download(path): name = os.path.basename(path) data = os.path.dirname(os.path.dirname(path)) + in_actions = os.environ.get("GITHUB_ACTIONS") == "true" + environment = None if os.name == "nt": + # PowerShell 7 when it is on the path, otherwise Windows + # PowerShell, which, started from PowerShell 7, inherits its + # module path and then cannot load its own commands. + power_shell = "pwsh" if shutil.which("pwsh") else "powershell" command = [ - "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", - "-File", os.path.join(data, "get-data-files.ps1"), name, - "-Source", ExampleUtils.DOWNLOAD_SOURCE] + power_shell, "-NoProfile", "-ExecutionPolicy", "Bypass", + "-File", os.path.join(data, "get-data-files.ps1"), name] + if not in_actions: + command += ["-Source", ExampleUtils.DOWNLOAD_SOURCE] + if power_shell == "powershell": + environment = dict(os.environ) + environment.pop("PSModulePath", None) text = subprocess.list2cmdline(command) else: - command = [ - "sh", os.path.join(data, "get-data-files.sh"), - "-s", ExampleUtils.DOWNLOAD_SOURCE, name] + command = ["sh", os.path.join(data, "get-data-files.sh")] + if not in_actions: + command += ["-s", ExampleUtils.DOWNLOAD_SOURCE] + command.append(name) text = shlex.join(command) # Flushed first, because the script writes to the same output. print(f"{name} is missing, so it is being downloaded by running\n" f"{text}", flush=True) try: - subprocess.run(command, check=False) + subprocess.run(command, check=False, env=environment) except OSError: pass if os.path.isfile(path): @@ -185,6 +211,132 @@ def __download(path): f"again.\n{text}") return None + @staticmethod + def has_property(engine, name): + + """! + True when the engine's data file has the property. The free Lite data + file has the registration properties, such as RegisteredName, and the + free ASN data file the ASN properties, such as AsnName. The data + file's own list is read, because the engine's properties are only + those it was built to give. + """ + + properties = engine.engine.getMetaData().getProperties() + name = name.lower() + for index in range(properties.getSize()): + if properties.getByIndex(index).getName().lower() == name: + return True + return False + + @staticmethod + def is_free_data_file(engine): + + """! + True when the engine uses one of the free data files, Lite or ASN. + Only the Enterprise data file has the range properties, so a check + for IpRangeStart tells them apart. + """ + + return not ExampleUtils.has_property(engine, "IpRangeStart") + + @staticmethod + def is_free_data_file_at(data_file): + + """! + True when the data file at the path is one of the free data files. + The data file is opened with the low memory profile, which reads + little of it. + """ + + with ExampleUtils.__open(data_file) as pipeline: + return ExampleUtils.is_free_data_file( + pipeline.flow_elements_list["ip"]) + + @staticmethod + def get_enterprise_value(engine, ip_data, property): + + """! + Format a property for display, or say that it needs the Enterprise + data file when the engine's data file does not have it. The value + is only read when the data file has the property. + """ + + if not ExampleUtils.has_property(engine, property): + return ExampleUtils.NEEDS_ENTERPRISE + return ExampleUtils.get_human_readable(ip_data, property) + + @staticmethod + def lacks_enterprise_properties(data_file, purpose, properties, output, + contact_us): + + """! + For an example that needs properties only the Enterprise data file + has, report when the data file has none of them, and then say so and + where to ask for a copy of the Enterprise data file. The data file is + opened with the low memory profile, which reads little of it. + @param purpose: What the example does, as the start of a sentence, + for example "Comparing known locations with IP Intelligence". + @param contact_us: The link to the contact page, tagged with the + example's file, so that an enquiry can be traced to the example. + """ + + with ExampleUtils.__open(data_file) as pipeline: + engine = pipeline.flow_elements_list["ip"] + if any(ExampleUtils.has_property(engine, name) + for name in properties): + return False + if len(properties) == 1: + names = properties[0] + else: + names = ", ".join(properties[:-1]) + " and " + properties[-1] + output(f"{purpose} needs the {names} properties, which only the " + "Enterprise data file has. The data file in use has none of " + "them, so the example stops here. Contact us for a copy of " + f"the Enterprise data file at {contact_us}") + return True + + @staticmethod + def get_enterprise_note(contact_us): + + """! + Where to ask for a copy of the Enterprise data file, written after an + example has shown properties that the data file in use does not + have. The link is tagged with the example's file, so that an enquiry + can be traced to the example. + """ + + return ("Some of these properties need the Enterprise data file. " + f"Contact us for a copy at {contact_us}") + + @staticmethod + def get_enterprise_offer(contact_us): + + """! + What the Enterprise data file adds and where to ask for a copy, + written after an example that shows no location properties has run + on a free data file. + """ + + return ("This example used a free data file. The Enterprise data " + "file adds the location, range and connection properties. " + f"Contact us for a copy at {contact_us}") + + @staticmethod + def __open(data_file): + + """! + A pipeline that reads the data file with the low memory profile. + """ + + from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder + return IpIntelligencePipelineBuilder( + data_file_path = data_file, + performance_profile = "LowMemory", + usage_sharing = False, + auto_update = False, + licence_keys = "").build() + @staticmethod def get_data_file_date(pipeline): engine = pipeline.flow_elements_list["ip"] @@ -194,8 +346,9 @@ def get_data_file_date(pipeline): def check_data_file(pipeline, logger): """! - Display information about the data file and log warnings if it - is old or a Lite file. + Display information about the data file and log a warning if it is + old. An example run on a free data file says what the Enterprise + data file adds in its own output. """ engine = pipeline.flow_elements_list["ip"] @@ -223,13 +376,6 @@ def check_data_file(pipeline, logger): f"or cxx/data/get-data-files.ps1 ip-intelligence -Update " f"on Windows.") - if product == "Lite": - logger.log("warning", - "This example is using the 'Lite' data file. This is " - "used for illustration, and has limited properties. A " - "more complete data file can be obtained from " - "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-example_utils.py&utm_term=lite-data-file") - @staticmethod def __get_env_variable(name): value = os.environ.get(name) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py index e418f786..d928f86f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py @@ -39,9 +39,10 @@ # true location and the returned location is then calculated, along with the # location confidence and whether an area was returned. # -# This example requires an enterprise IP intelligence data file (.ipi). To -# obtain an enterprise data file for testing, please contact us: -# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-compare_console.py&utm_term=top +# This example needs the location properties only the Enterprise IP +# intelligence data file has. With the free Lite data file, which it uses by +# default, it says which and stops. To obtain an Enterprise data file, please +# contact us: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-compare_console.py&utm_term=top # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -54,11 +55,19 @@ from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger -from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME # Name of the truth CSV file in the data repository. GEOIP_COMPARISON_EVIDENCE_FILE_NAME = "geoip_comparison_evidence.csv" +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-compare_console.py&utm_term=enterprise-data-file" + +# The properties the comparison reads, which only the Enterprise data file +# has. +LOCATION_PROPERTIES = ["Latitude", "Longitude", "LocationConfidence", "Country"] + # Mean radius of the earth in kilometres, used for the distance calculation. EARTH_RADIUS_KM = 6371.0 @@ -175,12 +184,19 @@ def main(argv): help="Maximum number of truth records to process") args = ap.parse_args(argv) - data_file = args.data_file or ExampleUtils.find_file(ENTERPRISE_DATAFILE_NAME) + data_file = args.data_file or ExampleUtils.find_file(LITE_DATAFILE_NAME) truth_file = args.truth_file or ExampleUtils.find_file( GEOIP_COMPARISON_EVIDENCE_FILE_NAME) logger = Logger(min_level="info") + if data_file and ExampleUtils.lacks_enterprise_properties( + data_file, + "Comparing known locations with IP Intelligence", + LOCATION_PROPERTIES, + print, + CONTACT_US): + return if data_file and truth_file: with open(args.output_file, "w", newline="", encoding="utf-8") as fout: CompareConsole().run( diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py index eb239813..6e33ef2b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py @@ -32,10 +32,10 @@ # 2. How to supply the browser language so the names are localized # 3. How to read the ordered "All" lists used to build a country dropdown # -# This example uses a local IP intelligence data file (.ipi). A data file that -# contains country information is needed to see weighted countries lead the -# list; with the network-only ASN data file the lists are simply every known -# country in alphabetical order. +# The country codes this example orders and translates are only in the +# Enterprise data file. With the free Lite data file, which it uses by +# default, the example says so and stops. Contact us for a copy: +# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-countries_console.py&utm_term=header # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -49,7 +49,15 @@ add_country_translation) from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_VALUES -from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-countries_console.py&utm_term=enterprise-data-file" + +# The properties the country lists are built from, which only the Enterprise +# data file has. +COUNTRY_PROPERTIES = ["CountryCodesGeographical", "CountryCodesPopulation"] # The browser language to translate the country names to. A value such as # "fr_FR" or a full Accept-Language header is accepted; leave blank to keep the @@ -146,12 +154,18 @@ def main(argv): data_file = ( argv[0] if len(argv) > 0 - else ExampleUtils.find_file(ASN_DATAFILE_NAME)) + else ExampleUtils.find_file(LITE_DATAFILE_NAME)) logger = Logger(min_level="info") if data_file: - CountriesConsole().run(data_file, logger, print) + if not ExampleUtils.lacks_enterprise_properties( + data_file, + "Listing the countries an IP address is most likely in", + COUNTRY_PROPERTIES, + print, + CONTACT_US): + CountriesConsole().run(data_file, logger, print) else: logger.log("error", "Failed to find an IP intelligence data file. Make sure the " diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py index 37200afd..a11c17a3 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py @@ -32,10 +32,11 @@ # - Daily auto-update # # ## License Key -# In order to run this example you will need a 51Degrees Enterprise license, -# which can be purchased from our pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=top. -# Supply the license key as the first command line argument or in the -# 'license_key' environment variable. +# Updating the data file needs a 51Degrees licence key for the Enterprise +# data file. Without one, the example shows the free ASN data file and says +# where to ask for a key. Supply the licence key as the first command line +# argument or in the 'license_key' environment variable. Contact us for a key: +# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=header # # ## Update on start-up # Configure the pipeline to download an Enterprise data file on start-up. @@ -69,6 +70,7 @@ from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME from fiftyone_pipeline_engines.datafile_update_service import DataFileUpdateService from fiftyone_pipeline_engines.datafile_update_service import UpdateStatus @@ -76,6 +78,10 @@ # a command line argument. UPDATE_EXAMPLE_LICENSE_KEY_NAME = "license_key" +# Where to ask for a licence key and a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=licence-key" + # The product type passed to the 51Degrees distributor. This identifies the # IP intelligence Enterprise product and may need to be adjusted to match # the product your license entitles you to. @@ -100,6 +106,33 @@ def clear(self): class DataFileUpdateConsole(): + def explain_license_key(self, data_file, output): + # Without a licence key the example cannot download a data file, so + # it shows the data file it would start from, the free ASN file + # unless another is given, and says what an update needs. + if not data_file: + data_file = ExampleUtils.find_file(ASN_DATAFILE_NAME) + if data_file and os.path.isfile(data_file): + pipeline = IpIntelligencePipelineBuilder( + data_file_path = data_file, + performance_profile = "LowMemory", + usage_sharing = False, + auto_update = False, + licence_keys = "").build() + with pipeline: + engine = pipeline.flow_elements_list["ip"].engine + published = engine.getPublishedTime() + date = datetime(published.getYear(), published.getMonth(), + published.getDay()) + output(f"The data file in use is the '{engine.getProduct()}' " + f"file published on {date:%Y-%m-%d}, at '{data_file}'.") + output("") + output("Updating the data file downloads the Enterprise data file, " + "which needs a licence key. Give the key as the first command " + "line argument, or in the environment variable " + f"'{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'. Contact us for a key " + f"and a copy at {CONTACT_US}") + def run(self, data_file, license_key, interactive, logger, output): logger.log("info", "Starting example") @@ -108,14 +141,8 @@ def run(self, data_file, license_key, interactive, logger, output): license_key = os.environ.get(UPDATE_EXAMPLE_LICENSE_KEY_NAME) if not license_key: - logger.log("error", - "In order to run this example you will need a 51Degrees " - "Enterprise license, which can be obtained on a trial basis " - "or purchased from our pricing page https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=run. " - "You must supply the license key as an argument to this " - "program, or in an environment variable named " - f"'{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'") - raise Exception("No license key available") + self.explain_license_key(data_file, output) + return # Work out where the downloaded file will be put. The directory must # exist; the file itself need not. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py index 4988da7d..70c2e121 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py @@ -32,10 +32,12 @@ # 2. How to pass input data (evidence) to the Pipeline # 3. How to retrieve the results # -# This example uses a local IP intelligence data file (.ipi). The data -# repository links to a freely available data file, which the example -# downloads when it is missing. A more complete data file can be supplied -# as the first command line argument. +# By default the example uses the free Lite IP intelligence data file, which +# it downloads when it is missing. The Lite data file has the registration +# properties, and the example shows each property only the Enterprise data +# file has as needing it. An Enterprise data file can be supplied as the first +# command line argument. Contact us for a copy: +# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=header # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -45,12 +47,15 @@ from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_VALUES -from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=enterprise-data-file" # The properties displayed for each IP address. These mirror the .NET -# getting started example. Properties that are not present in the data file -# being used (for example the network-only ASN data file) are reported as -# such rather than causing an error. +# getting started example. A property the data file in use does not have is +# shown as needing the Enterprise data file. OUTPUT_PROPERTIES = [ "RegisteredName", "RegisteredOwner", @@ -99,8 +104,12 @@ def run(self, data_file, logger, output): ExampleUtils.check_data_file(pipeline, logger) # carry out some sample lookups + needs_enterprise = False for values in EVIDENCE_VALUES: - self.analyseEvidence(values, pipeline, output) + needs_enterprise |= self.analyseEvidence( + values, pipeline, output) + if needs_enterprise: + output("\n" + ExampleUtils.get_enterprise_note(CONTACT_US)) def analyseEvidence(self, evidence, pipeline, output): @@ -141,11 +150,16 @@ def analyseEvidence(self, evidence, pipeline, output): # Display the results of the lookup. See the property # dictionary at https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=analyseevidence # for details of all available properties. + # The engine says which properties its data file has. + engine = pipeline.flow_elements_list["ip"] + needs_enterprise = False for name in OUTPUT_PROPERTIES: - self.outputValue(ip_data, name, message) + needs_enterprise |= self.outputValue( + engine, ip_data, name, message) output("".join(message)) + return needs_enterprise - def outputValue(self, ip_data, name, message): + def outputValue(self, engine, ip_data, name, message): # Individual result values have a wrapper called # `AspectPropertyValue`. This functions similarly to # a null-able type. @@ -153,22 +167,23 @@ def outputValue(self, ip_data, name, message): # `value` method will throw an exception. # `AspectPropertyValue` also includes the `no_value_message` # method, which describes why the value has not been set. - # `get_value` returns None when the property is not present in the - # data file at all (for example the network-only ASN data file), - # rather than raising. - value = ExampleUtils.get_value(ip_data, name.lower()) - if value is None: - message.append(f"\t{name}: This property is not present in the data file.\n") - elif value.has_value(): + # A property the data file does not have needs the Enterprise data + # file, and its value is not read. + if not ExampleUtils.has_property(engine, name): + message.append(f"\t{name}: {ExampleUtils.NEEDS_ENTERPRISE}\n") + return True + value = ip_data.get(name.lower()) + if value.has_value(): message.append(f"\t{name}: {ExampleUtils.format_value(value.value())}\n") else: message.append(f"\t{name}: {value.no_value_message()}\n") + return False def main(argv): # Use the supplied path for the data file or find the data file in the # project space. - data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(ASN_DATAFILE_NAME) + data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. logger = Logger(min_level="info") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py index 56ea9799..e6f25806 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py @@ -44,6 +44,10 @@ add_country_translation, build_country_options) from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_web-app.py&utm_term=enterprise-data-file" + app = Flask(__name__) pipeline = None @@ -69,6 +73,12 @@ def get_value(ip_data, property): def country_dropdown(translated): + # Only the Enterprise data file has the country codes the dropdown is + # built from. + if not ExampleUtils.has_property( + pipeline.flow_elements_list["ip"], "CountryCodesGeographical"): + return (f"

    The country dropdown {ExampleUtils.NEEDS_ENTERPRISE}. " + f"Contact us for a copy.

    ") # The geographical "All" lists are index-aligned: option i is # (CountryCodesGeographicalAll[i], CountryNamesGeographicalAllTranslated[i]), diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py index 3819776c..f9e4a0fb 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py @@ -25,7 +25,8 @@ # # This example shows how to list the properties available from the # on-premise IP intelligence engine, along with their types and -# descriptions. +# descriptions. By default it uses the free Lite data file, which it +# downloads when it is missing. # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -34,7 +35,11 @@ from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger -from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-metadata_console.py&utm_term=enterprise-data-file" class MetadataConsole(): @@ -59,9 +64,13 @@ def run(self, data_file, logger, output): if property_metadata.get("description"): output(f" {property_metadata['description']}") + if ExampleUtils.is_free_data_file(engine): + output("") + output(ExampleUtils.get_enterprise_offer(CONTACT_US)) + def main(argv): - data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(ENTERPRISE_DATAFILE_NAME) + data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) logger = Logger(min_level="info") @@ -69,10 +78,11 @@ def main(argv): MetadataConsole().run(data_file, logger, print) else: logger.log("error", - "Failed to find an IP intelligence data file. Supply its path " - "as the first command line argument. To obtain an enterprise " - "data file for testing, please contact us: " - "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-metadata_console.py&utm_term=data-file-required") + "Failed to find an IP intelligence data file. Make sure the " + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file, or supply the path to a " + "data file as the first command line argument.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py index 2f4f5e18..60fda04e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py @@ -28,11 +28,15 @@ # User-Agent (used by device detection) and an IP address (used by IP # intelligence), and both engines process the flow data. # -# Two data files are required: +# Two data files are used, which may be supplied as the first two command +# line arguments: # - a Device Detection data file (.hash) # - an IP Intelligence data file (.ipi) # -# They may be supplied as the first two command line arguments. +# By default the free Lite data files are used, which are downloaded when +# they are missing, and each property only the Enterprise data files have is +# shown as needing them. Contact us for a copy: +# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-mixed_console.py&utm_term=header # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -46,7 +50,11 @@ from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise import IpIntelligenceOnPremise from fiftyone_ipintelligence_examples.example_utils import ExampleUtils -from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME + +# Where to ask for a copy of the Enterprise data files, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-mixed_console.py&utm_term=enterprise-data-file" # Default name of the device detection data file (the lite file the data # repository links to). @@ -143,10 +151,15 @@ def run(self, device_file, ip_file, logger, output): # The pipeline is closed at the end of the with block, which closes # both engines and releases the native data they hold. with pipeline: + needs_enterprise = False for evidence in COMBINED_EVIDENCE: - self.analyse_evidence(evidence, pipeline, output) + needs_enterprise |= self.analyse_evidence( + evidence, pipeline, device_engine, ip_engine, output) + if needs_enterprise: + output("\n" + ExampleUtils.get_enterprise_note(CONTACT_US)) - def analyse_evidence(self, evidence, pipeline, output): + def analyse_evidence(self, evidence, pipeline, device_engine, ip_engine, + output): # The FlowData is closed at the end of the with block, which releases # the results of both engines straight away. @@ -165,36 +178,42 @@ def analyse_evidence(self, evidence, pipeline, output): data.process() message = [] + needs_enterprise = False # Device detection results, keyed under "device". message.append("Device Detection Results:\n") for name in DEVICE_PROPERTIES: - self.outputValue(data.device, name, message) + needs_enterprise |= self.outputValue( + device_engine, data.device, name, message) # IP intelligence results, keyed under "ip". message.append("\nIP Intelligence Results:\n") for name in IP_PROPERTIES: - self.outputValue(data.ip, name, message) + needs_enterprise |= self.outputValue( + ip_engine, data.ip, name, message) output("".join(message)) - - def outputValue(self, aspect_data, name, message): - # `get_value` returns None when the property is not present in the - # data file, rather than raising. - value = ExampleUtils.get_value(aspect_data, name.lower()) - if value is None: - message.append(f"\t{name}: This property is not present in the data file.\n") - elif value.has_value(): + return needs_enterprise + + def outputValue(self, engine, aspect_data, name, message): + # A property the engine's data file does not have needs the + # Enterprise data file, and its value is not read. + if not ExampleUtils.has_property(engine, name): + message.append(f"\t{name}: {ExampleUtils.NEEDS_ENTERPRISE}\n") + return True + value = aspect_data.get(name.lower()) + if value.has_value(): message.append(f"\t{name}: {ExampleUtils.format_value(value.value())}\n") else: message.append(f"\t{name}: {value.no_value_message()}\n") + return False def main(argv): # Use the supplied paths for the data files, or find the device detection # and IP intelligence files in the project space. device_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(HASH_DATAFILE_NAME) - ip_file = argv[1] if len(argv) > 1 else ExampleUtils.find_file(ASN_DATAFILE_NAME) + ip_file = argv[1] if len(argv) > 1 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. logger = Logger(min_level="info") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py index 4785249e..a9e92fb9 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py @@ -28,7 +28,9 @@ # # We create a pipeline using the on-premise engine, deserialize the # YAML evidence records, pass each entry to the pipeline and write the -# resolved properties out to another YAML file. +# resolved properties out to another YAML file. By default the free Lite +# data file is used, which is downloaded when it is missing, and Country, +# which only the Enterprise data file has, is recorded as needing it. # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -42,12 +44,17 @@ from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_FILE_NAME -from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME # Number of evidence records to process in this example. RECORD_LIMIT = 100 +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-offlineprocessing.py&utm_term=enterprise-data-file" + + class OfflineProcessing(): def run(self, data_file, evidence_yaml, logger, output): @@ -71,6 +78,7 @@ def run(self, data_file, evidence_yaml, logger, output): ExampleUtils.check_data_file(pipeline, logger) records = 0 + needs_enterprise = False with open(evidence_yaml, "r") as fin: for evidence in yaml.load_all(fin): if records >= RECORD_LIMIT: @@ -78,9 +86,14 @@ def run(self, data_file, evidence_yaml, logger, output): # write the yaml document separator output("---") - self.analyse_evidence(evidence, pipeline, output, yaml) + needs_enterprise |= self.analyse_evidence( + evidence, pipeline, output, yaml) records += 1 + if needs_enterprise: + # A comment keeps the output a valid YAML stream. + output("# " + ExampleUtils.get_enterprise_note(CONTACT_US)) + # write the yaml document end marker output("...") logger.log("info", f"Processed {records} evidence records") @@ -106,16 +119,20 @@ def analyse_evidence(self, evidence, pipeline, output, yaml): # Add the values that we want to store against the record. result["ip.registeredname"] = ExampleUtils.get_human_readable(ip_data, "registeredname") result["ip.registeredcountry"] = ExampleUtils.get_human_readable(ip_data, "registeredcountry") + # Only the Enterprise data file has Country. + result["ip.country"] = ExampleUtils.get_enterprise_value( + pipeline.flow_elements_list["ip"], ip_data, "country") buffer = io.BytesIO() yaml.dump(result, buffer) output(buffer.getvalue().decode("utf-8").rstrip()) + return result["ip.country"] == ExampleUtils.NEEDS_ENTERPRISE def main(argv): - # Use the supplied path for the data file or find the enterprise file - # in the project space. - data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(ENTERPRISE_DATAFILE_NAME) + # Use the supplied path for the data file or find the free Lite file in + # the project space, which is downloaded when it is missing. + data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) evidence_file = argv[1] if len(argv) > 1 else ExampleUtils.find_file(EVIDENCE_FILE_NAME) # Configure a logger to output to the console. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py index 1d910d81..c76f930a 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py @@ -55,13 +55,17 @@ from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger -from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_FILE_NAME # The IP intelligence properties used to measure performance. Restricting the # work to a single component keeps the benchmark focused on lookup speed. PERFORMANCE_PROPERTY = "registeredcountry" +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-performance_console.py&utm_term=enterprise-data-file" + # The number of records processed before timing starts, so that any lazy # initialisation is not included in the timed run. WARM_UP_RECORDS = 100 @@ -214,7 +218,9 @@ def main(argv): help="Performance profile: MaxPerformance (default) or LowMemory") args = ap.parse_args(argv) - data_file = args.data_file or ExampleUtils.find_file(ENTERPRISE_DATAFILE_NAME) + # By default the free ASN data file is used, which is far smaller than the + # Lite file and is downloaded when it is missing. + data_file = args.data_file or ExampleUtils.find_file(ASN_DATAFILE_NAME) evidence_file = args.evidence_file or ExampleUtils.find_file(EVIDENCE_FILE_NAME) logger = Logger(min_level="info") @@ -227,6 +233,9 @@ def main(argv): if args.json_output: with open(args.json_output, "w") as fout: json.dump(results, fout) + if ExampleUtils.is_free_data_file_at(data_file): + print("") + print(ExampleUtils.get_enterprise_offer(CONTACT_US)) else: logger.log("error", "Failed to find a data file or evidence file. Make sure the " diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py new file mode 100644 index 00000000..e417bd79 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py @@ -0,0 +1,133 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Runs the on-premise examples as they run from a fresh clone, with no data +# file setting and no licence key, so that they use the free data files. +# Each must finish, show the properties only the Enterprise data file has as +# needing it, and give the link to ask for a copy. + +import io +import os +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from fiftyone_ipintelligence_examples.example_utils import ExampleUtils +from fiftyone_ipintelligence_examples.onpremise import compare_console +from fiftyone_ipintelligence_examples.onpremise import countries_console +from fiftyone_ipintelligence_examples.onpremise import datafileupdate_console +from fiftyone_ipintelligence_examples.onpremise import gettingstarted_console +from fiftyone_ipintelligence_examples.onpremise import metadata_console +from fiftyone_ipintelligence_examples.onpremise import mixed_console +from fiftyone_ipintelligence_examples.onpremise import offlineprocessing +from fiftyone_ipintelligence_examples.onpremise import performance_console + +# The settings the examples read, which a fresh clone does not have. +SETTINGS = ["license_key", "LICENSE_KEY", ExampleUtils.DATA_FILE_ENV_VAR] + +# Options that keep the performance example short. They leave the data file +# to the example. +PERFORMANCE_OPTIONS = ["--record_limit", "200", "--workers", "1"] + +# The mixed example combines IP intelligence with device detection, which +# is an optional dependency. +try: + import fiftyone_devicedetection_onpremise # noqa: F401 + device_detection_available = True +except ImportError: + device_detection_available = False + + +def run_without_settings(main, argv=None): + """Runs an example's main with the settings taken out of the + environment, and returns what it printed.""" + printed = io.StringIO() + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in SETTINGS: + os.environ.pop(name, None) + main(argv or []) + return printed.getvalue() + + +class ExamplesWithoutSettingsTests(unittest.TestCase): + + def test_gettingstarted_console(self): + printed = run_without_settings(gettingstarted_console.main) + self.assertIn("Country: " + ExampleUtils.NEEDS_ENTERPRISE, printed) + self.assertIn( + ExampleUtils.get_enterprise_note(gettingstarted_console.CONTACT_US), + printed) + + def test_countries_console(self): + printed = run_without_settings(countries_console.main) + self.assertIn( + "Listing the countries an IP address is most likely in needs the " + "CountryCodesGeographical and CountryCodesPopulation properties", + printed) + self.assertIn(countries_console.CONTACT_US, printed) + + def test_metadata_console(self): + printed = run_without_settings(metadata_console.main) + self.assertIn( + ExampleUtils.get_enterprise_offer(metadata_console.CONTACT_US), + printed) + + def test_offlineprocessing(self): + printed = run_without_settings(offlineprocessing.main) + self.assertIn("ip.country: " + ExampleUtils.NEEDS_ENTERPRISE, printed) + self.assertIn( + "# " + ExampleUtils.get_enterprise_note( + offlineprocessing.CONTACT_US), + printed) + + def test_compare_console(self): + printed = run_without_settings(compare_console.main) + self.assertIn( + "Comparing known locations with IP Intelligence needs the " + "Latitude, Longitude, LocationConfidence and Country properties", + printed) + self.assertIn(compare_console.CONTACT_US, printed) + + def test_performance_console(self): + printed = run_without_settings( + performance_console.main, PERFORMANCE_OPTIONS) + self.assertIn( + ExampleUtils.get_enterprise_offer(performance_console.CONTACT_US), + printed) + + def test_datafileupdate_console(self): + printed = run_without_settings(datafileupdate_console.main) + self.assertIn("The data file in use is the", printed) + self.assertIn(datafileupdate_console.CONTACT_US, printed) + + @unittest.skipUnless( + device_detection_available, "device detection is not installed") + def test_mixed_console(self): + printed = run_without_settings(mixed_console.main) + self.assertIn(ExampleUtils.NEEDS_ENTERPRISE, printed) + self.assertIn( + ExampleUtils.get_enterprise_note(mixed_console.CONTACT_US), + printed) + + +if __name__ == "__main__": + unittest.main() From 5ad5980fd2d192c53c9856edf3e2966b55864ba1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 01:46:03 +0000 Subject: [PATCH 17/68] Build the released packages when a nightly publish has nothing to release (#45) --- ci/build-package.ps1 | 7 +++++-- ci/packages.Tests.ps1 | 30 ++++++++++++++++++++++++++++++ ci/packages.ps1 | 26 ++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/ci/build-package.ps1 b/ci/build-package.ps1 index f806b1f7..057d3167 100644 --- a/ci/build-package.ps1 +++ b/ci/build-package.ps1 @@ -6,9 +6,12 @@ $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'packages.ps1') # Builds the packages this publish releases, each at its own version from the -# release plan. $Version is common-ci's label for the run and is not used. +# release plan. When nothing is to be released, every package is built at its +# released version, so that the tests have packages to install and the nightly +# publish passes without publishing anything. $Version is common-ci's label for +# the run and is not used. -$plan = @(Get-ReleasePlan $RepoName) +$plan = @(Get-BuildPlan $RepoName) # fiftyone_pipeline_did carries the OWID source inside itself, copied in by the # pipeline product's script; see pipeline/ci/copy-owid-source.ps1. diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 2a2887d9..766d1380 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -249,6 +249,36 @@ Describe 'Get-ReleasePlan' { } } +Describe 'Get-BuildPlan' { + It 'is the release plan when a package is to be released' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('pipeline/engines/a.py') + $plan = @(Get-BuildPlan $script:Repo $script:Graph) + $plan.Name | Should -Be @('engines', 'dd', 'ipi') + ($plan | Where-Object Name -eq 'engines').Version | Should -Be '4.5.39' + } + + It 'builds every published package at its released version when nothing changed' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @() + @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 + $plan = @(Get-BuildPlan $script:Repo $script:Graph) + $plan.Name | Should -Be @('core', 'did', 'engines', 'location', 'dd', 'ipi') + ($plan | Where-Object Name -eq 'core').Version | Should -Be '4.5.38' + ($plan | Where-Object Name -eq 'dd').Version | Should -Be '4.5.96' + ($plan | Where-Object Name -eq 'location').Version | Should -Be '4.5.4' + ($plan | Where-Object Name -eq 'ipi').Version | Should -Be '4.5.0' + } + + It 'builds the released versions from a commit already released' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/a.py') + git -C $script:Repo tag 'dd/4.5.97' + ($plan = @(Get-BuildPlan $script:Repo $script:Graph)).Count | Should -Be 6 + ($plan | Where-Object Name -eq 'dd').Version | Should -Be '4.5.97' + } +} + Describe 'the real packages.json' { BeforeAll { $script:Real = Get-Packages (Split-Path $PSScriptRoot) diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 70e6af0c..429c43ff 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -292,3 +292,29 @@ function Get-ReleasePlan([string]$RepoPath, $Packages = $null) { } } } + +# What a build of HEAD makes: the release plan, or, when nothing changed since +# the last release, every published package at its released version. Nothing +# is published then, because package-update-required.ps1 finds the release +# plan empty, but the publish's tests still have the packages to install. +function Get-BuildPlan([string]$RepoPath, $Packages = $null) { + if ($null -eq $Packages) { + $Packages = Get-Packages $RepoPath + } + $plan = @(Get-ReleasePlan $RepoPath $Packages) + if ($plan.Count -gt 0) { + return $plan + } + $published = @($Packages | Where-Object { $_.Publish -eq $true }) + foreach ($name in (Sort-ByDependency $published @($published.Name))) { + $package = $published | Where-Object { $_.Name -eq $name } + $last = @(Get-ReleasedVersions $RepoPath $name)[-1] + [pscustomobject]@{ + Name = $name + Path = $package.Path + Version = "$last" + Previous = $last + Reason = 'already released, built for the tests' + } + } +} From 0507eefa25242e0c9080b0766725cb1650b7e190 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 02:44:45 +0000 Subject: [PATCH 18/68] Share only the evidence usage sharing accepts unless asked for all (#50) --- .../src/fiftyone_pipeline_core/evidence.py | 3 +- .../fiftyone_pipeline_core/tests/test_core.py | 18 ++ .../examples/usagesharing/51d.json | 4 +- .../share_usage.py | 68 ++-- .../share_usage_evidencekeyfilter.py | 161 ++++++---- .../test_share_usage_evidencekeyfilter.py | 168 ++++++++++ .../tests/test_usage.py | 296 ++++++++++++++++++ 7 files changed, 635 insertions(+), 83 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py index 627ca7d8..f7067647 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py @@ -98,7 +98,8 @@ def filter(self) -> None: if self.evidence_filtered: return - for key in self.evidence: + # Loop over a copy of the keys, as keys are removed on the way. + for key in list(self.evidence): if not any([ flow_element.filter_evidence_key(key) for flow_element diff --git a/pipeline/fiftyone_pipeline_core/tests/test_core.py b/pipeline/fiftyone_pipeline_core/tests/test_core.py index f26acde6..21407b33 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_core.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_core.py @@ -58,6 +58,24 @@ def testEvidenceKeyFilter(self): nullEvidence = testPipeline.flowdata.evidence.get("header.other-evidence") self.assertTrue(nullEvidence == None) + # Test that evidence no element accepts is removed without an error. + # The JavaScript and set header elements accept every key, so they are + # left out. + def testEvidenceKeyFilterRemovesKeys(self): + + pipeline = (PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}))\ + .add(ExampleFlowElement1())\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "test") + flowdata.evidence.add("header.accept-encoding", "gzip") + flowdata.evidence.add("header.host", "example.com") + flowdata.process() + self.assertEqual( + flowdata.evidence.get_all(), {"header.user-agent": "test"}) + # Test Getter methods def testGet(self): diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json index 90cfbada..7196a8cb 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json @@ -9,8 +9,10 @@ "interval": 1200, "requested_package_size": 50, "cookie": "", + "track_session": false, "query_whitelist": [], - "header_blacklist": [] + "header_blacklist": [], + "share_all": false } } ] diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 3173318e..210dfb0a 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -31,7 +31,8 @@ import requests from fiftyone_pipeline_engines.engine import Engine -from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter +from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter, \ + ShareUsageTrackerEvidenceKeyFilter from .share_usage_tracker import ShareUsageTracker # The maximum length of a piece of evidence's value which can be @@ -54,7 +55,9 @@ def __init__( header_blacklist = [], share_percentage = 1, endpoint = "https://devices-v4.51degrees.com/new.ashx", - request_timeout=REQUEST_TIMEOUT): + request_timeout=REQUEST_TIMEOUT, + share_all = False, + track_session = False): """! Constructor for ShareUsage element @@ -70,22 +73,32 @@ def __init__( reduce the overhead of sending HTTP messages. For example, the 51Degrees cloud service uses a value of 2500. @type cookie: string - @param cookie: If a cookie is being used to identify user - sessions, it can be specified here in order to reduce the - sharing of duplicated data. + @param cookie: The name of the cookie that holds the session id. + It has no effect unless track_session is True. The cookie itself + is not shared unless share_all is True. + @type track_session: bool + @param track_session: If True, the cookie named by cookie is used + to tell user sessions apart, so evidence already shared is shared + again when it comes from a new session. False by default. It + needs cookie to be set, and has no effect when share_all is True, + because every cookie is then shared and used to find repeats. @type query_whitelist: list - @param query_whitelist: By default query string and HTTP form + @param query_whitelist: By default query string and HTTP form parameters are not shared unless prefixed with '51D_'. - If you need to share query string parameters, a list can be - specified here. - @type query_blacklist: list - @param query_blacklist: By default, all HTTP headers - (except a few, such as Cookies) are shared. - Individual headers can be excluded from sharing by adding them + If you need to share query string parameters, a list can be + specified here. None shares every parameter. + @type header_blacklist: list + @param header_blacklist: By default, all HTTP headers + except Cookie are shared. + Individual headers can be excluded from sharing by adding them to this list. @type share_percentage : float @param share_percentage: approximate proportion of requests to be shared. 1 = 100%, 0.5 = 50%, etc.. + @type share_all: bool + @param share_all: If True, all evidence is shared apart from the + email address, the salt used for hashed email identifiers, and the + request's query string and path. False by default. """ super(ShareUsage, self).__init__() @@ -102,6 +115,25 @@ def __init__( self.share_percentage = share_percentage self.endpoint = endpoint self.request_timeout = request_timeout + self.share_all = share_all + self.track_session = track_session + + # The session id and sequence number are kept in evidence, as the + # usage data carries them separately. + included = None if query_whitelist is None else \ + list(query_whitelist) + ["session-id", "sequence"] + + # When sessions are tracked, the pipeline keeps the session cookie + # for the tracker. The data sent leaves it out unless all evidence + # is shared. + self.evidence_key_filter = ShareUsageEvidenceKeyFilter( + cookie, included, header_blacklist, track_session, share_all) + self.data_evidence_key_filter = ShareUsageEvidenceKeyFilter( + cookie, included, header_blacklist, False, share_all) + self.tracker_evidence_key_filter = \ + ShareUsageTrackerEvidenceKeyFilter( + cookie, included, header_blacklist, track_session, + share_all) # Add the share usage tracker which detects when # to send up sharing data @@ -188,11 +220,7 @@ def get_evidence_key_filter(self): """ - return ShareUsageEvidenceKeyFilter( - cookie = self.cookie, - query_whitelist = self.query_whitelist, - header_blacklist = self.header_blacklist - ) + return self.evidence_key_filter def share_send_usage(self): """! @@ -291,7 +319,7 @@ def process_internal(self, flow_data): """ if random.uniform(0, 1) <= self.share_percentage: - cachekey = self.get_evidence_key_filter().filter_evidence(flow_data.evidence.get_all()) + cachekey = self.tracker_evidence_key_filter.filter_evidence(flow_data.evidence.get_all()) cachekey = json.dumps(cachekey) share = self.tracker.track(cachekey) if share: @@ -327,7 +355,7 @@ def get_data_from_evidence(self, flow_data): f"The value '{value}' could not be parsed to an integer.") else: # Check if we can send this piece of evidence - if self.get_evidence_key_filter().filter_evidence_key(key): + if self.data_evidence_key_filter.filter_evidence_key(key): data.try_add(key, value) return data @@ -394,7 +422,7 @@ def try_add(self, key, value): category = "" field = key - first_separator = key.index(".") + first_separator = key.find(".") if first_separator > 0: category = key[0:first_separator] field = key[first_separator + 1:len(key)] diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index a9d0d2ac..5f2292f3 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -22,41 +22,64 @@ from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +# Cookies, session values and query string parameters whose names start with +# this are the ones 51Degrees sets, and are shared. +FIFTYONE_PREFIX = "51d_" + +# The Cookie header carries every cookie, including the ones the cookie rule +# leaves out, so it is not shared unless all evidence is. +BLOCKED_HEADERS = ("cookie",) + +# Evidence that is never shared, whatever the settings, matched as the whole +# key or after a '.'. 'id.email' and 'id.salt' are a raw email address and the +# salt used to make hashed email identifiers. 'request-query' is the whole query +# string, which would undo the rules for its parameters. 'request-path' would +# make each page a visitor opens look like a new session to the tracker. +NEVER_SHARED_KEYS = ("id.email", "id.salt", "request-query", "request-path") + class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter): """ - The ShareUsageEvidenceKeyFilter filters out all evidence - not needed by the 51Degrees ShareUsage service. - It allows for a specific whitelist of query strings, - a blacklist of headers and a specific cookie used for - session information + The ShareUsageEvidenceKeyFilter decides which evidence the 51Degrees + ShareUsage service is sent. Headers are shared unless they are blocked, + cookies and session values only when their names start with 51D_, and + query string and form parameters only when their names start with 51D_ or + are on the list to share. Any other evidence is shared. """ - def __init__(self, cookie = None, query_whitelist = [], header_blacklist = []): + def __init__( + self, + cookie = None, + query_whitelist = (), + header_blacklist = (), + include_session = False, + share_all = False): """! Constructor for ShareUsageEvidenceKeyFilter @type cookie: string - @param cookie: which cookie is used to track evidence + @param cookie: the name of the cookie that identifies the session @type query_whitelist: list - @param query_whitelist: list of query string whitelist evidence to keep - @type query_blacklist: list - @param query_blacklist: list of header evidence to exclude - @type - @param + @param query_whitelist: query string and form parameters to share as + well as those starting 51D_, or None to share every parameter + @type header_blacklist: list + @param header_blacklist: headers not to share + @type include_session: bool + @param include_session: whether the session cookie is included + @type share_all: bool + @param share_all: whether all evidence is shared, apart from the + evidence that is never shared """ - self.query_whitelist = query_whitelist + self.cookie = cookie.lower() if cookie else None - self.header_blacklist = header_blacklist + self.query_whitelist = None if query_whitelist is None else \ + [parameter.lower() for parameter in query_whitelist] - self.cookie = cookie + self.header_blacklist = \ + [header.lower() for header in header_blacklist or ()] - if cookie is None and \ - (query_whitelist is None or len(query_whitelist) == 0) and \ - (header_blacklist is None or len(header_blacklist) == 0): - self.share_all = True - else: - self.share_all = False + self.include_session = include_session + + self.share_all = share_all - def filter_evidence_key(self, key): """! @@ -69,42 +92,58 @@ def filter_evidence_key(self, key): @return: Is this key in the filter's keys list? """ - # get prefix and key of evidence - key_parts = key.lower().split(".") - - prefix = key_parts[0] - suffix = key_parts[1] - - result = self.share_all - - if not self.share_all: - if prefix == "header": - # Add the header to the list if the header name does not - # appear in the list of blocked headers - result = suffix not in self.header_blacklist - elif prefix == "cookie": - # Only add cookies that start with the 51Degrees cookie - # prefix - result = suffix.startswith("51d_") or \ - (self.include_session and suffix == self.cookie) - elif prefix == "session": - # Only session values that start with the 51Degrees - # cookie prefix - result = suffix.startswith("51d_") - elif prefix == "query": - # If no query string parameter filter was specified - # then share all of them. - # Otherwise, only include query string parameters that - # start with 51d_ or that have been specified in - # the constructor. - result = self.query_whitelist is None or \ - len(self.query_whitelist) == 0 or \ - suffix.startswith("51d_") or \ - suffix in self.query_whitelist - else: - # Add anything that is not a cookie, header, session - # variable or query parameter - result = True - - return result - \ No newline at end of file + key = key.lower() + + if any(key == never_shared or key.endswith("." + never_shared) + for never_shared in NEVER_SHARED_KEYS): + return False + + if self.share_all: + return True + + # The prefix is everything before the first dot and the field + # everything after it, so a field can itself contain dots. + prefix, separator, field = key.partition(".") + + if not separator: + return True + elif prefix == "header": + # Add the header to the list if the header name does not + # appear in the list of blocked headers + return field not in self.header_blacklist and \ + field not in BLOCKED_HEADERS + elif prefix == "cookie": + # Only add cookies that start with the 51Degrees cookie + # prefix + return field.startswith(FIFTYONE_PREFIX) or \ + (self.include_session and field == self.cookie) + elif prefix == "session": + # Only session values that start with the 51Degrees + # cookie prefix + return field.startswith(FIFTYONE_PREFIX) + elif prefix == "query": + # If no query string parameter list was given then share all + # of them. Otherwise, only include query string parameters + # that start with 51d_ or that are in the list. + return self.query_whitelist is None or \ + field.startswith(FIFTYONE_PREFIX) or \ + field in self.query_whitelist + else: + # Add anything that is not a cookie, header, session + # variable or query parameter + return True + +class ShareUsageTrackerEvidenceKeyFilter(ShareUsageEvidenceKeyFilter): + """ + The evidence the ShareUsageTracker compares to decide whether a request + repeats one already shared. The session id and sequence number are left + out, because they make every request different. + """ + + def filter_evidence_key(self, key): + + if key.lower() in ("query.session-id", "query.sequence"): + return False + + return super(ShareUsageTrackerEvidenceKeyFilter, self)\ + .filter_evidence_key(key) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py new file mode 100644 index 00000000..16184d52 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -0,0 +1,168 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_pipeline_engines_fiftyone.share_usage_evidencekeyfilter import \ + ShareUsageEvidenceKeyFilter + + +class ShareUsageEvidenceKeyFilterTests(unittest.TestCase): + + def test_default_rules(self): + """! + Check that by default headers other than Cookie are shared, cookies, + session values and query string parameters only when their names + start with 51D_, and any other evidence. + """ + + keys = ShareUsageEvidenceKeyFilter() + + self.assertTrue(keys.filter_evidence_key("header.user-agent")) + self.assertFalse(keys.filter_evidence_key("header.cookie")) + self.assertTrue(keys.filter_evidence_key("cookie.51d_screenpixelswidth")) + self.assertTrue(keys.filter_evidence_key("cookie.51D_ScreenPixelsWidth")) + self.assertFalse(keys.filter_evidence_key("cookie.sessionid")) + self.assertFalse(keys.filter_evidence_key("cookie.a.b")) + self.assertTrue(keys.filter_evidence_key("session.51d_value")) + self.assertFalse(keys.filter_evidence_key("session.org.example.attribute")) + self.assertTrue(keys.filter_evidence_key("query.51d_gethighentropyvalues")) + self.assertFalse(keys.filter_evidence_key("query.password")) + self.assertFalse(keys.filter_evidence_key("query.id.email")) + self.assertTrue(keys.filter_evidence_key("server.client-ip")) + self.assertTrue(keys.filter_evidence_key("nodot")) + + def test_field_is_everything_after_the_first_dot(self): + """! + Check that a name containing a dot is matched as a whole against + the parameters to share. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + + self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertFalse(keys.filter_evidence_key("query.valid")) + self.assertFalse(keys.filter_evidence_key("query.other.name")) + + def test_lists_ignore_case(self): + """! + Check that the headers not to share and the parameters to share + are matched without regard to case. + """ + + keys = ShareUsageEvidenceKeyFilter( + query_whitelist=["Campaign"], + header_blacklist=["X-Secret"]) + + self.assertFalse(keys.filter_evidence_key("header.x-secret")) + self.assertTrue(keys.filter_evidence_key("query.campaign")) + self.assertTrue(keys.filter_evidence_key("header.user-agent")) + + def test_no_parameter_list_shares_every_parameter(self): + """! + Check that None in place of the parameters to share shares every + parameter, and an empty list only those starting 51D_. + """ + + every = ShareUsageEvidenceKeyFilter(query_whitelist=None) + prefixed = ShareUsageEvidenceKeyFilter(query_whitelist=[]) + + self.assertTrue(every.filter_evidence_key("query.campaign")) + self.assertFalse(prefixed.filter_evidence_key("query.campaign")) + self.assertTrue(prefixed.filter_evidence_key("query.51d_campaign")) + + def test_session_cookie(self): + """! + Check that the session cookie is included only when asked for, and + that no other ordinary cookie is. + """ + + kept = ShareUsageEvidenceKeyFilter(cookie="SessionId", include_session=True) + sent = ShareUsageEvidenceKeyFilter(cookie="SessionId") + + self.assertTrue(kept.filter_evidence_key("cookie.sessionid")) + self.assertFalse(sent.filter_evidence_key("cookie.sessionid")) + self.assertFalse(kept.filter_evidence_key("cookie._ga")) + self.assertFalse(sent.filter_evidence_key("cookie._ga")) + + def test_share_all(self): + """! + Check that sharing everything shares the evidence the other rules + leave out, apart from the evidence that is never shared. + """ + + keys = ShareUsageEvidenceKeyFilter(share_all=True) + + self.assertTrue(keys.filter_evidence_key("header.cookie")) + self.assertTrue(keys.filter_evidence_key("cookie.sessionid")) + self.assertTrue(keys.filter_evidence_key("query.password")) + self.assertTrue(keys.filter_evidence_key("query.id.usage")) + self.assertTrue(keys.filter_evidence_key("server.request-method")) + self.assertFalse(keys.filter_evidence_key("query.id.email")) + self.assertFalse(keys.filter_evidence_key("header.id.email")) + self.assertFalse(keys.filter_evidence_key("query.id.salt")) + self.assertFalse(keys.filter_evidence_key("header.id.salt")) + self.assertFalse(keys.filter_evidence_key("server.request-query")) + self.assertFalse(keys.filter_evidence_key("server.request-path")) + + def test_never_shared_in_every_mode(self): + """! + Check that the email address, the salt for hashed email + identifiers, the query string and the path are never shared, + whether bare or after a prefix, and without regard to case. + """ + + for keys in ( + ShareUsageEvidenceKeyFilter(), + ShareUsageEvidenceKeyFilter(query_whitelist=None), + ShareUsageEvidenceKeyFilter(share_all=True)): + for key in ( + "id.email", "id.salt", + "query.id.email", "QUERY.ID.EMAIL", + "header.id.salt", "HEADER.ID.SALT", + "server.request-query", "SERVER.REQUEST-QUERY", + "server.request-path", "SERVER.REQUEST-PATH"): + self.assertFalse(keys.filter_evidence_key(key), key) + + def test_never_shared_keys_match_whole_parts(self): + """! + Check that a key that only ends with the same letters as a key + never shared is treated as an ordinary key. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + + self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertTrue( + ShareUsageEvidenceKeyFilter(share_all=True) + .filter_evidence_key("query.valid.email")) + + def test_naming_a_parameter_does_not_share_the_query_string(self): + """! + Check that sharing one parameter does not share the whole query + string, which carries every parameter. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["utm_source"]) + + self.assertTrue(keys.filter_evidence_key("query.utm_source")) + self.assertFalse(keys.filter_evidence_key("server.request-query")) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index 8329645b..4f1bea40 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -431,6 +431,302 @@ def test_includes_header(self): self.assertIn('&', app.received[0]) + def test_default_shares_only_accepted_evidence(self): + """! + Check that with no options the element shares headers and the + 51Degrees cookies and parameters, and leaves out other cookies, the + Cookie header, and other query string and form parameters. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("cookie.51d_screenpixelswidth", "1080") + flowdata.evidence.add("query.51d_gethighentropyvalues", "hev") + flowdata.evidence.add("cookie.sessionid", "cookie value") + flowdata.evidence.add("header.cookie", "sessionid=header value") + flowdata.evidence.add("query.password", "password value") + flowdata.evidence.add("query.id.email", "email value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('
    some user agent
    ', app.received[0]) + self.assertIn('1080', app.received[0]) + self.assertIn('hev', app.received[0]) + self.assertNotIn("cookie value", app.received[0]) + self.assertNotIn("header value", app.received[0]) + self.assertNotIn("password value", app.received[0]) + self.assertNotIn("email value", app.received[0]) + + def test_never_shared_evidence(self): + """! + Check that the email address, the salt for hashed email + identifiers, the query string and the path are not shared, though + the rules for their prefixes would share them. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("id.email", "email value") + flowdata.evidence.add("header.id.salt", "salt value") + flowdata.evidence.add("server.request-query", "query string value") + flowdata.evidence.add("server.request-path", "path value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("email value", app.received[0]) + self.assertNotIn("salt value", app.received[0]) + self.assertNotIn("query string value", app.received[0]) + self.assertNotIn("path value", app.received[0]) + + def test_share_all(self): + """! + Check that sharing all evidence shares what the other rules leave + out, but not the evidence that is never shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + share_all=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("cookie.sessionid", "cookie value") + flowdata.evidence.add("query.campaign", "campaign value") + flowdata.evidence.add("query.id.email", "email value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('cookie value', app.received[0]) + self.assertIn('campaign value', app.received[0]) + self.assertNotIn("email value", app.received[0]) + + def test_parameter_names_with_dots(self): + """! + Check that a query string parameter whose name contains a dot is + matched by its whole name against the parameters to share. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + query_whitelist=["valid.email"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.valid.email", "valid value") + flowdata.evidence.add("query.valid", "first part value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('valid value', app.received[0]) + self.assertNotIn("first part value", app.received[0]) + + def test_empty_query_whitelist(self): + """! + Check that an empty list of parameters to share shares only those + starting 51D_, and that None shares every parameter. + """ + + for query_whitelist, shared in (([], False), (None, True)): + app.received = [] + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + query_whitelist=query_whitelist, + header_blacklist=["x-secret"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.51d_param", "fiftyone value") + flowdata.evidence.add("query.campaign", "campaign value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("fiftyone value", app.received[0]) + self.assertEqual("campaign value" in app.received[0], shared) + + def test_session_cookie(self): + """! + Check that with session tracking on the session cookie is kept in + the evidence, but neither it nor any other ordinary cookie is + shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid", + track_session=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("cookie.sessionid", "session value") + flowdata.evidence.add("cookie._ga", "analytics value") + flowdata.process() + + self.waitForUsageThreads() + self.assertTrue(usage.get_evidence_key_filter().filter_evidence_key("cookie.sessionid")) + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("session value", app.received[0]) + self.assertNotIn("analytics value", app.received[0]) + + def test_session_cookie_without_session_tracking(self): + """! + Check that session tracking is off by default, and that naming the + session cookie then has no effect. The cookie is not kept, and + evidence already shared is not shared again for a new session. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid") + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session in ("session value 1", "session value 2"): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("cookie.sessionid", session) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertNotIn("session value", app.received[0]) + self.assertFalse(usage.get_evidence_key_filter().filter_evidence_key("cookie.sessionid")) + self.assertFalse(usage.track_session) + + def test_session_tracking(self): + """! + Check that with session tracking on, evidence already shared is + shared again for a new session but not for a repeat in the same + session, and that the session cookie is not shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid", + track_session=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session in ("session value 1", "session value 1", "session value 2"): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("cookie.sessionid", session) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 2) + received = "\n".join(app.received) + self.assertEqual(received.count("repeated user agent"), 2) + self.assertNotIn("session value", received) + + def test_ordinary_cookie_with_header_blacklist(self): + """! + Check that an ordinary cookie is left out without an error when + some headers are not to be shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + header_blacklist=["x-secret"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("header.x-secret", "secret value") + flowdata.evidence.add("cookie.csrftoken", "token value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("secret value", app.received[0]) + self.assertNotIn("token value", app.received[0]) + + def test_repeat_with_new_sequence_not_shared(self): + """! + Check that a request differing from one already shared only in + its session id and sequence number is not shared again. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session, sequence in (("session 1", "1"), ("session 2", "2")): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("query.session-id", session) + flowdata.evidence.add("query.sequence", sequence) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("session 1", app.received[0]) + + def test_evidence_without_a_prefix(self): + """! + Check that evidence whose key has no prefix is shared without an + error. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("nodot", "no prefix value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("no prefix value", app.received[0]) + # zzz is to make this test run last so no other tests depend on the receiver anymore # we depend on python unittests ordering tests alphabetically, which is not ideal, but works for now def test_zzz_timeout(self): From 8ba66c7022fa36c7ce246e9a2c4a9d4e3019c50e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 05:40:44 +0000 Subject: [PATCH 19/68] Fail the cloud tests when no resource key is set, and run the Cloud 51Did workflow (#58) * Fail the cloud tests when no resource key is set, and run the Cloud 51Did workflow Tests that exist to call the live cloud service skipped when no resource key reached them, and the 51Did live tests also skipped when the answer lacked what they checked, so a run with no key or with a key lacking a product reported success without the cloud being called. - Each test folder that calls the live service has the same live_cloud.py, which reads _51DEGREES_RESOURCE_KEY, then _51DEGREES_RESOURCE_KEY_BESPOKE, then resource_key, and gives the failure messages. Every skip for a missing key in the 51Did, Device Detection cloud, Device Detection example, IP Intelligence example and pipeline example tests is now a failure naming the variables, the product the key must carry and how to see what a key carries. - Where the 51Did answer lacks an identifier, the failure gives the cloud's null reason and the product. A host without the creator context fails. Without a licence key the redeem tests check the sealed result redeems as unreadable. - test_did_tier_live.py checks what a Basic key gets, being a 51Did for non-marketing and a refusal naming CloudV5FODiD for standard and personalized. - The live tests are marked, the unit steps deselect them through PYTEST_ADDOPTS, and the pipeline integration step now runs the 51Did live tests. The integration scripts fail when no TestResourceKey is passed, and the Device Detection one keeps the cloud package in the run when 51Degrees.csv is missing. - cloud-51did.yml moves to the root .github/workflows, where GitHub reads it, with its paths corrected. It fails when no key reaches it, and runs the Basic key only through the tier test. The licence-key tests in the pipeline examples and test_licensekey_live.py still skip, because CI has no cloud licence key yet (#13). * Wrap the tox.ini comment about the cloud test keys --- .../workflows/cloud-51did.yml | 47 +++-- device-detection/ci/run-integration-tests.ps1 | 13 +- device-detection/ci/run-unit-tests.ps1 | 21 +- .../tests/conftest.py | 32 +++ .../tests/live_cloud.py | 88 ++++++++ .../tests/test_cloud.py | 33 +-- .../tests/test_properties.py | 95 ++++----- .../fiftyone_devicedetection_cloud/tox.ini | 4 + .../readme.md | 5 +- .../tests/live_cloud.py | 88 ++++++++ .../tests/test_cloudexamples.py | 16 +- .../fiftyone_devicedetection_examples/tox.ini | 1 + ip-intelligence/ci/run-integration-tests.ps1 | 19 +- .../tests/live_cloud.py | 88 ++++++++ .../tests/test_examples.py | 28 ++- .../fiftyone_ipintelligence_examples/tox.ini | 2 + location/ci/run-integration-tests.ps1 | 5 +- pipeline/ci/run-integration-tests.ps1 | 24 ++- pipeline/ci/run-unit-tests.ps1 | 13 +- .../examples/readme.md | 6 +- .../examples/tests/helpers.py | 30 ++- .../examples/tests/live_cloud.py | 88 ++++++++ .../tests/test_cloud_mixed_console.py | 4 +- .../examples/tests/test_cloud_mixed_web.py | 4 +- .../examples/tox.ini | 15 +- .../fiftyone_pipeline_did/tests/conftest.py | 38 ++++ .../fiftyone_pipeline_did/tests/live_cloud.py | 88 ++++++++ .../tests/test_did_client_live.py | 188 +++++++++--------- .../tests/test_did_tier_live.py | 113 +++++++++++ pipeline/fiftyone_pipeline_did/tox.ini | 6 +- 30 files changed, 950 insertions(+), 252 deletions(-) rename {pipeline/.github => .github}/workflows/cloud-51did.yml (65%) create mode 100644 device-detection/fiftyone_devicedetection_cloud/tests/conftest.py create mode 100644 device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py create mode 100644 device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py create mode 100644 pipeline/fiftyone_pipeline_did/tests/conftest.py create mode 100644 pipeline/fiftyone_pipeline_did/tests/live_cloud.py create mode 100644 pipeline/fiftyone_pipeline_did/tests/test_did_tier_live.py diff --git a/pipeline/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml similarity index 65% rename from pipeline/.github/workflows/cloud-51did.yml rename to .github/workflows/cloud-51did.yml index 95b128b4..5f258dc1 100644 --- a/pipeline/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -1,9 +1,12 @@ name: Cloud 51Did Tests # Runs the live cloud 51Did integration test -# (fiftyone_pipeline_did/tests/test_did_client_live.py) once for every -# _51DEGREES_RESOURCE_KEY* key passed in below. Each run sets the single -# _51DEGREES_RESOURCE_KEY the test reads. +# (pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py) once for +# every _51DEGREES_RESOURCE_KEY* key passed to its step below. Each run sets +# the single _51DEGREES_RESOURCE_KEY the test reads first. Those keys carry +# the FODid product and the CloudV5FODiD product the marketing usages need. +# The Basic key carries FODid without CloudV5FODiD, so it runs only the +# tests of what the Basic tier is entitled to (test_did_tier_live.py). # # The secrets are named one by one rather than passed as toJSON(secrets). # GitHub marks a workflow that hands every secret to a step as possibly @@ -11,10 +14,9 @@ name: Cloud 51Did Tests # A new resource key therefore needs a line here as well as the organisation # secret. # -# The test skips itself when no key is set, so a run with no secrets is green -# and proves nothing. That is deliberate, because a pull request from a fork -# has no secrets, and it is the reason the loop below reports which key each -# run used. +# A run with no key fails, because a cloud test that passes without calling +# the cloud has proved nothing. A pull request from a fork gets no secrets, +# so run this workflow from a branch in this repository. on: workflow_dispatch: @@ -29,8 +31,8 @@ on: pull_request: branches: [main] paths: - - 'fiftyone_pipeline_did/**' - - 'owid-python/**' + - 'pipeline/fiftyone_pipeline_did/**' + - 'pipeline/owid-python/**' - '.github/workflows/cloud-51did.yml' jobs: @@ -39,7 +41,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: fiftyone_pipeline_did + working-directory: pipeline/fiftyone_pipeline_did steps: - uses: actions/checkout@v4 with: @@ -56,26 +58,27 @@ jobs: # runs. shell: pwsh working-directory: . - run: ./ci/copy-owid-source.ps1 + run: ./pipeline/ci/copy-owid-source.ps1 - name: Install tox run: python -m pip install --upgrade pip tox - name: Cloud 51Did test for each resource key secret shell: pwsh env: - _51DEGREES_RESOURCE_KEY_BASIC: ${{ vars._51DEGREES_RESOURCE_KEY_BASIC }} _51DEGREES_RESOURCE_KEY_BESPOKE: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} _51DEGREES_RESOURCE_KEY_51DID: ${{ vars._51DEGREES_RESOURCE_KEY_51DID }} - # The redemption tests skip without a license key, and the service - # only opens a sealed result for a license key of the account whose - # resource key sealed it. So each license key is passed only with - # the resource key of the same suffix. + # The redemption tests check a sealed result reads back only with a + # license key, and the service only opens a sealed result for a + # license key of the account whose resource key sealed it. So each + # license key is passed only with the resource key of the same + # suffix, and a resource key without one checks that the result + # redeems as unreadable. _51DEGREES_LICENSE_KEY_51DID: ${{ secrets._51DEGREES_LICENSE_KEY_51DID }} run: | $keys = Get-ChildItem env: | Where-Object { $_.Name -like '_51DEGREES_RESOURCE_KEY*' -and $_.Value } if (-not $keys) { - Write-Host 'No _51DEGREES_RESOURCE_KEY* secrets available; skipping.' - exit 0 + Write-Host '::error::No _51DEGREES_RESOURCE_KEY* secret reached this job. The tests need a key that carries the FODid product, and the CloudV5FODiD product for the marketing usages.' + exit 1 } $failed = $false foreach ($k in $keys) { @@ -84,10 +87,14 @@ jobs: $licenseName = $k.Name -replace '^_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY' $env:_51DEGREES_LICENSE_KEY = [Environment]::GetEnvironmentVariable($licenseName) Write-Host "License key: $(if ($env:_51DEGREES_LICENSE_KEY) { $licenseName } else { 'none' })" - # -rs prints why each skipped test skipped, which tells a run that - # lacked a key apart from one the service turned away. + # -rs lists any test that skipped, and none should. tox -- tests/test_did_client_live.py -v -rs if ($LASTEXITCODE -ne 0) { $failed = $true } Write-Host '::endgroup::' } if ($failed) { exit 1 } + - name: Cloud 51Did tier test with the Basic key + if: ${{ !cancelled() }} + env: + _51DEGREES_RESOURCE_KEY_BASIC: ${{ vars._51DEGREES_RESOURCE_KEY_BASIC }} + run: tox -- tests/test_did_tier_live.py -v -rs diff --git a/device-detection/ci/run-integration-tests.ps1 b/device-detection/ci/run-integration-tests.ps1 index f1483528..c2d33cc4 100644 --- a/device-detection/ci/run-integration-tests.ps1 +++ b/device-detection/ci/run-integration-tests.ps1 @@ -14,11 +14,14 @@ if ($IsWindows) { $packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_examples" if (!$Keys.TestResourceKey) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No Resource Key::No resource key was provided, so integration tests will not run." - return -} elseif (!(Test-Path $RepoName/fiftyone_devicedetection_cloud/tests/51Degrees.csv)) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No CSV File::CSV file wasn't found, so cloud tests will not run." - $packages = "fiftyone_devicedetection_examples" + throw "No TestResourceKey was passed, so the cloud tests cannot call " + + "the cloud service. The workflows set it from the " + + "_51DEGREES_RESOURCE_KEY_BESPOKE secret." +} +# The cloud package runs without the CSV file too, because only its property +# coverage tests read the file, and they say so when it is missing. +if (!(Test-Path $RepoName/fiftyone_devicedetection_cloud/tests/51Degrees.csv)) { + Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No CSV File::CSV file wasn't found, so the cloud property coverage tests will not run." } # nightly-publish-main workflow doesn't create the examples package, so diff --git a/device-detection/ci/run-unit-tests.ps1 b/device-detection/ci/run-unit-tests.ps1 index 7bb9dea9..4acebce4 100644 --- a/device-detection/ci/run-unit-tests.ps1 +++ b/device-detection/ci/run-unit-tests.ps1 @@ -9,11 +9,20 @@ if ($IsWindows) { Write-Output $env:TEMP } -# The cloud package's tests run against fixed cloud responses, so they need -# no resource key, no licence key and no network connection. Running them -# here means a broken cloud example shows as red on every build, rather than -# only when the integration tests happen to have a key to run with. -$packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_onpremise" -./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages +# The cloud package's tests against fixed cloud responses need no resource +# key, no licence key and no network connection. Running them here means a +# broken cloud example shows as red on every build, rather than only when the +# integration tests happen to have a key to run with. Its tests marked live +# call the cloud service, and no key reaches this step, so they are left to +# run-integration-tests.ps1. Only a package whose tox.ini passes +# PYTEST_ADDOPTS sees the selection. +$addopts = $env:PYTEST_ADDOPTS +$env:PYTEST_ADDOPTS = '-m "not live"' +try { + $packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_onpremise" + ./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/conftest.py b/device-detection/fiftyone_devicedetection_cloud/tests/conftest.py new file mode 100644 index 00000000..8890cf53 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_cloud/tests/conftest.py @@ -0,0 +1,32 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The marker that keeps the live cloud tests out of the unit test run, which +# device-detection/ci/run-unit-tests.ps1 deselects. The tests against fixed +# cloud responses carry no marker, so they run in both steps. + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py index c5369df1..177a42ed 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py @@ -24,33 +24,20 @@ import os import time +import pytest + from fiftyone_devicedetection_cloud.devicedetection_cloud_pipelinebuilder import DeviceDetectionCloudPipelineBuilder from fiftyone_devicedetection_cloud.devicedetection_cloud import DeviceDetectionCloud -# Resource key environment variables follow the 51Degrees convention, which -# is that every one of them starts with "_51DEGREES_RESOURCE_KEY". The name -# used before the convention was adopted is still read, so an existing setup -# keeps working. -RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" -LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key" - -MISSING_KEY_MESSAGE = ( - "No resource key found, so the tests that call the cloud service " - f"cannot run. Set the environment variable '{RESOURCE_KEY_ENV_VAR}' " - f"(the older name '{LEGACY_RESOURCE_KEY_ENV_VAR}' is still read). " - "Create a resource key for free at " - "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-tests-test_cloud.py&utm_term=resource-key-required") - +from .live_cloud import find_resource_key, no_resource_key -def get_resource_key(): - """! - The resource key from the environment, or None when neither the aligned - nor the older variable is set. - """ +pytestmark = pytest.mark.live - return (os.environ.get(RESOURCE_KEY_ENV_VAR) - or os.environ.get(LEGACY_RESOURCE_KEY_ENV_VAR)) +NO_KEY = no_resource_key( + "the device product", + "device detection properties such as IsMobile, and their metadata", + several=True) mobile_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" @@ -59,10 +46,10 @@ def get_resource_key(): class DeviceDetectionTests(unittest.TestCase): def setUp(self): - self.resource_key = get_resource_key() + self.resource_key = find_resource_key() if not self.resource_key: - self.skipTest(MISSING_KEY_MESSAGE) + self.fail(NO_KEY) def test_pipeline_builder_cloud_engine_init(self): diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_properties.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_properties.py index 43bbeb34..8a525e71 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_properties.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_properties.py @@ -23,6 +23,8 @@ import os import unittest +import pytest + from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import CloudRequestEngine from fiftyone_devicedetection_cloud.devicedetection_cloud import DeviceDetectionCloud @@ -32,6 +34,10 @@ is_same_type, ) +from .live_cloud import find_resource_key, no_resource_key + +pytestmark = pytest.mark.live + header_file_path = "./tests/51Degrees.csv" mobile_ua = ("Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) " "AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile" @@ -43,71 +49,54 @@ # TODO remove setheader properties from this list once UACH datafile is released. exclude_properties = ["setheaderbrowseraccept-ch", "setheaderplatformaccept-ch", "setheaderhardwareaccept-ch"] -# Resource key environment variables follow the 51Degrees convention, which -# is that every one of them starts with "_51DEGREES_RESOURCE_KEY". The name -# used before the convention was adopted is still read, so an existing setup -# keeps working. -RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" -LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key" - -resource_key = (os.environ.get(RESOURCE_KEY_ENV_VAR) - or os.environ.get(LEGACY_RESOURCE_KEY_ENV_VAR)) - -if not resource_key: - # Skipping rather than raising, so a run without a key shows as skipped - # and names the variable, instead of turning into a collection error. - import pytest - - pytest.skip( - "No resource key found, so the tests that call the cloud service " - f"cannot run. Set the environment variable '{RESOURCE_KEY_ENV_VAR}' " - f"(the older name '{LEGACY_RESOURCE_KEY_ENV_VAR}' is still read). " - "Create a resource key for free at " - "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-tests-test_properties.py&utm_term=resource-key-required", - allow_module_level=True) - -if not os.path.isfile(header_file_path): - # The property list comes from an asset the build fetches with a licence - # key. Without it these tests cannot run, so skip rather than turn a - # missing asset into a failure. - import pytest - - pytest.skip( - f"The property list file '{header_file_path}' is not present, so " - "the property coverage tests cannot run. It is fetched by " - "ci/fetch-assets.ps1 when a device detection licence key is " - "available.", - allow_module_level=True) - -# Get Properties list -properties_list = get_properties_from_header_file(header_file_path) - -# Create a simple pipeline to access the engine with and process it with flow data -cloudRequestEngine = CloudRequestEngine({"resource_key": resource_key}) -deviceDetectionCloudEngine = DeviceDetectionCloud() -pipeline = PipelineBuilder() \ - .add(cloudRequestEngine) \ - .add(deviceDetectionCloudEngine) \ - .build() - +NO_KEY = no_resource_key( + "the device product", + "every device detection property listed in " + header_file_path, + several=True) + class PropertyTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.resource_key = find_resource_key() + cls.pipeline = None + if cls.resource_key and os.path.isfile(header_file_path): + cls.properties_list = get_properties_from_header_file( + header_file_path) + cls.engine = DeviceDetectionCloud() + cls.pipeline = PipelineBuilder() \ + .add(CloudRequestEngine({"resource_key": cls.resource_key})) \ + .add(cls.engine) \ + .build() + + def setUp(self): + if not self.resource_key: + self.fail(NO_KEY) + if self.pipeline is None: + # The property list comes from an asset the build fetches with + # a licence key, so a run without it says which file is missing. + self.skipTest( + f"The property list file '{header_file_path}' is not " + "present, so the property coverage tests cannot run. It is " + "fetched by ci/fetch-assets.ps1 when a device detection " + "licence key is available.") + def test_available_properties(self): """! Tests whether the all the properties present in the engine when initialised with a resource key are accessible. """ - flowData = pipeline.create_flowdata() + flowData = self.pipeline.create_flowdata() flowData.evidence.add("header.user-agent", mobile_ua) flowData.process() - elementData = flowData.get(deviceDetectionCloudEngine.datakey) + elementData = flowData.get(self.engine.datakey) # Get dictionary of elementdata properties dd_property_dict = elementData.as_dictionary() #RUn test checks on all the properties available in header file - for property in properties_list: + for property in self.properties_list: property = property.lower() if(property not in exclude_properties): if(property in dd_property_dict): @@ -128,16 +117,16 @@ def test_value_types(self): Tests value types of the properties present present in the engine """ - flowData = pipeline.create_flowdata() + flowData = self.pipeline.create_flowdata() flowData.evidence.add("header.user-agent", chrome_ua) flowData.process() - elementData = flowData.get(deviceDetectionCloudEngine.datakey) + elementData = flowData.get(self.engine.datakey) # Get dictionary of elementdata properties dd_property_dict = elementData.as_dictionary() # Get properties from the engine - properties_list = deviceDetectionCloudEngine.get_properties() + properties_list = self.engine.get_properties() # Run test check valuetypes of properties for propertykey, propertymeta in properties_list.items(): diff --git a/device-detection/fiftyone_devicedetection_cloud/tox.ini b/device-detection/fiftyone_devicedetection_cloud/tox.ini index a0d62c6b..c58bf36f 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tox.ini +++ b/device-detection/fiftyone_devicedetection_cloud/tox.ini @@ -21,10 +21,14 @@ deps = -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_devicedetection_cloud {tty:--color=yes} {posargs} +# PYTEST_ADDOPTS carries the marker selection of ../ci/run-unit-tests.ps1, +# which leaves the live tests out of the unit step. pass_env = _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE resource_key license_key + PYTEST_ADDOPTS [testenv:pre-publish] package = skip diff --git a/device-detection/fiftyone_devicedetection_examples/readme.md b/device-detection/fiftyone_devicedetection_examples/readme.md index d25b881e..846ebc97 100644 --- a/device-detection/fiftyone_devicedetection_examples/readme.md +++ b/device-detection/fiftyone_devicedetection_examples/readme.md @@ -41,7 +41,10 @@ examples as they run from a fresh clone, with no licence key, and check that each one finishes on the free Lite data file and gives the link to ask for the Enterprise data file. -Cloud tests will only run with a valid 51Degrees resource key (see above) set as a resource_key operating system environment variable. +The cloud tests need a valid 51Degrees resource key (see above), and fail +without one. They read it from the environment variable +`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`, then +`resource_key`. For example, use following command to set resource_key on Linux: `export resource_key=MY-RESOURCE-KEY` diff --git a/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py b/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py index 79bf78c5..f5e10453 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py @@ -35,6 +35,13 @@ from fiftyone_devicedetection_examples.cloud.gettingstarted_console import GettingStartedConsole from fiftyone_devicedetection_examples.cloud.configurator_console import ConfiguratorConsole +from .live_cloud import find_resource_key, no_resource_key + +NO_KEY = no_resource_key( + "the device product, and the hardware product for the lookups", + "device detection properties, and for the hardware profiles the TAC " + "and native model lookups return", several=True) + # Text that shows an example printed a programming fault rather than a # result. An example that reaches any of these is broken, however little of # the data the resource key is entitled to. @@ -50,17 +57,12 @@ class DeviceDetectionExampleTests(unittest.TestCase): - # Init method - specify a resource key here, or set one in the - # environment variable named by ExampleUtils.RESOURCE_KEY_ENV_VAR. def setUp(self): - self.resource_key = ExampleUtils.get_resource_key() + self.resource_key = find_resource_key() self.logger = Logger() if not self.resource_key: - # Skipping rather than passing, so a run without a key is not - # mistaken for a run that proved something. The message names - # the variable that was wanted. - self.skipTest(ExampleUtils.get_missing_resource_key_message()) + self.fail(NO_KEY) def run_example(self, example): """! diff --git a/device-detection/fiftyone_devicedetection_examples/tox.ini b/device-detection/fiftyone_devicedetection_examples/tox.ini index 0b6222c0..b0932b5a 100644 --- a/device-detection/fiftyone_devicedetection_examples/tox.ini +++ b/device-detection/fiftyone_devicedetection_examples/tox.ini @@ -33,6 +33,7 @@ commands = {[common]commands} pass_env = _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE resource_key license_key run_performance_tests diff --git a/ip-intelligence/ci/run-integration-tests.ps1 b/ip-intelligence/ci/run-integration-tests.ps1 index 5e50e871..f57e296c 100644 --- a/ip-intelligence/ci/run-integration-tests.ps1 +++ b/ip-intelligence/ci/run-integration-tests.ps1 @@ -14,17 +14,18 @@ if ($IsWindows) { $packages = , "fiftyone_ipintelligence_examples" if (!$Keys.TestResourceKey) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No Resource Key::No resource key was provided, so cloud integration tests will not run." -} -else { - # Says what that key is entitled to, so a run with no IP data is read - # as an entitlement gap rather than as broken code. Prints product - # names and tiers only, never the key. Found through this script's own - # folder, because common-ci runs it from the common-ci checkout, where - # './ci/report-key-products.ps1' does not exist. - & (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey + throw "No TestResourceKey was passed, so the cloud tests cannot call " + + "the cloud service. The workflows set it from the " + + "_51DEGREES_RESOURCE_KEY_BESPOKE secret." } +# Says what that key is entitled to, so a run with no IP data is read as an +# entitlement gap rather than as broken code. Prints product names and tiers +# only, never the key. Found through this script's own folder, because +# common-ci runs it from the common-ci checkout, where +# './ci/report-key-products.ps1' does not exist. +& (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey + # nightly-publish-main workflow doesn't create the examples package, so # install-package.ps1 won't install it and its test dependencies won't get # installed. That's why we install it here, as a special case. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py index 0ada1f94..df742cdb 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py @@ -29,6 +29,8 @@ from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_FILE_NAME +from .live_cloud import find_resource_key, no_resource_key + # The tests run against the freely available ASN data file the data # repository links to, so they only rely on properties that the free data # file provides (location properties such as Latitude, Longitude and Areas @@ -36,7 +38,14 @@ data_file = ExampleUtils.find_file(ASN_DATAFILE_NAME) evidence_file = ExampleUtils.find_file(EVIDENCE_FILE_NAME) truth_file = ExampleUtils.find_file("geoip_comparison_evidence.csv") -resource_key = ExampleUtils.get_resource_key() +resource_key = find_resource_key() + +NO_IP_KEY = no_resource_key( + "the ip product", + "IP Intelligence properties such as RegisteredName, and their metadata") +NO_MIXED_KEY = no_resource_key( + "the device and ip products", + "device detection and IP Intelligence properties together") # The mixed examples combine IP intelligence with device detection, which is # an optional dependency. Their tests only run when it is installed. @@ -146,23 +155,23 @@ def test_onpremise_compare_console(self): # header + up to 20 records self.assertGreater(len(output.getvalue().splitlines()), 1) - @unittest.skipUnless( - bool(resource_key), "No resource_key environment variable set") def test_cloud_gettingstarted_console(self): + if not resource_key: + self.fail(NO_IP_KEY) from fiftyone_ipintelligence_examples.cloud.gettingstarted_console import GettingStartedConsole example = GettingStartedConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) - @unittest.skipUnless( - bool(resource_key), "No resource_key environment variable set") def test_cloud_metadata_console(self): + if not resource_key: + self.fail(NO_IP_KEY) from fiftyone_ipintelligence_examples.cloud.metadata_console import MetadataConsole example = MetadataConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) - @unittest.skipUnless( - bool(resource_key), "No resource_key environment variable set") def test_cloud_getallproperties_console(self): + if not resource_key: + self.fail(NO_IP_KEY) from fiftyone_ipintelligence_examples.cloud.getallproperties_console import GetAllPropertiesConsole example = GetAllPropertiesConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) @@ -176,9 +185,10 @@ def test_onpremise_mixed_console(self): example.run(hash_file, data_file, Logger(min_level="error"), lambda x: None) @unittest.skipUnless( - device_detection_available and bool(resource_key), - "device detection package or resource_key not available") + device_detection_available, "device detection package not available") def test_cloud_mixed_console(self): + if not resource_key: + self.fail(NO_MIXED_KEY) from fiftyone_ipintelligence_examples.cloud.mixed_console import MixedConsole example = MixedConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini index 3bc4fbd3..459f1a48 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini @@ -37,6 +37,8 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_examples {tty:--color=yes} {posargs} pass_env = + _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE resource_key license_key IPINTELLIGENCEDATAFILE diff --git a/location/ci/run-integration-tests.ps1 b/location/ci/run-integration-tests.ps1 index 1af0c1c0..bbbdb0bb 100644 --- a/location/ci/run-integration-tests.ps1 +++ b/location/ci/run-integration-tests.ps1 @@ -6,8 +6,9 @@ param ( ) if (!$Keys.TestResourceKey) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No Resource Key::No resource key was provided, so integration tests will not run." - return + throw "No TestResourceKey was passed, so the cloud tests cannot call " + + "the cloud service. The workflows set it from the " + + "_51DEGREES_RESOURCE_KEY_BESPOKE secret." } ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages . -Keys $Keys diff --git a/pipeline/ci/run-integration-tests.ps1 b/pipeline/ci/run-integration-tests.ps1 index 0ee466aa..5539b7ea 100644 --- a/pipeline/ci/run-integration-tests.ps1 +++ b/pipeline/ci/run-integration-tests.ps1 @@ -5,6 +5,12 @@ param ( [Hashtable]$Keys ) +if (!$Keys.TestResourceKey) { + throw "No TestResourceKey was passed, so the cloud tests cannot call " + + "the cloud service. The workflows set it from the " + + "_51DEGREES_RESOURCE_KEY_BESPOKE secret." +} + # The examples call the cloud service, so they are integration tests rather # than unit tests. Listing them here runs them on every Python version and # operating system in options.json, and in the nightly, which a standalone @@ -16,9 +22,23 @@ param ( # not a licence key for the cloud service, so the examples' licence-key tests # failed with it. They are not given it, and skip themselves as they did when # pipeline was its own repository, until a cloud licence key is passed. -$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples" +# +# The 51Did package runs only its tests marked live here, as its others ran +# in the unit step. Its tier tests need a key for another tier, which only +# the Cloud 51Did workflow has. Only a package whose tox.ini passes +# PYTEST_ADDOPTS sees the selection. Its build needs the OWID source, which +# build-project.ps1 copies, and the publish workflow's test job does not run +# that script, so it is copied here too. +& "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName +$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples", "fiftyone_pipeline_did" $pipelineKeys = $Keys.Clone() $pipelineKeys.Remove('DeviceDetection') -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $pipelineKeys +$addopts = $env:PYTEST_ADDOPTS +$env:PYTEST_ADDOPTS = '-m "live and not tier"' +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $pipelineKeys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/pipeline/ci/run-unit-tests.ps1 b/pipeline/ci/run-unit-tests.ps1 index c5af008e..441a2bcc 100644 --- a/pipeline/ci/run-unit-tests.ps1 +++ b/pipeline/ci/run-unit-tests.ps1 @@ -3,7 +3,16 @@ param ( [string]$RepoName ) -$packages = "fiftyone_pipeline_core", "fiftyone_pipeline_engines", "fiftyone_pipeline_engines_fiftyone", "fiftyone_pipeline_translation", "fiftyone_pipeline_did" -./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages +# The tests marked live call the cloud service, and no key reaches the unit +# step, so they are left to run-integration-tests.ps1. Only a package whose +# tox.ini passes PYTEST_ADDOPTS sees the selection. +$addopts = $env:PYTEST_ADDOPTS +$env:PYTEST_ADDOPTS = '-m "not live"' +try { + $packages = "fiftyone_pipeline_core", "fiftyone_pipeline_engines", "fiftyone_pipeline_engines_fiftyone", "fiftyone_pipeline_translation", "fiftyone_pipeline_did" + ./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index cd8b5b28..58b3ec35 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -90,8 +90,10 @@ then open . Set `PORT` to listen on another port. .venv/bin/python -m pytest ``` -The tests that call the cloud service skip themselves when no resource key -is set. +The tests that call the cloud service with a resource key fail when none is +set, naming the variables they read, which are `_51DEGREES_RESOURCE_KEY`, +`_51DEGREES_RESOURCE_KEY_BESPOKE` and `resource_key`, in that order. The +tests that use a licence key skip themselves when none is set. In CI the examples are run by [ci/run-integration-tests.ps1](../../ci/run-integration-tests.ps1) through common-ci, on every Python version and operating system in diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py index 74d791c1..60ec92d8 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py @@ -29,18 +29,21 @@ from fiftyone_pipeline_examples.example_utils import ExampleUtils +from live_cloud import find_resource_key, no_resource_key + +NO_KEY = no_resource_key( + "the device, ip and countrynamestranslated products", + "device detection and IP Intelligence properties, and the translated " + "country names", several=True) + # The ways a test can reach the cloud service, as the keyword arguments the -# examples' create_pipeline takes. Each is skipped when its key is not set. +# examples' create_pipeline takes, each passed through credentials(). A test +# with a resource key fails when none is set. A test with a licence key is +# skipped when none is set, because CI has no cloud licence key to give it. KEYS = [ pytest.param( - {"resource_key": ExampleUtils.get_resource_key()}, - id="resource key", - marks=pytest.mark.skipif( - not ExampleUtils.get_resource_key(), - reason=f"Set {ExampleUtils.RESOURCE_KEY_ENV_VAR} to a resource " - "key carrying device detection and IP intelligence " - "properties to run the cloud tests with a resource " - "key.")), + {"resource_key": find_resource_key()}, + id="resource key"), pytest.param( {"license_key": ExampleUtils.get_license_key()}, id="licence key", @@ -53,6 +56,15 @@ ] +def credentials(keys): + """The keyword arguments from KEYS, failing the test when they hold a + resource key that is not set.""" + + if "resource_key" in keys and not keys["resource_key"]: + pytest.fail(NO_KEY, pytrace=False) + return keys + + def cell_after(page, label): """ The text a browser shows in the table cell after the one holding label, diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_console.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_console.py index 190d940e..f4cb7bc4 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_console.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_console.py @@ -34,7 +34,7 @@ GettingStartedConsole, create_pipeline) from fiftyone_pipeline_examples.example_utils import ExampleUtils -from helpers import KEYS +from helpers import KEYS, credentials SEPARATOR = "=" * 79 @@ -45,7 +45,7 @@ def output(request): pipeline = create_pipeline( Logger(min_level="info"), cloud_endpoint=ExampleUtils.get_cloud_endpoint(), - **request.param) + **credentials(request.param)) GettingStartedConsole().run(pipeline, lines.append) return "\n".join(lines) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web.py index c3f7eb6d..fa66520c 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web.py @@ -37,7 +37,7 @@ create_app, create_pipeline) from fiftyone_pipeline_examples.example_utils import ExampleUtils -from helpers import KEYS, cell_after, country_options +from helpers import KEYS, cell_after, country_options, credentials DESKTOP_CHROME = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " @@ -51,7 +51,7 @@ @pytest.fixture(scope="module", params=KEYS) def key(request): - return request.param + return credentials(request.param) @pytest.fixture(scope="module") diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini index 1fae6a61..56e9bf52 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini @@ -29,15 +29,16 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_examples {tty:--color=yes} {posargs} # tox runs the tests in a clean environment and strips anything it is not -# told to keep, so without this the cloud tests never see a key and skip -# themselves on every run, which reads as a pass. resource_key and -# license_key are the names common-ci sets, from the secrets named in -# ci/run-integration-tests.ps1. The _51DEGREES names are the ones the -# examples document and the Examples workflow sets. The last three name a -# cloud service other than the public one: the first two are read by the -# examples, the third by the cloud request engine underneath them. +# told to keep, so without this the cloud tests never see a key and fail on +# every run. resource_key and license_key are the names common-ci sets, from +# the secrets named in ci/run-integration-tests.ps1. The _51DEGREES names are +# the ones the examples document, the Examples workflow sets and +# tests/live_cloud.py reads. The last three name a cloud service other than the +# public one: the first two are read by the examples, the third by the cloud +# request engine underneath them. pass_env = _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_LICENSE_KEY resource_key license_key diff --git a/pipeline/fiftyone_pipeline_did/tests/conftest.py b/pipeline/fiftyone_pipeline_did/tests/conftest.py new file mode 100644 index 00000000..8dd5e4f9 --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/conftest.py @@ -0,0 +1,38 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The markers that keep the live cloud tests out of the unit test run. +# pipeline/ci/run-unit-tests.ps1 deselects "live", and +# pipeline/ci/run-integration-tests.ps1 runs "live and not tier", because no +# key for another tier reaches it. The Cloud 51Did workflow runs the tier +# tests with the Basic key. + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + config.addinivalue_line( + "markers", + "tier: asserts what a key on one tier is entitled to, and needs the " + "key for that tier") diff --git a/pipeline/fiftyone_pipeline_did/tests/live_cloud.py b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py b/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py index 23d8fe90..6a41e265 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py @@ -20,25 +20,27 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -"""Live tests against the cloud, run only when a resource key is set. - -The key is read from ``resource_key``, the variable the repository's other -live tests and its CI use, or from ``_51DEGREES_RESOURCE_KEY`` as the -example reads it. Without either the tests are skipped, not failed. An -optional licence key is read from ``license_key`` or -``_51DEGREES_LICENSE_KEY``, and the endpoint from ``FOD_CLOUD_API_URL`` as -everywhere else. Every test here costs uses against the resource key. +"""Live tests against the 51Degrees cloud service. + +The resource key is read from the names in ``live_cloud.py``, and every +test fails when none is set. The key has to carry the FODid product, and +the CloudV5FODiD product for the marketing usages, which +``test_did_tier_live.py`` shows a Basic key does not get. An optional +licence key is read from ``license_key`` or ``_51DEGREES_LICENSE_KEY``, and +the endpoint from ``FOD_CLOUD_API_URL`` as everywhere else. Every test here +costs uses against the resource key. """ import asyncio import json import os -import sys import unittest import urllib.error import urllib.parse import urllib.request +import pytest + from fiftyone_pipeline_did import ( ContextResult, DidClient, @@ -49,11 +51,22 @@ ) from fiftyone_pipeline_did.did_client import USER_AGENT -RESOURCE_KEY = os.environ.get("resource_key") \ - or os.environ.get("_51DEGREES_RESOURCE_KEY") +from .live_cloud import answered_without, find_resource_key, no_resource_key + +pytestmark = pytest.mark.live + +RESOURCE_KEY = find_resource_key() LICENCE_KEY = os.environ.get("license_key") \ or os.environ.get("_51DEGREES_LICENSE_KEY") +FODID = "the FODid product" +MARKETING = "the CloudV5FODiD product" + +NO_KEY = no_resource_key( + FODID + ", and " + MARKETING + " for the marketing usages", + "FODiD.IdProbGlobal and FODiD.IdProbLic under each id.usage", + several=True) + def run(coroutine): """Runs one client call to completion on a fresh event loop, as the @@ -62,14 +75,27 @@ def run(coroutine): return asyncio.run(coroutine) -@unittest.skipUnless( - RESOURCE_KEY, - "set resource_key (or _51DEGREES_RESOURCE_KEY) to run the live tests") +def null_reason(fodid): + """The reason the cloud gave for answering without a 51Did.""" + return fodid.get("idprobglobalnullreason") \ + or fodid.get("idproblicnullreason") + + +def no_creator_context(endpoint, detail): + """What to fail with when the host does not offer the creator context, + which the 51Degrees cloud does.""" + return ("The cloud service at {0} {1}, so the creator context could not " + "be checked. Point FOD_CLOUD_API_URL at a service that offers " + "it, or leave it unset for the 51Degrees cloud.").format( + endpoint, detail) + + class DidClientLiveTests(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.client = DidClient(RESOURCE_KEY, LICENCE_KEY) + def setUp(self): + if not RESOURCE_KEY: + self.fail(NO_KEY) + self.client = DidClient(RESOURCE_KEY, LICENCE_KEY) def create(self): """Creates a 51Did through the cloud ``json`` endpoint, the route @@ -88,10 +114,10 @@ def create(self): body = json.loads(response.read().decode("utf-8")) fodid = body.get("fodid") or {} value = fodid.get("idproblic") or fodid.get("idprobglobal") - self.assertTrue( - value, "the resource key must include a 51Did property " - "(idprobglobal or idproblic); the json answer was: " - + json.dumps(body)[:300]) + if not value: + self.fail(answered_without( + "a 51Did for id.usage=non-marketing", null_reason(fodid), + FODID)) return FodId.from_base64(value) def test_created_identifier_verifies_offline_and_through_the_cloud(self): @@ -108,8 +134,10 @@ def test_garbage_result_redeems_as_unreadable(self): try: result = run(self.client.redeem(fod_id, "not-base64url!!", "x")) except DidNotSupportedError as error: - self.skipTest("the host does not offer the creator context: " - + str(error)) + self.fail(no_creator_context( + self.client.endpoint, + "answered that it does not offer the creator context (" + + str(error) + ")")) self.assertEqual(200, result.status_code) self.assertEqual(ContextResult.UNREADABLE, result.context) @@ -123,9 +151,8 @@ def test_garbage_result_redeems_as_unreadable(self): def identifiers_for(self, name, value): """Asks the ``json`` endpoint for a 51Did with the given query - parameter and returns every identifier it answered with. An empty - list means the resource key is not entitled to that usage, which - the caller reports rather than fails.""" + parameter, and returns every identifier it answered with and the + ``fodid`` member, which holds the reason when there is none.""" url = "{0}{1}.json?{2}&values=FODiD.IdProbGlobal" \ "&values=FODiD.IdProbLic".format( self.client.endpoint, @@ -137,11 +164,12 @@ def identifiers_for(self, name, value): with urllib.request.urlopen(request, timeout=30) as response: body = json.loads(response.read().decode("utf-8")) fodid = body.get("fodid") or {} - return [ + identifiers = [ FodId.from_base64(fodid[field]) for field in ("idprobglobal", "idproblic") if fodid.get(field) ] + return identifiers, fodid def assert_aligned(self, label, fod_id, usage, terms, indirect): """Asserts the terms and every field the flags byte carries, read @@ -170,35 +198,23 @@ def test_every_usage_reads_back_the_terms_and_flags(self): ends at the match key and reads as no terms. """ cases = [ - ("non-marketing", Usage.NON_MARKETING, None), - ("standard", Usage.STANDARD, self.MODEL_TERMS_FOR_MARKETING_2), + ("non-marketing", Usage.NON_MARKETING, None, FODID), + ("standard", Usage.STANDARD, self.MODEL_TERMS_FOR_MARKETING_2, + MARKETING), ("personalized", Usage.PERSONALIZED, - self.MODEL_TERMS_FOR_MARKETING_2), + self.MODEL_TERMS_FOR_MARKETING_2, MARKETING), ] - checked = 0 - for name, usage, terms in cases: - identifiers = self.identifiers_for("id.usage", name) + for name, usage, terms, product in cases: + identifiers, fodid = self.identifiers_for("id.usage", name) if not identifiers: - print("id.usage={0}: no identifier returned, so this key is " - "not entitled to that usage.".format(name), - file=sys.stderr) - continue + self.fail(answered_without( + "a 51Did for id.usage=" + name, null_reason(fodid), + product)) for index, fod_id in enumerate(identifiers): self.assert_aligned( "{0}[{1}]".format(name, index), fod_id, usage, terms, False) - if terms is not None: - checked += len(identifiers) - - # Reported as a skip rather than printed, because a runner shows - # the counts and not the output, so a run that proved nothing would - # otherwise look identical to one that proved everything. - if checked == 0: - self.skipTest( - "this resource key returned no marketing 51Did, so no terms " - "address was read and this run did not prove it; use a key " - "entitled to the standard or personalized usage") def test_consent_string_sets_the_usage_is_indirect_bit(self): """A consent management platform sends an IAB TCF consent string @@ -219,39 +235,28 @@ def test_consent_string_sets_the_usage_is_indirect_bit(self): the Appendix 1 standard set of 1, 2, 7, 8 and 11. """ cases = [ - ("AAAAAAAAAAAAAAAAAAAAAAAAAP_w", Usage.PERSONALIZED), - ("AAAAAAAAAAAAAAAAAAAAAAAAAMMg", Usage.STANDARD), + ("AAAAAAAAAAAAAAAAAAAAAAAAAP_w", Usage.PERSONALIZED, + "personalized"), + ("AAAAAAAAAAAAAAAAAAAAAAAAAMMg", Usage.STANDARD, "standard"), ] - proven = 0 - for tc_string, usage in cases: + for tc_string, usage, name in cases: # No id.usage is sent. A stated usage wins over a consent # string, so sending one would leave the bit clear and this # would prove the opposite of what it says. - identifiers = self.identifiers_for("tcstring", tc_string) + identifiers, fodid = self.identifiers_for("tcstring", tc_string) if not identifiers: - print("consent string granting {0}: no identifier returned, " - "so this key is not entitled to that marketing " - "usage.".format(usage), file=sys.stderr) - continue + self.fail(answered_without( + "a 51Did for a consent string granting the " + name + + " usage", null_reason(fodid), MARKETING)) for index, fod_id in enumerate(identifiers): # A consent string granting a marketing usage produces a # marketing identifier, so the terms travel with it too. self.assert_aligned( - "consent/{0}[{1}]".format(usage, index), fod_id, usage, + "consent/{0}[{1}]".format(name, index), fod_id, usage, self.MODEL_TERMS_FOR_MARKETING_2, True) - proven += len(identifiers) - - # Same reasoning as the usage test above. - if proven == 0: - self.skipTest( - "this resource key returned no identifier for either consent " - "string, so the usage is indirect bit was never read and " - "this run did not prove it") - -@unittest.skipUnless( - RESOURCE_KEY, - "set resource_key (or _51DEGREES_RESOURCE_KEY) to run the live tests") + + class BrowserCreatorContextLiveTests(unittest.TestCase): """The whole browser path against the live service, as a page runs it. @@ -264,6 +269,10 @@ class BrowserCreatorContextLiveTests(unittest.TestCase): service's contract with every one of these packages, and a name the service adds, drops or renames has to be seen here rather than by a customer. + + Without a licence key the sealed result cannot be opened, so each test + then checks that it redeems as unreadable, which is what a caller with + no licence key gets. """ # A browser the service will judge as a browser, and a second one @@ -297,9 +306,10 @@ class BrowserCreatorContextLiveTests(unittest.TestCase): "transport", "device", "browserip", "connectionip", "asn", "platformname", "platformversion", "browsername", "browserversion") - @classmethod - def setUpClass(cls): - cls.client = DidClient(RESOURCE_KEY, LICENCE_KEY) + def setUp(self): + if not RESOURCE_KEY: + self.fail(NO_KEY) + self.client = DidClient(RESOURCE_KEY, LICENCE_KEY) def get(self, url, user_agent): request = urllib.request.Request( @@ -325,14 +335,10 @@ def create(self, usage="standard"): value = fodid.get("idproblic") or fodid.get("idprobglobal") if not value: # The probabilistic reason in full, because it names every - # snippet result the service is still waiting for, and a cut - # dump of the whole member hides them behind the email ones. - reason = fodid.get("idprobglobalnullreason") \ - or fodid.get("idproblicnullreason") \ - or json.dumps(fodid)[:300] - self.skipTest( - "this resource key returned no identifier for a browser " - "shaped request: " + reason) + # snippet result the service is still waiting for. + self.fail(answered_without( + "a 51Did for a browser shaped request with id.usage=" + + usage, null_reason(fodid), MARKETING)) return FodId.from_base64(value) def verify_context(self, fod_id, user_agent): @@ -346,13 +352,16 @@ def verify_context(self, fod_id, user_agent): urllib.parse.urlencode(query)) status, body = self.get(url, user_agent) if status == 404: - self.skipTest("the host does not offer the creator context") + self.fail(no_creator_context( + self.client.endpoint, + "answered 404 from id/verify-context")) self.assertEqual(200, status, body[:300]) answer = json.loads(body) if "result" not in answer: - self.skipTest( - "this host holds no context secret, so it answered in the " - "open with " + body[:200]) + self.fail(no_creator_context( + self.client.endpoint, + "holds no creator context secret, so it answered in the " + "open with " + body[:200])) return answer["result"] def test_a_page_creates_reads_back_verifies_and_redeems(self): @@ -364,19 +373,18 @@ def test_a_page_creates_reads_back_verifies_and_redeems(self): result = self.verify_context(fod_id, self.CHROME) redeemed = run(self.client.redeem(fod_id, result)) if not self.client.has_licence_key: - # Without a licence key the sealed result cannot be opened, - # which the service reports as unreadable. self.assertEqual(ContextResult.UNREADABLE, redeemed.context) - self.skipTest("set license_key to redeem a sealed result") + return self.assertEqual(ContextResult.VERIFIED, redeemed.context) self.assertEqual("verified", redeemed.signature.value) def test_another_browser_redeems_as_a_mismatch_naming_every_factor(self): - if not LICENCE_KEY: - self.skipTest("set license_key to redeem a sealed result") fod_id = self.create() result = self.verify_context(fod_id, self.SAFARI) redeemed = run(self.client.redeem(fod_id, result)) + if not self.client.has_licence_key: + self.assertEqual(ContextResult.UNREADABLE, redeemed.context) + return self.assertEqual(ContextResult.MISMATCH, redeemed.context) self.assertEqual("verified", redeemed.signature.value, "the identifier is genuine whichever browser " diff --git a/pipeline/fiftyone_pipeline_did/tests/test_did_tier_live.py b/pipeline/fiftyone_pipeline_did/tests/test_did_tier_live.py new file mode 100644 index 00000000..e52b5f98 --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/test_did_tier_live.py @@ -0,0 +1,113 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""What a key on the Basic tier is entitled to, asked of the live cloud. + +A Basic key carries the FODid product, so it gets a 51Did for the +non-marketing usage. It does not carry the CloudV5FODiD product, so the +service refuses it the standard and personalized usages and says which +product is missing. The key is read from ``_51DEGREES_RESOURCE_KEY_BASIC`` +only, because these tests assert what that one tier gets, and they fail +when it is not set. +""" + +import json +import urllib.parse +import urllib.request +import unittest + +import pytest + +from fiftyone_pipeline_did import DidClient, FodId, Usage +from fiftyone_pipeline_did.did_client import USER_AGENT + +from .live_cloud import answered_without, find_resource_key, no_resource_key + +pytestmark = [pytest.mark.live, pytest.mark.tier] + +BASIC_KEY_NAMES = ("_51DEGREES_RESOURCE_KEY_BASIC",) +BASIC_KEY = find_resource_key(BASIC_KEY_NAMES) + +FIELDS = ("idprobglobal", "idproblic") + + +class BasicTierLiveTests(unittest.TestCase): + + def setUp(self): + if not BASIC_KEY: + self.fail(no_resource_key( + "the FODid product on the Basic tier", + "a 51Did under each id.usage, expecting the marketing " + "usages to be refused", several=True, + names=BASIC_KEY_NAMES)) + self.endpoint = DidClient(BASIC_KEY).endpoint + + def fodid_for(self, usage): + """The ``fodid`` member of the ``json`` endpoint's answer for the + usage.""" + url = "{0}{1}.json?{2}".format( + self.endpoint, urllib.parse.quote(BASIC_KEY, safe=""), + urllib.parse.urlencode([ + ("id.usage", usage), + ("values", "FODiD.IdProbGlobal"), + ("values", "FODiD.IdProbLic")])) + request = urllib.request.Request( + url, data=b"", headers={"User-Agent": USER_AGENT}, + method="POST") + with urllib.request.urlopen(request, timeout=30) as response: + body = json.loads(response.read().decode("utf-8")) + return body.get("fodid") or {} + + def test_non_marketing_usage_gets_a_51did(self): + fodid = self.fodid_for("non-marketing") + for field in FIELDS: + if not fodid.get(field): + self.fail(answered_without( + field + " for id.usage=non-marketing", + fodid.get(field + "nullreason"), "the FODid product")) + self.assertEqual( + Usage.NON_MARKETING, FodId.from_base64(fodid[field]).usage, + field) + + def test_marketing_usages_are_refused_naming_the_product(self): + for usage in ("standard", "personalized"): + fodid = self.fodid_for(usage) + for field in FIELDS: + self.assertFalse( + fodid.get(field), + "A Basic key got {0} for id.usage={1}, which only a key " + "that carries the CloudV5FODiD product should get. Check " + "the key is still on the Basic tier.".format( + field, usage)) + name = "IdProbGlobal" if field == "idprobglobal" \ + else "IdProbLic" + self.assertEqual( + "Can not form {0}. For marketing (personalized or " + "standard) usages a CloudV5FODiD product needs to be a " + "part of the license.".format(name), + fodid.get(field + "nullreason"), + "the reason given for {0} with id.usage={1}".format( + field, usage)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_did/tox.ini b/pipeline/fiftyone_pipeline_did/tox.ini index d80d881b..edf34540 100644 --- a/pipeline/fiftyone_pipeline_did/tox.ini +++ b/pipeline/fiftyone_pipeline_did/tox.ini @@ -12,13 +12,17 @@ package = wheel wheel_build_env = .pkg # tox runs the tests in a clean environment and strips anything it is not # told to keep, so without this the live cloud tests never see the resource -# key and skip themselves on every run, which reads as a pass. +# key and fail on every run. PYTEST_ADDOPTS carries the markers the CI +# scripts select with. passenv = _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE + _51DEGREES_RESOURCE_KEY_BASIC _51DEGREES_LICENSE_KEY resource_key license_key FOD_CLOUD_API_URL + PYTEST_ADDOPTS deps = {[common]deps} # owid is provided by the owid-python submodule (../owid-python), not From bb11a56329a38b060b1785ff1b18e4112fd8043f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 06:16:24 +0000 Subject: [PATCH 20/68] Make the example profile comments and the Lite file name match the code (#65) - The IP Intelligence data file update example now names the LowMemory profile for the pipeline that downloads and reloads the Enterprise data file, rather than taking the engine's default. - The Device Detection getting started example's comment said it uses the low memory profile, but the code sets MaxPerformance. The comment now says MaxPerformance, and why it suits the free Lite data file. - The IP Intelligence performance example's header said worker processes share a memory mapped data file under LowMemory, as if that were the default. It now says the default MaxPerformance loads the whole file into each worker, and that -p LowMemory lets the workers share the operating system's file cache. - The IP Intelligence package readme named the Lite data file 51Degrees-LiteIpiV41.ipi, which is CI's name for it. The data repository's scripts save it as 51Degrees-LiteV41.ipi. --- .../onpremise/gettingstarted_console.py | 8 +++++--- ip-intelligence/fiftyone_ipintelligence/readme.md | 2 +- .../onpremise/datafileupdate_console.py | 3 +++ .../onpremise/performance_console.py | 9 ++++++--- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index 12c4cff6..4742eed1 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -65,9 +65,11 @@ def run(self, data_file, logger, output): # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run pipeline = DeviceDetectionPipelineBuilder( data_file_path = data_file, - # We use the low memory profile as its performance is - # sufficient for this example. See the documentation for - # more detail on this and other configuration options: + # We use the MaxPerformance profile, which loads the whole data + # file into memory. The free Lite data file is about 35 MB, so + # that costs little memory and gives the fastest detections. + # See the documentation for more detail on this and other + # configuration options: # https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run # https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run diff --git a/ip-intelligence/fiftyone_ipintelligence/readme.md b/ip-intelligence/fiftyone_ipintelligence/readme.md index 1b1c9eec..4b546d11 100644 --- a/ip-intelligence/fiftyone_ipintelligence/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence/readme.md @@ -25,7 +25,7 @@ pipeline = IpIntelligencePipelineBuilder(resource_key="YOUR_RESOURCE_KEY").build # On-premise pipeline = IpIntelligencePipelineBuilder( - data_file_path="51Degrees-LiteIpiV41.ipi", + data_file_path="51Degrees-LiteV41.ipi", licence_keys="", usage_sharing=False).build() diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py index a11c17a3..54971693 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py @@ -188,6 +188,9 @@ def run(self, data_file, license_key, interactive, logger, output): # is configured to use a temporary copy. data_file_path = data_file, create_temp_data_copy = True, + # The Enterprise data file is large, so the LowMemory profile + # reads it from disk rather than holding parts of it in memory. + performance_profile = "LowMemory", # The update listener notifies us when an update is complete. data_file_update_service = update_service, # A license key is required for automatic updates. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py index c76f930a..e1001011 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py @@ -35,9 +35,12 @@ # # The native engine holds the Python GIL for the duration of each lookup, so # threads cannot run lookups in parallel. Parallelism therefore uses worker -# processes, each building its own pipeline. With the LowMemory profile the -# memory mapped data file is shared between the processes by the operating -# system, so a large enterprise data file is not duplicated per worker. +# processes, each building its own pipeline. With the default MaxPerformance +# profile each worker loads the whole data file into its own memory, so the +# memory used grows with the number of workers. With -p LowMemory each worker +# reads the data file from disk as it needs it, and the operating system's +# file cache holds one copy for all the workers, so a large Enterprise data +# file is not held once per worker. # # Required PyPi Dependencies: # - fiftyone_ipintelligence From f3f0d561610be22467c2fb661531a0d2f4306620 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 06:16:42 +0000 Subject: [PATCH 21/68] Use the Balanced performance profile by default for IP Intelligence (#62) The IP Intelligence on-premise specification gives Balanced as the default performance profile, and the C library's default IP Intelligence configuration is the Balanced one, but the Python engine defaulted to LowMemory. The engine now defaults to "Balanced", which its profile switch already maps to the native Balanced configuration, and the docstrings in the engine and both builders say so. Balanced loads some collections into memory and caches others, where LowMemory reads everything from the data file, so a caller relying on the default uses more memory. Naming "LowMemory" keeps the previous behaviour. A new test, on the free ASN data file, compares the collection settings of the configuration an engine built at its defaults gives the native code with those of each profile, because the engine keeps no name for the profile in use. --- .../ipintelligence_pipelinebuilder.py | 4 +- .../ipintelligence_onpremise.py | 6 +- ...pintelligence_onpremise_pipelinebuilder.py | 4 +- .../tests/test_performance_profile_default.py | 118 ++++++++++++++++++ 4 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_performance_profile_default.py diff --git a/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py b/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py index 8576e4cf..e17ee5c5 100644 --- a/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py +++ b/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py @@ -82,7 +82,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index 37b45904..9f15784e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -57,7 +57,7 @@ def __init__( polling_interval=30, update_time_maximum_randomisation=10, create_temp_data_copy=True, - performance_profile="LowMemory", + performance_profile="Balanced", reuse_temp_file=False, concurrency=multiprocessing.cpu_count(), update_on_start=False, @@ -84,7 +84,9 @@ def __init__( @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py index adb27093..182d39a8 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py @@ -62,7 +62,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_performance_profile_default.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_performance_profile_default.py new file mode 100644 index 00000000..65211ba7 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_performance_profile_default.py @@ -0,0 +1,118 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import os +import unittest + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule import ConfigIpiSwig +from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise import IpIntelligenceOnPremise +from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise_pipelinebuilder import IpIntelligenceOnPremisePipelineBuilder + +from .constants import TestConstants + +# The free ASN data file, which needs no licence key. +DATA_FILE = os.path.normpath(TestConstants.get_asn_data_file_path()) + +# The collections whose settings the configuration can be asked for. The +# getters for components, properties, values and profiles hand Python a +# setting the configuration owns, which Python then frees, corrupting the +# heap, so those four are left out. +COLLECTIONS = ( + "Strings", "Maps", "Graphs", "ProfileGroups", "ProfileOffsets", + "PropertyTypes", "Graph") + +# Each profile name the engine accepts and the native setter it maps to. +PROFILES = { + "LowMemory": "setLowMemory", + "MaxPerformance": "setMaxPerformance", + "Balanced": "setBalanced", + "BalancedTemp": "setBalancedTemp", + "HighPerformance": "setHighPerformance", +} + + +def collection_settings(config): + """ + Whether each collection is loaded into memory and how many items it + caches, which are what a performance profile sets. + """ + settings = {} + for name in COLLECTIONS: + collection = getattr(config, "get" + name)() + settings[name] = (collection.getLoaded(), collection.getCapacity()) + return settings + + +def profile_settings(setter): + """The collection settings the native code gives a profile.""" + config = ConfigIpiSwig() + getattr(config, setter)() + return collection_settings(config) + + +@unittest.skipUnless( + os.path.isfile(DATA_FILE), + "The free ASN data file is not at {0}. Download it with " + "cxx/data/get-data-files.ps1 or cxx/data/get-data-files.sh, after " + "'git submodule update --init --recursive'.".format(DATA_FILE)) +class PerformanceProfileDefaultTests(unittest.TestCase): + """ + The engine keeps no name for the profile in use, so these tests compare + the collection settings of the configuration the engine gives the + native code with the settings each profile has there. MaxPerformance + and LowMemory have the same collection settings, and Balanced and + BalancedTemp do too, so neither pair can be told apart this way. + """ + + def _engine_settings(self, **settings): + engine = IpIntelligenceOnPremise( + data_file_path=DATA_FILE, licence_keys="", **settings) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + self.assertIsNotNone(engine.engine) + return collection_settings(engine.config) + + def test_balanced_and_low_memory_differ(self): + self.assertNotEqual( + profile_settings("setBalanced"), profile_settings("setLowMemory")) + + def test_engine_defaults_use_balanced(self): + self.assertEqual( + profile_settings("setBalanced"), self._engine_settings()) + + def test_builder_defaults_use_balanced(self): + pipeline = IpIntelligenceOnPremisePipelineBuilder( + data_file_path=DATA_FILE, usage_sharing=False, + licence_keys="").build() + self.addCleanup(pipeline.close) + engine = pipeline.get_element("ip") + self.assertEqual( + profile_settings("setBalanced"), + collection_settings(engine.config)) + + def test_each_named_profile_is_used(self): + for name, setter in PROFILES.items(): + with self.subTest(name): + self.assertEqual( + profile_settings(setter), + self._engine_settings(performance_profile=name)) From d6ebf5ef3f5234ab49f317a07bacd5de04bc4b14 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 12:44:21 +0000 Subject: [PATCH 22/68] Accept the specification's translation option names, and the behavior as text (#64) * Accept the missing translation behavior as text, and refuse an unknown one TranslationEngineBase and Translator stored the behavior they were given without checking it, and compared it with the MissingTranslationBehavior members. A configuration file can only give text, so a configuration naming "EmptyString" or "flow_error" matched no member and a missing translation kept its original value, with nothing to say the setting had been ignored. MissingTranslationBehavior.from_value now returns a member as it is, and turns text naming a member into that member. It takes the specification's names (Original, EmptyString, FlowError) and the members' own names and values, ignoring case and underscores, and raises ValueError for anything else. The engine and the translator use it, so an unknown value stops the engine being built. The parameter is still named behavior. * Accept the specification's name for the missing translation behavior The specification names the Translation Engine's options SourceElementDataKey and MissingTranslationBehavior. Python already took source_element_data_key, but named the second option behavior. TranslationEngineBase and Translator now take missing_translation_behavior. behavior still works as a keyword-only deprecated alias that raises a DeprecationWarning, and when both are given missing_translation_behavior wins. Neither given means Original, as before. A value given by position is taken as missing_translation_behavior, as it was taken as behavior. The IP Intelligence translation engines now pass every option by its name, so they need a fiftyone_pipeline_translation that takes the new name. Their package requires fiftyone_pipeline_translation 4.5.43 or later, the version the release plan gives this commit. * Leave the IP Intelligence translation engines as they were on rc The IP Intelligence translation engines pass the missing translation behavior by position. A value passed by position is taken as missing_translation_behavior by the new translation package and as behavior by the old one, so the engines work with either release and raise no deprecation warning without any change. Passing it by name needed a minimum version of fiftyone_pipeline_translation that cannot be known until the release that carries the new name, so the engines and their package's requirements are back to their rc content. * Take None as not given for the missing translation behavior rc treated None as Original, because the translator took any value that was not EMPTY_STRING or FLOW_ERROR as Original. Refusing it would stop a configuration holding null, or code passing an unset option on as None, from building after a patch release. The repository's other engines use None to mean not set, so the translation engine and Translator now do the same. Both parameters default to None, None under either name gives Original without a warning, and anything else that names no member is still refused. --- .../missing_translation_behavior.py | 54 ++++ .../translation_engine_base.py | 22 +- .../translator.py | 22 +- .../tests/test_translation.py | 265 ++++++++++++++++++ 4 files changed, 354 insertions(+), 9 deletions(-) diff --git a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/missing_translation_behavior.py b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/missing_translation_behavior.py index 794ec987..619f843b 100644 --- a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/missing_translation_behavior.py +++ b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/missing_translation_behavior.py @@ -20,6 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +import warnings from enum import Enum @@ -38,3 +39,56 @@ class MissingTranslationBehavior(Enum): # Add a not-found error to the flow data and return None. FLOW_ERROR = "flow_error" + + @classmethod + def from_value(cls, value): + + """! + The member a value names. A member is returned as it is. Text names + a member by the specification's name (Original, EmptyString or + FlowError) or by the member's own name or value, ignoring case and + underscores, because a configuration file cannot give a member. + + @param value: A member, or text naming one. + + @rtype: MissingTranslationBehavior + @return: The member named. + + @raises ValueError: When the value names no member. + """ + + if isinstance(value, cls): + return value + if isinstance(value, str): + key = value.replace("_", "").lower() + for member in cls: + if member.name.replace("_", "").lower() == key: + return member + raise ValueError( + f"{value!r} is not a missing translation behavior. Use " + f"Original, EmptyString or FlowError.") + + +def _resolve(missing_translation_behavior, behavior): + + """! + The member the constructor arguments name. None means not given, as for + the other engines' optional settings, so a setting passed on as None + gives ORIGINAL. behavior is the deprecated name for + missing_translation_behavior, so using it warns, and when both are given + missing_translation_behavior wins. Neither given means ORIGINAL. + """ + + if behavior is not None: + # Two levels up is the caller of the constructor that called this. + warnings.warn( + "The 'behavior' argument is deprecated. Use " + "'missing_translation_behavior', the name the specification " + "gives the option.", + DeprecationWarning, + stacklevel=3) + if missing_translation_behavior is None: + missing_translation_behavior = behavior + if missing_translation_behavior is None: + return MissingTranslationBehavior.ORIGINAL + return MissingTranslationBehavior.from_value(missing_translation_behavior) diff --git a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translation_engine_base.py b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translation_engine_base.py index 4f349e76..8be1d175 100644 --- a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translation_engine_base.py +++ b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translation_engine_base.py @@ -31,7 +31,8 @@ from .languages import Languages, validate_locale from .translator import Translator -from .missing_translation_behavior import MissingTranslationBehavior +from .missing_translation_behavior import ( + MissingTranslationBehavior, _resolve) class TranslationEngineBase(FlowElement): @@ -65,7 +66,9 @@ def __init__( translations, sources, fixed_language=None, - behavior=MissingTranslationBehavior.ORIGINAL): + missing_translation_behavior=None, + *, + behavior=None): """! Constructor. @@ -86,8 +89,17 @@ def __init__( always translates to this language, otherwise the language is resolved from the evidence. - @type behavior: MissingTranslationBehavior - @param behavior: Behavior when a translation is missing for a value. + @type missing_translation_behavior: MissingTranslationBehavior or + string + @param missing_translation_behavior: Behavior when a translation is + missing for a value, as a member or as text naming one, such as + "EmptyString". ORIGINAL when not given or None. Any other value + raises ValueError. + + @type behavior: MissingTranslationBehavior or string + @param behavior: Deprecated name for missing_translation_behavior, + which raises a DeprecationWarning when used. When both are given, + missing_translation_behavior wins. """ super().__init__() @@ -100,6 +112,8 @@ def __init__( if not source_element_data_key or not source_element_data_key.strip(): raise ValueError("The source element key must be configured.") + behavior = _resolve(missing_translation_behavior, behavior) + self.datakey = "translation" self.source_element_data_key = source_element_data_key.strip() self._behavior = behavior diff --git a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translator.py b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translator.py index 19be2cd3..0a014c25 100644 --- a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translator.py +++ b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translator.py @@ -22,7 +22,8 @@ from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue -from .missing_translation_behavior import MissingTranslationBehavior +from .missing_translation_behavior import ( + MissingTranslationBehavior, _resolve) class Translator: @@ -41,7 +42,9 @@ class Translator: def __init__( self, translations=None, - behavior=MissingTranslationBehavior.ORIGINAL): + missing_translation_behavior=None, + *, + behavior=None): """! Constructor. @@ -50,15 +53,24 @@ def __init__( @param translations: Map of source value to translated value. Lookups are case-insensitive. - @type behavior: MissingTranslationBehavior - @param behavior: Behavior when a translation is missing for a value. + @type missing_translation_behavior: MissingTranslationBehavior or + string + @param missing_translation_behavior: Behavior when a translation is + missing for a value, as a member or as text naming one, such as + "EmptyString". ORIGINAL when not given or None. Any other value + raises ValueError. + + @type behavior: MissingTranslationBehavior or string + @param behavior: Deprecated name for missing_translation_behavior, + which raises a DeprecationWarning when used. When both are given, + missing_translation_behavior wins. """ translations = translations or {} # Store with lower-cased keys so lookups are case-insensitive. self._translations = { str(key).lower(): value for key, value in translations.items()} - self._behavior = behavior + self._behavior = _resolve(missing_translation_behavior, behavior) def translate(self, value, errors=None): diff --git a/pipeline/fiftyone_pipeline_translation/tests/test_translation.py b/pipeline/fiftyone_pipeline_translation/tests/test_translation.py index ebafc0d9..644e23b9 100644 --- a/pipeline/fiftyone_pipeline_translation/tests/test_translation.py +++ b/pipeline/fiftyone_pipeline_translation/tests/test_translation.py @@ -21,6 +21,7 @@ # ********************************************************************* import unittest +import warnings from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary @@ -183,5 +184,269 @@ def test_locale_from_filename(self): TranslationEngineBase._get_language_name("countries.fr_FR.yml")) +# Each text the missing translation behavior accepts, and the member it +# names. The specification's names, the members' names and values, and +# other mixes of case and underscores are all accepted. +BEHAVIOR_TEXTS = { + "Original": MissingTranslationBehavior.ORIGINAL, + "original": MissingTranslationBehavior.ORIGINAL, + "ORIGINAL": MissingTranslationBehavior.ORIGINAL, + "EmptyString": MissingTranslationBehavior.EMPTY_STRING, + "emptystring": MissingTranslationBehavior.EMPTY_STRING, + "empty_string": MissingTranslationBehavior.EMPTY_STRING, + "EMPTY_STRING": MissingTranslationBehavior.EMPTY_STRING, + "Empty_String": MissingTranslationBehavior.EMPTY_STRING, + "FlowError": MissingTranslationBehavior.FLOW_ERROR, + "flowerror": MissingTranslationBehavior.FLOW_ERROR, + "flow_error": MissingTranslationBehavior.FLOW_ERROR, + "FLOW_ERROR": MissingTranslationBehavior.FLOW_ERROR, +} + +# Values that name no member, each refused when the engine is built. +UNKNOWN_BEHAVIORS = ( + "", "Empty String", "Origin", "Missing", "flow-error", 1, + ["Original"]) + +# The specification's name for each member. +SPECIFICATION_NAMES = { + "Original": MissingTranslationBehavior.ORIGINAL, + "EmptyString": MissingTranslationBehavior.EMPTY_STRING, + "FlowError": MissingTranslationBehavior.FLOW_ERROR, +} + +# The value a missing translation of "Spain" gives under each member, with +# the number of flow errors it adds. +MISSING_RESULTS = { + MissingTranslationBehavior.ORIGINAL: (["Allemagne", "Spain"], 0), + MissingTranslationBehavior.EMPTY_STRING: (["Allemagne", ""], 0), + MissingTranslationBehavior.FLOW_ERROR: (["Allemagne", None], 1), +} + + +class MissingTranslationBehaviorTests(unittest.TestCase): + + """ + The missing translation behavior given as a member or as text, as a + configuration file gives it, with anything else refused when the engine + is built, apart from None, which means not given. The option takes the + specification's name, missing_translation_behavior, and still takes the + old name, behavior, with a DeprecationWarning. + """ + + def _engine(self, behavior): + return TranslationEngineBase( + "source", + [("Names", "NamesTranslated")], + FR_SOURCE, + "fr_FR", + behavior) + + def _missing(self, pipeline): + """The translated names and the flow errors for a source holding a + name with a translation and one without. The pipeline keeps errors + on the flow data rather than raising them, so a flow error can be + counted.""" + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) + flowdata.process() + translated = flowdata.get("translation").get("NamesTranslated") + if isinstance(translated, AspectPropertyValue): + translated = translated.value() + return translated, len(flowdata.errors) + + def _pipeline(self, behavior): + return self._pipeline_with(self._engine(behavior)) + + @staticmethod + def _pipeline_with(engine): + source = MockSourceElement("source", { + "Names": AspectPropertyValue(value=["Germany", "Spain"]) + }) + return PipelineBuilder({"suppress_process_exceptions": True}).add( + source).add(engine).build() + + @staticmethod + def _engine_with(**options): + """An engine built with every option given by its name.""" + return TranslationEngineBase( + source_element_data_key="source", + translations=[("Names", "NamesTranslated")], + sources=FR_SOURCE, + fixed_language="fr_FR", + **options) + + def test_from_value_names_each_member(self): + for text, member in BEHAVIOR_TEXTS.items(): + with self.subTest(text): + self.assertIs( + member, MissingTranslationBehavior.from_value(text)) + + def test_from_value_returns_a_member_as_it_is(self): + for member in MissingTranslationBehavior: + with self.subTest(member.name): + self.assertIs( + member, MissingTranslationBehavior.from_value(member)) + + def test_each_text_takes_effect(self): + for text, member in BEHAVIOR_TEXTS.items(): + with self.subTest(text): + self.assertEqual( + MISSING_RESULTS[member], + self._missing(self._pipeline(text))) + + def test_each_member_takes_effect(self): + for member, expected in MISSING_RESULTS.items(): + with self.subTest(member.name): + self.assertEqual( + expected, self._missing(self._pipeline(member))) + + def test_unknown_value_is_refused_when_the_engine_is_built(self): + for value in UNKNOWN_BEHAVIORS: + with self.subTest(repr(value)): + with self.assertRaises(ValueError) as raised: + self._engine(value) + self.assertIn("EmptyString", str(raised.exception)) + + def test_translator_takes_text(self): + translator = Translator({"Germany": "Allemagne"}, "EmptyString") + self.assertEqual("", translator.translate("Spain")) + + def test_specification_name_takes_effect_without_a_warning(self): + for text, member in SPECIFICATION_NAMES.items(): + with self.subTest(text): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + engine = self._engine_with( + missing_translation_behavior=text) + self.assertEqual( + MISSING_RESULTS[member], + self._missing(self._pipeline_with(engine))) + + def test_old_name_still_takes_effect_with_a_warning(self): + for text, member in SPECIFICATION_NAMES.items(): + with self.subTest(text): + with self.assertWarns(DeprecationWarning) as warned: + engine = self._engine_with(behavior=text) + self.assertIn( + "missing_translation_behavior", str(warned.warning)) + self.assertEqual( + MISSING_RESULTS[member], + self._missing(self._pipeline_with(engine))) + + def test_specification_name_wins_when_both_are_given(self): + with self.assertWarns(DeprecationWarning): + engine = self._engine_with( + missing_translation_behavior="EmptyString", + behavior="FlowError") + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior.EMPTY_STRING], + self._missing(self._pipeline_with(engine))) + + def test_neither_name_given_is_original(self): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + engine = self._engine_with() + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior.ORIGINAL], + self._missing(self._pipeline_with(engine))) + + def test_none_given_by_either_name_is_original(self): + # None means not given, so it gives Original without a warning. + translations = {"Germany": "Allemagne"} + for given in ("by position", "missing_translation_behavior", + "behavior"): + with self.subTest(given): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + if given == "by position": + engine = self._engine(None) + translator = Translator(translations, None) + else: + engine = self._engine_with(**{given: None}) + translator = Translator(translations, **{given: None}) + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior.ORIGINAL], + self._missing(self._pipeline_with(engine))) + self.assertEqual("Spain", translator.translate("Spain")) + + def test_translator_takes_both_names(self): + translations = {"Germany": "Allemagne"} + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + translator = Translator( + translations, missing_translation_behavior="EmptyString") + self.assertEqual("", translator.translate("Spain")) + with self.assertWarns(DeprecationWarning): + translator = Translator(translations, behavior="EmptyString") + self.assertEqual("", translator.translate("Spain")) + with self.assertWarns(DeprecationWarning): + translator = Translator( + translations, + missing_translation_behavior="Original", + behavior="EmptyString") + self.assertEqual("Spain", translator.translate("Spain")) + + def test_configuration_text_takes_effect(self): + for text, member in (("EmptyString", "EMPTY_STRING"), + ("flow_error", "FLOW_ERROR"), + ("Original", "ORIGINAL")): + with self.subTest(text): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + pipeline = PipelineBuilder().build_from_configuration( + self._configuration(text)) + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior[member]], + self._missing(pipeline)) + + def test_configuration_with_an_unknown_value_is_refused(self): + with self.assertRaises(ValueError): + PipelineBuilder().build_from_configuration( + self._configuration("EmptyStrings")) + + def test_configuration_with_the_old_name_still_takes_effect(self): + with self.assertWarns(DeprecationWarning): + pipeline = PipelineBuilder().build_from_configuration( + self._configuration("EmptyString", "behavior")) + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior.EMPTY_STRING], + self._missing(pipeline)) + + @staticmethod + def _configuration(behavior, name="missing_translation_behavior"): + """A pipeline configuration as a configuration file gives one, so + every value in it is text, a list or a dictionary. Every option has + the specification's name, unless the behavior's name is given.""" + return { + "PipelineOptions": { + "suppress_process_exceptions": True, + "Elements": [ + { + "elementPath": __name__, + "elementName": "MockSourceElement", + "elementParameters": { + "datakey": "source", + "contents": {"Names": ["Germany", "Spain"]}, + }, + }, + { + "elementPath": ( + "fiftyone_pipeline_translation." + "translation_engine_base"), + "elementName": "TranslationEngineBase", + "elementParameters": { + "source_element_data_key": "source", + "translations": [["Names", "NamesTranslated"]], + "sources": FR_SOURCE, + "fixed_language": "fr_FR", + name: behavior, + }, + }, + ] + } + } + + if __name__ == "__main__": unittest.main() From 88be76bfa6ea414de6f2373235d8bce19e18479f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 14:45:51 +0000 Subject: [PATCH 23/68] Package text says what each package does in the words people search for (#68) Name the competitor brands people search for in the remaining package text. Text only, no code change. --- .../fiftyone_devicedetection_shared/setup.py | 15 ++++++++++++++- .../fiftyone_ipintelligence_shared/setup.py | 13 ++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_shared/setup.py b/device-detection/fiftyone_devicedetection_shared/setup.py index 07b89cdf..c57092a3 100644 --- a/device-detection/fiftyone_devicedetection_shared/setup.py +++ b/device-detection/fiftyone_devicedetection_shared/setup.py @@ -46,7 +46,20 @@ def read(file_name): "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=documentation", "Source": "https://github.com/51Degrees/51degrees-python", }, - description=("The 51Degrees Pipeline API is a generic web request intelligence and data processing solution with the ability to add a range of 51Degrees and/or custom plug ins (Engines). This project contains 51Degrees Device Detection engines that can be used with the Pipeline API."), + description=("Device Detection and User-Agent parser for Python that detects the device, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints. This package holds the code shared by the cloud and on-premise engines for the 51Degrees Pipeline API. An alternative to UAParser.js, DeviceAtlas, and WURFL."), + keywords=[ + "51degrees", + "device-detection", + "user-agent", + "user-agent-parser", + "client-hints", + "browser-detection", + "mobile-detection", + "crawler-detection", + "wurfl", + "deviceatlas", + "uaparser" + ], long_description=read("readme.md"), long_description_content_type='text/markdown', python_requires=">=3.10", diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/setup.py b/ip-intelligence/fiftyone_ipintelligence_shared/setup.py index 7f89e1d0..a4c4934c 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_shared/setup.py @@ -45,7 +45,18 @@ def read(file_name): "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=documentation", "Source": "https://github.com/51Degrees/51degrees-python", }, - description=("Shared code used by the cloud and on-premise variants of the 51Degrees IP Intelligence engines for the 51Degrees Pipeline API."), + description=("IP Intelligence and IP geolocation for Python that finds an IP address's likely location (country, region, town, and coordinates), connection type, and network. This package holds the code shared by the cloud and on-premise engines for the 51Degrees Pipeline API. An alternative to MaxMind and IPinfo."), + keywords=[ + "51degrees", + "ip-intelligence", + "ip-geolocation", + "geoip", + "ip-lookup", + "asn", + "ip-to-country", + "maxmind", + "ipinfo" + ], long_description=read("readme.md"), long_description_content_type='text/markdown', python_requires=">=3.10", From a474f045675a890896f1c7cfc328e2b438412b77 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 15:15:29 +0000 Subject: [PATCH 24/68] Check the callback URL as the specification's six rules say (#66) * Check the callback URL as the specification's six rules say Refuse an IPv6 zone identifier, an '@', '?' or '#' anywhere in the host value, and a name ending in a number unless it is an IPv4 address in dotted decimal form. The logged URL shows *** in place of everything in the host before its last '@'. * Test that a port which is present is not empty * Refuse a port that is present but empty A host such as https://example.com otherwise reads as the host https with an empty port. RFC 3986 section 3.2.3 allows an empty port, but section 6.2.3 treats it as equal to none, so the callback URL does not carry one. --- .../fiftyone_pipeline_core/callback_url.py | 79 ++++----- .../tests/test_callback_url.py | 151 +++++++++++++++--- .../tests/test_javascriptbuilder_variables.py | 45 ++++++ 3 files changed, 206 insertions(+), 69 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py index b18fbb13..15854691 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py @@ -25,9 +25,9 @@ and host usually come from the request, and the script holds the URL in a single-quoted string, so the URL is checked before it is written into the script. A URL that is not valid, including one whose protocol is not http or -https, is refused. In the user information, in anything after the host and -port, and in the endpoint, the characters a URL cannot hold or that could end -the string or the script are percent-encoded. +https, is refused. In anything after the authority, and in the endpoint, the +characters a URL cannot hold or that could end the string or the script are +percent-encoded. See https://github.com/51Degrees/specifications/blob/main/pipeline-specification/pipeline-elements/javascript-builder.md#callback-url """ @@ -67,19 +67,18 @@ _ASCII_LOWER = str.maketrans( "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") -# Where the authority ends, which is at the first '/', '?' or '#' of the -# host. -AUTHORITY_END_PATTERN = re.compile(r"[/?#]") +# The characters the host value must not hold. +FORBIDDEN_PATTERN = re.compile(r"[?#@]") # A label of a host name, being ASCII letters, digits, '-' and '_'. LABEL_PATTERN = re.compile(r"[A-Za-z0-9_-]+") -# An IPv6 zone identifier, being unreserved characters as RFC 3986 section -# 2.3 defines them. -ZONE_PATTERN = re.compile(r"[A-Za-z0-9._~-]+") +# A last label that makes the host a number, being only digits, or 0x or 0X +# and any number of hexadecimal digits. +NUMBER_LABEL_PATTERN = re.compile(r"[0-9]+|0[xX][0-9A-Fa-f]*") -# A port as it follows the host, being ':' and any number of digits. -PORT_PATTERN = re.compile(r":[0-9]*") +# A port as it follows the host, being ':' and at least one digit. +PORT_PATTERN = re.compile(r":[0-9]+") # One group of an IPv6 address. HEX_GROUP_PATTERN = re.compile(r"[0-9A-Fa-f]{1,4}") @@ -147,18 +146,19 @@ def callback_url(protocol, host, endpoint): Build the callback URL from its parts and check it. One slash separates the host and the endpoint, being added when neither has one and removed when both do. The protocol and the host are written in lower case and - the rest as given, apart from the percent-encoding. + the rest as given, apart from the percent-encoding. The host value holds + no '?', '#' or '@'. @type protocol: str @param protocol: The protocol, http or https in any case @type host: str - @param host: The host, which can start with user information and end - with a port, a path, a query or a fragment + @param host: The host, which can end with a port and a path @type endpoint: str @param endpoint: The endpoint @rtype: CallbackUrl @return: The callback URL, or None when the protocol, host or endpoint - is not set + is not set. The URL of an invalid one is as it was built, for the log, + with '***' in place of everything in the host before its last '@'. """ if is_blank(protocol) or is_blank(host) or is_blank(endpoint): @@ -169,12 +169,14 @@ def callback_url(protocol, host, endpoint): endpoint = "/" + endpoint elif host_has_slash and endpoint_has_slash: endpoint = endpoint[1:] - built = protocol + "://" + host + endpoint - if not is_scheme(protocol): + at = host.rfind("@") + built = (protocol + "://" + ("***" + host[at:] if at != -1 else host) + + endpoint) + if not is_scheme(protocol) or FORBIDDEN_PATTERN.search(host): return CallbackUrl(False, built) - match = AUTHORITY_END_PATTERN.search(host) - end = len(host) if match is None else match.start() - authority = _checked_authority(host[:end]) + end = host.find("/") + end = len(host) if end == -1 else end + authority = _checked_host_port(host[:end]) if authority is None: return CallbackUrl(False, built) return CallbackUrl( @@ -183,40 +185,21 @@ def callback_url(protocol, host, endpoint): + _encoded(host[end:]) + _encoded(endpoint)) -def _checked_authority(authority): - - """! - The user information, host and port as they are written into the URL, - or None when they are not valid. The user information ends at the first - '@', so a second '@' is part of the host and refused. - """ - - user, at, host_port = authority.partition("@") - if not at: - return _checked_host_port(authority) - host_port = _checked_host_port(host_port) - if host_port is None: - return None - return _encoded(user) + "@" + host_port - - def _checked_host_port(host_port): """! The host and optional port in lower case, or None when they are not - valid. The host is an IPv6 address in brackets with an optional zone - identifier, or a name of labels separated by dots with a dot allowed at - the end. The port is ':' followed by digits up to 65535, or by nothing. + valid. The host is an IPv6 address in brackets, or a name of labels + separated by dots with a dot allowed at the end, which is an IPv4 address + when its last label is a number. The port is ':' followed by digits up to + 65535. """ if host_port.startswith("["): close = host_port.find("]") if close == -1: return None - address, separator, zone = host_port[1:close].partition("%25") - if separator and ZONE_PATTERN.fullmatch(zone) is None: - return None - if not _is_ipv6(address): + if not _is_ipv6(host_port[1:close]): return None host = host_port[:close + 1] port = host_port[close + 1:] @@ -224,9 +207,13 @@ def _checked_host_port(host_port): host, colon, port = host_port.partition(":") port = colon + port name = host[:-1] if host.endswith(".") else host - for label in name.split("."): + labels = name.split(".") + for label in labels: if LABEL_PATTERN.fullmatch(label) is None: return None + if (NUMBER_LABEL_PATTERN.fullmatch(labels[-1]) + and IPV4_PATTERN.fullmatch(host) is None): + return None if port and not _is_port(port): return None return host.lower() + port @@ -236,7 +223,7 @@ def _is_port(text): """! Whether the text is ':' followed by digits whose value is no more than - 65535, or by nothing. + 65535. """ if PORT_PATTERN.fullmatch(text) is None: diff --git a/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py index 129fee0d..29e8be4d 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py @@ -69,21 +69,22 @@ def test_missing_when_a_part_is_empty(self): @parameterized.expand([ ("EXAMPLE.com", "https://example.com/51dpipeline/json"), ("example.com:8080", "https://example.com:8080/51dpipeline/json"), - ("example.com:", "https://example.com:/51dpipeline/json"), ("example.com:65535", "https://example.com:65535/51dpipeline/json"), ("a_b.com", "https://a_b.com/51dpipeline/json"), ("-ab.com", "https://-ab.com/51dpipeline/json"), ("127.0.0.1:80", "https://127.0.0.1:80/51dpipeline/json"), + ("1.2.3.4", "https://1.2.3.4/51dpipeline/json"), + ("1.2.3.4:8080", "https://1.2.3.4:8080/51dpipeline/json"), + ("1.2.3.4/path", "https://1.2.3.4/path/51dpipeline/json"), + ("0.0.0.0", "https://0.0.0.0/51dpipeline/json"), + ("101.2.3.4", "https://101.2.3.4/51dpipeline/json"), + ("example.co1", "https://example.co1/51dpipeline/json"), + ("123.example.com", "https://123.example.com/51dpipeline/json"), + ("0x7f.example.com", "https://0x7f.example.com/51dpipeline/json"), ("[::1]", "https://[::1]/51dpipeline/json"), ("[::1]:8080", "https://[::1]:8080/51dpipeline/json"), - ("[fe80::1%25eth0]", "https://[fe80::1%25eth0]/51dpipeline/json"), - ("user:pw@example.com", - "https://user:pw@example.com/51dpipeline/json"), - ("u x@example.com", "https://u%20x@example.com/51dpipeline/json"), - ("u\tx@example.com", "https://u%09x@example.com/51dpipeline/json"), ("example.com.", "https://example.com./51dpipeline/json"), ("example.com/path", "https://example.com/path/51dpipeline/json"), - ("example.com?x", "https://example.com?x/51dpipeline/json"), ("example.com/a b", "https://example.com/a%20b/51dpipeline/json"), ("example.com/a%zzb", "https://example.com/a%25zzb/51dpipeline/json"), @@ -110,7 +111,6 @@ def test_accepts_the_host(self, host, expected): ("[::1",), ("[v1.x]",), ("[1.2.3.4]",), - ("a@b@example.com",), ("a..b.com",), ("%41.com",), ("a\\b.com",), @@ -155,11 +155,6 @@ def test_refuses_the_zone_or_the_text_after_the_bracket(self, host): self.assertFalse(_url("https", host).valid, repr(host)) - def test_accepts_a_zone_followed_by_a_port(self): - self.assertEqual( - _valid("https://[fe80::1%25eth0]:8080/51dpipeline/json"), - _url("https", "[fe80::1%25eth0]:8080")) - def test_checks_the_port_by_its_value(self): """A port goes by its value, so leading zeros are allowed, and a port too long to read is refused rather than read as small enough.""" @@ -178,15 +173,14 @@ def test_checks_the_port_by_its_value(self): self.assertFalse(_url("https", host).valid, host) def test_encodes_the_characters_rule_8_lists(self): - """Each character rule 8 lists, in the user information, after the - host and in the endpoint, and a '%' without two hexadecimal digits - after it, even at the end.""" + """Each character rule 8 lists, after the host and in the endpoint, + and a '%' without two hexadecimal digits after it, even at the + end.""" self.assertEqual( - _valid("https://u%22%5E%60%7B%7C%7D@example.com/%22%5E%60%7B%7C%7D" + _valid("https://example.com/%22%5E%60%7B%7C%7D" "/j%22%5E%60%7B%7C%7D"), - callback_url( - "https", 'u"^`{|}@example.com/"^`{|}', '/j"^`{|}')) + callback_url("https", 'example.com/"^`{|}', '/j"^`{|}')) self.assertEqual( _valid("https://example.com/a%25/51dpipeline/json"), _url("https", "example.com/a%")) @@ -200,6 +194,120 @@ def test_encodes_the_characters_rule_8_lists(self): _valid("https://example.com/json%25"), callback_url("https", "example.com", "/json%")) + @parameterized.expand([ + ("[fe80::1%25eth0]",), + ("[fe80::1%25eth0]:8080",), + ("[FE80::1%25ETH0]",), + ("[fe80::1%251]",), + ("[::1%25lo]",), + ("[fe80::1%]",), + ("[fe80::1%25eth0]/path",), + ]) + def test_refuses_the_zone_identifier(self, host): + """Rule 4 refuses an IPv6 zone identifier.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("user@example.com",), + ("user:pw@example.com",), + ("@example.com",), + (":@example.com",), + ("user@example.com:8080",), + ("user@[::1]",), + ("user@example.com/path",), + ("u%41@example.com",), + ("u x@example.com",), + ("u\tx@example.com",), + ("u\"x@example.com",), + ("u'x@example.com",), + ("u\\x@example.com",), + ("u\nx@example.com",), + ("u\u0001x@example.com",), + ("a@b@example.com",), + ("example.com/a@b",), + ("exa mple.com/a@b",), + ("user:12/ss@example.com",), + ("user:pa/ss@example.com",), + ("user:pa?ss@example.com",), + ]) + def test_refuses_the_user_information(self, host): + """Rule 3 refuses an '@' anywhere in the host value.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("example.com?x",), + ("example.com#x",), + ("example.com?",), + ("example.com#",), + ("example.com:8080?x",), + ("[::1]#x",), + ("example.com/path?x",), + ("example.com/path#x",), + ("example.com?');alert(1);//",), + ("example.com#');alert(1);//",), + ]) + def test_refuses_the_query_or_fragment(self, host): + """Rule 2 refuses a '?' or '#' anywhere in the host value.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("example.123",), + ("example.0x1f",), + ("example.0X",), + ("1.2.3.999",), + ("1.2.3",), + ("127.1",), + ("0x7f.0.0.1",), + ("01.2.3.4",), + ("00.2.3.4",), + ("example.123.",), + ("1.2.3.4.",), + ("1.2.3.4.5",), + ("1",), + ("300.1.1.1",), + ("0x7f.1",), + ("1.2.3.4.:8080",), + ]) + def test_refuses_a_name_ending_in_a_number(self, host): + """Rule 4 reads a name whose last label is a number as an IPv4 + address written as four decimal numbers, each starting with 0 only + when it is 0 alone.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("example.com:",), + ("example.com:/path",), + ("[::1]:",), + ("https://example.com",), + ("http://example.com",), + ]) + def test_refuses_the_empty_port(self, host): + """Rule 5 refuses a port that is present but empty, so a protocol + written into the host gives the host "https" with an empty port.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("user:secret@example.com", + "https://***@example.com/51dpipeline/json"), + ("a@b@example.com", "https://***@example.com/51dpipeline/json"), + ("u@[::1]:80x", "https://***@[::1]:80x/51dpipeline/json"), + ("user:pa/ss@example.com", + "https://***@example.com/51dpipeline/json"), + ("user:pa?ss@example.com", + "https://***@example.com/51dpipeline/json"), + ("exa mple.com/a@b", "https://***@b/51dpipeline/json"), + ]) + def test_the_logged_url_holds_no_user_information(self, host, logged): + """Rule 6 shows *** in place of everything in the host before its + last '@', wherever in the host value that '@' is.""" + + self.assertEqual(CallbackUrl(False, logged), _url("https", host)) + def test_the_invalid_url_is_the_one_built(self): self.assertEqual( CallbackUrl(False, "https://exa mple.com/51dpipeline/json"), @@ -224,10 +332,7 @@ def test_checks_the_scheme(self): @parameterized.expand([ ("example.com/');alert(1);//",), - ("example.com?');alert(1);//",), - ("example.com#');alert(1);//",), - ("u'x@example.com",), - ("u\\x@example.com",), + ('example.com/a"b',), ("example.com/a\\b",), ("example.com/",), ]) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py index 7b601e61..9ac0e771 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py @@ -202,6 +202,51 @@ def test_refused_url_is_left_out_and_logged_once(self): self.assertEqual(1, len(errors), errors) self.assertIn("'https://exa mple.com/json'", errors[0]) + @parameterized.expand([ + ("user:secret@example.com", "'https://***@example.com/json'"), + ("a@b@example.com", "'https://***@example.com/json'"), + ("u@[::1]:80x", "'https://***@[::1]:80x/json'"), + ("user:pa/ss@example.com", "'https://***@example.com/json'"), + ("user:pa?ss@example.com", "'https://***@example.com/json'"), + ("exa mple.com/a@b", "'https://***@b/json'"), + ]) + def test_refused_host_is_logged_without_its_user_information( + self, host, logged): + """The log never holds what comes before the last '@' of the host + value. Each host has an element of its own because a refusal is + logged once per element.""" + + rendered, errors = _render_with_hosts([host]) + + self.assertNotIn("_url", rendered[0]) + self.assertEqual(1, len(errors), errors) + self.assertIn(logged, errors[0]) + self.assertNotIn("secret", errors[0]) + self.assertNotIn("'pa", errors[0]) + self.assertNotIn("pa/", errors[0]) + + def test_refused_configured_host_is_logged_without_user_information( + self): + """A host from the settings is refused and logged the same way.""" + + logger = MemoryLogger("error") + pipeline = PipelineBuilder({"javascript_builder_settings": { + "minify": False, "endpoint": "/json", + "host": "user:secret@example.com"}}).add_logger( + logger).build() + captured = _CapturedAll() + with mock.patch.object( + javascriptbuilder.chevron, "render", captured.render): + pipeline.create_flowdata().process() + errors = [ + entry["message"] for entry in logger.memory_log + if entry["level"] == "error"] + + self.assertNotIn("_url", captured.variables[0]) + self.assertEqual(1, len(errors), errors) + self.assertIn("'https://***@example.com/json'", errors[0]) + self.assertNotIn("secret", errors[0]) + def test_refused_url_is_logged_on_one_line(self): """The URL is quoted and escaped in the log, so a line break in it cannot start a new log line.""" From 6f4a187effed242bb0f037af95089e8f5d12efbb Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:06:18 +0000 Subject: [PATCH 25/68] Test that a licence key is masked in logs and errors --- .../tests/test_licence_key_masking.py | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 pipeline/fiftyone_pipeline_engines/tests/test_licence_key_masking.py diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_licence_key_masking.py b/pipeline/fiftyone_pipeline_engines/tests/test_licence_key_masking.py new file mode 100644 index 00000000..535ac41c --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines/tests/test_licence_key_masking.py @@ -0,0 +1,128 @@ + +""" +A licence key is shown as its last 4 characters, each hidden character +replaced by an asterisk, wherever a key or a URL that carries one is logged or +raised. A key of 4 or fewer characters is replaced entirely. +""" + +import unittest +from unittest import mock + +import requests + +from fiftyone_pipeline_core.keymask import mask_licence_key +from fiftyone_pipeline_core.keymask import mask_licence_keys_in_text +from fiftyone_pipeline_engines.datafile_update_service import DataFileUpdateService + +from .test_datafile_update_service import RecordingPipeline + +KEY = "ABCDEFGHIJKL" +URL = "https://example.com/update?Type=Hash&licenseKeys=" + KEY + "&Product=V4" +MASKED_URL = ( + "https://example.com/update?Type=Hash&licenseKeys=********IJKL" + "&Product=V4") + + +class MaskTests(unittest.TestCase): + + def test_last_four_characters_are_shown(self): + self.assertEqual("********IJKL", mask_licence_key(KEY)) + self.assertEqual("*2345", mask_licence_key("12345")) + + def test_four_or_fewer_characters_are_replaced_entirely(self): + self.assertEqual("****", mask_licence_key("abcd")) + self.assertEqual("***", mask_licence_key("abc")) + self.assertEqual("*", mask_licence_key("a")) + self.assertEqual("", mask_licence_key("")) + + def test_a_key_in_a_url_is_masked(self): + self.assertEqual(MASKED_URL, mask_licence_keys_in_text(URL)) + + def test_every_key_of_a_list_is_masked(self): + for separator in ("|", "%7C", "%7c"): + text = "x?licenseKeys=AAAAAAAA" + separator + "BBBBB&a=b" + self.assertEqual( + "x?licenseKeys=****AAAA" + separator + "*BBBB&a=b", + mask_licence_keys_in_text(text), separator) + + def test_other_spellings_of_the_parameter_are_masked(self): + for name in ("licenseKey", "LicenceKeys", "licence", "license_keys"): + self.assertEqual( + "u?" + name + "=********IJKL", + mask_licence_keys_in_text("u?" + name + "=" + KEY), name) + + def test_text_without_a_key_is_unchanged(self): + text = "https://example.com/update?Type=Hash&Product=V4" + self.assertEqual(text, mask_licence_keys_in_text(text)) + + +class StandInDataFile: + + def __init__(self): + self.identifier = "id" + self.update_on_start = False + self.auto_update = False + self.file_system_watcher = False + self.updating = False + self.verify_if_modified_since = False + self.verify_md5 = False + self.decompress = False + self.download = False + self.create_temp_copy = False + self.data = b"old" + + class Engine: + datakey = "device" + self.flow_element = Engine() + + def get_update_url(self): + return URL + + def refresh(self): + pass + + +def answer(status): + reply = mock.Mock() + reply.status_code = status + reply.content = b"x" + reply.headers = {} + return reply + + +class UpdateServiceLogTests(unittest.TestCase): + + def update(self, reply): + pipeline = RecordingPipeline() + service = DataFileUpdateService(pipeline) + with mock.patch.object(requests, "get", lambda *a, **k: reply): + service.update_data_file(StandInDataFile()) + return pipeline.logs + + def test_no_message_holds_the_key(self): + for status in (304, 403, 429, 500): + logs = self.update(answer(status)) + self.assertTrue(logs, status) + for level, message in logs: + self.assertNotIn(KEY, message, status) + self.assertNotIn("ABCDEFGH", message, status) + + def test_the_masked_url_is_in_the_message(self): + logs = self.update(answer(403)) + self.assertTrue(any(MASKED_URL in message for level, message in logs), + logs) + + def test_an_error_raised_by_the_request_does_not_hold_the_key(self): + def fail(*args, **kwargs): + raise requests.ConnectionError("failed for url: " + URL) + + service = DataFileUpdateService(RecordingPipeline()) + with mock.patch.object(requests, "get", fail): + with self.assertRaises(requests.RequestException) as caught: + service.update_data_file(StandInDataFile()) + self.assertNotIn(KEY, str(caught.exception)) + self.assertIn(MASKED_URL, str(caught.exception)) + + +if __name__ == "__main__": + unittest.main() From b919b6b9441a0678505647bd52417f5240cfe41b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:07:37 +0000 Subject: [PATCH 26/68] Show only the last four characters of a licence key in logs and errors Each hidden character is an asterisk, and a key of four or fewer characters is hidden entirely. The update service masks the URL it logs and the message of a failed request, and the Cloud Request Engine masks the URL in its no-data message. --- .../cloudrequestengine.py | 4 +- .../src/fiftyone_pipeline_core/keymask.py | 71 +++++++++++++++++++ .../datafile_update_service.py | 20 ++++-- 3 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/keymask.py diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py index 42cd6be8..6a6ed7d0 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py @@ -30,6 +30,7 @@ import requests from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.keymask import mask_licence_keys_in_text from fiftyone_pipeline_engines.engine import Engine from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary @@ -354,7 +355,8 @@ def validate_response( # in the response then add an explanation to the list of # messages. if not messages and not has_data: - message = Constants.MESSAGE_NO_DATA_IN_RESPONSE.format(cloud_response.url) + message = Constants.MESSAGE_NO_DATA_IN_RESPONSE.format( + mask_licence_keys_in_text(cloud_response.url)) messages.append(message) # If there were no errors returned but the response code was non diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/keymask.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/keymask.py new file mode 100644 index 00000000..b3d32f2a --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/keymask.py @@ -0,0 +1,71 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import re + +# How many characters at the end of a key are shown. +VISIBLE_CHARACTERS = 4 + +# A licence key in a URL query string or in text that holds one, whatever the +# parameter is called (licenseKeys, licenceKey, license_keys and so on). +_LICENCE_PARAMETER = re.compile( + r"(licen[cs]e(?:_?keys?)?=)([^&\s'\"#]*)", re.IGNORECASE) + +# The separator of a list of keys, written as it is or percent encoded. +_KEY_SEPARATOR = re.compile(r"(\||%7C)", re.IGNORECASE) + + +def mask_licence_key(key): + """! + A licence key with each character hidden by an asterisk, except the last + four. A key of four or fewer characters is hidden entirely. + @type key: str + @param key: The licence key + @rtype: str + @return: The masked key, of the same length + """ + + key = str(key) + if len(key) <= VISIBLE_CHARACTERS: + return "*" * len(key) + return "*" * (len(key) - VISIBLE_CHARACTERS) + key[-VISIBLE_CHARACTERS:] + + +def _mask_parameter(match): + parts = _KEY_SEPARATOR.split(match.group(2)) + masked = [mask_licence_key(part) if index % 2 == 0 else part + for index, part in enumerate(parts)] + return match.group(1) + "".join(masked) + + +def mask_licence_keys_in_text(text): + """! + Text, such as a URL or an error message, with every licence key in a + licence parameter masked as :func:`mask_licence_key` does. A list of keys + separated by a vertical bar is masked key by key. + @type text: str + @param text: The text that may hold a licence key + @rtype: str + @return: The text with the keys masked + """ + + return _LICENCE_PARAMETER.sub(_mask_parameter, str(text)) diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py index c4c6b047..7ded5ad9 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py @@ -28,6 +28,7 @@ import threading import time import requests +from fiftyone_pipeline_core.keymask import mask_licence_keys_in_text # gzip decompress alternative for python 2 from io import StringIO @@ -102,6 +103,8 @@ def update_data_file(self, data_file): # Get the update url url = data_file.get_update_url() + # The URL carries the licence key, so only a masked copy is logged. + shown_url = mask_licence_keys_in_text(url) headers = {} @@ -114,23 +117,28 @@ def update_data_file(self, data_file): except: pass - response = requests.get(url, headers=headers) + try: + response = requests.get(url, headers=headers) + except requests.RequestException as e: + # The message of a failed request names the URL, key included. + raise requests.RequestException( + mask_licence_keys_in_text(e)) from None if response.status_code != 200: # Handle errors if response.status_code == 429: - self.pipeline.log("error", "Too many requests to " + url + "for engine " + data_file.flow_element.datakey) + self.pipeline.log("error", "Too many requests to " + shown_url + "for engine " + data_file.flow_element.datakey) status = UpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS elif response.status_code == 304: - self.pipeline.log("warn", "No data update available from " + url + "for engine " + data_file.flow_element.datakey) + self.pipeline.log("warn", "No data update available from " + shown_url + "for engine " + data_file.flow_element.datakey) status = UpdateStatus.AUTO_UPDATE_NOT_NEEDED elif response.status_code == 403: - self.pipeline.log("error", "Access denied from " + url + "for engine " + data_file.flow_element.datakey) + self.pipeline.log("error", "Access denied from " + shown_url + "for engine " + data_file.flow_element.datakey) status = UpdateStatus.AUTO_UPDATE_ERR_403_FORBIDDEN else: - self.pipeline.log("error", str(response.status_code) + url + "for engine " + data_file.flow_element.datakey) + self.pipeline.log("error", str(response.status_code) + shown_url + "for engine " + data_file.flow_element.datakey) status = UpdateStatus.AUTO_UPDATE_HTTPS_ERR data_file.updating = False @@ -150,7 +158,7 @@ def update_data_file(self, data_file): hash_md5.update(data) if header_md5 != hash_md5.hexdigest(): - self.pipeline.log("error", "MD5 doesn't match from " + url + "for engine " + data_file.flow_element.datakey) + self.pipeline.log("error", "MD5 doesn't match from " + shown_url + "for engine " + data_file.flow_element.datakey) status = UpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED data_file.updating = False self.__emit(status, data_file) From 147ed13c37790484552c370c448c1a526a582aca Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:20:20 +0000 Subject: [PATCH 27/68] Refuse a 51Did text with white space inside it, or both base64 alphabets (#56) * Refuse a 51Did text with white space inside it, or both base64 alphabets The 51Did specification says a reader ignores white space before and after the text, being any character with the Unicode White_Space property, refuses any other character outside the alphabet, white space inside the text included, and refuses a text holding a character from each alphabet, checking that before it maps "-" and "_" onto "+" and "/". FodId.to_standard_base64 now removes exactly the White_Space characters around the text, and answers None for a text holding any other character outside the two alphabets and the padding, or holding a character from each alphabet. The base64 readers report InvalidBase64 for such a text. White space alone still reports MissingInput. Before this change the reader read a mixed text as if it were clean, and read a text with U+001C to U+001F around it, because str.strip() removes those characters although they are not White_Space. White space inside the text was already refused by the OWID decoder. The client's length check and to_base64_url trim the same characters. * Say the alphabet check is made separately in the fod_id comment --- pipeline/fiftyone_pipeline_did/readme.md | 15 +- .../src/fiftyone_pipeline_did/did_client.py | 10 +- .../src/fiftyone_pipeline_did/fod_id.py | 99 +++++++++-- .../fiftyone_pipeline_did/tests/test_fodid.py | 162 ++++++++++++++++++ 4 files changed, 258 insertions(+), 28 deletions(-) diff --git a/pipeline/fiftyone_pipeline_did/readme.md b/pipeline/fiftyone_pipeline_did/readme.md index fe51f236..3c2f369c 100644 --- a/pipeline/fiftyone_pipeline_did/readme.md +++ b/pipeline/fiftyone_pipeline_did/readme.md @@ -112,11 +112,12 @@ url_safe = fod_id.as_base64_url() # URL-safe alphabet, no padding, for a link ``` `from_base64` accepts the standard alphabet the cloud issues and the -URL-safe alphabet a page puts in a link, with or without padding. On an -identifier carrying a creator context the License Id field holds an -encrypted value that only 51Degrees can turn back into a licence -identifier, so `license_id` is the field's raw value and identifies -nothing outside 51Degrees. +URL-safe alphabet a page puts in a link, with or without padding. White +space around the value is ignored. A value with white space inside it, or +with characters from both alphabets, is refused. On an identifier carrying +a creator context the License Id field holds an encrypted value that only +51Degrees can turn back into a licence identifier, so `license_id` is the +field's raw value and identifies nothing outside 51Degrees. ### The usage an identifier was created for @@ -267,9 +268,9 @@ same whichever language parsed the bytes. | Status | Meaning | | --- | --- | | `PARSED` | A structurally valid 51Did. The signature has not been checked | -| `MISSING_INPUT` | `None`, an empty string or an empty buffer | +| `MISSING_INPUT` | `None`, an empty string, a string of white space alone or an empty buffer | | `INVALID_INPUT_TYPE` | Not a string (base64 reader) or not a bytes-like object (byte reader) | -| `INVALID_BASE64` | The text is not base64 in either alphabet | +| `INVALID_BASE64` | The text is not base64 in one alphabet, so there are no bytes. White space inside the text, and characters from both alphabets, report this | | `UNSUPPORTED_VERSION` | The first byte names an envelope version this package does not know | | `UNEXPECTED_END` | The data stopped in the middle of an envelope field | | `INVALID_DOMAIN_ENCODING` | The creator domain is not terminated or is longer than the OWID maximum | diff --git a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py index 6297b286..1a28e810 100644 --- a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py +++ b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py @@ -65,7 +65,7 @@ from ._owid import Version from ._layout import GUID_LENGTH, HEADER_LENGTH, MATCH_KEY_LENGTH -from .fod_id import DATE_EPOCH, FodId, _date_minutes +from .fod_id import DATE_EPOCH, FodId, _date_minutes, _trim_white_space from .id_type import IdType #: The public cloud API base, used when neither the ``endpoint`` argument @@ -934,10 +934,10 @@ def _ensure_parses(value: str) -> None: def _ensure_encoded_size(value: str) -> None: """Refuses obviously malformed text before it is parsed, a key is - fetched or the cloud is called. Surrounding whitespace is stripped - before measuring, as the reader strips it before decoding, so the two - measure the same characters.""" - if len(value.strip()) > _MAXIMUM_ENCODED_LENGTH: + fetched or the cloud is called. White space around the value is + removed before measuring, as the reader removes it before decoding, + so the two measure the same characters.""" + if len(_trim_white_space(value)) > _MAXIMUM_ENCODED_LENGTH: raise ValueError( "The identifier is far longer than any 51Did the cloud " "issues.") diff --git a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py index a71e388f..0953b187 100644 --- a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py +++ b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py @@ -22,6 +22,7 @@ from __future__ import annotations +import re from datetime import datetime, timezone from enum import Enum from typing import NamedTuple, Optional, Tuple @@ -82,7 +83,9 @@ class FodIdParseStatus(Enum): MISSING_INPUT = "MissingInput" #: The input was supplied in a form this surface cannot read. INVALID_INPUT_TYPE = "InvalidInputType" - #: The string is not valid base 64, so there are no bytes to read. + #: The string is not valid base 64 in one alphabet, so there are no + #: bytes to read. White space inside the string, and characters from + #: both alphabets, report this. INVALID_BASE64 = "InvalidBase64" #: The first byte names an envelope version this package does not know. UNSUPPORTED_VERSION = "UnsupportedVersion" @@ -303,11 +306,17 @@ def try_from_base64(cls, value) -> FodIdParseResult: The cloud issues a 51Did in the standard alphabet with padding, and a page puts one in a link in the URL-safe alphabet (``-`` and ``_``) without padding. Both are accepted, with or without padding, by - normalising to the standard form before the envelope is read. - - The value may be anything at all, as external data is. ``None`` and - the empty string report :attr:`FodIdParseStatus.MISSING_INPUT`, - anything other than a string reports + normalising to the standard form before the envelope is read. White + space around the value is ignored. A value with white space or any + other character outside the alphabet inside it, or with characters + from both alphabets, reports + :attr:`FodIdParseStatus.INVALID_BASE64`, as + :meth:`to_standard_base64` explains. + + The value may be anything at all, as external data is. ``None``, the + empty string and a string of white space alone report + :attr:`FodIdParseStatus.MISSING_INPUT`, anything other than a string + reports :attr:`FodIdParseStatus.INVALID_INPUT_TYPE`, and every other failure names its reason. The signature is not checked. """ @@ -345,15 +354,33 @@ def from_base64(cls, base64: str) -> "FodId": return cls._from_read_or_raise(_read_base64(base64), "base64") @staticmethod - def to_standard_base64(value: str) -> str: + def to_standard_base64(value: str) -> Optional[str]: """Restores a base64 string in either alphabet to the standard alphabet with padding, which is the form the OWID library decodes. - ``-`` becomes ``+`` and ``_`` becomes ``/``, then ``==`` is added - when the length modulo 4 is 2 and ``=`` when it is 3. A value already - in the standard padded form is returned unchanged. + White space before and after the text is removed first, being every + character with the Unicode White_Space property, so a value read + from a file or a header with a trailing line break reads as the + clean value does. Then ``-`` becomes ``+`` and ``_`` becomes ``/``, + and ``==`` is added when the length modulo 4 is 2 and ``=`` when it + is 3. A value already in the standard padded form is returned + unchanged apart from that trim. + + Answers ``None`` for a text the 51Did specification says to refuse, + being one with white space or any other character outside the + alphabet left inside it, or one holding a character from each + alphabet, such as both ``+`` and ``-``. The issuer writes neither, + so either is a sign of a damaged value, such as a ``+`` a form + decoder turned into a space or a value converted only in part. """ - cleaned = value.strip().replace("-", "+").replace("_", "/") + text = _trim_white_space(value) + if _BASE64_CHARACTERS.fullmatch(text) is None: + return None + # Checked before the mapping below, which would hide a mix. + if (_has_any(text, _STANDARD_ONLY) and + _has_any(text, _URL_SAFE_ONLY)): + return None + cleaned = text.replace("-", "+").replace("_", "/") remainder = len(cleaned) % 4 if remainder == 2: cleaned += "==" @@ -365,9 +392,12 @@ def to_standard_base64(value: str) -> str: def to_base64_url(value: str) -> str: """Converts a base64 string in either alphabet to the URL-safe alphabet without padding, the inverse of :meth:`to_standard_base64`, - so a 51Did can be placed in a URL without further encoding. + so a 51Did can be placed in a URL without further encoding. White + space around the value is removed as :meth:`to_standard_base64` + removes it. """ - return value.strip().replace("+", "-").replace("/", "_").rstrip("=") + return _trim_white_space(value).replace("+", "-").replace( + "/", "_").rstrip("=") @classmethod def from_byte_array(cls, buffer: bytes) -> "FodId": @@ -546,12 +576,49 @@ def signature_status(self, public_pem: str) -> SignatureStatus: return self._owid.signature_status(public_pem) +#: Every character with the Unicode White_Space property, which is what the +#: 51Did specification means by white space. Written out because +#: ``str.strip()`` and ``str.isspace()`` also take U+001C to U+001F, which +#: do not have that property and so are outside the alphabet. +_WHITE_SPACE = ( + "\t\n\x0b\x0c\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005" + "\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000") + +#: The characters a 51Did text may hold once the white space around it is +#: removed, being both alphabets and the padding. Which alphabet the text +#: uses is checked separately, and where the padding sits is left to the OWID +#: reader's decoder. +_BASE64_CHARACTERS = re.compile(r"[A-Za-z0-9+/_=-]*") + +#: The characters for the values 62 and 63 that only the standard alphabet +#: uses. +_STANDARD_ONLY = "+/" + +#: The characters for the values 62 and 63 that only the URL-safe alphabet +#: uses. +_URL_SAFE_ONLY = "-_" + + +def _trim_white_space(value: str) -> str: + """The value without the white space before and after it.""" + return value.strip(_WHITE_SPACE) + + +def _has_any(text: str, characters: str) -> bool: + """Whether the text holds any of the characters.""" + return any(character in text for character in characters) + + def _read_base64(value) -> ParseResult: - """The OWID read of a base64 string in either alphabet. Anything that is - not a string goes to the OWID reader as given, so the reason it reports - (nothing supplied, or a type it cannot read) is the one carried.""" + """The OWID read of a base64 string in either alphabet. A string the + 51Did rules refuse reports invalid base 64, as the OWID reader does for + any other text it cannot decode. Anything that is not a string goes to + the OWID reader as given, so the reason it reports (nothing supplied, or + a type it cannot read) is the one carried.""" if isinstance(value, str): value = FodId.to_standard_base64(value) + if value is None: + return ParseResult(False, None, ParseStatus.INVALID_BASE64) return Owid.parse(value) diff --git a/pipeline/fiftyone_pipeline_did/tests/test_fodid.py b/pipeline/fiftyone_pipeline_did/tests/test_fodid.py index 3b6b445d..dad12a7a 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_fodid.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_fodid.py @@ -852,6 +852,168 @@ def test_raising_and_non_raising_readers_agree_on_success(self): self.assertEqual(raising.match_key, result.value.match_key) +#: Every character with the Unicode White_Space property, which is what the +#: 51Did specification means by white space. +WHITE_SPACE = ( + "\t\n\x0b\x0c\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005" + "\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000") + +#: Characters ``str.strip()`` removes or that look like white space, but +#: which do not have the White_Space property, so they are outside the +#: alphabet wherever they sit. U+001C to U+001F are the ones Python's own +#: ``str.isspace()`` takes, and U+FEFF and U+200B are zero width +#: characters that cannot be seen in the text. +NOT_WHITE_SPACE = "\x1c\x1d\x1e\x1f\ufeff\u200b" + + +def _forms(standard): + """The standard text, and the URL-safe text with and without its + padding.""" + url_safe = standard.replace("+", "-").replace("/", "_") + return (standard, url_safe, url_safe.rstrip("=")) + + +def _spaced(text, space, count): + """The text with the white space put in at a number of points spread + evenly through it.""" + spaced = text + for point in range(count, 0, -1): + at = len(text) * point // (count + 1) + spaced = spaced[:at] + space + spaced[at:] + return spaced + + +def _show(text): + """The text with each character outside printable ASCII written as its + code, so a failure message shows where it is.""" + return "".join( + c if " " < c <= "~" else "\\u{0:04X}".format(ord(c)) for c in text) + + +class FodIdBase64TextTests(unittest.TestCase): + """How the base64 readers treat the text around and inside a 51Did, as + the identifier layout page of the 51Did specification sets out. White + space before and after the text is ignored, any other character outside + the alphabet is refused, white space inside the text included, and a + text holding a character from each alphabet is refused.""" + + def setUp(self): + self.factory = FodIdTestFactory() + self.standard = self._standard_with_both_specials() + + def _standard_with_both_specials(self): + """The standard text of a real signed 51Did holding both ``+`` and + ``/``, so every form below holds both characters of its alphabet. + The signature differs on every signing, so a few tries find one.""" + for _ in range(100): + standard = self.factory.signed_owid_base64(canonical_payload()) + if "+" in standard and "/" in standard: + return standard + self.fail("No signed envelope contained both + and / in 100 tries.") + + def assert_reads_as_standard(self, text): + result = FodId.try_from_base64(text) + self.assertTrue(result.ok, "{0} for {1}".format( + result.status.value, _show(text))) + self.assertEqual(self.standard, result.value.as_base64()) + self.assertEqual( + self.standard, FodId.from_base64(text).as_base64()) + + def assert_invalid_base64(self, text): + """Both readers refuse the text as invalid base64, and the step + they share gives nothing for it.""" + result = FodId.try_from_base64(text) + self.assertFalse(result.ok, "Parsed {0}".format(_show(text))) + self.assertIsNone(result.value) + self.assertIs(FodIdParseStatus.INVALID_BASE64, result.status, + _show(text)) + with self.assertRaises(OwidError) as raised: + FodId.from_base64(text) + self.assertIn("InvalidBase64", str(raised.exception)) + self.assertIsNone(FodId.to_standard_base64(text), _show(text)) + + def test_the_text_holds_both_specials_in_every_form(self): + standard, url_safe, unpadded = _forms(self.standard) + self.assertTrue("+" in standard and "/" in standard) + self.assertTrue("-" in url_safe and "_" in url_safe) + self.assertTrue("-" in unpadded and "_" in unpadded) + self.assertNotIn("=", unpadded) + + def test_every_form_parses(self): + for text in _forms(self.standard): + with self.subTest(text=text): + self.assert_reads_as_standard(text) + + def test_white_space_around_the_text_parses(self): + for before, after in ((" ", "\n"), ("\t", "\r\n"), + ("\u00a0", "\u00a0"), ("\u3000", "\u3000")): + for text in _forms(self.standard): + with self.subTest(before=_show(before), text=text): + self.assert_reads_as_standard(before + text + after) + + def test_every_white_space_character_around_the_text_parses(self): + self.assertEqual(25, len(WHITE_SPACE)) + for space in WHITE_SPACE: + for text in _forms(self.standard): + with self.subTest(space=_show(space), text=text): + self.assert_reads_as_standard(space + text + space) + + def test_white_space_inside_the_text_is_invalid_base64(self): + for space, count in ((" ", 1), (" ", 2), (" ", 3), ("\t", 4), + ("\r\n", 2), ("\u00a0", 4), ("\u3000", 1)): + for text in _forms(self.standard): + with self.subTest(space=_show(space), count=count, + text=text): + self.assert_invalid_base64(_spaced(text, space, count)) + + def test_every_white_space_character_inside_the_text_is_refused(self): + for space in WHITE_SPACE: + for text in _forms(self.standard): + with self.subTest(space=_show(space), text=text): + self.assert_invalid_base64(_spaced(text, space, 1)) + + def test_characters_that_are_not_white_space_are_refused_anywhere(self): + # Python's own strip() removes U+001C to U+001F, so they are + # checked around the text as well as inside it. + for character in NOT_WHITE_SPACE: + for text in _forms(self.standard): + for spaced in (character + text, text + character, + _spaced(text, character, 1)): + with self.subTest(text=_show(spaced)): + self.assert_invalid_base64(spaced) + + def test_both_alphabets_is_invalid_base64(self): + url_safe = FodId.to_base64_url(self.standard) + for text, from_, to in ((self.standard, "+", "-"), + (self.standard, "/", "_"), + (url_safe, "-", "+"), + (url_safe, "_", "/")): + with self.subTest(from_=from_, to=to): + at = text.index(from_) + self.assert_invalid_base64(text[:at] + to + text[at + 1:]) + + def test_white_space_alone_is_still_missing_input(self): + for text in (" ", "\t\r\n", "\u00a0", "\u3000", WHITE_SPACE): + with self.subTest(text=_show(text)): + result = FodId.try_from_base64(text) + self.assertFalse(result.ok) + self.assertIsNone(result.value) + self.assertIs(FodIdParseStatus.MISSING_INPUT, result.status) + + def test_to_standard_base64_restores_alphabet_and_padding(self): + for value, expected in (("-_", "+/=="), ("+/==", "+/=="), + ("", ""), (" YQ\n", "YQ=="), + ("\u3000-_\u3000", "+/==")): + with self.subTest(value=_show(value)): + self.assertEqual(expected, FodId.to_standard_base64(value)) + + def test_to_standard_base64_gives_none_for_a_text_it_refuses(self): + for value in ("YW Jj", "YW\tJj", "YW\r\nJj", "YW\u00a0Jj", + "YWJj\x1c", "+_", "-/", "+-", "/_"): + with self.subTest(value=_show(value)): + self.assertIsNone(FodId.to_standard_base64(value)) + + class FodIdTermsTests(unittest.TestCase): """The Terms byte, which says which terms document the identifier was created under so that the terms travel with the identifier. The byte is From f55f0d9c67e28f435bdda2007a17f139e841cd8d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:20:38 +0000 Subject: [PATCH 28/68] Use the specification's defaults for the performance profile and Update Matched User Agent (#57) * Leave Update Matched User Agent on by default, as the native code sets it The Device Detection on-premise specification says the option is on by default, so the UserAgents property holds the matched characters of the evidence. The C library sets it on, but the Python engine took update_matched_useragent=False and always passed it on, so UserAgents was a list of empty strings unless a caller turned the option on. The engine now takes None and only sets the option when a value is given, so the native default applies, as the .NET builder does. The docstrings in the engine and both builders say the option is on by default, and the match metrics example's comment no longer says it has to be set. A new test builds a pipeline on the free Lite data file with the option left unset, set on and set off, and skips when that file is not there. * Use the Balanced performance profile by default, as the native code does The Device Detection on-premise specification gives Balanced as the default performance profile, and the C library's default configuration is the Balanced one, but the Python engine defaulted to LowMemory. The engine now defaults to "Balanced", which its profile switch already maps to the native Balanced configuration, and the docstrings in the engine and both builders say so. Balanced loads some collections into memory and caches others, where LowMemory reads everything from the data file, so a caller relying on the default uses more memory. Naming "LowMemory" keeps the previous behaviour. A new test compares the collection settings of the configuration an engine built at its defaults gives the native code with those of each profile, because the engine keeps no name for the profile in use. * Say which profiles the default profile test cannot tell apart MaxPerformance and LowMemory have the same collection settings, as do Balanced and BalancedTemp, so the test's comparison of collection settings cannot separate either pair. The test's docstring now says so. * Say what the matched User-Agent length limit does, as the C header does The max_matched_useragent_length docstrings in the engine and both builders said the option is ignored when update_matched_useragent is false, and named it max_matched_useragents_length. The C library's header (src/dd/config-dd.h) says the limit applies to a single header, a pseudo header and the matched User-Agent, that a longer value is ignored and detection moves on, and that the limit applies whether or not the matched User-Agent is recorded. The docstrings now say the same, give the native default of 500, and spell the option's name correctly. * Drop the cloud builder's false matched User-Agent settings The cloud builder's docstring listed max_matched_useragents_length, misspelled, as ignored when update_matched_useragent is false, which the C header contradicts. The cloud builder reads neither setting, so both entries are removed rather than corrected. The match metrics example comment now says the option is set to show where to turn it off, as the line sets it to True. --- .../devicedetection_pipelinebuilder.py | 22 +++- .../devicedetection_cloud_pipelinebuilder.py | 4 - .../onpremise/match_metrics.py | 5 +- .../devicedetection_onpremise.py | 31 +++-- ...vicedetection_onpremise_pipelinebuilder.py | 22 +++- .../tests/test_performance_profile_default.py | 115 ++++++++++++++++++ .../tests/test_update_matched_useragent.py | 89 ++++++++++++++ 7 files changed, 264 insertions(+), 24 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_performance_profile_default.py create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_update_matched_useragent.py diff --git a/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py b/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py index 13a1f624..1a735f87 100644 --- a/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py @@ -83,10 +83,20 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @param licence_keys: licencekeys to use for the data file update service @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type max_matched_useragents_length : int - @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: - @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent + @type max_matched_useragent_length: int + @param max_matched_useragent_length: Limit in bytes on the length of + a single header such as User-Agent, of a pseudo header including the + separators between its values, and of the matched User-Agent. A value + longer than the limit is ignored and detection moves on to the next + header. The limit applies whether or not update_matched_useragent is + on. Left unset, the native default of 500 applies. A larger limit + raises the memory each set of results holds. + @type update_matched_useragent: bool + @param update_matched_useragent: True if the detection should record + the matched characters from the target User-Agent, which the + UserAgents property returns. On by default, as the native code sets + it. Set it to False when UserAgents is not needed, which saves + recording the characters. @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @type difference: int @@ -94,7 +104,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type allow_unmatched : bool @param allow_unmatched: True if there should be at least one matched node in order for the results to be considered valid. By default, this is false @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py index 4f59cada..8380803d 100644 --- a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py @@ -81,10 +81,6 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @param licence_keys: licencekeys to use for the data file update service @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type max_matched_useragents_length : int - @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: - @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @type difference: int diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py index dd53ed68..8f364ac8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py @@ -90,8 +90,9 @@ def run(self, data_file, show_descs, logger, output): # https://51degrees.com/documentation/_device_detection__hash.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=run#DeviceDetection_Hash_DataSetProduction_Performance use_predictive_graph = True, use_performance_graph = False, - # We want to show the matching evidence characters as part of this example, so we have to set - # this flag to true. + # This example shows the matched evidence characters, which + # needs this option on. It is on by default, and is set here to + # show where to turn it off. update_matched_useragent = True).add_logger(logger).build() # The pipeline is closed at the end of the with block, which diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index 632751ae..5f3d168c 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -61,8 +61,8 @@ def __init__( polling_interval=30, update_time_maximum_randomisation=10, create_temp_data_copy=True, - update_matched_useragent=False, - performance_profile="LowMemory", + update_matched_useragent=None, + performance_profile="Balanced", reuse_temp_file=False, concurrency=multiprocessing.cpu_count(), update_on_start=False, @@ -90,10 +90,20 @@ def __init__( @param licence_keys: licencekeys to use for the data file update service @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type max_matched_useragents_length : int - @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: - @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent + @type max_matched_useragent_length: int + @param max_matched_useragent_length: Limit in bytes on the length of + a single header such as User-Agent, of a pseudo header including the + separators between its values, and of the matched User-Agent. A value + longer than the limit is ignored and detection moves on to the next + header. The limit applies whether or not update_matched_useragent is + on. Left unset, the native default of 500 applies. A larger limit + raises the memory each set of results holds. + @type update_matched_useragent: bool + @param update_matched_useragent: True if the detection should record + the matched characters from the target User-Agent, which the + UserAgents property returns. On by default, as the native code sets + it. Set it to False when UserAgents is not needed, which saves + recording the characters. @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @type difference: int @@ -101,7 +111,9 @@ def __init__( @type allow_unmatched : bool @param allow_unmatched: True if there should be at least one matched node in order for the results to be considered valid. By default, this is false @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int @@ -209,7 +221,10 @@ def __init__( config.setUseTempFile(True) config.setReuseTempFile(reuse_temp_file) - config.setUpdateMatchedUserAgent(update_matched_useragent) + + # Left unset, the native default applies, which is on. + if update_matched_useragent is not None: + config.setUpdateMatchedUserAgent(update_matched_useragent) if allow_unmatched == True or allow_unmatched == False: config.setAllowUnmatched(allow_unmatched) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py index 133b9f2e..28ce1b70 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py @@ -80,10 +80,20 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @param licence_keys: licencekeys to use for the data file update service @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type max_matched_useragents_length : int - @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: - @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent + @type max_matched_useragent_length: int + @param max_matched_useragent_length: Limit in bytes on the length of + a single header such as User-Agent, of a pseudo header including the + separators between its values, and of the matched User-Agent. A value + longer than the limit is ignored and detection moves on to the next + header. The limit applies whether or not update_matched_useragent is + on. Left unset, the native default of 500 applies. A larger limit + raises the memory each set of results holds. + @type update_matched_useragent: bool + @param update_matched_useragent: True if the detection should record + the matched characters from the target User-Agent, which the + UserAgents property returns. On by default, as the native code sets + it. Set it to False when UserAgents is not needed, which saves + recording the characters. @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @type difference: int @@ -91,7 +101,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type allow_unmatched : bool @param allow_unmatched: True if there should be at least one matched node in order for the results to be considered valid. By default, this is false @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_performance_profile_default.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_performance_profile_default.py new file mode 100644 index 00000000..a40f04de --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_performance_profile_default.py @@ -0,0 +1,115 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import os +import unittest + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule import ConfigHashSwig +from fiftyone_devicedetection_onpremise.devicedetection_onpremise import DeviceDetectionOnPremise +from fiftyone_devicedetection_onpremise.devicedetection_onpremise_pipelinebuilder import DeviceDetectionOnPremisePipelineBuilder + +# The free Lite data file, where cxx/data/get-data-files.ps1 and +# get-data-files.sh download it and where CI links it in. +DATA_FILE = os.path.normpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..", "..", "cxx", + "data", "device-detection", "51Degrees-LiteV4.1.hash")) + +# The collections whose settings the configuration exposes. +COLLECTIONS = ( + "Strings", "Properties", "Values", "Profiles", "Nodes", "ProfileOffsets") + +# Each profile name the engine accepts and the native setter it maps to. +PROFILES = { + "LowMemory": "setLowMemory", + "MaxPerformance": "setMaxPerformance", + "Balanced": "setBalanced", + "BalancedTemp": "setBalancedTemp", + "HighPerformance": "setHighPerformance", +} + + +def collection_settings(config): + """ + Whether each collection is loaded into memory and how many items it + caches, which are what a performance profile sets. + """ + settings = {} + for name in COLLECTIONS: + collection = getattr(config, "get" + name)() + settings[name] = (collection.getLoaded(), collection.getCapacity()) + return settings + + +def profile_settings(setter): + """The collection settings the native code gives a profile.""" + config = ConfigHashSwig() + getattr(config, setter)() + return collection_settings(config) + + +@unittest.skipUnless( + os.path.isfile(DATA_FILE), + "The free Lite data file is not at {0}. Download it with " + "cxx/data/get-data-files.ps1 or cxx/data/get-data-files.sh, after " + "'git submodule update --init --recursive'.".format(DATA_FILE)) +class PerformanceProfileDefaultTests(unittest.TestCase): + """ + The engine keeps no name for the profile in use, so these tests compare + the collection settings of the configuration the engine gives the + native code with the settings each profile has there. MaxPerformance + and LowMemory have the same collection settings, and Balanced and + BalancedTemp do too, so neither pair can be told apart this way. + """ + + def _engine_settings(self, **settings): + engine = DeviceDetectionOnPremise( + data_file_path=DATA_FILE, licence_keys="", **settings) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + self.assertIsNotNone(engine.engine) + return collection_settings(engine.config) + + def test_balanced_and_low_memory_differ(self): + self.assertNotEqual( + profile_settings("setBalanced"), profile_settings("setLowMemory")) + + def test_engine_defaults_use_balanced(self): + self.assertEqual( + profile_settings("setBalanced"), self._engine_settings()) + + def test_builder_defaults_use_balanced(self): + pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=DATA_FILE, usage_sharing=False, + licence_keys="").build() + self.addCleanup(pipeline.close) + engine = pipeline.get_element("device") + self.assertEqual( + profile_settings("setBalanced"), + collection_settings(engine.config)) + + def test_each_named_profile_is_used(self): + for name, setter in PROFILES.items(): + with self.subTest(name): + self.assertEqual( + profile_settings(setter), + self._engine_settings(performance_profile=name)) diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_update_matched_useragent.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_update_matched_useragent.py new file mode 100644 index 00000000..155053c5 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_update_matched_useragent.py @@ -0,0 +1,89 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import os +import unittest + +from fiftyone_devicedetection_onpremise.devicedetection_onpremise_pipelinebuilder import DeviceDetectionOnPremisePipelineBuilder + +# The free Lite data file, where cxx/data/get-data-files.ps1 and +# get-data-files.sh download it and where CI links it in. +DATA_FILE = os.path.normpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..", "..", "cxx", + "data", "device-detection", "51Degrees-LiteV4.1.hash")) + +MOBILE_UA = ( + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) " + "AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114") + +# The character the native code writes where the evidence was not matched. +UNMATCHED = "_" + + +@unittest.skipUnless( + os.path.isfile(DATA_FILE), + "The free Lite data file is not at {0}. Download it with " + "cxx/data/get-data-files.ps1 or cxx/data/get-data-files.sh, after " + "'git submodule update --init --recursive'.".format(DATA_FILE)) +class UpdateMatchedUserAgentTests(unittest.TestCase): + """ + The Update Matched User Agent option is on unless it is turned off, so + the UserAgents property holds the characters of the evidence that were + matched. + """ + + def _user_agents(self, **settings): + pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=DATA_FILE, usage_sharing=False, licence_keys="", + **settings).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) + flowdata.evidence.add("header.user-agent", MOBILE_UA) + flowdata.process() + return flowdata.device.useragents.value() + + def assert_matched_from_the_evidence(self, user_agents): + """ + Each matched User-Agent has the evidence's characters where they were + matched, and at least one character was matched. + """ + self.assertGreaterEqual(len(user_agents), 1) + matched = 0 + for user_agent in user_agents: + for index, character in enumerate(user_agent): + if character != UNMATCHED: + self.assertEqual(MOBILE_UA[index], character, user_agent) + matched += 1 + self.assertGreater(matched, 0, user_agents) + + def test_builder_defaults_populate_user_agents(self): + self.assert_matched_from_the_evidence(self._user_agents()) + + def test_turned_on_populates_user_agents(self): + self.assert_matched_from_the_evidence( + self._user_agents(update_matched_useragent=True)) + + def test_turned_off_leaves_user_agents_empty(self): + user_agents = self._user_agents(update_matched_useragent=False) + self.assertGreaterEqual(len(user_agents), 1) + self.assertEqual([""] * len(user_agents), user_agents) From 2b56529594c166ab85aeddfd675e48d2e39133b8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 10:35:44 +0000 Subject: [PATCH 29/68] Read a 51Did factor word the package does not list as a mismatch (#75) * Test that an unknown 51Did factor word is a mismatch * Read a 51Did factor word the package does not list as a mismatch * Test that a 51Did factor word is read without regard to case * Compare a 51Did factor word without regard to case, as the specification says --- .../src/fiftyone_pipeline_did/did_client.py | 13 +++++-- .../tests/test_did_client.py | 34 ++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py index 1a28e810..ff88994f 100644 --- a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py +++ b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py @@ -174,7 +174,7 @@ class FactorResult(str, Enum): """The outcome of one factor in a mismatch or a partly misconfigured result. The cloud reports ``null`` for a factor that was not compared, which is passed through as ``None``, and a word this package does not - list is read the same way.""" + list is read as :attr:`MISMATCH`.""" VERIFIED = "verified" MISMATCH = "mismatch" @@ -415,6 +415,12 @@ def _context_of(text: str) -> ContextResult: def _factor_of(value: Any) -> FactorValue: + # Nothing was compared. + if value is None: + return None + # The cloud writes lower case words, and the comparison ignores case. + if isinstance(value, str): + value = value.lower() if value == "verified": return FactorResult.VERIFIED if value == "mismatch": @@ -425,7 +431,10 @@ def _factor_of(value: Any) -> FactorValue: return FactorResult.MISCONFIGURED if value == "notrecorded": return FactorResult.NOT_RECORDED - return None + # A word this package does not list, such as one a newer service sends, + # is a mismatch, so it never reads as a factor that matched or one that + # could not be compared. + return FactorResult.MISMATCH class DidClient: diff --git a/pipeline/fiftyone_pipeline_did/tests/test_did_client.py b/pipeline/fiftyone_pipeline_did/tests/test_did_client.py index 09a0c721..518045b3 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_did_client.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_did_client.py @@ -1188,10 +1188,42 @@ def test_a_factor_the_creator_did_not_record_is_its_own_outcome(self): json.loads(REDEEMED_WITH_NOT_RECORDED_FACTORS)["factors"], result.to_dict()["factors"]) - def test_a_factor_value_this_package_does_not_list_reads_as_none(self): + # A word this package does not list is a mismatch, so a value a newer + # service sends never reads as a factor that matched or one that could + # not be compared. + def test_a_factor_word_this_package_does_not_list_is_a_mismatch(self): result = self.redeem(200, REDEEMED_WITH_AN_UNKNOWN_FACTOR_VALUE) + self.assertIs(FactorResult.MISMATCH, result.factors["transport"]) + self.assertEqual("mismatch", result.to_dict()["factors"]["transport"]) + + def test_a_factor_that_was_not_compared_is_still_none(self): + body = json.dumps({ + "signature": "verified", "context": "mismatch", + "factors": {"transport": None}}) + result = self.redeem(200, body) self.assertIsNone(result.factors["transport"]) + def test_each_listed_factor_word_is_read_only_from_its_own_word(self): + for word, expected in ( + ("verified", FactorResult.VERIFIED), + ("mismatch", FactorResult.MISMATCH), + ("misconfigured", FactorResult.MISCONFIGURED), + ("notrecorded", FactorResult.NOT_RECORDED), + # The words are compared without regard to case. + ("Verified", FactorResult.VERIFIED), + ("MISMATCH", FactorResult.MISMATCH), + ("Misconfigured", FactorResult.MISCONFIGURED), + ("NotRecorded", FactorResult.NOT_RECORDED), + ("", FactorResult.MISMATCH), + ("not recorded", FactorResult.MISMATCH), + (7, FactorResult.MISMATCH), + (True, FactorResult.MISMATCH)): + body = json.dumps({ + "signature": "verified", "context": "mismatch", + "factors": {"transport": word}}) + result = self.redeem(200, body) + self.assertIs(expected, result.factors["transport"], repr(word)) + def test_the_factor_outcomes_are_the_words_the_cloud_writes(self): self.assertEqual( ["verified", "mismatch", "misconfigured", "notrecorded"], From 3da13a680dc927df9f15b7aa799de2b5e7cfa0f7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 10:35:56 +0000 Subject: [PATCH 30/68] Accept only the evidence each element reads, and none by default (#63) * Test that the elements the builder adds accept only the evidence they read * Accept only the evidence the sequence element, the JavaScript builder, the JSON bundler and the set header element read * Keep the evidence the cloud request engine tests need with an element that accepts every key, and test what is no longer sent * Describe the reserved evidence test's element without a claim about the live service * Test that a cloud engine accepts no evidence and that a cloud pipeline sends only what the service lists * Accept no evidence in a cloud engine, as it reads only the cloud request engine's results * Test that an element which declares no evidence accepts none * Accept no evidence in an element that declares none, as the specification asks * Wrap the import and fix the blank lines in the default elements test --- .../cloudengine.py | 13 +- .../tests/classes/accepteverykeyelement.py | 44 ++++++ .../tests/test_cloudengine_evidence.py | 70 +++++++++ .../tests/test_cloudrequestengine.py | 6 +- .../tests/test_credentials.py | 31 +++- .../src/fiftyone_pipeline_core/flowelement.py | 8 +- .../javascriptbuilder.py | 6 +- .../src/fiftyone_pipeline_core/jsonbundler.py | 10 ++ .../fiftyone_pipeline_core/sequenceelement.py | 4 +- .../setheaderelement.py | 10 ++ .../tests/test_default_elements_accept.py | 139 ++++++++++++++++++ 11 files changed, 332 insertions(+), 9 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/classes/accepteverykeyelement.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_evidence.py create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_default_elements_accept.py diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py index 9f285f75..79bb5007 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py @@ -20,6 +20,8 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter from fiftyone_pipeline_engines.engine import Engine from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue @@ -42,7 +44,7 @@ class CloudEngine(Engine): It requires the 51Degrees cloudRequestEngine to be placed in a pipeline before it. It takes that raw JSON response and parses it to extract the device part. - It also uses this data to generate a list of properties and an evidence key filter + It also uses this data to generate a list of properties. """ @@ -52,6 +54,15 @@ def __init__(self): self.datakey = "CloudEngineBase" # This should be overriden + def get_evidence_key_filter(self): + + """! + A cloud engine reads only the cloud request engine's results, so it + accepts no evidence. + """ + + return BasicListEvidenceKeyFilter([]) + def on_registration(self, pipeline): """! diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/classes/accepteverykeyelement.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/classes/accepteverykeyelement.py new file mode 100644 index 00000000..c7adc06b --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/classes/accepteverykeyelement.py @@ -0,0 +1,44 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +from fiftyone_pipeline_core.evidence_keyfilter import EvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement + + +class AcceptEveryKeyElement(FlowElement): + """! + An element that accepts every evidence key and does nothing else. A + pipeline keeps only the evidence one of its elements accepts, and the + stand-in service accepts query.User-Agent alone, so a test of how the + cloud request engine treats other evidence adds this element to keep + that evidence in the flow data. + """ + + def __init__(self): + super(AcceptEveryKeyElement, self).__init__() + self.datakey = "accepteverykey" + + def get_evidence_key_filter(self): + return EvidenceKeyFilter() + + def process_internal(self, flowdata): + pass diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_evidence.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_evidence.py new file mode 100644 index 00000000..ba9f157a --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_evidence.py @@ -0,0 +1,70 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""A cloud engine reads only the cloud request engine's results, so it +accepts no evidence, and a cloud pipeline sends the service only the +evidence the service lists and the elements the builder adds read.""" + +from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine +from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import \ + CloudRequestEngine +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase + +LICENSE_KEY = "a-licence-key" +REQUESTED = ["device.ismobile", "device.istablet"] + + +class CloudEngineEvidenceTests(CloudRequestEngineTestsBase): + + def test_a_cloud_engine_accepts_no_evidence(self): + engine = CloudEngine() + for key in ("query.user-agent", "header.user-agent", "header.x-test", + "cookie.51d_profileids", "query.session-id", + "server.client-ip"): + self.assertFalse(engine.filter_evidence_key(key), key) + + def test_a_cloud_pipeline_sends_only_the_evidence_the_service_lists(self): + """ + The stand-in service lists query.User-Agent alone, so of this + evidence only the user agent reaches it. + """ + cloud = CloudRequestEngine({ + "license_key": LICENSE_KEY, + "requested_properties": REQUESTED, + "http_client": self.mock_http(), + }) + engine = CloudEngine() + engine.datakey = "device" + + pipeline = PipelineBuilder().add(cloud).add(engine).build() + data = pipeline.create_flowdata() + data.evidence.add("query.User-Agent", "iPhone") + data.evidence.add("header.x-test", "a header") + data.evidence.add("cookie.other", "a cookie") + data.process() + + request_type, url, content = cloud.http_client.requests[-1] + self.assertEqual("iPhone", content["user-agent"]) + self.assertNotIn("x-test", content) + self.assertNotIn("other", content) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py index 649f7866..1c39b024 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py @@ -29,6 +29,7 @@ from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_cloudrequestengine.constants import Constants +from .classes.accepteverykeyelement import AcceptEveryKeyElement from .classes.constants import Constants as TestConstants from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase @@ -111,8 +112,11 @@ def test_evidence_precedence(self, warn, evidence1, evidence2): "http_client": self.mock_http(), }) + # The stand-in service accepts query.User-Agent alone, so an element + # that accepts every key keeps the rest of the evidence for the + # engine to order. builder = PipelineBuilder() - pipeline = builder.add(engine).build() + pipeline = builder.add(AcceptEveryKeyElement()).add(engine).build() data = pipeline.create_flowdata() diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py index a5a2b8ab..786ae020 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py @@ -28,6 +28,7 @@ from fiftyone_pipeline_cloudrequestengine.constants import Constants from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from .classes.accepteverykeyelement import AcceptEveryKeyElement from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase RESOURCE_KEY = "resource_key" @@ -269,7 +270,11 @@ def test_reserved_evidence_is_left_out_of_the_body(self): "http_client": self.mock_http(), }) - pipeline = PipelineBuilder().add(engine).build() + # The stand-in service lists query.User-Agent alone, so an element + # that accepts every key keeps this evidence for the engine, which + # leaves it out whatever reaches it. + pipeline = PipelineBuilder().add(AcceptEveryKeyElement()) \ + .add(engine).build() data = pipeline.create_flowdata() data.evidence.add("query.license", "another-key") data.evidence.add("header.resource", "another-key") @@ -302,6 +307,30 @@ def test_reserved_evidence_is_left_out_of_the_body(self): for message in messages: self.assertNotIn("another-key", message) + def test_evidence_no_element_accepts_is_not_sent(self): + """ + A default pipeline keeps only the evidence one of its elements + accepts, so a header or cookie the service does not ask for is not + sent to it. + """ + engine = CloudRequestEngine({ + "license_key": LICENSE_KEY, + "requested_properties": REQUESTED, + "http_client": self.mock_http(), + }) + + pipeline = PipelineBuilder().add(engine).build() + data = pipeline.create_flowdata() + data.evidence.add("query.User-Agent", "iPhone") + data.evidence.add("header.x-test", "a header") + data.evidence.add("cookie.other", "a cookie") + data.process() + + request_type, url, content = engine.http_client.requests[-1] + self.assertEqual("iPhone", content["user-agent"]) + self.assertNotIn("x-test", content) + self.assertNotIn("other", content) + def test_properties_added_later_are_normalised_and_checked(self): """ A caller may add to the list after the engine is built, once the diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py index 73e6922d..b04553a0 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py @@ -20,7 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -from .evidence_keyfilter import EvidenceKeyFilter +from .basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter class FlowElement(object): @@ -91,14 +91,16 @@ def close(self): def get_evidence_key_filter(self): """! - Filter FlowData evidence using the FlowElement's EvidenceKeyFilter + The evidence this element accepts. An element overrides this to name + the evidence that can change its results. One that does not accepts + no evidence, so it never widens what the pipeline keeps. @rtype: EvidenceKeyFilter @return: Returns an EvidenceKeyFilter """ - return EvidenceKeyFilter() + return BasicListEvidenceKeyFilter([]) def filter_evidence(self, flowdata): diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py index bdb85c50..cb96e9c8 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py @@ -215,8 +215,10 @@ def url_encode(value): class JavaScriptBuilderEvidenceKeyFilter(EvidenceKeyFilter): - def filter(self, key): - if "query" in key: + # The query string, which the script sends back to the callback, and the + # host and protocol the callback URL is built from. + def filter_evidence_key(self, key): + if key.startswith("query."): return True if key == "header.host" or key == "header.protocol": diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index 9f2a8ca7..575508ba 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -20,6 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from .flowelement import FlowElement from .elementdata_dictionary import ElementDataDictionary @@ -47,6 +48,15 @@ def __init__(self): self.property_cache = {} + def get_evidence_key_filter(self): + + """! + The element reads only the results of other elements, so it accepts + no evidence. + """ + + return BasicListEvidenceKeyFilter([]) + def process_internal(self, flowdata): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py index c4056232..cd85a857 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py @@ -28,7 +28,9 @@ class SequenceElementEvidenceKeyFilter(EvidenceKeyFilter): - def filter(self, key): + # The session id and the sequence. The result changes as the sequence + # grows, so they are part of the ETag. + def filter_evidence_key(self, key): if (key == "query.sequence" or key == "query.session-id"): return True diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py index 8f5402d7..083c5617 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py @@ -20,6 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from .basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from .flowelement import FlowElement from .elementdata_dictionary import ElementDataDictionary from .messages import Messages @@ -46,6 +47,15 @@ def __init__(self): self.exclude_from_messages = True + def get_evidence_key_filter(self): + + """! + The element reads only the results of other elements, so it accepts + no evidence. + """ + + return BasicListEvidenceKeyFilter([]) + def process_internal(self, flowdata): """! diff --git a/pipeline/fiftyone_pipeline_core/tests/test_default_elements_accept.py b/pipeline/fiftyone_pipeline_core/tests/test_default_elements_accept.py new file mode 100644 index 00000000..07d009b6 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_default_elements_accept.py @@ -0,0 +1,139 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The elements PipelineBuilder adds accept only the evidence they read, so +a default pipeline accepts what its engines read and no more. The ETag and +the Vary header of the client-side endpoints are built from what the +pipeline accepts.""" + +import unittest + +from flask import Flask, request + +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.javascriptbuilder import \ + JavascriptBuilderElement +from fiftyone_pipeline_core.jsonbundler import JSONBundlerElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_core.sequenceelement import SequenceElement +from fiftyone_pipeline_core.setheaderelement import SetHeaderElement + +from .classes.exampleflowelement1 import ExampleFlowElement1 + + +class DeclaresNothingElement(FlowElement): + """An element that does not say which evidence it accepts.""" + + def __init__(self): + super(DeclaresNothingElement, self).__init__() + self.datakey = "declaresnothing" + + def process_internal(self, flowdata): + pass + + +NOT_READ = ("header.if-none-match", "header.x-test", "header.x-query", + "header.cookie", "cookie.query", "server.client-ip") + + +def default_pipeline(): + """A pipeline with the elements the builder adds and one element that + accepts header.user-agent alone.""" + return PipelineBuilder().add(ExampleFlowElement1()).build() + + +def accepted(pipeline, key): + return any(element.filter_evidence_key(key) + for element in pipeline.flow_elements) + + +class DefaultElementsAcceptTests(unittest.TestCase): + + def test_a_default_pipeline_accepts_only_what_its_elements_read(self): + pipeline = default_pipeline() + + for key in NOT_READ: + self.assertFalse(accepted(pipeline, key), key) + for key in ("header.user-agent", "query.session-id", + "query.sequence", "query.fod-js-object-name", + "header.host", "header.protocol"): + self.assertTrue(accepted(pipeline, key), key) + + def test_each_element_accepts_only_what_it_reads(self): + sequence = SequenceElement() + self.assertTrue(sequence.filter_evidence_key("query.session-id")) + self.assertTrue(sequence.filter_evidence_key("query.sequence")) + self.assertFalse(sequence.filter_evidence_key("query.other")) + + builder = JavascriptBuilderElement({}) + self.assertTrue(builder.filter_evidence_key("query.anything")) + self.assertTrue(builder.filter_evidence_key("header.host")) + self.assertTrue(builder.filter_evidence_key("header.protocol")) + self.assertFalse(builder.filter_evidence_key("header.x-query")) + self.assertFalse(builder.filter_evidence_key("cookie.query")) + + for element in (SetHeaderElement(), JSONBundlerElement()): + for key in NOT_READ + ("header.user-agent", "query.sequence"): + self.assertFalse(element.filter_evidence_key(key), + type(element).__name__ + " " + key) + + def test_processing_sets_aside_the_evidence_no_element_reads(self): + flowdata = default_pipeline().create_flowdata() + flowdata.evidence.add("header.user-agent", "a browser") + flowdata.evidence.add("header.if-none-match", '"a-tag"') + flowdata.process() + + self.assertEqual("a browser", + flowdata.evidence.get("header.user-agent")) + self.assertIsNone(flowdata.evidence.get("header.if-none-match")) + + def test_only_the_headers_the_elements_read_are_named_for_vary(self): + """The client-side endpoints name in Vary the request headers the + pipeline accepts.""" + + pipeline = default_pipeline() + with Flask(__name__).test_request_context("/", headers={ + "User-Agent": "a browser", "Accept": "*/*", + "Accept-Language": "en", "Cookie": "a=b", + "If-None-Match": '"a-tag"', "X-Test": "x"}): + named = sorted(name for name, _ in request.headers + if accepted(pipeline, "header." + name.lower())) + + self.assertEqual(["Host", "User-Agent"], named) + + def test_an_element_that_declares_nothing_accepts_nothing(self): + element = DeclaresNothingElement() + for key in NOT_READ + ("header.user-agent", "query.session-id"): + self.assertFalse(element.filter_evidence_key(key), key) + + def test_an_element_that_declares_nothing_widens_nothing(self): + pipeline = PipelineBuilder() \ + .add(ExampleFlowElement1()) \ + .add(DeclaresNothingElement()) \ + .build() + for key in NOT_READ: + self.assertFalse(accepted(pipeline, key), key) + self.assertTrue(accepted(pipeline, "header.user-agent")) + + +if __name__ == "__main__": + unittest.main() From 451c17ed3ccbce23f84efac5ce0f80c027229dc5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 16:02:44 +0000 Subject: [PATCH 31/68] Take 51degrees-cxx rc cfad8978 and generate the IP Intelligence wrapper again (#72) * Take 51degrees-cxx rc 865cee97 and generate the IP Intelligence wrapper again Moves the cxx submodule from e866f6d3, 4.5.103, to the 51degrees-cxx rc commit 865cee97. Generated hash_python_wrap.cxx, DeviceDetectionHashEngineModule.py, ipi_python_wrap.cxx and IpIntelligenceEngineModule.py again with SWIG 4.4.1, the version in the committed files, from the staged source the way PreBuild.sh does. Run against e866f6d3 the same command gives files byte for byte the same as the committed ones, so every difference comes from cxx. Only ipi_python_wrap.cxx changes. The four ConfigIpiSwig getters that return a CollectionConfig no longer pass SWIG_POINTER_OWN, because the cxx interface now scopes its newobject directives to MetaData and the configuration keeps ownership of those references. The Device Detection wrapper and both module files are unchanged. The setup.py files list the cxx sources by glob, so the new src/common/date.c is picked up with no change there. * Move cxx to the 51degrees-cxx rc commit b4f769ad Takes the eight 51degrees-cxx pull requests merged into rc after 865cee97, among them the rename of src/common/Date.cpp to date_pp.cpp so it no longer clashes with date.c on a case-insensitive file system. * Move cxx to the 51degrees-cxx rc commit cfad8978 Takes 51degrees-cxx #130, which writes the text of each IP Intelligence value with its weight in C, and #146, which gives a reason and a message when a property has no value in the matched results. --- cxx | 2 +- .../fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cxx b/cxx index e866f6d3..cfad8978 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit e866f6d3cea936587790f87967eff7863778b30e +Subproject commit cfad8978827292b97ad08f306891edea09ece304 diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx index 0d472453..022ec85e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx @@ -45763,7 +45763,7 @@ SWIGINTERN PyObject *_wrap_ConfigIpiSwig_getComponents(PyObject *self, PyObject SWIG_exception(SWIG_ValueError, e.what()); } } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_CollectionConfig, SWIG_POINTER_OWN | 0 ); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_CollectionConfig, 0 | 0 ); return resultobj; fail: return NULL; @@ -45873,7 +45873,7 @@ SWIGINTERN PyObject *_wrap_ConfigIpiSwig_getProperties(PyObject *self, PyObject SWIG_exception(SWIG_ValueError, e.what()); } } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_CollectionConfig, SWIG_POINTER_OWN | 0 ); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_CollectionConfig, 0 | 0 ); return resultobj; fail: return NULL; @@ -45928,7 +45928,7 @@ SWIGINTERN PyObject *_wrap_ConfigIpiSwig_getValues(PyObject *self, PyObject *arg SWIG_exception(SWIG_ValueError, e.what()); } } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_CollectionConfig, SWIG_POINTER_OWN | 0 ); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_CollectionConfig, 0 | 0 ); return resultobj; fail: return NULL; @@ -45983,7 +45983,7 @@ SWIGINTERN PyObject *_wrap_ConfigIpiSwig_getProfiles(PyObject *self, PyObject *a SWIG_exception(SWIG_ValueError, e.what()); } } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_CollectionConfig, SWIG_POINTER_OWN | 0 ); + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_CollectionConfig, 0 | 0 ); return resultobj; fail: return NULL; From 45ef646c3d8082d43c44960666a30e04ed5f613f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 17:50:17 +0000 Subject: [PATCH 32/68] Test that the cloud's errors text shows only the masked licence key --- .../tests/test_licence_key_errors.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licence_key_errors.py diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licence_key_errors.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licence_key_errors.py new file mode 100644 index 00000000..8d92ff4c --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licence_key_errors.py @@ -0,0 +1,72 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The cloud service names the licence key in the errors it answers with, and +the engine's exception shows only the key's last four characters.""" + +import json +from unittest.mock import Mock + +from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import \ + CloudRequestEngine +from fiftyone_pipeline_cloudrequestengine.cloudrequestexception import \ + CloudRequestException + +from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase + +KEY = "ABCDEFGHIJKL" +MASKED = "********IJKL" + + +def answer(status, text): + reply = Mock() + reply.status_code = status + reply.text = text + reply.json = lambda: json.loads(text) + reply.headers = {} + reply.url = "https://cloud.51degrees.com/api/v4/json" + return reply + + +class LicenceKeyErrorTests(CloudRequestEngineTestsBase): + + def engine(self): + return CloudRequestEngine({ + "license_key": KEY, + "requested_properties": ["device.ismobile"], + "http_client": self.mock_http(), + }) + + def test_the_errors_text_shows_only_the_masked_key(self): + text = json.dumps({"errors": [ + "The license key is not valid. License key(s): '" + KEY + "'"]}) + with self.assertRaises(CloudRequestException) as caught: + self.engine().validate_response(answer(400, text)) + self.assertNotIn(KEY[:8], str(caught.exception)) + self.assertIn(MASKED, str(caught.exception)) + + def test_content_that_is_not_json_shows_only_the_masked_key(self): + with self.assertRaises(CloudRequestException) as caught: + self.engine().validate_response( + answer(500, "Licence " + KEY + " refused")) + self.assertNotIn(KEY[:8], str(caught.exception)) + self.assertIn(MASKED, str(caught.exception)) From 930b6775c11f21c8c691a2a77de2e73fda470463 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 17:50:33 +0000 Subject: [PATCH 33/68] Show only the last four characters of the licence key in the cloud's errors text --- .../cloudrequestengine.py | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py index 6a6ed7d0..13b7f215 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py @@ -30,6 +30,7 @@ import requests from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.keymask import mask_licence_key from fiftyone_pipeline_core.keymask import mask_licence_keys_in_text from fiftyone_pipeline_engines.engine import Engine from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary @@ -335,15 +336,15 @@ def validate_response( try: json_response = cloud_response.json() except (JSONDecodeError, requests.exceptions.JSONDecodeError): - raise CloudRequestException( + raise CloudRequestException(self.masked( f'Cloud request engine properties list request returned code "{cloud_response.status_code}"' f' with non-JSON content "{cloud_response.text}"' - ) + )) except Exception as e: - raise CloudRequestException( + raise CloudRequestException(self.masked( f'Cloud request engine properties list request returned code "{cloud_response.status_code}"' f' with content "{cloud_response.text}".\nError: "{type(e).__name__}: {e}"' - ) + )) has_errors = "errors" in json_response and len(json_response["errors"]) has_data = len(json_response) > (1 if has_errors else 0) @@ -380,11 +381,25 @@ def validate_response( ) raise CloudRequestException( - exception_message, + self.masked(exception_message), cloud_response.status_code, cloud_response.headers, ) + def masked(self, text): + + """! + Text from or about a cloud answer with the licence key in use shown + only as its last four characters. The cloud service names the key in + some of its errors, and a URL may carry it in its query. + """ + + text = mask_licence_keys_in_text(text) + if self.license_key != "": + text = text.replace( + self.license_key, mask_licence_key(self.license_key)) + return text + def make_cloud_request(self, type, url, content = None): """! From 7f689f29616ed0a8f64f3a52739dff8f9c2675b1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 19:21:27 +0000 Subject: [PATCH 34/68] Accept no evidence for usage sharing, and share what it finds in the flow data and the web request (#61) * Test that usage sharing accepts no evidence and still shares what it finds * Accept no evidence for usage sharing, and share what it finds in the flow data and the web request * Check the tracker reads the session cookie the element no longer accepts * Test the request view's parts and that the Cookie header is never shared * Never share the Cookie header, and give the request view its client IP, protocol, method, path and session values * Give the cookie in the request view tests a value Flask reads * Read the session through the object behind Flask's proxy * Test that session numbers and booleans reach the evidence as text, and wrap two test lines * Add session numbers and booleans to the evidence as text --- .../tests/test_cloud_mixed_web_offline.py | 11 + .../src/fiftyone_pipeline_core/evidence.py | 27 ++- .../src/fiftyone_pipeline_core/web.py | 43 +++- .../tests/test_evidence_unaccepted.py | 131 +++++++++++ .../share_usage.py | 60 ++++- .../share_usage_evidencekeyfilter.py | 12 +- .../test_share_usage_accepts_no_evidence.py | 208 ++++++++++++++++++ .../test_share_usage_evidencekeyfilter.py | 10 +- .../tests/test_usage.py | 11 +- 9 files changed, 481 insertions(+), 32 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_evidence_unaccepted.py create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py index 47fe209e..8ba78fef 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py @@ -302,3 +302,14 @@ def test_licence_key_gets_an_engine_only_for_what_it_asks_for(cloud): def test_licence_key_never_reaches_the_page(cloud): cloud(EVERY_SECTION) assert "a-licence-key" not in page_for(license_key="a-licence-key") + + +def test_the_web_request_is_not_sent_to_the_cloud(cloud): + # The web integration adds the request itself as server.http-request for + # the usage sharing element, and it is never evidence another element + # reads. + stub = cloud(EVERY_SECTION) + page_for(resource_key="a-resource-key") + + url, data = stub.requests[-1] + assert not any("http-request" in key for key in data), data diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py index f7067647..02b45ce7 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py @@ -20,6 +20,11 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +# The key the web integration gives the web request itself, which the usage +# sharing element reads. No element uses it as evidence, so filter() always +# sets it aside, whatever the elements' filters accept. +HTTP_REQUEST_KEY = "server.http-request" + class Evidence: def __init__(self, flowdata): @@ -32,6 +37,7 @@ def __init__(self, flowdata): self.evidence = {} self.evidence_filtered = False self.flowdata = flowdata + self.unaccepted = {} def add(self, key, value): """! @@ -89,10 +95,25 @@ def get_all(self): return self.evidence + def get_unaccepted(self): + """! + The evidence filter() set aside, being the keys no FlowElement in + the pipeline accepts and the web request itself. The usage sharing + element accepts no evidence, and shares what it finds here and in + get_all(). + + @rtype: dict + @return: The evidence set aside, empty until filter() runs + """ + + return self.unaccepted + def filter(self) -> None: """! Filter evidence using `filter_evidence_key` and - `filter_evidence` methods of each FlowElement in the pipeline + `filter_evidence` methods of each FlowElement in the pipeline. The + keys that do not pass, and the web request under HTTP_REQUEST_KEY, + move to get_unaccepted(). """ if self.evidence_filtered: @@ -100,11 +121,11 @@ def filter(self) -> None: # Loop over a copy of the keys, as keys are removed on the way. for key in list(self.evidence): - if not any([ + if str(key).lower() == HTTP_REQUEST_KEY or not any([ flow_element.filter_evidence_key(key) for flow_element in self.flowdata.pipeline.flow_elements ]): - del self.evidence[key] + self.unaccepted[key] = self.evidence.pop(key) self.evidence_filtered = True diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py index d3f96cf0..768fdd8b 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py @@ -20,13 +20,30 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -from flask import request +from types import MappingProxyType + +from flask import has_request_context, request, session + +from .evidence import HTTP_REQUEST_KEY + +def request_line_path(request): + + """! + The path of the request line as the server received it, when the server + says what that was, and otherwise the path Flask decoded. + """ + + raw = request.environ.get("RAW_URI") or request.environ.get("REQUEST_URI") + if raw: + return raw.split("?", 1)[0].split("#", 1)[0] + return request.path def webevidence(request): """! - Get evidence from a web request (gets headers, cookies, query parameters - and the fields of a posted form) + Get evidence from a web request (gets headers, cookies, session values, + query parameters and the fields of a posted form, and the request itself + as server.http-request) @type request: Request @param request: A Request object @@ -59,6 +76,15 @@ def webevidence(request): webevidence["query." + field] = value + # Values stored in the session. They are read without marking the session + # as accessed, which would add a Vary: Cookie header to the response, and + # only plain values are taken. Numbers and booleans are added as text, + # because the on-premise engines take evidence values as text only. + if has_request_context(): + for name, value in dict.items(session._get_current_object()): + if isinstance(value, (str, int, float, bool)): + webevidence["session." + name] = str(value) + webevidence["server.client-ip"] = request.remote_addr webevidence["server.host-ip"] = request.host @@ -68,6 +94,17 @@ def webevidence(request): else: webevidence["header.protocol"] = "http" + # The request itself, for the usage sharing element, which accepts no + # evidence and shares what it finds. A read-only view of the parts above + # and the method and path of the request line, keyed as evidence is. It + # leaves out the Cookie header, which cookies already cover. Evidence + # keeps it from every other element. + view = {key: value for key, value in webevidence.items() + if key not in ("header.cookie", "server.host-ip")} + view["server.request-method"] = request.method + view["server.request-path"] = request_line_path(request) + webevidence[HTTP_REQUEST_KEY] = MappingProxyType(view) + return webevidence def set_response_header(flowData, response): diff --git a/pipeline/fiftyone_pipeline_core/tests/test_evidence_unaccepted.py b/pipeline/fiftyone_pipeline_core/tests/test_evidence_unaccepted.py new file mode 100644 index 00000000..0ffc97a2 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_evidence_unaccepted.py @@ -0,0 +1,131 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The evidence a pipeline's elements do not accept, and the web request the +web integration adds as server.http-request.""" + +import unittest + +from flask import Flask, request, session + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_core.web import webevidence + +from .classes.exampleflowelement1 import ExampleFlowElement1 + +# No JavaScript or set header elements, so the only filter is that of +# ExampleFlowElement1, which accepts header.user-agent. +NO_DEFAULT_ELEMENTS = { + "add_javascript_builder": False, + "use_setheader_properties": False, +} + + +class UnacceptedEvidenceTests(unittest.TestCase): + + def test_evidence_no_element_accepts_is_set_aside(self): + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ + .add(ExampleFlowElement1()).build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "a browser") + flowdata.evidence.add("header.x-test", "set aside") + flowdata.process() + + self.assertEqual( + {"header.user-agent": "a browser"}, flowdata.evidence.get_all()) + self.assertEqual( + {"header.x-test": "set aside"}, + flowdata.evidence.get_unaccepted()) + + def test_the_web_request_is_added_and_set_aside(self): + """No element reads server.http-request as evidence, even an element + that accepts every key, as the JavaScript builder does.""" + + pipeline = PipelineBuilder().add(ExampleFlowElement1()).build() + app = Flask(__name__) + with app.test_request_context( + "/?campaign=a", headers={"User-Agent": "a browser", + "X-Test": "a header"}): + evidence = webevidence(request) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add_from_dict(evidence) + flowdata.process() + + self.assertNotIn("server.http-request", flowdata.evidence.get_all()) + view = flowdata.evidence.get_unaccepted()["server.http-request"] + self.assertEqual("a header", view["header.x-test"]) + self.assertEqual("a browser", view["header.user-agent"]) + self.assertEqual("a", view["query.campaign"]) + # A read-only view of the request, not evidence an element can + # change. + with self.assertRaises(TypeError): + view["header.x-test"] = "changed" + + def test_the_web_request_has_every_part_the_specification_names(self): + app = Flask(__name__) + app.secret_key = "a secret" + with app.test_request_context( + "/page?campaign=a", + headers={"User-Agent": "a browser", + "Cookie": "sessionid=cookievalue"}, + environ_base={"REMOTE_ADDR": "203.0.113.5"}): + session["51D_one"] = "first" + session["theme"] = "dark" + view = webevidence(request)["server.http-request"] + + self.assertEqual("a browser", view["header.user-agent"]) + self.assertEqual("cookievalue", view["cookie.sessionid"]) + self.assertEqual("a", view["query.campaign"]) + self.assertEqual("first", view["session.51D_one"]) + self.assertEqual("dark", view["session.theme"]) + self.assertEqual("203.0.113.5", view["server.client-ip"]) + self.assertEqual("http", view["header.protocol"]) + self.assertEqual("GET", view["server.request-method"]) + self.assertEqual("/page", view["server.request-path"]) + # The Cookie header carries every cookie, and cookies are their own + # part of the view. + self.assertNotIn("header.cookie", view) + + def test_session_numbers_and_booleans_are_added_as_text(self): + """The on-premise engines take evidence values as text only, so a + number or a boolean kept in the session must not reach them as + anything else.""" + + app = Flask(__name__) + app.secret_key = "a secret" + with app.test_request_context("/"): + session["visits"] = 3 + session["ratio"] = 0.5 + session["consented"] = True + session["basket"] = ["left", "out"] + evidence = webevidence(request) + + view = evidence["server.http-request"] + for source in (evidence, view): + self.assertEqual("3", source["session.visits"]) + self.assertEqual("0.5", source["session.ratio"]) + self.assertEqual("True", source["session.consented"]) + self.assertNotIn("session.basket", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 210dfb0a..ec2a657a 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -29,12 +29,19 @@ from importlib.metadata import version import requests +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter from fiftyone_pipeline_engines.engine import Engine from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter, \ ShareUsageTrackerEvidenceKeyFilter from .share_usage_tracker import ShareUsageTracker +# The key the web integration gives the web request itself. Written out +# rather than imported, so the element still loads beside an older +# fiftyone_pipeline_core. +HTTP_REQUEST_KEY = "server.http-request" + # The maximum length of a piece of evidence's value which can be # added to the usage data being sent. SHARE_USAGE_MAX_EVIDENCE_LENGTH = 512 @@ -123,11 +130,13 @@ def __init__( included = None if query_whitelist is None else \ list(query_whitelist) + ["session-id", "sequence"] - # When sessions are tracked, the pipeline keeps the session cookie - # for the tracker. The data sent leaves it out unless all evidence + # The element accepts no evidence, so that caching, the ETag and the + # Vary header never depend on it. It shares what it finds instead, + # and these filters decide which of that is shared and which tells + # a repeat apart. When sessions are tracked the tracker reads the + # session cookie, which the data sent leaves out unless all evidence # is shared. - self.evidence_key_filter = ShareUsageEvidenceKeyFilter( - cookie, included, header_blacklist, track_session, share_all) + self.evidence_key_filter = BasicListEvidenceKeyFilter([]) self.data_evidence_key_filter = ShareUsageEvidenceKeyFilter( cookie, included, header_blacklist, False, share_all) self.tracker_evidence_key_filter = \ @@ -214,14 +223,40 @@ def get_evidence_key_filter(self): """! - The share useage element comes with its own evidence - key filter that uses the whitelists and blacklists to - determine which evidence to share - + The usage sharing element accepts no evidence. See found_evidence + for what it shares. """ return self.evidence_key_filter + def found_evidence(self, flow_data): + + """! + The evidence the element finds to share. These are the values of the + web request, when the web integration added it as + server.http-request, and every evidence value in the flow data, + whether an element accepted it or not. A key found in both places + takes the evidence value. The web request itself is never one of + them. + + @type flow_data: FlowData + @param flow_data: The flow data being processed + @rtype: dict + @return: The evidence found, before the sharing rules apply + """ + + evidence = flow_data.evidence + # An older fiftyone_pipeline_core sets nothing aside. + unaccepted = getattr(evidence, "get_unaccepted", dict)() + web_request = unaccepted.get(HTTP_REQUEST_KEY) or \ + evidence.get(HTTP_REQUEST_KEY) or {} + found = {} + for source in (web_request, unaccepted, evidence.get_all()): + for key, value in source.items(): + if str(key).lower() != HTTP_REQUEST_KEY: + found[key] = value + return found + def share_send_usage(self): """! Internal method to send the share usage bundle to the 51Degrees servers @@ -319,8 +354,11 @@ def process_internal(self, flow_data): """ if random.uniform(0, 1) <= self.share_percentage: - cachekey = self.tracker_evidence_key_filter.filter_evidence(flow_data.evidence.get_all()) - cachekey = json.dumps(cachekey) + # A hash of the values that would be shared, as the element + # accepts no evidence of its own. + cachekey = self.tracker_evidence_key_filter.filter_evidence( + self.found_evidence(flow_data)) + cachekey = json.dumps(cachekey, sort_keys=True, default=str) share = self.tracker.track(cachekey) if share: self.tracker.set_cache_value(cachekey) @@ -336,7 +374,7 @@ def get_data_from_evidence(self, flow_data): """ data = ShareUsageData() - for key, value in flow_data.evidence.get_all().items(): + for key, value in self.found_evidence(flow_data).items(): if key == 'server.client-ip': # The client IP is dealt with separately for backwards # compatibility purposes. diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index 5f2292f3..dc4d4264 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -26,16 +26,15 @@ # this are the ones 51Degrees sets, and are shared. FIFTYONE_PREFIX = "51d_" -# The Cookie header carries every cookie, including the ones the cookie rule -# leaves out, so it is not shared unless all evidence is. -BLOCKED_HEADERS = ("cookie",) - # Evidence that is never shared, whatever the settings, matched as the whole # key or after a '.'. 'id.email' and 'id.salt' are a raw email address and the # salt used to make hashed email identifiers. 'request-query' is the whole query # string, which would undo the rules for its parameters. 'request-path' would # make each page a visitor opens look like a new session to the tracker. -NEVER_SHARED_KEYS = ("id.email", "id.salt", "request-query", "request-path") +# 'header.cookie' is the Cookie header, which carries every cookie, and the +# cookies that are shared are shared as their own 'cookie.' evidence. +NEVER_SHARED_KEYS = ( + "id.email", "id.salt", "request-query", "request-path", "header.cookie") class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter): """ @@ -110,8 +109,7 @@ def filter_evidence_key(self, key): elif prefix == "header": # Add the header to the list if the header name does not # appear in the list of blocked headers - return field not in self.header_blacklist and \ - field not in BLOCKED_HEADERS + return field not in self.header_blacklist elif prefix == "cookie": # Only add cookies that start with the 51Degrees cookie # prefix diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py new file mode 100644 index 00000000..e683f856 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_accepts_no_evidence.py @@ -0,0 +1,208 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The usage sharing element accepts no evidence, so a pipeline's accepted +evidence, and the cache keys, ETag and Vary header built from it, never +depend on it. It shares what it finds in the flow data and in the web +request instead, by the same rules as before.""" + +import unittest + +from flask import Flask, request + +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_core.web import webevidence +from fiftyone_pipeline_engines_fiftyone.share_usage import ShareUsage + +# No JavaScript or set header elements, so the filters are those of the +# usage sharing element and of UserAgentElement alone. +NO_DEFAULT_ELEMENTS = { + "add_javascript_builder": False, + "use_setheader_properties": False, +} + +CHROME = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36") + + +class UserAgentElement(FlowElement): + """A stand-in for an engine, accepting the User-Agent alone.""" + + def __init__(self): + super(UserAgentElement, self).__init__() + self.datakey = "useragent" + + def process_internal(self, flowdata): + pass + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(["header.user-agent"]) + + +def usage_element(**settings): + """A usage sharing element that keeps what it would share, because its + package size is never reached, and sends nothing.""" + + return ShareUsage( + requested_package_size=1000, endpoint="http://127.0.0.1:9", + **settings) + + +def pipeline_with(usage): + return PipelineBuilder(NO_DEFAULT_ELEMENTS) \ + .add(UserAgentElement()).add(usage).build() + + +def request_evidence(headers, path="/"): + """The evidence the web integration makes from a request.""" + + with Flask(__name__).test_request_context(path, headers=headers): + return webevidence(request) + + +class ShareUsageAcceptsNoEvidenceTests(unittest.TestCase): + + def test_the_pipeline_accepts_no_header_for_usage_sharing(self): + usage = usage_element() + pipeline = pipeline_with(usage) + + for key in ("header.if-none-match", "header.x-test", + "cookie.51d_screenpixelswidth", "server.client-ip"): + self.assertFalse(usage.get_evidence_key_filter() + .filter_evidence_key(key), key) + self.assertFalse(any( + element.filter_evidence_key(key) + for element in pipeline.flow_elements), key) + self.assertTrue(any( + element.filter_evidence_key("header.user-agent") + for element in pipeline.flow_elements)) + + def test_a_revalidation_has_the_same_accepted_evidence(self): + """The ETag of a client-side endpoint hashes the evidence the + pipeline accepts, so a revalidation, which adds If-None-Match, has + to leave that evidence as it was.""" + + pipeline = pipeline_with(usage_element()) + accepted = [] + for headers in ({"User-Agent": CHROME}, + {"User-Agent": CHROME, "If-None-Match": '"a-tag"'}): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add_from_dict(request_evidence(headers)) + flowdata.process() + accepted.append(dict(flowdata.evidence.get_all())) + + self.assertEqual(accepted[0], accepted[1]) + self.assertNotIn("header.if-none-match", accepted[1]) + + def test_the_web_request_is_shared_by_the_same_rules(self): + usage = usage_element(header_blacklist=["x-blocked"]) + flowdata = pipeline_with(usage).create_flowdata() + flowdata.evidence.add_from_dict(request_evidence({ + "User-Agent": CHROME, + "X-Test": "only usage sharing reads this", + "X-Blocked": "never shared", + "Cookie": "51D_ScreenPixelsWidth=1920; other=private", + }, "/?51D_GetHighEntropyValues=abc&password=secret")) + flowdata.process() + + shared = "".join(usage.share_data) + self.assertIn( + '
    only usage sharing reads this
    ', + shared) + self.assertIn('
    ', shared) + self.assertIn( + '1920', shared) + self.assertIn( + 'abc', shared) + for left_out in ('Name="x-blocked"', 'Name="cookie"', "private", + "secret", "http-request"): + self.assertNotIn(left_out, shared) + + def test_the_method_is_shared_and_the_path_and_cookie_header_are_not(self): + """Even when all evidence is shared.""" + + usage = usage_element(share_all=True) + flowdata = pipeline_with(usage).create_flowdata() + flowdata.evidence.add_from_dict(request_evidence({ + "User-Agent": CHROME, + "Cookie": "sessionid=cookievalue", + }, "/page?a=1")) + flowdata.process() + + shared = "".join(usage.share_data) + self.assertIn("request-method", shared) + self.assertIn('cookievalue', + shared) + self.assertNotIn("/page", shared) + self.assertNotIn('
    ', shared) + + def test_a_key_in_both_places_is_shared_once_with_the_evidence_value(self): + usage = usage_element() + flowdata = pipeline_with(usage).create_flowdata() + flowdata.evidence.add_from_dict( + request_evidence({"X-Test": "from the request"})) + flowdata.evidence.add("header.x-test", "from the evidence") + flowdata.process() + + shared = "".join(usage.share_data) + self.assertEqual(1, shared.count('Name="x-test"')) + self.assertIn( + '
    from the evidence
    ', shared) + + def test_evidence_added_by_hand_is_shared_as_before(self): + usage = usage_element() + flowdata = pipeline_with(usage).create_flowdata() + flowdata.evidence.add("header.user-agent", CHROME) + flowdata.evidence.add("header.x-test", "by hand") + flowdata.evidence.add("cookie.51D_ScreenPixelsWidth", "1920") + flowdata.evidence.add("cookie.other", "private") + flowdata.evidence.add("server.client-ip", "203.0.113.42") + flowdata.process() + + shared = "".join(usage.share_data) + self.assertIn('
    by hand
    ', shared) + self.assertIn( + '1920', shared) + self.assertIn("203.0.113.42", shared) + self.assertNotIn("private", shared) + + def test_a_repeat_is_told_apart_by_what_would_be_shared(self): + """The tracker's hash is of the values that would be shared, so a + header no other element accepts still tells two requests apart, + and a repeat of the same request is shared once.""" + + usage = usage_element() + pipeline = pipeline_with(usage) + for value in ("first", "first", "second"): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add_from_dict( + request_evidence({"User-Agent": CHROME, "X-Test": value})) + flowdata.process() + + self.assertEqual(2, len(usage.share_data)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py index 16184d52..317ccf5a 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -112,7 +112,7 @@ def test_share_all(self): keys = ShareUsageEvidenceKeyFilter(share_all=True) - self.assertTrue(keys.filter_evidence_key("header.cookie")) + self.assertFalse(keys.filter_evidence_key("header.cookie")) self.assertTrue(keys.filter_evidence_key("cookie.sessionid")) self.assertTrue(keys.filter_evidence_key("query.password")) self.assertTrue(keys.filter_evidence_key("query.id.usage")) @@ -127,8 +127,9 @@ def test_share_all(self): def test_never_shared_in_every_mode(self): """! Check that the email address, the salt for hashed email - identifiers, the query string and the path are never shared, - whether bare or after a prefix, and without regard to case. + identifiers, the query string, the path and the Cookie header are + never shared, whether bare or after a prefix, and without regard to + case. """ for keys in ( @@ -140,7 +141,8 @@ def test_never_shared_in_every_mode(self): "query.id.email", "QUERY.ID.EMAIL", "header.id.salt", "HEADER.ID.SALT", "server.request-query", "SERVER.REQUEST-QUERY", - "server.request-path", "SERVER.REQUEST-PATH"): + "server.request-path", "SERVER.REQUEST-PATH", + "header.cookie", "HEADER.COOKIE"): self.assertFalse(keys.filter_evidence_key(key), key) def test_never_shared_keys_match_whole_parts(self): diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index 4f1bea40..4438b8d3 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -574,9 +574,9 @@ def test_empty_query_whitelist(self): def test_session_cookie(self): """! - Check that with session tracking on the session cookie is kept in - the evidence, but neither it nor any other ordinary cookie is - shared. + Check that with session tracking on the tracker reads the session + cookie, which the element does not accept as evidence, and that + neither it nor any other ordinary cookie is shared. """ usage = ShareUsage( @@ -595,7 +595,10 @@ def test_session_cookie(self): flowdata.process() self.waitForUsageThreads() - self.assertTrue(usage.get_evidence_key_filter().filter_evidence_key("cookie.sessionid")) + self.assertFalse(usage.get_evidence_key_filter() + .filter_evidence_key("cookie.sessionid")) + self.assertTrue(usage.tracker_evidence_key_filter + .filter_evidence_key("cookie.sessionid")) self.assertEqual(len(app.received), 1) self.assertIn("some user agent", app.received[0]) self.assertNotIn("session value", app.received[0]) From 4c8a0e189758e81c6a58c459e20511ffa3f50943 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 19:25:11 +0000 Subject: [PATCH 35/68] Write the JSON Builder element and meta-property keys in lower case (#77) * Test that the JSON Builder writes keys in lower case * Write the JSON Builder's element and meta-property keys in lower case * Store the lower case test element's data under the lower case key * Set the mixed case test element's data through set_element_data, as a real element does * Keep Element Data under the data key in lower case, which is how FlowData.get reads it --- .../src/fiftyone_pipeline_core/flowdata.py | 3 +- .../src/fiftyone_pipeline_core/jsonbundler.py | 8 +-- .../tests/test_jsonbundler_lower_case.py | 69 +++++++++++++++++++ 3 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_lower_case.py diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py index c6f83c7c..914d7596 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py @@ -203,7 +203,8 @@ def set_element_data(self, element_data): """ - self.data[element_data.flow_element.datakey] = element_data + # Kept under the data key in lower case, which is how get reads it. + self.data[element_data.flow_element.datakey.lower()] = element_data def set_error(self, flow_error): diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index 575508ba..bb441dd1 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -113,7 +113,7 @@ def process_internal(self, flowdata): if len(delayed_evidence_properties_list): delayed_evidence_properties[propertykey] = list(map(lambda x: - flow_element.datakey + '.' + x, delayed_evidence_properties_list)) + flow_element.datakey.lower() + '.' + x, delayed_evidence_properties_list)) self.property_cache[flow_element.datakey] = { @@ -125,7 +125,7 @@ def process_internal(self, flowdata): # Create empty area for FlowElement properties to go - output[flow_element.datakey] = {} + output[flow_element.datakey.lower()] = {} for propertykey, propertymeta in properties.items(): value = None @@ -135,13 +135,13 @@ def process_internal(self, flowdata): if propertykey in property_cache["delayExecutionList"]: - output[flow_element.datakey][propertykey.lower() + "delayexecution"] = True + output[flow_element.datakey.lower()][propertykey.lower() + "delayexecution"] = True # Check if property has any delayed execution evidence properties and set in JSON if yes if propertykey in property_cache["evidenceProperties"]: - output[flow_element.datakey][propertykey.lower() + 'evidenceproperties'] = property_cache["evidenceProperties"][propertykey] + output[flow_element.datakey.lower()][propertykey.lower() + 'evidenceproperties'] = property_cache["evidenceProperties"][propertykey] try: diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_lower_case.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_lower_case.py new file mode 100644 index 00000000..028eac4a --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_lower_case.py @@ -0,0 +1,69 @@ + +""" +The JSON Builder writes the key of each top-level entry and of each sub-entry, +including the meta-Property suffixes, in lower case, apart from +javascriptProperties, as the JSON Builder specification says. +""" + +import unittest + +from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + + +class MixedCaseElement(FlowElement): + + def __init__(self): + super(MixedCaseElement, self).__init__() + self.datakey = "MixedCase" + self.properties = { + "DelayedProp": { + "name": "DelayedProp", "type": "string", + "delayexecution": True}, + "OtherProp": { + "name": "OtherProp", "type": "string", + "evidenceproperties": ["delayedprop"]}, + "EmptyProp": {"name": "EmptyProp", "type": "string"}} + + def process_internal(self, flowdata): + # Set as a real element sets it, under its mixed case data key. + flowdata.set_element_data(ElementDataDictionary( + self, {"DelayedProp": "a", "OtherProp": "b", + "EmptyProp": AspectPropertyValue( + no_value_message="No value")})) + + +class JsonBuilderLowerCaseTests(unittest.TestCase): + + def process(self): + pipeline = PipelineBuilder({}).add(MixedCaseElement()).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.process() + return flowdata.jsonbundler.json + + def test_every_key_is_lower_case_but_javascript_properties(self): + json = self.process() + self.assertIn("javascriptProperties", json) + self.assertIn("mixedcase", json) + for key, value in json.items(): + if key == "javascriptProperties": + continue + self.assertEqual(key.lower(), key) + for sub_key in value: + self.assertEqual(sub_key.lower(), sub_key, sub_key) + + def test_the_meta_property_keys_are_lower_case(self): + section = self.process()["mixedcase"] + self.assertTrue(section["delayedpropdelayexecution"]) + self.assertIn("otherpropevidenceproperties", section) + self.assertEqual("No value", section["emptypropnullreason"]) + self.assertIsNone(section["emptyprop"]) + self.assertEqual("a", section["delayedprop"]) + self.assertEqual("b", section["otherprop"]) + + +if __name__ == "__main__": + unittest.main() From a85be0214dd9ad6851be706dde84f288f5aed4cd Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 19:25:22 +0000 Subject: [PATCH 36/68] Send only the Evidence the cloud accepts, never resource, license, licence or values (#79) * Test that reserved names in Evidence are not sent in the request * Send only the Evidence the cloud accepts, never resource, license, licence or values * Keep the other evidence for the engine in the reserved evidence helper, now that no default element accepts it * Wrap the reserved evidence helper's pipeline line --- .../constants.py | 6 +- .../tests/test_credentials.py | 82 +++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py index 8014286d..f53bee2a 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py @@ -71,13 +71,15 @@ class Constants: LICENSE_PARAMETER = "license" VALUES_PARAMETER = "values" - # The cloud service advertises these three among its accepted evidence + # The cloud service reads these names among its accepted evidence # keys, so evidence such as "query.license" would otherwise go into the # request body beside the engine's own credentials, with no way to say # which one the service applied. Evidence whose key strips to one of # them is left out of every request. + LICENCE_SPELLING = "licence" RESERVED_EVIDENCE_SUFFIXES = ( - RESOURCE_PARAMETER, LICENSE_PARAMETER, VALUES_PARAMETER) + RESOURCE_PARAMETER, LICENSE_PARAMETER, LICENCE_SPELLING, + VALUES_PARAMETER) # The path, below the base URL, of the data endpoint that takes no # resource key in its route, used when the engine authenticates on a diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py index 786ae020..09f01bfe 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py @@ -307,6 +307,88 @@ def test_reserved_evidence_is_left_out_of_the_body(self): for message in messages: self.assertNotIn("another-key", message) + def send_with_reserved_evidence(self, settings): + """ + Process one flow data holding evidence named resource, license, licence + and values in mixed case under each prefix, with other evidence beside + it, and give back the body of the data request. + """ + + settings["http_client"] = self.mock_http() + engine = CloudRequestEngine(settings) + # The stand-in service lists query.User-Agent alone, so an element + # that accepts every key keeps the other evidence for the engine. + pipeline = PipelineBuilder().add(AcceptEveryKeyElement()) \ + .add(engine).build() + data = pipeline.create_flowdata() + data.evidence.add("query.Resource", "evidence_resource") + data.evidence.add("query.LICENSE", "evidence_license") + data.evidence.add("query.Licence", "evidence_licence") + data.evidence.add("query.Values", "evidence_values") + data.evidence.add("header.resource", "header_resource") + data.evidence.add("cookie.License", "cookie_license") + data.evidence.add("query.other", "query_other") + data.evidence.add("header.other", "header_other") + data.evidence.add("cookie.cookie-only", "cookie_only") + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + data.process() + + return engine.http_client.requests[-1][2] + + def assert_other_evidence_sent(self, content): + self.assertEqual("query_other", content["other"]) + self.assertEqual("cookie_only", content["cookie-only"]) + for value in content.values(): + self.assertNotIn("evidence_", str(value)) + self.assertNotIn("header_resource", str(value)) + self.assertNotIn("cookie_license", str(value)) + + def test_evidence_never_supplies_resource_or_license(self): + """ + With a Resource Key and a License Key configured, the body carries + those two and none of the evidence named resource, license or + values, in any case and under any prefix. + """ + + content = self.send_with_reserved_evidence({ + "resource_key": RESOURCE_KEY, + "license_key": LICENSE_KEY, + }) + + self.assertEqual(RESOURCE_KEY, content[Constants.RESOURCE_PARAMETER]) + self.assertEqual(LICENSE_KEY, content[Constants.LICENSE_PARAMETER]) + self.assertNotIn(Constants.VALUES_PARAMETER, content) + self.assertEqual( + [], [name for name in content if name.lower() in ( + "resource", "license", "licence", "values") and name not in ( + Constants.RESOURCE_PARAMETER, Constants.LICENSE_PARAMETER)]) + self.assert_other_evidence_sent(content) + + def test_evidence_never_supplies_values(self): + """ + With a License Key and Requested Properties configured, the body + carries the configured list as values and the License Key as + license, and no resource, whatever the evidence holds. + """ + + content = self.send_with_reserved_evidence({ + "license_key": LICENSE_KEY, + "requested_properties": REQUESTED, + }) + + self.assertEqual(LICENSE_KEY, content[Constants.LICENSE_PARAMETER]) + self.assertEqual( + "device.ismobile,device.istablet", + content[Constants.VALUES_PARAMETER]) + self.assertNotIn(Constants.RESOURCE_PARAMETER, content) + self.assertEqual( + [], [name for name in content if name.lower() in ( + "resource", "license", "licence", "values") and name not in ( + Constants.LICENSE_PARAMETER, Constants.VALUES_PARAMETER)]) + self.assert_other_evidence_sent(content) + def test_evidence_no_element_accepts_is_not_sent(self): """ A default pipeline keeps only the evidence one of its elements From 14362b133c3f19ef55d357c3b50a81aca0b04f0b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 19:27:02 +0000 Subject: [PATCH 37/68] Write a WKT value to JSON as an object holding the text, and read it back (#82) * Test that a WKT value is an object in JSON * Write a WKT value to JSON as an object holding the text, and read it back The JSON Builder writes a shape in well-known text as an object with one member value, and the Cloud Engine gives the text from such an object. * Key the WKT test's property list by the names the cloud service gives * Find a WKT property in the cloud property list by its name in any case --- .../cloudengine.py | 12 ++++ .../tests/test_cloudengine_wkt.py | 60 +++++++++++++++++++ .../src/fiftyone_pipeline_core/jsonbundler.py | 8 +++ .../tests/test_jsonbundler_wkt.py | 50 ++++++++++++++++ 4 files changed, 130 insertions(+) create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_wkt.py create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_wkt.py diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py index 79bb5007..18a40a91 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py @@ -24,6 +24,7 @@ BasicListEvidenceKeyFilter from fiftyone_pipeline_engines.engine import Engine from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.jsonbundler import WKT_TYPES from .clouddata import CloudData @@ -133,8 +134,19 @@ def process_internal(self, flowdata): result = {} + # A shape in well-known text arrives as an object holding the text. + # The property list is keyed by the names the cloud service gives, + # such as Areas, and the response by the same names in lower case. + wkt_keys = { + name.lower() for name, meta in self.properties.items() + if str(meta.get("type", "")).lower() in WKT_TYPES} + for key, value in engineData.items(): + if (key.lower() in wkt_keys and isinstance(value, dict) + and list(value) == ["value"]): + value = value["value"] + if key + "nullreason" in engineData: result[key] = AspectPropertyValue(no_value_message=engineData[key + "nullreason"]) else: diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_wkt.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_wkt.py new file mode 100644 index 00000000..6e938d08 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_wkt.py @@ -0,0 +1,60 @@ + +""" +A WKT value in a response from the remote server is an object with one member, +value, holding the WKT text, and the engine gives the text. +""" + +import json +import unittest + +from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +AREA = "POLYGON((1 2, 3 4, 5 6, 1 2))" + + +class FakeCloudRequestEngine(FlowElement): + + def __init__(self, answer): + super(FakeCloudRequestEngine, self).__init__() + self.datakey = "cloud" + self.answer = answer + # Keyed by the names the cloud service gives, as + # CloudRequestEngine.get_engine_properties keys them. + self.flow_element_properties = { + "ip": {"Areas": {"name": "Areas", "type": "WktString"}, + "Name": {"name": "Name", "type": "String"}}} + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary( + self, {"cloud": json.dumps(self.answer)})) + + +def read(answer): + engine = CloudEngine() + engine.datakey = "ip" + pipeline = PipelineBuilder({ + "add_javascript_builder": False, "use_setheader_properties": False}) \ + .add(FakeCloudRequestEngine(answer)).add(engine).build() + flowdata = pipeline.create_flowdata() + flowdata.process() + pipeline.close() + return flowdata.get("ip") + + +class CloudEngineWktTests(unittest.TestCase): + + def test_the_text_is_taken_out_of_the_object(self): + data = read({"ip": {"areas": {"value": AREA}, "name": "a place"}}) + self.assertEqual(AREA, data.get("areas").value()) + self.assertEqual("a place", data.get("name").value()) + + def test_text_that_is_already_plain_is_kept(self): + data = read({"ip": {"areas": AREA, "name": "a place"}}) + self.assertEqual(AREA, data.get("areas").value()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index bb441dd1..be38237c 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -24,6 +24,10 @@ from .flowelement import FlowElement from .elementdata_dictionary import ElementDataDictionary +# The property types that hold a shape in well-known text. The JSON holds such +# a value as an object with one member, value, holding the text. +WKT_TYPES = ("wkt", "wktstring", "wkb", "wkbr") + class JSONBundlerElement(FlowElement): """! @@ -184,6 +188,10 @@ def process_internal(self, flowdata): continue + if (isinstance(value, str) + and str(propertymeta.get("type", "")).lower() in WKT_TYPES): + value = {"value": value} + output[flow_element.datakey.lower()][propertykey.lower()] = value if value is None: output[flow_element.datakey.lower()][propertykey.lower() + "nullreason"] = null_reason diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_wkt.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_wkt.py new file mode 100644 index 00000000..04f0b18f --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_wkt.py @@ -0,0 +1,50 @@ + +""" +A WKT value is written to JSON as an object with one member, value, holding +the WKT text, as the value types page of the specification says. +""" + +import unittest + +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +AREA = "POLYGON((1 2, 3 4, 5 6, 1 2))" + + +class AreasElement(FlowElement): + + def __init__(self, type_name): + super(AreasElement, self).__init__() + self.datakey = "areas" + self.properties = { + "areas": {"name": "Areas", "type": type_name}, + "name": {"name": "Name", "type": "string"}} + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary( + self, {"Areas": AREA, "Name": "a place"})) + + +def json_of(type_name): + pipeline = PipelineBuilder({}).add(AreasElement(type_name)).build() + flowdata = pipeline.create_flowdata() + flowdata.process() + pipeline.close() + return flowdata.jsonbundler.json["areas"] + + +class JsonBuilderWktTests(unittest.TestCase): + + def test_a_wkt_value_is_an_object_holding_the_text(self): + for type_name in ("WktString", "wkb", "WKBR", "wkt"): + self.assertEqual( + {"value": AREA}, json_of(type_name)["areas"], type_name) + + def test_other_values_are_written_as_they_are(self): + self.assertEqual("a place", json_of("WktString")["name"]) + + +if __name__ == "__main__": + unittest.main() From 353c579a8f89e58eeeae97219bc84bfb7553c05f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 19:31:33 +0000 Subject: [PATCH 38/68] Build a Cloud Aspect Engine whose product the key lacks, and log it once (#74) * Test that a Cloud Aspect Engine with no product builds and logs once * Build a Cloud Aspect Engine whose product the key lacks, and log it once The engine builds with empty Property metadata and logs the missing product once, where it raised and failed the Pipeline build. The Location engine's own check that raised is removed. * Test that the missing product is logged at warning level * Log a Cloud Aspect Engine's missing product at warning level, as the specification says --- .../src/fiftyone_location/location_cloud.py | 9 -- .../cloudengine.py | 14 ++- .../tests/test_cloudengine_missing_product.py | 97 +++++++++++++++++++ 3 files changed, 106 insertions(+), 14 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_product.py diff --git a/location/src/fiftyone_location/location_cloud.py b/location/src/fiftyone_location/location_cloud.py index ec25062d..596a1b21 100644 --- a/location/src/fiftyone_location/location_cloud.py +++ b/location/src/fiftyone_location/location_cloud.py @@ -44,12 +44,3 @@ def __init__(self, location_provider="fiftyonedegrees"): self.datakey = "location_digitalelement" else: raise Exception("The location provider " + location_provider + " was not recognized.") - - def on_registration(self, pipeline): - available_properties = pipeline.flow_elements_list.get( - "cloud", {}).flow_element_properties - if "location" not in available_properties: - raise Exception("Location data was not available. " - "Check that this key is authorised " - "for geolocation data") - return super().on_registration(pipeline) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py index 18a40a91..46a8cb78 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py @@ -83,11 +83,15 @@ def on_registration(self, pipeline): # The accessible properties come from the cloud service for the # key in use, a resource key or a licence key, so a section missing - # from them means that key does not carry the product at all. - if not self.datakey in pipeline.flow_elements_list["cloud"].flow_element_properties: - raise Exception("The key in use does not include access to any properties under the engine with key " + self.datakey + " that was added to the pipeline. For more details on resource keys, see our explainer: https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-cloudengine.py&utm_term=engine-properties-not-accessible " + "Available engine data keys are: " + str([e for e in pipeline.flow_elements_list["cloud"].flow_element_properties])) - - self.properties = pipeline.flow_elements_list["cloud"].flow_element_properties[self.datakey] + # from them means that key does not carry the product at all. That + # is not a configuration error, so the engine is built with no + # properties and the missing product is logged here, once. + carried = pipeline.flow_elements_list["cloud"].flow_element_properties + if not self.datakey in carried: + pipeline.log("warning", "The key in use does not include access to any properties under the engine with key " + self.datakey + " that was added to the pipeline. For more details on resource keys, see our explainer: https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-cloudengine.py&utm_term=engine-properties-not-accessible " + "Available engine data keys are: " + str([e for e in carried])) + self.properties = {} + else: + self.properties = carried[self.datakey] # Add a special message to the pipeline # when requesting an element that does not exist diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_product.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_product.py new file mode 100644 index 00000000..8604ea23 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_product.py @@ -0,0 +1,97 @@ + +""" +A Cloud Aspect Engine whose product the key does not include builds with +empty Property metadata, logs the missing product once, and logs nothing +about it on each response, as the Cloud Aspect Engine specification says. +""" + +import json +import unittest + +from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.logger import Logger +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + + +class RecordingLogger(Logger): + + def __init__(self): + super(RecordingLogger, self).__init__("debug") + self.logged = [] + + def log_internal(self, level, log): + self.logged.append((level, log["message"])) + + +class FakeCloudRequestEngine(FlowElement): + """Stands in for the Cloud Request Engine of a key that carries only the + device product.""" + + def __init__(self): + super(FakeCloudRequestEngine, self).__init__() + self.datakey = "cloud" + self.flow_element_properties = { + "device": {"ismobile": {"name": "IsMobile", "type": "Boolean"}}} + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary( + self, {"cloud": json.dumps({"device": {"ismobile": True}})})) + + +def location_engine(): + engine = CloudEngine() + engine.datakey = "location" + return engine + + +class MissingProductTests(unittest.TestCase): + + def build(self): + logger = RecordingLogger() + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + builder.add_logger(logger) + engine = location_engine() + pipeline = builder.add(FakeCloudRequestEngine()).add(engine).build() + self.addCleanup(pipeline.close) + return pipeline, engine, logger + + def testPipelineBuildsWithEmptyMetadata(self): + pipeline, engine, logger = self.build() + self.assertEqual({}, engine.get_properties()) + + def testMissingProductIsLoggedOnce(self): + pipeline, engine, logger = self.build() + missing = [entry for entry in logger.logged + if "location" in entry[1]] + self.assertEqual(1, len(missing), logger.logged) + # The specification asks for the warning level. + self.assertEqual("warning", missing[0][0]) + + def testNothingIsLoggedOnEachResponse(self): + pipeline, engine, logger = self.build() + before = len(logger.logged) + for _ in range(3): + flowdata = pipeline.create_flowdata() + flowdata.process() + self.assertEqual(0, len(flowdata.errors)) + self.assertEqual({}, flowdata.get("location").as_dictionary()) + self.assertEqual(before, len(logger.logged), logger.logged) + + def testEngineWithItsProductStillBuildsWithItsProperties(self): + logger = RecordingLogger() + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + builder.add_logger(logger) + engine = CloudEngine() + engine.datakey = "device" + pipeline = builder.add(FakeCloudRequestEngine()).add(engine).build() + self.addCleanup(pipeline.close) + self.assertEqual(["ismobile"], list(engine.get_properties())) + self.assertEqual([], logger.logged) + + +if __name__ == "__main__": + unittest.main() From 7864c7a3c3f847db2ea61a44f6e8828b6ed40c3e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 19:40:01 +0000 Subject: [PATCH 39/68] Refuse the Window property names the specification lists as the object name (#73) * Test that the object name refuses the Window properties the specification lists * Refuse the Window property names the specification lists as the object name * Say in the object name comment why eval and arguments are refused --- .../javascriptbuilder.py | 20 ++++- .../test_javascriptbuilder_object_name.py | 75 +++++++++++++++++++ 2 files changed, 91 insertions(+), 4 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py index cb96e9c8..59bdf9bd 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py @@ -60,9 +60,12 @@ # Names the object cannot have. These are the reserved words of the language, # including those reserved only in strict mode and the literals null, true -# and false, plus the three global values a top level var cannot replace, so -# the object would silently never be created, and the name of the constructor -# the script itself defines. +# and false. Then eval and arguments, which strict mode code cannot declare, +# and eval would also replace the page's own eval function. Then the three +# global values and the properties of the browser's Window object that a +# top level var cannot replace, so the object would never be created, and +# location, which would navigate the page away. Last, the name of the +# constructor the script itself defines. RESERVED_WORDS = frozenset([ "await", "break", "case", "catch", "class", "const", "continue", "debugger", "default", "delete", "do", "else", "enum", "export", @@ -71,7 +74,16 @@ "package", "private", "protected", "public", "return", "static", "super", "switch", "this", "throw", "true", "try", "typeof", "var", "void", "while", "with", "yield", - "Infinity", "NaN", "undefined", "fiftyoneDegreesManager"]) + "Infinity", "NaN", "undefined", + "eval", "arguments", + "window", "document", "top", "name", "status", "history", "navigator", + "closed", "location", "frameElement", "isSecureContext", + "crossOriginIsolated", "trustedTypes", "crypto", "indexedDB", + "localStorage", "sessionStorage", "caches", "cookieStore", + "speechSynthesis", "styleMedia", "crashReport", "credentialless", + "customElements", "documentPictureInPicture", "fence", "launchQueue", + "originAgentCluster", + "fiftyoneDegreesManager"]) def is_valid_object_name(name): diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py index 8342461f..c6e2e400 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py @@ -49,6 +49,24 @@ from .classes.memorylogger import MemoryLogger from .test_javascriptbuilder import TestEngine +# The names the specification refuses that are not reserved words of the +# language, being eval, arguments, and the properties of the browser's Window +# object that a top level var cannot replace or that navigate the page away. +REFUSED_NAMES = [ + "eval", "arguments", "window", "document", "top", "name", "status", + "history", "navigator", "closed", "location", "frameElement", + "isSecureContext", "crossOriginIsolated", "trustedTypes", "crypto", + "indexedDB", "localStorage", "sessionStorage", "caches", "cookieStore", + "speechSynthesis", "styleMedia", "crashReport", "credentialless", + "customElements", "documentPictureInPicture", "fence", "launchQueue", + "originAgentCluster"] + +# Window properties that a top level var can replace, so the names stay +# allowed, and near misses of refused names. +ALLOWED_NAMES = [ + "self", "frames", "parent", "opener", "length", "windows", "Window", + "cachesX"] + NODE = shutil.which("node") HARNESS = os.path.join(os.path.dirname(__file__), "object_name_harness.js") @@ -205,6 +223,50 @@ def test_invalid_name_from_evidence_is_ignored( if check_text: self.assertNotIn(name, warnings[0]) + @parameterized.expand([[name] for name in REFUSED_NAMES]) + def test_refused_window_name_from_evidence_is_ignored(self, name): + """A requested name the specification refuses is not declared. The + configured name is used, the script parses and runs, and a warning + is logged.""" + + logger = MemoryLogger("warning") + script = _render( + {"minify": False}, + {Constants.EVIDENCE_OBJECT_NAME: name}, + logger) + + self.assert_uses_name(script, "fod") + self.assert_parses(script) + + result = self.run_script(script, "fod") + self.assertIsNone(result["error"]) + self.assertTrue(result["exists"]) + self.assertEqual(["fod"], result["globals"]) + + warnings = [ + entry["message"] for entry in logger.memory_log + if entry["level"] == "warning" + and Constants.EVIDENCE_OBJECT_NAME in entry["message"]] + self.assertEqual(1, len(warnings), str(logger.memory_log)) + + @parameterized.expand([[name] for name in ALLOWED_NAMES]) + def test_allowed_window_name_from_evidence_is_used(self, name): + """A name the specification allows is declared and the script + parses.""" + + logger = MemoryLogger("warning") + script = _render( + {"minify": False}, + {Constants.EVIDENCE_OBJECT_NAME: name}, + logger) + + self.assert_uses_name(script, name) + self.assert_parses(script) + self.assertEqual([], [ + entry for entry in logger.memory_log + if entry["level"] == "warning" + and Constants.EVIDENCE_OBJECT_NAME in entry["message"]]) + def test_invalid_name_from_evidence_uses_configured_name(self): """The fallback is the configured name, not always 'fod'.""" @@ -239,6 +301,19 @@ def test_invalid_configured_name_is_refused(self, _, name): with self.assertRaises(ValueError): JavascriptBuilderElement({"obj_name": name}) + @parameterized.expand([[name] for name in REFUSED_NAMES]) + def test_refused_window_name_is_refused(self, name): + """A name the specification refuses is refused when the builder is + created.""" + + with self.assertRaises(ValueError): + JavascriptBuilderElement({"obj_name": name}) + + @parameterized.expand([[name] for name in ALLOWED_NAMES]) + def test_allowed_window_name_is_accepted(self, name): + element = JavascriptBuilderElement({"obj_name": name}) + self.assertEqual(name, element.settings["_objName"]) + @parameterized.expand([ ["plain", "myFod"], ["underscore", "_fod"], From 23d692762531a08cf283f1ff628f19290e42d17e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 20:22:36 +0000 Subject: [PATCH 40/68] Fail the JSON Builder when no Sequence Element has produced a sequence number (#76) * Test that the JSON Builder needs a sequence number * Fail the JSON Builder when no Sequence Element has produced a sequence number * Build the JavaScript script tests on a pipeline with a Sequence Element The JSON Builder fails without a sequence number, so a pipeline without a Sequence Element no longer reaches the script. The builder's reading of text values stays covered by the get_sequence tests. * Test that each on-premise web example's configuration gives the JSON Builder a sequence number * Add the Sequence Element to the User-Agent Client Hints web example's configuration * Keep the caller's integer sequence in the JSON Builder tests with an element that accepts it, now that no default element does --- .../useragentclienthints_web/config.json | 4 + .../tests/test_web_example_configs.py | 63 +++++++++++ .../src/fiftyone_pipeline_core/jsonbundler.py | 14 ++- .../tests/test_javascriptbuilder_sequence.py | 93 +++------------- .../test_jsonbundler_sequence_required.py | 100 ++++++++++++++++++ 5 files changed, 191 insertions(+), 83 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_examples/tests/test_web_example_configs.py create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_sequence_required.py diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/config.json b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/config.json index b355a2e4..d0ceed13 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/config.json +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/config.json @@ -11,6 +11,10 @@ "licence_keys": "" } }, + { + "elementName": "SequenceElement", + "elementPath": "fiftyone_pipeline_core.sequenceelement" + }, { "elementName": "JSONBundlerElement", "elementPath": "fiftyone_pipeline_core.jsonbundler" diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_web_example_configs.py b/device-detection/fiftyone_devicedetection_examples/tests/test_web_example_configs.py new file mode 100644 index 00000000..b49ead9d --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_web_example_configs.py @@ -0,0 +1,63 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The on-premise web examples build their pipelines from config.json, which +must place a Sequence Element before the JSON Builder, or the JSON Builder +fails on every request. Only the pipeline core elements of each file are +built, so no data file is needed.""" + +import json +import unittest +from pathlib import Path + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +EXAMPLES = Path(__file__).resolve().parent.parent.joinpath( + "src", "fiftyone_devicedetection_examples", "onpremise") +WEB_EXAMPLES = ("gettingstarted_web", "useragentclienthints_web") +CORE = "fiftyone_pipeline_core." + + +class WebExampleConfigTests(unittest.TestCase): + + def test_json_builder_has_a_sequence_number(self): + for example in WEB_EXAMPLES: + with self.subTest(example): + config = json.loads( + EXAMPLES.joinpath(example, "config.json").read_text()) + options = config["PipelineOptions"] + options["Elements"] = [ + element for element in options["Elements"] + if element["elementPath"].startswith(CORE)] + pipeline = PipelineBuilder().build_from_configuration(config) + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "a browser") + flowdata.process() + + self.assertEqual({}, flowdata.errors) + self.assertIn( + "json", flowdata.get("jsonbundler").as_dictionary()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index be38237c..37362a02 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -71,6 +71,16 @@ def process_internal(self, flowdata): """ + # The sequence number is the one the Sequence Element has put in the + # Evidence as a whole number. Text in the query string does not + # count, and without a number no JSON is set. + sequence = flowdata.evidence.get("query.sequence") + if isinstance(sequence, bool) or not isinstance(sequence, int): + raise Exception( + "The JSON Builder needs the sequence number that a Sequence " + "Element produces. A Sequence Element needs to be placed " + "before it in the Pipeline.") + # Get every property on every FlowElement # Storing JavaScript properties in an extra section @@ -196,9 +206,7 @@ def process_internal(self, flowdata): if value is None: output[flow_element.datakey.lower()][propertykey.lower() + "nullreason"] = null_reason - sequence = flowdata.evidence.get("query.sequence") - - if sequence is None or sequence < 10: + if sequence < 10: if "type" in propertymeta and propertymeta["type"].lower() == "javascript": diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py index 2e42c300..db057848 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py @@ -41,25 +41,19 @@ from parameterized import parameterized from fiftyone_pipeline_core import javascriptbuilder -from fiftyone_pipeline_core.javascriptbuilder import JavascriptBuilderElement -from fiftyone_pipeline_core.jsonbundler import JSONBundlerElement from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder NODE = shutil.which("node") -def _render(evidence=None, minify=False, sequence_element=False): - """Render the script. With sequence_element False the pipeline has only - the JSON bundler and the JavaScript builder.""" +def _render(evidence=None, minify=False, sequence_element=True): + """Render the script. The JSON builder fails without a sequence number, + so the pipeline always has the SequenceElement.""" - settings = {"javascript_builder_settings": {"minify": minify}} - if not sequence_element: - settings["add_javascript_builder"] = False - builder = PipelineBuilder(settings) if not sequence_element: - builder.add(JSONBundlerElement()) - builder.add(JavascriptBuilderElement({"minify": minify})) - pipeline = builder.build() + raise ValueError("The JSON builder needs the SequenceElement") + settings = {"javascript_builder_settings": {"minify": minify}} + pipeline = PipelineBuilder(settings).build() flowdata = pipeline.create_flowdata() for key, value in (evidence or {}).items(): flowdata.evidence.add(key, value) @@ -92,54 +86,15 @@ def assert_parses(self, script): "the rendered script does not parse: " + result.stderr) @parameterized.expand([["unminified", False], ["minified", True]]) - def test_script_parses_without_sequence_element(self, _, minify): - """With no SequenceElement and no query evidence the script is - rendered with sequence 1 and an empty session id, and parses.""" + def test_script_parses(self, _, minify): + """With no query evidence the script is rendered with sequence 1, and + parses.""" script = _render(minify=minify) self.assert_parses(script) if not minify: self.assertEqual(["var sequence = 1;"], _line(script, "sequence")) - self.assertEqual( - ['var sessionId = "";'], _line(script, "sessionId")) - - @parameterized.expand([ - ["text", "abc"], - ["negative", "-1"], - ["zero", "0"], - ["too_large", "99999999999"], - ["empty", ""], - ["statement", "1;b"], - ["decimal", "1.5"], - ["underscore", "1_0"], - ["just_too_large", "2147483648"], - ["file_separator", chr(0x1C) + "5"], - ["group_separator", chr(0x1D) + "5"], - ["record_separator", chr(0x1E) + "5"], - ["unit_separator", chr(0x1F) + "5"], - ["trailing_file_separator", "5" + chr(0x1C)], - ["null", chr(0) + "5"], - ["no_break_space", chr(0xA0) + "5"], - ["line_separator", chr(0x2028) + "5"], - ["arabic_indic_digit", chr(0x665)], - ["full_width_digit", chr(0xFF15)], - ["five_thousand_digits", "9" * 5000], - ["plus_alone", "+"], - ["spaces_alone", " "], - ["zeros", "0000"], - ]) - def test_unusable_sequence_from_query_becomes_one(self, _, value): - """A query.sequence that is not a positive 32 bit integer is rendered - as 1, as the pipeline specification says, so the script parses and - nothing else is written in its place.""" - - script = _render({ - "query.session-id": "abc", - "query.sequence": value}) - - self.assertEqual(["var sequence = 1;"], _line(script, "sequence")) - self.assert_parses(script) @parameterized.expand([ ["text", "abc"], @@ -171,30 +126,8 @@ def test_unusable_sequence_with_sequence_element_becomes_one( ['var sessionId = "abc";'], _line(script, "sessionId")) self.assert_parses(script) - @parameterized.expand([ - ["text", "7", 7], - ["spaces", " 7 ", 7], - ["number", 12, 12], - ["largest", "2147483647", 2147483647], - ]) - def test_usable_sequence_from_query_is_rendered(self, _, value, expected): - script = _render({ - "query.session-id": "abc", - "query.sequence": value}) - - self.assertEqual( - ["var sequence = " + str(expected) + ";"], - _line(script, "sequence")) - self.assert_parses(script) - - @parameterized.expand([ - ["without_sequence_element", False], - ["with_sequence_element", True], - ]) - def test_session_id_from_query_is_rendered(self, _, sequence_element): - script = _render( - {"query.session-id": "abc-123"}, - sequence_element=sequence_element) + def test_session_id_from_query_is_rendered(self): + script = _render({"query.session-id": "abc-123"}) self.assertEqual( ['var sessionId = "abc-123";'], _line(script, "sessionId")) @@ -223,7 +156,7 @@ def test_longest_session_id_is_rendered(self): ["space", "a b", False], ["new_line", "ab\n", False], ] - for element in [False, True] + for element in [True] ]) def test_unsafe_session_id_is_rendered_empty( self, _, value, check_text, sequence_element): @@ -331,7 +264,7 @@ def test_any_sequence_value_reaches_the_script(self): chr(0x1F) + "5", "5" + chr(0x1C), chr(0) + "5", "9" * 5000, "+", " "] for value in values: - for element in [False, True]: + for element in [True]: with self.subTest(value=repr(value), element=element): script = _render( {"query.session-id": "abc", diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_sequence_required.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_sequence_required.py new file mode 100644 index 00000000..8cc8058b --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_sequence_required.py @@ -0,0 +1,100 @@ + +""" +The JSON Builder fails with an error, and sets no JSON, when no Sequence +Element has produced a sequence number, as the JSON Builder specification +says. +""" + +import unittest + +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.jsonbundler import JSONBundlerElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + + +class KeepsTheSequence(FlowElement): + """Accepts query.sequence and does nothing else, as an element of the + caller's that keeps the integer it adds would. The JSON Builder accepts + no evidence, so without such an element the pipeline sets the value + aside.""" + + def __init__(self): + super(KeepsTheSequence, self).__init__() + self.datakey = "keepsthesequence" + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(["query.sequence"]) + + def process_internal(self, flowdata): + pass + + +def bundler_only(suppress=False, keep_sequence=False): + builder = PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False, + "suppress_process_exceptions": suppress}) + if keep_sequence: + builder.add(KeepsTheSequence()) + return builder.add(JSONBundlerElement()).build() + + +class JsonBuilderSequenceTests(unittest.TestCase): + + def test_no_sequence_number_is_an_error_that_names_the_sequence_element( + self): + pipeline = bundler_only() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + with self.assertRaises(Exception) as caught: + flowdata.process() + self.assertIn("Sequence Element", str(caught.exception)) + + def test_no_json_is_set_when_the_error_is_suppressed(self): + pipeline = bundler_only(suppress=True) + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.process() + self.assertEqual(1, len(flowdata.errors)) + self.assertNotIn("jsonbundler", flowdata.data) + + def test_a_sequence_number_from_the_query_string_does_not_count(self): + for value in ("3", "", "abc", True, 3.0): + pipeline = bundler_only(suppress=True) + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.sequence", value) + flowdata.process() + self.assertEqual(1, len(flowdata.errors), repr(value)) + self.assertNotIn("jsonbundler", flowdata.data, repr(value)) + + def test_an_integer_sequence_number_in_the_evidence_counts(self): + pipeline = bundler_only(keep_sequence=True) + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.sequence", 3) + flowdata.process() + self.assertIn("javascriptProperties", flowdata.jsonbundler.json) + + def test_the_default_pipeline_has_a_sequence_number(self): + pipeline = PipelineBuilder({}).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.process() + self.assertEqual(0, len(flowdata.errors)) + self.assertIn("javascriptProperties", flowdata.jsonbundler.json) + + def test_a_sequence_of_ten_or_more_leaves_out_the_javascript_properties( + self): + pipeline = bundler_only(keep_sequence=True) + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.sequence", 10) + flowdata.process() + self.assertEqual([], flowdata.jsonbundler.json["javascriptProperties"]) + + +if __name__ == "__main__": + unittest.main() From fc68b1ebcb6b8da5891a2d56610198496620c672 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 20:27:47 +0000 Subject: [PATCH 41/68] Write a list of records in the JSON Builder (#85) * Test that the JSON Builder writes a list of records * Write a list of records in the JSON Builder A Property whose value is a list of Element Data is written as a list of objects. A Property with no value is null beside its nullreason, and a null that was not assigned no value has no entry. The list branch no longer replaces the output with its own list, and an empty list is written as an empty list. * Test that a top-level null not assigned no value has no entry in the JSON Builder * Leave a top-level null not assigned no value out of the JSON, as the specification says * Test that a null JavaScript Property is still listed and a no value with no message has a text reason * List a null JavaScript Property, and give a no value with no message the reason Unknown --- .../src/fiftyone_pipeline_core/jsonbundler.py | 67 ++++++++++--- .../tests/test_jsonbundler_records.py | 95 +++++++++++++++++++ 2 files changed, 150 insertions(+), 12 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index 37362a02..030cba4a 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -61,6 +61,37 @@ def get_evidence_key_filter(self): return BasicListEvidenceKeyFilter([]) + @staticmethod + def record_json(record): + + """! + A record, being the Properties of one Element Data in a list, as an + object. A Property that has been assigned no value is null with its + nullreason beside it, and a Property that is null without one has no + entry. + @type record: dict + @param record: Property name to value or aspect property value + @rtype: dict + @return: The record as it is written to JSON + """ + + written = {} + + for key, value in record.items(): + key = str(key).lower() + + if hasattr(value, "has_value"): + if value.has_value(): + written[key] = value.value() + else: + written[key] = None + written[key + "nullreason"] = ( + value.no_value_message() or "Unknown") + elif value is not None: + written[key] = value + + return written + def process_internal(self, flowdata): """! @@ -144,6 +175,7 @@ def process_internal(self, flowdata): for propertykey, propertymeta in properties.items(): value = None null_reason = "Unknown" + no_value = False # Check if property has delayed execution and set in JSON if yes @@ -172,21 +204,28 @@ def process_internal(self, flowdata): value = value_container.value() else: value = None - null_reason = value_container.no_value_message() + no_value = True + null_reason = ( + value_container.no_value_message() + or null_reason) - # Check if list of aspect property values + # A list of records, such as the profiles of a hardware + # profile lookup, or of aspect property values - elif isinstance(value_container, list) and isinstance(value_container[0], object): + elif isinstance(value_container, list): - output = [] + items = [] for item in value_container: - if item.has_value(): - output.append(item.value()) + if isinstance(item, dict): + items.append(self.record_json(item)) + elif hasattr(item, "has_value"): + if item.has_value(): + items.append(item.value()) else: - null_reason = item.no_value_message + items.append(item) - value = output + value = items else: @@ -197,14 +236,18 @@ def process_internal(self, flowdata): # Catching missing property exceptions and other errors continue - + if (isinstance(value, str) and str(propertymeta.get("type", "")).lower() in WKT_TYPES): value = {"value": value} - output[flow_element.datakey.lower()][propertykey.lower()] = value - if value is None: - output[flow_element.datakey.lower()][propertykey.lower() + "nullreason"] = null_reason + # A null that was not assigned 'no value', which an element + # that is not an engine can give, has no entry of its own. A + # JavaScript Property is still listed below. + if value is not None or no_value: + output[flow_element.datakey.lower()][propertykey.lower()] = value + if value is None: + output[flow_element.datakey.lower()][propertykey.lower() + "nullreason"] = null_reason if sequence < 10: diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py new file mode 100644 index 00000000..e6ff5cf0 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py @@ -0,0 +1,95 @@ + +""" +A Property whose value is a list of Element Data is written as a list of +objects, each following the null rules of the JSON Builder specification, and +the other Properties are written beside it. +""" + +import unittest + +from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + + +class ProfilesElement(FlowElement): + + def __init__(self): + super(ProfilesElement, self).__init__() + self.datakey = "hardware" + self.properties = { + "profiles": {"name": "Profiles", "type": "array"}, + "vendor": {"name": "Vendor", "type": "string"}, + "model": {"name": "Model", "type": "string"}, + "colour": {"name": "Colour", "type": "string"}, + "script": {"name": "Script", "type": "javascript"}, + "size": {"name": "Size", "type": "string"}} + + def process_internal(self, flowdata): + records = [ + {"hardwarename": AspectPropertyValue(value="Phone"), + "hardwarevendor": AspectPropertyValue(no_value_message="No data"), + "hardwaremodel": None}, + {"hardwarename": AspectPropertyValue(value="Tablet")}, + {"hardwarename": AspectPropertyValue()}] + flowdata.set_element_data(ElementDataDictionary( + self, {"profiles": records, + "vendor": AspectPropertyValue(value="Acme"), + "model": None, + "colour": AspectPropertyValue( + no_value_message="No colour"), + "script": None, + "size": AspectPropertyValue()})) + + +class JsonBuilderRecordTests(unittest.TestCase): + + def process_json(self): + pipeline = PipelineBuilder({}).add(ProfilesElement()).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.process() + return flowdata.jsonbundler.json + + def process(self): + return self.process_json()["hardware"] + + def test_each_record_is_an_object(self): + self.assertEqual( + [{"hardwarename": "Phone", + "hardwarevendor": None, + "hardwarevendornullreason": "No data"}, + {"hardwarename": "Tablet"}, + {"hardwarename": None, "hardwarenamenullreason": "Unknown"}], + self.process()["profiles"]) + + def test_a_null_that_was_not_assigned_no_value_has_no_entry(self): + self.assertNotIn("hardwaremodel", self.process()["profiles"][0]) + + def test_the_other_properties_are_still_written(self): + self.assertEqual("Acme", self.process()["vendor"]) + + def test_a_top_level_null_not_assigned_no_value_has_no_entry(self): + hardware = self.process() + self.assertNotIn("model", hardware) + self.assertNotIn("modelnullreason", hardware) + + def test_a_null_javascript_property_is_still_listed(self): + json = self.process_json() + self.assertNotIn("script", json["hardware"]) + self.assertIn("hardware.script", json["javascriptProperties"]) + + def test_no_value_with_no_message_has_a_text_reason(self): + hardware = self.process() + self.assertIsNone(hardware["size"]) + self.assertEqual("Unknown", hardware["sizenullreason"]) + + def test_a_top_level_property_assigned_no_value_is_null(self): + hardware = self.process() + self.assertIsNone(hardware["colour"]) + self.assertEqual("No colour", hardware["colournullreason"]) + + +if __name__ == "__main__": + unittest.main() From 993566b96fd65b69643746cd37b4f2a9cfb3cdc7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 10 Nov 2026 20:34:33 +0000 Subject: [PATCH 42/68] Mark a usage sharing value with invalid characters as escaped, as the specification says (#71) * Test the usage sharing record attributes * Mark a value with invalid characters as escaped in the usage sharing record * Expect escaped in the usage sharing invalid character test * Test the usage sharing record escapes and limit * Write invalid characters as backslash x and four hex digits, and cut values at 10,000 characters A character XML 1.0 does not allow is written as its code in four lower-case hexadecimal digits after a backslash and x, and the element is marked escaped. A value is cut to 10,000 characters (code points) before the markup characters are written, and the element is marked truncated. * Test that escapes do not count towards the usage sharing length limit * Test that the session id, client IP and names leave the usage record parseable, and the date and warning * Write the session id, client IP and Name attribute through the record's escaping, leave out entries with no XML name, and give the date in UTC * Import re apart from the imports the back-off draft adds * Test that the session id and client IP carry no attributes * Write the session id and client IP with no escaped or truncated attribute, as the record gives them none --- .../share_usage.py | 147 +++++++++------ .../tests/test_share_usage_record.py | 172 ++++++++++++++++++ .../tests/test_usage.py | 13 +- 3 files changed, 267 insertions(+), 65 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index ec2a657a..d0a1f913 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -21,6 +21,7 @@ # ********************************************************************* import random +import re import json import gzip import platform @@ -44,7 +45,7 @@ # The maximum length of a piece of evidence's value which can be # added to the usage data being sent. -SHARE_USAGE_MAX_EVIDENCE_LENGTH = 512 +SHARE_USAGE_MAX_EVIDENCE_LENGTH = 10000 SHARE_USAGE_VERSION = '1.1' @@ -212,7 +213,7 @@ def get_flow_elements(self): # make up the pipeline so a warning is logged # but otherwise, the system can continue as normal. self._log( - "warn", + "warning", "Share usage element registered " f"to {'too many' if len(self.pipelines) > 0 else 'no'}" " pipelines. Unable to send share usage information.") @@ -299,40 +300,30 @@ def add_to_share_usage(self, data): # --- write variable data # The SessionID used to track a series of requests - xml += f"{data.session_id}" + xml += plain_element("SessionId", data.session_id) # The sequence number of the request in a series of requests. xml += f"{data.sequence}" # The client IP of the request - xml += f"{data.client_ip}" + xml += plain_element("ClientIP", data.client_ip) - # The UTC date/time this entry was written - date = datetime.datetime.now().isoformat() + # The UTC date and time this entry was written, to the second. + date = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%S") xml += f"{date}" - # Write all other evidence data that has been included. + # Write all other evidence data that has been included. The prefix, + # or a key with no prefix, is the element name. An entry whose + # element name is not an XML name is left out, because it would make + # the whole batch unreadable. for category_key, category_value in data.evidence_data.items(): for entry_key, entry_value in category_value.items(): - replaced_string = ReplacedString(entry_value) - # Write start element if len(category_key) > 0: - xml += f"<{category_key} Name=\"{entry_key}\"" - else: - xml += f"<{entry_key}" - # Write any attributes - if replaced_string.replaced: - xml += ' replaced="true"' - if replaced_string.truncated: - xml += ' truncated="true"' - # End the start element - xml += ">" - # Write the value - xml += replaced_string.result - # Write end element - if len(category_key) > 0: - xml += f"" - else: - xml += f"" - + if XML_NAME.match(category_key): + xml += record_element( + category_key, entry_value, entry_key) + elif XML_NAME.match(entry_key): + xml += record_element(entry_key, entry_value) + xml += "" self.share_data.append(xml) @@ -399,49 +390,91 @@ def get_data_from_evidence(self, flow_data): return data -# a set of valid XML character values (ignoring valid controls x09, x0a, x0d, x85) -VALID_XML_CHARS = [*range(int("0x20", 16), int("0x7F", 16))] -VALID_XML_CHARS.extend(range(int("0x40", 16), int("0x100", 16))) +# The characters that XML 1.0 section 2.2 allows, as ranges of code points. +VALID_XML_RANGES = ( + (0x9, 0xA), (0xD, 0xD), (0x20, 0xD7FF), (0xE000, 0xFFFD), + (0x10000, 0x10FFFF)) + +# The characters that are written as entities in element text. +XML_ENTITIES = { + '&': '&', + '"': '"', + "'": ''', + '<': '<', + '>': '>', +} + -def get_is_valid_char_map(): - maxChar = int('0x100', 16) - isValidChar = {} - for c in range(maxChar): - isValidChar[c] = c in VALID_XML_CHARS - return isValidChar +def is_valid_xml_char(code): + return any(low <= code <= high for low, high in VALID_XML_RANGES) -# an array describing whether a character value is valid -IS_VALID_XML_CHAR = get_is_valid_char_map() + +# The characters XML 1.0 allows in a name, without the colon, which names a +# namespace. +XML_NAME_START = ( + "A-Z_a-z\u00c0-\u00d6\u00d8-\u00f6\u00f8-\u02ff\u0370-\u037d" + "\u037f-\u1fff\u200c-\u200d\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff" + "\uf900-\ufdcf\ufdf0-\ufffd\U00010000-\U000effff") +XML_NAME = re.compile( + "[" + XML_NAME_START + "][" + XML_NAME_START + + "\\-.0-9\u00b7\u0300-\u036f\u203f-\u2040]*\\Z") + + +def plain_element(tag, value): + """! + An element of a usage sharing record that is not an Evidence entry, with + its value made to fit the XML by ReplacedString and no attributes, as + the record gives these elements none. + """ + return "<" + tag + ">" + ReplacedString(value).result + "" + + +def record_element(tag, value, name=None): + """! + One element of a usage sharing record. The value, and the Name attribute + when there is one, are made to fit the XML by ReplacedString, and the + element is marked escaped or truncated when either changed. + """ + replaced = ReplacedString(value) + escaped = replaced.escaped + xml = "<" + tag + if name is not None: + replaced_name = ReplacedString(name) + escaped = escaped or replaced_name.escaped + xml += ' Name="' + replaced_name.result + '"' + if escaped: + xml += ' escaped="true"' + if replaced.truncated: + xml += ' truncated="true"' + return xml + ">" + replaced.result + "" class ReplacedString: """! - replace characters that cause problems in XML with the "Replacement character" + Makes an Evidence value fit in the usage sharing XML. The value is cut to + SHARE_USAGE_MAX_EVIDENCE_LENGTH characters (code points, so a character is + never split), a character XML 1.0 does not allow is written as a backslash, + "x" and its code in four lower-case hexadecimal digits, and the markup + characters are written as entities. `escaped` and `truncated` say what + changed. """ def __init__(self, text): self.result = "" - self.replaced = False + self.escaped = False self.truncated = False if text: - escapedText = text\ - .replace('&', '&')\ - .replace('"', '"')\ - .replace('\'', ''')\ - .replace('<', '<')\ - .replace('>', '>') - - length = min(len(escapedText), SHARE_USAGE_MAX_EVIDENCE_LENGTH) - self.truncated = len(escapedText) > SHARE_USAGE_MAX_EVIDENCE_LENGTH - self.result = "".join(map(self.map_char, list(escapedText[0:length]))) - + if not isinstance(text, str): + text = str(text) + self.truncated = len(text) > SHARE_USAGE_MAX_EVIDENCE_LENGTH + self.result = "".join( + map(self.map_char, text[0:SHARE_USAGE_MAX_EVIDENCE_LENGTH])) def map_char(self, char): - charV = ord(char) - if charV < len(IS_VALID_XML_CHAR) and IS_VALID_XML_CHAR[charV]: - return char - else: - self.replaced = True - return chr(int("0xFFFD", 16)) + code = ord(char) + if not is_valid_xml_char(code): + self.escaped = True + return "\\x%04x" % code + return XML_ENTITIES.get(char, char) class ShareUsageData: """! diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py new file mode 100644 index 00000000..93992946 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py @@ -0,0 +1,172 @@ + +""" +How the usage sharing element writes an Evidence value that was changed, as +the Processing section of the usage sharing element specification says, +including its invalid character and length rules. +""" + +import re +import unittest +import xml.etree.ElementTree as ElementTree + +from fiftyone_pipeline_core.logger import Logger +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines_fiftyone.share_usage import ( + SHARE_USAGE_MAX_EVIDENCE_LENGTH, ShareUsage, ShareUsageData) + +BS = chr(92) + + +class ShareUsageRecordTests(unittest.TestCase): + + def record(self, value, key="header.user-agent", session_id="session", + client_ip=""): + element = ShareUsage(requested_package_size=1000) + self.addCleanup(element.close) + element.constant_xml = "test" + data = ShareUsageData() + data.session_id = session_id + data.client_ip = client_ip + data.sequence = 1 + data.try_add(key, value) + element.add_to_share_usage(data) + return element.share_data[0] + + def test_session_id_is_written_so_the_record_parses(self): + session_id = 'a"): + xml = self.record("a value", key=key) + device = ElementTree.fromstring(xml) + self.assertNotIn("a value", xml, key) + self.assertEqual("session", device.find("SessionId").text, key) + + def test_date_sent_is_utc_to_the_second(self): + device = ElementTree.fromstring(self.record("Mozilla/5.0")) + self.assertRegex( + device.find("DateSent").text, + re.compile(r"\A\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\Z")) + + def test_registered_to_too_many_pipelines_logs_a_warning(self): + logged = [] + + class Recorder(Logger): + def log_internal(self, level, log): + logged.append(level) + + element = ShareUsage(requested_package_size=1000) + self.addCleanup(element.close) + pipelines = [] + for _ in range(2): + builder = PipelineBuilder({"add_javascript_builder": False, + "use_setheader_properties": False}) + builder.add_logger(Recorder("debug")) + pipelines.append(builder.build()) + element.pipelines = pipelines + self.assertEqual([], element.get_flow_elements()) + self.assertEqual(["warning", "warning"], logged) + + def test_limit_is_ten_thousand(self): + self.assertEqual(10000, SHARE_USAGE_MAX_EVIDENCE_LENGTH) + + def test_unchanged_value_has_no_attributes(self): + xml = self.record("Mozilla/5.0") + self.assertIn('
    Mozilla/5.0
    ', xml) + + def test_invalid_character_is_written_as_hex_and_marked_escaped(self): + xml = self.record("a" + chr(0x1f) + "b") + self.assertIn( + '
    a' + BS + 'x001fb
    ', + xml) + + def test_other_invalid_characters(self): + for code, text in ((0x0, "0000"), (0x8, "0008"), (0xb, "000b"), + (0xc, "000c"), (0xe, "000e"), (0xd800, "d800"), + (0xdfff, "dfff"), (0xfffe, "fffe"), + (0xffff, "ffff")): + xml = self.record("x" + chr(code) + "y") + self.assertIn("x" + BS + "x" + text + "y", xml, hex(code)) + self.assertIn(' escaped="true"', xml, hex(code)) + + def test_characters_xml_allows_are_kept(self): + value = "t" + chr(9) + chr(10) + chr(13) + chr(0x18c) + chr(0xfffd) + xml = self.record(value) + self.assertIn(">" + value + "<", xml) + self.assertNotIn("escaped", xml) + + def test_markup_characters_are_still_entities(self): + xml = self.record("a&bd\"e'") + self.assertIn(">a&b<c>d"e'<", xml) + self.assertNotIn("escaped", xml) + + def test_value_of_the_limit_is_not_truncated(self): + xml = self.record("a" * 10000) + self.assertIn(">" + "a" * 10000 + "<", xml) + self.assertNotIn("truncated", xml) + + def test_longer_value_is_cut_and_marked_truncated(self): + xml = self.record("a" * 10001) + self.assertIn( + ' truncated="true">' + "a" * 10000 + "
    ", xml) + + def test_length_is_counted_before_markup_characters_are_written(self): + xml = self.record("&" * 10000) + self.assertNotIn("truncated", xml) + + def test_escapes_do_not_count_towards_the_limit(self): + xml = self.record(chr(1) + "a" * 9999) + self.assertIn(' escaped="true"', xml) + self.assertNotIn("truncated", xml) + self.assertIn(chr(92) + "x0001" + "a" * 9999 + "<", xml) + + def test_character_outside_the_basic_plane_is_never_split(self): + emoji = chr(0x1f600) + xml = self.record("a" * 9999 + emoji + "tail") + self.assertIn( + ' truncated="true">' + "a" * 9999 + emoji + "
    ", xml) + xml = self.record("a" * 9998 + emoji + emoji + "z") + self.assertIn( + ' truncated="true">' + "a" * 9998 + emoji + emoji + "", + xml) + self.assertNotIn("escaped", xml) + + def test_invalid_character_in_a_long_value_sets_both(self): + xml = self.record(chr(1) + "a" * 10000) + self.assertIn(' escaped="true"', xml) + self.assertIn(' truncated="true"', xml) + + def test_changed_value_is_written_without_cdata_or_bad_schema(self): + xml = self.record("a" + chr(1) + "b") + self.assertNotIn("CDATA", xml) + self.assertNotIn("BadSchema", xml) + self.assertNotIn("replaced", xml) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index 4438b8d3..66e9b4e5 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -304,20 +304,17 @@ def test_invalid_char(self): requested_package_size=1, endpoint=testEndpoint) - replacementChar = chr(int("0xFFFD", 16)) - pipeline = (PipelineBuilder())\ .add(usage)\ .build() flowdata = pipeline.create_flowdata() - flowdata.evidence.add("header.user-agent", "1ƌ2") + flowdata.evidence.add("header.user-agent", "1" + chr(0x1f) + "2") flowdata.process() self.waitForUsageThreads() self.assertEqual(len(app.received), 1) - self.assertNotIn("ƌ", app.received[0]) - self.assertIn(replacementChar, app.received[0]) - self.assertIn(f"
    1{replacementChar}2
    ", app.received[0]) + self.assertNotIn(chr(0x1f), app.received[0]) + self.assertIn('
    1\\x001f2
    ', app.received[0]) def test_truncated_value(self): """! @@ -328,7 +325,7 @@ def test_truncated_value(self): requested_package_size=1, endpoint=testEndpoint) - ua = "X" * 1000 + ua = "X" * 10001 pipeline = (PipelineBuilder())\ .add(usage)\ @@ -341,7 +338,7 @@ def test_truncated_value(self): self.assertEqual(len(app.received), 1) self.assertIn("truncated=\"true\"", app.received[0]) self.assertNotIn(ua, app.received[0]) - self.assertIn(f'
    {"X" * 512}
    ', app.received[0]) + self.assertIn(f'
    {"X" * 10000}
    ', app.received[0]) def test_send_more_than_once(self): """! From cb7cdc2da1e6e69329e179315b55563ccb39a9c0 Mon Sep 17 00:00:00 2001 From: Automation51D <60396166+Automation51D@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:40:33 +0100 Subject: [PATCH 43/68] Merged Pull Request '#96 update-submodules/rc->rc: Update submodules' Co-authored-by: Automation51D <51DCI@51Degrees.com> --- cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cxx b/cxx index cfad8978..c6d4e4d9 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit cfad8978827292b97ad08f306891edea09ece304 +Subproject commit c6d4e4d907e8da83845c095e80374280f8db29c5 From 66ace716d4f7dad1f8da9a1599d3896ed5197ba1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 08:56:53 +0000 Subject: [PATCH 44/68] Read each on-premise property by its index in the build-time names (#93) * Test that Element Data reads each property by its build-time index * Test with the native engines that a property keeps its build-time index, and time a read by name and by index * Read each on-premise property by its index in the build-time names, and build an engine given no names with the data set's names * Make the index table from the build-time names, count reads by set, and time building with every name * Register each engine before the native tests read it, and close each flow data they make * Test that an engine built with names keeps each build-time index, and that an engine built with no names reads by name as before The tests cover an engine built with a list of names in an order that differs from the data set's, and a list that is a subset of what the data set holds, which is the superset case. Each name keeps its place in the list given, and reads the same value as an engine built with every name. A read by name, a read by index and the typed read are checked to give the same value, and their times are printed. An engine built with no names is checked to have an empty table and to keep reading by name, which is what it does today. * Keep an engine built with no property names reading by name An engine built with no names was built with every name in the data set, read first from a low memory engine, so that its properties had fixed indexes too. Processing was measured at 3 to 7 times slower that way, because an engine holding every name walks each component's graph during process where one built with no names walked a graph only when a property of that component was first read. The fastest approach is always used, so that part is removed. An engine built with no names is built exactly as it is today, its table of names to indexes is empty, and every read of its properties goes by name as before. Nothing about it changes. An engine built with a list of property names still reads every property by its index in that list. That half is unchanged and is where the gain is. The no-names half follows once the native change is in and 51degrees-cxx has had its second release. * Test that restricted property names ignore case, and give the superset test the names CI has A restricted property list given as the data set spells the names, for example IsMobile, must still let flowdata.device.ismobile be read, as access-to-results.md says string comparisons must be case insensitive. The new core test gives "Boolean" and reads "boolean", and checks that a name outside the list is refused in any case. The IP Intelligence superset test asked for more than 6 names and the data file in CI lists 3. It now builds with every other name in reverse order, so the data set still holds a name the engine was not given, that name sits between two given ones, and each given name's index differs from its place in the data set. * Compare restricted property names ignoring case AspectData.get refused a property whose name was in the engine's restricted list in another case, so an engine built with "IsMobile" refused flowdata.device.ismobile with "Property ismobile was excluded from device". access-to-results.md says string comparisons must be case insensitive. set_restricted_properties now also keeps the names in lower case, and AspectData.get compares the key in lower case against them. The error still names the key as the caller gave it. --- .../devicedetection_onpremise.py | 25 +++ .../swig_data.py | 18 +- .../tests/test_read_by_index.py | 186 ++++++++++++++++++ .../tests/test_swig_data_index.py | 87 ++++++++ .../ipintelligence_onpremise.py | 25 +++ .../swig_data.py | 32 +-- .../tests/test_read_by_index.py | 148 ++++++++++++++ .../tests/test_swig_data_index.py | 86 ++++++++ .../fiftyone_pipeline_engines/aspectdata.py | 4 +- .../src/fiftyone_pipeline_engines/engine.py | 5 +- .../tests/test_engines.py | 25 +++ 11 files changed, 618 insertions(+), 23 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_index.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_index.py diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index 5f3d168c..3a1a9af7 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -180,6 +180,10 @@ def __init__( properties_list = RequiredPropertiesConfigSwig(restricted_properties_vector) + # The property names the engine is built with. A property's index is + # its place in this list, which is fixed at build. + self.build_property_names = list(restricted_properties_vector) + # Assemble configuration config = ConfigHashSwig() @@ -299,6 +303,12 @@ def init_engine(self): self.engine = engine + # The table from each build-time name to its index, made once here + # so every read goes by index. An engine built with no names has no + # build-time list, so the table is empty and every read goes by + # name, which is the native behaviour it has today. + self.property_indexes = self.read_property_indexes() + # Get keys and add to evidenceKey filter evidence_keys = engine.getKeys() @@ -334,6 +344,21 @@ def init_engine(self): for key, property in self.get_metric_properties().items(): self.properties[key] = property + def read_property_indexes(self): + + """! + The index of each property the engine was built with, keyed by its + name in lower case. The native code gives a property the place of + its name among the names the engine was built with, the first place + where a name is given twice. An engine built with no names has an + empty table, so every read of its properties goes by name. + """ + + indexes = {} + for index, name in enumerate(self.build_property_names): + indexes.setdefault(name.lower(), index) + return indexes + def get_metric_properties(self): return { "deviceid": { "name": 'DeviceId', diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index 17987a59..1c5d19a0 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -135,23 +135,29 @@ def get_internal(self, key): value = None + # Read by the property's index from the engine's build-time + # table. A name outside the table is read by name, which gives + # the native no value message. + target = getattr(self.flow_element, "property_indexes", {}).get( + key, this_property["name"]) + if this_property["type"] == "bool": - value = self.swig_results.getValueAsBool(this_property["name"]) + value = self.swig_results.getValueAsBool(target) if this_property["type"] == "string": - value = self.swig_results.getValueAsString(this_property["name"]) + value = self.swig_results.getValueAsString(target) if this_property["type"] == "javascript": - value = self.swig_results.getValueAsString(this_property["name"]) + value = self.swig_results.getValueAsString(target) if this_property["type"] == "int": - value = self.swig_results.getValueAsInteger(this_property["name"]) + value = self.swig_results.getValueAsInteger(target) if this_property["type"] == "double": - value = self.swig_results.getValueAsDouble(this_property["name"]) + value = self.swig_results.getValueAsDouble(target) if this_property["type"] == "string[]": - value = self.swig_results.getValues(this_property["name"]) + value = self.swig_results.getValues(target) # Check for value / no value message diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py new file mode 100644 index 00000000..a4d38f84 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py @@ -0,0 +1,186 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""An engine built with a list of property names reads each one by its +index in that list, with the native engine and the Lite data file. Also +prints how long a read by name and a read by index take. An engine built +with no names keeps reading by name until the native change lands.""" + +import time +import unittest +import warnings + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_devicedetection_onpremise.devicedetection_onpremise import \ + DeviceDetectionOnPremise + +data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" +mobile_ua = ("Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) " + "AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile" + "/11D167 Safari/9537.53") +# Reads timed for the figures in the warnings. +TIMED_READS = 20000 + + +def shown(value): + """What a caller sees of a value, whether or not it has one.""" + if value.has_value(): + return ("value", value.value()) + return ("no value", value.no_value_message()) + + +def process(engine): + pipeline = PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}).add(engine).build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", mobile_ua) + flowdata.process() + return pipeline, flowdata + + +def data_set_names(engine): + """The names the data set holds, in the data set's own order. Only the + tests use this order, to build engines whose name lists differ from it. + """ + metadata = engine.engine.getMetaData().getProperties() + return [metadata.getByIndex(x).getName() + for x in range(metadata.getSize()) + if metadata.getByIndex(x).getAvailable()] + + +class ReadByIndexTests(unittest.TestCase): + + def names_from_the_data_set(self): + """The property names the data set holds, read from an engine built + with no names, which still lists them all.""" + engine = DeviceDetectionOnPremise( + data_file_path=data_file, licence_keys="", auto_update=False) + pipeline, flowdata = process(engine) + self.addCleanup(pipeline.close) + self.addCleanup(flowdata.close) + return data_set_names(engine), flowdata + + def test_an_engine_with_no_names_keeps_reading_by_name(self): + """An engine built with no names has an empty table, so nothing + about it changes until the native change lands.""" + engine = DeviceDetectionOnPremise( + data_file_path=data_file, licence_keys="", auto_update=False) + pipeline, flowdata = process(engine) + self.addCleanup(pipeline.close) + self.addCleanup(flowdata.close) + self.assertEqual([], engine.build_property_names) + self.assertEqual({}, engine.property_indexes) + # Reading still works, by name, exactly as before. + self.assertTrue(flowdata.device.ismobile.value()) + + def test_the_table_follows_the_order_the_names_were_given_in(self): + names, _ = self.names_from_the_data_set() + given = list(reversed(names[:6])) + engine = DeviceDetectionOnPremise( + data_file_path=data_file, licence_keys="", auto_update=False, + restricted_properties=list(given)) + pipeline, flowdata = process(engine) + self.addCleanup(pipeline.close) + self.addCleanup(flowdata.close) + # The index of each name is its place in the list given, not the + # data set's order. + for index, name in enumerate(given): + self.assertEqual(index, engine.property_indexes[name.lower()], + name) + + def test_names_in_another_order_read_the_same_values(self): + names, all_data = self.names_from_the_data_set() + chosen = names[:6] + reversed_engine = DeviceDetectionOnPremise( + data_file_path=data_file, licence_keys="", auto_update=False, + restricted_properties=list(reversed(chosen))) + second, reversed_data = process(reversed_engine) + self.addCleanup(second.close) + self.addCleanup(reversed_data.close) + for name in chosen: + self.assertEqual( + shown(all_data.device.get(name)), + shown(reversed_data.device.get(name)), name) + + def test_a_superset_data_set_keeps_each_build_time_index(self): + """The engine is built with a short list of names and the data set + holds many more, which is the superset case. Each name keeps its + place in the list given, and reads the value the data set holds.""" + names, all_data = self.names_from_the_data_set() + self.assertGreater(len(names), 6, "data set must hold a superset") + chosen = list(reversed(names[:6])) + engine = DeviceDetectionOnPremise( + data_file_path=data_file, licence_keys="", auto_update=False, + restricted_properties=list(chosen)) + pipeline, flowdata = process(engine) + self.addCleanup(pipeline.close) + self.addCleanup(flowdata.close) + for index, name in enumerate(chosen): + self.assertEqual(index, engine.property_indexes[name.lower()], + name) + self.assertEqual( + shown(all_data.device.get(name)), + shown(flowdata.device.get(name)), name) + + def test_a_read_by_name_and_by_index_give_the_same_value_and_time(self): + """A read by name, a read by index and the typed read all give the + same value, with the time each read takes for the figures.""" + names, _ = self.names_from_the_data_set() + engine = DeviceDetectionOnPremise( + data_file_path=data_file, licence_keys="", auto_update=False, + restricted_properties=list(names)) + pipeline, flowdata = process(engine) + self.addCleanup(pipeline.close) + self.addCleanup(flowdata.close) + results = flowdata.device.swig_results + index = engine.property_indexes["ismobile"] + self.assertEqual( + results.getValueAsBool("IsMobile").getValue(), + results.getValueAsBool(index).getValue()) + self.assertEqual( + results.getValueAsBool("IsMobile").getValue(), + flowdata.device.ismobile.value()) + + start = time.perf_counter() + for _ in range(TIMED_READS): + results.getValueAsBool("IsMobile") + by_name = time.perf_counter() - start + start = time.perf_counter() + for _ in range(TIMED_READS): + results.getValueAsBool(index) + by_index = time.perf_counter() - start + # The typed read is the whole path a caller takes, through the + # table and on to the native getter. + start = time.perf_counter() + for _ in range(TIMED_READS): + flowdata.device.get("ismobile") + typed = time.perf_counter() - start + warnings.warn( + f"IsMobile read {TIMED_READS} times, native by name " + f"{by_name * 1e9 / TIMED_READS:.0f} ns a read, native by index " + f"{by_index * 1e9 / TIMED_READS:.0f} ns a read, typed read " + f"{typed * 1e9 / TIMED_READS:.0f} ns a read") + + +if __name__ == "__main__": + unittest.main() diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_index.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_index.py new file mode 100644 index 00000000..a8fb7ea0 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig_data_index.py @@ -0,0 +1,87 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Element Data reads each property by its index in the list of names the +engine was built with, through the engine's table, so the same index names +the same property after a reload.""" + +import unittest + +from fiftyone_devicedetection_onpremise.swig_data import SwigData + + +class Value: + + def __init__(self, value): + self.value = value + + def hasValue(self): + return True + + def getValue(self): + return self.value + + +class RecordingResults: + """Stands in for the native results and records what each read asks for.""" + + def __init__(self): + self.asked = [] + + def read(self, target): + self.asked.append(target) + return Value(True) + + getValueAsBool = read + getValueAsString = read + getValueAsInteger = read + + +class FakeElement: + datakey = "device" + properties = { + "ismobile": {"name": "IsMobile", "type": "bool"}, + "browsername": {"name": "BrowserName", "type": "string"}, + "notbuilt": {"name": "NotBuilt", "type": "int"}} + # The build-time names were BrowserName then IsMobile. + property_indexes = {"browsername": 0, "ismobile": 1} + + +class SwigDataIndexTests(unittest.TestCase): + + def test_each_read_goes_by_the_build_time_index(self): + results = RecordingResults() + data = SwigData(FakeElement(), results) + self.assertTrue(data.get("IsMobile").value()) + data.get("browsername") + # Each read asks the native results once or more, always by the + # index. + self.assertEqual({1, 0}, set(results.asked)) + + def test_a_name_outside_the_table_is_read_by_name(self): + results = RecordingResults() + SwigData(FakeElement(), results).get("notbuilt") + self.assertEqual({"NotBuilt"}, set(results.asked)) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index 9f15784e..fc707307 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -149,6 +149,10 @@ def __init__( properties_list = RequiredPropertiesConfigSwig(restricted_properties_vector) + # The property names the engine is built with. A property's index is + # its place in this list, which is fixed at build. + self.build_property_names = list(restricted_properties_vector) + # Assemble configuration config = ConfigIpiSwig() @@ -240,6 +244,12 @@ def init_engine(self): self.engine = engine + # The table from each build-time name to its index, made once here + # so every read goes by index. An engine built with no names has no + # build-time list, so the table is empty and every read goes by + # name, which is the native behaviour it has today. + self.property_indexes = self.read_property_indexes() + # Get keys and add to evidenceKey filter evidence_keys = engine.getKeys() @@ -275,6 +285,21 @@ def init_engine(self): for metric_property_name, metric_property in self.get_metric_properties().items(): self.properties[metric_property_name] = metric_property + def read_property_indexes(self): + + """! + The index of each property the engine was built with, keyed by its + name in lower case. The native code gives a property the place of + its name among the names the engine was built with, the first place + where a name is given twice. An engine built with no names has an + empty table, so every read of its properties goes by name. + """ + + indexes = {} + for index, name in enumerate(self.build_property_names): + indexes.setdefault(name.lower(), index) + return indexes + def get_metric_properties(self): """! diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 40413cf3..50c53f98 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -215,31 +215,35 @@ def get_internal(self, key): if this_property: results = self.swig_results - name = this_property["name"] + # Read by the property's index from the engine's build-time + # table. A name outside the table is read by name, which gives + # the native no value message. + target = getattr(self.flow_element, "property_indexes", {}).get( + key, this_property["name"]) property_type = this_property["type"].lower() # Mcc is weighted even though its metadata type is "string" # (ip-intelligence-dotnet parity). - if property_type == "string" and name.lower() == "mcc": + if property_type == "string" and key == "mcc": property_type = "weightedstring" if property_type in ("weightedstring", "weightedwkbr", "weightedwkb"): return self.weighted_list_to_value( - results.getValuesAsWeightedUTF8StringList(name), + results.getValuesAsWeightedUTF8StringList(target), convert=self.utf8_string_to_str) if property_type == "weightedint": return self.weighted_list_to_value( - results.getValuesAsWeightedIntegerList(name)) + results.getValuesAsWeightedIntegerList(target)) if property_type in ("weighteddouble", "weightedsingle"): return self.weighted_list_to_value( - results.getValuesAsWeightedDoubleList(name)) + results.getValuesAsWeightedDoubleList(target)) if property_type == "weightedbool": return self.weighted_list_to_value( - results.getValuesAsWeightedBoolList(name)) + results.getValuesAsWeightedBoolList(target)) if property_type in ("wkb", "wkbr"): # WKB geometry properties (e.g. Areas) are exposed as their @@ -247,35 +251,35 @@ def get_internal(self, key): # matching the .NET binding. The weighted-WKT list getter # (getValuesAsWeightedWKTStringList) is not used as it is not # safely exposed through the Python SWIG wrapper. - return self.single_value(results.getValueAsString(name)) + return self.single_value(results.getValueAsString(target)) if property_type == "ipaddress": # IP address properties (e.g. IpRangeStart / IpRangeEnd) are # read as their string form. This matches the .NET binding, # which reads IP properties via getValueAsString rather than # the lower-level getValueAsIpAddress accessor. - return self.single_value(results.getValueAsString(name)) + return self.single_value(results.getValueAsString(target)) if property_type in ("string", "javascript"): return self.single_value( - results.getValueAsUTF8String(name), + results.getValueAsUTF8String(target), convert=self.utf8_string_to_str) if property_type == "int": - return self.single_value(results.getValueAsInteger(name)) + return self.single_value(results.getValueAsInteger(target)) if property_type in ("double", "single"): - return self.single_value(results.getValueAsDouble(name)) + return self.single_value(results.getValueAsDouble(target)) if property_type == "bool": - return self.single_value(results.getValueAsBool(name)) + return self.single_value(results.getValueAsBool(target)) if property_type == "string[]": - value = results.getValues(name) + value = results.getValues(target) if value.hasValue(): return AspectPropertyValue(value=list(value.getValue())) return AspectPropertyValue(no_value_message=value.getNoValueMessage()) # Unknown type, fall back to string - return self.single_value(results.getValueAsString(name)) + return self.single_value(results.getValueAsString(target)) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py new file mode 100644 index 00000000..1bb785d3 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py @@ -0,0 +1,148 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""An engine built with a list of property names reads each one by its +index in that list, with the native engine and the data file the tests use. +An engine built with no names keeps reading by name until the native +change lands.""" + +import os +import unittest + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise import \ + IpIntelligenceOnPremise + +from .constants import TestConstants + +data_file = TestConstants.get_data_file_path() + + +def shown(value): + """What a caller sees of a value, whether or not it has one.""" + if value.has_value(): + return ("value", value.value()) + return ("no value", value.no_value_message()) + + +def process(engine): + pipeline = PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}).add(engine).build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.client-ip", "8.8.8.8") + flowdata.process() + return pipeline, flowdata + + +def data_set_names(engine): + """The names the data set holds, in the data set's own order. Only the + tests use this order, to build engines whose name lists differ from it. + """ + metadata = engine.engine.getMetaData().getProperties() + return [metadata.getByIndex(x).getName() + for x in range(metadata.getSize()) + if metadata.getByIndex(x).getAvailable()] + + +@unittest.skipUnless( + os.path.isfile(data_file), + "No IP intelligence data file found at '%s'." % data_file) +class ReadByIndexTests(unittest.TestCase): + + def names_from_the_data_set(self): + """The property names the data set holds, read from an engine built + with no names, which still lists them all.""" + engine = IpIntelligenceOnPremise( + data_file_path=data_file, licence_keys="") + pipeline, flowdata = process(engine) + self.addCleanup(pipeline.close) + self.addCleanup(flowdata.close) + return data_set_names(engine), flowdata + + def test_an_engine_with_no_names_keeps_reading_by_name(self): + """An engine built with no names has an empty table, so nothing + about it changes until the native change lands.""" + engine = IpIntelligenceOnPremise( + data_file_path=data_file, licence_keys="") + pipeline, flowdata = process(engine) + self.addCleanup(pipeline.close) + self.addCleanup(flowdata.close) + self.assertEqual([], engine.build_property_names) + self.assertEqual({}, engine.property_indexes) + # Reading still works, by name, exactly as before. + self.assertTrue(flowdata.ip.get(data_set_names(engine)[0])) + + def test_the_table_follows_the_order_the_names_were_given_in(self): + names, _ = self.names_from_the_data_set() + given = list(reversed(names[:6])) + engine = IpIntelligenceOnPremise( + data_file_path=data_file, licence_keys="", + restricted_properties=list(given)) + pipeline, flowdata = process(engine) + self.addCleanup(pipeline.close) + self.addCleanup(flowdata.close) + # The index of each name is its place in the list given, not the + # data set's order. + for index, name in enumerate(given): + self.assertEqual(index, engine.property_indexes[name.lower()], + name) + + def test_names_in_another_order_read_the_same_values(self): + names, all_data = self.names_from_the_data_set() + chosen = names[:6] + reversed_engine = IpIntelligenceOnPremise( + data_file_path=data_file, licence_keys="", + restricted_properties=list(reversed(chosen))) + second, reversed_data = process(reversed_engine) + self.addCleanup(second.close) + self.addCleanup(reversed_data.close) + for name in chosen: + self.assertEqual( + shown(all_data.ip.get(name)), + shown(reversed_data.ip.get(name)), name) + + def test_a_superset_data_set_keeps_each_build_time_index(self): + """The engine is built with every other name, in reverse order, so + the data set holds names the engine was not given, which is the + superset case, and the second name sits between two given ones. + Each name keeps its place in the list given, and reads the value the + data set holds.""" + names, all_data = self.names_from_the_data_set() + self.assertGreaterEqual(len(names), 3, "data set must hold a superset") + chosen = list(reversed(names[::2])) + engine = IpIntelligenceOnPremise( + data_file_path=data_file, licence_keys="", + restricted_properties=list(chosen)) + pipeline, flowdata = process(engine) + self.addCleanup(pipeline.close) + self.addCleanup(flowdata.close) + for index, name in enumerate(chosen): + self.assertEqual(index, engine.property_indexes[name.lower()], + name) + self.assertEqual( + shown(all_data.ip.get(name)), + shown(flowdata.ip.get(name)), name) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_index.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_index.py new file mode 100644 index 00000000..31afa244 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_index.py @@ -0,0 +1,86 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Element Data reads each property by its index in the list of names the +engine was built with, through the engine's table, so the same index names +the same property after a reload.""" + +import unittest + +from fiftyone_ipintelligence_onpremise.swig_data import SwigData + + +class Value: + + def __init__(self, value): + self.value = value + + def hasValue(self): + return True + + def getValue(self): + return self.value + + +class RecordingResults: + """Stands in for the native results and records what each read asks for.""" + + def __init__(self): + self.asked = [] + + def read(self, target): + self.asked.append(target) + return Value(True) + + getValueAsBool = read + getValueAsInteger = read + + +class FakeElement: + datakey = "ip" + properties = { + "isproxy": {"name": "IsProxy", "type": "bool"}, + "accuracyradius": {"name": "AccuracyRadius", "type": "int"}, + "notbuilt": {"name": "NotBuilt", "type": "int"}} + # The build-time names were AccuracyRadius then IsProxy. + property_indexes = {"accuracyradius": 0, "isproxy": 1} + + +class SwigDataIndexTests(unittest.TestCase): + + def test_each_read_goes_by_the_build_time_index(self): + results = RecordingResults() + data = SwigData(FakeElement(), results) + self.assertTrue(data.get("IsProxy").value()) + data.get("accuracyradius") + # Each read asks the native results once or more, always by the + # index. + self.assertEqual({1, 0}, set(results.asked)) + + def test_a_name_outside_the_table_is_read_by_name(self): + results = RecordingResults() + SwigData(FakeElement(), results).get("notbuilt") + self.assertEqual({"NotBuilt"}, set(results.asked)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py index 98c731c9..bb85f9bf 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/aspectdata.py @@ -62,9 +62,9 @@ def get(self, key): @return The value stored under the chosen key """ - if hasattr(self.flow_element, "restricted_properties"): + if hasattr(self.flow_element, "restricted_property_keys"): - if not key in self.flow_element.restricted_properties: + if key.lower() not in self.flow_element.restricted_property_keys: raise Exception("Property " + key + " was excluded from " + self.flow_element.datakey) try: diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py index 5ef3556a..8729ab87 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py @@ -69,7 +69,10 @@ def set_restricted_properties(self, properties_list): """ self.restricted_properties = properties_list - + # Names in lower case, as property names are compared ignoring case. + self.restricted_property_keys = { + name.lower() for name in properties_list} + def in_cache(self, flowdata): diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_engines.py b/pipeline/fiftyone_pipeline_engines/tests/test_engines.py index 39813240..d2441630 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_engines.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_engines.py @@ -113,6 +113,31 @@ def testRestrictedProperty(self): self.assertTrue(result == "Property integer was excluded from example") + # Restricted property names are compared ignoring case + def testRestrictedPropertyIgnoresCase(self): + + engine = ExampleAspectEngine() + + engine.set_restricted_properties(["Boolean"]) + + testPipeline = PipelineBuilder().add(engine).build() + + flowdata = testPipeline.create_flowdata() + + flowdata.process() + + # A name given in another case is in the list. + self.assertTrue(flowdata.example.get("boolean")) + self.assertTrue(flowdata.example.boolean) + + # A name outside the list is refused in any case. + for key in ("integer", "Integer"): + with self.assertRaises(Exception) as refused: + flowdata.example.get(key) + self.assertEqual( + "Property " + key + " was excluded from example", + str(refused.exception)) + # Test restricting properties def testMissingPropertyService(self): From 45ac25a24b08f00e7dc337265e9362903738b5c6 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 10:44:38 +0000 Subject: [PATCH 45/68] Report the client address the native code looks up, chosen as the specification says (#90) * Read the client addresses in their own module, with no change in what is read * Test that an IPv4-mapped address is reported as Ip and an address with spaces is not read * Report an IPv4-mapped address as its IPv4 address, and read an address as given, as the Address forms section says * Test that a value holding a zone index is not read * Refuse a client IP value that holds a zone index, as the Zone indexes section says * Test that an IPv4 address with a leading zero is not read Reading the client IP value says the Engine must not read an IPv4 address with a leading zero in any part, such as 01.2.3.4, and must read 0.0.0.0. ipaddress refuses a leading zero, so the rule holds today, and this test catches a Python whose ipaddress reads one. * Let client_ips be given the engine's client IP keys A structural change only, so the tests that follow can give the keys. Nothing reads the parameter yet. * Test that the client address is chosen as the native code chooses it Choosing the IP address says the native code uses one value, from the client IP keys the data file lists, the query keys first in the data file's order and the server keys only when no query key supplied an address, and that the Property values describe that one address. Reading the client IP value reads a port, which is dropped, and an IPv6 address in brackets. evidence.md says keys are case insensitive. The tests give client_ips the keys as an engine lists them and check each of those, and that only the one address looked up is reported. This commit is the tests alone, so six of them fail. * Choose the client address as the native code does, and report only that one The engine reported any evidence value that was an address, from a fixed pair of keys first and then from any key at all, matched with letter case, and it could not read a port or brackets. So the Ip or IpV6 it reported could differ from the address the native code looked up, which the other Property values describe. As .NET's IpiOnPremiseEngine does, the engine now takes the client IP keys from the native engine's own list, every query key in the data file's order and then every server key, and reads each from the evidence in any letter case. read_client_ip reads a whole value only, being an address, an IPv4 address with a port, or an IPv6 address in brackets with or without a port, and drops the port, as Reading the client IP value says. Only the values read reach the native code, each as its address alone under the engine's spelling of the key, so the first of them is the address the native code looks up, and that one address is reported, in Ip or in IpV6. --- .../client_ip.py | 144 ++++++++++++++++ .../ipintelligence_onpremise.py | 61 +++---- .../tests/test_client_ip.py | 154 ++++++++++++++++++ 3 files changed, 325 insertions(+), 34 deletions(-) create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py new file mode 100644 index 00000000..98f0316c --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/client_ip.py @@ -0,0 +1,144 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import ipaddress +import re + +# The prefixes the native code reads the client IP keys under, in the order +# it reads them. +PREFIXES = ("query.", "server.") + +# An IPv6 address in brackets, with or without a port, and an IPv4 address +# with a port. A port is one or more ASCII digits. +BRACKETED = re.compile(r"\[([^\[\]]*)\](?::([0-9]+))?") +WITH_PORT = re.compile(r"([0-9.]+):([0-9]+)") +MAX_PORT = 65535 + + +def client_ip_keys(keys): + + """! + The client IP evidence keys in the order the native code reads them, + every query key in the order given, then every server key in the + order given. + + @type keys: list + @param keys: the engine's evidence keys, in the data file's order + @rtype: list + """ + + return [key for prefix in PREFIXES for key in keys + if key.lower().startswith(prefix)] + + +def read_client_ip(value): + + """! + The address a client IP value holds, or None when the whole value is + not one address, an IPv4 address with a port, or an IPv6 address in + brackets with or without a port. A port is dropped. An IPv4-mapped IPv6 + address is its IPv4 address. A value holding a zone index is not read. + + @rtype: ipaddress.IPv4Address or ipaddress.IPv6Address or None + """ + + text = str(value) + # ipaddress reads a zone index, such as fe80::1%eth0, and the whole + # value must be refused rather than read with the zone dropped. The + # percent sign only ever starts a zone index here, in the plain and the + # %25 encoded form alike. + if "%" in text: + return None + version = None + port = None + match = BRACKETED.fullmatch(text) + if match: + text, port, version = match.group(1), match.group(2), 6 + else: + match = WITH_PORT.fullmatch(text) + if match: + text, port, version = match.group(1), match.group(2), 4 + if port is not None and int(port) > MAX_PORT: + return None + try: + address = ipaddress.ip_address(text) + except ValueError: + return None + if version is not None and address.version != version: + return None + if address.version == 6 and address.ipv4_mapped is not None: + address = address.ipv4_mapped + return address + + +def readable_client_ips(evidence, keys): + + """! + Each client IP value that is read, as (key, address), in the order the + native code reads the keys. A key is matched with the evidence in any + letter case, and keeps the spelling the engine gives it. + + @type evidence: dict + @param evidence: every evidence key and value + @type keys: list + @param keys: the engine's evidence keys, in the data file's order + @rtype: list + """ + + supplied = {} + for key, value in evidence.items(): + supplied.setdefault(str(key).lower(), value) + found = [] + for key in client_ip_keys(keys): + value = supplied.get(key.lower()) + if value is None: + continue + address = read_client_ip(value) + if address is not None: + found.append((key, address)) + return found + + +def client_ips(evidence, keys): + + """! + The address the engine reports in Ip or IpV6, being the first client IP + value that is read, which is the one the native code looks up, as the + Choosing the IP address and Reading the client IP value sections of the + specification say. A value that is not read does not hide a value under + a key that comes later. + + @type evidence: dict + @param evidence: every evidence key and value + @type keys: list + @param keys: the engine's evidence keys, in the data file's order + @rtype: tuple + @returns: (ipv4 or None, ipv6 or None), with at most one set + """ + + found = readable_client_ips(evidence, keys) + if not found: + return (None, None) + address = found[0][1] + if address.version == 4: + return (str(address), None) + return (None, str(address)) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index fc707307..865506ba 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -21,7 +21,6 @@ # ********************************************************************* from __future__ import absolute_import -import ipaddress import multiprocessing import os.path @@ -33,6 +32,7 @@ from .IpIntelligenceEngineModule import * +from .client_ip import client_ip_keys, client_ips, readable_client_ips from .swig_data import SwigData from .ipintelligence_datafile import IpIntelligenceDataFile @@ -257,9 +257,13 @@ def init_engine(self): evidence_keys_list = [] for x in range(evidence_keys.size()): - evidence_keys_list.append(evidence_keys.__getitem__(x).lower()) + evidence_keys_list.append(evidence_keys.__getitem__(x)) - self.evidence_keys_list = evidence_keys_list + # The client IP keys in the order the native code reads them, with + # the engine's own spelling, which the native code matches. + self.client_ip_keys = client_ip_keys(evidence_keys_list) + + self.evidence_keys_list = [key.lower() for key in evidence_keys_list] # Get properties list @@ -373,39 +377,15 @@ def get_client_ips(self, flow_data): """! - Get the IPv4 and IPv6 addresses from the evidence, using the same - priority order as the other IP intelligence engines: - query.client-ip first, then server.client-ip, then any other - parseable value in the evidence accepted by the engine. + The address the engine reports in Ip or IpV6 for the flow data's + evidence, which is the one the native code looks up. @rtype: tuple - @returns: (ipv4 or None, ipv6 or None) + @returns: (ipv4 or None, ipv6 or None), with at most one set """ - evidence = flow_data.evidence.get_all() - - ordered = [] - for key in ["query.client-ip", "server.client-ip"]: - if key in evidence: - ordered.append(evidence[key]) - for key, value in evidence.items(): - if value not in ordered: - ordered.append(value) - - ipv4 = None - ipv6 = None - for value in ordered: - try: - parsed = ipaddress.ip_address(str(value).strip()) - except ValueError: - continue - if parsed.version == 4 and ipv4 is None: - ipv4 = str(parsed) - elif parsed.version == 6 and ipv6 is None: - ipv6 = str(parsed) - - return (ipv4, ipv6) + return client_ips(flow_data.evidence.get_all(), self.client_ip_keys) def process_internal(self, flow_data): @@ -423,14 +403,27 @@ def process_internal(self, flow_data): if getattr(self, "engine", None) is None: raise Exception(ENGINE_CLOSED) + # Only the client IP values that are read reach the native code, + # each as its address alone, so the first of them is the address + # the native code looks up and the one reported in Ip or IpV6. + found = readable_client_ips( + flow_data.evidence.get_all(), self.client_ip_keys) + evidence = EvidenceIpiSwig() - for key, value in flow_data.evidence.get_all().items(): - evidence.__setitem__(key, value) + for key, address in found: + evidence.__setitem__(key, str(address)) result = self.engine.process(evidence) - ipv4, ipv6 = self.get_client_ips(flow_data) + ipv4 = None + ipv6 = None + if found: + address = found[0][1] + if address.version == 4: + ipv4 = str(address) + else: + ipv6 = str(address) data = SwigData(self, result, echo_ip=ipv4, echo_ipv6=ipv6) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py new file mode 100644 index 00000000..2b02fc8d --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_client_ip.py @@ -0,0 +1,154 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The client addresses the engine reports in Ip and IpV6, as the Choosing +the IP address, Reading the client IP value and Address forms sections of +the IP Intelligence on-premise specification say.""" + +import unittest + +from fiftyone_ipintelligence_onpremise.client_ip import client_ips + + +# The client IP keys as an engine might list them, in the data file's order +# with both prefixes mixed together. +KEYS = ["server.true-client-ip-51d", "query.true-client-ip-51d", + "server.client-ip", "query.client-ip"] + + +class ClientIpTests(unittest.TestCase): + + def test_an_ipv4_mapped_address_is_its_ipv4_address(self): + for text in ("::ffff:8.8.8.8", "::ffff:808:808", + "0:0:0:0:0:ffff:808:808"): + self.assertEqual( + ("8.8.8.8", None), + client_ips({"query.client-ip": text}, KEYS), text) + + def test_ipv4_compatible_and_embedded_addresses_stay_ipv6(self): + self.assertEqual( + (None, "::808:808"), + client_ips({"query.client-ip": "::8.8.8.8"}, KEYS)) + self.assertEqual( + (None, "64:ff9b::808:808"), + client_ips({"query.client-ip": "64:ff9b::8.8.8.8"}, KEYS)) + + def test_an_address_with_spaces_is_not_read(self): + self.assertEqual( + (None, None), client_ips({"query.client-ip": " 8.8.8.8 "}, KEYS)) + + def test_a_zone_index_is_not_read(self): + for text in ("fe80::1%eth0", "fe80::1%1", "[fe80::1%eth0]", + "[fe80::1%25eth0]", "::ffff:8.8.8.8%1", + "::ffff:8.8.8.8%eth0"): + self.assertEqual( + (None, None), + client_ips({"query.client-ip": text}, KEYS), text) + + def test_a_zone_index_does_not_hide_a_later_value(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"query.client-ip": "fe80::1%eth0", + "server.client-ip": "1.2.3.4"}, KEYS)) + + def test_the_client_ip_query_comes_first(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"server.client-ip": "5.6.7.8", + "query.client-ip": "1.2.3.4"}, KEYS)) + + def test_a_leading_zero_is_not_read(self): + # The rule is kept by ipaddress, which refuses a leading zero, so + # this catches a Python that reads one. + for text in ("01.2.3.4", "1.02.3.4", "1.2.3.04", "00.0.0.0", + "::ffff:01.2.3.4"): + self.assertEqual( + (None, None), + client_ips({"query.client-ip": text}, KEYS), text) + self.assertEqual( + ("0.0.0.0", None), client_ips({"query.client-ip": "0.0.0.0"}, KEYS)) + + +class ClientIpChoiceTests(unittest.TestCase): + + def test_only_the_client_ip_keys_are_read(self): + self.assertEqual( + (None, None), + client_ips({"header.x-real-ip": "1.2.3.4"}, KEYS)) + + def test_the_query_keys_come_first_in_the_data_files_order(self): + self.assertEqual( + ("9.9.9.9", None), + client_ips({"server.true-client-ip-51d": "5.6.7.8", + "query.client-ip": "1.2.3.4", + "query.true-client-ip-51d": "9.9.9.9"}, KEYS)) + + def test_the_server_keys_are_read_when_no_query_key_has_an_address(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"server.client-ip": "5.6.7.8", + "server.true-client-ip-51d": "1.2.3.4", + "query.client-ip": "unknown"}, KEYS)) + + def test_the_keys_match_in_any_letter_case(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"Server.Client-IP": "5.6.7.8", + "QUERY.CLIENT-IP": "1.2.3.4"}, KEYS)) + + def test_only_the_address_looked_up_is_reported(self): + self.assertEqual( + ("1.2.3.4", None), + client_ips({"query.client-ip": "1.2.3.4", + "server.client-ip": "2001:db8::1"}, KEYS)) + + def test_a_port_and_brackets_are_read_and_the_port_dropped(self): + for text, expected in ( + ("203.0.113.1:8080", ("203.0.113.1", None)), + ("203.0.113.1:0", ("203.0.113.1", None)), + ("203.0.113.1:65535", ("203.0.113.1", None)), + ("[2001:db8::1]", (None, "2001:db8::1")), + ("[2001:db8::1]:8080", (None, "2001:db8::1")), + ("[::ffff:8.8.8.8]:80", ("8.8.8.8", None))): + self.assertEqual( + expected, client_ips({"query.client-ip": text}, KEYS), text) + + def test_an_ipv6_address_without_brackets_has_no_port(self): + self.assertEqual( + (None, "2001:db8::1:80"), + client_ips({"query.client-ip": "2001:db8::1:80"}, KEYS)) + + def test_the_values_the_specification_does_not_read(self): + for text in ("203.0.113.0/24", "203.0.113.1 x", + "203.0.113.1, 198.51.100.7", "203.0.113.1 ", + "[203.0.113.1]", "[2001:db8::1", "2001:db8::1]:80", + "203.0.113.1:", "203.0.113.1:65536", + "203.0.113.1:８ï¼", "256.0.113.1", + "203.0.113", "2001:db8::1::2", "2001:db8:1:2:3:4:5:6:7", + "2001:db8::12345", "unknown", "", "[fe80::1%eth0]:80"): + self.assertEqual( + (None, None), client_ips({"query.client-ip": text}, KEYS), + text) + + +if __name__ == "__main__": + unittest.main() From ec1e4bcdadbd84815cee7803587000907c42430c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 10:45:53 +0000 Subject: [PATCH 46/68] Rename the data file variable and read every key under one _51DEGREES_ name (#97) * Rename the data file environment variable to _51DEGREES_DD_PATH A POSIX shell cannot set a variable whose name starts with a digit, so 51DEGREES_DD_PATH could not be exported the way the examples described. Every name now starts with an underscore, and the old name is not read anywhere. A new check reads every file git tracks and fails on a name that starts with the digit, run by its own workflow on each pull request. * Fail the environment name check on a fallback name, in any case for a digit-initial name Each variable has one name with nothing read in its place, so the check now also fails on a bare RESOURCE_KEY or LICENSE_KEY read or set as an environment variable, and on a LEGACY_..._ENV_VAR constant, the shape the examples' fallbacks take. This is the check 51degrees-php uses, with the forms Python reads an environment variable by. A digit-initial name is matched in any case, since no tracked file holds one in lower case. The fallback names are matched with case, because the lower case resource_key and license_key are still the one name some examples and tests read. git lists paths unquoted, so a path holding a non-ASCII character is still read, and the check says how many paths it read and which it skipped as not files. This commit is the check alone, before the fallbacks are removed, so it fails. * Fail the environment name check on a bare key name held in a string The 51Did example reads its keys through env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY"), a fallback the read forms do not see. A string holding RESOURCE_KEY or LICENSE_KEY and nothing else is now matched as well, and is only ever such a name. Still the check alone, before the fallbacks are removed, so it fails on 12 names. * Read each key from its _51DEGREES_ name alone, with no fallback Each variable has one name with nothing read in its place. - The Device Detection and cloud request engine examples read the resource key from _51DEGREES_RESOURCE_KEY and the licence key from _51DEGREES_LICENSE_KEY, and no longer fall back to resource_key and license_key. The LEGACY_ constants go with them. - The 51Did example reads _51DEGREES_RESOURCE_KEY and _51DEGREES_LICENSE_KEY, and no longer RESOURCE_KEY and LICENSE_KEY. - The live tests' live_cloud.py, in all five copies, no longer tries resource_key after the two _51DEGREES_ names, and the 51Did client and licence key live tests read _51DEGREES_LICENSE_KEY alone. - KeyUtils.get_named_key in both shared packages, and the IP Intelligence examples' reader, read the name given and no longer its upper case form. common-ci sets only resource_key, so the IP Intelligence and pipeline integration scripts now set _51DEGREES_RESOURCE_KEY from the test key, as the Device Detection one already did. The READMEs and AGENTS.md name the one variable each example reads. * Name the environment name check's job for both of its checks * Make the environment name check's matching a function its tests can call Dot-sourcing the script now defines Find-EnvironmentNameProblem without reading the repository, and the scan leaves out the script's own tests, which name every form it fails on. What the check matches is unchanged. * Test the environment name check, with names in lower case Windows reads an environment variable name in any case, so a lower case old name is still the old name there, and the check must fail on it. The tests give each form the check must find, in upper, lower and mixed case, and the lines it must leave alone, being the _51DEGREES_ names, the builder's resource_key setting, a placeholder and a longer name. The workflow runs them before the check. This commit is the tests alone, so the lower case fallback forms fail. * Match environment names in any case, and read every key under its _51DEGREES_ name Windows reads an environment variable name in any case, so the check now matches the fallback names in any case, as it already did the digit-initial ones. A key name in a string is now matched only as a second name after a _51DEGREES_ one, the shape of a fallback, because the builder setting is also called resource_key. Matching in any case finds the names that had no _51DEGREES_ form at all, so each now reads its _51DEGREES_ name, as 51degrees-node does. - The location examples and tests, the IP Intelligence examples and the cloud request engine's own tests read _51DEGREES_RESOURCE_KEY rather than resource_key. - The two data file update examples read _51DEGREES_LICENSE_KEY rather than license_key, and the tests that run the examples without settings clear that name. - The Device Detection integration script sets _51DEGREES_LICENSE_KEY from the data file licence key, and the location one sets _51DEGREES_RESOURCE_KEY, since common-ci sets only the older names. - Every tox.ini passes the _51DEGREES_ names through in place of resource_key and license_key, which nothing reads now. - The READMEs, AGENTS.md and example comments name the new variables. --- .../workflows/environment-variable-names.yml | 30 +++++ AGENTS.md | 13 +- ci/check-environment-variable-names.Tests.ps1 | 54 ++++++++ ci/check-environment-variable-names.ps1 | 117 ++++++++++++++++++ device-detection/ci/run-integration-tests.ps1 | 13 +- .../tests/live_cloud.py | 5 +- .../fiftyone_devicedetection_cloud/tox.ini | 3 +- .../readme.md | 9 +- .../example_utils.py | 19 +-- .../onpremise/datafileupdate_console.py | 2 +- .../onpremise/gettingstarted_web/app.py | 2 +- .../tests/live_cloud.py | 5 +- .../tests/test_cloudexamples.py | 5 +- .../tests/test_examples_without_settings.py | 2 +- .../fiftyone_devicedetection_examples/tox.ini | 3 +- .../tox.ini | 3 +- .../key_utils.py | 5 +- device-detection/readme.md | 8 +- ip-intelligence/ci/run-integration-tests.ps1 | 4 + .../fiftyone_ipintelligence_cloud/readme.md | 2 +- .../fiftyone_ipintelligence_cloud/tox.ini | 4 +- .../readme.md | 2 +- .../cloud/gettingstarted_web/app.py | 2 +- .../example_utils.py | 7 +- .../onpremise/datafileupdate_console.py | 4 +- .../tests/live_cloud.py | 5 +- .../tests/test_examples_without_settings.py | 2 +- .../fiftyone_ipintelligence_examples/tox.ini | 3 +- .../fiftyone_ipintelligence_onpremise/tox.ini | 4 +- .../key_utils.py | 5 +- .../tests/test_shared.py | 4 +- .../fiftyone_ipintelligence_shared/tox.ini | 4 +- .../tox.ini | 4 +- ip-intelligence/readme.md | 2 +- location/ci/run-integration-tests.ps1 | 4 + location/examples/cloud/gettingstarted.py | 6 +- location/examples/cloud/web.py | 6 +- location/tests/test_examples.py | 44 +++---- location/tests/test_properties51D.py | 6 +- location/tox.ini | 4 +- pipeline/ci/run-integration-tests.ps1 | 4 + .../examples/readme.md | 10 +- .../example_utils.py | 16 +-- .../examples/tests/live_cloud.py | 5 +- .../examples/tox.ini | 2 - .../tests/test_cloudengine.py | 2 +- .../tests/test_licensekey_live.py | 10 +- .../tox.ini | 3 +- pipeline/fiftyone_pipeline_core/tox.ini | 4 +- .../examples/creator_context_web/server.py | 26 ++-- pipeline/fiftyone_pipeline_did/readme.md | 4 +- .../fiftyone_pipeline_did/tests/live_cloud.py | 5 +- .../tests/test_did_client_live.py | 5 +- pipeline/fiftyone_pipeline_did/tox.ini | 2 - pipeline/fiftyone_pipeline_engines/tox.ini | 4 +- .../tox.ini | 4 +- pipeline/readme.md | 2 +- 57 files changed, 346 insertions(+), 188 deletions(-) create mode 100644 .github/workflows/environment-variable-names.yml create mode 100644 ci/check-environment-variable-names.Tests.ps1 create mode 100644 ci/check-environment-variable-names.ps1 diff --git a/.github/workflows/environment-variable-names.yml b/.github/workflows/environment-variable-names.yml new file mode 100644 index 00000000..e6bcaf04 --- /dev/null +++ b/.github/workflows/environment-variable-names.yml @@ -0,0 +1,30 @@ +name: Environment variable names + +on: + pull_request: + push: + branches: + - main + - rc + workflow_dispatch: + +permissions: + contents: read + +jobs: + environment-variable-names: + name: No tracked file names a variable starting with a digit or a fallback name + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Test the check + shell: pwsh + run: Invoke-Pester -Path ./ci/check-environment-variable-names.Tests.ps1 -CI + + - name: Check the names + shell: pwsh + run: ./ci/check-environment-variable-names.ps1 diff --git a/AGENTS.md b/AGENTS.md index 81f3a40b..7d729849 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -207,17 +207,16 @@ which is the configuration the example's own message names, and a key for IP Intelligence in the [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples). -The Device Detection cloud example takes the key as its first argument, or -from `_51DEGREES_RESOURCE_KEY`, or from the older name `resource_key`, which -the Device Detection README uses. The IP Intelligence cloud example takes it -as its first argument or from `resource_key`. +The Device Detection and IP Intelligence cloud examples take the key as +their first argument, or from `_51DEGREES_RESOURCE_KEY`. ``` -resource_key= python -m fiftyone_devicedetection_examples.cloud.gettingstarted_console -resource_key= python -m fiftyone_ipintelligence_examples.cloud.gettingstarted_console +_51DEGREES_RESOURCE_KEY= python -m fiftyone_devicedetection_examples.cloud.gettingstarted_console +_51DEGREES_RESOURCE_KEY= python -m fiftyone_ipintelligence_examples.cloud.gettingstarted_console ``` -In PowerShell, set `$env:resource_key = ''` first. Without a key, each +In PowerShell, set `$env:_51DEGREES_RESOURCE_KEY = ''` first. Without +a key, each example logs an error that names the variable and the configurator. ### Pitfalls diff --git a/ci/check-environment-variable-names.Tests.ps1 b/ci/check-environment-variable-names.Tests.ps1 new file mode 100644 index 00000000..b3b7e8bd --- /dev/null +++ b/ci/check-environment-variable-names.Tests.ps1 @@ -0,0 +1,54 @@ +# Tests for ci/check-environment-variable-names.ps1. +# +# Each case is one line of text and what the check must make of it. Names are +# matched in any case, because Windows reads an environment variable name in +# any case, so a lower case old name is still the old name there. + +BeforeAll { + . (Join-Path $PSScriptRoot 'check-environment-variable-names.ps1') + + function Get-Kind([string]$Line) { + @(Find-EnvironmentNameProblem $Line | ForEach-Object { $_.Kind }) + } +} + +Describe 'Find-EnvironmentNameProblem' { + + It 'finds the digit-initial name ' -ForEach @( + @{ Line = '$env:51DEGREES_DD_PATH = "file.hash"' } + @{ Line = 'export 51DEGREES_IPI_PATH=file.ipi' } + @{ Line = 'os.environ.get("51degrees_dd_path")' } + @{ Line = '$env:51Degrees_Resource_Key = "key"' } + ) { + Get-Kind $Line | Should -Contain 'digit' + } + + It 'finds the fallback name ' -ForEach @( + @{ Line = 'os.environ.get("RESOURCE_KEY")' } + @{ Line = 'os.environ.get("resource_key")' } + @{ Line = 'os.environ["license_key"]' } + @{ Line = 'os.getenv(''License_Key'')' } + @{ Line = 'if "resource_key" in os.environ:' } + @{ Line = '$env:resource_key = "key"' } + @{ Line = '[Environment]::SetEnvironmentVariable("license_key", $k)' } + @{ Line = 'RESOURCE = env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY")' } + @{ Line = 'LICENCE = env("_51DEGREES_LICENSE_KEY", "license_key")' } + @{ Line = 'LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key"' } + @{ Line = 'legacy_license_key_env_var = "x"' } + ) { + Get-Kind $Line | Should -Contain 'fallback' + } + + It 'leaves alone' -ForEach @( + @{ Line = 'os.environ.get("_51DEGREES_RESOURCE_KEY")' } + @{ Line = '$env:_51DEGREES_DD_PATH = "file.hash"' } + @{ Line = '$env:_51DEGREES_LICENSE_KEY_51DID = $k' } + @{ Line = 'RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY"' } + @{ Line = 'builder = Builder({"resource_key": "YOUR_RESOURCE_KEY"})' } + @{ Line = 'Builder(resource_key=key, license_key=licence)' } + @{ Line = '$env:PAID_RESOURCE_KEY = $Keys.TestResourceKey' } + @{ Line = 'Set the 51D_ cookie for the 51Did example.' } + ) { + Get-Kind $Line | Should -BeNullOrEmpty + } +} diff --git a/ci/check-environment-variable-names.ps1 b/ci/check-environment-variable-names.ps1 new file mode 100644 index 00000000..cd34fc77 --- /dev/null +++ b/ci/check-environment-variable-names.ps1 @@ -0,0 +1,117 @@ +# Every 51Degrees environment variable name in this repository starts with an +# underscore, because a POSIX shell cannot set a name that starts with a digit, +# and each variable has one name with nothing read in its place. This reads +# every file git tracks, which leaves out build output and the submodules, and +# fails on any name that starts with the digit and on any fallback name. +# +# Names are matched in any case, because Windows reads an environment +# variable name in any case, so a lower case old name is still the old name +# there. +# +# Dot-sourcing the script defines Find-EnvironmentNameProblem without reading +# the repository, which is how its tests use it. + +[CmdletBinding()] +param( + # The root of the checkout to read. The default is the repository this + # script lives in. + [string]$RepoRoot = (Join-Path $PSScriptRoot '..') +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# The start of a name a shell cannot set, split so this file does not match +# itself. +$digitInitial = '5' + '1DEGREES_' + +# A name is not matched when a name character comes before it, so a name with +# a leading underscore is left alone. +$script:DigitInitialPattern = [regex]::new( + "(? python -m fiftyone_devicedetection_examples.cloud.gettingstarted_console +_51DEGREES_RESOURCE_KEY= python -m fiftyone_devicedetection_examples.cloud.gettingstarted_console ``` -For some of the onpremise examples you might need to provide a `license_key` as an environment variable, f.e.: +For some of the onpremise examples you might need to provide a `_51DEGREES_LICENSE_KEY` as an environment variable, f.e.: ```bash -license_key= python -m fiftyone_devicedetection_examples.onpremise.datafileupdate_console +_51DEGREES_LICENSE_KEY= python -m fiftyone_devicedetection_examples.onpremise.datafileupdate_console ``` ```bash diff --git a/ip-intelligence/ci/run-integration-tests.ps1 b/ip-intelligence/ci/run-integration-tests.ps1 index f57e296c..a64f611e 100644 --- a/ip-intelligence/ci/run-integration-tests.ps1 +++ b/ip-intelligence/ci/run-integration-tests.ps1 @@ -19,6 +19,10 @@ if (!$Keys.TestResourceKey) { "_51DEGREES_RESOURCE_KEY_BESPOKE secret." } +# The tests read the resource key from _51DEGREES_RESOURCE_KEY, and common-ci +# sets only the older resource_key, so the key is set here. +$env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey + # Says what that key is entitled to, so a run with no IP data is read as an # entitlement gap rather than as broken code. Prints product names and tiers # only, never the key. Found through this script's own folder, because diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md b/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md index 83d568f9..ab94098e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md @@ -32,7 +32,7 @@ print(flowdata.ip.registeredname.value()) - `fiftyone_ipintelligence_cloud` - the IP intelligence cloud engine and pipeline builder. - `tests` - unit tests. These mock the cloud service so no resource key is - required; set the `resource_key` environment variable to also run the live + required; set the `_51DEGREES_RESOURCE_KEY` environment variable to also run the live cloud integration test. ## Find out more diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini b/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini index 668932c9..996a428f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini @@ -22,8 +22,8 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_cloud {tty:--color=yes} {posargs} pass_env = - resource_key - license_key + _51DEGREES_RESOURCE_KEY + _51DEGREES_LICENSE_KEY [testenv:pre-publish] package = skip diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md index dd0f3423..61c3f9f5 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md @@ -10,7 +10,7 @@ Usage examples for the 51Degrees IP intelligence engines. ## Cloud examples -Require a resource key in the `resource_key` environment variable. +Require a resource key in the `_51DEGREES_RESOURCE_KEY` environment variable. Obtain one from the [51Degrees Configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=cloud-examples). - `cloud/gettingstarted_console.py` - resolve IP addresses using the diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py index 0ee1ecdc..04b2fcf0 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py @@ -27,7 +27,7 @@ # visitor (or an IP supplied as the "client-ip" query parameter) using # the 51Degrees cloud service. # -# A resource key must be supplied in the `resource_key` environment +# A resource key must be supplied in the `_51DEGREES_RESOURCE_KEY` environment # variable. Obtain one from https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_web-app.py&utm_term=top # # Run with: diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py index 3ffd5499..34fe549a 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py @@ -38,7 +38,7 @@ class ExampleUtils: # The default environment variable used to get the resource key # to use when running examples. - RESOURCE_KEY_ENV_VAR = "resource_key" + RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" ENDPOINT_ENV_VAR = "cloud_endpoint" @@ -378,7 +378,4 @@ def check_data_file(pipeline, logger): @staticmethod def __get_env_variable(name): - value = os.environ.get(name) - if not value: - value = os.environ.get(name.upper()) - return value + return os.environ.get(name) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py index 54971693..6e7a8ebe 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py @@ -35,7 +35,7 @@ # Updating the data file needs a 51Degrees licence key for the Enterprise # data file. Without one, the example shows the free ASN data file and says # where to ask for a key. Supply the licence key as the first command line -# argument or in the 'license_key' environment variable. Contact us for a key: +# argument or in the '_51DEGREES_LICENSE_KEY' environment variable. Contact us for a key: # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=header # # ## Update on start-up @@ -76,7 +76,7 @@ # Environment variable checked for the license key if one is not supplied as # a command line argument. -UPDATE_EXAMPLE_LICENSE_KEY_NAME = "license_key" +UPDATE_EXAMPLE_LICENSE_KEY_NAME = "_51DEGREES_LICENSE_KEY" # Where to ask for a licence key and a copy of the Enterprise data file, tagged with this # example. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py index 239a8852..23eb9f4f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py @@ -32,12 +32,11 @@ import os # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud -# 51Did workflow sets it to each key in turn, and resource_key last because -# it is the older name, which common-ci still sets. +# 51Did workflow sets it to each key in turn, and each product's +# integration test script sets it too. RESOURCE_KEY_NAMES = ( "_51DEGREES_RESOURCE_KEY", "_51DEGREES_RESOURCE_KEY_BESPOKE", - "resource_key", ) ACCESSIBLE_PROPERTIES_URL = ( diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py index e417bd79..8d4cf2d3 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py @@ -42,7 +42,7 @@ from fiftyone_ipintelligence_examples.onpremise import performance_console # The settings the examples read, which a fresh clone does not have. -SETTINGS = ["license_key", "LICENSE_KEY", ExampleUtils.DATA_FILE_ENV_VAR] +SETTINGS = ["_51DEGREES_LICENSE_KEY", ExampleUtils.DATA_FILE_ENV_VAR] # Options that keep the performance example short. They leave the data file # to the example. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini index 459f1a48..f1490076 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini @@ -39,8 +39,7 @@ commands = pass_env = _51DEGREES_RESOURCE_KEY _51DEGREES_RESOURCE_KEY_BESPOKE - resource_key - license_key + _51DEGREES_LICENSE_KEY IPINTELLIGENCEDATAFILE TMPDIR TEMP diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini b/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini index 211e4571..ff935225 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini @@ -34,8 +34,8 @@ commands = python setup.py build_clib build_ext {[common]commands} pass_env = - resource_key - license_key + _51DEGREES_RESOURCE_KEY + _51DEGREES_LICENSE_KEY IPINTELLIGENCEDATAFILE TMPDIR TEMP diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/key_utils.py b/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/key_utils.py index 66088cd7..e4ea13a4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/key_utils.py +++ b/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/key_utils.py @@ -24,8 +24,7 @@ class KeyUtils: - # Obtain a key either from environment variable or from a property. - # Try resource key as env var, then as upper case env var, the system property + # A key from the environment variable of exactly that name. @staticmethod def get_named_key(key_name): - return os.environ.get(key_name) or os.environ.get(key_name.upper()) + return os.environ.get(key_name) diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/tests/test_shared.py b/ip-intelligence/fiftyone_ipintelligence_shared/tests/test_shared.py index a5128b04..ca2f4c5d 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/tests/test_shared.py +++ b/ip-intelligence/fiftyone_ipintelligence_shared/tests/test_shared.py @@ -40,11 +40,11 @@ def test_merge_two_dicts(self): def test_get_named_key(self): import os - os.environ["IPINTELLIGENCETESTKEY"] = "value" + os.environ["ipintelligencetestkey"] = "value" try: self.assertEqual(KeyUtils.get_named_key("ipintelligencetestkey"), "value") finally: - del os.environ["IPINTELLIGENCETESTKEY"] + del os.environ["ipintelligencetestkey"] if __name__ == "__main__": diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini b/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini index 98972fd0..c304f959 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini @@ -21,8 +21,8 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_shared {tty:--color=yes} {posargs} pass_env = - resource_key - license_key + _51DEGREES_RESOURCE_KEY + _51DEGREES_LICENSE_KEY [testenv:pre-publish] package = skip diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini b/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini index b055ee03..fabb28ba 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini @@ -19,8 +19,8 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_translation {tty:--color=yes} {posargs} pass_env = - resource_key - license_key + _51DEGREES_RESOURCE_KEY + _51DEGREES_LICENSE_KEY [testenv:pre-publish] package = skip diff --git a/ip-intelligence/readme.md b/ip-intelligence/readme.md index 7a1be44e..cad2a5c8 100644 --- a/ip-intelligence/readme.md +++ b/ip-intelligence/readme.md @@ -122,7 +122,7 @@ cd fiftyone_ipintelligence_examples && python -m pytest tests ``` - The cloud tests run against a mocked cloud service, so no resource - key is required. Set the `resource_key` environment variable to also + key is required. Set the `_51DEGREES_RESOURCE_KEY` environment variable to also exercise the live cloud service. - The on-premise tests require a data file (see Data files above). Set the `IPINTELLIGENCEDATAFILE` environment variable to use a data file diff --git a/location/ci/run-integration-tests.ps1 b/location/ci/run-integration-tests.ps1 index bbbdb0bb..01ae9b5c 100644 --- a/location/ci/run-integration-tests.ps1 +++ b/location/ci/run-integration-tests.ps1 @@ -11,6 +11,10 @@ if (!$Keys.TestResourceKey) { "_51DEGREES_RESOURCE_KEY_BESPOKE secret." } +# The examples and tests read the resource key from _51DEGREES_RESOURCE_KEY, +# and common-ci sets only the older resource_key, so the key is set here. +$env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages . -Keys $Keys exit $LASTEXITCODE diff --git a/location/examples/cloud/gettingstarted.py b/location/examples/cloud/gettingstarted.py index a0052c3f..fcd10ddf 100644 --- a/location/examples/cloud/gettingstarted.py +++ b/location/examples/cloud/gettingstarted.py @@ -38,10 +38,10 @@ # You need to create a resource key at https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-gettingstarted.py&utm_term=top # and paste it into the code, replacing !!YOUR_RESOURCE_KEY!! below. -# Alternatively, add a resource_key environment variable +# Alternatively, add a _51DEGREES_RESOURCE_KEY environment variable import os -if "resource_key" in os.environ: - resource_key = os.environ["resource_key"] +if "_51DEGREES_RESOURCE_KEY" in os.environ: + resource_key = os.environ["_51DEGREES_RESOURCE_KEY"] else: resource_key = "!!YOUR_RESOURCE_KEY!!" diff --git a/location/examples/cloud/web.py b/location/examples/cloud/web.py index 93c8e3aa..759b821d 100644 --- a/location/examples/cloud/web.py +++ b/location/examples/cloud/web.py @@ -39,10 +39,10 @@ # You need to create a resource key at https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-web.py&utm_term=top # and paste it into the code, replacing !!YOUR_RESOURCE_KEY!! below. -# Alternatively, add a resource_key environment variable +# Alternatively, add a _51DEGREES_RESOURCE_KEY environment variable import os -if "resource_key" in os.environ: - resource_key = os.environ["resource_key"] +if "_51DEGREES_RESOURCE_KEY" in os.environ: + resource_key = os.environ["_51DEGREES_RESOURCE_KEY"] else: resource_key = "!!YOUR_RESOURCE_KEY!!" diff --git a/location/tests/test_examples.py b/location/tests/test_examples.py index 91e71c40..c1e6c276 100644 --- a/location/tests/test_examples.py +++ b/location/tests/test_examples.py @@ -1,24 +1,24 @@ -# ********************************************************************* -# This Original Work is copyright of 51 Degrees Mobile Experts Limited. -# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, -# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. -# -# This Original Work is licensed under the European Union Public Licence -# (EUPL) v.1.2 and is subject to its terms as set out below. -# -# If a copy of the EUPL was not distributed with this file, You can obtain -# one at https://opensource.org/licenses/EUPL-1.2. -# -# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -# amended by the European Commission) shall be deemed incompatible for -# the purposes of the Work and the provisions of the compatibility -# clause in Article 5 of the EUPL shall not apply. -# -# If using the Work as, or as part of, a network application, by -# including the attribution notice(s) required under Article 5 of the EUPL -# in the end user terms of the application under an appropriate heading, -# such notice(s) shall fulfill the requirements of that article. -# ********************************************************************* +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* import os import sys @@ -28,7 +28,7 @@ class LocationExampleTests(unittest.TestCase): def test_cloud_getting_started(self): - if "resource_key" in os.environ: + if "_51DEGREES_RESOURCE_KEY" in os.environ: # this is required to import the example code, # since pytest sets sys.path to the test directory sys.path.append(str(Path(__file__).parent.parent.absolute() / "examples")) diff --git a/location/tests/test_properties51D.py b/location/tests/test_properties51D.py index a5d6ac80..16c347c8 100644 --- a/location/tests/test_properties51D.py +++ b/location/tests/test_properties51D.py @@ -29,11 +29,11 @@ location_LAT = '51.4578261'; location_LON = '-0.975922996290084'; -if "resource_key" in os.environ: - resource_key = os.environ["resource_key"] +if "_51DEGREES_RESOURCE_KEY" in os.environ: + resource_key = os.environ["_51DEGREES_RESOURCE_KEY"] else: raise Exception("To run the cloud tests, please set a valid 51Degrees " - "cloud resource key as the resource_key environment variable.") + "cloud resource key as the _51DEGREES_RESOURCE_KEY environment variable.") # Create a simple pipeline to access the engine with and process it with flow data pipeline = LocationPipelineBuilder(resource_key = resource_key, location_provider = "fiftyonedegrees").build() diff --git a/location/tox.ini b/location/tox.ini index 504de885..e74b3a9f 100644 --- a/location/tox.ini +++ b/location/tox.ini @@ -16,8 +16,8 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_location {tty:--color=yes} {posargs} pass_env = - resource_key - license_key + _51DEGREES_RESOURCE_KEY + _51DEGREES_LICENSE_KEY [testenv:pre-publish] package = skip diff --git a/pipeline/ci/run-integration-tests.ps1 b/pipeline/ci/run-integration-tests.ps1 index 5539b7ea..a4845edd 100644 --- a/pipeline/ci/run-integration-tests.ps1 +++ b/pipeline/ci/run-integration-tests.ps1 @@ -33,6 +33,10 @@ if (!$Keys.TestResourceKey) { $packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples", "fiftyone_pipeline_did" $pipelineKeys = $Keys.Clone() $pipelineKeys.Remove('DeviceDetection') +# The tests read the resource key from _51DEGREES_RESOURCE_KEY, and common-ci +# sets only the older resource_key, so the key is set here. +$env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey + $addopts = $env:PYTEST_ADDOPTS $env:PYTEST_ADDOPTS = '-m "live and not tier"' try { diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index 58b3ec35..6d6bd486 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -32,13 +32,13 @@ The examples call the cloud service with a resource key or a licence key. A resource key carries the list of properties it returns. Create one carrying device detection and IP intelligence properties for free with the [configurator](https://configure.51degrees.com?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-readme.md&utm_term=resource-key), -and set the `_51DEGREES_RESOURCE_KEY` environment variable to it. The older -name `resource_key` is also read. A resource key limited to particular +and set the `_51DEGREES_RESOURCE_KEY` environment variable to it. A +resource key limited to particular domains only answers a request whose origin is one of them, so set `cloud_request_origin` to one of its domains. A licence key can be used on its own instead, by setting -`_51DEGREES_LICENSE_KEY`, or the older name `license_key`. A licence key +`_51DEGREES_LICENSE_KEY`. A licence key carries no list of properties, so the examples ask the cloud service for exactly the properties they show, and the web example also asks for the properties its client-side script needs. A licence key identifies an @@ -91,8 +91,8 @@ then open . Set `PORT` to listen on another port. ``` The tests that call the cloud service with a resource key fail when none is -set, naming the variables they read, which are `_51DEGREES_RESOURCE_KEY`, -`_51DEGREES_RESOURCE_KEY_BESPOKE` and `resource_key`, in that order. The +set, naming the variables they read, which are `_51DEGREES_RESOURCE_KEY` +and `_51DEGREES_RESOURCE_KEY_BESPOKE`, in that order. The tests that use a licence key skip themselves when none is set. In CI the examples are run by [ci/run-integration-tests.ps1](../../ci/run-integration-tests.ps1) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py index cf58f209..02bf4daf 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py @@ -35,18 +35,10 @@ class ExampleUtils: # "_51DEGREES_RESOURCE_KEY". RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" - # Also read for the resource key, being the name the other Python - # examples accept, so one setup runs them all. - LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key" - # The environment variable the examples read a licence key from, to use # in place of a resource key. LICENSE_KEY_ENV_VAR = "_51DEGREES_LICENSE_KEY" - # Also read for the licence key, being the name the other Python - # examples and tests accept. - LEGACY_LICENSE_KEY_ENV_VAR = "license_key" - # The environment variable naming a cloud service other than the public # one, for example one hosted privately. The value includes the api/v4 # path, as in https://cloud.51degrees.com/api/v4/. @@ -62,15 +54,11 @@ class ExampleUtils: @staticmethod def get_resource_key(): - return (os.environ.get(ExampleUtils.RESOURCE_KEY_ENV_VAR) or - os.environ.get(ExampleUtils.LEGACY_RESOURCE_KEY_ENV_VAR) or - "") + return os.environ.get(ExampleUtils.RESOURCE_KEY_ENV_VAR) or "" @staticmethod def get_license_key(): - return (os.environ.get(ExampleUtils.LICENSE_KEY_ENV_VAR) or - os.environ.get(ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR) or - "") + return os.environ.get(ExampleUtils.LICENSE_KEY_ENV_VAR) or "" @staticmethod def get_cloud_endpoint(): diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py index 239a8852..23eb9f4f 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py @@ -32,12 +32,11 @@ import os # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud -# 51Did workflow sets it to each key in turn, and resource_key last because -# it is the older name, which common-ci still sets. +# 51Did workflow sets it to each key in turn, and each product's +# integration test script sets it too. RESOURCE_KEY_NAMES = ( "_51DEGREES_RESOURCE_KEY", "_51DEGREES_RESOURCE_KEY_BESPOKE", - "resource_key", ) ACCESSIBLE_PROPERTIES_URL = ( diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini index 56e9bf52..c594a096 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini @@ -40,8 +40,6 @@ pass_env = _51DEGREES_RESOURCE_KEY _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_LICENSE_KEY - resource_key - license_key cloud_endpoint cloud_request_origin FOD_CLOUD_API_URL diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py index 8de87213..dff83016 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py @@ -33,7 +33,7 @@ class CloudEngineTests(unittest.TestCase): def setUp(self): - self.resource_key = os.environ.get("resource_key") + self.resource_key = os.environ.get("_51DEGREES_RESOURCE_KEY") self.assertIsNotNone( self.resource_key, diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licensekey_live.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licensekey_live.py index 974da1ae..6db15de5 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licensekey_live.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licensekey_live.py @@ -29,13 +29,9 @@ from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder -# The licence key the live tests authenticate with, read from either of -# the variables the other live tests and the examples in this repository -# accept, so an environment set up for them runs these tests too. -LICENSE_KEY = ( - os.environ.get("license_key") - or os.environ.get("_51DEGREES_LICENSE_KEY") - or "").strip() +# The licence key the live tests authenticate with, read from the variable +# the examples in this repository read it from. +LICENSE_KEY = (os.environ.get("_51DEGREES_LICENSE_KEY") or "").strip() USER_AGENT = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) " diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini index 9f6dae66..646020a0 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini @@ -19,8 +19,7 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_cloudrequestengine {tty:--color=yes} {posargs} pass_env = - resource_key - license_key + _51DEGREES_RESOURCE_KEY _51DEGREES_LICENSE_KEY [testenv:pre-publish] diff --git a/pipeline/fiftyone_pipeline_core/tox.ini b/pipeline/fiftyone_pipeline_core/tox.ini index baa342ad..9a03f3a3 100644 --- a/pipeline/fiftyone_pipeline_core/tox.ini +++ b/pipeline/fiftyone_pipeline_core/tox.ini @@ -21,8 +21,8 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_core {tty:--color=yes} {posargs} pass_env = - resource_key - license_key + _51DEGREES_RESOURCE_KEY + _51DEGREES_LICENSE_KEY [testenv:pre-publish] package = skip diff --git a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py index 3b910e9e..57037b7f 100644 --- a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py +++ b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py @@ -61,9 +61,9 @@ two uses every time. The creation call is a further use. The public key list the offline check needs is fetched once and cached for a day. -Environment variables. ``_51DEGREES_RESOURCE_KEY`` (or the older -``RESOURCE_KEY``) is required. ``_51DEGREES_LICENSE_KEY`` (or -``LICENSE_KEY``) is optional, and the comment in ``run`` says why. +Environment variables. ``_51DEGREES_RESOURCE_KEY`` is required. +``_51DEGREES_LICENSE_KEY`` is optional, and the comment in ``run`` says +why. ``FOD_CLOUD_API_URL`` is the cloud API base including the ``/api/v4/`` segment, defaulting to ``https://cloud.51degrees.com/api/v4/``, and is the same variable the cloud request engine and the ``DidClient`` honour. @@ -102,18 +102,14 @@ DEFAULT_API = "https://cloud.51degrees.com/api/v4/" -def env(*names): - """The first of the named environment variables that is set and not - empty, so the aligned name is tried before the older one.""" - for name in names: - value = os.environ.get(name) - if value: - return value - return None +def env(name): + """The named environment variable when it is set and not empty, + otherwise None.""" + return os.environ.get(name) or None -RESOURCE = env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY") -LICENCE = env("_51DEGREES_LICENSE_KEY", "LICENSE_KEY") or "" +RESOURCE = env("_51DEGREES_RESOURCE_KEY") +LICENCE = env("_51DEGREES_LICENSE_KEY") or "" # The cloud API base, normalised to end in exactly one slash so that # every URL is the base followed by its path. The page receives the same # value through its __API__ placeholder and builds its two cloud calls @@ -281,8 +277,8 @@ def _is_json(text): def run(): if not RESOURCE: - sys.exit("Set _51DEGREES_RESOURCE_KEY (or RESOURCE_KEY) to the " - "resource key of the page.") + sys.exit("Set _51DEGREES_RESOURCE_KEY to the resource key of the " + "page.") if not LICENCE: # Only an account that holds licence keys needs one to redeem, # because the licence key is what keeps redemption to the acting diff --git a/pipeline/fiftyone_pipeline_did/readme.md b/pipeline/fiftyone_pipeline_did/readme.md index 3c2f369c..38ae2b05 100644 --- a/pipeline/fiftyone_pipeline_did/readme.md +++ b/pipeline/fiftyone_pipeline_did/readme.md @@ -655,8 +655,8 @@ unreachable cloud answers 502 with `{ "error": ... }`. | Variable | Meaning | | --- | --- | -| `_51DEGREES_RESOURCE_KEY` | Required. The page's resource key, public by nature. The older `RESOURCE_KEY` is read when the aligned name is not set | -| `_51DEGREES_LICENSE_KEY` | Optional. A licence key of the same account, held server side only. Only an account that holds licence keys needs one to redeem, so an account holding none runs without it. The older `LICENSE_KEY` is read when the aligned name is not set | +| `_51DEGREES_RESOURCE_KEY` | Required. The page's resource key, public by nature | +| `_51DEGREES_LICENSE_KEY` | Optional. A licence key of the same account, held server side only. Only an account that holds licence keys needs one to redeem, so an account holding none runs without it | | `FOD_CLOUD_API_URL` | Optional. The cloud API base including the `/api/v4/` segment, defaulting to `https://cloud.51degrees.com/api/v4/`. This is the same variable the cloud request engine honours. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons, which is the private hosting option of the cloud service. Both run the same service, so the demo works unchanged against either | | `PORT` | The port to listen on, defaulting to `5100` | diff --git a/pipeline/fiftyone_pipeline_did/tests/live_cloud.py b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py index 239a8852..23eb9f4f 100644 --- a/pipeline/fiftyone_pipeline_did/tests/live_cloud.py +++ b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py @@ -32,12 +32,11 @@ import os # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud -# 51Did workflow sets it to each key in turn, and resource_key last because -# it is the older name, which common-ci still sets. +# 51Did workflow sets it to each key in turn, and each product's +# integration test script sets it too. RESOURCE_KEY_NAMES = ( "_51DEGREES_RESOURCE_KEY", "_51DEGREES_RESOURCE_KEY_BESPOKE", - "resource_key", ) ACCESSIBLE_PROPERTIES_URL = ( diff --git a/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py b/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py index 6a41e265..69f95236 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py @@ -26,7 +26,7 @@ test fails when none is set. The key has to carry the FODid product, and the CloudV5FODiD product for the marketing usages, which ``test_did_tier_live.py`` shows a Basic key does not get. An optional -licence key is read from ``license_key`` or ``_51DEGREES_LICENSE_KEY``, and +licence key is read from ``_51DEGREES_LICENSE_KEY``, and the endpoint from ``FOD_CLOUD_API_URL`` as everywhere else. Every test here costs uses against the resource key. """ @@ -56,8 +56,7 @@ pytestmark = pytest.mark.live RESOURCE_KEY = find_resource_key() -LICENCE_KEY = os.environ.get("license_key") \ - or os.environ.get("_51DEGREES_LICENSE_KEY") +LICENCE_KEY = os.environ.get("_51DEGREES_LICENSE_KEY") FODID = "the FODid product" MARKETING = "the CloudV5FODiD product" diff --git a/pipeline/fiftyone_pipeline_did/tox.ini b/pipeline/fiftyone_pipeline_did/tox.ini index edf34540..50b464d3 100644 --- a/pipeline/fiftyone_pipeline_did/tox.ini +++ b/pipeline/fiftyone_pipeline_did/tox.ini @@ -19,8 +19,6 @@ passenv = _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_RESOURCE_KEY_BASIC _51DEGREES_LICENSE_KEY - resource_key - license_key FOD_CLOUD_API_URL PYTEST_ADDOPTS deps = diff --git a/pipeline/fiftyone_pipeline_engines/tox.ini b/pipeline/fiftyone_pipeline_engines/tox.ini index c36d9445..1e8e9619 100644 --- a/pipeline/fiftyone_pipeline_engines/tox.ini +++ b/pipeline/fiftyone_pipeline_engines/tox.ini @@ -18,8 +18,8 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_engines {tty:--color=yes} {posargs} pass_env = - resource_key - license_key + _51DEGREES_RESOURCE_KEY + _51DEGREES_LICENSE_KEY [testenv:pre-publish] package = skip diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tox.ini b/pipeline/fiftyone_pipeline_engines_fiftyone/tox.ini index 21f6519e..cddb3cd4 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tox.ini +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tox.ini @@ -19,8 +19,8 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_engines_fiftyone {tty:--color=yes} {posargs} pass_env = - resource_key - license_key + _51DEGREES_RESOURCE_KEY + _51DEGREES_LICENSE_KEY [testenv:pre-publish] package = skip diff --git a/pipeline/readme.md b/pipeline/readme.md index ea7419ec..3b5fbf45 100755 --- a/pipeline/readme.md +++ b/pipeline/readme.md @@ -51,7 +51,7 @@ and examples that are available. To run tests: * Go to each directory (for example `fiftyone_pipeline_core`) * Run `tox -e py` -The tests that call the 51Degrees cloud service read a resource key from the `resource_key` environment variable. Without one, those tests fail in `fiftyone_pipeline_cloudrequestengine` and are skipped in `fiftyone_pipeline_did`. +The tests that call the 51Degrees cloud service need a resource key. The cloud request engine's own tests read it from the `_51DEGREES_RESOURCE_KEY` environment variable, and its examples' tests and the 51Did tests from `_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`. Without one, those tests fail. ## Examples From a10b1cbbb19c9ec18dfd0c4af1ce36f588d9406a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 13:54:39 +0000 Subject: [PATCH 47/68] Give the specification's JSON Builder error text with no sequence number (#101) * Test the exact JSON Builder error text with no sequence number * Give the specification's JSON Builder error text with no sequence number Every language gives the same text when no Sequence Element has produced a sequence number, as the JSON Builder page of the specification states. --- .../src/fiftyone_pipeline_core/jsonbundler.py | 5 ++--- .../tests/test_jsonbundler_sequence_required.py | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index 030cba4a..6ddf50a3 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -108,9 +108,8 @@ def process_internal(self, flowdata): sequence = flowdata.evidence.get("query.sequence") if isinstance(sequence, bool) or not isinstance(sequence, int): raise Exception( - "The JSON Builder needs the sequence number that a Sequence " - "Element produces. A Sequence Element needs to be placed " - "before it in the Pipeline.") + "The JSON Builder needs a sequence number. Place a Sequence " + "Element before the JSON Builder in the Pipeline.") # Get every property on every FlowElement # Storing JavaScript properties in an extra section diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_sequence_required.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_sequence_required.py index 8cc8058b..45100d5f 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_sequence_required.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_sequence_required.py @@ -50,7 +50,10 @@ def test_no_sequence_number_is_an_error_that_names_the_sequence_element( flowdata = pipeline.create_flowdata() with self.assertRaises(Exception) as caught: flowdata.process() - self.assertIn("Sequence Element", str(caught.exception)) + self.assertEqual( + "The JSON Builder needs a sequence number. Place a Sequence " + "Element before the JSON Builder in the Pipeline.", + str(caught.exception)) def test_no_json_is_set_when_the_error_is_suppressed(self): pipeline = bundler_only(suppress=True) From 77ea967260ceb3693dd9282be6962cfc75d4815c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 14:19:53 +0000 Subject: [PATCH 48/68] Give IP Intelligence latitude and longitude to three decimal places (#81) * Test that IP Intelligence latitude and longitude have three decimal places * Give IP Intelligence latitude and longitude to three decimal places The native code gives the coordinates in text and in the WKT of Areas to three places, and the numbers read through the weighted and single value getters are rounded to match. The unused sixteen place setting is removed. * Test that a half rounds away from zero and a coordinate rounding to zero has no sign * Round a half away from zero and give a coordinate that rounds to zero no sign --- .../swig_data.py | 46 ++++++- .../tests/test_swig_data_decimal_places.py | 117 ++++++++++++++++++ 2 files changed, 158 insertions(+), 5 deletions(-) create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_decimal_places.py diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 50c53f98..49fe459d 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -20,6 +20,9 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +import math +from decimal import Decimal, ROUND_HALF_UP + from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue @@ -27,9 +30,15 @@ from .constants import RESULTS_CLOSED -# Number of decimal places used when converting WKB geometry -# properties (e.g. Areas) to their WKT string form. -WKT_DECIMAL_PLACES = 16 +# Latitude and Longitude are given to this many decimal places, as the +# native code gives them in text and in the WKT of Areas. +COORDINATE_DECIMAL_PLACES = 3 + +# The step a coordinate is rounded to, worked out from the places above so +# that the number of places is written down once. +COORDINATE_QUANTUM = Decimal(1).scaleb(-COORDINATE_DECIMAL_PLACES) + +COORDINATE_PROPERTIES = ("latitude", "longitude") class OnPremiseMissingPropertyService(MissingPropertyService): @@ -166,6 +175,26 @@ def ip_address_value(self, value): return AspectPropertyValue(no_value_message="The IP address type is invalid.") + @staticmethod + def round_coordinate(value): + + """! + A latitude or longitude to COORDINATE_DECIMAL_PLACES places. + + A value half way between two places rounds away from zero, as the + native code rounds it, where Python's own round() would round it to + the nearer even place. A value that rounds to zero from below is + given as zero without a sign, because the native code writes the + same value as text without one. A value that is not finite is given + as it is, having no places to round. + """ + + if math.isfinite(value) is False: + return value + rounded = float(Decimal(value).quantize( + COORDINATE_QUANTUM, rounding=ROUND_HALF_UP)) + return 0.0 if rounded == 0 else rounded + def single_value(self, value, convert=None): """! @@ -237,9 +266,15 @@ def get_internal(self, key): return self.weighted_list_to_value( results.getValuesAsWeightedIntegerList(target)) + # Only the coordinates are rounded. + round_coordinate = ( + self.round_coordinate if key in COORDINATE_PROPERTIES + else None) + if property_type in ("weighteddouble", "weightedsingle"): return self.weighted_list_to_value( - results.getValuesAsWeightedDoubleList(target)) + results.getValuesAsWeightedDoubleList(target), + convert=round_coordinate) if property_type == "weightedbool": return self.weighted_list_to_value( @@ -269,7 +304,8 @@ def get_internal(self, key): return self.single_value(results.getValueAsInteger(target)) if property_type in ("double", "single"): - return self.single_value(results.getValueAsDouble(target)) + return self.single_value( + results.getValueAsDouble(target), convert=round_coordinate) if property_type == "bool": return self.single_value(results.getValueAsBool(target)) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_decimal_places.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_decimal_places.py new file mode 100644 index 00000000..3de26193 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_decimal_places.py @@ -0,0 +1,117 @@ + +""" +IP Intelligence Latitude and Longitude are given to 3 decimal places, as the +value types page of the specification says. +""" + +import math +import unittest + +from fiftyone_ipintelligence_onpremise import swig_data +from fiftyone_ipintelligence_onpremise.swig_data import SwigData + + +class Weighted: + + def __init__(self, value, weight): + self.value = value + self.weight = weight + + def getValue(self): + return self.value + + def getWeight(self): + return self.weight + + +class WeightedList: + + def __init__(self, items): + self.items = items + + def hasValue(self): + return True + + def getValue(self): + return self.items + + +class Single: + + def __init__(self, value): + self.value = value + + def hasValue(self): + return True + + def getValue(self): + return self.value + + +class FakeResults: + + def getValuesAsWeightedDoubleList(self, name): + return WeightedList([Weighted(18.542741775512695, 65535), + Weighted(73.91278076171875, 100)]) + + def getValueAsDouble(self, name): + return Single(18.542741775512695) + + +class FakeElement: + datakey = "ip" + + def __init__(self, type_name, name): + self.properties = {name.lower(): {"name": name, "type": type_name}} + + +def read(type_name, name): + data = SwigData(FakeElement(type_name, name), FakeResults()) + return data.get_internal(name.lower()).value() + + +class DecimalPlacesTests(unittest.TestCase): + + def test_weighted_latitude_and_longitude_have_three_places(self): + for type_name in ("weighteddouble", "weightedsingle"): + for name in ("Latitude", "Longitude"): + values = [item["value"] for item in read(type_name, name)] + self.assertEqual([18.543, 73.913], values, (type_name, name)) + + def test_weights_are_unchanged(self): + weights = [item["weighting"] for item in read("weighteddouble", "Latitude")] + self.assertEqual([65535, 100], weights) + + def test_single_latitude_has_three_places(self): + for type_name in ("double", "single"): + self.assertEqual(18.543, read(type_name, "Latitude")) + + def test_other_numbers_are_not_rounded(self): + values = [item["value"] for item in read("weighteddouble", "Accuracy")] + self.assertEqual([18.542741775512695, 73.91278076171875], values) + + def test_the_unused_sixteen_place_setting_is_gone(self): + self.assertFalse(hasattr(swig_data, "WKT_DECIMAL_PLACES")) + + def test_a_half_rounds_away_from_zero(self): + for value, expected in ((0.0625, 0.063), + (-0.0625, -0.063), + (51.3125, 51.313)): + self.assertEqual(expected, SwigData.round_coordinate(value), value) + + def test_a_value_that_rounds_to_zero_from_below_has_no_sign(self): + for value in (-0.0004, -0.0): + rounded = SwigData.round_coordinate(value) + self.assertEqual(0.0, rounded, value) + self.assertFalse(math.copysign(1.0, rounded) < 0, + f"{value} was given as minus zero") + + def test_a_value_that_is_not_finite_is_kept(self): + infinity = float("inf") + self.assertEqual(infinity, SwigData.round_coordinate(infinity)) + self.assertEqual(-infinity, SwigData.round_coordinate(-infinity)) + self.assertTrue(math.isnan(SwigData.round_coordinate(float("nan")))) + + +if __name__ == "__main__": + unittest.main() From f69f2643e8ea49d3a1112d9507f4780b7a247459 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 14:55:51 +0000 Subject: [PATCH 49/68] Send every Evidence value to the cloud apart from the reserved names, under the name after the first dot, with prefixes ranked as evidence.md lists them (#99) * Test that the Cloud Request Engine sends evidence the cloud does not list * Send the evidence no element accepts to the cloud as well * Test the reserved names with credentials the engine accepts A resource key with requested properties is refused when the engine is built, so the reserved name test now sends twice, once with a resource key and a licence key and no property list, and once with a licence key alone and the property list it needs. * Test that every evidence value is sent under the name after the first dot, in its own letter case The two rc tests that asserted a header and a cookie no element accepts are left out now assert they are sent, under decision 1. The sent name is everything after the first dot, as .NET's CloudRequestEngine.AddQueryData takes it, and keeps the letter case of the evidence key. * Send each evidence value under the name after the first dot, in its own letter case The prefix is the text before the first dot, so query.a.license is sent as a.license rather than dropped as the reserved name license, and the name keeps the letter case of the evidence key, as .NET's AddQueryData does. A name already added in other letters is the same name, so precedence still sends it once. * Test that the Cloud Request Engine ranks Evidence prefixes in evidence.md's order * Rank server, fiftyone and location Evidence in evidence.md's order, then other prefixes alphabetically --- .../cloudrequestengine.py | 101 ++++-- .../constants.py | 5 + .../tests/test_all_evidence_sent.py | 299 ++++++++++++++++++ .../tests/test_cloudengine_evidence.py | 16 +- .../tests/test_cloudrequestengine.py | 2 +- .../tests/test_credentials.py | 18 +- 6 files changed, 403 insertions(+), 38 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_all_evidence_sent.py diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py index 13b7f215..945b8a76 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py @@ -30,6 +30,7 @@ import requests from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.evidence import HTTP_REQUEST_KEY from fiftyone_pipeline_core.keymask import mask_licence_key from fiftyone_pipeline_core.keymask import mask_licence_keys_in_text from fiftyone_pipeline_engines.engine import Engine @@ -504,14 +505,19 @@ def report_properties_not_covered(self, result): def get_content(self, flowData): """! - Generate the Content to send in the POST request. The evidence keys - e.g. 'query.' and 'header.' have an order of precedence. These are - added to the evidence in reverse order, if there is conflict then - the queryData value is overwritten. - - 'query.' evidence should take precedence over all other evidence. - If there are evidence keys other than 'query.' that conflict then - this is unexpected so a warning will be logged. + Generate the Content to send in the POST request. One name under + several prefixes is sent once, with the value of the prefix that + evidence.md ranks highest, being query, header, cookie, server, + fiftyone and location, then any other prefix alphabetically. The + groups are added lowest precedence first, so a later value replaces + an earlier one, and a replacement by a key other than a query key + is logged as a warning. + + The engine sends every evidence value in the flow data, including + the values no element in the pipeline accepts, because what the + cloud service uses changes over time and knowing about the other + evidence is important to continued optimization. The reserved names + and the web request view are left out. @param: flowData: FlowData @return: Evidence Dictionary @@ -519,8 +525,8 @@ def get_content(self, flowData): queryData = {} - evidence = flowData.evidence.get_all() - # Add evidence in reverse alphabetical order, excluding special keys. + evidence = self.sendable_evidence(flowData.evidence) + # Add the other evidence lowest precedence first. self.add_query_data(queryData, evidence, self.get_selected_evidence(evidence, Constants.EVIDENCE_OTHER)) # Add cookie evidence. self.add_query_data(queryData, evidence, self.get_selected_evidence(evidence, Constants.EVIDENCE_COOKIE_PREFIX)) @@ -530,6 +536,28 @@ def get_content(self, flowData): self.add_query_data(queryData, evidence, self.get_selected_evidence(evidence, Constants.EVIDENCE_QUERY_PREFIX)) return queryData + def sendable_evidence(self, evidence): + + """! + The evidence the engine sends, being every value in the flow data, + whether an element accepted it or not, apart from the web request + view. The reserved names are left out by add_query_data, which + warns as it does so. An accepted value wins over a set aside one of + the same key. + + @param evidence: The flow data's Evidence + @return: The evidence keys and values to send + """ + + sendable = {} + # An older fiftyone_pipeline_core sets nothing aside. + unaccepted = getattr(evidence, "get_unaccepted", dict)() + for source in (unaccepted, evidence.get_all()): + for key, value in source.items(): + if str(key).lower() != HTTP_REQUEST_KEY: + sendable[key] = value + return sendable + def add_query_data(self, query_data, all_evidence, evidence): """! @@ -543,10 +571,14 @@ def add_query_data(self, query_data, all_evidence, evidence): for evidenceKey, evidenceValue in evidence.items(): - # Get the key parts - evidenceKeyParts = evidenceKey.split(Constants.EVIDENCE_SEPERATOR) - prefix = evidenceKeyParts[0].lower() - suffix = evidenceKeyParts[-1].lower() + # The prefix is the text before the first separator, and the + # name sent is the rest of the key in the letter case given, as + # .NET's AddQueryData takes it. A key with no separator is both. + prefix, separator, suffix = evidenceKey.partition( + Constants.EVIDENCE_SEPERATOR) + if not separator: + suffix = evidenceKey + prefix = prefix.lower() # The service reads its credentials and the list of requested # properties from these names, so evidence carrying one of them @@ -555,13 +587,19 @@ def add_query_data(self, query_data, all_evidence, evidence): # say so as the conflict case below does, so a caller who set # it expecting an effect can see why there was none. The value # is not logged, as it may be a credential. - if suffix in Constants.RESERVED_EVIDENCE_SUFFIXES: + if suffix.lower() in Constants.RESERVED_EVIDENCE_SUFFIXES: warnings.warn( Constants.WARNING_RESERVED_EVIDENCE.format(evidenceKey)) continue + # Evidence keys ignore letter case, so a name already added in + # other letters is the same name, and the later, higher + # precedence value replaces it under its own spelling. + existing = next((name for name in query_data + if name.lower() == suffix.lower()), None) + # Check and add the evidence to the query parameters. - if (not suffix in query_data.keys()): + if existing is None: query_data[suffix] = evidenceValue # If the queryParameter exists already. else: @@ -572,7 +610,7 @@ def add_query_data(self, query_data, all_evidence, evidence): if (prefix.lower() != Constants.EVIDENCE_QUERY_PREFIX): conflicts = {} for key, value in all_evidence.items(): - if(key.lower() != evidenceKey.lower() and suffix in key.lower()): + if(key.lower() != evidenceKey.lower() and suffix.lower() in key.lower()): conflicts[key] = value warningMessage = Constants.WARNING_MESSAGE \ @@ -585,6 +623,7 @@ def add_query_data(self, query_data, all_evidence, evidence): warnings.warn(warningMessage + conflictStr) # Overwrite the existing queryParameter value. + del query_data[existing] query_data[suffix] = evidenceValue def get_selected_evidence(self, evidence, type): @@ -604,15 +643,37 @@ def get_selected_evidence(self, evidence, type): not self.key_has_prefix(key, Constants.EVIDENCE_HTTPHEADER_PREFIX) and \ not self.key_has_prefix(key, Constants.EVIDENCE_COOKIE_PREFIX)): selected_evidence[key] = value - selected_evidence = dict(sorted(selected_evidence.items(), reverse=True)) - + # Lowest precedence first, so the value added last wins. + selected_evidence = dict(sorted( + selected_evidence.items(), + key=lambda item: self.other_prefix_rank(item[0]), + reverse=True)) + else: for key, value in evidence.items(): if self.key_has_prefix(key, type): selected_evidence[key] = value return selected_evidence - + + def other_prefix_rank(self, key): + + """! + The sort key of an evidence key that is not query, header or cookie + evidence, where a lower one wins. server, fiftyone and location + come first, in the order evidence.md lists them, then any other + prefix alphabetically, all ignoring letter case. + + @param key: The evidence key + @return: A tuple that sorts higher precedence first + """ + + prefix = key.partition(Constants.EVIDENCE_SEPERATOR)[0].lower() + if prefix in Constants.OTHER_PREFIX_PRECEDENCE: + return (Constants.OTHER_PREFIX_PRECEDENCE.index(prefix), "", + key.lower()) + return (len(Constants.OTHER_PREFIX_PRECEDENCE), prefix, key.lower()) + def key_has_prefix(self, itemKey, prefix): """! diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py index f53bee2a..5b198db4 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py @@ -53,6 +53,11 @@ class Constants: # processing. EVIDENCE_QUERY_PREFIX = "query" + # The prefixes after query, header and cookie, highest precedence + # first, as evidence.md lists them. Any other prefix ranks after these, + # alphabetically. + OTHER_PREFIX_PRECEDENCE = ("server", "fiftyone", "location") + # other evidence constant EVIDENCE_OTHER = "other" diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_all_evidence_sent.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_all_evidence_sent.py new file mode 100644 index 00000000..ddb6e847 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_all_evidence_sent.py @@ -0,0 +1,299 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The Cloud Request Engine sends every Evidence value in the Flow Data, +apart from the reserved names and the web request view, as the Processing +section of cloud-request-engine.md says at e972d136.""" + +import warnings + +from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import \ + CloudRequestEngine +from fiftyone_pipeline_cloudrequestengine.constants import Constants +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase + +RESOURCE_KEY = "resource_key" +LICENSE_KEY = "a-licence-key" + + +class TestAllEvidenceSent(CloudRequestEngineTestsBase): + """! + A cloud-only pipeline, so the Cloud Request Engine is the only element + and its accepted Evidence is the list the stand-in service gives, which + holds query.User-Agent alone. Everything else is Evidence no element in + the pipeline accepts, and the engine still sends it. + """ + + def send(self, evidence, settings=None): + """ + Process one flow data holding the evidence given, through a + pipeline whose only element is the Cloud Request Engine, and give + back the body of the data request. + """ + + settings = dict(settings or {}) + settings.setdefault("resource_key", RESOURCE_KEY) + settings["http_client"] = self.mock_http() + engine = CloudRequestEngine(settings) + pipeline = PipelineBuilder().add(engine).build() + data = pipeline.create_flowdata() + for key, value in evidence.items(): + data.evidence.add(key, value) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + data.process() + + return engine.http_client.requests[-1][2] + + def test_evidence_the_service_does_not_list_is_sent(self): + """ + The stand-in service lists none of these three names, so a store + that keeps only the accepted Evidence loses all three before the + engine reads them. The specification says the engine makes the + request "with every Evidence value in the Flow Data, apart from the + reserved names and the web request view". + """ + + content = self.send({ + "query.my-own-thing": "a value", + "header.sec-ch-ua-platform": "Windows", + "cookie.51D_experiment": "b", + }) + + self.assertEqual("a value", content["my-own-thing"]) + self.assertEqual("Windows", content["sec-ch-ua-platform"]) + self.assertEqual("b", content["51D_experiment"]) + + def test_evidence_the_service_lists_is_still_sent(self): + """ + An accepted key is still sent, so reading the set aside Evidence as + well has not displaced the Evidence the engine sent before. + """ + + content = self.send({ + "query.User-Agent": "a browser", + "query.my-own-thing": "a value", + }) + + self.assertEqual("a browser", content["User-Agent"]) + self.assertEqual("a value", content["my-own-thing"]) + + def test_query_evidence_wins_whether_listed_or_not(self): + """ + Query evidence takes precedence over header and cookie evidence of + the same name, whether the service lists the name or not. + """ + + content = self.send({ + "cookie.my-own-thing": "from the cookie", + "header.my-own-thing": "from the header", + "query.my-own-thing": "from the query", + }) + + self.assertEqual("from the query", content["my-own-thing"]) + + def test_each_prefix_wins_over_every_later_one(self): + """ + evidence.md lists query, header, cookie, server, fiftyone and + location from highest to lowest precedence, and ranks any other + prefix alphabetically after them. Each pair of prefixes gives one + name, added higher first so the order added cannot decide it, and + the higher prefix's value is the one sent. + """ + + order = ("query", "header", "cookie", "server", "fiftyone", + "location", "apple", "zebra") + evidence = {} + for i, higher in enumerate(order): + for lower in order[i + 1:]: + name = "n-" + higher + "-" + lower + evidence[higher + "." + name] = higher + evidence[lower + "." + name] = lower + + content = self.send(evidence) + + wrong = [] + for i, higher in enumerate(order): + for lower in order[i + 1:]: + name = "n-" + higher + "-" + lower + if content.get(name) != higher: + wrong.append(name + " sent " + str(content.get(name))) + self.assertEqual([], wrong) + + def test_a_prefix_ranks_the_same_in_any_letter_case(self): + """ + Evidence keys ignore letter case, so Location ranks as location and + SERVER as server. + """ + + content = self.send({ + "fiftyone.b": "fiftyone", + "Location.b": "location", + "SERVER.c": "server", + "Location.c": "location", + }) + + self.assertEqual("fiftyone", content["b"]) + self.assertEqual("server", content["c"]) + + def test_other_prefixes_rank_by_the_prefix_alone(self): + """ + Other prefixes rank alphabetically by the prefix, so ab comes + before ab-c, although the key ab-c.n sorts before ab.n because a + hyphen sorts before a dot. + """ + + content = self.send({"ab.n": "ab", "ab-c.n": "ab-c"}) + + self.assertEqual("ab", content["n"]) + + def test_a_reserved_name_is_never_sent_from_evidence(self): + """ + The service reads its credentials and the property list from these + names, so Evidence carrying one of them would sit beside the + engine's own values with no way to know which one the service + applied. The specification says Evidence under any prefix whose key + strips to values, resource, license or licence, in any letter case, + MUST be left out. + """ + + evidence = { + "query.resource": "evidence_resource", + "query.License": "evidence_license", + "header.LICENCE": "evidence_licence", + "cookie.values": "evidence_values", + "query.my-own-thing": "a value", + } + + # A resource key and a licence key, with no property list, so + # nothing the engine sends is called values. + content = self.send(evidence, {"license_key": LICENSE_KEY}) + self.assertEqual(RESOURCE_KEY, content[Constants.RESOURCE_PARAMETER]) + self.assertEqual(LICENSE_KEY, content[Constants.LICENSE_PARAMETER]) + self.assertNotIn(Constants.VALUES_PARAMETER, content) + self.assertNotIn(Constants.LICENCE_SPELLING, content) + for name, value in content.items(): + self.assertNotIn("evidence_", str(value), name) + self.assertEqual("a value", content["my-own-thing"]) + + # A licence key alone, with the property list it needs, so nothing + # the engine sends is called resource. + content = self.send(evidence, { + "resource_key": "", + "license_key": LICENSE_KEY, + "requested_properties": ["device.ismobile"], + }) + self.assertNotIn(Constants.RESOURCE_PARAMETER, content) + self.assertEqual(LICENSE_KEY, content[Constants.LICENSE_PARAMETER]) + self.assertEqual( + "device.ismobile", content[Constants.VALUES_PARAMETER]) + self.assertNotIn(Constants.LICENCE_SPELLING, content) + for name, value in content.items(): + self.assertNotIn("evidence_", str(value), name) + self.assertEqual("a value", content["my-own-thing"]) + + def test_the_web_request_view_is_never_sent(self): + """ + A web integration adds the web request itself under + server.http-request. It is a view of the whole request and not a + value to send, so the specification says it MUST be left out. + """ + + content = self.send({ + "server.http-request": {"headers": {"user-agent": "a browser"}}, + "query.my-own-thing": "a value", + }) + + self.assertEqual( + [], [name for name in content if "http-request" in name.lower()]) + self.assertEqual("a value", content["my-own-thing"]) + + def test_a_reserved_name_is_left_out_under_any_prefix_and_case(self): + """ + The letter case of the key does not change which name the service + reads, so a reserved name under any prefix is left out whatever its + case. + """ + + content = self.send({ + "header.ReSoUrCe": "evidence_resource", + "cookie.LiCeNsE": "evidence_license", + "query.VALUES": "evidence_values", + }) + + for name, value in content.items(): + self.assertNotIn("evidence_", str(value), name) + + def test_the_name_sent_is_everything_after_the_first_dot(self): + """ + The prefix is the text before the first dot, and removing it leaves + the rest of the key, dots included. So a.license is not the + reserved name license, and a key with no dot is sent whole. + """ + + content = self.send({ + "query.a.license": "after the first dot", + "header.x.y.z": "three parts", + "nodot": "no prefix", + }) + + self.assertEqual("after the first dot", content["a.license"]) + self.assertEqual("three parts", content["x.y.z"]) + self.assertEqual("no prefix", content["nodot"]) + self.assertNotIn("z", content) + + def test_the_name_sent_keeps_the_letter_case_given(self): + """ + The name is sent in the letter case of the evidence key, as .NET + sends it, rather than lower-cased. + """ + + content = self.send({ + "cookie.51D_Experiment": "b", + "header.Sec-CH-UA-Platform": "Windows", + }) + + self.assertEqual("b", content["51D_Experiment"]) + self.assertEqual("Windows", content["Sec-CH-UA-Platform"]) + self.assertNotIn("51d_experiment", content) + self.assertNotIn("sec-ch-ua-platform", content) + + def test_one_name_in_two_letter_cases_is_sent_once(self): + """ + Evidence keys ignore letter case, so header.User-Agent and + query.user-agent name the same value, and the query value wins + under the precedence order. The name is sent once, spelled as the + winning key spells it. + """ + + content = self.send({ + "header.User-Agent": "from the header", + "query.user-agent": "from the query", + }) + + self.assertEqual( + ["user-agent"], + [name for name in content if name.lower() == "user-agent"]) + self.assertEqual("from the query", content["user-agent"]) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_evidence.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_evidence.py index ba9f157a..748a4b59 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_evidence.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_evidence.py @@ -21,8 +21,8 @@ # ********************************************************************* """A cloud engine reads only the cloud request engine's results, so it -accepts no evidence, and a cloud pipeline sends the service only the -evidence the service lists and the elements the builder adds read.""" +accepts no evidence, and a cloud pipeline still sends the service every +evidence value apart from the reserved names.""" from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import \ @@ -44,10 +44,10 @@ def test_a_cloud_engine_accepts_no_evidence(self): "server.client-ip"): self.assertFalse(engine.filter_evidence_key(key), key) - def test_a_cloud_pipeline_sends_only_the_evidence_the_service_lists(self): + def test_a_cloud_pipeline_sends_every_evidence_value(self): """ - The stand-in service lists query.User-Agent alone, so of this - evidence only the user agent reaches it. + The stand-in service lists query.User-Agent alone, and no element + accepts the header or the cookie, yet all three reach it. """ cloud = CloudRequestEngine({ "license_key": LICENSE_KEY, @@ -65,6 +65,6 @@ def test_a_cloud_pipeline_sends_only_the_evidence_the_service_lists(self): data.process() request_type, url, content = cloud.http_client.requests[-1] - self.assertEqual("iPhone", content["user-agent"]) - self.assertNotIn("x-test", content) - self.assertNotIn("other", content) + self.assertEqual("a header", content["x-test"]) + self.assertEqual("a cookie", content["other"]) + self.assertEqual("iPhone", content["User-Agent"]) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py index 1c39b024..4e246298 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py @@ -202,7 +202,7 @@ def test_get_content(self, evidence, expected_value): result = engine.get_content(data) - self.assertEqual(expected_value, result["user-agent"]) + self.assertEqual(expected_value, result["User-Agent"]) def test_no_exception_on_cloud_request_engine_init(self): """ diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py index 09f01bfe..75b08646 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py @@ -86,7 +86,7 @@ def test_resource_key_alone_is_accepted(self): self.assertEqual(RESOURCE_KEY, content[Constants.RESOURCE_PARAMETER]) self.assertNotIn(Constants.LICENSE_PARAMETER, content) self.assertNotIn(Constants.VALUES_PARAMETER, content) - self.assertEqual("iPhone", content["user-agent"]) + self.assertEqual("iPhone", content["User-Agent"]) def test_resource_key_with_licence_key_is_accepted(self): """ @@ -291,7 +291,7 @@ def test_reserved_evidence_is_left_out_of_the_body(self): self.assertEqual( "device.ismobile,device.istablet", content[Constants.VALUES_PARAMETER]) - self.assertEqual("iPhone", content["user-agent"]) + self.assertEqual("iPhone", content["User-Agent"]) # Each piece of evidence left out is named once, without its # value, as the value may be a credential. @@ -389,11 +389,11 @@ def test_evidence_never_supplies_values(self): Constants.LICENSE_PARAMETER, Constants.VALUES_PARAMETER)]) self.assert_other_evidence_sent(content) - def test_evidence_no_element_accepts_is_not_sent(self): + def test_evidence_no_element_accepts_is_sent(self): """ - A default pipeline keeps only the evidence one of its elements - accepts, so a header or cookie the service does not ask for is not - sent to it. + The engine sends every evidence value in the flow data apart from + the reserved names, so a header and a cookie that no element in a + default pipeline accepts are sent as well. """ engine = CloudRequestEngine({ "license_key": LICENSE_KEY, @@ -409,9 +409,9 @@ def test_evidence_no_element_accepts_is_not_sent(self): data.process() request_type, url, content = engine.http_client.requests[-1] - self.assertEqual("iPhone", content["user-agent"]) - self.assertNotIn("x-test", content) - self.assertNotIn("other", content) + self.assertEqual("a header", content["x-test"]) + self.assertEqual("a cookie", content["other"]) + self.assertEqual("iPhone", content["User-Agent"]) def test_properties_added_later_are_normalised_and_checked(self): """ From 026a46be916a9e082f9380c89b949e6378fd45d8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 15:05:49 +0000 Subject: [PATCH 50/68] Pause usage sharing sends after repeated failures (#53) * Test that usage sharing pauses sending after failed sends Usage sharing in Python has no back-off. Every package is sent while the service is failing, and is lost. These tests expect the back-off .NET is being given, at its default settings of ten failed sends within 100 seconds and a first pause of two seconds: - after ten failed sends nothing is sent during the pause, and the evidence processed during it is all sent once the pause ends and the service answers, with no more requests processed; - closing while paused returns at once and sends nothing. The test receiver can now answer with another status, and records every request it gets as well as the data it accepts. The tests fail until the back-off is added. * Pause usage sharing sends after repeated failures Python now has the same back-off as .NET, with the same five settings and defaults, as constructor arguments. - After failures_to_enter_recovery sends fail within failures_window_seconds, sending pauses for initial_recovery_delay_seconds. A send fails when it raises or the answer is not 200. - When a pause ends one send is tried. If it fails the delay is multiplied by recovery_multiplier, up to max_recovery_delay_seconds. If it succeeds the back-off ends. - Processing is not affected. Evidence is queued during a pause, and new evidence is dropped only when the queue is full. The new max_queue_size argument sets the size, by default 1000 or ten times the requested package size if that is larger, as in .NET. The queue had no limit before, which a pause would have needed. - One thread sends at a time, as in .NET and Java. It sends until fewer than requested_package_size entries are queued. A timer that does not keep the process running starts sending when a pause ends, so the queue drains without more requests being processed. - Closing cancels the timer and sends nothing while paused, logging how many entries were not sent. ShareUsageBackOff holds the rules and takes the time from the caller, so its tests set the time. send_thread now returns whether the service accepted the message. * Test the full queue log, the failed answer log, message size, close and the repeat boundary * Log a full queue and the content of a failed answer, send up to twice the package size, send what is queued on close and share a repeat at the interval * Build the Devices message in one place, say the message size in the docstring, and give the constants their units * Guard the failure deque against reading from an empty queue * Take the message inside the lock so the name is always bound --- .../share_usage.py | 283 +++++++++++++-- .../share_usage_backoff.py | 161 +++++++++ .../share_usage_tracker.py | 2 +- .../tests/test_share_usage_backoff.py | 122 +++++++ .../tests/test_share_usage_tracker.py | 82 +++++ .../tests/test_usage.py | 322 +++++++++++++++++- 6 files changed, 948 insertions(+), 24 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_backoff.py create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_backoff.py create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_tracker.py diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index d0a1f913..a94a1c8e 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -26,6 +26,8 @@ import gzip import platform import datetime +import threading +import time from concurrent.futures import ThreadPoolExecutor from importlib.metadata import version @@ -34,6 +36,10 @@ BasicListEvidenceKeyFilter from fiftyone_pipeline_engines.engine import Engine +from .share_usage_backoff import ShareUsageBackOff, \ + FAILURES_TO_ENTER_RECOVERY_DEFAULT, FAILURES_WINDOW_SECONDS_DEFAULT, \ + INITIAL_RECOVERY_DELAY_SECONDS_DEFAULT, \ + MAX_RECOVERY_DELAY_SECONDS_DEFAULT, RECOVERY_MULTIPLIER_DEFAULT from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter, \ ShareUsageTrackerEvidenceKeyFilter from .share_usage_tracker import ShareUsageTracker @@ -52,6 +58,30 @@ # The default number of seconds to wait for the server connection before failing the request when sending usage data REQUEST_TIMEOUT = 60 +# The default maximum number of entries waiting to be sent, unless ten +# times the requested package size is larger. +MAX_QUEUE_SIZE_DEFAULT = 1000 + +# While the queue is full, a message is logged for the first entry discarded +# and then once for every this many more, never one for each entry. +QUEUE_FULL_LOG_EVERY_ENTRIES = 1000 + +# The most characters of an answer's content that are logged. +LOGGED_CONTENT_CHARACTERS = 500 + +# The longest single wait during a pause. Longer pauses are waited out in +# parts, as a timer cannot wait for any length of time on every platform. +LONGEST_WAIT_SECONDS = 86400 + + +def devices_message(entries): + """! + The text of one message, being the queued Device entries inside the + Devices element. + """ + return (f'' + f'{"".join(entries)}') + class ShareUsage(Engine): def __init__( @@ -65,7 +95,13 @@ def __init__( endpoint = "https://devices-v4.51degrees.com/new.ashx", request_timeout=REQUEST_TIMEOUT, share_all = False, - track_session = False): + track_session = False, + max_queue_size = None, + failures_to_enter_recovery = FAILURES_TO_ENTER_RECOVERY_DEFAULT, + failures_window_seconds = FAILURES_WINDOW_SECONDS_DEFAULT, + initial_recovery_delay_seconds = INITIAL_RECOVERY_DELAY_SECONDS_DEFAULT, + max_recovery_delay_seconds = MAX_RECOVERY_DELAY_SECONDS_DEFAULT, + recovery_multiplier = RECOVERY_MULTIPLIER_DEFAULT): """! Constructor for ShareUsage element @@ -107,6 +143,26 @@ def __init__( @param share_all: If True, all evidence is shared apart from the email address, the salt used for hashed email identifiers, and the request's query string and path. False by default. + @type max_queue_size: int + @param max_queue_size: The most entries held while waiting to be + sent. When the queue is full, new entries are dropped. The default + is 1000, or ten times requested_package_size if that is larger. + @type failures_to_enter_recovery: int + @param failures_to_enter_recovery: The number of failed sends within + failures_window_seconds that pauses sending, from 1 to 100. Usage + data is still queued while sending is paused. 10 by default. + @type failures_window_seconds: int + @param failures_window_seconds: The time in seconds over which + failed sends are counted. 100 by default. + @type initial_recovery_delay_seconds: float + @param initial_recovery_delay_seconds: The length in seconds of the + first pause after failed sends. 2.0 by default. + @type max_recovery_delay_seconds: float + @param max_recovery_delay_seconds: The longest a pause can be, in + seconds. 300.0 by default. + @type recovery_multiplier: float + @param recovery_multiplier: What the pause is multiplied by each time + the send tried when a pause ends fails. 2.0 by default. """ super(ShareUsage, self).__init__() @@ -150,30 +206,100 @@ def __init__( self.tracker = ShareUsageTracker(interval = interval) - # Initialise share data list + # The entries waiting to be sent, oldest first. self.share_data = [] + # The entries discarded since the queue last had room. + self._discarded = 0 + + # A message holds at most twice the requested package size. + self._message_size = 2 * requested_package_size + + self.max_queue_size = max_queue_size if max_queue_size is not None \ + else max(MAX_QUEUE_SIZE_DEFAULT, 10 * requested_package_size) + if self.max_queue_size < requested_package_size: + raise ValueError( + "max_queue_size cannot be smaller than " + "requested_package_size.") + + # Pauses sending after repeated failed sends. + self.backoff = ShareUsageBackOff( + failures_to_enter_recovery, + failures_window_seconds, + initial_recovery_delay_seconds, + max_recovery_delay_seconds, + recovery_multiplier) + + # Guards the queue, the sending flag and the resume timer. + self._lock = threading.Lock() + # True while a thread is sending. + self._sending = False + # Starts sending again when a pause ends. + self._resume_timer = None + # Set when the element closes. + self._closing = threading.Event() + self.constant_xml = None - # Reusable thread pool for sending usage in background - # According to the doc on ThreadPoolExecutor - all threads will be joined before the interpreter can exit - # this ensures that all the data will be sent correctly before the actual exit happens + # One thread sends at a time, so a failing service gets one request + # at a time. According to the doc on ThreadPoolExecutor - all threads + # will be joined before the interpreter can exit, so a message being + # sent is finished before the actual exit happens. - self.thread_pool = ThreadPoolExecutor(thread_name_prefix='ShareUsage') + self.thread_pool = ThreadPoolExecutor( + max_workers=1, thread_name_prefix='ShareUsage') def __del__(self): - self.thread_pool.shutdown(wait=True) + thread_pool = getattr(self, "thread_pool", None) + if thread_pool is not None: + thread_pool.shutdown(wait=True) def close(self): """! - Wait for any usage data already being sent, then stop the threads - that send it. The pipeline calls this when it is closed. + Wait for any usage data already being sent, then send what is still + queued and stop the threads that send it. The pipeline calls this + when it is closed. Closing does not wait for a pause after failed + sends, and sends nothing while paused. """ super(ShareUsage, self).close() + self._closing.set() + with self._lock: + if self._resume_timer is not None: + self._resume_timer.cancel() + self._resume_timer = None self.thread_pool.shutdown(wait=True) + self._send_remaining() + + def _send_remaining(self): + """! + Send the entries still queued when closing, even if there are fewer + than the requested package size. Nothing is sent while paused, or + after a message fails, and the entries left are discarded and + counted in the log. + """ + + while True: + with self._lock: + queued = len(self.share_data) + if queued == 0: + return + if self.backoff.remaining(time.monotonic()) > 0: + break + share_data = self.share_data[:self._message_size] + del self.share_data[:self._message_size] + data = devices_message(share_data) + if not self.send_thread(self.endpoint, data): + with self._lock: + queued = len(self.share_data) + break + if queued > 0: + self._log( + "info", + f"{queued} usage sharing entries were not sent, as sending " + "had failed when usage sharing closed.") def get_constant_xml(self): if self.constant_xml is None: @@ -260,29 +386,126 @@ def found_evidence(self, flow_data): def share_send_usage(self): """! - Internal method to send the share usage bundle to the 51Degrees servers + Internal method to start sending the queued usage data to the + 51Degrees servers. Nothing starts if a thread is already sending, + if fewer than requested_package_size entries are queued, or while + sending is paused after failed sends, in which case a timer starts + sending when the pause ends. """ - share_data = self.share_data - self.share_data = [] + with self._lock: + if self._sending or self._closing.is_set(): + return + pause = self.backoff.remaining(time.monotonic()) + if pause > 0: + self._schedule_resume(pause) + return + if len(self.share_data) < self.requested_package_size: + return + self._sending = True + try: + self.thread_pool.submit(self._send_while_queued) + except RuntimeError: + # The thread pool has been shut down, so nothing more is sent. + with self._lock: + self._sending = False - data = f'{"".join(share_data)}' + def _schedule_resume(self, pause): + """! + Start sending when the pause ends. Called with the lock held. + """ + + if self._resume_timer is None and not self._closing.is_set(): + self._resume_timer = threading.Timer( + min(pause, LONGEST_WAIT_SECONDS), self._resume) + # The timer does not keep the process running. + self._resume_timer.daemon = True + self._resume_timer.start() - self.thread_pool.submit(self.send_thread, self.endpoint, data) + def _resume(self): + with self._lock: + self._resume_timer = None + self.share_send_usage() + + def _send_while_queued(self): + """! + Send messages of up to twice requested_package_size entries until + fewer than requested_package_size are queued, or a failed send pauses + sending. + """ + + try: + self._send_messages() + except Exception as e: + with self._lock: + self._sending = False + self._log("error", f"ShareUsage failed sending data: {e}") + + def _send_messages(self): + while True: + with self._lock: + share_data = self.share_data[:self._message_size] + del self.share_data[:self._message_size] + if not share_data: + self._sending = False + return + + data = devices_message(share_data) + accepted = self.send_thread(self.endpoint, data) + + with self._lock: + resumed = False + pause = 0 + if accepted: + resumed = self.backoff.succeeded() + else: + pause = self.backoff.failed(time.monotonic()) + if pause > 0: + self._schedule_resume(pause) + done = pause > 0 or \ + len(self.share_data) < self.requested_package_size + if done: + self._sending = False + + if resumed: + self._log("info", "Usage sharing resumed after a send succeeded.") + if pause > 0: + self._log( + "warning", + f"Usage sharing paused for {pause} seconds after failed " + "sends. Usage data is still queued, up to the maximum " + "queue size, and sending starts again when the pause " + "ends.") + if done: + return def send_thread(self, endpoint, data): + """! + Send one message of usage data. + + @return: True if the service accepted it + """ + data = gzip.compress(bytearray(data, encoding='utf8')) # setting a reasonable time out and catching any exceptions try: - requests.post(endpoint, headers={"Content-Encoding": "gzip", "Content-Type": "text/xml"}, data=data, + response = requests.post(endpoint, headers={"Content-Encoding": "gzip", "Content-Type": "text/xml"}, data=data, timeout=self.request_timeout) except Exception as e: - print(e) self._log( "error", f"ShareUsage failed sending {len(data)} bytes of data: {e}" ) + return False + if response.status_code != 200: + self._log( + "warning", + f"ShareUsage sent {len(data)} bytes of data and the service " + f"answered {response.status_code} with content " + f"'{response.text[:LOGGED_CONTENT_CHARACTERS]}'.") + return False + return True def add_to_share_usage(self, data): """! @@ -325,13 +548,33 @@ def add_to_share_usage(self, data): xml += record_element(entry_key, entry_value) xml += "" - - self.share_data.append(xml) + + with self._lock: + # When the queue is full, the newest entries are dropped. + if len(self.share_data) >= self.max_queue_size: + self._discarded += 1 + discarded = self._discarded + log_full = (discarded == 1 or + discarded % QUEUE_FULL_LOG_EVERY_ENTRIES == 0) + else: + discarded = 0 + log_full = False + self._discarded = 0 + self.share_data.append(xml) + + if discarded > 0: + if log_full: + self._log( + "warning", + f"The usage sharing queue is full at " + f"{self.max_queue_size} entries, so usage data is being " + f"discarded. Discarded since the queue last had room: " + f"{discarded}.") + return # Send share usage data if data size greater than requested # package size - if len(self.share_data) >= self.requested_package_size: - self.share_send_usage() + self.share_send_usage() def process_internal(self, flow_data): diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_backoff.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_backoff.py new file mode 100644 index 00000000..53996ede --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_backoff.py @@ -0,0 +1,161 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +from collections import deque +import threading + +# The default number of failed sends within the failures window that +# pauses sending. +FAILURES_TO_ENTER_RECOVERY_DEFAULT = 10 + +# The range of failed sends that can be set to pause sending. +FAILURES_TO_ENTER_RECOVERY_MIN = 1 +FAILURES_TO_ENTER_RECOVERY_MAX = 100 + +# The default time in seconds over which failed sends are counted. +FAILURES_WINDOW_SECONDS_DEFAULT = 100 + +# The default length in seconds of the first pause after failed sends. +INITIAL_RECOVERY_DELAY_SECONDS_DEFAULT = 2.0 + +# The default longest pause in seconds. +MAX_RECOVERY_DELAY_SECONDS_DEFAULT = 300.0 + +# The default multiplier applied to the pause each time the send tried +# when a pause ends fails. +RECOVERY_MULTIPLIER_DEFAULT = 2.0 + + +class ShareUsageBackOff: + """! + Decides when usage sharing may send. Sending pauses for the initial + delay once the set number of sends fail within the window. A failed + send after a pause multiplies the delay, up to the maximum, and a + successful send ends the back-off. Only sending pauses, so evidence is + still queued while paused. Each method takes the time from the caller, + in seconds. + """ + + def __init__( + self, + failures_to_enter_recovery = FAILURES_TO_ENTER_RECOVERY_DEFAULT, + failures_window_seconds = FAILURES_WINDOW_SECONDS_DEFAULT, + initial_recovery_delay_seconds = INITIAL_RECOVERY_DELAY_SECONDS_DEFAULT, + max_recovery_delay_seconds = MAX_RECOVERY_DELAY_SECONDS_DEFAULT, + recovery_multiplier = RECOVERY_MULTIPLIER_DEFAULT): + + if not (isinstance(failures_to_enter_recovery, int) + and FAILURES_TO_ENTER_RECOVERY_MIN <= failures_to_enter_recovery + <= FAILURES_TO_ENTER_RECOVERY_MAX): + raise ValueError( + "failures_to_enter_recovery must be from " + f"{FAILURES_TO_ENTER_RECOVERY_MIN} to " + f"{FAILURES_TO_ENTER_RECOVERY_MAX}. " + f"Received: {failures_to_enter_recovery}.") + if not failures_window_seconds >= 1: + raise ValueError("failures_window_seconds must be 1 or more.") + if not initial_recovery_delay_seconds > 0: + raise ValueError( + "initial_recovery_delay_seconds must be more than 0.") + if not max_recovery_delay_seconds > 0: + raise ValueError( + "max_recovery_delay_seconds must be more than 0.") + if not recovery_multiplier >= 1: + raise ValueError("recovery_multiplier must be 1 or more.") + + self.failures_to_enter_recovery = failures_to_enter_recovery + self.failures_window_seconds = failures_window_seconds + self.initial_recovery_delay_seconds = initial_recovery_delay_seconds + self.max_recovery_delay_seconds = max_recovery_delay_seconds + self.recovery_multiplier = recovery_multiplier + + self._lock = threading.Lock() + # When each recent failed send happened, oldest first. Empty while + # backing off. + self._failures = deque() + # The length of the current pause, or zero when not backing off. + self._delay = 0 + # When sending may start again. + self._resume_at = None + + @property + def backing_off(self): + """! + True from the start of a pause until a send succeeds. + """ + with self._lock: + return self._delay > 0 + + def remaining(self, now): + """! + The seconds left before sending may start again, or zero if it may + start now. + """ + with self._lock: + if self._resume_at is None or self._resume_at <= now: + return 0 + return self._resume_at - now + + def failed(self, now): + """! + Record a failed send. + + @return: the new pause in seconds if this failure started or + lengthened one, otherwise zero + """ + with self._lock: + if self._delay > 0: + # The send tried when the last pause ended failed. + self._delay = min( + self._delay * self.recovery_multiplier, + self.max_recovery_delay_seconds) + else: + self._failures.append(now) + # The emptiness check comes first so that nothing is read + # from an empty deque. The constructor refuses a window + # below one second, so the deque always keeps the failure + # just added. + while (self._failures + and self._failures[0] + < now - self.failures_window_seconds): + self._failures.popleft() + if len(self._failures) < self.failures_to_enter_recovery: + return 0 + self._failures.clear() + self._delay = min( + self.initial_recovery_delay_seconds, + self.max_recovery_delay_seconds) + self._resume_at = now + self._delay + return self._delay + + def succeeded(self): + """! + Record a successful send. + + @return: True if this ended a back-off + """ + with self._lock: + if self._delay == 0: + return False + self._delay = 0 + self._resume_at = None + return True diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_tracker.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_tracker.py index 694c3de5..ef7f3d99 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_tracker.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_tracker.py @@ -59,7 +59,7 @@ def match(self, key, value): difference = time.time() - value - if difference > self.interval: + if difference >= self.interval: self.set_cache_value(key, value) return True else: diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_backoff.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_backoff.py new file mode 100644 index 00000000..762b3093 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_backoff.py @@ -0,0 +1,122 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_pipeline_engines_fiftyone.share_usage_backoff import \ + ShareUsageBackOff + + +def create(failures=3, initial=2, maximum=5): + """! + Three failures within a ten second window start a two second pause, + rising by two each time up to five. + """ + return ShareUsageBackOff(failures, 10, initial, maximum, 2) + + +class ShareUsageBackOffTests(unittest.TestCase): + """! + The rules that decide when usage sharing may send after failed sends, + with each test setting the time. + """ + + def test_pauses_after_failures_within_window(self): + backoff = create() + + self.assertEqual(backoff.failed(0), 0) + self.assertEqual(backoff.failed(1), 0) + self.assertEqual(backoff.remaining(1), 0, + "Sending paused before enough sends failed.") + self.assertFalse(backoff.backing_off) + + self.assertEqual(backoff.failed(2), 2) + self.assertTrue(backoff.backing_off) + self.assertEqual(backoff.remaining(3), 1) + self.assertEqual(backoff.remaining(4), 0) + + def test_failures_outside_window_do_not_count(self): + backoff = create() + + backoff.failed(0) + backoff.failed(1) + + self.assertEqual(backoff.failed(11.5), 0, + "Failures older than the window were counted.") + self.assertFalse(backoff.backing_off) + + def test_successes_between_failures_do_not_reset(self): + """! + The window counts failures, not failures in a row, so a high + failure rate pauses sending even when some sends succeed. + """ + backoff = create() + + backoff.failed(0) + self.assertFalse(backoff.succeeded()) + backoff.failed(1) + self.assertFalse(backoff.succeeded()) + + self.assertEqual(backoff.failed(2), 2) + + def test_failed_send_after_pause_multiplies_delay(self): + backoff = create(failures=1) + + self.assertEqual(backoff.failed(0), 2) + self.assertEqual(backoff.failed(2), 4) + self.assertEqual(backoff.failed(6), 5, + "The delay grew past the maximum.") + self.assertEqual(backoff.failed(11), 5) + self.assertEqual(backoff.remaining(11), 5) + + def test_success_ends_back_off(self): + backoff = create(failures=2) + backoff.failed(0) + backoff.failed(1) + backoff.failed(3) + + self.assertTrue(backoff.succeeded(), + "A success did not end the back-off.") + self.assertFalse(backoff.backing_off) + self.assertEqual(backoff.remaining(4), 0) + + # The next pause needs the full number of failures again, and + # starts from the initial delay. + self.assertEqual(backoff.failed(8), 0) + self.assertEqual(backoff.failed(9), 2) + + def test_maximum_below_initial_limits_first_pause(self): + backoff = create(failures=1, initial=10, maximum=3) + + self.assertEqual(backoff.failed(0), 3) + + def test_invalid_settings_are_rejected(self): + for settings in [ + (0, 10, 2, 5, 2), + (101, 10, 2, 5, 2), + (1, 0, 2, 5, 2), + (1, 10, 0, 5, 2), + (1, 10, float("nan"), 5, 2), + (1, 10, 2, 0, 2), + (1, 10, 2, 5, 0.5)]: + with self.assertRaises(ValueError, msg=str(settings)): + ShareUsageBackOff(*settings) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_tracker.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_tracker.py new file mode 100644 index 00000000..63cc9737 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_tracker.py @@ -0,0 +1,82 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. + +import unittest +from unittest import mock + +from fiftyone_pipeline_engines_fiftyone.share_usage_tracker import \ + ShareUsageTracker + +# The tracker reads the time through this name. +TIME = "fiftyone_pipeline_engines_fiftyone.share_usage_tracker.time.time" + +INTERVAL_SECONDS = 60 +START = 1000000.0 + + +class ShareUsageTrackerTests(unittest.TestCase): + + def track_at(self, tracker, key, seconds): + """! + Track the key with the clock at the given seconds after the start. + """ + with mock.patch(TIME, return_value=START + seconds): + return tracker.track(key) + + def share_at(self, tracker, key, seconds): + """! + Track the key as the element does, storing the time when it is shared. + """ + with mock.patch(TIME, return_value=START + seconds): + shared = tracker.track(key) + if shared: + tracker.set_cache_value(key) + return shared + + def test_repeat_inside_the_interval_is_not_shared(self): + tracker = ShareUsageTracker(interval=INTERVAL_SECONDS) + self.assertTrue(self.share_at(tracker, "a", 0)) + + self.assertFalse( + self.track_at(tracker, "a", INTERVAL_SECONDS - 1)) + + def test_repeat_at_the_interval_is_shared(self): + tracker = ShareUsageTracker(interval=INTERVAL_SECONDS) + self.assertTrue(self.share_at(tracker, "a", 0)) + + self.assertTrue(self.track_at(tracker, "a", INTERVAL_SECONDS)) + + def test_repeat_after_the_interval_is_shared(self): + tracker = ShareUsageTracker(interval=INTERVAL_SECONDS) + self.assertTrue(self.share_at(tracker, "a", 0)) + + self.assertTrue(self.track_at(tracker, "a", INTERVAL_SECONDS + 1)) + + def test_repeat_inside_the_interval_leaves_the_time(self): + tracker = ShareUsageTracker(interval=INTERVAL_SECONDS) + self.assertTrue(self.share_at(tracker, "a", 0)) + self.assertFalse(self.share_at(tracker, "a", 30)) + + self.assertTrue(self.track_at(tracker, "a", INTERVAL_SECONDS)) + + +if __name__ == '__main__': + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index 66e9b4e5..1fd819cb 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -20,9 +20,13 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +import contextlib +import io import platform +import re import sys import threading +import time from time import sleep import unittest from wsgiref.simple_server import make_server @@ -59,25 +63,48 @@ def get_received(self): app = Flask(__name__) app.received = [] +# When each request arrived, whatever the answer. +app.attempts = [] +# The status the receiver answers with. Only data it accepts is received. +app.status = 200 +# The content the receiver answers with. +app.body = "" @app.route('/', methods=['POST']) def server(): + app.attempts.append(time.monotonic()) data = gzip.decompress(request.data).decode("utf8") - str(app.received.append(data)) - return "" + if app.status == 200: + app.received.append(data) + return app.body, app.status + + +def client_ips(received): + """! + The client IPs in the usage data received. + """ + ips = [] + for data in received: + ips.extend(re.findall("([^<]*)", data)) + return ips ###################################### # The Tests class UsageTests(unittest.TestCase): - + @classmethod def setUpClass(cls): cls.startReceiver() def setUp(self): app.received = [] + app.attempts = [] + app.status = 200 + app.body = "" + self.requests = 0 + self.logs = [] @classmethod def tearDownClass(cls): @@ -97,6 +124,295 @@ def waitForUsageThreads(self): # allowing the thread pool in ShareUsage to start sending data and release the GIL sleep(0.1) + def wait_for(self, condition, message): + """! + Wait for a condition, failing the test with the message if it is not + met within 30 seconds. + """ + end = time.monotonic() + 30 + while not condition(): + if time.monotonic() > end: + self.fail(message()) + sleep(0.02) + + def back_off_pipeline(self, **settings): + """! + A pipeline whose usage sharing element sends each entry on its own. + The element is closed when the test ends. + """ + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + **settings) + pipeline = PipelineBuilder().add(usage).build() + self.addCleanup(usage.close) + return usage, pipeline + + def process_unique(self, pipeline): + """! + Process a request whose evidence no other request has, and return + its client IP. + """ + ip = f"10.0.{self.requests // 250}.{self.requests % 250 + 1}" + self.requests += 1 + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("server.client-ip", ip) + flowdata.process() + return ip + + def fail_sends(self, pipeline, count): + """! + Process requests one at a time, waiting for each to be sent, so each + is a separate failed send. + """ + app.status = 500 + for sent in range(1, count + 1): + self.process_unique(pipeline) + self.wait_for( + lambda: len(app.attempts) == sent, + lambda: f"Send {sent} was not made.") + + def test_back_off_pauses_then_drains_queue(self): + """! + Ten failed sends pause sending for two seconds, the defaults. + Nothing is sent during the pause, the evidence processed during it + is queued, and all of it is sent once the pause ends and the + service answers, with no more requests processed. + """ + + usage, pipeline = self.back_off_pipeline() + self.fail_sends(pipeline, 10) + paused_at = time.monotonic() + + queued = [self.process_unique(pipeline) for i in range(5)] + sleep(0.7) + self.assertEqual( + len(app.attempts), 10, + "Sends were made during the pause that ten failed sends " + "should have started.") + + app.status = 200 + self.wait_for( + lambda: all(ip in client_ips(app.received) for ip in queued), + lambda: "The evidence processed during the pause was not all " + f"sent once the service answered. Sent: " + f"{client_ips(app.received)}") + waited = app.attempts[10] - paused_at + self.assertGreaterEqual( + waited, 1.5, + f"Sending resumed {waited:.2f} seconds after the pause started, " + "before the two second pause ended.") + + def test_back_off_close_while_paused(self): + """! + Closing while paused returns at once rather than waiting for the + pause to end, and sends nothing. + """ + + usage, pipeline = self.back_off_pipeline() + self.fail_sends(pipeline, 10) + self.process_unique(pipeline) + self.process_unique(pipeline) + + start = time.monotonic() + usage.close() + took = time.monotonic() - start + + self.assertLess( + took, 1.5, + f"Closing took {took:.2f} seconds, as it waited for the pause.") + self.assertEqual( + len(app.attempts), 10, + "A send was made during the pause or when closing.") + + def test_back_off_delay_grows_to_maximum(self): + """! + Each failed send after a pause multiplies the delay, and the delay + stops growing at the maximum. + """ + + usage, pipeline = self.back_off_pipeline( + failures_to_enter_recovery=1, + initial_recovery_delay_seconds=0.5, + max_recovery_delay_seconds=1.0) + app.status = 500 + for i in range(40): + self.process_unique(pipeline) + + self.wait_for( + lambda: len(app.attempts) >= 4, + lambda: "Fewer than four sends were made.") + gaps = [app.attempts[i] - app.attempts[i - 1] for i in range(1, 4)] + shown = ", ".join(f"{gap:.2f}" for gap in gaps) + self.assertGreaterEqual( + gaps[0], 0.4, + f"The first pause was not the initial delay. Gaps: {shown}") + self.assertGreaterEqual( + gaps[1], 0.9, + f"The second pause was not doubled. Gaps: {shown}") + self.assertTrue( + 0.9 <= gaps[2] < 1.9, + f"The third pause was not held at the maximum. Gaps: {shown}") + + def test_back_off_full_queue_drops_new_evidence(self): + """! + When the queue fills during a pause, new evidence is dropped and + what was queued first is sent once sending resumes. + """ + + usage, pipeline = self.back_off_pipeline( + max_queue_size=4, + failures_to_enter_recovery=1, + initial_recovery_delay_seconds=2, + max_recovery_delay_seconds=2) + self.fail_sends(pipeline, 1) + + kept = [self.process_unique(pipeline) for i in range(4)] + dropped = [self.process_unique(pipeline) for i in range(6)] + app.status = 200 + + self.wait_for( + lambda: all(ip in client_ips(app.received) for ip in kept), + lambda: "The evidence queued during the pause was not all sent. " + f"Sent: {client_ips(app.received)}") + sleep(0.5) + for ip in dropped: + self.assertNotIn( + ip, client_ips(app.received), + "Evidence processed when the queue was full was sent.") + + def test_full_queue_logs_without_a_message_per_entry(self): + """! + Entries discarded because the queue is full are logged, but not with + a message for each entry discarded. + """ + + usage, pipeline = self.back_off_pipeline( + max_queue_size=4, + failures_to_enter_recovery=1, + initial_recovery_delay_seconds=2, + max_recovery_delay_seconds=2) + usage._log = lambda level, message: self.logs.append((level, message)) + self.fail_sends(pipeline, 1) + self.logs.clear() + + for i in range(10): + self.process_unique(pipeline) + + full = [log for log in self.logs if "full" in log[1]] + self.assertGreaterEqual( + len(full), 1, "Discarding entries was not logged.") + self.assertLess( + len(full), 6, "A message was logged for each entry discarded.") + self.assertEqual("warning", full[0][0]) + + def test_failed_answer_logs_status_and_content(self): + """! + A send answered with anything other than 200 logs the status code and + the content of the answer. + """ + + usage, pipeline = self.back_off_pipeline() + usage._log = lambda level, message: self.logs.append((level, message)) + app.body = "the service says no" + + self.fail_sends(pipeline, 1) + + self.wait_for( + lambda: any("500" in log[1] for log in self.logs), + lambda: f"The status code was not logged. Logged: {self.logs}") + failed = [log for log in self.logs if "500" in log[1]][0] + self.assertIn("the service says no", failed[1]) + + def test_failed_send_prints_nothing(self): + """! + A send that cannot be made is logged and does not print. + """ + + usage, pipeline = self.back_off_pipeline(request_timeout=1) + usage._log = lambda level, message: self.logs.append((level, message)) + usage.endpoint = "http://127.0.0.1:1" + printed = io.StringIO() + + with contextlib.redirect_stdout(printed): + self.process_unique(pipeline) + self.wait_for( + lambda: any("failed sending" in log[1] for log in self.logs), + lambda: f"The failed send was not logged. Logged: {self.logs}") + + self.assertEqual("", printed.getvalue()) + + def test_message_holds_up_to_twice_the_package_size(self): + """! + A message holds the requested package size or more entries, up to + twice the package size, so a queue that filled during a pause drains + in fewer messages. + """ + + usage = ShareUsage( + requested_package_size=2, + endpoint=testEndpoint, + failures_to_enter_recovery=1, + initial_recovery_delay_seconds=1, + max_recovery_delay_seconds=1) + pipeline = PipelineBuilder().add(usage).build() + self.addCleanup(usage.close) + app.status = 500 + self.process_unique(pipeline) + self.process_unique(pipeline) + self.wait_for( + lambda: len(app.attempts) == 1, + lambda: "The first send was not made.") + for i in range(6): + self.process_unique(pipeline) + + app.status = 200 + self.wait_for( + lambda: len(client_ips(app.received)) == 6, + lambda: f"Not all entries were sent: {client_ips(app.received)}") + + sizes = [len(client_ips([data])) for data in app.received] + self.assertEqual( + [4, 2], sizes, + "The messages did not hold up to twice the package size.") + + def test_close_sends_fewer_entries_than_the_package_size(self): + """! + Closing sends the entries still queued, even when there are fewer + than the requested package size. + """ + + usage = ShareUsage( + requested_package_size=5, + endpoint=testEndpoint) + pipeline = PipelineBuilder().add(usage).build() + ips = [self.process_unique(pipeline) for i in range(2)] + self.assertEqual([], app.received) + + usage.close() + + self.assertEqual(ips, client_ips(app.received)) + + def test_back_off_invalid_settings(self): + """! + Settings the pause cannot work with are rejected. + """ + + for settings in [ + {"failures_to_enter_recovery": 0}, + {"failures_to_enter_recovery": 101}, + {"failures_to_enter_recovery": 2.5}, + {"failures_window_seconds": 0}, + {"initial_recovery_delay_seconds": 0}, + {"max_recovery_delay_seconds": -1}, + {"recovery_multiplier": 0.5}, + {"max_queue_size": 0}]: + with self.assertRaises(ValueError, msg=str(settings)): + ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + **settings) + def test_data_received(self): """! Check that data is received from the share usage element. From cb2b1c9a119116e93098482ffca0721d87ea1d6a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 15:13:03 +0000 Subject: [PATCH 51/68] Show the root logo in every README, add one contributing guide, and fix the README links (#46) * Show the root logo in every README, add one contributing guide, and fix the README links Twenty READMEs showed an older 51Degrees wordmark from 51degrees.com rather than the logo the root README shows, and the fiftyone_pipeline_did package README had none. Each now shows the root logo as a Markdown image from raw.githubusercontent.com, using .github/images/logo-160.png, a 160 pixel copy of .github/images/logo.png, the size the root README shows it at. The repository had no contributing guide of its own. One CONTRIBUTING.md at the root now links the 51Degrees contributing guide. The Developer Documentation link at the top of thirteen READMEs went to the main page of the product's generated API reference, which has no content, and the translation package's went to the documentation home page. They now go to the product's documentation. The dependencies page that four READMEs linked is gone, and the next sentence in each already links the tested versions page, so the sentence is removed. * Link each product's quick start from the READMEs The documentation links went to section pages of the 51Degrees documentation, such as _device_detection__index.html, which only list the pages in the section. They now go to the page in each section that is most helpful to a developer and the most recently kept up to date. Those are the Device Detection, IP Intelligence and Reverse Geocoding quick starts, which show how to choose between the cloud and on-premise and how to start in each language, and Pipeline Basics for the Pipeline API, which has no quick start. The campaign tags of each link are unchanged. * Point each Find out more documentation line at the documentation home --- .github/images/logo-160.png | Bin 0 -> 7216 bytes CONTRIBUTING.md | 5 +++++ .../fiftyone_devicedetection/readme.md | 6 +++--- .../fiftyone_devicedetection_cloud/readme.md | 6 +++--- .../fiftyone_devicedetection_examples/readme.md | 4 ++-- .../fiftyone_devicedetection_onpremise/readme.md | 6 +++--- .../fiftyone_devicedetection_shared/readme.md | 6 +++--- device-detection/readme.md | 5 ++--- .../fiftyone_ipintelligence/readme.md | 6 +++--- .../fiftyone_ipintelligence_cloud/readme.md | 6 +++--- .../fiftyone_ipintelligence_examples/readme.md | 4 ++-- .../fiftyone_ipintelligence_onpremise/readme.md | 6 +++--- .../fiftyone_ipintelligence_shared/readme.md | 6 +++--- .../readme.md | 6 +++--- ip-intelligence/readme.md | 8 +++----- location/readme.md | 7 +++---- .../readme.md | 6 +++--- pipeline/fiftyone_pipeline_core/readme.md | 6 +++--- pipeline/fiftyone_pipeline_did/readme.md | 2 ++ pipeline/fiftyone_pipeline_engines/readme.md | 6 +++--- .../fiftyone_pipeline_engines_fiftyone/readme.md | 6 +++--- pipeline/fiftyone_pipeline_translation/readme.md | 6 +++--- pipeline/readme.md | 5 ++--- 23 files changed, 63 insertions(+), 61 deletions(-) create mode 100644 .github/images/logo-160.png create mode 100644 CONTRIBUTING.md mode change 100755 => 100644 pipeline/readme.md diff --git a/.github/images/logo-160.png b/.github/images/logo-160.png new file mode 100644 index 0000000000000000000000000000000000000000..cbcbc612069c53c0843e8cd301bd09c8c60729ad GIT binary patch literal 7216 zcmbuEWlS7S^saaD;?Cmk6j-3RYoWNcEV4KhEmoYxr8vc97b))UUbGY`#a#*%cP;Se zcfZ}_CO0>Ck~8yRGBYQW$vN+wCrVRY2?vt`6953$P~4}Hi#}GyvQn2#*8uuKTC=ppv{BatljDht z+tz-H^pV66#?u(7Tb`m2ssd|ZlUE3(Ey+b=Hp4R{O&YV@86R62_lB?0?uy*NeRtJa z=_8C*xVu0&La?3+vZB7hvKKy`IoX;$>yuV0ZTM}T|(KT}W5njKlnh1z-A zr2;Fv4K}V>i6>#Bq+O~kTk7G+pLUoxKR38>&;MUe3AG}J9m*` z@q;5zer*2MC?mfwt>#nPm^viqsL9t(CknXPy;WA&?hrTHDGUO`y<3OZ--hKBkf14Y zf*g4j8Ii&oCbHwOA3MECCyur!?^fJ*r?0f?7APt{0gj!I1eTudQ0i8?zDuY^o;S!#Yy zacfg47bC^QtW<*W8&e!NRcj7ED8PQooav!jAz&|tL$=fpjkdVYt_y!HrzP8oRL0}5 zNP!yGY=!j^y9Uyo%I5TqSu*WaA$9H53&rM=zV|0EbIy;@)0ES{7qc}!T0DTYdxN0r zwR*VUqTIB@W76q>bU4em=7gQ*iLNo@c=3LbA)acvt`mSQb|k)2K*oAhF@1{N_Jyy- zT`>jfWr5(uh{H8cID@&?18BR6^`+NboltT=OalQsQX z>&>fRO-3z87NF^6f%@uTI)-o%`fT_27tau9G4pFU^+34N-FW+g_kfM2eJQ2+G#8s&JKmo|o&gg5kF(tje$isv7 zo(MV6rA-txI$_S~bZKy>6|}c!^}ny$p0|kG`Sw-aa^U=r7ySLR>@{n2Nh3?4>BTr0 zO{x5nG~eO+EG?W(&)!mp24df!CY?)wY+eZ^RAjEP@EU^{EGX5RRwl{xhS450;EX)L_kQJ^*IDw~ z6itcK@1i(sm`C2MG2vwbUCzH0VnFk zoV@F6(LlES!l(mxc*_y{O2FsGkCKr4L`|X1-ppa>9Qs>{R=P3>7UjXbkx;^kf)(7+k?owTm>o0-|+(Qy$e ziqUl*-o>wIp7SYANxp;Q4@FO=!hOER(^DQE8xZEBqn+VqN6*3N?h9Wh z{FH~4l%yxE__XKc(W9ZE4ewrrUb`_#ePqdE&Kk*0wyO&U$bj;b&DEWGV4gq8PW=&h z#sLczG9|I4tVoQ1D3+p0f`vcUOc->Q2-)>#SVvoBVv>dD)pD$L-hdkf*_7K><-=o? zTP*12F^eet(FaqhI@C3d9eu$3z+4X{YfFJPYM6Gdg84+j!Swswo$Hx*g6%}#)9pxL zBj%c4E)&byC2klP3{*>F*x%>(x#R)hnYo9IT55P+b7E_~vqwuWv0n}KFQjrXj#}(P z^RX7(``Tdi51+BIv6cwqSxD4tODL4~Ki@^%|7wp*9*mKD4uSpRyK27Dik~yYcM1V%YJ9-#jJi}G*JRRCxj@|wr2j;vZoWk z_TO}S%hLH{aqHn})4$#CS*38a&EufuY$xmP$+6iYhCc1b8-xB9>6{-{ci&m12DFrD zlwr^R$U}I;Pf{b`yY~ojTV+K?pc_3X@TEQ)THPrqA>w+L-%i zVcHI8vf;Z=5@0NlCS;0M?g(}7Vg1G36ZM0o-*)L!wI#%ee>D)sFmaT&#FS$vYWAmp zaHVS;)~Sn>ZdSLkM6Y+G^nXjL^n5LiFT3iZ=Rp|ujYSaia=r`!C0c*t|7@zW`1Bj1?n{9ZLL zd`df7v>zEVbkgubta_}PRagm7=&~lm(EFcpCCHBNu`h6Blfx zO1G<(Yl!R0xlN=~jX7tlrB6}GHd{JhW6s={Oq_`P1_kCG7FXs_;2EX`g(&VUfX$^@ z9Ta5E`=3nAS*1;ZGLfaITa{WRS`sViwS+U!r|@4no;qzu<9oNg#<~HnQoU|Gik}7& zSRT66LU2v|+pHj1h%OC$6sjh3)U>W=9(K~`t%}^)Oe5|e5;qlvM%;6iY}0(E_>Z0O z4pcqwgK?L>J4_xrc^s<0P8VpOacS(jce6c4U(%_L&EqE{`Rnnaj+ECnFim_?6Cz~A zOb+wh6yVnnR%z6tiQ4M+OI>Umn*Y3Shyh*1o`Gyd6UrpZdU)pH?z6fr+mGbo)1eV1WzZQS`5 zN2aoazn_nrfSPq#G7c*crD|R^9R(Je!uohtoIpZ)3^|i_KLX9~y3nTD0Hl_AYD#29 znLp%)Wy%`Os4J+n;Tb@p=PLbICWkLr)rbu~QQ-8~toQB@mNYBtyvpYEUw zF30T)cN-c7&3FP88(eBBAd9_&tp0=NJ(R)0!J%OIE?87_lrdhM&(TS^>d=v#qt@bE zOmU8r;mDdjwh^$|RFFoe&8^s#*Rf zq1y4t)pTyu72`c#9pSV3ls!egV2&9;hOJ*&&bvnZvU@;jZ6O&} zzEe~1nRlu%lTdh&d6W_4teFg|?`)h==3KSOsMT)v z@UMWd)pLd9Rq|Fn>V_7R6}Az;9tX-2q04u7pl#lZsCw(kYQ2m9YhmgwQ?kIOYyr`c zZ0(d|R%n5~oKM>;xL@k#?sF8}{+Y#RRkd27=GjCNK|3e&@u{Wp<1?4V41jung#r;= z0&92sHrCfAPL}-%RBck1Ad=HsVw*r^%B!s8jKzwk-Id>69fB;1609_pJVd&6Pic3X3C?PByLn8lqNMqkbMe)tl42TL>ws!abE3`#`Q!N|#Uf$B@i1 zzs(GkkLmrkn89-yDR2L%=rW|8&FEhf8#trz8xuL%&soVy6~cv5C&Z`U4~=Mwqa@tI zW+E_`d+?C3yy4=QW1P>bz0Xa4g)2jG6j+lXp9|82Wp)y5&7nlO|+!97Q@lI3SC4$rlBE#CJ1 zc7h6bm^vI<_wlrffbrAK4p3WAkWAUke#@-Uziin&-Xla|!d1aXlFFrIqeK7MqhqSd z(^a7-U>e*7CHa_n`J530od8Tk7GbzjGKE3GJQ_?U*gr=qZPCw`6{%v4R%~YOSr!`i zZdG~bYACPSfziLO_O0m5HS+?$I(^6G3bwKLn-s)yygP9w{cp4p%aYaSANTXm#CVXy zR2swX04gu#tIJDOtr3qtjEL48gJk0qGeV1tA}fb3Vf~HyKNl^RF^m||p7aAtryaPb zo>LLk?bA^`n1nYGKjWDZ(aOo@*j{edUTzG9`z!IkH#mp0{bW&~*;o7D8{A9iG0sTG zif`#EEuw+mE7EVO^GMKV@bYyVPcfu)te^ zcsaRrUk%yQu@7@CUMUJpH;L&Rbw@|w?K|?%2#YVJVj(f5F~g?r(zRRf`WNT+^B6?o zO>lalibh5G`5H51yno#hY{chkk0u247)U!~8U6b+=^6HCd&TO~y1^6yZ&?CwTnOy(ecO@qnM!t-`_0L{*35-)fQ1(@ z4MN*cMl3n)y;1JZmUc55$E!VW&s-*&gPN%x^Y$+;KF5d~pvcx~eL`B@`jk>U}D{=AYXSdyxK9eSo6r8Qhx zTU!992YfxsvW&9wB>A2&a!W+oy@uUO7Ea%Fa^bwznMk|rg8rh{w#<9HQkF%WMSYh~ z_uWYf;tfQ3TziRdiERJfe^PA6);&Rje-e7VVmVQ&z}@|T!`~rOSUHXJ@|wh?&NGJX z*sUt(xR;A?WkUC>X$OAW{mhM$&edAUDT(Vd86YBxV`Cny*(@bnyJ`L7+k76SVXTm+ z2_k!7F9dO>2>bm}QI|UU{SnGvp)26zvso6#m93L&RIWe{9?9tZ?z0-DSlr;$^hwPn zN1ciBm3>LSJk|UlgP_^~2xEw@nGm!3qlqZE60oS8I_2z_Hs!pc`nPCl;jHN$1`VtX z%tsQh^M;y2o4v1ciEYJ8tZXs~6)T-r`szC0q3BKpNZbx8T_K1!^M((-XsFxU7xnjt zwiRjOLQ|y?wZ@3s?aqpp&Y7%D!{)W7!$Kiv+GZ!rRd}G*j+iEz&YV!Pz<8*aiEqIa zDSB2^H9W`cQAgawy3nH<0jCH=wyz+&0|S7?;~{Ms^_p-E@_dOK^2mQmU8b0g0_;OJv+itFq5i z;7hWG2i%h%8?>jk2i!8g{0vn+Jw2C@8crS#L7?@+`teOQ8-#>~p(h(jis`th-Kz;_ z%AI@K$2iT1TMm;Oa}eiTiEfDe+iye{K14Go0_m>42Z{&web<%EEl)N`!l#6LqU>$Z zj*n=QvpaSN2+Q3+2>DHQC470JNtvGEwC>*dW@l#qG5fcbkRZ?XWn0>emq=G>sB!fa z!P{sm?&m_zlEdJ;gXrjU;84NFK8^G2j2=SxWEu%ZtN-I`6x1EH3J;HX6&xs=C#|8O zvDAh*c_5BfNaZmc+^v3EBJZXssv{CTk@Ck;;*m%Y3pf+taiDzpoMG@ns_MK<-n(_M z{^~N+icqEVNN@~f3bF%oRb4Q%=2>8|(cHS_-Kb(AGJaJKJQ`cI^$wDO{GNw+Etr|o z92*is>M3q+bdxuBI*?~qOyJpx@g_FKS&pHKCa->j34?QXCjCwNc-tr6_Ys_harnp zqwh&6S7^WP(I0Go-2=%`CjFL04k4p5?1nz3u1{UPB=f=NewXE9U4c{{M;hnm*a&S+ zEX^=+Ad4QC6=RorYaH+c3*%y@lo2>cLf@Ay%lPtu&$6lkur|$c3nuurjLQz3E$o02To0C z1=DQ@UrcP(!yd_B0%5&|gZ4KTLoA)<&t{Z!Sq?dRW|zIzoUal+s4-g z8Gk#?4w+Psv-wZcxXt#;>`NqXKQ(x3a?jNyf9++}hVe)U#O$+8UKCCwiQSB?yc5^W zGS)TxtRI_~Icz0ZQJ<%e#TFM62(B=w563gQPs{e$$?$mB3O!7)f*A4oY|6{mHA&c{ zHHR}!siOmWH5-1Zh2@?f`eqR=s4LiAso=I;QO2zJi4_Pf%hzm-V01XJESgsyndjTe zF6!E+yON*z*TMXda$r|L?Q(s2mu*XE(`;;EhH^P)F#G%c)At-CDuvpHp5?LhB{P$d ztp`befB$Uv%>$8LjpyWP4Kb3nn}fpCN9*_=A0fFxxgq(5im>WW0%P4=`@2q$9ETl0 z0z4n~k3Y;(Z?T_~!IfUFqNL(cQ0d{N4uuGnjt_l`sXx_CT5%O=dMzZE=NKX=|IQ6UUfc2xMlr9k%S$#Cfl~iWrEy? z4lgh#o?$P6eVl1;@Kcf6m+N9i>N z7Ir)gtLsFg<-L1$=%G~m_5mBsl8+kkl5$fasV$_9CMr}Oeab#<4e1Gp^X`5dRdW&G zHXfn?vsh_bRH`Q0^o~8$M`7PSOQR0%K1>)f^o}w|2HC)e{%O#y_;#}@To<{ z`>W^^#uq8n6epk)SuLHkFbdBa;6e(p*?;XUcU8FO@3Shr#pvP-%Sk9|8b<+dui{c3 z?r%jgrAnrgQO%$6ch?TGSTaLrnpi9d4F85)(q%#%5TfX}X*wT$;tJEl9@*BT2)_jx_TzHlK zOXsENMT9>hZO5G?i7`?w?fN587Z;fPO1i_K{X-t|r|x!rqpdH1sU!S?ehhhSx-M#4 zoXq7$HYo}%etJy>0!81O`G>7-)$4_WP~ju~Qz2SYO%S!Y>P%Ic{emRKA&HAFAOCEU zAu>)y?96D}!Kh*qIQ;ZE)-(kKnN%ZP0L}cubB3E_*qRD8`PFN|T>-~K_$)kcQcWsDe;(GUvi^q%UI@W|#rqd{@E<(@-!aqrU~RS$_Wv_U_&;s_LMHzAb2C{0$etA0*2-^Ci8Erc21Fk<^v4z>k>H;H z3wKO#uA8CfPICcl10#q}q1^N?nPJOvfwWQJB4jV(4-Td7bfRj-4PhrD>oY3yih&1* zIpa8DKPL-UTh-;in8y(0qI+QrPa4P+lh*_o2I}>wqp}f@qe$A){yr8LQR5G!Z5U%` zX2^S?IEg2~IYLJx6Sv*bk6-la9gH~{njt%~wP}?2_OIkD#?*QYBSo1;TP=#&YkU^_ zJnJJnWfO0jvllLxMK)L>hXGzZUBxJT+(BF)ZS4i23-#sK0GN4BJ$G z`?04zNbMCqQ1)Sv+wikp+KSx|j8~_Ez>@~m0l%5MM8WsZiIGZEgXyh3zOQ56t%!VH zM3#8vM|9g3=}@0q{i>@8z#@J5A5O#SIS8B%l0r$vPjS*d78hGUv0y#rIt82}dXO3T f7Du(qpQL(wbGF1bYsOzlAV66`UA{`zH28l2u;%J^ literal 0 HcmV?d00001 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..ea856fa8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,5 @@ +# Contributing + +Contributions are welcome. The [51Degrees contributing guide](https://github.com/51Degrees/.github/blob/main/CONTRIBUTING.md) +explains where a change goes and how to propose it. It applies to every +package in this repository. diff --git a/device-detection/fiftyone_devicedetection/readme.md b/device-detection/fiftyone_devicedetection/readme.md index 72ffdfb4..8ebe1ab6 100644 --- a/device-detection/fiftyone_devicedetection/readme.md +++ b/device-detection/fiftyone_devicedetection/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - Device Detection -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=51degrees-device-detection-engines-device-detection "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=51degrees-device-detection-engines-device-detection "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=51degrees-device-detection-engines-device-detection "View all available properties and values") +[Developer Documentation](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=51degrees-device-detection-engines-device-detection "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=51degrees-device-detection-engines-device-detection "View all available properties and values") ## Introduction @@ -29,7 +29,7 @@ To run the tests use: ## Find out more - [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-device-detection) -- [Device Detection documentation](https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-documentation) +- [Device Detection documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-documentation) - [Getting started](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-getting-started) - [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-pricing) - https://github.com/51Degrees/51degrees-python diff --git a/device-detection/fiftyone_devicedetection_cloud/readme.md b/device-detection/fiftyone_devicedetection_cloud/readme.md index 10fe2fe4..07923fdc 100644 --- a/device-detection/fiftyone_devicedetection_cloud/readme.md +++ b/device-detection/fiftyone_devicedetection_cloud/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - Cloud -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=51degrees-device-detection-engines-cloud "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=51degrees-device-detection-engines-cloud "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=51degrees-device-detection-engines-cloud "View all available properties and values") +[Developer Documentation](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=51degrees-device-detection-engines-cloud "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=51degrees-device-detection-engines-cloud "View all available properties and values") ## Introduction @@ -43,7 +43,7 @@ To run the tests use: ## Find out more - [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-device-detection) -- [Device Detection documentation](https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-documentation) +- [Device Detection documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-documentation) - [Getting started](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-getting-started) - [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-pricing) - https://github.com/51Degrees/51degrees-python diff --git a/device-detection/fiftyone_devicedetection_examples/readme.md b/device-detection/fiftyone_devicedetection_examples/readme.md index b2041654..244df89b 100644 --- a/device-detection/fiftyone_devicedetection_examples/readme.md +++ b/device-detection/fiftyone_devicedetection_examples/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - Examples -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-readme.md&utm_term=51degrees-device-detection-engines-examples "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-readme.md&utm_term=51degrees-device-detection-engines-examples "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-readme.md&utm_term=51degrees-device-detection-engines-examples "View all available properties and values") +[Developer Documentation](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-readme.md&utm_term=51degrees-device-detection-engines-examples "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-readme.md&utm_term=51degrees-device-detection-engines-examples "View all available properties and values") ## Introduction diff --git a/device-detection/fiftyone_devicedetection_onpremise/readme.md b/device-detection/fiftyone_devicedetection_onpremise/readme.md index 5a83973d..eb1b0178 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/readme.md +++ b/device-detection/fiftyone_devicedetection_onpremise/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - On-Premise -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=51degrees-device-detection-engines-on-premise "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=51degrees-device-detection-engines-on-premise "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=51degrees-device-detection-engines-on-premise "View all available properties and values") +[Developer Documentation](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=51degrees-device-detection-engines-on-premise "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=51degrees-device-detection-engines-on-premise "View all available properties and values") ## Introduction @@ -81,7 +81,7 @@ To run the tests use: ## Find out more - [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-device-detection) -- [Device Detection documentation](https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-documentation) +- [Device Detection documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-documentation) - [Getting started](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-getting-started) - [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-pricing) - https://github.com/51Degrees/51degrees-python diff --git a/device-detection/fiftyone_devicedetection_shared/readme.md b/device-detection/fiftyone_devicedetection_shared/readme.md index 1a5deb11..3d1a780f 100644 --- a/device-detection/fiftyone_devicedetection_shared/readme.md +++ b/device-detection/fiftyone_devicedetection_shared/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - Shared -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=51degrees-device-detection-engines-shared "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=51degrees-device-detection-engines-shared "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=51degrees-device-detection-engines-shared "View all available properties and values") +[Developer Documentation](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=51degrees-device-detection-engines-shared "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=51degrees-device-detection-engines-shared "View all available properties and values") ## Introduction @@ -27,7 +27,7 @@ To run the tests use: ## Find out more - [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-device-detection) -- [Device Detection documentation](https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-documentation) +- [Device Detection documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-documentation) - [Getting started](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-getting-started) - [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-pricing) - https://github.com/51Degrees/51degrees-python diff --git a/device-detection/readme.md b/device-detection/readme.md index f5ebd952..62f44556 100644 --- a/device-detection/readme.md +++ b/device-detection/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-device-detection-engines "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-device-detection-engines "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-device-detection-engines "View all available properties and values") +[Developer Documentation](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-device-detection-engines "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-device-detection-engines "View all available properties and values") ## Introduction @@ -24,7 +24,6 @@ git submodule update --init --recursive ## Dependencies -For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page. The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. diff --git a/ip-intelligence/fiftyone_ipintelligence/readme.md b/ip-intelligence/fiftyone_ipintelligence/readme.md index 4b546d11..8d33e4d7 100644 --- a/ip-intelligence/fiftyone_ipintelligence/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence/readme.md @@ -1,8 +1,8 @@ # 51Degrees IP Intelligence Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=51degrees-ip-intelligence-engines "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=51degrees-ip-intelligence-engines "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=51degrees-ip-intelligence-engines "developer documentation") ## Introduction @@ -38,7 +38,7 @@ print(flowdata.ip.registeredname.value()) ## Find out more - [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-ip-intelligence) -- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-documentation) +- [IP Intelligence documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-documentation) - [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-getting-started) - [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-pricing) - https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md b/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md index ab94098e..fc189448 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md @@ -1,8 +1,8 @@ # 51Degrees IP Intelligence Engines - Cloud -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=51degrees-ip-intelligence-engines-cloud "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=51degrees-ip-intelligence-engines-cloud "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=51degrees-ip-intelligence-engines-cloud "developer documentation") ## Introduction @@ -38,7 +38,7 @@ print(flowdata.ip.registeredname.value()) ## Find out more - [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-ip-intelligence) -- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-documentation) +- [IP Intelligence documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-documentation) - [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-getting-started) - [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-pricing) - https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md index 61c3f9f5..127fc3af 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md @@ -1,8 +1,8 @@ # 51Degrees IP Intelligence Engines - Examples -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=51degrees-ip-intelligence-engines-examples "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=51degrees-ip-intelligence-engines-examples "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=51degrees-ip-intelligence-engines-examples "developer documentation") ## Introduction diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md index 569058a1..5cd4011b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md @@ -1,8 +1,8 @@ # 51Degrees IP Intelligence Engines - OnPremise -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=51degrees-ip-intelligence-engines-onpremise "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=51degrees-ip-intelligence-engines-onpremise "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=51degrees-ip-intelligence-engines-onpremise "developer documentation") ## Introduction @@ -59,7 +59,7 @@ print(flowdata.ip.registeredname.value()) ## Find out more - [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-ip-intelligence) -- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-documentation) +- [IP Intelligence documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-documentation) - [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-getting-started) - [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-pricing) - https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/readme.md b/ip-intelligence/fiftyone_ipintelligence_shared/readme.md index ced1fe62..9f1e6138 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_shared/readme.md @@ -1,8 +1,8 @@ # 51Degrees IP Intelligence Engines - Shared -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=51degrees-ip-intelligence-engines-shared "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=51degrees-ip-intelligence-engines-shared "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=51degrees-ip-intelligence-engines-shared "developer documentation") ## Introduction @@ -30,7 +30,7 @@ instead. ## Find out more - [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-ip-intelligence) -- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-documentation) +- [IP Intelligence documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-documentation) - [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-getting-started) - [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-pricing) - https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/readme.md b/ip-intelligence/fiftyone_ipintelligence_translation/readme.md index 7f12b228..7efb4146 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_translation/readme.md @@ -1,8 +1,8 @@ # IP Intelligence Country Translation -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=ip-intelligence-country-translation "Data rewards the curious") **Python IP Intelligence Country Translation** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python IP Intelligence Country Translation** -[Developer Documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=ip-intelligence-country-translation "Developer Documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=ip-intelligence-country-translation "Developer Documentation") ## Introduction @@ -37,7 +37,7 @@ Built on top of: ## Find out more - [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-ip-intelligence) -- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-documentation) +- [IP Intelligence documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-documentation) - [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-getting-started) - [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-pricing) - https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/readme.md b/ip-intelligence/readme.md index cad2a5c8..c2424801 100644 --- a/ip-intelligence/readme.md +++ b/ip-intelligence/readme.md @@ -1,8 +1,8 @@ # 51Degrees IP Intelligence Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines "developer documentation") ## Introduction @@ -19,9 +19,7 @@ design of this API. ## Dependencies -For runtime dependencies, see our -[dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) -page. The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. ### Git submodules diff --git a/location/readme.md b/location/readme.md index e3c7ee9d..dd21ab30 100644 --- a/location/readme.md +++ b/location/readme.md @@ -1,9 +1,9 @@ # 51Degrees Geo-Location Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=51degrees-geo-location-engines) +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **v4 Location Python** -[Developer Documentation](https://51degrees.com/location-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=51degrees-geo-location-engines "Developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_reverse_geocoding__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=51degrees-geo-location-engines "Developer documentation") ## Introduction @@ -13,7 +13,6 @@ The Pipeline is a generic web request intelligence and data processing solution ## Dependencies -For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=dependencies) page. The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. ## Installation and Examples @@ -64,7 +63,7 @@ Execute `$env:FLASK_APP = "web"` where `web` is the example file, and start your ## Find out more - [Geolocation](https://51degrees.com/geolocation?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-geolocation) -- [Reverse geocoding documentation](https://51degrees.com/documentation/_reverse_geocoding__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-documentation) +- [Reverse geocoding documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-documentation) - [Getting started](https://51degrees.com/documentation/_reverse_geocoding__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-getting-started) - [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-pricing) - https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md index 5979c0e8..5f378302 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md @@ -1,8 +1,8 @@ # Cloud Request Engine -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=cloud-request-engine "Data rewards the curious") **Python Pipeline Cloud Request Engine** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python Pipeline Cloud Request Engine** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=cloud-request-engine "Developer Documentation") +[Developer Documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=cloud-request-engine "Developer Documentation") ## Introduction @@ -83,7 +83,7 @@ It is used by the cloud versions of the following 51Degrees engines: ## Find out more - [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-build-your-own) -- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-documentation) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-documentation) - [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-examples) - [Cloud service documentation](https://51degrees.com/documentation/_services__cloud__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-cloud-service) - https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_core/readme.md b/pipeline/fiftyone_pipeline_core/readme.md index ad3470f1..0e7a2e34 100644 --- a/pipeline/fiftyone_pipeline_core/readme.md +++ b/pipeline/fiftyone_pipeline_core/readme.md @@ -1,8 +1,8 @@ # Pipeline Core -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=pipeline-core "Data rewards the curious") **Python Pipeline Core** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python Pipeline Core** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=pipeline-core "Developer Documentation") +[Developer Documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=pipeline-core "Developer Documentation") ## Introduction @@ -29,6 +29,6 @@ Engines created by 51Degrees: ## Find out more - [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=find-out-more-build-your-own) -- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=find-out-more-documentation) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=find-out-more-documentation) - [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=find-out-more-examples) - https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_did/readme.md b/pipeline/fiftyone_pipeline_did/readme.md index 38ae2b05..23b7990d 100644 --- a/pipeline/fiftyone_pipeline_did/readme.md +++ b/pipeline/fiftyone_pipeline_did/readme.md @@ -1,5 +1,7 @@ # fiftyone_pipeline_did +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") + Strongly typed Python reader and cloud client for the 51Did (51Degrees Identifier) returned by the 51Degrees Cloud service. Mirrors the .NET `FiftyOne.Did` package. diff --git a/pipeline/fiftyone_pipeline_engines/readme.md b/pipeline/fiftyone_pipeline_engines/readme.md index b9993940..29a470cf 100644 --- a/pipeline/fiftyone_pipeline_engines/readme.md +++ b/pipeline/fiftyone_pipeline_engines/readme.md @@ -1,8 +1,8 @@ # Pipeline Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=pipeline-engines "Data rewards the curious") **Python Pipeline Engines** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python Pipeline Engines** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=pipeline-engines "Developer Documentation") +[Developer Documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=pipeline-engines "Developer Documentation") ## Introduction @@ -33,6 +33,6 @@ Engines created by 51Degrees: ## Find out more - [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=find-out-more-build-your-own) -- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=find-out-more-documentation) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=find-out-more-documentation) - [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=find-out-more-examples) - https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/readme.md b/pipeline/fiftyone_pipeline_engines_fiftyone/readme.md index 8dba8ae2..282c8e28 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/readme.md +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/readme.md @@ -1,8 +1,8 @@ # Pipeline Engines Fiftyone -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=pipeline-engines-fiftyone "Data rewards the curious") **Python Pipeline Engines** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python Pipeline Engines** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=pipeline-engines-fiftyone "Developer Documentation") +[Developer Documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=pipeline-engines-fiftyone "Developer Documentation") ## Introduction @@ -19,6 +19,6 @@ It includes a ShareUsage engine that sends usage data to 51Degrees in zipped bat ## Find out more - [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=find-out-more-build-your-own) -- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=find-out-more-documentation) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=find-out-more-documentation) - [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=find-out-more-examples) - https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_translation/readme.md b/pipeline/fiftyone_pipeline_translation/readme.md index 4c0450eb..181723d1 100644 --- a/pipeline/fiftyone_pipeline_translation/readme.md +++ b/pipeline/fiftyone_pipeline_translation/readme.md @@ -1,8 +1,8 @@ # Pipeline Translation -![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=pipeline-translation "Data rewards the curious") **Python Pipeline Translation** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python Pipeline Translation** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=pipeline-translation "Developer Documentation") +[Developer Documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=pipeline-translation "Developer Documentation") ## Introduction @@ -28,6 +28,6 @@ This package is built on top of: ## Find out more - [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=find-out-more-build-your-own) -- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=find-out-more-documentation) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=find-out-more-documentation) - [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=find-out-more-examples) - https://github.com/51Degrees/51degrees-python diff --git a/pipeline/readme.md b/pipeline/readme.md old mode 100755 new mode 100644 index 3b5fbf45..e873cbdd --- a/pipeline/readme.md +++ b/pipeline/readme.md @@ -1,8 +1,8 @@ # 51Degrees Pipeline API -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-pipeline-api "Data rewards the curious") **Python Pipeline** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-python/main/.github/images/logo-160.png "Data rewards the curious") **Python Pipeline** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-pipeline-api "Developer Documentation") +[Developer Documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-pipeline-api "Developer Documentation") ## Introduction This repository contains the components of the Python implementation of the 51Degrees Pipeline API. @@ -23,7 +23,6 @@ The examples are described under Examples below. The `owid-python` folder is a g ## Dependencies -For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page. The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. ## Installation From 1769421a6d23a226b7cd9916e8f3f07888dd778a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 17:08:07 +0000 Subject: [PATCH 52/68] Move cxx to 7c4ac436 and read by index on an engine built with no names (#102) * Move cxx to the 51degrees-cxx release commit 7c4ac436 7c4ac436 is the second cxx release of 11 November, from 51Degrees/51degrees-cxx#150. It carries 51Degrees/51degrees-cxx#148, which keeps the first load's property order for an engine built with no names. No SWIG interface changes beyond a comment, so the committed wrappers stay as they are. * Test that an engine built with no names reads by index An engine built with no names takes the names the native code gives after the first load as its list. Its table holds every one of them, each index names its property after a reload, and a read through the table gives what an engine built with the same names gives. The Device Detection test also times a read by name, a read by index, the typed read and a build of the table. * Read by index on an engine built with no names An engine built with no names now builds its index table from the names the native code gives after the first load, through getRequiredProperties. Since 51Degrees/51degrees-cxx#148 those names are every property of the first data set, sorted by name, and a reload does not change them, so the index of each one is fixed in the same way as for an engine built with names. --- cxx | 2 +- .../devicedetection_onpremise.py | 18 ++-- .../tests/test_read_by_index.py | 102 ++++++++++++++++-- .../ipintelligence_onpremise.py | 18 ++-- .../tests/test_read_by_index.py | 66 ++++++++++-- 5 files changed, 175 insertions(+), 31 deletions(-) diff --git a/cxx b/cxx index c6d4e4d9..7c4ac436 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit c6d4e4d907e8da83845c095e80374280f8db29c5 +Subproject commit 7c4ac436e47cd15beed304cad2b8151d50a1efd1 diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index 3a1a9af7..ef2be582 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -181,7 +181,8 @@ def __init__( properties_list = RequiredPropertiesConfigSwig(restricted_properties_vector) # The property names the engine is built with. A property's index is - # its place in this list, which is fixed at build. + # its place in this list, which is fixed at build. An empty list + # means the engine takes its names from the native code once built. self.build_property_names = list(restricted_properties_vector) # Assemble configuration @@ -304,9 +305,7 @@ def init_engine(self): self.engine = engine # The table from each build-time name to its index, made once here - # so every read goes by index. An engine built with no names has no - # build-time list, so the table is empty and every read goes by - # name, which is the native behaviour it has today. + # so every read goes by index. self.property_indexes = self.read_property_indexes() # Get keys and add to evidenceKey filter @@ -350,12 +349,17 @@ def read_property_indexes(self): The index of each property the engine was built with, keyed by its name in lower case. The native code gives a property the place of its name among the names the engine was built with, the first place - where a name is given twice. An engine built with no names has an - empty table, so every read of its properties goes by name. + where a name is given twice. An engine built with no names takes + the names the native code gives after the first load, which are + every property in that data set sorted by name and which a reload + does not change. """ + names = self.build_property_names + if not names: + names = list(self.engine.getRequiredProperties()) indexes = {} - for index, name in enumerate(self.build_property_names): + for index, name in enumerate(names): indexes.setdefault(name.lower(), index) return indexes diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py index a4d38f84..981b715b 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_read_by_index.py @@ -23,7 +23,8 @@ """An engine built with a list of property names reads each one by its index in that list, with the native engine and the Lite data file. Also prints how long a read by name and a read by index take. An engine built -with no names keeps reading by name until the native change lands.""" +with no names takes the names the native code gives after the first load as +its list, and reads by index too.""" import time import unittest @@ -39,6 +40,8 @@ "/11D167 Safari/9537.53") # Reads timed for the figures in the warnings. TIMED_READS = 20000 +# Builds of the index table timed for the figures in the warnings. +TIMED_TABLE_BUILDS = 200 def shown(value): @@ -58,6 +61,15 @@ def process(engine): return pipeline, flowdata +def first_places(names): + """Each name in lower case against its first place in the list, which + is the index the native code gives.""" + indexes = {} + for index, name in enumerate(names): + indexes.setdefault(name.lower(), index) + return indexes + + def data_set_names(engine): """The names the data set holds, in the data set's own order. Only the tests use this order, to build engines whose name lists differ from it. @@ -80,19 +92,95 @@ def names_from_the_data_set(self): self.addCleanup(flowdata.close) return data_set_names(engine), flowdata - def test_an_engine_with_no_names_keeps_reading_by_name(self): - """An engine built with no names has an empty table, so nothing - about it changes until the native change lands.""" + def no_names_engine(self): engine = DeviceDetectionOnPremise( data_file_path=data_file, licence_keys="", auto_update=False) pipeline, flowdata = process(engine) self.addCleanup(pipeline.close) self.addCleanup(flowdata.close) - self.assertEqual([], engine.build_property_names) - self.assertEqual({}, engine.property_indexes) - # Reading still works, by name, exactly as before. + return engine, pipeline, flowdata + + def test_an_engine_with_no_names_reads_by_index(self): + """An engine built with no names takes the names the native code + gives after the first load as its list, so its table holds every + one of them and each index names its property.""" + engine, _, flowdata = self.no_names_engine() + required = list(engine.engine.getRequiredProperties()) + self.assertIn("IsMobile", required) + self.assertEqual(first_places(required), engine.property_indexes) + results = flowdata.device.swig_results + for name, index in engine.property_indexes.items(): + self.assertEqual(name, results.getPropertyName(index).lower()) + # Reading through the table gives what an engine built with the + # same names gives. + named = DeviceDetectionOnPremise( + data_file_path=data_file, licence_keys="", auto_update=False, + restricted_properties=list(required)) + second, named_data = process(named) + self.addCleanup(second.close) + self.addCleanup(named_data.close) + for name in ("ismobile", "browsername", "platformname"): + self.assertEqual( + shown(named_data.device.get(name)), + shown(flowdata.device.get(name)), name) self.assertTrue(flowdata.device.ismobile.value()) + def test_a_reload_keeps_the_table_of_an_engine_with_no_names(self): + """A reload leaves an engine built with no names with its table, + and each index still names the same property.""" + engine, pipeline, _ = self.no_names_engine() + required = list(engine.engine.getRequiredProperties()) + table = dict(engine.property_indexes) + self.assertEqual(first_places(required), table) + engine.engine.refreshData() + self.assertEqual(required, list(engine.engine.getRequiredProperties())) + self.assertEqual(table, engine.property_indexes) + flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) + flowdata.evidence.add("header.user-agent", mobile_ua) + flowdata.process() + results = flowdata.device.swig_results + for name, index in table.items(): + self.assertEqual(name, results.getPropertyName(index).lower()) + self.assertTrue(flowdata.device.ismobile.value()) + + def test_an_engine_with_no_names_times_each_read(self): + """With no names, a read by name, a read by index and the typed + read give the same value, with the time each read takes and the + time to build the index table, for the figures.""" + engine, _, flowdata = self.no_names_engine() + results = flowdata.device.swig_results + index = list(engine.engine.getRequiredProperties()).index("IsMobile") + self.assertEqual( + results.getValueAsBool("IsMobile").getValue(), + results.getValueAsBool(index).getValue()) + self.assertEqual( + results.getValueAsBool("IsMobile").getValue(), + flowdata.device.ismobile.value()) + + start = time.perf_counter() + for _ in range(TIMED_READS): + results.getValueAsBool("IsMobile") + by_name = time.perf_counter() - start + start = time.perf_counter() + for _ in range(TIMED_READS): + results.getValueAsBool(index) + by_index = time.perf_counter() - start + start = time.perf_counter() + for _ in range(TIMED_READS): + flowdata.device.get("ismobile") + typed = time.perf_counter() - start + start = time.perf_counter() + for _ in range(TIMED_TABLE_BUILDS): + engine.read_property_indexes() + table = time.perf_counter() - start + warnings.warn( + f"No names, IsMobile read {TIMED_READS} times, native by name " + f"{by_name * 1e9 / TIMED_READS:.0f} ns a read, native by index " + f"{by_index * 1e9 / TIMED_READS:.0f} ns a read, typed read " + f"{typed * 1e9 / TIMED_READS:.0f} ns a read, index table " + f"{table * 1e9 / TIMED_TABLE_BUILDS:.0f} ns a build") + def test_the_table_follows_the_order_the_names_were_given_in(self): names, _ = self.names_from_the_data_set() given = list(reversed(names[:6])) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index 865506ba..054026ba 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -150,7 +150,8 @@ def __init__( properties_list = RequiredPropertiesConfigSwig(restricted_properties_vector) # The property names the engine is built with. A property's index is - # its place in this list, which is fixed at build. + # its place in this list, which is fixed at build. An empty list + # means the engine takes its names from the native code once built. self.build_property_names = list(restricted_properties_vector) # Assemble configuration @@ -245,9 +246,7 @@ def init_engine(self): self.engine = engine # The table from each build-time name to its index, made once here - # so every read goes by index. An engine built with no names has no - # build-time list, so the table is empty and every read goes by - # name, which is the native behaviour it has today. + # so every read goes by index. self.property_indexes = self.read_property_indexes() # Get keys and add to evidenceKey filter @@ -295,12 +294,17 @@ def read_property_indexes(self): The index of each property the engine was built with, keyed by its name in lower case. The native code gives a property the place of its name among the names the engine was built with, the first place - where a name is given twice. An engine built with no names has an - empty table, so every read of its properties goes by name. + where a name is given twice. An engine built with no names takes + the names the native code gives after the first load, which are + every property in that data set sorted by name and which a reload + does not change. """ + names = self.build_property_names + if not names: + names = list(self.engine.getRequiredProperties()) indexes = {} - for index, name in enumerate(self.build_property_names): + for index, name in enumerate(names): indexes.setdefault(name.lower(), index) return indexes diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py index 1bb785d3..f8ccdfdc 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_read_by_index.py @@ -22,8 +22,8 @@ """An engine built with a list of property names reads each one by its index in that list, with the native engine and the data file the tests use. -An engine built with no names keeps reading by name until the native -change lands.""" +An engine built with no names takes the names the native code gives after +the first load as its list, and reads by index too.""" import os import unittest @@ -54,6 +54,15 @@ def process(engine): return pipeline, flowdata +def first_places(names): + """Each name in lower case against its first place in the list, which + is the index the native code gives.""" + indexes = {} + for index, name in enumerate(names): + indexes.setdefault(name.lower(), index) + return indexes + + def data_set_names(engine): """The names the data set holds, in the data set's own order. Only the tests use this order, to build engines whose name lists differ from it. @@ -79,18 +88,57 @@ def names_from_the_data_set(self): self.addCleanup(flowdata.close) return data_set_names(engine), flowdata - def test_an_engine_with_no_names_keeps_reading_by_name(self): - """An engine built with no names has an empty table, so nothing - about it changes until the native change lands.""" + def no_names_engine(self): engine = IpIntelligenceOnPremise( data_file_path=data_file, licence_keys="") pipeline, flowdata = process(engine) self.addCleanup(pipeline.close) self.addCleanup(flowdata.close) - self.assertEqual([], engine.build_property_names) - self.assertEqual({}, engine.property_indexes) - # Reading still works, by name, exactly as before. - self.assertTrue(flowdata.ip.get(data_set_names(engine)[0])) + return engine, pipeline, flowdata + + def test_an_engine_with_no_names_reads_by_index(self): + """An engine built with no names takes the names the native code + gives after the first load as its list, so its table holds every + one of them and each index names its property.""" + engine, _, flowdata = self.no_names_engine() + required = list(engine.engine.getRequiredProperties()) + self.assertNotEqual([], required) + self.assertEqual(first_places(required), engine.property_indexes) + results = flowdata.ip.swig_results + for name, index in engine.property_indexes.items(): + self.assertEqual(name, results.getPropertyName(index).lower()) + # Reading through the table gives what an engine built with the + # same names gives. + named = IpIntelligenceOnPremise( + data_file_path=data_file, licence_keys="", + restricted_properties=list(required)) + second, named_data = process(named) + self.addCleanup(second.close) + self.addCleanup(named_data.close) + for name in engine.property_indexes: + if name in engine.properties: + self.assertEqual( + shown(named_data.ip.get(name)), + shown(flowdata.ip.get(name)), name) + + def test_a_reload_keeps_the_table_of_an_engine_with_no_names(self): + """A reload leaves an engine built with no names with its table, + and each index still names the same property.""" + engine, pipeline, _ = self.no_names_engine() + required = list(engine.engine.getRequiredProperties()) + table = dict(engine.property_indexes) + self.assertEqual(first_places(required), table) + self.assertNotEqual({}, table) + engine.engine.refreshData() + self.assertEqual(required, list(engine.engine.getRequiredProperties())) + self.assertEqual(table, engine.property_indexes) + flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) + flowdata.evidence.add("query.client-ip", "8.8.8.8") + flowdata.process() + results = flowdata.ip.swig_results + for name, index in table.items(): + self.assertEqual(name, results.getPropertyName(index).lower()) def test_the_table_follows_the_order_the_names_were_given_in(self): names, _ = self.names_from_the_data_set() From c4804124c95d84053a0c09080156ff2bde270e6a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 18:45:01 +0000 Subject: [PATCH 53/68] Bound usage sharing cleanup at five seconds and take fifty as the minimum entries per message (#103) * Test that closing stops after five seconds and fifty entries start a message * Bound usage sharing cleanup at five seconds and take fifty as the minimum entries per message * Write the specification's numbers in the tests rather than reading them from the element --- .../share_usage.py | 123 ++++++++++++++---- .../tests/test_usage.py | 96 ++++++++++++++ 2 files changed, 194 insertions(+), 25 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index a94a1c8e..943674b8 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -58,10 +58,22 @@ # The default number of seconds to wait for the server connection before failing the request when sending usage data REQUEST_TIMEOUT = 60 +# The default number of entries that must be queued before a message is +# started. A message holds from this to twice this number. +REQUESTED_PACKAGE_SIZE_DEFAULT = 50 + # The default maximum number of entries waiting to be sent, unless ten # times the requested package size is larger. MAX_QUEUE_SIZE_DEFAULT = 1000 +# The time in seconds that closing may spend sending the entries still +# queued. Closing stops after this, so it does not hold up the process that +# closes it. It is not a user setting. +CLEANUP_TIMEOUT_SECONDS = 5 + +# How long closing waits between looks at a send that is already running. +CLEANUP_POLL_SECONDS = 0.01 + # While the queue is full, a message is logged for the first entry discarded # and then once for every this many more, never one for each entry. QUEUE_FULL_LOG_EVERY_ENTRIES = 1000 @@ -87,7 +99,7 @@ class ShareUsage(Engine): def __init__( self, interval = 1200, - requested_package_size = 10, + requested_package_size = REQUESTED_PACKAGE_SIZE_DEFAULT, cookie = None, query_whitelist = [], header_blacklist = [], @@ -110,10 +122,11 @@ def __init__( multiple times within this time limit (in seconds) then they will only be shared once. @type requested_package_size: int - @param requested_package_size: The usage element will group data into - single requests before sending it. This setting controls the minimum - number of entries before data is sent. If you are sharing large - amounts of data, increasing this value is recommended in order to + @param requested_package_size: The usage element will group data into + single requests before sending it. This setting controls the minimum + number of entries before data is sent, and a message holds from this + to twice this number. 50 by default. If you are sharing large + amounts of data, increasing this value is recommended in order to reduce the overhead of sending HTTP messages. For example, the 51Degrees cloud service uses a value of 2500. @type cookie: string @@ -238,6 +251,9 @@ def __init__( self._resume_timer = None # Set when the element closes. self._closing = threading.Event() + # The time closing may spend sending what is still queued. Not a + # user setting, and here so that a test can shorten it. + self.cleanup_timeout_seconds = CLEANUP_TIMEOUT_SECONDS self.constant_xml = None @@ -252,7 +268,11 @@ def __init__( def __del__(self): thread_pool = getattr(self, "thread_pool", None) if thread_pool is not None: - thread_pool.shutdown(wait=True) + closing = getattr(self, "_closing", None) + # Closing has already given sending all the time it is allowed, + # so waiting again here would undo that bound. + thread_pool.shutdown( + wait=closing is None or not closing.is_set()) def close(self): @@ -262,6 +282,11 @@ def close(self): when it is closed. Closing does not wait for a pause after failed sends, and sends nothing while paused. + Closing gives up after cleanup_timeout_seconds, discards the entries + still queued and logs a warning, so that it does not hold up the + process that closes it. A message already going out when that time + passes is left to the request timeout on its own thread. + """ super(ShareUsage, self).close() @@ -270,36 +295,84 @@ def close(self): if self._resume_timer is not None: self._resume_timer.cancel() self._resume_timer = None - self.thread_pool.shutdown(wait=True) - self._send_remaining() + # Sending runs on its own thread, so that a service which does not + # answer cannot hold closing up for longer than the bound. + deadline = time.monotonic() + self.cleanup_timeout_seconds + # Set when sending stops for a reason of its own rather than for + # want of time, so that a thread which stops exactly at the + # deadline is still counted as having run out of time. + done = threading.Event() + flush = threading.Thread( + target=self._send_remaining, args=(deadline, done), + name="ShareUsageClose") + # The thread does not keep the process running. + flush.daemon = True + flush.start() + flush.join(self.cleanup_timeout_seconds) + if not done.is_set(): + with self._lock: + queued = len(self.share_data) + self._log( + "warning", + "Usage sharing stopped sending when closing down, because " + "it did not finish within " + f"{self.cleanup_timeout_seconds} seconds. {queued} entries " + "were left in the queue and discarded.") + self.thread_pool.shutdown(wait=False) - def _send_remaining(self): + def _send_remaining(self, deadline, done): """! Send the entries still queued when closing, even if there are fewer - than the requested package size. Nothing is sent while paused, or - after a message fails, and the entries left are discarded and - counted in the log. + than the requested package size. A send already running finishes + first, so only one message goes out at a time. Nothing is sent while + paused, after a message fails, or once deadline has passed, and the + entries left are discarded. Closing logs the ones left when the time + ran out, and this logs the ones left for the other two reasons. + + @type deadline: float + @param deadline: The monotonic time sending has to stop at. + @type done: threading.Event + @param done: Set when sending stops for a reason other than the + deadline. """ + while time.monotonic() < deadline: + with self._lock: + if not self._sending: + break + time.sleep(CLEANUP_POLL_SECONDS) while True: + # None while sending is paused, so nothing is sent. + share_data = None with self._lock: queued = len(self.share_data) + if time.monotonic() >= deadline: + return if queued == 0: + done.set() return - if self.backoff.remaining(time.monotonic()) > 0: - break - share_data = self.share_data[:self._message_size] - del self.share_data[:self._message_size] - data = devices_message(share_data) - if not self.send_thread(self.endpoint, data): + if self.backoff.remaining(time.monotonic()) == 0: + share_data = self.share_data[:self._message_size] + del self.share_data[:self._message_size] + if share_data is None: + self._log( + "info", + f"{queued} usage sharing entries were not sent, as " + "sending was paused after failed sends when usage " + "sharing closed.") + done.set() + return + if not self.send_thread(self.endpoint, devices_message( + share_data)): with self._lock: queued = len(self.share_data) - break - if queued > 0: - self._log( - "info", - f"{queued} usage sharing entries were not sent, as sending " - "had failed when usage sharing closed.") + if queued > 0: + self._log( + "info", + f"{queued} usage sharing entries were not sent, as " + "sending had failed when usage sharing closed.") + done.set() + return def get_constant_xml(self): if self.constant_xml is None: @@ -462,7 +535,7 @@ def _send_messages(self): pause = self.backoff.failed(time.monotonic()) if pause > 0: self._schedule_resume(pause) - done = pause > 0 or \ + done = pause > 0 or self._closing.is_set() or \ len(self.share_data) < self.requested_package_size if done: self._sending = False diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index 1fd819cb..c56eee51 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -43,6 +43,12 @@ shareUsageVersion = "1.1" +# The numbers the specification gives, written out here rather than read +# from the element, so that a test fails when the element does not keep to +# them. +CLEANUP_BOUND_SECONDS = 5 +MINIMUM_ENTRIES_PER_MESSAGE = 50 + class ReceiverThread(threading.Thread): @@ -69,10 +75,16 @@ def get_received(self): app.status = 200 # The content the receiver answers with. app.body = "" +# How long in seconds the receiver holds a request before answering. +app.delay = 0 +# Set to let a held request go at once. +app.delay_release = threading.Event() @app.route('/', methods=['POST']) def server(): app.attempts.append(time.monotonic()) + if app.delay: + app.delay_release.wait(app.delay) data = gzip.decompress(request.data).decode("utf8") if app.status == 200: app.received.append(data) @@ -103,6 +115,8 @@ def setUp(self): app.attempts = [] app.status = 200 app.body = "" + app.delay = 0 + app.delay_release.clear() self.requests = 0 self.logs = [] @@ -393,6 +407,88 @@ def test_close_sends_fewer_entries_than_the_package_size(self): self.assertEqual(ips, client_ips(app.received)) + def test_close_stops_sending_after_the_cleanup_bound(self): + """! + Closing stops after the five seconds the specification allows, + rather than waiting for a service that does not answer, and logs a + warning counting the entries it discarded. + """ + + app.delay = CLEANUP_BOUND_SECONDS + 5 + # Let the held request go as soon as the test is done with it. + self.addCleanup(app.delay_release.set) + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + usage._log = lambda level, message: self.logs.append( + (level, message)) + pipeline = PipelineBuilder().add(usage).build() + self.process_unique(pipeline) + self.wait_for( + lambda: len(app.attempts) == 1, + lambda: "The first send was not made, so nothing was holding " + "sending up when closing.") + for i in range(5): + self.process_unique(pipeline) + + start = time.monotonic() + usage.close() + took = time.monotonic() - start + + self.assertLess( + took, CLEANUP_BOUND_SECONDS + 2, + f"Closing took {took:.2f} seconds, so it waited for the service " + "rather than stopping at the bound.") + self.assertGreaterEqual( + took, CLEANUP_BOUND_SECONDS - 0.5, + f"Closing took {took:.2f} seconds, so it gave up before the " + "bound.") + warnings = [log for log in self.logs if "did not finish" in log[1]] + self.assertEqual( + 1, len(warnings), + f"The entries discarded were not logged once. Logged: " + f"{self.logs}") + self.assertEqual( + ("warning", + "Usage sharing stopped sending when closing down, because it " + f"did not finish within {CLEANUP_BOUND_SECONDS} seconds. 5 " + "entries were left in the queue and discarded."), + warnings[0]) + + # Let the held message go and wait for it to arrive, so that + # nothing from this element lands in another test. + app.delay_release.set() + self.wait_for( + lambda: len(app.received) == 1, + lambda: "The held message did not arrive once it was let go.") + + def test_minimum_entries_per_message_default(self): + """! + The default minimum entries per message is the 50 the specification + gives, so no message is started before 50 entries are queued. + """ + + usage = ShareUsage(endpoint=testEndpoint) + pipeline = PipelineBuilder().add(usage).build() + self.addCleanup(usage.close) + self.assertEqual( + MINIMUM_ENTRIES_PER_MESSAGE, usage.requested_package_size, + "The default is not the 50 the specification gives.") + + for i in range(MINIMUM_ENTRIES_PER_MESSAGE - 1): + self.process_unique(pipeline) + sleep(0.5) + self.assertEqual( + 0, len(app.attempts), + "A message was started before 50 entries were queued.") + + self.process_unique(pipeline) + self.wait_for( + lambda: len(client_ips(app.received)) + == MINIMUM_ENTRIES_PER_MESSAGE, + lambda: "The fiftieth entry did not start a message. Sent: " + f"{len(client_ips(app.received))}") + def test_back_off_invalid_settings(self): """! Settings the pause cannot work with are rejected. From b29423f1ddb0c62bca1a20d9a79bd16b6b8defc0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 11 Nov 2026 19:34:19 +0000 Subject: [PATCH 54/68] Give every environment variable one _51DEGREES_ name (#100) * Make the name check fail on the names that remain, red The check already failed on a name starting with the digit and on a RESOURCE_KEY or LICENSE_KEY read behind a _51DEGREES_ one. It now also fails on FOD_CLOUD_API_URL, cloud_endpoint, IPINTELLIGENCEDATAFILE, ipintelligencetestkey and run_performance_tests, read or set as an environment variable or held in a constant to read later. A pipeline setting carries the same word as an old variable name, so only the environment forms are matched and a setting written by its key is left alone, which the tests state. This commit is the check alone, so it fails over the repository as it stands and names every place the next commit changes. * Make the name check fail on cloud_request_origin too The examples read it for a resource key limited to particular domains, and it was missed when the other old names were added. * Read every environment variable by its one _51DEGREES_ name The cloud endpoint was read under two names, FOD_CLOUD_API_URL by the cloud request engine and the 51Did client, and cloud_endpoint by the examples. Both become _51DEGREES_CLOUD_ENDPOINT, which the Rust library already reads, and the constants that hold the name move with it. cloud_request_origin becomes _51DEGREES_CLOUD_REQUEST_ORIGIN, IPINTELLIGENCEDATAFILE becomes _51DEGREES_IPI_PATH, and run_performance_tests becomes _51DEGREES_RUN_PERFORMANCE_TESTS. The shared key helper test set a variable called ipintelligencetestkey as scaffolding rather than as a 51Degrees setting, so it takes _51DEGREES_TEST_NAMED_KEY rather than the real resource key name, which the test would otherwise overwrite for whatever ran next. A pipeline setting and a function parameter carry the same words as some of these names and are left alone, because only the environment variable moves. * Leave a fake resource key in a test fixture alone The constant clause matched any assignment of an old name, so it also matched resourceKey = "resource_key" in the cloud request engine's and the IP Intelligence cloud tests, where the name is the fake key sent on the wire rather than a variable. It now matches only a constant whose own name says it holds a variable name, and the names that are never anything but a variable are matched wherever they appear. The tests state both. * Treat cloud_request_origin as a setting as well, and stop the check naming names cloud_request_origin is a pipeline builder parameter and a setting key across the device detection and IP Intelligence builders, so it joins the names matched only where a variable is read, set or named in a constant. The names matched wherever they appear were written out in the pattern and in the comment above it, so the check failed on its own source. Each is now built from two pieces, as the digit-initial start already was. * Leave the IpIntelligenceDataFile class alone The names matched wherever they appear were matched in any case, so the IP Intelligence data file's old name also matched the class name IpIntelligenceDataFile, which differs from it by case alone. Those four names now keep their case, and the tests state it. * Say which group of old names keeps its case The paragraph about keeping case sat under the paragraph about RESOURCE_KEY, LICENSE_KEY, cloud_endpoint and cloud_request_origin, which are matched in any case, so it read as though it described them. The (?-i:) is on the other group. The paragraph now sits with the group it describes and each paragraph names the variable that holds its four. Comment only, and the two behaviours it describes are already pinned by the Pester cases for the IpIntelligenceDataFile class and for a lower case read of the shared names. * See an old name braced as a PowerShell variable The check misses an old name written as ${env:NAME}, as Env:\NAME or as a workflow ${{ env.NAME }}, because its prefix wants the dollar and the env token adjacent. The CI scripts here are PowerShell, where a name is braced whenever it sits beside other text, so the check cannot see a form the files it watches really use. The Go, Node, Java and PHP checks all match the token on its own. These cases fail until this one does the same. * Match the env token on its own, so a braced old name is seen The prefix wanted the dollar sign and the env token next to each other, so ${env:NAME}, Env:\NAME and a workflow ${{ env.NAME }} went past it. Matching the token on its own, with a following backslash allowed, closes all three and brings this check into line with the Go, Node, Java and PHP ones. --- ci/check-environment-variable-names.Tests.ps1 | 31 ++++++++++ ci/check-environment-variable-names.ps1 | 57 +++++++++++++++---- device-detection/ci/run-integration-tests.ps1 | 2 +- .../readme.md | 4 +- .../example_utils.py | 2 +- .../tests/test_onpremiseexamples.py | 4 +- .../fiftyone_devicedetection_examples/tox.ini | 2 +- ip-intelligence/ci/fetch-assets.ps1 | 2 +- .../example_utils.py | 4 +- .../fiftyone_ipintelligence_examples/tox.ini | 2 +- .../tests/constants.py | 4 +- .../tests/test_ipintelligence.py | 4 +- .../fiftyone_ipintelligence_onpremise/tox.ini | 2 +- .../tests/test_shared.py | 6 +- ip-intelligence/readme.md | 2 +- .../examples/readme.md | 6 +- .../cloud/mixed/gettingstarted_console.py | 6 +- .../cloud/mixed/gettingstarted_web/app.py | 6 +- .../example_utils.py | 4 +- .../examples/tox.ini | 16 +++--- .../readme.md | 2 +- .../cloudrequestengine.py | 4 +- .../constants.py | 2 +- .../tests/test_cloudrequestengine.py | 8 +-- .../examples/creator_context_web/server.py | 4 +- pipeline/fiftyone_pipeline_did/readme.md | 6 +- .../src/fiftyone_pipeline_did/did_client.py | 6 +- .../tests/test_did_client.py | 12 ++-- .../tests/test_did_client_live.py | 4 +- pipeline/fiftyone_pipeline_did/tox.ini | 2 +- 30 files changed, 140 insertions(+), 76 deletions(-) diff --git a/ci/check-environment-variable-names.Tests.ps1 b/ci/check-environment-variable-names.Tests.ps1 index b3b7e8bd..7ba15f49 100644 --- a/ci/check-environment-variable-names.Tests.ps1 +++ b/ci/check-environment-variable-names.Tests.ps1 @@ -35,18 +35,49 @@ Describe 'Find-EnvironmentNameProblem' { @{ Line = 'LICENCE = env("_51DEGREES_LICENSE_KEY", "license_key")' } @{ Line = 'LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key"' } @{ Line = 'legacy_license_key_env_var = "x"' } + @{ Line = 'self.baseURL = os.environ.get("FOD_CLOUD_API_URL")' } + @{ Line = 'FOD_CLOUD_API_URL = "FOD_CLOUD_API_URL"' } + @{ Line = 'RESOURCE_KEY_ENV_VAR = "resource_key"' } + @{ Line = 'ENDPOINT_ENV_VAR = "cloud_endpoint"' } + @{ Line = 'ORIGIN_ENV_VAR = "cloud_request_origin"' } + @{ Line = 'os.environ.get("cloud_request_origin")' } + @{ Line = '$env:cloud_endpoint = "https://cloud.51degrees.com/api/v4/"' } + @{ Line = '${env:cloud_endpoint} = "https://cloud.51degrees.com/api/v4/"' } + @{ Line = '${Env:RESOURCE_KEY}' } + @{ Line = 'Remove-Item Env:\license_key' } + @{ Line = ' url: ${{ env.cloud_endpoint }}' } + @{ Line = 'env = os.environ.get("IPINTELLIGENCEDATAFILE")' } + @{ Line = 'os.environ["ipintelligencetestkey"] = "value"' } + @{ Line = 'if "run_performance_tests" in os.environ:' } + @{ Line = 'saved = os.environ.pop("FOD_CLOUD_API_URL", None)' } ) { Get-Kind $Line | Should -Contain 'fallback' } + # cloud_endpoint is a pipeline setting as well as an old variable name, + # and only the variable moves, so a setting written by its key is left + # alone. It 'leaves alone' -ForEach @( @{ Line = 'os.environ.get("_51DEGREES_RESOURCE_KEY")' } + @{ Line = 'pipeline_settings["cloud_endpoint"] = cloud_endpoint' } + @{ Line = 'resourceKey = "resource_key"' } + @{ Line = 'class IpIntelligenceDataFile(DataFile):' } + @{ Line = 'if isinstance(datafile, IpIntelligenceDataFile):' } + @{ Line = 'RESOURCE_KEY = "resource_key"' } + @{ Line = 'testResourceKey = "resource_key"' } + @{ Line = 'def __init__(self, resource_key, cloud_request_origin=None):' } + @{ Line = '"cloud_request_origin": cloud_request_origin,' } + @{ Line = 'cloudRequestEngineSettings["cloud_endpoint"] = cloudEndPoint' } + @{ Line = 'def run(self, resource_key, logger, cloud_endpoint = None):' } + @{ Line = 'ENDPOINT_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT"' } @{ Line = '$env:_51DEGREES_DD_PATH = "file.hash"' } @{ Line = '$env:_51DEGREES_LICENSE_KEY_51DID = $k' } @{ Line = 'RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY"' } @{ Line = 'builder = Builder({"resource_key": "YOUR_RESOURCE_KEY"})' } @{ Line = 'Builder(resource_key=key, license_key=licence)' } @{ Line = '$env:PAID_RESOURCE_KEY = $Keys.TestResourceKey' } + @{ Line = '${env:_51DEGREES_RESOURCE_KEY} = $Keys.TestResourceKey' } + @{ Line = ' PAID_RESOURCE_KEY: ${{ env.PAID_RESOURCE_KEY }}' } @{ Line = 'Set the 51D_ cookie for the 51Did example.' } ) { Get-Kind $Line | Should -BeNullOrEmpty diff --git a/ci/check-environment-variable-names.ps1 b/ci/check-environment-variable-names.ps1 index cd34fc77..6c9766d2 100644 --- a/ci/check-environment-variable-names.ps1 +++ b/ci/check-environment-variable-names.ps1 @@ -31,18 +31,53 @@ $script:DigitInitialPattern = [regex]::new( "(? Date: Wed, 11 Nov 2026 23:37:47 +0000 Subject: [PATCH 55/68] Use the standard licence text in one LICENSE, and the standard header in every file (#104) * Use the standard licence text once, with its additional terms in NOTICE The root LICENSE and ip-intelligence/LICENSE held the EUPL v1.2 text and the two additional terms in Windows-1252, and pipeline/LICENSE held the 51Degrees header form. The root LICENSE now holds the text byte for byte as 51degrees-cxx has it, NOTICE holds the two additional terms as 51degrees-cxx has them, and the two other copies are removed. Neither copy sat in a package folder, and every setup.py declares EUPL-1.2. Nine test files had no licence header and get the standard one. Three headers had a closing line missing or written in the C comment form, and now end as the standard header does. * Keep the additional terms in the file headers rather than in NOTICE The two additional terms apply where each file's header states them, so the repository has LICENSE alone, without a NOTICE file. The README's licence line goes back to what it said before. --- LICENSE | 112 +++---- .../devicedetection_datafile.py | 2 +- ip-intelligence/LICENSE | 286 ----------------- .../ipintelligence_datafile.py | 2 +- .../tests/test_swig_data_decimal_places.py | 21 ++ pipeline/LICENSE | 298 ------------------ .../tests/test_cloudengine_missing_product.py | 21 ++ .../tests/test_cloudengine_wkt.py | 21 ++ .../tests/test_jsonbundler_lower_case.py | 21 ++ .../tests/test_jsonbundler_records.py | 21 ++ .../test_jsonbundler_sequence_required.py | 21 ++ .../tests/test_jsonbundler_wkt.py | 21 ++ .../tests/test_licence_key_masking.py | 21 ++ .../tests/test_share_usage_record.py | 21 ++ .../tests/test_share_usage_tracker.py | 1 + 15 files changed, 242 insertions(+), 648 deletions(-) delete mode 100644 ip-intelligence/LICENSE delete mode 100755 pipeline/LICENSE diff --git a/LICENSE b/LICENSE index c9c8de6a..a74f0443 100644 --- a/LICENSE +++ b/LICENSE @@ -1,44 +1,44 @@ European Union Public Licence V. 1.2 -EUPL © the European Union 2007, 2016 +EUPL © the European Union 2007, 2016 -This European Union Public Licence (the ‘EUPL’) applies to the Work (as +This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined below) which is provided under the terms of this Licence. Any use of the Work, other than as authorised under this Licence is prohibited (to the extent such use is covered by a right of the copyright holder of the Work). The Work is provided under the terms of this Licence when the Licensor (as defined below) has placed the following notice immediately following the -copyright notice for the Work: “Licensed under the EUPL”, or has expressed by +copyright notice for the Work: “Licensed under the EUPLâ€, or has expressed by any other means his willingness to license under the EUPL. 1. Definitions In this Licence, the following terms have the following meaning: -— ‘The Licence’: this Licence. -— ‘The Original Work’: the work or software distributed or communicated by the - ‘Licensor under this Licence, available as Source Code and also as - ‘Executable Code as the case may be. -— ‘Derivative Works’: the works or software that could be created by the - ‘Licensee, based upon the Original Work or modifications thereof. This - ‘Licence does not define the extent of modification or dependence on the - ‘Original Work required in order to classify a work as a Derivative Work; - ‘this extent is determined by copyright law applicable in the country - ‘mentioned in Article 15. -— ‘The Work’: the Original Work or its Derivative Works. -— ‘The Source Code’: the human-readable form of the Work which is the most +— ‘The Licence’: this Licence. +— ‘The Original Work’: the work or software distributed or communicated by the + ‘Licensor under this Licence, available as Source Code and also as + ‘Executable Code as the case may be. +— ‘Derivative Works’: the works or software that could be created by the + ‘Licensee, based upon the Original Work or modifications thereof. This + ‘Licence does not define the extent of modification or dependence on the + ‘Original Work required in order to classify a work as a Derivative Work; + ‘this extent is determined by copyright law applicable in the country + ‘mentioned in Article 15. +— ‘The Work’: the Original Work or its Derivative Works. +— ‘The Source Code’: the human-readable form of the Work which is the most convenient for people to study and modify. -— ‘The Executable Code’: any code which has generally been compiled and which +— ‘The Executable Code’: any code which has generally been compiled and which is meant to be interpreted by a computer as a program. -— ‘The Licensor’: the natural or legal person that distributes or communicates +— ‘The Licensor’: the natural or legal person that distributes or communicates the Work under the Licence. -— ‘Contributor(s)’: any natural or legal person who modifies the Work under +— ‘Contributor(s)’: any natural or legal person who modifies the Work under the Licence, or otherwise contributes to the creation of a Derivative Work. -— ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of +— ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of the Work under the terms of the Licence. -— ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, +— ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, renting, distributing, communicating, transmitting, or otherwise making available, online or offline, copies of the Work or providing access to its essential functionalities at the disposal of any other natural or legal @@ -50,15 +50,15 @@ The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, sublicensable licence to do the following, for the duration of copyright vested in the Original Work: -— use the Work in any circumstance and for all usage, -— reproduce the Work, -— modify the Work, and make Derivative Works based upon the Work, -— communicate to the public, including the right to make available or display +— use the Work in any circumstance and for all usage, +— reproduce the Work, +— modify the Work, and make Derivative Works based upon the Work, +— communicate to the public, including the right to make available or display the Work or copies thereof to the public and perform publicly, as the case may be, the Work, -— distribute the Work or copies thereof, -— lend and rent the Work or copies thereof, -— sublicense rights in the Work or copies thereof. +— distribute the Work or copies thereof, +— lend and rent the Work or copies thereof, +— sublicense rights in the Work or copies thereof. Those rights can be exercised on any media, supports and formats, whether now known or later invented, as far as the applicable law permits so. @@ -104,7 +104,7 @@ Copyleft clause: If the Licensee distributes or communicates copies of the Original Works or Derivative Works, this Distribution or Communication will be done under the terms of this Licence or of a later version of this Licence unless the Original Work is expressly distributed only under this version of -the Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee +the Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee (becoming Licensor) cannot offer or impose any additional terms or conditions on the Work or Derivative Work that alter or restrict the terms of the Licence. @@ -113,7 +113,7 @@ Compatibility clause: If the Licensee Distributes or Communicates Derivative Works or copies thereof based upon both the Work and another work licensed under a Compatible Licence, this Distribution or Communication can be done under the terms of this Compatible Licence. For the sake of this clause, -‘Compatible Licence’ refers to the licences listed in the appendix attached to +‘Compatible Licence’ refers to the licences listed in the appendix attached to this Licence. Should the Licensee's obligations under the Compatible Licence conflict with his/her obligations under this Licence, the obligations of the Compatible Licence shall prevail. @@ -146,9 +146,9 @@ terms of this Licence. The Work is a work in progress, which is continuously improved by numerous Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. +‘bugs’ inherent to this type of development. -For the above reason, the Work is provided under the Licence on an ‘as is’ +For the above reason, the Work is provided under the Licence on an ‘as is’ basis and without warranties of any kind concerning the Work, including without limitation merchantability, fitness for a particular purpose, absence of defects or errors, accuracy, non-infringement of intellectual property @@ -181,8 +181,8 @@ liability. 10. Acceptance of the Licence -The provisions of this Licence can be accepted by clicking on an icon ‘I -agree’ placed under the bottom of a window displaying the text of this Licence +The provisions of this Licence can be accepted by clicking on an icon ‘I +agree’ placed under the bottom of a window displaying the text of this Licence or by affirming consent in any other similar way, in accordance with the rules of applicable law. Clicking on that icon indicates your clear and irrevocable acceptance of this Licence and all of its terms and conditions. @@ -232,12 +232,12 @@ their choice. 14. Jurisdiction Without prejudice to specific agreement between parties, -— any litigation resulting from the interpretation of this License, arising +— any litigation resulting from the interpretation of this License, arising between the European Union institutions, bodies, offices or agencies, as a Licensor, and any Licensee, will be subject to the jurisdiction of the Court of Justice of the European Union, as laid down in article 272 of the Treaty on the Functioning of the European Union, -— any litigation arising between other parties and resulting from the +— any litigation arising between other parties and resulting from the interpretation of this License, will be subject to the exclusive jurisdiction of the competent court where the Licensor resides or conducts its primary business. @@ -245,42 +245,30 @@ Without prejudice to specific agreement between parties, 15. Applicable Law Without prejudice to specific agreement between parties, -— this Licence shall be governed by the law of the European Union Member State +— this Licence shall be governed by the law of the European Union Member State where the Licensor has his seat, resides or has his registered office, -— this licence shall be governed by Belgian law if the Licensor has no seat, +— this licence shall be governed by Belgian law if the Licensor has no seat, residence or registered office inside a European Union Member State. Appendix -‘Compatible Licences’ according to Article 5 EUPL are: -— GNU General Public License (GPL) v. 2, v. 3 -— GNU Affero General Public License (AGPL) v. 3 -— Open Software License (OSL) v. 2.1, v. 3.0 -— Eclipse Public License (EPL) v. 1.0 -— CeCILL v. 2.0, v. 2.1 -— Mozilla Public Licence (MPL) v. 2 -— GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -— Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for +‘Compatible Licences’ according to Article 5 EUPL are: +— GNU General Public License (GPL) v. 2, v. 3 +— GNU Affero General Public License (AGPL) v. 3 +— Open Software License (OSL) v. 2.1, v. 3.0 +— Eclipse Public License (EPL) v. 1.0 +— CeCILL v. 2.0, v. 2.1 +— Mozilla Public Licence (MPL) v. 2 +— GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 +— Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for works other than software -— European Union Public Licence (EUPL) v. 1.1, v. 1.2 -— Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or +— European Union Public Licence (EUPL) v. 1.1, v. 1.2 +— Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong Reciprocity (LiLiQ-R+) -— The European Commission may update this Appendix to later versions of the +— The European Commission may update this Appendix to later versions of the above licences without producing a new version of the EUPL, as long as they provide the rights granted in Article 2 of this Licence and protect the covered Source Code from exclusive appropriation. -— All other changes or additions to this Appendix require the production of a +— All other changes or additions to this Appendix require the production of a new EUPL version. - -/* ********************************************************************* -* The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -* amended by the European Commission) shall be deemed incompatible for -* the purposes of the Work and the provisions of the compatibility -* clause in Article 5 of the EUPL shall not apply. -* -* If using the Work as, or as part of, a network application, by -* including the attribution notice(s) required under Article 5 of the EUPL -* in the end user terms of the application under an appropriate heading, -* such notice(s) shall fulfil the requirements of that article. -* ********************************************************************* */ diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_datafile.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_datafile.py index 5a423013..a586ed76 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_datafile.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_datafile.py @@ -18,7 +18,7 @@ # including the attribution notice(s) required under Article 5 of the EUPL # in the end user terms of the application under an appropriate heading, # such notice(s) shall fulfill the requirements of that article. -# **********************************************************************/ +# ********************************************************************* from __future__ import absolute_import from fiftyone_pipeline_engines.datafile import DataFile diff --git a/ip-intelligence/LICENSE b/ip-intelligence/LICENSE deleted file mode 100644 index c9c8de6a..00000000 --- a/ip-intelligence/LICENSE +++ /dev/null @@ -1,286 +0,0 @@ -European Union Public Licence -V. 1.2 - -EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as -defined below) which is provided under the terms of this Licence. Any use of -the Work, other than as authorised under this Licence is prohibited (to the -extent such use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: “Licensed under the EUPL”, or has expressed by -any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: -— ‘The Licence’: this Licence. -— ‘The Original Work’: the work or software distributed or communicated by the - ‘Licensor under this Licence, available as Source Code and also as - ‘Executable Code as the case may be. -— ‘Derivative Works’: the works or software that could be created by the - ‘Licensee, based upon the Original Work or modifications thereof. This - ‘Licence does not define the extent of modification or dependence on the - ‘Original Work required in order to classify a work as a Derivative Work; - ‘this extent is determined by copyright law applicable in the country - ‘mentioned in Article 15. -— ‘The Work’: the Original Work or its Derivative Works. -— ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -— ‘The Executable Code’: any code which has generally been compiled and which - is meant to be interpreted by a computer as a program. -— ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. -— ‘Contributor(s)’: any natural or legal person who modifies the Work under - the Licence, or otherwise contributes to the creation of a Derivative Work. -— ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. -— ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright -vested in the Original Work: - -— use the Work in any circumstance and for all usage, -— reproduce the Work, -— modify the Work, and make Derivative Works based upon the Work, -— communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case - may be, the Work, -— distribute the Work or copies thereof, -— lend and rent the Work or copies thereof, -— sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make -effective the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights -to any patents held by the Licensor, to the extent necessary to make use of -the rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, -in a notice following the copyright notice attached to the Work, a repository -where the Source Code is easily and freely accessible for as long as the -Licensor continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits -from any exception or limitation to the exclusive rights of the rights owners -in the Work, of the exhaustion of those rights or of other applicable -limitations thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and -a copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of -the Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions -on the Work or Derivative Work that alter or restrict the terms of the -Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed -under a Compatible Licence, this Distribution or Communication can be done -under the terms of this Compatible Licence. For the sake of this clause, -‘Compatible Licence’ refers to the licences listed in the appendix attached to -this Licence. Should the Licensee's obligations under the Compatible Licence -conflict with his/her obligations under this Licence, the obligations of the -Compatible Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the -Work, the Licensee will provide a machine-readable copy of the Source Code or -indicate a repository where this Source will be easily and freely available -for as long as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade -names, trademarks, service marks, or names of the Licensor, except as required -for reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she -brings to the Work are owned by him/her or licensed to him/her and that he/she -has the power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ -basis and without warranties of any kind concerning the Work, including -without limitation merchantability, fitness for a particular purpose, absence -of defects or errors, accuracy, non-infringement of intellectual property -rights other than copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a -condition for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the -use of the Work, including without limitation, damages for loss of goodwill, -work stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such -damage. However, the Licensor will be liable under statutory product liability -laws as far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional -agreement, defining obligations or services consistent with this Licence. -However, if accepting obligations, You may act only on your own behalf and on -your sole responsibility, not on behalf of the original Licensor or any other -Contributor, and only if You agree to indemnify, defend, and hold each -Contributor harmless for any liability incurred by, or claims asserted against -such Contributor by the fact You have accepted any warranty or additional -liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I -agree’ placed under the bottom of a window displaying the text of this Licence -or by affirming consent in any other similar way, in accordance with the rules -of applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this -Licence, such as the use of the Work, the creation by You of a Derivative Work -or the Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of -electronic communication by You (for example, by offering to download the Work -from a remote location) the distribution channel or media (for example, a -website) must at least provide to the public the information requested by the -applicable law regarding the Licensor, the Licence and the way it may be -accessible, concluded, stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. Such a termination -will not terminate the licences of any person who has received the Work from -the Licensee under the Licence, provided such persons remain in full -compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions -of this Licence or updated versions of the Appendix, so far this is required -and reasonable, without reducing the scope of the rights granted by the -Licence. New versions of the Licence will be published with a unique version -number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, -— any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty - on the Functioning of the European Union, -— any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive - jurisdiction of the competent court where the Licensor resides or conducts - its primary business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, -— this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, -— this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: -— GNU General Public License (GPL) v. 2, v. 3 -— GNU Affero General Public License (AGPL) v. 3 -— Open Software License (OSL) v. 2.1, v. 3.0 -— Eclipse Public License (EPL) v. 1.0 -— CeCILL v. 2.0, v. 2.1 -— Mozilla Public Licence (MPL) v. 2 -— GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -— Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -— European Union Public Licence (EUPL) v. 1.1, v. 1.2 -— Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or - Strong Reciprocity (LiLiQ-R+) - -— The European Commission may update this Appendix to later versions of the - above licences without producing a new version of the EUPL, as long as they - provide the rights granted in Article 2 of this Licence and protect the - covered Source Code from exclusive appropriation. -— All other changes or additions to this Appendix require the production of a - new EUPL version. - -/* ********************************************************************* -* The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -* amended by the European Commission) shall be deemed incompatible for -* the purposes of the Work and the provisions of the compatibility -* clause in Article 5 of the EUPL shall not apply. -* -* If using the Work as, or as part of, a network application, by -* including the attribution notice(s) required under Article 5 of the EUPL -* in the end user terms of the application under an appropriate heading, -* such notice(s) shall fulfil the requirements of that article. -* ********************************************************************* */ diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_datafile.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_datafile.py index 89472bf2..1423fca3 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_datafile.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_datafile.py @@ -18,7 +18,7 @@ # including the attribution notice(s) required under Article 5 of the EUPL # in the end user terms of the application under an appropriate heading, # such notice(s) shall fulfill the requirements of that article. -# **********************************************************************/ +# ********************************************************************* from __future__ import absolute_import from fiftyone_pipeline_engines.datafile import DataFile diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_decimal_places.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_decimal_places.py index 3de26193..738b315a 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_decimal_places.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_swig_data_decimal_places.py @@ -1,3 +1,24 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* """ IP Intelligence Latitude and Longitude are given to 3 decimal places, as the diff --git a/pipeline/LICENSE b/pipeline/LICENSE deleted file mode 100755 index 43159c60..00000000 --- a/pipeline/LICENSE +++ /dev/null @@ -1,298 +0,0 @@ -/* ********************************************************************* -* This Original Work is copyright of 51 Degrees Mobile Experts Limited. -* Copyright 2019 51 Degrees Mobile Experts Limited, 5 Charlotte Close, -* Caversham, Reading, Berkshire, United Kingdom RG4 7BY. -* -* This Original Work is licensed under the European Union Public Licence (EUPL) -* v.1.2 and is subject to its terms as set out below. -* -* If a copy of the EUPL was not distributed with this file, You can obtain -* one at https://opensource.org/licenses/EUPL-1.2. -* ********************************************************************* */ - -European Union Public Licence -V. 1.2 - -EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as -defined below) which is provided under the terms of this Licence. Any use of -the Work, other than as authorised under this Licence is prohibited (to the -extent such use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: “Licensed under the EUPLâ€, or has expressed by -any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: -— ‘The Licence’: this Licence. -— ‘The Original Work’: the work or software distributed or communicated by the - ‘Licensor under this Licence, available as Source Code and also as - ‘Executable Code as the case may be. -— ‘Derivative Works’: the works or software that could be created by the - ‘Licensee, based upon the Original Work or modifications thereof. This - ‘Licence does not define the extent of modification or dependence on the - ‘Original Work required in order to classify a work as a Derivative Work; - ‘this extent is determined by copyright law applicable in the country - ‘mentioned in Article 15. -— ‘The Work’: the Original Work or its Derivative Works. -— ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -— ‘The Executable Code’: any code which has generally been compiled and which - is meant to be interpreted by a computer as a program. -— ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. -— ‘Contributor(s)’: any natural or legal person who modifies the Work under - the Licence, or otherwise contributes to the creation of a Derivative Work. -— ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. -— ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright -vested in the Original Work: - -— use the Work in any circumstance and for all usage, -— reproduce the Work, -— modify the Work, and make Derivative Works based upon the Work, -— communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case - may be, the Work, -— distribute the Work or copies thereof, -— lend and rent the Work or copies thereof, -— sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make -effective the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights -to any patents held by the Licensor, to the extent necessary to make use of -the rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, -in a notice following the copyright notice attached to the Work, a repository -where the Source Code is easily and freely accessible for as long as the -Licensor continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits -from any exception or limitation to the exclusive rights of the rights owners -in the Work, of the exhaustion of those rights or of other applicable -limitations thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and -a copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of -the Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions -on the Work or Derivative Work that alter or restrict the terms of the -Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed -under a Compatible Licence, this Distribution or Communication can be done -under the terms of this Compatible Licence. For the sake of this clause, -‘Compatible Licence’ refers to the licences listed in the appendix attached to -this Licence. Should the Licensee's obligations under the Compatible Licence -conflict with his/her obligations under this Licence, the obligations of the -Compatible Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the -Work, the Licensee will provide a machine-readable copy of the Source Code or -indicate a repository where this Source will be easily and freely available -for as long as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade -names, trademarks, service marks, or names of the Licensor, except as required -for reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she -brings to the Work are owned by him/her or licensed to him/her and that he/she -has the power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ -basis and without warranties of any kind concerning the Work, including -without limitation merchantability, fitness for a particular purpose, absence -of defects or errors, accuracy, non-infringement of intellectual property -rights other than copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a -condition for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the -use of the Work, including without limitation, damages for loss of goodwill, -work stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such -damage. However, the Licensor will be liable under statutory product liability -laws as far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional -agreement, defining obligations or services consistent with this Licence. -However, if accepting obligations, You may act only on your own behalf and on -your sole responsibility, not on behalf of the original Licensor or any other -Contributor, and only if You agree to indemnify, defend, and hold each -Contributor harmless for any liability incurred by, or claims asserted against -such Contributor by the fact You have accepted any warranty or additional -liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I -agree’ placed under the bottom of a window displaying the text of this Licence -or by affirming consent in any other similar way, in accordance with the rules -of applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this -Licence, such as the use of the Work, the creation by You of a Derivative Work -or the Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of -electronic communication by You (for example, by offering to download the Work -from a remote location) the distribution channel or media (for example, a -website) must at least provide to the public the information requested by the -applicable law regarding the Licensor, the Licence and the way it may be -accessible, concluded, stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. Such a termination -will not terminate the licences of any person who has received the Work from -the Licensee under the Licence, provided such persons remain in full -compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions -of this Licence or updated versions of the Appendix, so far this is required -and reasonable, without reducing the scope of the rights granted by the -Licence. New versions of the Licence will be published with a unique version -number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, -— any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty - on the Functioning of the European Union, -— any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive - jurisdiction of the competent court where the Licensor resides or conducts - its primary business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, -— this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, -— this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: -— GNU General Public License (GPL) v. 2, v. 3 -— GNU Affero General Public License (AGPL) v. 3 -— Open Software License (OSL) v. 2.1, v. 3.0 -— Eclipse Public License (EPL) v. 1.0 -— CeCILL v. 2.0, v. 2.1 -— Mozilla Public Licence (MPL) v. 2 -— GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -— Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -— European Union Public Licence (EUPL) v. 1.1, v. 1.2 -— Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or - Strong Reciprocity (LiLiQ-R+) - -— The European Commission may update this Appendix to later versions of the - above licences without producing a new version of the EUPL, as long as they - provide the rights granted in Article 2 of this Licence and protect the - covered Source Code from exclusive appropriation. -— All other changes or additions to this Appendix require the production of a - new EUPL version. - -/* ********************************************************************* -* The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -* amended by the European Commission) shall be deemed incompatible for -* the purposes of the Work and the provisions of the compatibility -* clause in Article 5 of the EUPL shall not apply. -* -* If using the Work as, or as part of, a network application, by -* including the attribution notice(s) required under Article 5 of the EUPL -* in the end user terms of the application under an appropriate heading, -* such notice(s) shall fulfil the requirements of that article. -* ********************************************************************* */ diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_product.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_product.py index 8604ea23..67ee11e7 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_product.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_missing_product.py @@ -1,3 +1,24 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* """ A Cloud Aspect Engine whose product the key does not include builds with diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_wkt.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_wkt.py index 6e938d08..6449b064 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_wkt.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine_wkt.py @@ -1,3 +1,24 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* """ A WKT value in a response from the remote server is an object with one member, diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_lower_case.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_lower_case.py index 028eac4a..7699beef 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_lower_case.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_lower_case.py @@ -1,3 +1,24 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* """ The JSON Builder writes the key of each top-level entry and of each sub-entry, diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py index e6ff5cf0..70281e88 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_records.py @@ -1,3 +1,24 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* """ A Property whose value is a list of Element Data is written as a list of diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_sequence_required.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_sequence_required.py index 45100d5f..2a515894 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_sequence_required.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_sequence_required.py @@ -1,3 +1,24 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* """ The JSON Builder fails with an error, and sets no JSON, when no Sequence diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_wkt.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_wkt.py index 04f0b18f..8369c907 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_wkt.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_wkt.py @@ -1,3 +1,24 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* """ A WKT value is written to JSON as an object with one member, value, holding diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_licence_key_masking.py b/pipeline/fiftyone_pipeline_engines/tests/test_licence_key_masking.py index 535ac41c..8f87c1a6 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_licence_key_masking.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_licence_key_masking.py @@ -1,3 +1,24 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* """ A licence key is shown as its last 4 characters, each hidden character diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py index 93992946..e70d2230 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_record.py @@ -1,3 +1,24 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* """ How the usage sharing element writes an Evidence value that was changed, as diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_tracker.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_tracker.py index 63cc9737..20429c1d 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_tracker.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_tracker.py @@ -18,6 +18,7 @@ # including the attribution notice(s) required under Article 5 of the EUPL # in the end user terms of the application under an appropriate heading, # such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* import unittest from unittest import mock From d79dae4eac69c9de600ae4c445ac8a6d3440cb8c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 02:17:13 +0000 Subject: [PATCH 56/68] Do not cache a Cloud Request Engine response that holds a fodid (#83) * Test that a Cloud Request Engine response with a fodid is not cached * Do not cache a Cloud Request Engine response that holds a fodid An engine can say a result must not be stored in its cache. The Cloud Request Engine says so for a response with a top-level fodid, in any case, which holds a 51Did created for the one request. Such a response is therefore never served from the cache either. * Remove a statement the fodid cache test does not need * Test that a cached response holding a fodid is not served The cache tests covered the store half only. Two tests now reach an entry already in the cache, one ordinary and one holding a fodid, through a cache that answers every lookup with the entry it holds. * Do not serve a cached response that holds a fodid is_cacheable now takes the result rather than the flowdata, so in_cache can ask it about an entry it has found, as .NET, Node.js and PHP do. An entry the engine may not serve counts as a miss. --- .../cloudrequestengine.py | 26 ++++ .../tests/test_cloudrequestengine_cache.py | 146 ++++++++++++++++++ .../src/fiftyone_pipeline_engines/engine.py | 36 ++++- 3 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine_cache.py diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py index a4162609..ff598ef1 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py @@ -456,6 +456,32 @@ def process_internal(self, flowdata): return + def is_cacheable(self, data): + + """! + A response whose JSON has a top-level member named fodid, in any + case, holds a 51Did created for the one request that asked for it, + so it is neither stored in the cache nor served from it. A fodid + below the top level is not one, and a response that cannot be read + as a JSON object is cached as before. + + @type data: ElementData + @param data: A response the engine has just made, or one found in + the cache + + @rtype: bool + @return: False if the response holds a fodid + """ + + try: + answer = json.loads(data.get("cloud")) + except (AttributeError, JSONDecodeError, KeyError, TypeError, + ValueError): + return True + if not isinstance(answer, dict): + return True + return not any(str(name).lower() == "fodid" for name in answer) + def report_properties_not_covered(self, result): """! diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine_cache.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine_cache.py new file mode 100644 index 00000000..d49db793 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine_cache.py @@ -0,0 +1,146 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +A cache never stores or serves a Cloud Request Engine response that holds a +value created for one request, being a top-level member named fodid in any +case, as the caching page of the specification says. +""" + +import json + +from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import \ + CloudRequestEngine +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_engines.aspectdata_dictionary import \ + AspectDataDictionary +from fiftyone_pipeline_engines.basic_dictionary_cache import \ + BasicDictionaryCache + +from .classes.cloudrequestengine_testbase import CloudRequestEngineTestsBase +from .classes.mockrequestclient import MockRequestClient + + +class CountingClient(MockRequestClient): + """Answers every data request with the same text and counts them.""" + + def __init__(self, answer): + super(CountingClient, self).__init__() + self.answer = answer + self.data_requests = 0 + + def request(self, type, url, content, originHeader): + response = super(CountingClient, self).request( + type, url, content, originHeader) + if "resource_key.json" in url: + self.data_requests += 1 + response.text = self.answer + return response + + +class AlwaysHitCache(BasicDictionaryCache): + """Answers every lookup with the one entry it holds, so that a reader + reaches an entry the engine never stored itself.""" + + def __init__(self): + super(AlwaysHitCache, self).__init__() + self.entry = None + + def get_cache_value(self, key): + return self.entry + + +class TestCloudRequestEngineCache(CloudRequestEngineTestsBase): + + def pipeline_for(self, client, cache): + """A pipeline holding one Cloud Request Engine with the cache given. + Gives the engine and the pipeline.""" + + engine = CloudRequestEngine({ + "resource_key": "resource_key", "http_client": client}) + engine.set_cache(cache) + pipeline = PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}).add(engine).build() + self.addCleanup(pipeline.close) + return engine, pipeline + + def answer_for(self, pipeline): + """One request through the pipeline. Gives the response text the + request ended with.""" + + data = pipeline.create_flowdata() + data.evidence.add("query.User-Agent", "iPhone") + data.process() + return data.get("cloud").get("cloud") + + def requests_for(self, answer): + """Two requests with the same evidence, through a cache. Gives how + many reached the cloud service.""" + + client = CountingClient(answer) + _, pipeline = self.pipeline_for(client, BasicDictionaryCache()) + for _ in range(2): + self.assertEqual(answer, self.answer_for(pipeline)) + return client.data_requests + + def found_in_cache(self, entry_text): + """One request whose cache answers with an entry holding entry_text, + where the cloud service answers with something else. Gives the + response text the request ended with and how many requests reached + the service.""" + + answer = json.dumps({"device": {"ismobile": False}}) + client = CountingClient(answer) + cache = AlwaysHitCache() + engine, pipeline = self.pipeline_for(client, cache) + cache.entry = AspectDataDictionary(engine, {"cloud": entry_text}) + return self.answer_for(pipeline), client.data_requests + + def test_an_ordinary_response_is_cached(self): + answer = json.dumps({"device": {"ismobile": True}}) + self.assertEqual(1, self.requests_for(answer)) + + def test_a_response_with_a_fodid_is_neither_stored_nor_served(self): + for name in ("fodid", "FodId", "FODID"): + answer = json.dumps({"device": {"ismobile": True}, name: "abc"}) + self.assertEqual(2, self.requests_for(answer), name) + + def test_a_fodid_below_the_top_level_does_not_count(self): + answer = json.dumps({"device": {"fodid": "abc"}}) + self.assertEqual(1, self.requests_for(answer)) + + def test_an_entry_without_a_fodid_is_served(self): + """The cache really is read, so the test below says something.""" + + entry_text = json.dumps({"device": {"ismobile": True}}) + served, requests = self.found_in_cache(entry_text) + self.assertEqual(entry_text, served) + self.assertEqual(0, requests) + + def test_an_entry_holding_a_fodid_is_not_served(self): + for name in ("fodid", "FodId", "FODID"): + entry_text = json.dumps({"device": {"ismobile": True}, + name: "abc"}) + served, requests = self.found_in_cache(entry_text) + self.assertNotEqual(entry_text, served, name) + self.assertEqual(1, requests, name) diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py index 8729ab87..4894bca8 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py @@ -74,11 +74,33 @@ def set_restricted_properties(self, properties_list): name.lower() for name in properties_list} + def is_cacheable(self, data): + + """! + Whether a result may be stored in the cache and served from it. A + result that holds a value created for the one request that asked for + it must not be, so an engine whose results can hold one overrides + this. + + @type data: ElementData + @param data: A result the engine has just made, or one found in the + cache + + @rtype: bool + @return: True if the result may be stored and served + + """ + + return True + def in_cache(self, flowdata): """! - A method to check if a flowdata's evidence is in the cache - + A method to check if a flowdata's evidence is in the cache. An entry + the engine may not serve, because it holds a value created for the + one request that asked for it, counts as a miss and is left where it + is, so the engine processes the flowdata as if nothing was there. + @type FlowData: FlowData @param FlowData: @@ -86,14 +108,14 @@ def in_cache(self, flowdata): @return: True or false: a flowdata's evidence is in the cache """ - + keys = self.filter_evidence(flowdata) cacheKey = json.dumps(keys) cached = self.cache.get_cache_value(cacheKey) - if cached is not None: + if cached is not None and self.is_cacheable(cached): flowdata.set_element_data(cached) return True @@ -130,8 +152,10 @@ def process(self, flowdata): return True else: self.process_internal(flowdata) - cacheKey = json.dumps(self.filter_evidence(flowdata)) - self.cache.set_cache_value(cacheKey, flowdata.get(self.datakey)) + data = flowdata.get(self.datakey) + if self.is_cacheable(data): + cacheKey = json.dumps(self.filter_evidence(flowdata)) + self.cache.set_cache_value(cacheKey, data) else: From 8ac723611876a92810229bf5e0726d688ea02e34 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 02:17:42 +0000 Subject: [PATCH 57/68] Treat evidence keys that differ only in letter case as one key (#95) * Test that evidence keys ignore letter case * Treat evidence keys that differ only in letter case as one key, as the specification says * Test that the elements reading evidence keys ignore letter case get_all gives each key in the letters it was added with, so the Sequence Element's key filter and the JavaScript Builder's reads are tested with keys in other letters. * Match evidence keys in lower case where an element reads them The Sequence Element's key filter and the JavaScript Builder's key filter, session id, sequence and query parameters all matched the keys get_all gives with letter case, so a key added in other letters was missed. Each now matches in lower case, and a name sent on keeps the letters it was added with. --- .../src/fiftyone_pipeline_core/evidence.py | 14 +- .../javascriptbuilder.py | 29 +++-- .../fiftyone_pipeline_core/sequenceelement.py | 11 +- .../tests/test_evidence_key_case.py | 122 ++++++++++++++++++ 4 files changed, 162 insertions(+), 14 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_evidence_key_case.py diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py index 02b45ce7..ee6b8f5d 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py @@ -35,6 +35,9 @@ def __init__(self, flowdata): @param flowdata: Parent FlowData """ self.evidence = {} + # Each key in lower case to the key as first added, because keys + # ignore letter case. + self.keys = {} self.evidence_filtered = False self.flowdata = flowdata self.unaccepted = {} @@ -50,6 +53,11 @@ def add(self, key, value): @param value: a piece of evidence """ + folded = str(key).lower() + if folded in self.keys: + key = self.keys[folded] + else: + self.keys[folded] = key self.evidence[key] = value def add_from_dict(self, evidence_dictionary): @@ -77,9 +85,10 @@ def get(self, key): @return: A piece of evidence """ - if key in self.evidence: + found = self.keys.get(str(key).lower()) + if found is not None and found in self.evidence: - return self.evidence[key] + return self.evidence[found] else: @@ -127,5 +136,6 @@ def filter(self) -> None: in self.flowdata.pipeline.flow_elements ]): self.unaccepted[key] = self.evidence.pop(key) + del self.keys[str(key).lower()] self.evidence_filtered = True diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py index 59bdf9bd..86a84fc4 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py @@ -228,14 +228,17 @@ def url_encode(value): class JavaScriptBuilderEvidenceKeyFilter(EvidenceKeyFilter): # The query string, which the script sends back to the callback, and the - # host and protocol the callback URL is built from. + # host and protocol the callback URL is built from. Evidence keys ignore + # letter case, so each is matched in lower case. def filter_evidence_key(self, key): - if key.startswith("query."): + folded = str(key).lower() + + if folded.startswith("query."): return True - if key == "header.host" or key == "header.protocol": + if folded == "header.host" or folded == "header.protocol": return True - + return False @@ -413,12 +416,22 @@ def process_internal(self, flowdata): + self.settings["_objName"] + "' is used.") query_params = self.get_evidence_key_filter().filter_evidence(flowdata.evidence.get_all()) + # Evidence keys ignore letter case, and get_all gives each key in + # the letters it was added with, so a key is matched here in lower + # case and keeps those letters where it is sent on. + folded = {str(key).lower(): key for key in query_params} + + def query_param(name): + key = folded.get(name) + return None if key is None else query_params[key] + # Both are written into the script, so each always has a safe value. # The session id is written inside quotes and is empty when absent or # not safe, and the sequence is written as bare code, so it is always # a positive number. - variables["_sessionId"] = get_session_id(query_params.get("query.session-id")) - variables["_sequence"] = get_sequence(query_params.get("query.sequence")) + variables["_sessionId"] = get_session_id( + query_param("query.session-id")) + variables["_sequence"] = get_sequence(query_param("query.sequence")) # The session id and the sequence are left out, because the script # appends both to its own request after it has taken the record of @@ -435,8 +448,8 @@ def process_internal(self, flowdata): (url_encode(param.split(".", 1)[1]), url_encode(query_params[param])) for param in query_params.keys() - if param.startswith("query.") - and param not in EXCLUDED_PARAMETERS + if str(param).lower().startswith("query.") + and str(param).lower() not in EXCLUDED_PARAMETERS ]) # Written without spaces, as the .NET builder writes it. variables["_parameters"] = json.dumps( diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py index cd85a857..2aaf8283 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py @@ -29,12 +29,15 @@ class SequenceElementEvidenceKeyFilter(EvidenceKeyFilter): # The session id and the sequence. The result changes as the sequence - # grows, so they are part of the ETag. + # grows, so they are part of the ETag. Evidence keys ignore letter case, + # so each is matched in lower case. def filter_evidence_key(self, key): - - if (key == "query.sequence" or key == "query.session-id"): + + folded = str(key).lower() + + if (folded == "query.sequence" or folded == "query.session-id"): return True - + return False diff --git a/pipeline/fiftyone_pipeline_core/tests/test_evidence_key_case.py b/pipeline/fiftyone_pipeline_core/tests/test_evidence_key_case.py new file mode 100644 index 00000000..48d26770 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_evidence_key_case.py @@ -0,0 +1,122 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Evidence keys ignore letter case, as the Evidence page of the pipeline +specification says, so header.User-Agent and header.user-agent are the same +key. get_all gives each key in the letters it was added with, so an element +that matches a key it holds matches it in lower case.""" + +import json +import unittest +from unittest import mock + +from fiftyone_pipeline_core import javascriptbuilder +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_core.sequenceelement import SequenceElement + +from .classes.exampleflowelement1 import ExampleFlowElement1 + + +def flowdata(): + return PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(ExampleFlowElement1()).build().create_flowdata() + + +class _Captured: + """Records the variables of the last render and renders nothing.""" + + def __init__(self): + self.variables = None + + def render(self, template, variables): + self.variables = dict(variables) + return "" + + +def rendered_variables(evidence): + """The variables the JavaScript Builder renders its script from, after a + pipeline holding the Sequence Element has processed the evidence given.""" + + pipeline = PipelineBuilder({"javascript_builder_settings": { + "minify": False, "endpoint": "/json"}}).build() + data = pipeline.create_flowdata() + for key, value in evidence.items(): + data.evidence.add(key, value) + captured = _Captured() + with mock.patch.object( + javascriptbuilder.chevron, "render", captured.render): + data.process() + return captured.variables + + +class EvidenceKeyCaseTests(unittest.TestCase): + + def test_a_key_is_read_in_any_case(self): + data = flowdata() + data.evidence.add("header.User-Agent", "a browser") + self.assertEqual("a browser", data.evidence.get("header.user-agent")) + self.assertEqual("a browser", data.evidence.get("HEADER.USER-AGENT")) + + def test_keys_that_differ_only_in_case_are_one_key(self): + data = flowdata() + data.evidence.add("header.User-Agent", "first") + data.evidence.add("header.user-agent", "second") + self.assertEqual(1, len(data.evidence.get_all())) + self.assertEqual("second", data.evidence.get("header.User-Agent")) + + def test_a_key_in_any_case_survives_filtering(self): + data = flowdata() + data.evidence.add("Header.User-Agent", "a browser") + data.process() + self.assertEqual("a browser", data.evidence.get("header.user-agent")) + + def test_the_sequence_element_accepts_its_keys_in_any_case(self): + """Without this the two keys move to the evidence set aside, and the + Sequence Element then starts a session of its own.""" + + data = PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(SequenceElement()).build().create_flowdata() + data.evidence.add("Query.Session-Id", "abc-123") + data.evidence.add("QUERY.SEQUENCE", "4") + data.process() + self.assertEqual({}, data.evidence.get_unaccepted()) + self.assertEqual("abc-123", data.evidence.get("query.session-id")) + + def test_the_script_takes_the_session_and_sequence_in_any_case(self): + variables = rendered_variables({ + "Query.Session-Id": "abc-123", "QUERY.SEQUENCE": "4"}) + self.assertEqual("abc-123", variables["_sessionId"]) + # The Sequence Element counts this request as the next one. + self.assertEqual(5, variables["_sequence"]) + + def test_the_script_sends_a_query_value_whose_key_is_in_any_case(self): + variables = rendered_variables({"Query.Example": "a value"}) + self.assertEqual( + {"Example": "a+value"}, json.loads(variables["_parameters"])) + + +if __name__ == "__main__": + unittest.main() From b6519a91a4077e71e6e1a45fab7dbb9b38c6ce0f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 11:47:50 +0000 Subject: [PATCH 58/68] Take the free data files from the run's asset cache (#106) Each test job downloaded the Asn data file again with the data repository's script, outside the asset cache that the run fills once and every test job restores. They are now fetched as assets with the others, restored from the cache in each test job, and copied or linked to where the tests and examples look for them, so no test job downloads a data file. The prefetch in Configure fetches every product's files, because the cache key does not depend on the products a change touches. The notes say CI takes every data file from the assets, and that a new data file needs an entry in common-ci's steps/fetch-assets.ps1. Invoke-ProductStep takes the products to run when it is given them. The Device Detection script's call to the data repository's script downloaded nothing, so it is removed. --- AGENTS.md | 7 +++++-- ci/dispatch.ps1 | 13 ++++++++----- ci/fetch-assets.ps1 | 8 +++++++- device-detection/ci/fetch-assets.ps1 | 9 --------- ip-intelligence/ci/fetch-assets.ps1 | 21 +++++++++------------ 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7d729849..1b697ff7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -300,8 +300,11 @@ The data files come through cxx's own `data` submodule, at `cxx/data/device-detection` and `cxx/data/ip-intelligence`, where each free file is a `.link` that `cxx/data/get-data-files.ps1` or `cxx/data/get-data-files.sh` downloads. Each product's `ci/fetch-assets.ps1` -gets the files its tests need, and the examples' `example_utils.py` runs the -script for a file that is missing. The IP Intelligence on-premise package must +gets the files its tests need from common-ci's assets, which each run caches +once, so no test job downloads a data file. A file added to the data +repository needs an entry in common-ci's `steps/fetch-assets.ps1` before CI +can use it. The examples' `example_utils.py` runs the script for a file +that is missing. The IP Intelligence on-premise package must compile its own copy of the common C library with large data file support; the Device Detection one without it. diff --git a/ci/dispatch.ps1 b/ci/dispatch.ps1 index 4f3365d8..4249d40b 100644 --- a/ci/dispatch.ps1 +++ b/ci/dispatch.ps1 @@ -8,13 +8,16 @@ . (Join-Path $PSScriptRoot 'packages.ps1') -# Runs $Step for each affected product. Every product is run even when an -# earlier one fails, so one run reports every broken product, and the step -# fails at the end if any did. +# Runs $Step for each affected product, or for each of $Products when given. +# Every product is run even when an earlier one fails, so one run reports +# every broken product, and the step fails at the end if any did. function Invoke-ProductStep([string]$Step, [string]$RepoName, - [Collections.IDictionary]$Options) { + [Collections.IDictionary]$Options, [string[]]$Products) { $failed = [Collections.Generic.List[string]]::new() - foreach ($product in (Get-AffectedProducts $RepoName)) { + if (-not $Products) { + $Products = Get-AffectedProducts $RepoName + } + foreach ($product in $Products) { $script = Join-Path $RepoName $product 'ci' "$Step.ps1" if ((Test-Path $script) -eq $false) { Write-Output "$product has no $Step step" diff --git a/ci/fetch-assets.ps1 b/ci/fetch-assets.ps1 index 191c3119..802476f1 100644 --- a/ci/fetch-assets.ps1 +++ b/ci/fetch-assets.ps1 @@ -13,4 +13,10 @@ param ( $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') -Invoke-ProductStep 'fetch-assets' $RepoName $PSBoundParameters +# The prefetch in the Configure job fills the asset cache that every run that +# day restores, whatever it changes, so it fetches for every product. +$fetchProducts = $null +if ($env:GITHUB_JOB -eq 'Configure') { + $fetchProducts = @((Get-Packages $RepoName).Product | Select-Object -Unique) +} +Invoke-ProductStep 'fetch-assets' $RepoName $PSBoundParameters $fetchProducts diff --git a/device-detection/ci/fetch-assets.ps1 b/device-detection/ci/fetch-assets.ps1 index 45cb2875..a2501740 100644 --- a/device-detection/ci/fetch-assets.ps1 +++ b/device-detection/ci/fetch-assets.ps1 @@ -30,12 +30,3 @@ foreach ($asset in $assets) { default {New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/$_" -Path "$deviceDetectionData/$_"} } } - -# Downloads any other file the data repository links to in the folder. The -# script keeps a file that is already there, so the assets linked in above -# are not downloaded again. -$global:LASTEXITCODE = 0 -& "$data/get-data-files.ps1" device-detection -if ($LASTEXITCODE -ne 0) { - throw "Not every data file in $deviceDetectionData was downloaded" -} diff --git a/ip-intelligence/ci/fetch-assets.ps1 b/ip-intelligence/ci/fetch-assets.ps1 index 6153ecfd..3340931d 100644 --- a/ip-intelligence/ci/fetch-assets.ps1 +++ b/ip-intelligence/ci/fetch-assets.ps1 @@ -29,18 +29,15 @@ foreach ($asset in $assets) { } } -# The ASN file is not an asset, so the data repository's script downloads it -# beside its link in cxx/data/ip-intelligence, where the tests look for it. -# The examples look for the Lite data file in the data repository, under the -# name its script saves it as, so the asset is linked there too rather than -# downloaded again. -New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/51Degrees-LiteIpiV41.ipi" -Path "$PSScriptRoot/../../cxx/data/ip-intelligence/51Degrees-LiteV41.ipi" - -$global:LASTEXITCODE = 0 -& "$PSScriptRoot/../../cxx/data/get-data-files.ps1" 51Degrees-IPIV4AsnIpiV41.ipi -if ($LASTEXITCODE -ne 0) { - throw "The ASN data file was not downloaded" -} +# The tests look for the Asn file in cxx/data/ip-intelligence, and the examples +# look there for the Lite data file, under the name the data repository saves +# it as. Both are assets, so a test job restores them from the run's cache +# rather than downloading them. +$ipIntelligenceData = "$PSScriptRoot/../../cxx/data/ip-intelligence" +New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/51Degrees-LiteIpiV41.ipi" -Path "$ipIntelligenceData/51Degrees-LiteV41.ipi" +$asn = '51Degrees-IPIV4AsnIpiV41.ipi' +./steps/fetch-assets.ps1 -Assets $asn +Copy-Item "assets/$asn" $ipIntelligenceData # If the Enterprise file was downloaded, prefer it for tests by exposing # it through the environment variable used by the test suites. From 8b0b6248e33f34bc1b2110890147664ab625fe9d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 15:06:47 +0000 Subject: [PATCH 59/68] Align the data update service to the specification: HTTP date, expected time, update from memory and logging (#69) * Align the data update service to the specification Send If-Modified-Since as an HTTP date, wait for the data update expected time before the first poll, add update_from_memory, and log the actions with the messages and levels in the Logging section of the data updates specification. * Treat a data file that is already current as a finished update in the update examples With If-Modified-Since sent as an HTTP date, the Distributor answers 304 when the file is current, which the examples reported as a failure. * Test that no update message holds the licence key, the watcher needs automatic updates, a file data file is written, and content checks carry on * Register the watched data files in the watcher tests with automatic updates on * Mask the licence key in every update message, create the watcher only with automatic updates, write update_from_memory's bytes to a data file kept on disk, and carry on without an MD5 header or with content that is not compressed * Test that a callback that raises does not stop the checks, and a connection failure keeps polling * Log a callback that raises, so the other callbacks and the next check still run * Test that the watcher refresh is routed as the polling refresh is * Route the watcher refresh through the polling refresh * Test that a callback's error is logged with .NET's message for any other error DataUpdateService.cs logs an error from an update complete callback with MessageAutoUpdateUnhandledError at Error level. The test now asserts that text, so it fails while Python writes its own sentence. * Test that a callback that raises stops the callbacks after it data-updates.md at e89679e9 says the callbacks after the one that raised are not called. On update polling the error is logged with the last row's message, which is .NET's MessageAutoUpdateUnhandledError, and on a programmatic check it passes to the caller. The tests cover polling, a programmatic check and an update from memory, and that polling still schedules its next check. This commit is the tests alone, so they fail. * Test that a callback that raises is logged and stops the callbacks after it data-updates.md at e89679e9 says the callbacks after the one that raised are not called, and that on update polling and the file watcher the error is logged with the last row's message, .NET's MessageAutoUpdateUnhandledError. Passing the error to the caller of a programmatic check, an update from memory or the update on start-up waits for next week, so on those routes it is logged in the same way for now. This commit is the tests alone, so they fail. * Stop at the first callback that raises, and log it with .NET's message DataUpdateService.cs wraps the callback loop in one try, so the first callback that raises stops the rest, and LogUpdateException logs it with MessageAutoUpdateUnhandledError, "An unhandled error occurred while checking for automatic updates for engine '{0}'", at Error. data-updates.md says the same. Python now does both, on every route, and the next check is still scheduled. * Attach the error a callback raised to the log entry The errors table in the data updates specification says an error of any other kind is logged with the error attached, and that a callback's error is logged in the same way. __emit wrote the message and bound no variable, so the error the callback raised was lost. It is now written after the message, which is what .NET, Java, Node.js and Go each do. The new test file takes the licence header every other file in the repository carries. * Name only the languages that attach a callback's error The comment on the expected message said .NET, Node.js and Go write the error the callback raised, and Java as well. Java's data update service calls each onUpdateComplete in a bare loop with no catch (DataUpdateServiceDefault.java on its rc, the loops after the update and after the file watcher's refresh), so a callback that raises there is not caught and nothing is logged. The comment now names the three that do. * Test that a callback's error is masked in the log entry * Mask every licence key in a callback's error before logging it --- .../onpremise/datafileupdate_console.py | 3 + .../onpremise/datafileupdate_console.py | 3 + .../datafile_update_service.py | 367 ++++++++--- .../tests/test_datafile_update_http.py | 574 ++++++++++++++++++ .../tests/test_datafile_update_service.py | 52 +- .../tests/test_licence_key_masking.py | 36 +- 6 files changed, 945 insertions(+), 90 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index 2a8d0fd2..572c7a82 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -354,6 +354,9 @@ def run(self, data_file, license_key, interactive, logger, output): output(f"Update on file modification complete, status: {update_event.status}") else: output("Update on file modification timed out") + elif update_event.status == UpdateStatus.AUTO_UPDATE_NOT_NEEDED: + output("The data file is already the latest published, " + "so there is nothing to download") else: logger.log("error", "Auto update was not successful, abandoning example") error_message = f"Auto update failed: {update_event.status}" diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py index 6e7a8ebe..1ef6fcf4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py @@ -231,6 +231,9 @@ def run(self, data_file, license_key, interactive, logger, output): f"{update_event.status}") else: output("Update on file modification timed out") + elif update_event.status == UpdateStatus.AUTO_UPDATE_NOT_NEEDED: + output("The data file is already the latest published, " + "so there is nothing to download") else: logger.log("error", "Auto update was not successful, abandoning example") error_message = f"Auto update failed: {update_event.status}" diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py index 7ded5ad9..93568b01 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py @@ -20,6 +20,9 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +import calendar +import datetime +from email.utils import formatdate from enum import Enum import gzip import shutil @@ -27,10 +30,9 @@ import random import threading import time +import zlib import requests from fiftyone_pipeline_core.keymask import mask_licence_keys_in_text -# gzip decompress alternative for python 2 -from io import StringIO class UpdateStatus(Enum): @@ -57,6 +59,10 @@ class UpdateStatus(Enum): # Refreshing the engine with the new data caused an error to occur. AUTO_UPDATE_REFRESH_FAILED = 10 +# The first two bytes of gzip compressed content (RFC 1952 section 2.3.1). +GZIP_MAGIC = b"\x1f\x8b" + + class DataFileUpdateService(): """! @@ -79,8 +85,20 @@ def __init__(self, pipeline = None): self.on_complete_methods = [] def __emit(self, status, data_file): - for method in self.on_complete_methods: - method(status, data_file) + # A callback that raises stops the callbacks after it, as .NET's service + # does, and its error is logged with .NET's message for any other + # error and the error itself after it, so that the next check still + # runs. The error is a caller's, so it is masked as any other text + # that may name the update URL is. + try: + for method in self.on_complete_methods: + method(status, data_file) + except Exception as e: + self._log( + "error", + "An unhandled error occurred while checking for automatic " + "updates for engine '" + self._engine_type(data_file) + + "'. Error detail: " + mask_licence_keys_in_text(e)) def update_data_file(self, data_file): @@ -100,103 +118,126 @@ def update_data_file(self, data_file): if data_file.updating: pass + self._log("info", "Checking for update") + # Get the update url url = data_file.get_update_url() # The URL carries the licence key, so only a masked copy is logged. shown_url = mask_licence_keys_in_text(url) + engine = self._engine_type(data_file) headers = {} status = UpdateStatus.AUTO_UPDATE_IN_PROGRESS # Add if modified since header if enabled and available + since = None if data_file.verify_if_modified_since: - try: - headers["If-Modified-Since"] = data_file.get_date_published().strftime('%Y-%m-%dT%H:%M:%SZ') - except: - pass - + since = self._if_modified_since(data_file) + if since: + headers["If-Modified-Since"] = since + + self._log( + "info", + "Checking for update from '" + shown_url + "' for Engine '" + + engine + "'") + + retry = self._retry_text(data_file) + try: response = requests.get(url, headers=headers) except requests.RequestException as e: # The message of a failed request names the URL, key included. - raise requests.RequestException( - mask_licence_keys_in_text(e)) from None + self._log( + "warning", + "An error occurred when connecting to " + shown_url + + " in order to check for data file updates for " + engine + + "." + retry + " Error detail: " + + mask_licence_keys_in_text(e)) + return self._update_failed( + data_file, UpdateStatus.AUTO_UPDATE_HTTPS_ERR) + + if response.status_code == 304: + self._log( + "info", + "No data newer than " + (since or "the data in use") + + " found at '" + shown_url + "' for Engine '" + engine + + "'") + return self._update_failed( + data_file, UpdateStatus.AUTO_UPDATE_NOT_NEEDED) if response.status_code != 200: # Handle errors if response.status_code == 429: - self.pipeline.log("error", "Too many requests to " + shown_url + "for engine " + data_file.flow_element.datakey) status = UpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS - elif response.status_code == 304: - self.pipeline.log("warn", "No data update available from " + shown_url + "for engine " + data_file.flow_element.datakey) - status = UpdateStatus.AUTO_UPDATE_NOT_NEEDED elif response.status_code == 403: - self.pipeline.log("error", "Access denied from " + shown_url + "for engine " + data_file.flow_element.datakey) status = UpdateStatus.AUTO_UPDATE_ERR_403_FORBIDDEN else: - self.pipeline.log("error", str(response.status_code) + shown_url + "for engine " + data_file.flow_element.datakey) status = UpdateStatus.AUTO_UPDATE_HTTPS_ERR - data_file.updating = False - self.__emit(status, data_file) - self.check_next_update(data_file, run_after_error = True) - return False + self._log( + "warning", + "An error occurred while downloading a data file update " + "for " + engine + " from " + shown_url + "." + retry + + " Error detail: HTTP status code " + + str(response.status_code)) + return self._update_failed(data_file, status) - # Verify MD5 hash if enabled + # Verify the MD5 hash if enabled and the answer gives one. - if data_file.verify_md5: - header_md5 = response.headers["content-md5"] + header_md5 = response.headers.get("content-md5") + if data_file.verify_md5 and header_md5 is None: + self._log( + "warning", + "No MD5 hash included in data update response for '" + + shown_url + "'. Unable to verify data integrity") + elif data_file.verify_md5: import hashlib hash_md5 = hashlib.md5() for data in response.iter_content(8192): hash_md5.update(data) - - if header_md5 != hash_md5.hexdigest(): - self.pipeline.log("error", "MD5 doesn't match from " + shown_url + "for engine " + data_file.flow_element.datakey) - status = UpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED - data_file.updating = False - self.__emit(status, data_file) - self.check_next_update(data_file) - return False + if header_md5 != hash_md5.hexdigest(): + self._log( + "warning", + "An error occurred during the integrity check of new " + "data file for " + engine + "." + retry + + " Error detail: The MD5 hash of the data does not " + "match the content-md5 header.") + return self._update_failed( + data_file, + UpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED) + + self._log( + "info", + "Downloaded new data from '" + shown_url + "' for Engine '" + + engine + "'") data = response.content - # Decompress if set - if data_file.decompress: - # Python 3 version - if(hasattr(gzip, "decompress")): - data = gzip.decompress(response.content) - else: - compressed_stream = StringIO(response.content) - gzipper = gzip.GzipFile(fileobj=compressed_stream) - data = gzipper.read() + # Decompress if set and the content is compressed. + if data_file.decompress and data[:2] == GZIP_MAGIC: + try: + data = gzip.decompress(data) + except (OSError, EOFError, zlib.error) as e: + self._log( + "warning", + "An error occurred during the integrity check of new " + "data file for " + engine + "." + retry + + " Error detail: " + str(e)) + return self._update_failed( + data_file, UpdateStatus.AUTO_UPDATE_ERR_READING_STREAM) # Download datafile if needed, or pass through memory if data_file.download: - - if data_file.create_temp_copy: - - filename = data_file.temp_directory + "/" + data_file.identifier + str(time.time()) - - else: - - filename = data_file.path - - with open(filename, 'wb') as f: - f.write(data) - - if data_file.create_temp_copy: - - # Move to real file - - shutil.move(filename, data_file.path) + failed = self._write_data_file(data_file, data) + if failed is not None: + return self._update_failed(data_file, failed) else: @@ -209,19 +250,184 @@ def update_data_file(self, data_file): # Done. Refresh the file. - try: - data_file.refresh() - status = UpdateStatus.AUTO_UPDATE_SUCCESS - except: - status = UpdateStatus.AUTO_UPDATE_REFRESH_FAILED + status = self._refresh(data_file) data_file.updating = False - + self.__emit(status, data_file) self.check_next_update(data_file) return True + def update_from_memory(self, data_file, data): + + """! + Refresh a data file from the bytes given, without downloading + anything. A data file kept on disk has the bytes written to its path + first, as the update route writes a download, and the engine is then + refreshed from the file. + @type data_file: DataFile + @param data_file: Datafile to update + @type data: bytes + @param data: The new content of the data file + @rtype: UpdateStatus + @return: AUTO_UPDATE_SUCCESS, AUTO_UPDATE_REFRESH_FAILED if the + engine could not use the data (the data it had is kept), the status + of a failed write for a data file kept on disk, or + AUTO_UPDATE_NOT_NEEDED if the data file's engine has been closed + """ + + if not self.__is_registered(data_file): + return UpdateStatus.AUTO_UPDATE_NOT_NEEDED + + if data_file.download and getattr(data_file, "path", None): + data_file.updating = True + try: + status = self._write_data_file(data_file, data) + if status is None: + status = self._refresh(data_file) + finally: + data_file.updating = False + self.__emit(status, data_file) + return status + + previous = data_file.data + data_file.data = data + data_file.updating = True + try: + status = self._refresh(data_file) + finally: + data_file.updating = False + + if status != UpdateStatus.AUTO_UPDATE_SUCCESS: + data_file.data = previous + + self.__emit(status, data_file) + + return status + + def _write_data_file(self, data_file, data): + # Write new data to the data file's path, through a temporary copy + # when the data file asks for one. None when written, or the status + # of the failure, which is logged. + if data_file.create_temp_copy: + filename = os.path.join( + data_file.temp_directory, + data_file.identifier + str(time.time())) + else: + filename = data_file.path + + try: + with open(filename, 'wb') as f: + f.write(data) + except OSError as e: + self._log( + "error", + "An error occurred when writing to " + filename + + ". Error detail: " + str(e)) + return UpdateStatus.AUTO_UPDATE_ERR_READING_STREAM + + if data_file.create_temp_copy: + try: + shutil.move(filename, data_file.path) + except (OSError, shutil.Error): + self._log( + "error", + "An error occurred when copying a data file to " + "replace the existing one at '" + data_file.path + + "'.") + return UpdateStatus.AUTO_UPDATE_NEW_FILE_CANT_RENAME + return None + + def _refresh(self, data_file): + # Refresh the engine with the data file's new content. + engine = self._engine_type(data_file) + self._log( + "info", + "Attempting to refresh Engine '" + engine + "' with new data") + try: + data_file.refresh() + except Exception as e: + self._log( + "error", + "An error occurred while applying a data file update to " + + engine + ". Error detail: " + str(e)) + return UpdateStatus.AUTO_UPDATE_REFRESH_FAILED + return UpdateStatus.AUTO_UPDATE_SUCCESS + + def _update_failed(self, data_file, status): + # An update that ends without new data. The next check is the next + # scheduled one, never an immediate repeat of a check on start-up. + data_file.updating = False + self.__emit(status, data_file) + self.check_next_update(data_file, run_after_error=True) + return False + + def _log(self, level, message): + pipeline = getattr(self, "pipeline", None) + if pipeline is not None: + pipeline.log(level, message) + + @staticmethod + def _engine_type(data_file): + return type(getattr(data_file, "flow_element", data_file)).__name__ + + @staticmethod + def _retry_text(data_file): + # Only an automatic update is tried again. + if getattr(data_file, "auto_update", False): + return " Update will be attempted again later." + return "" + + @staticmethod + def _http_date(moment): + # An HTTP date, as RFC 9110 section 5.6.7 requires. A time with no + # zone is taken as UTC. + if isinstance(moment, datetime.datetime): + fields = moment.utctimetuple() + else: + fields = moment.timetuple() + return formatdate(calendar.timegm(fields), usegmt=True) + + def _if_modified_since(self, data_file): + # The publication date of the data file, or the time the file was + # last modified when it has no publication date. None when neither + # is known. + try: + published = data_file.get_date_published() + except Exception: + published = None + if published is None: + path = getattr(data_file, "path", None) + try: + published = datetime.datetime.fromtimestamp( + os.path.getmtime(path), datetime.timezone.utc) + except (OSError, TypeError): + return None + return self._http_date(published) + + @staticmethod + def _seconds_until_update_expected(data_file): + # Seconds from now until the data file says new data is expected, or + # 0 when it does not say or the time has passed. A date with no time + # is the default of a data file that does not know. + try: + expected = data_file.get_next_update() + except Exception: + return 0 + utc = datetime.timezone.utc + if isinstance(expected, datetime.datetime): + if expected.tzinfo is None: + expected = expected.replace(tzinfo=utc) + elif isinstance(expected, datetime.date): + if expected <= datetime.date.today(): + return 0 + expected = datetime.datetime( + expected.year, expected.month, expected.day, tzinfo=utc) + else: + return 0 + return max(0, (expected - datetime.datetime.now(utc)).total_seconds()) + def async_update_data_file(self, data_file): """! @@ -237,7 +443,9 @@ def async_update_data_file(self, data_file): def check_next_update(self, data_file, run_after_error = False): """! Function to periodically check if a datafile needs updating - also updates the datafile on start if it is set with the update_on_start option + also updates the datafile on start if it is set with the update_on_start option. + Polling does not start until after the time the data file expects + new data, if it gives one. @type data_file: DataFile @param data_file: Datafile to check updates for """ @@ -246,13 +454,17 @@ def check_next_update(self, data_file, run_after_error = False): return if data_file.update_on_start and not data_file.updated_on_start and not run_after_error: + self._log("info", "Updating on start-up") self.update_data_file(data_file) elif data_file.auto_update: - + random_offset = random.randint(0, data_file.update_time_maximum_randomisation) interval = random_offset + (data_file.polling_interval * 60) + until_expected = self._seconds_until_update_expected(data_file) + if until_expected > 0: + interval = until_expected + random_offset t = threading.Timer(interval, self.async_update_data_file, args=[data_file]) t.daemon = True t.name = "AutoUpdate - Scheduled" @@ -277,7 +489,10 @@ def register_data_file(self, data_file): # Watch for changes in file and refresh if # file_system_watcher is enabled on data_file - if data_file.file_system_watcher: + # The watcher is an automatic update, so it only runs with + # automatic updates on. + if data_file.auto_update and data_file.file_system_watcher: + self._log("info", "Creating file system watcher") data_file.last_changed_time = os.stat(data_file.path).st_mtime self.track_local_data_file_changes(data_file) @@ -305,10 +520,12 @@ def __is_registered(self, data_file): def track_local_data_file_changes(self, data_file): """! Check for local changes to a datafile. Used by - the file system watcher. A refresh that fails is logged, and - reported with AUTO_UPDATE_REFRESH_FAILED as on the update route. - The next check is scheduled whatever happens, so that a later file - is still picked up, until the data file is unregistered. + the file system watcher. The refresh takes the same route as the + polling one, so the attempt is logged, a failure carries the + errors table's message and level, and the result is reported to + the update complete callbacks either way. The next check is + scheduled whatever happens, so that a later file is still picked + up, until the data file is unregistered. @type data_file: DataFile @param data_file: Datafile to check updates for """ @@ -326,13 +543,7 @@ def track_local_data_file_changes(self, data_file): changed_time > data_file.last_changed_time and not data_file.updating): data_file.last_changed_time = changed_time - data_file.refresh() - except Exception as e: - self.pipeline.log( - "error", - "Refreshing from the changed data file '" + data_file.path + - "' failed. " + str(e)) - self.__emit(UpdateStatus.AUTO_UPDATE_REFRESH_FAILED, data_file) + self.__emit(self._refresh(data_file), data_file) finally: if self.__is_registered(data_file): t = threading.Timer( diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py new file mode 100644 index 00000000..a1e938eb --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py @@ -0,0 +1,574 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import datetime +import gzip +import os +import random +import shutil +import tempfile +import threading +import unittest +from email.utils import formatdate +from unittest import mock + +import requests + +from fiftyone_pipeline_engines.datafile_update_service import DataFileUpdateService +from fiftyone_pipeline_engines.datafile_update_service import UpdateStatus + +from .test_datafile_update_service import RecordingPipeline +from .test_datafile_update_service import ScheduledCheck + +URL = "https://example.com/update" + +# An update URL that carries a licence key, which no message may hold in +# full. Only its last four characters may be shown. +KEY = "ABCDEFGHIJKL" +KEYED_URL = "https://example.com/update?licenseKeys=" + KEY + "&Type=HashV41" +MASKED_URL = ( + "https://example.com/update?licenseKeys=********IJKL&Type=HashV41") + + +class FakeEngine: + """Stands in for the flow element of a data file.""" + datakey = "fake" + + +class HttpDataFile: + """Stands in for a data file that is kept in memory and updated over + HTTP.""" + + def __init__(self, path=None): + self.flow_element = FakeEngine() + self.identifier = "fake-data" + self.update_on_start = False + self.auto_update = False + self.file_system_watcher = False + self.updating = False + self.verify_if_modified_since = True + self.verify_md5 = False + self.decompress = False + self.download = False + self.create_temp_copy = False + self.polling_interval = 30 + self.update_time_maximum_randomisation = 0 + self.data = b"old" + self.refreshed = [] + self.published = datetime.datetime(2026, 11, 3, 10, 30, 0) + self.next_update = datetime.datetime(2026, 11, 3, 10, 30, 0) + self.refresh_error = None + self.url = URL + if path is not None: + self.path = path + + def get_update_url(self): + return self.url + + def get_date_published(self): + if isinstance(self.published, Exception): + raise self.published + return self.published + + def get_next_update(self): + return self.next_update + + def refresh(self): + if self.refresh_error: + raise self.refresh_error + self.refreshed.append(self.data) + + +def response(status, content=b"new"): + reply = mock.Mock() + reply.status_code = status + reply.content = content + reply.headers = {} + return reply + + +class UpdateServiceTestCase(unittest.TestCase): + + def setUp(self): + self.pipeline = RecordingPipeline() + self.service = DataFileUpdateService(self.pipeline) + self.statuses = [] + self.service.on_complete( + lambda status, data_file: self.statuses.append(status)) + self.data_file = HttpDataFile() + self.pending = [] + + def get(self, reply): + """Replaces the HTTP request and keeps the headers it was sent.""" + self.sent = [] + + def fake_get(url, headers=None, **kwargs): + self.sent.append(headers) + if isinstance(reply, Exception): + raise reply + return reply + + patch = mock.patch.object(requests, "get", fake_get) + patch.start() + self.addCleanup(patch.stop) + + def hold_timers(self): + """Replaces the timers, which are kept in self.pending and never run.""" + timer = mock.patch.object( + threading, "Timer", + lambda interval, function, args=None: ScheduledCheck( + self.pending, function, args or [])) + timer.start() + self.addCleanup(timer.stop) + + +# If-Modified-Since is an HTTP date (RFC 9110 section 5.6.7), set to the +# publication date, or the file system last modified time without one. +class IfModifiedSinceTests(UpdateServiceTestCase): + + def testPublishedDateIsSentAsHttpDate(self): + self.get(response(304)) + self.service.update_data_file(self.data_file) + self.assertEqual( + "Tue, 03 Nov 2026 10:30:00 GMT", + self.sent[0]["If-Modified-Since"]) + + def testLastModifiedTimeIsUsedWithoutPublishedDate(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder, ignore_errors=True) + path = os.path.join(folder, "data.dat") + with open(path, "w") as file: + file.write("x") + os.utime(path, (1700000000, 1700000000)) + self.data_file = HttpDataFile(path) + self.data_file.published = Exception("no published date") + self.get(response(304)) + self.service.update_data_file(self.data_file) + self.assertEqual( + formatdate(1700000000, usegmt=True), + self.sent[0]["If-Modified-Since"]) + self.assertEqual( + "Tue, 14 Nov 2023 22:13:20 GMT", + self.sent[0]["If-Modified-Since"]) + + def testNoHeaderWhenDisabled(self): + self.data_file.verify_if_modified_since = False + self.get(response(304)) + self.service.update_data_file(self.data_file) + self.assertNotIn("If-Modified-Since", self.sent[0]) + + +# Polling does not start until after the time the data update is expected, +# then waits the random offset as well. +class UpdateExpectedTimeTests(UpdateServiceTestCase): + + def setUp(self): + super(UpdateExpectedTimeTests, self).setUp() + self.intervals = [] + timer = mock.patch.object( + threading, "Timer", + lambda interval, function, args=None: self.record(interval)) + timer.start() + self.addCleanup(timer.stop) + randomise = mock.patch.object(random, "randint", lambda a, b: 5) + randomise.start() + self.addCleanup(randomise.stop) + self.data_file.auto_update = True + + def record(self, interval): + self.intervals.append(interval) + return ScheduledCheck([], None, []) + + @staticmethod + def utc_now(): + return datetime.datetime.now(datetime.timezone.utc).replace( + tzinfo=None) + + def testFirstCheckWaitsUntilExpectedTime(self): + self.data_file.next_update = ( + self.utc_now() + datetime.timedelta(hours=2)) + self.service.register_data_file(self.data_file) + self.assertEqual(1, len(self.intervals)) + self.assertAlmostEqual(7200 + 5, self.intervals[0], delta=10) + + def testPollingIntervalIsUsedOnceExpectedTimeHasPassed(self): + self.data_file.next_update = ( + self.utc_now() - datetime.timedelta(hours=2)) + self.service.register_data_file(self.data_file) + self.assertEqual([30 * 60 + 5], self.intervals) + + def testPlainDateOfTodayIsNotInTheFuture(self): + self.data_file.next_update = datetime.date.today() + self.service.register_data_file(self.data_file) + self.assertEqual([30 * 60 + 5], self.intervals) + + +# A memory data file can be refreshed from the bytes given. +class UpdateFromMemoryTests(UpdateServiceTestCase): + + def testMemoryDataFileIsRefreshedFromTheBytes(self): + status = self.service.update_from_memory(self.data_file, b"fresh") + self.assertEqual(UpdateStatus.AUTO_UPDATE_SUCCESS, status) + self.assertEqual(b"fresh", self.data_file.data) + self.assertEqual([b"fresh"], self.data_file.refreshed) + self.assertEqual([UpdateStatus.AUTO_UPDATE_SUCCESS], self.statuses) + + def testFailedRefreshIsReportedAndLogged(self): + self.data_file.refresh_error = Exception("boom") + status = self.service.update_from_memory(self.data_file, b"fresh") + self.assertEqual(UpdateStatus.AUTO_UPDATE_REFRESH_FAILED, status) + self.assertIn( + ("error", + "An error occurred while applying a data file update to " + "FakeEngine. Error detail: boom"), + self.pipeline.logs) + + def testFileDataFileIsWrittenThenRefreshed(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder, ignore_errors=True) + path = os.path.join(folder, "data.dat") + with open(path, "wb") as file: + file.write(b"old") + self.data_file = HttpDataFile(path) + self.data_file.download = True + status = self.service.update_from_memory(self.data_file, b"fresh") + self.assertEqual(UpdateStatus.AUTO_UPDATE_SUCCESS, status) + with open(path, "rb") as file: + self.assertEqual(b"fresh", file.read()) + self.assertEqual(1, len(self.data_file.refreshed)) + self.assertEqual([UpdateStatus.AUTO_UPDATE_SUCCESS], self.statuses) + + def testFailedWriteIsLoggedAndNotRefreshed(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder, ignore_errors=True) + path = os.path.join(folder, "missing", "data.dat") + self.data_file = HttpDataFile(path) + self.data_file.download = True + status = self.service.update_from_memory(self.data_file, b"fresh") + self.assertEqual(UpdateStatus.AUTO_UPDATE_ERR_READING_STREAM, status) + self.assertEqual([], self.data_file.refreshed) + errors = [message for level, message in self.pipeline.logs + if level == "error"] + self.assertEqual(1, len(errors), self.pipeline.logs) + self.assertTrue(errors[0].startswith( + "An error occurred when writing to " + path + + ". Error detail: "), errors[0]) + + def testUnregisteredDataFileIsLeftAlone(self): + self.service.unregister_data_file(self.data_file) + self.service.update_from_memory(self.data_file, b"fresh") + self.assertEqual([], self.data_file.refreshed) + + +# The messages and levels are those of the Logging section of the data +# updates specification. +class LoggingTests(UpdateServiceTestCase): + + def levels(self): + return [level for level, message in self.pipeline.logs] + + def testSuccessfulUpdateLogsEachAction(self): + self.get(response(200)) + self.service.update_data_file(self.data_file) + self.assertEqual([ + ("info", "Checking for update"), + ("info", + "Checking for update from '" + URL + "' for Engine " + "'FakeEngine'"), + ("info", + "Downloaded new data from '" + URL + "' for Engine " + "'FakeEngine'"), + ("info", + "Attempting to refresh Engine 'FakeEngine' with new data"), + ], self.pipeline.logs) + + def testNotModifiedIsInformationNotWarning(self): + self.get(response(304)) + self.service.update_data_file(self.data_file) + self.assertIn( + ("info", + "No data newer than Tue, 03 Nov 2026 10:30:00 GMT found at '" + + URL + "' for Engine 'FakeEngine'"), + self.pipeline.logs) + for level in ("warn", "warning", "error"): + self.assertNotIn(level, self.levels()) + + def testRefusedDownloadIsWarning(self): + self.get(response(403)) + self.service.update_data_file(self.data_file) + self.assertIn( + ("warning", + "An error occurred while downloading a data file update for " + "FakeEngine from " + URL + ". Error detail: HTTP status code " + "403"), + self.pipeline.logs) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_ERR_403_FORBIDDEN], self.statuses) + + def testAutomaticUpdateSaysItWillBeTriedAgain(self): + self.data_file.auto_update = True + self.hold_timers() + self.get(response(500)) + self.service.update_data_file(self.data_file) + self.assertIn( + ("warning", + "An error occurred while downloading a data file update for " + "FakeEngine from " + URL + ". Update will be attempted again " + "later. Error detail: HTTP status code 500"), + self.pipeline.logs) + + def testConnectionFailureIsWarning(self): + self.get(requests.ConnectionError("boom")) + self.assertFalse(self.service.update_data_file(self.data_file)) + self.assertIn( + ("warning", + "An error occurred when connecting to " + URL + " in order to " + "check for data file updates for FakeEngine. Error detail: " + "boom"), + self.pipeline.logs) + self.assertEqual([UpdateStatus.AUTO_UPDATE_HTTPS_ERR], self.statuses) + + def testFailedRefreshIsError(self): + self.get(response(200)) + self.data_file.refresh_error = Exception("boom") + self.service.update_data_file(self.data_file) + self.assertIn( + ("error", + "An error occurred while applying a data file update to " + "FakeEngine. Error detail: boom"), + self.pipeline.logs) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_REFRESH_FAILED], self.statuses) + + def testUpdateOnStartIsLogged(self): + self.data_file.update_on_start = True + self.data_file.updated_on_start = False + with mock.patch.object(self.service, "update_data_file"): + self.service.register_data_file(self.data_file) + self.assertIn(("info", "Updating on start-up"), self.pipeline.logs) + + def testFileSystemWatcherIsLogged(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder, ignore_errors=True) + data_file = HttpDataFile(os.path.join(folder, "data.dat")) + with open(data_file.path, "w") as file: + file.write("x") + data_file.file_system_watcher = True + data_file.auto_update = True + self.hold_timers() + self.service.register_data_file(data_file) + self.assertIn( + ("info", "Creating file system watcher"), self.pipeline.logs) + + def testNoFileSystemWatcherWithoutAutomaticUpdates(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder, ignore_errors=True) + data_file = HttpDataFile(os.path.join(folder, "data.dat")) + with open(data_file.path, "w") as file: + file.write("x") + data_file.file_system_watcher = True + data_file.auto_update = False + self.hold_timers() + self.service.register_data_file(data_file) + self.assertNotIn( + ("info", "Creating file system watcher"), self.pipeline.logs) + self.assertEqual([], self.pending) + self.assertIsNone(getattr(data_file, "watcher_timer", None)) + + +# No message holds a licence key in full, as the Logging section of the data +# updates specification and the License keys section of logging.md say. +# A callback that raises stops the callbacks after it, as data-updates.md +# says, and its error is logged with .NET's message for any other error, +# with the error the callback raised attached. Polling carries on. +class CallbackTests(UpdateServiceTestCase): + + UNHANDLED = ("An unhandled error occurred while checking for automatic " + "updates for engine 'FakeEngine'") + + # The error the callback raised is attached, as the errors table asks + # and as .NET, Node.js and Go each write it. + UNHANDLED_WITH_CAUSE = UNHANDLED + ". Error detail: callback failed" + + def fail_first(self): + def fails(status, data_file): + raise RuntimeError("callback failed") + + self.service.on_complete_methods.insert(0, fails) + + def errors(self): + return [message for level, message in self.pipeline.logs + if level == "error"] + + def testACallbackErrorOnPollingIsLoggedAndStopsTheCallbacks(self): + self.fail_first() + self.data_file.auto_update = True + self.hold_timers() + self.get(response(304)) + self.service.async_update_data_file(self.data_file).join() + self.assertEqual([], self.statuses) + self.assertEqual(1, len(self.pending)) + self.assertEqual([self.UNHANDLED_WITH_CAUSE], self.errors()) + + def testACallbackErrorOnAProgrammaticCheckIsLoggedAndStopsTheCallbacks( + self): + self.fail_first() + self.data_file.auto_update = True + self.hold_timers() + self.get(response(304)) + self.service.update_data_file(self.data_file) + self.assertEqual([], self.statuses) + self.assertEqual(1, len(self.pending)) + self.assertEqual([self.UNHANDLED_WITH_CAUSE], self.errors()) + + def testACallbackErrorOnAnUpdateFromMemoryIsLoggedAndStopsTheCallbacks( + self): + self.fail_first() + self.service.update_from_memory(self.data_file, b"fresh") + self.assertEqual([], self.statuses) + self.assertEqual([self.UNHANDLED_WITH_CAUSE], self.errors()) + + def testAConnectionFailureKeepsPolling(self): + self.data_file.auto_update = True + self.hold_timers() + self.get(requests.ConnectionError("boom")) + self.assertFalse(self.service.update_data_file(self.data_file)) + self.assertEqual([UpdateStatus.AUTO_UPDATE_HTTPS_ERR], self.statuses) + self.assertEqual(1, len(self.pending)) + + +class LicenceKeyTests(UpdateServiceTestCase): + + def setUp(self): + super(LicenceKeyTests, self).setUp() + self.data_file.url = KEYED_URL + + def assertNoKey(self): + self.assertTrue(self.pipeline.logs) + for level, message in self.pipeline.logs: + self.assertNotIn(KEY[:8], message) + + def testSuccessfulUpdateShowsOnlyTheMaskedKey(self): + self.get(response(200)) + self.service.update_data_file(self.data_file) + self.assertNoKey() + self.assertIn( + ("info", + "Checking for update from '" + MASKED_URL + "' for Engine " + "'FakeEngine'"), + self.pipeline.logs) + self.assertIn( + ("info", + "Downloaded new data from '" + MASKED_URL + "' for Engine " + "'FakeEngine'"), + self.pipeline.logs) + + def testEachFailedAnswerShowsOnlyTheMaskedKey(self): + for status in (304, 403, 429, 500): + self.pipeline.logs.clear() + self.get(response(status)) + self.service.update_data_file(self.data_file) + self.assertNoKey() + self.assertTrue(any(MASKED_URL in message + for level, message in self.pipeline.logs), + status) + + def testFailedConnectionShowsOnlyTheMaskedKey(self): + # The message of a failed request names the URL, key included. + self.get(requests.ConnectionError("failed for url: " + KEYED_URL)) + self.service.update_data_file(self.data_file) + self.assertNoKey() + self.assertIn( + ("warning", + "An error occurred when connecting to " + MASKED_URL + + " in order to check for data file updates for FakeEngine. " + "Error detail: failed for url: " + MASKED_URL), + self.pipeline.logs) + + +# A missing MD5 header, content that is not compressed and content that +# cannot be read each end as the data updates specification says, and the +# checks carry on. +class ContentTests(UpdateServiceTestCase): + + def testMissingMd5HeaderIsWarnedAndTheUpdateCarriesOn(self): + self.data_file.verify_md5 = True + self.data_file.url = KEYED_URL + self.get(response(200)) + self.service.update_data_file(self.data_file) + self.assertEqual([b"new"], self.data_file.refreshed) + self.assertEqual([UpdateStatus.AUTO_UPDATE_SUCCESS], self.statuses) + self.assertIn( + ("warning", + "No MD5 hash included in data update response for '" + + MASKED_URL + "'. Unable to verify data integrity"), + self.pipeline.logs) + + def testCompressedContentIsDecompressed(self): + self.data_file.decompress = True + self.get(response(200, gzip.compress(b"new"))) + self.service.update_data_file(self.data_file) + self.assertEqual([b"new"], self.data_file.refreshed) + + def testContentThatIsNotCompressedIsUsedAsItIs(self): + self.data_file.decompress = True + self.get(response(200, b"plain")) + self.service.update_data_file(self.data_file) + self.assertEqual([b"plain"], self.data_file.refreshed) + self.assertEqual([UpdateStatus.AUTO_UPDATE_SUCCESS], self.statuses) + + def testCorruptCompressedContentIsWarnedAndCheckedAgainLater(self): + self.data_file.decompress = True + self.data_file.auto_update = True + self.hold_timers() + self.get(response(200, gzip.compress(b"new")[:12])) + self.assertFalse(self.service.update_data_file(self.data_file)) + self.assertEqual([], self.data_file.refreshed) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_ERR_READING_STREAM], self.statuses) + warnings = [message for level, message in self.pipeline.logs + if level == "warning"] + self.assertEqual(1, len(warnings), self.pipeline.logs) + self.assertTrue(warnings[0].startswith( + "An error occurred during the integrity check of new data file " + "for FakeEngine. Update will be attempted again later. Error " + "detail: "), warnings[0]) + self.assertEqual(1, len(self.pending)) + + def testFailedMd5CheckOnStartUpIsNotRepeatedAtOnce(self): + reply = response(200) + reply.headers = {"content-md5": "not the hash"} + reply.iter_content = lambda size: [b"new"] + self.data_file.verify_md5 = True + self.data_file.update_on_start = True + self.data_file.updated_on_start = False + self.hold_timers() + self.get(reply) + self.service.update_data_file(self.data_file) + self.assertEqual(1, len(self.sent)) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED], + self.statuses) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py index 5fd9c872..6cbe5f29 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py @@ -119,31 +119,71 @@ def run_next_check(self): check = self.pending.pop(0) check.function(*check.args) + def register_watched(self, data_file): + # The watcher runs only with automatic updates on. The update checks + # those would schedule are left out, so only the watcher's are + # pending. + data_file.auto_update = True + with mock.patch.object(self.service, "check_next_update"): + self.service.register_data_file(data_file) + # A refresh that fails is logged and reported, the data in use is kept, # and a later good file is picked up. def testFileSystemWatcherKeepsWatchingAfterFailedRefresh(self): data_file = StandInDataFile(self.path) - self.service.register_data_file(data_file) + self.register_watched(data_file) self.write("unusable") self.run_next_check() self.assertEqual("version 1", data_file.data) self.assertEqual( [UpdateStatus.AUTO_UPDATE_REFRESH_FAILED], self.statuses) - self.assertEqual(1, len(self.pipeline.logs)) - self.assertEqual("error", self.pipeline.logs[0][0]) - self.assertIn("The data file cannot be used.", self.pipeline.logs[0][1]) + errors = [log for log in self.pipeline.logs if log[0] == "error"] + self.assertEqual(1, len(errors)) + self.assertIn("The data file cannot be used.", errors[0][1]) self.assertEqual(1, len(self.pending)) self.write("version 2") self.run_next_check() self.assertEqual("version 2", data_file.data) + # The watcher's refresh takes the same route as the polling one, so it + # logs the attempt, uses the table's failure message and level, and + # reports success to the update complete callbacks. + def testFileSystemWatcherRefreshIsRoutedAsPollingIs(self): + data_file = StandInDataFile(self.path) + self.register_watched(data_file) + + self.write("version 2") + self.run_next_check() + + self.assertEqual("version 2", data_file.data) + self.assertEqual([UpdateStatus.AUTO_UPDATE_SUCCESS], self.statuses) + attempts = [log for log in self.pipeline.logs + if "Attempting to refresh Engine" in log[1]] + self.assertEqual(1, len(attempts)) + self.assertEqual("info", attempts[0][0]) + + # A refresh that fails from the watcher carries the message and level of + # the specification's errors table, as the polling route's does. + def testFileSystemWatcherFailedRefreshUsesTheTableMessage(self): + data_file = StandInDataFile(self.path) + self.register_watched(data_file) + + self.write("unusable") + self.run_next_check() + + errors = [log for log in self.pipeline.logs if log[0] == "error"] + self.assertEqual(1, len(errors)) + self.assertIn( + "An error occurred while applying a data file update to", + errors[0][1]) + # A file that is missing at a check, for example part way through being # replaced, is looked for again at the next check. def testFileSystemWatcherKeepsWatchingWhileFileIsMissing(self): data_file = StandInDataFile(self.path) - self.service.register_data_file(data_file) + self.register_watched(data_file) os.remove(self.path) self.run_next_check() @@ -160,7 +200,7 @@ def testFileSystemWatcherKeepsWatchingWhileFileIsMissing(self): # nothing is left holding the data file. def testUnregisterStopsFileSystemWatcher(self): data_file = StandInDataFile(self.path) - self.service.register_data_file(data_file) + self.register_watched(data_file) check = self.pending[0] self.service.unregister_data_file(data_file) diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_licence_key_masking.py b/pipeline/fiftyone_pipeline_engines/tests/test_licence_key_masking.py index 8f87c1a6..4f437f50 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_licence_key_masking.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_licence_key_masking.py @@ -133,16 +133,40 @@ def test_the_masked_url_is_in_the_message(self): self.assertTrue(any(MASKED_URL in message for level, message in logs), logs) - def test_an_error_raised_by_the_request_does_not_hold_the_key(self): + def test_a_failed_request_logs_only_the_masked_key(self): + # The update service logs a failed request rather than raising it, + # and the message of the request's error names the URL. def fail(*args, **kwargs): raise requests.ConnectionError("failed for url: " + URL) - service = DataFileUpdateService(RecordingPipeline()) + pipeline = RecordingPipeline() + service = DataFileUpdateService(pipeline) with mock.patch.object(requests, "get", fail): - with self.assertRaises(requests.RequestException) as caught: - service.update_data_file(StandInDataFile()) - self.assertNotIn(KEY, str(caught.exception)) - self.assertIn(MASKED_URL, str(caught.exception)) + service.update_data_file(StandInDataFile()) + self.assertTrue(pipeline.logs) + for level, message in pipeline.logs: + self.assertNotIn(KEY, message) + self.assertTrue(any(MASKED_URL in message + for level, message in pipeline.logs), + pipeline.logs) + + def test_a_callback_error_logs_only_the_masked_key(self): + # A callback's error is written into the log entry, and a callback + # that passes on a request error carries the update URL with it. + def fails(status, data_file): + raise requests.ConnectionError("failed for url: " + URL) + + pipeline = RecordingPipeline() + service = DataFileUpdateService(pipeline) + service.on_complete(fails) + with mock.patch.object(requests, "get", lambda *a, **k: answer(304)): + service.update_data_file(StandInDataFile()) + self.assertTrue(pipeline.logs) + for level, message in pipeline.logs: + self.assertNotIn(KEY, message) + self.assertTrue(any(MASKED_URL in message + for level, message in pipeline.logs), + pipeline.logs) if __name__ == "__main__": From 3c786b7364a6adb59e1cb0402b5e15aee14fced9 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 15:08:19 +0000 Subject: [PATCH 60/68] Give each web integration element its own switch, put Sequence first and give it Element Data (#70) * Give each web integration element its own switch and put Sequence first The Sequence, JSON Builder, JavaScript Builder and Set Headers elements each have a switch that defaults to on, and add_javascript_builder and use_setheader_properties work as before. The Sequence Element is the first element of the pipeline and also writes the session-id and sequence Element Data, as well as its Evidence. * Include the Sequence Element in the JSON, declare its properties and say the Lite file has few Properties The JSON Builder specification requires a top-level entry for each Element Data and excludes only Set Headers, the Cloud Request Engine and Usage Sharing, so the JSON Builder no longer skips the Sequence Element. The Sequence Element declares session-id and sequence as the Sequence Element specification lists them. The Device Detection example comments now say what the required examples specification says of the Lite data file. * Write an empty session id to the Sequence Element Data when the script cannot be given it The Sequence Element's session id now goes into the JSON that is written into the script, so an id the script cannot take is empty there, as it is in the script's own sessionId. * Expect the incremented sequence in the usage sharing record now that Sequence is first * Give the native engines evidence as text, as the Sequence Element now adds a number before them * Read the session id and the sequence independently in the Sequence Element A sequence with no session id now counts up with a new id. A session id that is not a non-empty string is absent, a sequence in text must be decimal digits, and the Evidence and the Element Data are written from the same values. The JSON written into the script escapes the characters that could end the script. * Check the escaped script end in the session id test * Take a sequence of 0 to 2147483646, as the specification says The Sequence Element took a sequence of 1 to 2147483647 and added one, so 2147483647 gave an output of 2147483648, which leaves a 32 bit integer. The specification sets the top at 2,147,483,646 so that adding one stays within one, which is the range 51degrees-node and 51degrees-java take. * Say the Lite data file has a small number of Properties in the last example too datafileupdate_console.py still said "limited capabilities", which is the wording 51degrees-node also replaced in its own update example. * Say plainly in the docstring what add_set_header_element does --- .../tests/test_cloud.py | 6 +- .../onpremise/datafileupdate_console.py | 3 +- .../onpremise/gettingstarted_console.py | 4 +- .../onpremise/match_metrics.py | 2 +- .../onpremise/metadata_console.py | 2 +- .../onpremise/offlineprocessing.py | 2 +- .../devicedetection_onpremise.py | 3 +- .../tests/test_automatic_data_file_updates.py | 22 +- .../tests/test_devicedetection.py | 18 +- .../javascriptbuilder.py | 8 +- .../src/fiftyone_pipeline_core/jsonbundler.py | 1 - .../fiftyone_pipeline_core/pipelinebuilder.py | 70 ++++-- .../fiftyone_pipeline_core/sequenceelement.py | 55 +++-- .../fiftyone_pipeline_core/tests/test_core.py | 3 +- .../tests/test_javascriptbuilder.py | 8 +- .../tests/test_javascriptbuilder_sequence.py | 7 +- .../tests/test_web_integration_elements.py | 223 ++++++++++++++++++ .../tests/test_usage.py | 4 +- 18 files changed, 369 insertions(+), 72 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_web_integration_elements.py diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py index 177a42ed..7d769b84 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py @@ -59,8 +59,8 @@ def test_pipeline_builder_cloud_engine_init(self): pipeline = DeviceDetectionCloudPipelineBuilder(resource_key = self.resource_key).build() - self.assertTrue(pipeline.flow_elements[0].datakey == "cloud") - self.assertTrue(pipeline.flow_elements[1].datakey == "device") + self.assertIn("cloud", pipeline.flow_elements_list) + self.assertIn("device", pipeline.flow_elements_list) def test_properties_cloud(self): """! @@ -69,7 +69,7 @@ def test_properties_cloud(self): pipeline = DeviceDetectionCloudPipelineBuilder(resource_key = self.resource_key).build() - properties = pipeline.flow_elements[1].get_properties() + properties = pipeline.get_element("device").get_properties() self.assertTrue(len(properties.keys()) > 0) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index 572c7a82..eeaa8492 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -52,7 +52,8 @@ # ``` # # ## Lite Data File -# Lite data files (free-to-use, limited capabilities, no license key required) are created roughly +# Lite data files (free-to-use, with a small number of Properties, no +# license key required) are created roughly # once a month and cannot be updated using auto-update, they may be downloaded with the # scripts in the [data](https://github.com/51Degrees/data) repository on GitHub. # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index 4742eed1..8e44043d 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -175,8 +175,8 @@ def main(argv): # a command line parameter. # # Note that the Lite data file is only used for illustration, and has - # limited accuracy and capabilities. The example shows the properties - # only the Enterprise data file has as needing it. Contact us for a copy + # a small number of Properties. The example shows the properties only + # the Enterprise data file has as needing it. Contact us for a copy # of the Enterprise data file: # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py index 8f364ac8..2d66f811 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py @@ -190,7 +190,7 @@ def main(argv): # a command line parameter. # # Note that the Lite data file is only used for illustration, and has - # limited accuracy and capabilities. + # a small number of Properties. # Contact us for a copy of the Enterprise data file: # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py index e3141e06..edae4a0e 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py @@ -127,7 +127,7 @@ def main(argv): # a command line parameter. # # Note that the Lite data file is only used for illustration, and has - # limited accuracy and capabilities. + # a small number of Properties. # Contact us for a copy of the Enterprise data file: # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py index c4b437f5..9bbe7063 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py @@ -223,7 +223,7 @@ def main(argv): # a command line parameter. # # Note that the Lite data file is only used for illustration, and has - # limited accuracy and capabilities. + # a small number of Properties. # Contact us for a copy of the Enterprise data file: # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index ef2be582..0fd31aac 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -477,8 +477,9 @@ def process_internal(self, flow_data): evidence = EvidenceDeviceDetectionSwig() + # The native evidence takes text, and the Sequence Element adds a number. for key, value in flow_data.evidence.get_all().items(): - evidence.__setitem__(key, value) + evidence.__setitem__(key, str(value)) result = self.engine.process(evidence) diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py index e1ac9944..47cfb443 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py @@ -73,7 +73,7 @@ def test_on_premise_data_update_url(self): with pipeline.create_flowdata() as flowdata: flowdata.process() - flow_element_data_file = pipeline.flow_elements[0].data_file + flow_element_data_file = pipeline.get_element("device").data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) self.assertFalse(flow_element_data_file.verify_md5) self._assertSignedUrl(flow_element_data_file.get_update_url()) @@ -92,7 +92,7 @@ def test_on_premise_data_old_property_data_file_update_base_url(self): with pipeline.create_flowdata() as flowdata: flowdata.process() - flow_element_data_file = pipeline.flow_elements[0].data_file + flow_element_data_file = pipeline.get_element("device").data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) self.assertFalse(flow_element_data_file.verify_md5) self._assertSignedUrl(flow_element_data_file.get_update_url()) @@ -114,7 +114,7 @@ def test_on_premise_data_update_url_config(self): with pipeline.create_flowdata() as flowdata: flowdata.process() - flow_element_data_file = pipeline.flow_elements[0].data_file + flow_element_data_file = pipeline.get_element("device").data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) self.assertFalse(flow_element_data_file.verify_md5) self._assertSignedUrl(flow_element_data_file.get_update_url()) @@ -136,7 +136,7 @@ def test_on_premise_data_config_old_property_data_file_update_base_url(self): with pipeline.create_flowdata() as flowdata: flowdata.process() - flow_element_data_file = pipeline.flow_elements[0].data_file + flow_element_data_file = pipeline.get_element("device").data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) self.assertFalse(flow_element_data_file.verify_md5) self._assertSignedUrl(flow_element_data_file.get_update_url()) @@ -155,7 +155,7 @@ def test_on_premise_data_update_verify_md5(self): with pipeline.create_flowdata() as flowdata: flowdata.process() - flow_element_data_file = pipeline.flow_elements[0].data_file + flow_element_data_file = pipeline.get_element("device").data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) self.assertTrue(flow_element_data_file.verify_md5) self._assertSignedUrl(flow_element_data_file.get_update_url()) @@ -174,7 +174,7 @@ def test_on_premise_data_update_verify_md5_old_property(self): with pipeline.create_flowdata() as flowdata: flowdata.process() - flow_element_data_file = pipeline.flow_elements[0].data_file + flow_element_data_file = pipeline.get_element("device").data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) self.assertTrue(flow_element_data_file.verify_md5) self._assertSignedUrl(flow_element_data_file.get_update_url()) @@ -196,7 +196,7 @@ def test_on_premise_data_update_verify_md5_config(self): with pipeline.create_flowdata() as flowdata: flowdata.process() - flow_element_data_file = pipeline.flow_elements[0].data_file + flow_element_data_file = pipeline.get_element("device").data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) self.assertTrue(flow_element_data_file.verify_md5) self._assertSignedUrl(flow_element_data_file.get_update_url()) @@ -218,7 +218,7 @@ def test_on_premise_data_update_verify_md5_config_old_property(self): with pipeline.create_flowdata() as flowdata: flowdata.process() - flow_element_data_file = pipeline.flow_elements[0].data_file + flow_element_data_file = pipeline.get_element("device").data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) self.assertTrue(flow_element_data_file.verify_md5) self._assertSignedUrl(flow_element_data_file.get_update_url()) @@ -238,7 +238,7 @@ def test_on_premise_data_update_use_url_formatter(self): with pipeline.create_flowdata() as flowdata: flowdata.process() - flow_element_data_file = pipeline.flow_elements[0].data_file + flow_element_data_file = pipeline.get_element("device").data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) self._assertNotSignedUrl(flow_element_data_file.get_update_url()) @@ -260,7 +260,7 @@ def test_on_premise_data_update_use_url_formatter_config(self): with pipeline.create_flowdata() as flowdata: flowdata.process() - flow_element_data_file = pipeline.flow_elements[0].data_file + flow_element_data_file = pipeline.get_element("device").data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) self._assertNotSignedUrl(flow_element_data_file.get_update_url()) @@ -278,7 +278,7 @@ def test_on_premise_data_update_status_rate_limit(self): with pipeline.create_flowdata() as flowdata: flowdata.process() - flow_element_data_file = pipeline.flow_elements[0].data_file + flow_element_data_file = pipeline.get_element("device").data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) self.assertTrue(flow_element_data_file.verify_md5) self._assertSignedUrl(flow_element_data_file.get_update_url()) diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py index 11ed21f6..af63494a 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py @@ -63,7 +63,7 @@ def test_pipeline_builder_shareusage_init(self): data_file_path=data_file, licence_keys="").build() self.addCleanup(pipeline.close) - self.assertTrue(pipeline.flow_elements[0].datakey == "shareusage") + self.assertIn("shareusage", pipeline.flow_elements_list) def test_on_premise_engine_datafile(self): """! @@ -75,7 +75,7 @@ def test_on_premise_engine_datafile(self): self.addCleanup(pipeline.close) self.assertTrue(isinstance( - pipeline.flow_elements[0].data_file, DeviceDetectionDataFile)) + pipeline.get_element("device").data_file, DeviceDetectionDataFile)) def test_properties_onpremise(self): """! @@ -87,7 +87,7 @@ def test_properties_onpremise(self): data_file_path=data_file, usage_sharing=False, licence_keys="").build() self.addCleanup(pipeline.close) - properties = pipeline.flow_elements[0].get_properties() + properties = pipeline.get_element("device").get_properties() self.assertTrue(len(properties.keys()) > 0) @@ -100,7 +100,7 @@ def test_evidencekey_filter_onpremise(self): data_file_path=data_file, usage_sharing=False, licence_keys="").build() self.addCleanup(pipeline.close) - evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( + evidence_key_filter = pipeline.get_element("device").get_evidence_key_filter( ) self.assertFalse(evidence_key_filter.filter_evidence_key("test.test")) @@ -310,7 +310,7 @@ def test_evidencekey_filter_contains_user_agent(self): data_file_path=data_file, usage_sharing=False, licence_keys="").build() self.addCleanup(pipeline.close) - evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( + evidence_key_filter = pipeline.get_element("device").get_evidence_key_filter( ) self.assertTrue( @@ -325,7 +325,7 @@ def test_evidencekey_filter_contains_device_stock_ua(self): data_file_path=data_file, usage_sharing=False, licence_keys="").build() self.addCleanup(pipeline.close) - evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( + evidence_key_filter = pipeline.get_element("device").get_evidence_key_filter( ) self.assertTrue(evidence_key_filter.filter_evidence_key( @@ -340,7 +340,7 @@ def test_evidencekey_filter_contains_query_params(self): data_file_path=data_file, usage_sharing=False, licence_keys="").build() self.addCleanup(pipeline.close) - evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( + evidence_key_filter = pipeline.get_element("device").get_evidence_key_filter( ) self.assertTrue(evidence_key_filter.filter_evidence_key( @@ -357,7 +357,7 @@ def test_evidencekey_filter_case_insensitive_keys(self): data_file_path=data_file, usage_sharing=False, licence_keys="").build() self.addCleanup(pipeline.close) - evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( + evidence_key_filter = pipeline.get_element("device").get_evidence_key_filter( ) self.assertTrue( @@ -378,7 +378,7 @@ def test_evidencekey_filter_overrides(self): data_file_path=data_file, usage_sharing=False, licence_keys="").build() self.addCleanup(pipeline.close) - evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( + evidence_key_filter = pipeline.get_element("device").get_evidence_key_filter( ) self.assertTrue(evidence_key_filter.filter_evidence_key( diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py index 86a84fc4..7f7bc795 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py @@ -368,7 +368,13 @@ def process_internal(self, flowdata): for key, value in self.settings.items(): variables[key] = value - variables["_jsonObject"] = json.dumps(flowdata.jsonbundler.json) + # The JSON is written into a script element, so the characters that + # could end the element or the script are written as escapes. + variables["_jsonObject"] = ( + json.dumps(flowdata.jsonbundler.json) + .replace("<", "\\u003c") + .replace(">", "\\u003e") + .replace("&", "\\u0026")) # Generate URL and autoUpdate params diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index 6ddf50a3..408a30d0 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -124,7 +124,6 @@ def process_internal(self, flowdata): for flow_element in flowdata.pipeline.flow_elements: if (flow_element.datakey == "jsonbundler" or - flow_element.datakey == "sequence" or flow_element.datakey == "javascriptbuilder" or flow_element.datakey == "set-headers"): continue diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py index c3fa52e0..e208b686 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py @@ -42,12 +42,23 @@ def __init__(self, settings= {}): Pipeline Builder constructor. @type settings: dictionary @param settings : settings for the pipeline builder including: - `add_javascript_builder (Bool)` - Whether to add the JavaScriptBuilder to the pipeline - (default true) + `add_javascript_builder (Bool)` - Whether to add the JavaScriptBuilder, with the + JSONBundler and SequenceElement it works with, to the pipeline (default true). + Set to False to add none of the three. + `add_sequence_element (Bool)` - Whether to add the SequenceElement, as the first + element of the pipeline (default true) + `add_json_builder_element (Bool)` - Whether to add the JSONBundler (default true). + The JavaScriptBuilder reads its output, so a pipeline that keeps the + JavaScriptBuilder needs a JSONBundler, added here or by the caller. + `add_javascript_builder_element (Bool)` - Whether to add the JavaScriptBuilder + alone (default true) `javascript_builder_settings (dict)` - Settings for the JavaScriptBuilder engine @rtype: PipelineBuilder `use_setheader_properties (Bool)` - Whether to add the SetHeaderElement to the pipeline (default true) + `add_set_header_element (Bool)` - Whether to add the SetHeaderElement alone + (default true). It is added only when this and use_setheader_properties + are both true. `suppress_process_exceptions (Bool)` - If False (the default) the first error thrown while processing is re-raised from flow_data.process(); if True, errors are stored on flow_data.errors and process() returns normally. Recommended True for web apps so a @@ -67,6 +78,14 @@ def __init__(self, settings= {}): else: self.add_javaScriptbuilder = True + self.add_sequence_element = settings.get("add_sequence_element", True) + self.add_json_builder_element = settings.get( + "add_json_builder_element", True) + self.add_javascript_builder_element = settings.get( + "add_javascript_builder_element", True) + self.add_set_header_element = settings.get( + "add_set_header_element", True) + if "javascript_builder_settings" in settings: self.javascriptbuilder_settings = settings["javascript_builder_settings"] @@ -90,10 +109,25 @@ def __init__(self, settings= {}): else: self.auto_close_elements = True + def get_sequence_elements(self): + """ + The SequenceElement, to be the first element of the pipeline, unless + add_javascript_builder or add_sequence_element is false + @rtype: list + @return: Returns a list of FlowElements + """ + + if self.add_javaScriptbuilder and self.add_sequence_element: + return [SequenceElement()] + + return [] + def get_javascript_elements(self): """ - Adds the JavaScriptBuilder, JSONBundler and SequenceElement to the pipeline if - If add_javascript_builder is set to true (the default) + The JSONBundler and the JavaScriptBuilder, to be added after the other + elements. Neither is added if add_javascript_builder is false, and + each has its own switch (add_json_builder_element and + add_javascript_builder_element). @rtype: list @return: Returns a list of FlowElements """ @@ -102,27 +136,28 @@ def get_javascript_elements(self): if (self.add_javaScriptbuilder): - flow_elements.append(SequenceElement()) - flow_elements.append(JSONBundlerElement()) - - if (hasattr(self, "javascriptbuilder_settings")): - flow_elements.append(JavascriptBuilderElement(self.javascriptbuilder_settings)) - else: - flow_elements.append(JavascriptBuilderElement()) + if self.add_json_builder_element: + flow_elements.append(JSONBundlerElement()) + + if self.add_javascript_builder_element: + if (hasattr(self, "javascriptbuilder_settings")): + flow_elements.append(JavascriptBuilderElement(self.javascriptbuilder_settings)) + else: + flow_elements.append(JavascriptBuilderElement()) return flow_elements def get_setheader_elements(self): """ Adds the SetHeaderElement to the pipeline if - If use_setheader_properties is set to true (the default) + If use_setheader_properties and add_set_header_element are true (the default) @rtype: list @return: Returns a list of FlowElements """ flow_elements = [] - if (self.use_setheader_properties): + if self.use_setheader_properties and self.add_set_header_element: flow_elements.append(SetHeaderElement()) return flow_elements @@ -153,10 +188,13 @@ def build(self): """ - self.flow_elements.extend(self.get_javascript_elements()) - self.flow_elements.extend(self.get_setheader_elements()) + flow_elements = ( + self.get_sequence_elements() + + self.flow_elements + + self.get_javascript_elements() + + self.get_setheader_elements()) - return Pipeline(self.flow_elements, logger=self.logger, suppress_process_exceptions=self.suppress_process_exceptions, data_file_update_service=self.data_file_update_service, auto_close_elements=self.auto_close_elements) + return Pipeline(flow_elements, logger=self.logger, suppress_process_exceptions=self.suppress_process_exceptions, data_file_update_service=self.data_file_update_service, auto_close_elements=self.auto_close_elements) def add_logger(self, logger): """ diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py index 2aaf8283..5092cd84 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py @@ -24,8 +24,17 @@ from .flowelement import FlowElement from .evidence_keyfilter import EvidenceKeyFilter from .javascriptbuilder import parse_sequence +from .elementdata_dictionary import ElementDataDictionary +import re import uuid +# Text that is a sequence, in ASCII decimal digits and nothing else. +DIGITS_PATTERN = re.compile(r"[0-9]+") + +# The largest sequence the element takes, so that adding one stays within a +# 32 bit signed integer. +MAX_INPUT_SEQUENCE = 2147483646 + class SequenceElementEvidenceKeyFilter(EvidenceKeyFilter): # The session id and the sequence. The result changes as the sequence @@ -57,6 +66,9 @@ def __init__(self): super(SequenceElement, self).__init__() self.datakey = "sequence" + self.properties = { + "session-id": {"type": "string"}, + "sequence": {"type": "int"}} self.exclude_from_messages = True @@ -71,30 +83,35 @@ def get_evidence_key_filter(self): def process_internal(self, flowdata): """! - The SequenceElement stores session data for requests for JavaScript + The SequenceElement stores session data for requests for JavaScript, + in its Element Data (session-id and sequence) and in Evidence @type flowdata: FlowData @param flowdata: The FlowData """ - if flowdata.evidence.get("query.session-id"): - - # Get current sequence number. A value that is absent or is not - # a positive 32 bit integer is treated as no sequence at all, so - # the request is sequence 1 rather than failing. The JavaScript - # builder renders any value it cannot use as 1 in the same way. - - sequence = parse_sequence(flowdata.evidence.get("query.sequence")) + # The session id and the sequence are read independently. A session + # id that is not a non-empty string is absent, and gets a new id. A + # sequence that is not a whole number from 0 to MAX_INPUT_SEQUENCE, + # written in decimal digits with no sign or spaces if it is text, is + # absent and gives 1, which is also what 0 gives. parse_sequence + # answers None for 0 and for anything it cannot read at all. + session_id = flowdata.evidence.get("query.session-id") + if not isinstance(session_id, str) or session_id == "": + session_id = str(uuid.uuid4()) + + sequence = flowdata.evidence.get("query.sequence") + if isinstance(sequence, str) and not DIGITS_PATTERN.fullmatch(sequence): + sequence = None + sequence = parse_sequence(sequence) + if sequence is None or sequence > MAX_INPUT_SEQUENCE: + sequence = 1 + else: + sequence = sequence + 1 - flowdata.evidence.add( - "query.sequence", - 1 if sequence is None else sequence + 1) + flowdata.evidence.add("query.session-id", session_id) + flowdata.evidence.add("query.sequence", sequence) - else: - flowdata.evidence.add( - "query.session-id", - str(uuid.uuid4()) - ) - - flowdata.evidence.add("query.sequence", 1) + flowdata.set_element_data(ElementDataDictionary( + self, {"session-id": session_id, "sequence": sequence})) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_core.py b/pipeline/fiftyone_pipeline_core/tests/test_core.py index 21407b33..9f0d88b9 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_core.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_core.py @@ -86,8 +86,9 @@ def testGet(self): def testGetWhere(self): testPipeline = TestPipeline() + # The example element's integer and the Sequence Element's sequence. getValue = len(testPipeline.flowdata.get_where("type", "int")) - self.assertTrue(getValue == 1) + self.assertTrue(getValue == 2) def testGetFromElement(self): diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder.py index a2d80b01..03f4e68d 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder.py @@ -229,7 +229,13 @@ def testJSONBundler(self): } } - self.assertEqual(FlowData.jsonbundler.json, expected) + # The Sequence Element has an entry too. Its session id is new on + # every request. + actual = dict(FlowData.jsonbundler.json) + sequence = actual.pop('sequence') + self.assertTrue(sequence['session-id']) + self.assertEqual(1, sequence['sequence']) + self.assertEqual(actual, expected) def testJavaScriptBuilder_Minify(self): diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py index db057848..cc7d03e8 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py @@ -144,7 +144,9 @@ def test_longest_session_id_is_rendered(self): @parameterized.expand([ [name + "_" + ("with" if element else "without") - + "_sequence_element", value, check_text, element] + + "_sequence_element", value, + # The JSON of the Sequence Element's data holds the long id as it is. + check_text and not (element and name == "too_long"), element] for name, value, check_text in [ ["quote", 'a"b', True], ["backslash", "a\\b", True], @@ -164,7 +166,8 @@ def test_unsafe_session_id_is_rendered_empty( one that is not 1 to 64 ASCII letters, digits and hyphens is rendered as an empty string, as the pipeline specification says. The SequenceElement keeps a session id it is given, so the check is the - builder's in both pipelines.""" + builder's in both pipelines. The JSON in the script cannot be ended + early by the id either.""" script = _render( {"query.session-id": value}, diff --git a/pipeline/fiftyone_pipeline_core/tests/test_web_integration_elements.py b/pipeline/fiftyone_pipeline_core/tests/test_web_integration_elements.py new file mode 100644 index 00000000..b7e049ac --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_web_integration_elements.py @@ -0,0 +1,223 @@ + +""" +The elements the web integration adds (Sequence, JSON Builder, JavaScript +Builder and Set Headers) each have a switch that defaults to on, and the +Sequence Element comes first, as the web integration specification says. +""" + +import unittest + +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + + +class UserElement(FlowElement): + + def __init__(self): + super(UserElement, self).__init__() + self.datakey = "user" + + def process_internal(self, flowdata): + pass + + +def datakeys(settings=None): + builder = PipelineBuilder(settings or {}) + builder.add(UserElement()) + pipeline = builder.build() + return [element.datakey for element in pipeline.flow_elements] + + +class WebIntegrationElementSwitchTests(unittest.TestCase): + + def testAllElementsAreAddedByDefaultWithSequenceFirst(self): + self.assertEqual( + ["sequence", "user", "jsonbundler", "javascriptbuilder", + "set-headers"], + datakeys()) + + def testSequenceSwitch(self): + self.assertEqual( + ["user", "jsonbundler", "javascriptbuilder", "set-headers"], + datakeys({"add_sequence_element": False})) + + def testJsonBuilderSwitch(self): + self.assertEqual( + ["sequence", "user", "javascriptbuilder", "set-headers"], + datakeys({"add_json_builder_element": False})) + + def testJavaScriptBuilderSwitch(self): + self.assertEqual( + ["sequence", "user", "jsonbundler", "set-headers"], + datakeys({"add_javascript_builder_element": False})) + + def testSetHeadersSwitch(self): + self.assertEqual( + ["sequence", "user", "jsonbundler", "javascriptbuilder"], + datakeys({"add_set_header_element": False})) + + def testExistingJavaScriptSwitchStillDropsThreeElements(self): + self.assertEqual( + ["user", "set-headers"], + datakeys({"add_javascript_builder": False})) + + def testExistingSetHeaderSwitchStillDropsSetHeaders(self): + self.assertEqual( + ["sequence", "user", "jsonbundler", "javascriptbuilder"], + datakeys({"use_setheader_properties": False})) + + def testEverySwitchOff(self): + self.assertEqual( + ["user"], + datakeys({ + "add_sequence_element": False, + "add_json_builder_element": False, + "add_javascript_builder_element": False, + "add_set_header_element": False})) + + +# The Sequence Element writes Element Data as well as Evidence. +class SequenceElementDataTests(unittest.TestCase): + + def process(self, evidence): + pipeline = PipelineBuilder({}).build() + flowdata = pipeline.create_flowdata() + for key, value in evidence.items(): + flowdata.evidence.add(key, value) + flowdata.process() + return flowdata + + def testFirstRequestHasNewSessionAndSequenceOne(self): + flowdata = self.process({}) + session_id = flowdata.get("sequence").get("session-id") + self.assertTrue(session_id) + self.assertEqual(1, flowdata.get("sequence").get("sequence")) + self.assertEqual(session_id, flowdata.evidence.get("query.session-id")) + self.assertEqual(1, flowdata.evidence.get("query.sequence")) + + def testCallbackKeepsSessionAndAddsOne(self): + flowdata = self.process( + {"query.session-id": "abc-123", "query.sequence": "3"}) + self.assertEqual("abc-123", flowdata.get("sequence").get("session-id")) + self.assertEqual(4, flowdata.get("sequence").get("sequence")) + self.assertEqual("abc-123", flowdata.evidence.get("query.session-id")) + self.assertEqual(4, flowdata.evidence.get("query.sequence")) + + # The JSON Builder specification requires a top-level entry for each + # Element Data, excluding only Set Headers, the Cloud Request Engine and + # Usage Sharing. + def testSequenceHasAnEntryInTheJsonSentToBrowsers(self): + flowdata = self.process({}) + entry = flowdata.jsonbundler.json["sequence"] + self.assertEqual( + flowdata.get("sequence").get("session-id"), entry["session-id"]) + self.assertEqual(1, entry["sequence"]) + + # The JSON is written into the script, so a session id from the query + # string must not end the script. + def testSessionIdIsKeptButCannotEndTheScript(self): + flowdata = self.process({"query.session-id": ""}) + self.assertEqual("", flowdata.get("sequence").get("session-id")) + self.assertEqual( + "", flowdata.jsonbundler.json["sequence"]["session-id"]) + self.assertNotIn("", flowdata.javascriptbuilder.javascript) + self.assertIn( + "\\u003c/script\\u003e", flowdata.javascriptbuilder.javascript) + + def testSequencePropertiesAreDescribed(self): + element = PipelineBuilder({}).build().flow_elements[0] + self.assertEqual("sequence", element.datakey) + self.assertEqual( + {"session-id": {"type": "string"}, "sequence": {"type": "int"}}, + element.get_properties()) + + +# The session id and the sequence are read independently, as the Process +# section of the Sequence Element specification says, and the Evidence and the +# Element Data are written from the same values. +class SequenceElementProcessTests(unittest.TestCase): + + def process(self, evidence): + pipeline = PipelineBuilder({}).build() + flowdata = pipeline.create_flowdata() + for key, value in evidence.items(): + flowdata.evidence.add(key, value) + flowdata.process() + data = flowdata.get("sequence") + session_id = data.get("session-id") + sequence = data.get("sequence") + self.assertEqual(session_id, flowdata.evidence.get("query.session-id")) + self.assertEqual(sequence, flowdata.evidence.get("query.sequence")) + return session_id, sequence + + def testSequenceWithNoSessionIdCountsUpWithANewId(self): + session_id, sequence = self.process({"query.sequence": "3"}) + self.assertTrue(session_id) + self.assertEqual(4, sequence) + + def testSessionIdWithNoSequenceGivesOne(self): + self.assertEqual(("abc", 1), self.process({"query.session-id": "abc"})) + + def testIntegerSequenceCountsUp(self): + self.assertEqual( + ("abc", 8), + self.process({"query.session-id": "abc", "query.sequence": 7})) + + def testSessionIdThatIsNotANonEmptyStringIsAbsent(self): + for value in (123, "", None, ["abc"]): + session_id, sequence = self.process( + {"query.session-id": value, "query.sequence": "2"}) + self.assertIsInstance(session_id, str) + self.assertTrue(session_id) + self.assertEqual(3, sequence) + + def testSessionIdOfZeroIsKept(self): + self.assertEqual( + ("0", 3), + self.process({"query.session-id": "0", "query.sequence": "2"})) + + def testSequenceThatIsNotWholeDigitsIsAbsent(self): + for value in ("abc", "", " 5 ", "+5", "-5", "5.0", True, "Ù£", None): + self.assertEqual( + ("abc", 1), + self.process( + {"query.session-id": "abc", "query.sequence": value}), + repr(value)) + + def testLeadingZerosAreAllowed(self): + self.assertEqual( + ("abc", 6), + self.process({"query.session-id": "abc", + "query.sequence": "00005"})) + + def testSequenceOfZeroGivesOne(self): + # "A sequence of 0 gives an output of 1, as an absent one does." + for value in (0, "0", "000"): + self.assertEqual( + ("abc", 1), + self.process( + {"query.session-id": "abc", "query.sequence": value}), + repr(value)) + + def testLargestSequenceCountsUp(self): + for value in (2147483646, "2147483646"): + self.assertEqual( + ("abc", 2147483647), + self.process( + {"query.session-id": "abc", "query.sequence": value}), + repr(value)) + + def testSequenceAboveTheLargestIsAbsent(self): + # Above 2,147,483,646 adding one would leave a 32 bit integer, so the + # value is not one the element can use and the output is 1. + for value in (2147483647, "2147483647", 2 ** 31, "2147483648", + 10 ** 400, "1" + "0" * 400): + self.assertEqual( + ("abc", 1), + self.process( + {"query.session-id": "abc", "query.sequence": value}), + repr(value)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index c56eee51..1a3fe94d 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -705,7 +705,9 @@ def test_session_and_sequence(self): self.waitForUsageThreads() self.assertEqual(len(app.received), 1) self.assertIn("1", app.received[0]) - self.assertIn("1", app.received[0]) + # The Sequence Element is the first element, so it has added one to the + # sequence by the time usage sharing reads it. + self.assertIn("2", app.received[0]) def test_invalid_char(self): """! From 612752ef1f74efe49da0887cac08f9140fe90f49 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 15:09:42 +0000 Subject: [PATCH 61/68] Set the client-side cache headers, with a weak ETag that is the same in every process and none after errors (#59) * Test the cache headers of the client-side endpoints * Set the client-side cache headers, with an ETag that is the same in every process and none after errors * Test that a POST with a matching ETag gets 412, and a GET or HEAD 304 * Answer a POST whose If-None-Match matches with 412, as RFC 9110 section 13.1.2 asks * Test that the web request value added for usage sharing is not in the ETag * Leave the web request value out of the ETag * Test that the client-side ETag is weak and If-None-Match is read as RFC 9110 says * Send the client-side ETag as a weak entity tag, and match If-None-Match only when it is "*" or a list of entity tags * Test that the client-side max age can be set * Let the caller set the client-side max age, which the specification says should be configurable * Read the session id from the minified script in the ETag test * Name every header in the evidence key filter in Vary, as the other languages do The Vary header named only the headers the request carried, so a cache would not re-fetch when a header the pipeline accepts appeared for the first time. The web integration specification's cache header detail says Vary carries all headers in the Pipeline Evidence key filter, and 51degrees-node, 51degrees-php and 51degrees-go all build it from each element's filter list. _vary now reads each element's evidence key filter list, and asks a filter that cannot list its keys about the request's own headers one by one, which is what PHP does. A key that is not a field name is left out, and the revalidation headers stay out as before. Three tests cover it. * Name Host in the Vary header expectations, which the JavaScript Builder adds The three new Vary tests expected only the headers their own element accepts. Run 43175480017 showed Host beside them, which is right, because the JavaScript Builder's evidence key filter accepts header.host to build the callback URL the client script posts to, and its filter is not a list so the request's own headers are asked of it one by one. * Test that Content-Length is the length of what was sent --- .../cloud/gettingstarted_web/app.py | 47 +- .../onpremise/gettingstarted_web/app.py | 49 +- location/examples/cloud/web.py | 21 +- .../cloud/mixed/gettingstarted_web/app.py | 21 +- .../tests/test_cloud_mixed_web_offline.py | 109 ++++ ...lient_side_evidence_custom_flow_element.py | 23 +- .../src/fiftyone_pipeline_core/web.py | 231 +++++++- .../tests/test_web_client_side.py | 492 ++++++++++++++++++ 8 files changed, 865 insertions(+), 128 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_web_client_side.py diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py index 8260065b..93f448b3 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py @@ -79,8 +79,8 @@ from flask.helpers import make_response from fiftyone_devicedetection_cloud.devicedetection_cloud_pipelinebuilder import DeviceDetectionCloudPipelineBuilder from fiftyone_pipeline_core.logger import Logger -from fiftyone_pipeline_core.web import webevidence, set_response_header -import json +from fiftyone_pipeline_core.web import ( + javascript_response, json_response, webevidence, set_response_header) import os import sys @@ -123,57 +123,28 @@ def run(self): GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5000))) # First we make a JSON route that will be called from the client side and will return - # a JSON encoded property database using any additional evidence provided by the client + # a JSON encoded property database using any additional evidence provided by the client. + # json_response processes the request and sets the cache headers, so a browser keeps a + # good answer for 30 minutes and never keeps one made while processing failed. @staticmethod @app.route('/json', methods=['POST']) def jsonroute(): - # Create the flowdata object for the JSON route - flowdata = GettingStartedWeb.pipeline.create_flowdata() - - # Add any information from the request (headers, cookies and additional - # client side provided information) - - flowdata.evidence.add_from_dict(webevidence(request)) - - # Process the flowdata - - flowdata.process() - - # Return the JSON from the JSONBundler engine - - return json.dumps(flowdata.jsonbundler.json) + return json_response(GettingStartedWeb.pipeline, request) # Next we serve the client-side JavaScript from its own route. The other # Pipeline APIs ship a web integration that intercepts '/51Degrees.core.js' # for this, so the page can reference the script by that name. There is no # such integration for Flask, so the example wires up the route itself. + # Query parameters are evidence, so the per-request 'fod-js-enable-cookies' + # override is picked up and applied by the JavaScriptBuilder engine. @staticmethod @app.route('/51Degrees.core.js') def core_js(): - # Create the flowdata object for the JavaScript route - flowdata = GettingStartedWeb.pipeline.create_flowdata() - - # Add any information from the request (headers, cookies and additional - # client side provided information). Query parameters are included, so - # the per-request 'fod-js-enable-cookies' override is picked up here and - # applied by the JavaScriptBuilder engine. - - flowdata.evidence.add_from_dict(webevidence(request)) - - # Process the flowdata - - flowdata.process() - - # Return the JavaScript from the JavaScriptBuilder engine - - response = make_response(flowdata.javascriptbuilder.javascript) - response.headers["Content-Type"] = "application/x-javascript" - - return response + return javascript_response(GettingStartedWeb.pipeline, request) # In the main route we dynamically update the screen's device property display # using the above JSON route diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index fe88d9af..080b512f 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -82,7 +82,8 @@ from flask import Flask, request, render_template from fiftyone_pipeline_core.logger import Logger from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder -from fiftyone_pipeline_core.web import webevidence, set_response_header +from fiftyone_pipeline_core.web import ( + javascript_response, json_response, webevidence, set_response_header) import json class GettingStartedWeb(): @@ -108,60 +109,28 @@ def run(self): GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5001))) # First we make a JSON route that will be called from the client side and will return - # a JSON encoded property database using any additional evidence provided by the client + # a JSON encoded property database using any additional evidence provided by the client. + # json_response processes the request and sets the cache headers, so a browser keeps a + # good answer for 30 minutes and never keeps one made while processing failed. @staticmethod @app.route('/json', methods=['POST']) def jsonroute(): - # Create the flowdata object for the JSON route. It is closed when the - # route returns, which releases the results of the detection straight - # away rather than when the garbage collector next runs. - with GettingStartedWeb.pipeline.create_flowdata() as flowdata: - - # Add any information from the request (headers, cookies and additional - # client side provided information) - - flowdata.evidence.add_from_dict(webevidence(request)) - - # Process the flowdata - - flowdata.process() - - # Return the JSON from the JSONBundler engine - - return json.dumps(flowdata.jsonbundler.json) + return json_response(GettingStartedWeb.pipeline, request) # Next we serve the client-side JavaScript from its own route. The other # Pipeline APIs ship a web integration that intercepts '/51Degrees.core.js' # for this, so the page can reference the script by that name. There is no # such integration for Flask, so the example wires up the route itself. + # Query parameters are evidence, so the per-request 'fod-js-enable-cookies' + # override is picked up and applied by the JavaScriptBuilder engine. @staticmethod @app.route('/51Degrees.core.js') def core_js(): - # Create the flowdata object for the JavaScript route. It is closed - # when the route returns. - with GettingStartedWeb.pipeline.create_flowdata() as flowdata: - - # Add any information from the request (headers, cookies and additional - # client side provided information). Query parameters are included, so - # the per-request 'fod-js-enable-cookies' override is picked up here and - # applied by the JavaScriptBuilder engine. - - flowdata.evidence.add_from_dict(webevidence(request)) - - # Process the flowdata - - flowdata.process() - - # Return the JavaScript from the JavaScriptBuilder engine - - response = make_response(flowdata.javascriptbuilder.javascript) - response.headers["Content-Type"] = "application/x-javascript" - - return response + return javascript_response(GettingStartedWeb.pipeline, request) # In the main route we dynamically update the screen's device property display # using the above JSON route diff --git a/location/examples/cloud/web.py b/location/examples/cloud/web.py index 759b821d..a5db8558 100644 --- a/location/examples/cloud/web.py +++ b/location/examples/cloud/web.py @@ -32,8 +32,7 @@ # This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/location/examples/cloud/web.py) from fiftyone_location.location_pipelinebuilder import LocationPipelineBuilder -from fiftyone_pipeline_core.web import webevidence -import json +from fiftyone_pipeline_core.web import json_response, webevidence # First create the location pipeline with the desired settings. @@ -68,24 +67,14 @@ app = Flask(__name__) # First we make a JSON route that will be called from the client side and will return - # a JSON encoded property database using any additional evidence provided by the client + # a JSON encoded property database using any additional evidence provided by the client. + # json_response processes the request and sets the cache headers, so a browser keeps a + # good answer for 30 minutes and never keeps one made while processing failed. @app.route('/json', methods=['POST']) def jsonroute(): - # Create the flowData object for the JSON route - flowData = pipeline.create_flowdata() - - # Add any information from the request (headers, cookies and additional - # client side provided information) - - flowData.evidence.add_from_dict(webevidence(request)) - - # Process the flowData - flowData.process() - - # Return the JSON from the JSONBundler engine - return json.dumps(flowData.jsonbundler.json) + return json_response(pipeline, request) # Helper function to get a property value if it exists and return # the reason why if it doesn't diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py index 4b2d4666..f86331cf 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py @@ -74,7 +74,6 @@ # - [fiftyone_pipeline_cloudrequestengine](https://pypi.org/project/fiftyone-pipeline-cloudrequestengine/) # - [flask](https://pypi.org/project/flask/) -import json import os import sys @@ -84,7 +83,8 @@ from fiftyone_pipeline_cloudrequestengine.cloudrequestexception import ( CloudRequestException) from fiftyone_pipeline_core.logger import Logger -from fiftyone_pipeline_core.web import set_response_header, webevidence +from fiftyone_pipeline_core.web import ( + javascript_response, json_response, set_response_header, webevidence) from fiftyone_pipeline_examples.cloud.engines import ( CountriesTranslationCloud, DeviceDetectionCloud, IpIntelligenceCloud) @@ -265,27 +265,20 @@ def index(): # The other Pipeline APIs ship web integrations that serve the # client-side script at '/51Degrees.core.js', so the page refers to it by # that name. There is no such integration for Flask, so the example adds - # the route itself. + # the route itself. javascript_response and json_response process the + # request and set the cache headers, so a browser keeps a good answer + # for 30 minutes and never keeps one made while processing failed. @app.route("/51Degrees.core.js") def core_js(): - flowdata = process() - - response = make_response(flowdata.javascriptbuilder.javascript) - response.headers["Content-Type"] = "application/x-javascript" - - return response + return javascript_response(pipeline, request) # The client-side script posts the evidence it gathers here, and gets # back the results of processing it as JSON. @app.route("/json", methods=["POST"]) def json_results(): - flowdata = process() - - return app.response_class( - json.dumps(flowdata.jsonbundler.json), - mimetype="application/json") + return json_response(pipeline, request) return app diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py index 8ba78fef..95471e1e 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web_offline.py @@ -27,6 +27,10 @@ """ import json +import os +import re +import subprocess +import sys import pytest import requests @@ -113,6 +117,9 @@ class StubCloud: def __init__(self, sections): self.sections = sections self.requests = [] + # While set, a request for results fails as it does when the service + # cannot be reached. + self.failing = False # The Origin header of each request, kept apart from the body. self.origins = [] @@ -131,6 +138,10 @@ def request(self, method, url, data=None, headers=None): if "evidencekeys" in url: return StubResponse(["header.user-agent", "query.client-ip"]) + if self.failing: + raise requests.exceptions.ConnectionError( + "The stand-in service cannot be reached.") + # A request that names the properties it wants gets those alone, # and no section it named nothing from. named = data.get("values") @@ -313,3 +324,101 @@ def test_the_web_request_is_not_sent_to_the_cloud(cloud): url, data = stub.requests[-1] assert not any("http-request" in key for key in data), data + + +# The cache headers of the client-side script and the client-side callback, +# as the cache header detail of the web integration specification sets them. +DESKTOP_CHROME = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") +CACHEABLE = "max-age=1800, private" +NOT_CACHEABLE = "private, no-cache" + +# A weak entity tag of RFC 9110 section 8.8.3, because the ETag depends on +# the evidence alone. +WEAK_ETAG = re.compile(r'W/"[0-9a-f]{32}"\Z') + + +def client_side(client, path, **headers): + """The response of one of the client-side endpoints.""" + + headers.setdefault("User-Agent", DESKTOP_CHROME) + if path == "/json": + return client.post(path, headers=headers) + return client.get(path, headers=headers) + + +def example_client(): + pipeline = create_pipeline( + Logger(min_level="error"), resource_key="a-resource-key") + return create_app(pipeline).test_client() + + +def script_etag(): + """The ETag of the client-side script for a fixed request, with the + stand-in answering. Run in a process of its own by the test that + compares processes, because it replaces requests.request for good.""" + + requests.request = StubCloud(EVERY_SECTION).request + response = client_side(example_client(), "/51Degrees.core.js") + return response.headers.get("ETag") + + +@pytest.mark.parametrize("path", ["/51Degrees.core.js", "/json"]) +def test_a_good_response_may_be_kept_by_the_browser_alone(cloud, path): + cloud(EVERY_SECTION) + response = client_side(example_client(), path) + + assert response.status_code == 200 + assert response.headers.get("Cache-Control") == CACHEABLE + assert WEAK_ETAG.match(response.headers.get("ETag", "")) + assert "User-Agent" in response.headers.get("Vary", "") + + +def test_the_etag_is_the_same_in_every_process(): + """Python seeds the hash of a string afresh in each process, so the two + processes are given different seeds.""" + + tests = os.path.dirname(os.path.abspath(__file__)) + code = ("import sys; sys.path.insert(0, sys.argv[1]); " + "from test_cloud_mixed_web_offline import script_etag; " + "print(script_etag())") + etags = [] + for seed in ("1", "2"): + finished = subprocess.run( + [sys.executable, "-c", code, tests], capture_output=True, + text=True, timeout=120, env=dict(os.environ, PYTHONHASHSEED=seed)) + assert finished.returncode == 0, finished.stderr + etags.append(finished.stdout.strip().splitlines()[-1]) + assert WEAK_ETAG.match(etags[0]), etags + assert etags[0] == etags[1] + + +@pytest.mark.parametrize("path", ["/51Degrees.core.js", "/json"]) +def test_a_response_to_a_failed_request_carries_no_etag(cloud, path): + stub = cloud(EVERY_SECTION) + client = example_client() + stub.failing = True + response = client_side(client, path) + + assert response.status_code == 200 + assert response.headers.get("Cache-Control") == NOT_CACHEABLE + assert "ETag" not in response.headers + + +# A 304 for the script, which is a GET, and a 412 for the callback, which is +# a POST, as RFC 9110 section 13.1.2 asks. +@pytest.mark.parametrize("path, status", [ + ("/51Degrees.core.js", 304), ("/json", 412)]) +def test_a_matching_etag_is_answered_while_processing_fails( + cloud, path, status): + stub = cloud(EVERY_SECTION) + client = example_client() + etag = client_side(client, path).headers.get("ETag") + assert etag, "a good response carries an ETag" + stub.failing = True + response = client_side(client, path, **{"If-None-Match": etag}) + + assert response.status_code == status + if status == 304: + assert response.headers.get("ETag") == etag diff --git a/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py b/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py index 6edda1c2..e25c5723 100644 --- a/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py +++ b/pipeline/fiftyone_pipeline_core/examples/client_side_evidence_custom_flow_element.py @@ -31,12 +31,11 @@ # a person is in the northern or southern hemispheres. from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder -from fiftyone_pipeline_core.web import webevidence +from fiftyone_pipeline_core.web import json_response, webevidence from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary -import json # Function to get star sign from month and day def getStarSign(month, day): @@ -168,26 +167,14 @@ def get_evidence_key_filter(self): app = Flask(__name__) -# Make a route that returns back JSON encded properties for client side scripts to pick up +# Make a route that returns back JSON encoded properties for client side scripts to pick +# up. json_response processes the request and sets the cache headers, so a browser keeps +# a good answer for 30 minutes and never keeps one made while processing failed. @app.route('/json', methods=['POST']) def jsonroute(): - # Create the flowdata object for the JSON route - flowdata = myPipeline.create_flowdata() - - # Add any information from the request (headers, cookies and additional - # client side provided information) - - flowdata.evidence.add_from_dict(webevidence(request)) - - # Process the flowdata - - flowdata.process() - - # Return the JSON from the JSONBundler engine - - return json.dumps(flowdata.jsonbundler.json) + return json_response(myPipeline, request) @app.route('/') diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py index 768fdd8b..b5d06b0e 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py @@ -20,12 +20,45 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +import hashlib +import json +import re from types import MappingProxyType -from flask import has_request_context, request, session +from flask import has_request_context, make_response, request, session from .evidence import HTTP_REQUEST_KEY +# How long the browser alone may keep a client-side response to a request +# whose processing recorded no errors, unless the caller gives another time. +CLIENT_SIDE_MAX_AGE_SECONDS = 1800 + +# The Cache-Control of a client-side response to a request whose processing +# recorded errors. It goes with no ETag, so a browser that kept the response +# has nothing to revalidate with and asks again in full once processing +# works, rather than having the faulty response confirmed by a 304. +_CACHE_CONTROL_AFTER_ERRORS = "private, no-cache" + +# The headers a browser adds when it revalidates or reloads. They differ +# between a request and its revalidation and no element uses them, so they +# are left out of the ETag and the Vary header. +_REVALIDATION_HEADERS = frozenset(( + "cache-control", "if-match", "if-modified-since", "if-none-match", + "if-range", "if-unmodified-since", "pragma")) + +# An entity tag of RFC 9110 section 8.8.3, being an optional W/ and an +# opaque tag in double quotes of visible characters other than the double +# quote. The second group is the opaque tag without its quotes. +_ENTITY_TAG = re.compile(r'(W/)?"([\x21\x23-\x7e\x80-\xff]*)"') + +# A field name of RFC 9110 section 5.1, in lower case. An evidence key that +# is not one would make the Vary header unreadable, so it is left out. +_FIELD_NAME = re.compile(r"[!#$%&'*+.^_`|~0-9a-z-]+\Z") + +# The optional whitespace around the commas of a list, RFC 9110 section +# 5.6.3. +_OWS = " \t" + def request_line_path(request): """! @@ -127,4 +160,198 @@ def set_response_header(flowData, response): if response_value != "": response.headers[response_key] = response_value - return response \ No newline at end of file + return response + +def javascript_response( + pipeline, request, max_age_seconds=CLIENT_SIDE_MAX_AGE_SECONDS): + + """! + Process a request for the client-side JavaScript and answer with the + JavaScript the pipeline's JavaScriptBuilder writes, with the cache headers + the web integration specification sets. + + The ETag is a weak entity tag, W/"...", hashed from the evidence the + request supplied that the pipeline's elements accept. A GET or HEAD + request whose If-None-Match header matches it, by the weak comparison of + RFC 9110 section 13.1.2, gets a 304 without being processed, so a + browser keeps a good copy while processing fails. A request by any other + method whose If-None-Match matches, such as a POST to the JSON endpoint, + gets 412 Precondition Failed. An If-None-Match that is neither "*" nor a + list of entity tags matches nothing. Otherwise the response carries + 'Cache-Control: max-age=1800, private', or the max age given, and the + ETag, or 'Cache-Control: private, no-cache' and no ETag when processing + recorded errors. Both carry a Vary header naming the headers the + pipeline's elements accept, whether or not the request carried them. + + @type pipeline: Pipeline + @param pipeline: A pipeline holding the JavaScriptBuilder + @type request: Request + @param request: The request for the JavaScript + @type max_age_seconds: int + @param max_age_seconds: How long the browser may keep the response + @rtype Response + @return The response to send + """ + + return _client_side_response( + pipeline, request, + lambda flowdata: flowdata.javascriptbuilder.javascript, + "application/x-javascript", max_age_seconds) + +def json_response( + pipeline, request, max_age_seconds=CLIENT_SIDE_MAX_AGE_SECONDS): + + """! + Process a request from the client-side JavaScript and answer with the + JSON the pipeline's JSONBundler writes, with the same cache headers as + javascript_response. + + @type pipeline: Pipeline + @param pipeline: A pipeline holding the JSONBundler + @type request: Request + @param request: The request from the client-side JavaScript + @type max_age_seconds: int + @param max_age_seconds: How long the browser may keep the response + @rtype Response + @return The response to send + """ + + return _client_side_response( + pipeline, request, + lambda flowdata: json.dumps(flowdata.jsonbundler.json), + "application/json", max_age_seconds) + +def _client_side_response( + pipeline, request, content, content_type, max_age_seconds): + + cache_control = "max-age=" + str(int(max_age_seconds)) + ", private" + flowdata = pipeline.create_flowdata() + try: + evidence = webevidence(request) + flowdata.evidence.add_from_dict(evidence) + + # From the request's own evidence, because processing adds to it. + # The sequence element gives a request with no session id a new one. + opaque_tag = _opaque_tag(pipeline, evidence) + # Weak, because the response can change while the evidence does not, + # for example when the data file is updated (RFC 9110 section 8.8.3). + etag = 'W/"' + opaque_tag + '"' + vary = _vary(pipeline, request) + + if _if_none_match_matches( + request.headers.get("If-None-Match"), opaque_tag): + if request.method not in ("GET", "HEAD"): + # RFC 9110 section 13.1.2 allows a 304 for GET and HEAD only. + response = make_response("", 412) + response.headers.pop("Content-Type", None) + return response + # With the headers of a good response that RFC 9110 section + # 15.4.5 asks a 304 to repeat. + response = make_response("", 304) + response.headers.pop("Content-Type", None) + response.headers["Cache-Control"] = cache_control + response.headers["ETag"] = etag + else: + flowdata.process() + response = make_response(content(flowdata)) + response.headers["Content-Type"] = content_type + if flowdata.errors: + response.headers["Cache-Control"] = _CACHE_CONTROL_AFTER_ERRORS + else: + response.headers["Cache-Control"] = cache_control + response.headers["ETag"] = etag + + if vary: + response.headers["Vary"] = vary + return response + finally: + flowdata.close() + +def _uses(pipeline, key): + + # Whether the key is evidence one of the pipeline's elements accepts. The + # web request itself, which the usage sharing element reads, is evidence + # no element uses, whatever the filters accept. + if key.lower() == HTTP_REQUEST_KEY: + return False + if key.startswith("header.") and \ + key[len("header."):] in _REVALIDATION_HEADERS: + return False + return any( + element.filter_evidence_key(key) + for element in pipeline.flow_elements) + +def _opaque_tag(pipeline, evidence): + + # SHA-256 rather than hash(), which Python seeds afresh in each process, + # so a browser revalidating against another process, after a restart or + # behind a load balancer, still matches. + digest = hashlib.sha256() + for key in sorted(evidence): + if _uses(pipeline, key): + digest.update(key.encode("utf-8")) + digest.update(b"\0") + digest.update(str(evidence[key]).encode("utf-8")) + digest.update(b"\0") + return digest.hexdigest()[:32] + +def _if_none_match_matches(if_none_match, opaque_tag): + + # RFC 9110 section 13.1.2. "*" matches, as there is always a response. + # A list of entity tags matches when one of them has the same opaque + # tag, whether or not either is weak. Anything else matches nothing. + if if_none_match is None: + return False + value = if_none_match.strip(_OWS) + if value == "*": + return True + tags = _entity_tags(value) + return tags is not None and opaque_tag in tags + +def _entity_tags(value): + + # The opaque tags of a list of entity tags, which may have whitespace + # around its commas and empty elements (RFC 9110 section 5.6.1.2), or + # None when the value is not such a list. + tags = [] + position = 0 + while True: + while position < len(value) and value[position] in _OWS: + position += 1 + if position == len(value): + return tags + if value[position] != ",": + tag = _ENTITY_TAG.match(value, position) + if tag is None: + return None + tags.append(tag.group(2)) + position = tag.end() + while position < len(value) and value[position] in _OWS: + position += 1 + if position == len(value): + return tags + if value[position] != ",": + return None + position += 1 + +def _vary(pipeline, request): + + # The headers in the pipeline's evidence key filter, whether or not this + # request carried them, so a cache re-fetches when one that was absent + # appears. An element whose filter is a list names every header in it. A + # filter that cannot list its keys is asked about the request's own + # headers one by one instead. + carried = [name.lower() for name, _ in request.headers] + names = set() + for element in pipeline.flow_elements: + listed = getattr(element.get_evidence_key_filter(), "list", None) + keys = listed if listed is not None else \ + ["header." + name for name in carried] + for key in keys: + key = str(key).lower() + if key.startswith("header.") and _uses(pipeline, key) and \ + _FIELD_NAME.match(key[len("header."):]): + names.add(key[len("header."):]) + return ", ".join( + "-".join(part.capitalize() for part in name.split("-")) + for name in sorted(names)) \ No newline at end of file diff --git a/pipeline/fiftyone_pipeline_core/tests/test_web_client_side.py b/pipeline/fiftyone_pipeline_core/tests/test_web_client_side.py new file mode 100644 index 00000000..ddc07912 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_web_client_side.py @@ -0,0 +1,492 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The cache headers of the client-side JavaScript and JSON responses, as +the cache header detail of the web integration specification sets them.""" + +import os +import re +import subprocess +import sys +from types import MappingProxyType +import unittest +from unittest import mock + +from flask import Flask, request + +from fiftyone_pipeline_core import web +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.evidence_keyfilter import EvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_core.web import javascript_response, json_response + +CHROME = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36") +SAFARI = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 " + "Safari/605.1.15") + +CACHEABLE = "max-age=1800, private" +NOT_CACHEABLE = "private, no-cache" + +# A weak entity tag of RFC 9110 section 8.8.3, because the ETag depends on +# the evidence alone and the response can change while the evidence does not. +WEAK_ETAG = re.compile(r'W/"[0-9a-f]{32}"\Z') + +# The session id the JavaScript builder writes into the script, which is +# minified by default. +SESSION_ID = re.compile(r'var sessionId\s*=\s*"([^"]*)"') + + +class Switchable(FlowElement): + """An element that fails while failing is set, as a cloud request does + while the service cannot be reached.""" + + failing = False + + def __init__(self): + super(Switchable, self).__init__() + self.datakey = "switchable" + self.properties = {"answer": {"type": "int"}} + + def process_internal(self, flowdata): + if Switchable.failing: + raise Exception("The stand-in service cannot be reached.") + flowdata.set_element_data(ElementDataDictionary(self, {"answer": 42})) + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(["header.user-agent"]) + + +class Listed(FlowElement): + """An element that accepts the evidence keys it is given, as most + elements do, so its filter can list them.""" + + def __init__(self, keys): + super(Listed, self).__init__() + self.datakey = "listed" + self.keys = keys + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary(self, {})) + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(self.keys) + + +class Unlisted(FlowElement): + """An element whose filter decides key by key and cannot list what it + accepts, as the cloud request engine's does.""" + + def __init__(self): + super(Unlisted, self).__init__() + self.datakey = "unlisted" + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary(self, {})) + + def get_evidence_key_filter(self): + return OnlyUserAgent() + + +class OnlyUserAgent(EvidenceKeyFilter): + + def filter_evidence_key(self, key): + return key.lower() == "header.user-agent" + + +def client(): + """A test client of an application serving the two client-side + endpoints.""" + + pipeline = PipelineBuilder({ + "javascript_builder_settings": {"endpoint": "/json"}, + "suppress_process_exceptions": True, + }).add(Switchable()).build() + + app = Flask(__name__) + + @app.route("/51Degrees.core.js") + def javascript(): + return javascript_response(pipeline, request) + + @app.route("/json", methods=["POST"]) + def results(): + return json_response(pipeline, request) + + return app.test_client() + + +def script_etag(): + """The ETag of the JavaScript for a fixed request. Run in a process of + its own by the test that compares processes.""" + + return client().get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}).headers["ETag"] + + +class ClientSideCacheTests(unittest.TestCase): + + def setUp(self): + Switchable.failing = False + self.client = client() + + def tearDown(self): + Switchable.failing = False + + def responses(self, headers): + """The JavaScript and the JSON responses to the same evidence.""" + return [ + self.client.get("/51Degrees.core.js", headers=headers), + self.client.post("/json", headers=headers), + ] + + def test_a_good_response_may_be_kept_by_the_browser_alone(self): + for response in self.responses({"User-Agent": CHROME}): + self.assertEqual(200, response.status_code) + self.assertEqual(CACHEABLE, response.headers.get("Cache-Control")) + etag = response.headers.get("ETag") + self.assertIsNotNone(etag, "a good response carries an ETag") + self.assertRegex(etag, WEAK_ETAG) + self.assertIn("User-Agent", response.headers.get("Vary", "")) + + def test_the_content_type_says_which_endpoint_answered(self): + script, results = self.responses({"User-Agent": CHROME}) + self.assertEqual("application/x-javascript", + script.headers["Content-Type"]) + self.assertEqual("application/json", results.headers["Content-Type"]) + self.assertIn(b"fiftyoneDegreesManager", script.data) + self.assertEqual(42, results.get_json()["switchable"]["answer"]) + + def test_the_content_length_is_the_length_of_what_was_sent(self): + for response in self.responses({"User-Agent": CHROME}): + self.assertEqual(str(len(response.data)), + response.headers.get("Content-Length")) + + def test_the_etag_follows_the_evidence(self): + first = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) + again = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) + other = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": SAFARI}) + self.assertEqual(first.headers["ETag"], again.headers["ETag"]) + self.assertNotEqual(first.headers["ETag"], other.headers["ETag"]) + + def test_the_etag_is_the_same_in_every_process(self): + """Python seeds the hash of a string afresh in each process, so the + two processes are given different seeds.""" + + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + code = ("import sys; sys.path.insert(0, sys.argv[1]); " + "from tests.test_web_client_side import script_etag; " + "print(script_etag())") + etags = [] + for seed in ("1", "2"): + finished = subprocess.run( + [sys.executable, "-c", code, root], capture_output=True, + text=True, cwd=root, timeout=120, + env=dict(os.environ, PYTHONHASHSEED=seed)) + self.assertEqual(0, finished.returncode, finished.stderr) + etags.append(finished.stdout.strip().splitlines()[-1]) + self.assertRegex(etags[0], WEAK_ETAG) + self.assertEqual(etags[0], etags[1]) + self.assertEqual(script_etag(), etags[0]) + + def test_the_etag_is_the_same_from_two_pipelines(self): + self.assertRegex(script_etag(), WEAK_ETAG) + self.assertEqual(script_etag(), script_etag()) + + def test_the_session_id_the_sequence_element_adds_is_not_in_the_etag( + self): + """The sequence element gives a request with no session id a new one + while processing, and the script carries it. The ETag covers only the + evidence the request supplied, so it is the same for both + requests.""" + + responses = [ + self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) + for _ in range(2)] + session_ids = [ + SESSION_ID.search(response.get_data(as_text=True)).group(1) + for response in responses] + self.assertTrue(session_ids[0], "the script carries a session id") + self.assertNotEqual(session_ids[0], session_ids[1]) + self.assertRegex(responses[0].headers["ETag"], WEAK_ETAG) + self.assertEqual( + responses[0].headers["ETag"], responses[1].headers["ETag"]) + + def test_revalidation_with_the_etag_is_answered_not_modified(self): + good = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) + for method in (self.client.get, self.client.head): + # A browser that revalidates sends Cache-Control as well when + # the page is reloaded, and neither header is evidence. + response = method("/51Degrees.core.js", headers={ + "User-Agent": CHROME, + "If-None-Match": good.headers["ETag"], + "Cache-Control": "max-age=0", + }) + self.assertEqual(304, response.status_code) + self.assertEqual(b"", response.data) + # The headers of the good response that RFC 9110 section 15.4.5 + # asks a 304 to repeat. + for name in ("ETag", "Cache-Control", "Vary"): + self.assertEqual(good.headers.get(name), + response.headers.get(name), name) + + def test_a_post_with_a_matching_etag_is_refused(self): + """RFC 9110 section 13.1.2 allows a 304 for GET and HEAD only, and + asks for 412 Precondition Failed for any other method.""" + + etag = self.client.post( + "/json", headers={"User-Agent": CHROME}).headers["ETag"] + response = self.client.post("/json", headers={ + "User-Agent": CHROME, "If-None-Match": etag}) + self.assertEqual(412, response.status_code) + self.assertEqual(b"", response.data) + self.assertNotIn("ETag", response.headers) + + def test_the_web_request_value_is_not_in_the_etag(self): + """The web integration may add the request itself as + server.http-request, for the usage sharing element. It holds the + request's every header, If-None-Match included, so it must not be + hashed, even though the JavaScript builder accepts every key.""" + + def with_web_request(request): + evidence = webevidence(request) + evidence["server.http-request"] = MappingProxyType(dict(evidence)) + return evidence + + webevidence = web.webevidence + with mock.patch.object(web, "webevidence", with_web_request): + etag = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) \ + .headers["ETag"] + response = self.client.get("/51Degrees.core.js", headers={ + "User-Agent": CHROME, "If-None-Match": etag}) + self.assertEqual(304, response.status_code) + + @staticmethod + def if_none_match_cases(etag): + """Values of If-None-Match, each with whether it matches the ETag by + RFC 9110 section 13.1.2. A value that is neither "*" nor a list of + entity tags matches nothing, even when it holds the ETag.""" + + quoted = etag[len("W/"):] + return [ + # The weak comparison ignores W/ on either side. + (etag, True), + (quoted, True), + ("*", True), + ('"other", ' + etag, True), + ('W/"other", ' + quoted, True), + # A list may hold a comma inside a tag, a tab or no space + # around a comma, and empty elements (RFC 9110 section 5.6.1.2). + ('W/"a,b", ' + etag, True), + ('"other",\t' + etag, True), + ('"other"\t,' + etag, True), + ('"other",' + etag, True), + (", " + etag + " ,", True), + ('"other"', False), + ('W/"other", "another"', False), + ("", False), + # Not a list of entity tags. + (etag + ", other", False), + ("other, " + etag, False), + ('"other" ' + etag, False), + ('"other"' + etag, False), + (etag + ' "other"', False), + (etag + ', "a b"', False), + (quoted[:-1] + ' "', False), + ("w/" + quoted, False), + ("W/ " + quoted, False), + ("*, " + etag, False), + (etag + ", *", False), + (quoted[1:-1], False), + (quoted[:-1], False), + ] + + def test_if_none_match_on_a_get_is_compared_weakly(self): + good = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) + self.assertRegex(good.headers["ETag"], WEAK_ETAG) + for value, matches in self.if_none_match_cases(good.headers["ETag"]): + with self.subTest(value=value): + response = self.client.get("/51Degrees.core.js", headers={ + "User-Agent": CHROME, "If-None-Match": value}) + self.assertEqual(304 if matches else 200, + response.status_code) + self.assertEqual(good.headers["ETag"], + response.headers.get("ETag")) + + def test_if_none_match_on_a_post_is_compared_weakly(self): + good = self.client.post("/json", headers={"User-Agent": CHROME}) + self.assertRegex(good.headers["ETag"], WEAK_ETAG) + for value, matches in self.if_none_match_cases(good.headers["ETag"]): + with self.subTest(value=value): + response = self.client.post("/json", headers={ + "User-Agent": CHROME, "If-None-Match": value}) + self.assertEqual(412 if matches else 200, + response.status_code) + if matches: + self.assertEqual(b"", response.data) + self.assertNotIn("ETag", response.headers) + else: + self.assertEqual( + 42, response.get_json()["switchable"]["answer"]) + + def test_if_none_match_fields_are_one_list(self): + """Several If-None-Match fields are one comma-separated list, as RFC + 9110 section 5.3 says.""" + + etag = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) \ + .headers["ETag"] + response = self.client.get("/51Degrees.core.js", headers=[ + ("User-Agent", CHROME), + ("If-None-Match", '"other"'), + ("If-None-Match", etag)]) + self.assertEqual(304, response.status_code) + + def test_a_response_to_a_failed_request_carries_no_etag(self): + Switchable.failing = True + for response in self.responses({"User-Agent": CHROME}): + self.assertEqual(200, response.status_code) + self.assertEqual( + NOT_CACHEABLE, response.headers.get("Cache-Control")) + self.assertNotIn("ETag", response.headers) + + def test_a_good_copy_is_kept_while_processing_fails(self): + etag = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) \ + .headers["ETag"] + Switchable.failing = True + response = self.client.get("/51Degrees.core.js", headers={ + "User-Agent": CHROME, "If-None-Match": etag}) + self.assertEqual(304, response.status_code) + + def test_a_failed_response_is_replaced_once_processing_works(self): + """The browser kept the failed response with no ETag, so it has + nothing to revalidate with and asks again in full.""" + + Switchable.failing = True + failed = self.client.get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) + Switchable.failing = False + headers = {"User-Agent": CHROME} + if "ETag" in failed.headers: + headers["If-None-Match"] = failed.headers["ETag"] + response = self.client.get("/51Degrees.core.js", headers=headers) + self.assertEqual(200, response.status_code) + self.assertEqual(CACHEABLE, response.headers.get("Cache-Control")) + + def test_the_max_age_can_be_set(self): + pipeline = PipelineBuilder({ + "javascript_builder_settings": {"endpoint": "/json"}, + }).add(Switchable()).build() + app = Flask(__name__) + + @app.route("/51Degrees.core.js") + def javascript(): + return javascript_response(pipeline, request, 60) + + @app.route("/json", methods=["POST"]) + def results(): + return json_response(pipeline, request, max_age_seconds=60) + + client = app.test_client() + for response in ( + client.get("/51Degrees.core.js", + headers={"User-Agent": CHROME}), + client.post("/json", headers={"User-Agent": CHROME})): + self.assertEqual("max-age=60, private", + response.headers.get("Cache-Control")) + + def test_vary_names_a_header_the_request_did_not_carry(self): + # The Vary header names every header in the pipeline's evidence key + # filter, not only the ones this request happened to carry, so a + # cache re-fetches when a header that was absent appears. + pipeline = PipelineBuilder({ + "javascript_builder_settings": {"endpoint": "/json"}, + }).add(Listed(["header.user-agent", "header.sec-ch-ua-model"])).build() + app = Flask(__name__) + + @app.route("/51Degrees.core.js") + def javascript(): + return javascript_response(pipeline, request) + + response = app.test_client().get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) + + # Host comes from the JavaScript Builder, which reads it to build + # the callback URL the script posts to. + self.assertEqual("Host, Sec-Ch-Ua-Model, User-Agent", + response.headers.get("Vary")) + + def test_vary_leaves_out_the_headers_a_browser_revalidates_with(self): + # If-None-Match and the rest differ between a request and its + # revalidation, so naming them would stop any response being reused. + pipeline = PipelineBuilder({ + "javascript_builder_settings": {"endpoint": "/json"}, + }).add(Listed(["header.user-agent", "header.if-none-match", + "header.cache-control"])).build() + app = Flask(__name__) + + @app.route("/51Degrees.core.js") + def javascript(): + return javascript_response(pipeline, request) + + response = app.test_client().get( + "/51Degrees.core.js", headers={"User-Agent": CHROME}) + + self.assertEqual("Host, User-Agent", response.headers.get("Vary")) + + def test_vary_falls_back_to_the_request_for_a_filter_that_cannot_list( + self): + # An element whose filter is not a list cannot say which headers it + # accepts, so the request's own headers are asked of it one by one. + pipeline = PipelineBuilder({ + "javascript_builder_settings": {"endpoint": "/json"}, + }).add(Unlisted()).build() + app = Flask(__name__) + + @app.route("/51Degrees.core.js") + def javascript(): + return javascript_response(pipeline, request) + + response = app.test_client().get( + "/51Degrees.core.js", + headers={"User-Agent": CHROME, "Accept-Language": "en"}) + + # User-Agent from the element below, Host from the JavaScript + # Builder, and no Accept-Language, which neither accepts. + self.assertEqual("Host, User-Agent", response.headers.get("Vary")) + + +if __name__ == "__main__": + unittest.main() From 435274d8f4c04e1c4bad4d738e893c3d965bc0f7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 15:10:38 +0000 Subject: [PATCH 62/68] Send requested properties beside a resource key, with or without a licence key (#89) * Test that requested properties beside a resource key are sent, with or without a licence key * Send requested properties beside a resource key, with or without a licence key The engine refused requested_properties whenever a resource key was set, because the cloud service was thought to ignore the list. The service adds the listed properties to those the resource key carries, so the engine now builds and sends the list as values, and the licence key too where there is one. It still refuses no key at all, and a licence key with no resource key or list, which the service refuses on every request. The message for properties an answer leaves out now says they are not covered by the keys in use, as a resource key can be the only one. * Test that a name left out beside a resource key alone is reported as not a property * Report a name left out beside a resource key alone as not a property, and drop history from two comments * Say the keys in use in the properties not covered warning, and drop a stale claim about discovery .NET, Java, Node.js and PHP all word the warning as not covered by the keys in use, because the licence behind a resource key and a licence key beside it both decide what comes back. The name not known warning gains the same closing line and the spelling advice those languages give. The discovery comment no longer says a licence key beside a resource key widens the answer, which the corrected Accepted combinations table replaced. Tests assert both wordings. * Test an answer that is not an object, and the message .NET and Java write The report compares the answer with the list it asked for, and an answer that parses as JSON but is not an object has nothing to compare, so the test pins that the request still succeeds and nothing is written. The warning for a name that is not a property is asserted as .NET and Java word it, both of them carrying the same sentence. * Guard the report against an answer that is not an object, and take .NET and Java's wording An answer that parses as JSON but is not an object has no sections to look in, so the report returned an AttributeError out of processing and failed a request that had succeeded. It now leaves such an answer alone, with the same guard and the same wider catch that is_cacheable uses on the same text. The warning for a name that is not a property is now word for word the sentence .NET and Java both write, so Python's two warnings match theirs exactly. --- .../readme.md | 12 +- .../cloudrequestengine.py | 54 ++++---- .../constants.py | 27 ++-- .../tests/test_credentials.py | 126 +++++++++++++++--- 4 files changed, 155 insertions(+), 64 deletions(-) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md index 022d2c05..de740e3b 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md @@ -29,10 +29,11 @@ property names to ask for, is also needed: | `resource_key` | `license_key` | `requested_properties` | Accepted | | --- | --- | --- | --- | | Yes | No | No | Yes. The resource key states which properties it carries. | -| Yes | Yes | No | Yes. The licence key adds the products it grants. | +| Yes | Yes | No | Yes. The answer holds what the resource key carries. | +| Yes | Yes | Yes | Yes. The service adds the listed properties the licence key grants. | +| Yes | No | Yes | Yes. The service adds the listed properties, as null with a reason where no licence grants one. | | No | Yes | Yes | Yes. A licence key names no properties, so the list says which are wanted. | | No | Yes | No | No. The service answers 400 to every such request. | -| Yes | Either | Yes | No. The service ignores the list while a resource key is present. | | No | No | Either | No. There is nothing to authenticate with. | A combination that is not accepted raises `ValueError` when the engine is @@ -47,8 +48,11 @@ key and name the properties it wants, so the credential stays on the server and each answer carries only what that caller needs. A licence key is sent in the body of every request, never in a URL. Where a request named properties and the answer left some out, the engine warns once that -they are not covered by the licence key; the properties that came back -are usable as normal. +they are not covered by the keys in use, and the properties that came back +are usable as normal. With a resource key and no licence key, a property no +licence grants comes back as null with a reason, so the warning says instead +that every property is answered and these names are not properties, and +asks the caller to check the spelling of each one. See the [Credentials section of the cloud request engine specification](https://github.com/51Degrees/specifications/blob/main/pipeline-specification/pipeline-elements/cloud-request-engine.md#credentials) and the rules in diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py index ff598ef1..5dfc6bc0 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py @@ -60,9 +60,8 @@ def __init__(self, settings = {}): license_key, and optionally 1) requested_properties, the fully qualified names of the properties to ask for, as a list or a comma separated string. - Required with a license_key alone and refused with a resource_key, - because the cloud service ignores the list whenever a resource key - is present + Required with a license_key alone. Beside a resource_key the cloud + service adds them to the properties the resource key carries 2) a cloud_endpoint to overwrite the default baseurl 3) an cloud_request_origin to use when making requests @@ -181,9 +180,7 @@ def add_requested_properties(self, names): Add to the properties each request asks for, after the engine is built. The names are trimmed and lower-cased as the constructor does, so a name given in the case the documentation uses still - matches the answer, and the credential rules are checked again, - because a list only has meaning on a licence key alone and the - constructor's check cannot see a list added later. + matches the answer. @type names: list or string @param names: the names to add, as a list or one comma separated @@ -192,7 +189,6 @@ def add_requested_properties(self, names): self.requested_properties.extend( self.setting_names({"names": names}, "names")) - self.check_credentials() def check_credentials(self): @@ -213,10 +209,6 @@ def check_credentials(self): if has_resource_key == False and has_requested_properties == False: raise ValueError(Constants.MESSAGE_LICENSE_KEY_NEEDS_PROPERTIES) - if has_resource_key == True and has_requested_properties == True: - raise ValueError( - Constants.MESSAGE_PROPERTIES_IGNORED_WITH_RESOURCE_KEY) - def get_credentials(self): """! @@ -277,12 +269,10 @@ def get_engine_properties(self): """ # Get properties for all engines. A resource key alone goes in the - # query string as it always has. A licence key identifies an - # account, so it is posted in the body instead, where it stays out - # of request logs. The cloud service reads both keys from either - # place, answers a licence key alone with everything the licence - # entitles, and widens a resource key's answer by the products a - # licence key sent beside it grants. + # query string. A licence key identifies an account, so it is + # posted in the body instead, where it stays out of request logs. + # The cloud service reads both keys from either place, and answers + # a licence key alone with every property the licence entitles. propertiesURL = self.baseURL + "accessibleProperties" @@ -430,7 +420,7 @@ def process_internal(self, flowdata): """ - # A resource key names the route, as it always has. A licence key + # A resource key names the route. A licence key # alone has no resource key to put there, so it uses the data # endpoint that takes none and travels in the body with the list # of properties wanted. @@ -488,13 +478,16 @@ def report_properties_not_covered(self, result): Compare the properties a request asked for with the ones its answer carries, and warn once about any it left out. - The cloud service drops a property the licence key does not cover - without naming it, whenever the request also asked for a property - the key does cover, so the engine has to notice the difference - itself. A property that is present but null counts as answered, + With a licence key, the cloud service drops a property the licence + key does not cover without naming it, whenever the request also + asked for a property the key does cover, so the engine has to + notice the difference itself. With a resource key and no licence + key, a property no licence grants comes back as null with a reason, + so a name left out is not a property, and the warning says that + instead. A property that is present but null counts as answered, because it carries a nullreason beside it saying why it has no value on this request, which is a different matter from - entitlement. The answer cannot change while the key and the list + entitlement. The answer cannot change while the keys and the list stay the same, so each distinct list is reported once. @type result: string @@ -507,11 +500,17 @@ def report_properties_not_covered(self, result): try: answer = json.loads(result) - except JSONDecodeError: + except (AttributeError, JSONDecodeError, TypeError, ValueError): # validate_response has already dealt with an answer that is # not JSON, so there is nothing to compare here. return + # An answer that parses but is not an object, such as an array, + # has no sections to look in. Reporting must not fail the request, + # which succeeded, so there is nothing to say. + if isinstance(answer, dict) == False: + return + not_covered = [] for name in sorted(requested): section, _, property_name = name.partition( @@ -524,9 +523,10 @@ def report_properties_not_covered(self, result): self.reported_not_covered.add(requested) if len(not_covered) > 0: - warnings.warn( - Constants.MESSAGE_PROPERTIES_NOT_COVERED.format( - ", ".join(not_covered))) + message = (Constants.MESSAGE_PROPERTIES_NOT_COVERED + if self.license_key != "" + else Constants.MESSAGE_PROPERTIES_NOT_KNOWN) + warnings.warn(message.format(", ".join(not_covered))) def get_content(self, flowData): diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py index 4ba8535f..60bd4d24 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py @@ -115,18 +115,23 @@ class Constants: "properties it wants. Set requested_properties to the fully " "qualified names wanted, for example " "['device.ismobile', 'device.iscrawler'].") - MESSAGE_PROPERTIES_IGNORED_WITH_RESOURCE_KEY = ( - "CloudRequestEngine has requested_properties and a resource key. " - "The cloud service ignores the list whenever a resource key is " - "present and answers with everything the resource key carries, so " - "the caller would believe the response had narrowed when it had " - "not. Either leave out requested_properties, or authenticate with " - "a licence key alone.") - # Logged once when a request named properties and the response left - # some of them out without saying so. + # Logged once when a request with a licence key named properties and + # the response left some of them out without saying so. Both the + # licence behind a resource key and a licence key beside it decide + # what comes back, so the message names the keys rather than one key. MESSAGE_PROPERTIES_NOT_COVERED = ( "The cloud service did not return the requested properties {} in " - "its answer. They are not covered by the licence key in use, so " - "they will be missing from every result. The other requested " + "its answer. They are not covered by the keys in use, so they " + "will be missing from every result. The other requested " "properties came back and are usable.") + # Logged once when a request with a resource key and no licence key + # named properties and the response left some of them out. Without a + # licence key a property no licence grants comes back as null with a + # reason, so a name left out is not a property. + MESSAGE_PROPERTIES_NOT_KNOWN = ( + "The cloud service did not return the requested properties {} in " + "its answer. Every property is answered, with a reason when it has " + "no value, so these names are not properties. Check the spelling " + "of each one. The other requested properties came back and are " + "usable.") diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py index 75b08646..e29c2e64 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_credentials.py @@ -90,9 +90,10 @@ def test_resource_key_alone_is_accepted(self): def test_resource_key_with_licence_key_is_accepted(self): """ - Row two of the table. The licence key adds the products it grants, - so it travels in the body of the data request and of the accessible - properties request, and no list of properties is sent. + Row two of the table. With no list the answer holds what the + resource key carries and the licence key adds no property, so the + licence key travels in the body of the data request and of the + accessible properties request, and no list of properties is sent. """ engine = CloudRequestEngine({ @@ -182,23 +183,39 @@ def test_licence_key_without_properties_is_refused(self): self.assertIn("400", str(context.exception)) self.assertIn("requested_properties", str(context.exception)) - def test_properties_with_resource_key_are_refused(self): + def test_properties_with_resource_key_are_sent(self): """ - Row five of the table, with and without a licence key beside the - resource key. The service ignores the list, so the engine refuses - to build and says the response would not narrow. + A list beside a resource key, with and without a licence key. The + service adds the listed properties to those the resource key + carries, so the engine builds and sends the list as values, with + the licence key in the body where there is one. """ for license_key in ("", LICENSE_KEY): - with self.assertRaises(ValueError) as context: - CloudRequestEngine({ - "resource_key": RESOURCE_KEY, - "license_key": license_key, - "requested_properties": REQUESTED, - "http_client": self.mock_http(), - }) + engine = CloudRequestEngine({ + "resource_key": RESOURCE_KEY, + "license_key": license_key, + "requested_properties": REQUESTED, + "http_client": self.mock_http(), + }) - self.assertIn("ignores the list", str(context.exception)) + requests = self.process(engine) + + for request_type, url, content in requests: + self.assertNotIn(LICENSE_KEY, url) + + request_type, url, content = self.data_request(requests) + self.assertTrue(url.endswith(RESOURCE_KEY + ".json?"), url) + self.assertEqual( + RESOURCE_KEY, content[Constants.RESOURCE_PARAMETER]) + self.assertEqual( + "device.ismobile,device.istablet", + content[Constants.VALUES_PARAMETER]) + if license_key == "": + self.assertNotIn(Constants.LICENSE_PARAMETER, content) + else: + self.assertEqual( + LICENSE_KEY, content[Constants.LICENSE_PARAMETER]) def test_no_credential_is_refused(self): """ @@ -413,14 +430,13 @@ def test_evidence_no_element_accepts_is_sent(self): self.assertEqual("a cookie", content["other"]) self.assertEqual("iPhone", content["User-Agent"]) - def test_properties_added_later_are_normalised_and_checked(self): + def test_properties_added_later_are_normalised_and_sent(self): """ A caller may add to the list after the engine is built, once the accessible properties say which client-side properties the key carries. The names added are trimmed and lower-cased as the - constructor's are, so they match the answer, and the rule that a - list only goes with a licence key alone is checked again, because - the constructor cannot see a list added after it ran. + constructor's are, so they match the answer, and they are sent + beside a resource key as well as beside a licence key alone. """ engine = CloudRequestEngine({ @@ -447,10 +463,13 @@ def test_properties_added_later_are_normalised_and_checked(self): "http_client": self.mock_http(), }) - with self.assertRaises(ValueError) as context: - engine.add_requested_properties(REQUESTED) + engine.add_requested_properties(REQUESTED) - self.assertIn("ignores the list", str(context.exception)) + request_type, url, content = self.data_request( + self.process(engine)) + self.assertEqual( + "device.ismobile,device.istablet", + content[Constants.VALUES_PARAMETER]) def test_properties_the_answer_leaves_out_are_reported_once(self): """ @@ -483,10 +502,73 @@ def test_properties_the_answer_leaves_out_are_reported_once(self): str(warning.message) for warning in caught if "not covered" in str(warning.message)] self.assertEqual(1, len(messages), messages) + self.assertIn("not covered by the keys in use", messages[0]) self.assertIn("device.iscrawler", messages[0]) self.assertNotIn("device.istablet", messages[0]) self.assertNotIn("device.ismobile", messages[0]) + def test_a_name_left_out_beside_a_resource_key_alone_is_not_a_property( + self): + """ + With a resource key, a list and no licence key, a property no + licence grants comes back as null with its nullreason, so a name the + answer leaves out is not a property at all. The engine says so, + once, and does not name a licence key that was never given. + """ + + engine = CloudRequestEngine({ + "resource_key": RESOURCE_KEY, + "requested_properties": ["device.value", "device.notaproperty"], + "http_client": self.mock_http(), + }) + + pipeline = PipelineBuilder().add(engine).build() + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + for _ in range(2): + data = pipeline.create_flowdata() + data.evidence.add("query.User-Agent", "iPhone") + data.process() + + messages = [ + str(warning.message) for warning in caught + if "device.notaproperty" in str(warning.message)] + self.assertEqual(1, len(messages), messages) + self.assertIn( + "Every property is answered, with a reason when it has no " + "value, so these names are not properties", messages[0]) + self.assertIn("Check the spelling of each one", messages[0]) + self.assertNotIn("licence key", messages[0]) + self.assertNotIn("device.value", messages[0]) + + def test_an_answer_that_is_not_an_object_does_not_fail_the_request(self): + """ + The report compares the answer with the list, and an answer that + parses as JSON but is not an object has nothing to compare. It + must not fail the request, because the request itself succeeded, + so the engine leaves the answer alone and says nothing. + """ + + engine = CloudRequestEngine({ + "license_key": LICENSE_KEY, + "requested_properties": REQUESTED, + "http_client": self.mock_http( + license_json_response='["device", "ismobile"]'), + }) + + pipeline = PipelineBuilder().add(engine).build() + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + data = pipeline.create_flowdata() + data.evidence.add("query.User-Agent", "iPhone") + data.process() + + self.assertEqual( + [], [str(warning.message) for warning in caught + if "did not return" in str(warning.message)]) + def test_no_report_when_every_property_came_back(self): """ An answer carrying every requested property raises no warning, and From 58111e7fbe07bf273868c50bf4e2c87cb89bb9be Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 19:19:17 +0000 Subject: [PATCH 63/68] Take the data update messages and levels .NET and 51degrees-node write (#98) * Align the data update service to the specification Send If-Modified-Since as an HTTP date, wait for the data update expected time before the first poll, add update_from_memory, and log the actions with the messages and levels in the Logging section of the data updates specification. * Treat a data file that is already current as a finished update in the update examples With If-Modified-Since sent as an HTTP date, the Distributor answers 304 when the file is current, which the examples reported as a failure. * Test that no update message holds the licence key, the watcher needs automatic updates, a file data file is written, and content checks carry on * Register the watched data files in the watcher tests with automatic updates on * Mask the licence key in every update message, create the watcher only with automatic updates, write update_from_memory's bytes to a data file kept on disk, and carry on without an MD5 header or with content that is not compressed * Test that a callback that raises does not stop the checks, and a connection failure keeps polling * Log a callback that raises, so the other callbacks and the next check still run * Test that the watcher refresh is routed as the polling refresh is * Route the watcher refresh through the polling refresh * Test the data update messages and levels against .NET * Align the data update messages and levels to .NET Each event in the data update service now carries the message and level of the .NET data update service, which the Errors table of data-updates.md follows. A failed connection, a 429, a 403, any other status, a failed integrity check and a failed refresh move from Warning to Error and take .NET wording. A failed write to the data file becomes a Warning and the engine is still refreshed from the data in memory. Every check is reported to the update complete callbacks once it ends. A watcher event with no newer file reports not needed, and one that arrived whilst another was being applied reports in progress. * Test that a callback's error is logged with .NET's message for any other error DataUpdateService.cs logs an error from an update complete callback with MessageAutoUpdateUnhandledError at Error level. The test now asserts that text, so it fails while Python writes its own sentence. * Test that a callback that raises stops the callbacks after it data-updates.md at e89679e9 says the callbacks after the one that raised are not called. On update polling the error is logged with the last row's message, which is .NET's MessageAutoUpdateUnhandledError, and on a programmatic check it passes to the caller. The tests cover polling, a programmatic check and an update from memory, and that polling still schedules its next check. This commit is the tests alone, so they fail. * Test that a callback that raises is logged and stops the callbacks after it data-updates.md at e89679e9 says the callbacks after the one that raised are not called, and that on update polling and the file watcher the error is logged with the last row's message, .NET's MessageAutoUpdateUnhandledError. Passing the error to the caller of a programmatic check, an update from memory or the update on start-up waits for next week, so on those routes it is logged in the same way for now. This commit is the tests alone, so they fail. * Stop at the first callback that raises, and log it with .NET's message DataUpdateService.cs wraps the callback loop in one try, so the first callback that raises stops the rest, and LogUpdateException logs it with MessageAutoUpdateUnhandledError, "An unhandled error occurred while checking for automatic updates for engine '{0}'", at Error. data-updates.md says the same. Python now does both, on every route, and the next check is still scheduled. * Test that an error from the errors table is logged as .NET logs it data-updates.md at e89679e9, lines 476 to 485, says an error from a row at Error level is logged at Error as "Error during check for data update" with the error attached, and the error carries the row's text. An error of any other kind, a callback's included, is logged with the last row's message, also with the error attached. Python's logger takes a string, so the error follows the entry text, its cause after it as "Error detail:". This commit is the tests alone, so they fail. * Log an error from the errors table as .NET does, with its cause An error from a row at Error level is now logged as "Error during check for data update", with the error after it, being the row's text and then its cause, as .NET logs a DataUpdateException with the exception attached. Any other error, a callback's included, is logged with the last row's message and the error after it. Python's logger takes a string, so the error follows as "Error detail: ", as the other Python messages put a cause. The cause of a failed connection is masked, as it can name the URL with its licence key. * Test that the error entry is written exactly as 51degrees-node writes it 51degrees-node writes an error from a row of the errors table as "Error during check for data update. Error detail: ", with ". Error detail: " after it when there is a cause, and nothing more when there is none. The entry constant now holds that text, and a new test writes out two entries in full, one without a cause and one with. This commit is the tests alone, so they fail. * Write the error entry as 51degrees-node does An error from a row of the errors table is now logged as "Error during check for data update. Error detail: ", followed by ". Error detail: " when there is a cause, which is the text 51degrees-node writes at f471d65d (checkErrorText and withCause in dataFileUpdateService.js), so the two languages' logs match character for character. * Test the start-up, refresh and integrity texts as 51degrees-node writes them 51degrees-node at f471d65d writes "Updating on start-up", as the data-updates.md table at e89679e9 does, names the one try in a refresh after a check ("... after 1 tries.") and ends a refresh from memory with a full stop, and writes the integrity row as the table does, with no space before "data update". The tests now expect those texts, so they fail. * Write the start-up, refresh and integrity texts as 51degrees-node does "Updating on start-up" takes the hyphen the data-updates.md table at e89679e9 and 51degrees-node write. A failed refresh after a check or a watched change names its one try ("... after 1 tries."), and one from update_from_memory ends with a full stop, as the table's two rows and 51degrees-node's refreshDataFile do. The integrity row has no space before "data update", as the table writes it. * Test that a failed write ends the update and that an untouched watched file is not reported A data file kept on disk is read back by its engine, so the tests now use a data file whose refresh reads the file, as the on-premise engines do. They cover a download whose write fails, a temporary copy that cannot be written, an update from memory whose write fails, and a watched file nobody has written to. * End the update when the data file cannot be written, and report only the changes the watcher finds The engine of a data file kept on disk reads that file, so letting a failed write through reported a success with the old data still in use. Both routes now end with the failure, under the errors table row for the copy that replaces the file, or the row for any other error when the temporary copy is the one that cannot be written, which is what 51degrees-node writes. The watcher looks every ten seconds where .NET and 51degrees-node are told by the operating system, so a look that finds the file untouched is not an event and is not reported. The statuses take the numbers .NET gives them. * Attach the error a callback raised to the log entry The errors table in the data updates specification says an error of any other kind is logged with the error attached, and that a callback's error is logged in the same way. __emit wrote the message and bound no variable, so the error the callback raised was lost. It is now written after the message, which is what .NET, Java, Node.js and Go each do. The new test file takes the licence header every other file in the repository carries. * Name only the languages that attach a callback's error The comment on the expected message said .NET, Node.js and Go write the error the callback raised, and Java as well. Java's data update service calls each onUpdateComplete in a bare loop with no catch (DataUpdateServiceDefault.java on its rc, the loops after the update and after the file watcher's refresh), so a callback that raises there is not caught and nothing is logged. The comment now names the three that do. * One constant for the unhandled error entry, with the cause written once The merge of the data update alignment brought a second constant for the same entry. This branch already built the text with the cause attached, so appending the cause again made the expectation read it twice. The one constant carries the whole entry, and the comment saying why the cause is attached stays with it. * Test that a callback's error is masked in the log entry * Mask every licence key in a callback's error before logging it * Pair a failed temporary copy with the status its message names The errors table pairs the any other error message with AUTO_UPDATE_UNKNOWN_ERROR, and a temporary copy that cannot be written has no row of its own, so it is that error. The test asserted the copy-over-the-existing-file status instead. * A failed temporary copy reports the unknown error status The write failure logged the any other error message and returned the copy-over-the-existing-file status, which the errors table pairs with a different message. Each branch now returns the status its own row gives. --- .../datafile_update_service.py | 202 +++++++---- .../tests/test_datafile_update_http.py | 339 +++++++++++++++--- .../tests/test_datafile_update_service.py | 14 +- 3 files changed, 428 insertions(+), 127 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py index 93568b01..7d5c6972 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py @@ -58,10 +58,20 @@ class UpdateStatus(Enum): AUTO_UPDATE_NEW_FILE_CANT_RENAME = 9 # Refreshing the engine with the new data caused an error to occur. AUTO_UPDATE_REFRESH_FAILED = 10 + # 11 is .NET's AUTO_UPDATE_NO_CONFIGURATION, which this service has no + # route to, so the two below keep the numbers .NET gives them. + # The data file is checking for updates but has no temporary file path. + AUTO_UPDATE_TEMP_PATH_NOT_SET = 12 + # Any other error during a check for an update. + AUTO_UPDATE_UNKNOWN_ERROR = 13 # The first two bytes of gzip compressed content (RFC 1952 section 2.3.1). GZIP_MAGIC = b"\x1f\x8b" +# The tries made to refresh an engine from a data file, which the message +# for a failed refresh names. +REFRESH_TRIES = 1 + class DataFileUpdateService(): @@ -94,11 +104,7 @@ def __emit(self, status, data_file): for method in self.on_complete_methods: method(status, data_file) except Exception as e: - self._log( - "error", - "An unhandled error occurred while checking for automatic " - "updates for engine '" + self._engine_type(data_file) + - "'. Error detail: " + mask_licence_keys_in_text(e)) + self._log_unhandled(data_file, e) def update_data_file(self, data_file): @@ -143,21 +149,24 @@ def update_data_file(self, data_file): "Checking for update from '" + shown_url + "' for Engine '" + engine + "'") - retry = self._retry_text(data_file) - try: response = requests.get(url, headers=headers) except requests.RequestException as e: # The message of a failed request names the URL, key included. - self._log( - "warning", - "An error occurred when connecting to " + shown_url + - " in order to check for data file updates for " + engine + - "." + retry + " Error detail: " + + self._log_check_error( + "Error accessing data update service at '" + shown_url + + "' for engine '" + engine + "'", mask_licence_keys_in_text(e)) return self._update_failed( data_file, UpdateStatus.AUTO_UPDATE_HTTPS_ERR) + if response is None: + self._log_check_error( + "No response from data update service at '" + shown_url + + "' for engine '" + engine + "'") + return self._update_failed( + data_file, UpdateStatus.AUTO_UPDATE_HTTPS_ERR) + if response.status_code == 304: self._log( "info", @@ -169,21 +178,25 @@ def update_data_file(self, data_file): if response.status_code != 200: - # Handle errors + # Each response that is not a success and not a 304 carries the + # message and status of the matching row of the errors table. if response.status_code == 429: status = UpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS + message = ("Too many requests to '" + shown_url + + "' for engine '" + engine + "'") elif response.status_code == 403: status = UpdateStatus.AUTO_UPDATE_ERR_403_FORBIDDEN + message = ("Access denied to data update service at '" + + shown_url + "' for engine '" + engine + "'") else: status = UpdateStatus.AUTO_UPDATE_HTTPS_ERR + message = ("HTTP status code '" + + str(response.status_code) + + "' from data update service at '" + shown_url + + "' for engine '" + engine + "'") - self._log( - "warning", - "An error occurred while downloading a data file update " - "for " + engine + " from " + shown_url + "." + retry + - " Error detail: HTTP status code " + - str(response.status_code)) + self._log_check_error(message) return self._update_failed(data_file, status) # Verify the MD5 hash if enabled and the answer gives one. @@ -203,12 +216,11 @@ def update_data_file(self, data_file): hash_md5.update(data) if header_md5 != hash_md5.hexdigest(): - self._log( - "warning", - "An error occurred during the integrity check of new " - "data file for " + engine + "." + retry + - " Error detail: The MD5 hash of the data does not " - "match the content-md5 header.") + self._log_check_error( + "Integrity check failed. MD5 hash in HTTP response '" + + str(header_md5) + "' for '" + engine + "'data update " + "does not match calculated hash for the downloaded " + "file '" + hash_md5.hexdigest() + "'.") return self._update_failed( data_file, UpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED) @@ -225,16 +237,18 @@ def update_data_file(self, data_file): try: data = gzip.decompress(data) except (OSError, EOFError, zlib.error) as e: - self._log( - "warning", - "An error occurred during the integrity check of new " - "data file for " + engine + "." + retry + - " Error detail: " + str(e)) + # Content that cannot be read is any other error, which + # carries the last row of the errors table. + self._log_unhandled(data_file, e) return self._update_failed( - data_file, UpdateStatus.AUTO_UPDATE_ERR_READING_STREAM) + data_file, UpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR) # Download datafile if needed, or pass through memory if data_file.download: + # The engine of a data file kept on disk reads that file, so a + # write that failed leaves it with the data it already had and + # the update ends there, as it does in .NET and + # 51degrees-node. failed = self._write_data_file(data_file, data) if failed is not None: return self._update_failed(data_file, failed) @@ -273,7 +287,8 @@ def update_from_memory(self, data_file, data): @rtype: UpdateStatus @return: AUTO_UPDATE_SUCCESS, AUTO_UPDATE_REFRESH_FAILED if the engine could not use the data (the data it had is kept), the status - of a failed write for a data file kept on disk, or + of a failed write for a data file kept on disk, which is not + refreshed because its engine reads the file, or AUTO_UPDATE_NOT_NEEDED if the data file's engine has been closed """ @@ -295,7 +310,7 @@ def update_from_memory(self, data_file, data): data_file.data = data data_file.updating = True try: - status = self._refresh(data_file) + status = self._refresh(data_file, from_file=False) finally: data_file.updating = False @@ -309,8 +324,15 @@ def update_from_memory(self, data_file, data): def _write_data_file(self, data_file, data): # Write new data to the data file's path, through a temporary copy # when the data file asks for one. None when written, or the status - # of the failure, which is logged. + # of the failure, which is logged. Every failure leaves the file on + # disk as it was, so the caller ends the update. if data_file.create_temp_copy: + if not getattr(data_file, "temp_directory", None): + self._log_check_error( + "The data file '" + str(data_file.identifier) + + "' is checking for updates but does not have a " + "temporary file path configured.") + return UpdateStatus.AUTO_UPDATE_TEMP_PATH_NOT_SET filename = os.path.join( data_file.temp_directory, data_file.identifier + str(time.time())) @@ -321,26 +343,35 @@ def _write_data_file(self, data_file, data): with open(filename, 'wb') as f: f.write(data) except OSError as e: - self._log( - "error", - "An error occurred when writing to " + filename + - ". Error detail: " + str(e)) - return UpdateStatus.AUTO_UPDATE_ERR_READING_STREAM + if filename == data_file.path: + # Writing straight to the data file is the copy that + # replaces it, so it carries that row of the errors table. + self._log_check_error( + "An error occurred when copying a data file to replace " + "the existing one at '" + str(data_file.path) + "'.", e) + return UpdateStatus.AUTO_UPDATE_NEW_FILE_CANT_RENAME + # A temporary copy that cannot be written has no row of its own, + # so it is any other error, as 51degrees-node writes it, with + # the status that row of the table gives. + self._log_unhandled(data_file, e) + return UpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR if data_file.create_temp_copy: try: shutil.move(filename, data_file.path) - except (OSError, shutil.Error): - self._log( - "error", + except (OSError, shutil.Error) as e: + self._log_check_error( "An error occurred when copying a data file to " "replace the existing one at '" + data_file.path + - "'.") + "'.", e) return UpdateStatus.AUTO_UPDATE_NEW_FILE_CANT_RENAME return None - def _refresh(self, data_file): - # Refresh the engine with the data file's new content. + def _refresh(self, data_file, from_file=True): + # Refresh the engine with the data file's new content. A refresh + # after a check or a watched change is from the data file, and one + # from update_from_memory is not, as the two rows of the errors + # table say. engine = self._engine_type(data_file) self._log( "info", @@ -348,10 +379,11 @@ def _refresh(self, data_file): try: data_file.refresh() except Exception as e: - self._log( - "error", - "An error occurred while applying a data file update to " + - engine + ". Error detail: " + str(e)) + self._log_check_error( + "An error occurred when applying a data update to engine '" + + engine + "'" + + (" after " + str(REFRESH_TRIES) + " tries." if from_file + else "."), e) return UpdateStatus.AUTO_UPDATE_REFRESH_FAILED return UpdateStatus.AUTO_UPDATE_SUCCESS @@ -368,16 +400,40 @@ def _log(self, level, message): if pipeline is not None: pipeline.log(level, message) + def _log_check_error(self, text, cause=None): + # An error from a row of the errors table at Error level is logged + # as .NET logs a DataUpdateException, under .NET's entry text with + # the error, being the row's text and then its cause, after it, in + # the words 51degrees-node writes. + self._log( + "error", + self._with_cause( + self._with_cause("Error during check for data update", text), + cause)) + + def _log_unhandled(self, data_file, cause): + # Any other error is logged with the last row's message, and the + # error after it. + self._log( + "error", + self._with_cause( + "An unhandled error occurred while checking for automatic " + "updates for engine '" + self._engine_type(data_file) + "'", + cause)) + @staticmethod - def _engine_type(data_file): - return type(getattr(data_file, "flow_element", data_file)).__name__ + def _with_cause(text, cause): + # The cause may be a caller's error or a request's, either of which + # can name the update URL, so every key in it is masked here rather + # than at each call. + if cause is None: + return text + return (text + ("" if text.endswith(".") else ".") + + " Error detail: " + mask_licence_keys_in_text(cause)) @staticmethod - def _retry_text(data_file): - # Only an automatic update is tried again. - if getattr(data_file, "auto_update", False): - return " Update will be attempted again later." - return "" + def _engine_type(data_file): + return type(getattr(data_file, "flow_element", data_file)).__name__ @staticmethod def _http_date(moment): @@ -520,12 +576,13 @@ def __is_registered(self, data_file): def track_local_data_file_changes(self, data_file): """! Check for local changes to a datafile. Used by - the file system watcher. The refresh takes the same route as the - polling one, so the attempt is logged, a failure carries the - errors table's message and level, and the result is reported to - the update complete callbacks either way. The next check is - scheduled whatever happens, so that a later file is still picked - up, until the data file is unregistered. + the file system watcher. A file that has been written to is an + event, and the refresh takes the same route as the polling one, so + the attempt is logged, a failure carries the errors table's message + and level, and the result is reported to the update complete + callbacks either way. The next check is scheduled whatever happens, + so that a later file is still picked up, until the data file is + unregistered. @type data_file: DataFile @param data_file: Datafile to check updates for """ @@ -538,12 +595,21 @@ def track_local_data_file_changes(self, data_file): # replaced, is looked for again at the next check. changed_time = None try: - # Don't change if data_file is updating - if (changed_time is not None and - changed_time > data_file.last_changed_time and - not data_file.updating): - data_file.last_changed_time = changed_time - self.__emit(self._refresh(data_file), data_file) + if (changed_time is None or + changed_time <= data_file.last_changed_time): + # A file nobody has written to is not an event, so nothing + # is reported. The watchers of .NET and 51degrees-node are + # told by the operating system and say nothing whilst a + # file is untouched, and this check stands in for that. + return + if data_file.updating: + # A newer file found whilst an update is being applied is + # reported as in progress, and the time is not taken, so + # the next check looks at the file again. + self.__emit(UpdateStatus.AUTO_UPDATE_IN_PROGRESS, data_file) + return + data_file.last_changed_time = changed_time + self.__emit(self._refresh(data_file), data_file) finally: if self.__is_registered(data_file): t = threading.Timer( diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py index a1e938eb..fa30d679 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py @@ -98,6 +98,22 @@ def refresh(self): self.refreshed.append(self.data) +class FileDataFile(HttpDataFile): + """Stands in for a data file kept on disk, whose engine reads the file + rather than the data held in memory, as the on-premise engines do.""" + + def __init__(self, path): + super(FileDataFile, self).__init__(path) + self.download = True + + def refresh(self): + if self.refresh_error: + raise self.refresh_error + with open(self.path, "rb") as file: + self.data = file.read() + self.refreshed.append(self.data) + + def response(status, content=b"new"): reply = mock.Mock() reply.status_code = status @@ -106,6 +122,12 @@ def response(status, content=b"new"): return reply +# The entry an error from a row of the errors table is logged under, with +# the error, being the row's text and then its cause, after it, as .NET logs +# a DataUpdateException. The text is the one 51degrees-node writes. +CHECK_ERROR = "Error during check for data update. Error detail: " + + class UpdateServiceTestCase(unittest.TestCase): def setUp(self): @@ -238,8 +260,8 @@ def testFailedRefreshIsReportedAndLogged(self): self.assertEqual(UpdateStatus.AUTO_UPDATE_REFRESH_FAILED, status) self.assertIn( ("error", - "An error occurred while applying a data file update to " - "FakeEngine. Error detail: boom"), + CHECK_ERROR + "An error occurred when applying a data update " + "to engine 'FakeEngine'. Error detail: boom"), self.pipeline.logs) def testFileDataFileIsWrittenThenRefreshed(self): @@ -248,30 +270,37 @@ def testFileDataFileIsWrittenThenRefreshed(self): path = os.path.join(folder, "data.dat") with open(path, "wb") as file: file.write(b"old") - self.data_file = HttpDataFile(path) - self.data_file.download = True + self.data_file = FileDataFile(path) status = self.service.update_from_memory(self.data_file, b"fresh") self.assertEqual(UpdateStatus.AUTO_UPDATE_SUCCESS, status) with open(path, "rb") as file: self.assertEqual(b"fresh", file.read()) - self.assertEqual(1, len(self.data_file.refreshed)) + self.assertEqual([b"fresh"], self.data_file.refreshed) self.assertEqual([UpdateStatus.AUTO_UPDATE_SUCCESS], self.statuses) - def testFailedWriteIsLoggedAndNotRefreshed(self): + # The engine of a data file kept on disk reads that file, so a write + # that failed leaves it with the data it had. The update ends with the + # failure rather than reporting a success the engine did not have. + def testAFailedWriteFromMemoryEndsTheUpdate(self): folder = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, folder, ignore_errors=True) path = os.path.join(folder, "missing", "data.dat") - self.data_file = HttpDataFile(path) - self.data_file.download = True + self.data_file = FileDataFile(path) status = self.service.update_from_memory(self.data_file, b"fresh") - self.assertEqual(UpdateStatus.AUTO_UPDATE_ERR_READING_STREAM, status) + self.assertEqual( + UpdateStatus.AUTO_UPDATE_NEW_FILE_CANT_RENAME, status) self.assertEqual([], self.data_file.refreshed) - errors = [message for level, message in self.pipeline.logs - if level == "error"] - self.assertEqual(1, len(errors), self.pipeline.logs) - self.assertTrue(errors[0].startswith( - "An error occurred when writing to " + path + - ". Error detail: "), errors[0]) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_NEW_FILE_CANT_RENAME], self.statuses) + self.assertEqual( + [], [message for level, message + in self.pipeline.logs if level == "warning"]) + self.assertTrue( + [message for level, message in self.pipeline.logs + if level == "error" and message.startswith( + CHECK_ERROR + "An error occurred when copying a data file " + "to replace the existing one at '" + path + "'.")], + self.pipeline.logs) def testUnregisteredDataFileIsLeftAlone(self): self.service.unregister_data_file(self.data_file) @@ -279,8 +308,8 @@ def testUnregisteredDataFileIsLeftAlone(self): self.assertEqual([], self.data_file.refreshed) -# The messages and levels are those of the Logging section of the data -# updates specification. +# The messages and levels are those of the .NET data update service, which +# the Errors table of the data updates specification follows. class LoggingTests(UpdateServiceTestCase): def levels(self): @@ -312,38 +341,66 @@ def testNotModifiedIsInformationNotWarning(self): for level in ("warn", "warning", "error"): self.assertNotIn(level, self.levels()) - def testRefusedDownloadIsWarning(self): + def testTheErrorEntryIsWrittenExactly(self): + # Written out in full, so a change to any character fails. + self.get(response(403)) + self.service.update_data_file(self.data_file) + self.get(requests.ConnectionError("boom")) + self.service.update_data_file(self.data_file) + errors = [message for level, message in self.pipeline.logs + if level == "error"] + self.assertEqual( + ["Error during check for data update. Error detail: Access " + "denied to data update service at " + "'https://example.com/update' for engine 'FakeEngine'", + "Error during check for data update. Error detail: Error " + "accessing data update service at " + "'https://example.com/update' for engine 'FakeEngine'. " + "Error detail: boom"], + errors) + + def testRefusedDownloadIsError(self): self.get(response(403)) self.service.update_data_file(self.data_file) self.assertIn( - ("warning", - "An error occurred while downloading a data file update for " - "FakeEngine from " + URL + ". Error detail: HTTP status code " - "403"), + ("error", + CHECK_ERROR + "Access denied to data update service at '" + + URL + "' for engine 'FakeEngine'"), self.pipeline.logs) self.assertEqual( [UpdateStatus.AUTO_UPDATE_ERR_403_FORBIDDEN], self.statuses) - def testAutomaticUpdateSaysItWillBeTriedAgain(self): + def testTooManyRequestsIsError(self): + self.get(response(429)) + self.service.update_data_file(self.data_file) + self.assertIn( + ("error", + CHECK_ERROR + "Too many requests to '" + URL + + "' for engine 'FakeEngine'"), + self.pipeline.logs) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS], + self.statuses) + + def testAnyOtherStatusNamesTheCode(self): self.data_file.auto_update = True self.hold_timers() self.get(response(500)) self.service.update_data_file(self.data_file) self.assertIn( - ("warning", - "An error occurred while downloading a data file update for " - "FakeEngine from " + URL + ". Update will be attempted again " - "later. Error detail: HTTP status code 500"), + ("error", + CHECK_ERROR + "HTTP status code '500' from data update service " + "at '" + URL + "' for engine 'FakeEngine'"), self.pipeline.logs) + self.assertEqual([UpdateStatus.AUTO_UPDATE_HTTPS_ERR], self.statuses) - def testConnectionFailureIsWarning(self): + def testConnectionFailureIsError(self): self.get(requests.ConnectionError("boom")) self.assertFalse(self.service.update_data_file(self.data_file)) self.assertIn( - ("warning", - "An error occurred when connecting to " + URL + " in order to " - "check for data file updates for FakeEngine. Error detail: " - "boom"), + ("error", + CHECK_ERROR + "Error accessing data update service at '" + + URL + "' for engine 'FakeEngine'. Error detail: boom"), self.pipeline.logs) self.assertEqual([UpdateStatus.AUTO_UPDATE_HTTPS_ERR], self.statuses) @@ -351,14 +408,34 @@ def testFailedRefreshIsError(self): self.get(response(200)) self.data_file.refresh_error = Exception("boom") self.service.update_data_file(self.data_file) + # A refresh after a check names the one try made, as the row for a + # refresh from the data file does. self.assertIn( ("error", - "An error occurred while applying a data file update to " - "FakeEngine. Error detail: boom"), + CHECK_ERROR + "An error occurred when applying a data update " + "to engine 'FakeEngine' after 1 tries. Error detail: boom"), self.pipeline.logs) self.assertEqual( [UpdateStatus.AUTO_UPDATE_REFRESH_FAILED], self.statuses) + def testIntegrityCheckFailureIsError(self): + reply = response(200) + reply.headers = {"content-md5": "not the hash"} + reply.iter_content = lambda size: [b"new"] + self.get(reply) + self.data_file.verify_md5 = True + self.service.update_data_file(self.data_file) + errors = [message for level, message in self.pipeline.logs + if level == "error"] + self.assertEqual(1, len(errors), self.pipeline.logs) + self.assertTrue(errors[0].startswith( + CHECK_ERROR + "Integrity check failed. MD5 hash in HTTP response " + "'not the hash' for 'FakeEngine'data update does not match " + "calculated hash for the downloaded file '"), errors[0]) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED], + self.statuses) + def testUpdateOnStartIsLogged(self): self.data_file.update_on_start = True self.data_file.updated_on_start = False @@ -395,6 +472,160 @@ def testNoFileSystemWatcherWithoutAutomaticUpdates(self): self.assertIsNone(getattr(data_file, "watcher_timer", None)) +# A data file kept on disk that cannot put the new file in place, and one +# with no temporary file path, carry .NET's message and status. +class FileReplacementTests(UpdateServiceTestCase): + + def setUp(self): + super(FileReplacementTests, self).setUp() + self.folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.folder, ignore_errors=True) + self.path = os.path.join(self.folder, "data.dat") + with open(self.path, "w") as file: + file.write("old") + self.data_file = HttpDataFile(self.path) + self.data_file.download = True + self.data_file.create_temp_copy = True + self.data_file.temp_directory = self.folder + + def testFailedCopyNamesThePath(self): + move = mock.patch.object( + shutil, "move", + mock.Mock(side_effect=OSError("in use"))) + move.start() + self.addCleanup(move.stop) + status = self.service.update_from_memory(self.data_file, b"fresh") + self.assertEqual( + UpdateStatus.AUTO_UPDATE_NEW_FILE_CANT_RENAME, status) + self.assertIn( + ("error", + CHECK_ERROR + "An error occurred when copying a data file to " + "replace the existing one at '" + self.path + + "'. Error detail: in use"), + self.pipeline.logs) + + def testNoTemporaryPathIsReported(self): + self.data_file.temp_directory = None + status = self.service.update_from_memory(self.data_file, b"fresh") + self.assertEqual( + UpdateStatus.AUTO_UPDATE_TEMP_PATH_NOT_SET, status) + self.assertIn( + ("error", + CHECK_ERROR + "The data file '" + self.data_file.identifier + + "' is checking for updates but does not have a temporary " + "file path configured."), + self.pipeline.logs) + + +# A download is written to the data file's path and the engine then reads +# that file, so a write that failed leaves the engine with the data it had +# and the update ends there, as it does in .NET and 51degrees-node. +class DownloadedFileTests(UpdateServiceTestCase): + + def setUp(self): + super(DownloadedFileTests, self).setUp() + self.folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.folder, ignore_errors=True) + self.path = os.path.join(self.folder, "data.dat") + with open(self.path, "wb") as file: + file.write(b"old") + self.data_file = FileDataFile(self.path) + + def errors(self): + return [message for level, message in self.pipeline.logs + if level == "error"] + + def testADownloadedFileIsWrittenAndReadBackByTheEngine(self): + self.get(response(200, b"new")) + self.service.update_data_file(self.data_file) + with open(self.path, "rb") as file: + self.assertEqual(b"new", file.read()) + self.assertEqual([b"new"], self.data_file.refreshed) + self.assertEqual([UpdateStatus.AUTO_UPDATE_SUCCESS], self.statuses) + + def testAFailedWriteEndsTheUpdateAndLeavesTheEngineAlone(self): + self.data_file = FileDataFile( + os.path.join(self.folder, "missing", "data.dat")) + self.get(response(200, b"new")) + self.service.update_data_file(self.data_file) + self.assertEqual([], self.data_file.refreshed) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_NEW_FILE_CANT_RENAME], self.statuses) + self.assertTrue( + [message for message in self.errors() + if message.startswith( + CHECK_ERROR + "An error occurred when copying a data file " + "to replace the existing one at '" + + self.data_file.path + "'.")], + self.pipeline.logs) + + def testAFailedTemporaryCopyIsAnyOtherError(self): + self.data_file.create_temp_copy = True + self.data_file.temp_directory = os.path.join(self.folder, "missing") + self.get(response(200, b"new")) + self.service.update_data_file(self.data_file) + self.assertEqual([], self.data_file.refreshed) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR], self.statuses) + self.assertTrue( + [message for message in self.errors() + if message.startswith( + "An unhandled error occurred while checking for automatic " + "updates for engine 'FakeEngine'.")], + self.pipeline.logs) + with open(self.path, "rb") as file: + self.assertEqual(b"old", file.read()) + + +# Every check is reported to the update complete callbacks once it ends, and +# a check that needed no update is reported as not needed, as the Reporting +# each check section says and as .NET does. The watcher's checks are the +# changes it finds, as they are in .NET and 51degrees-node, so a look that +# finds the file untouched is not a check and is not reported. +class ReportingEachCheckTests(UpdateServiceTestCase): + + def testNotModifiedIsReportedAsNotNeeded(self): + self.get(response(304)) + self.service.update_data_file(self.data_file) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_NOT_NEEDED], self.statuses) + + # A watched file nobody has written to is not an event, so the watcher + # reports nothing, as the watchers of .NET and 51degrees-node do. Both + # are told by the operating system, where this one looks every ten + # seconds, so reporting each look would tell every callback that no + # update was needed six times a minute for ever. + def testAnUnchangedWatchedFileIsNotReported(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder, ignore_errors=True) + path = os.path.join(folder, "data.dat") + with open(path, "w") as file: + file.write("x") + data_file = HttpDataFile(path) + data_file.last_changed_time = os.stat(path).st_mtime + data_file.registered = True + self.hold_timers() + self.service.track_local_data_file_changes(data_file) + self.service.track_local_data_file_changes(data_file) + self.assertEqual([], self.statuses) + self.assertEqual([], self.pipeline.logs) + + def testAWatcherEventWhilstUpdatingIsReportedAsInProgress(self): + folder = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, folder, ignore_errors=True) + path = os.path.join(folder, "data.dat") + with open(path, "w") as file: + file.write("x") + data_file = HttpDataFile(path) + data_file.last_changed_time = os.stat(path).st_mtime - 10 + data_file.registered = True + data_file.updating = True + self.hold_timers() + self.service.track_local_data_file_changes(data_file) + self.assertEqual( + [UpdateStatus.AUTO_UPDATE_IN_PROGRESS], self.statuses) + + # No message holds a licence key in full, as the Logging section of the data # updates specification and the License keys section of logging.md say. # A callback that raises stops the callbacks after it, as data-updates.md @@ -402,12 +633,11 @@ def testNoFileSystemWatcherWithoutAutomaticUpdates(self): # with the error the callback raised attached. Polling carries on. class CallbackTests(UpdateServiceTestCase): - UNHANDLED = ("An unhandled error occurred while checking for automatic " - "updates for engine 'FakeEngine'") - # The error the callback raised is attached, as the errors table asks # and as .NET, Node.js and Go each write it. - UNHANDLED_WITH_CAUSE = UNHANDLED + ". Error detail: callback failed" + UNHANDLED = ("An unhandled error occurred while checking for automatic " + "updates for engine 'FakeEngine'. Error detail: " + "callback failed") def fail_first(self): def fails(status, data_file): @@ -427,7 +657,7 @@ def testACallbackErrorOnPollingIsLoggedAndStopsTheCallbacks(self): self.service.async_update_data_file(self.data_file).join() self.assertEqual([], self.statuses) self.assertEqual(1, len(self.pending)) - self.assertEqual([self.UNHANDLED_WITH_CAUSE], self.errors()) + self.assertEqual([self.UNHANDLED], self.errors()) def testACallbackErrorOnAProgrammaticCheckIsLoggedAndStopsTheCallbacks( self): @@ -438,14 +668,14 @@ def testACallbackErrorOnAProgrammaticCheckIsLoggedAndStopsTheCallbacks( self.service.update_data_file(self.data_file) self.assertEqual([], self.statuses) self.assertEqual(1, len(self.pending)) - self.assertEqual([self.UNHANDLED_WITH_CAUSE], self.errors()) + self.assertEqual([self.UNHANDLED], self.errors()) def testACallbackErrorOnAnUpdateFromMemoryIsLoggedAndStopsTheCallbacks( self): self.fail_first() self.service.update_from_memory(self.data_file, b"fresh") self.assertEqual([], self.statuses) - self.assertEqual([self.UNHANDLED_WITH_CAUSE], self.errors()) + self.assertEqual([self.UNHANDLED], self.errors()) def testAConnectionFailureKeepsPolling(self): self.data_file.auto_update = True @@ -493,15 +723,15 @@ def testEachFailedAnswerShowsOnlyTheMaskedKey(self): status) def testFailedConnectionShowsOnlyTheMaskedKey(self): - # The message of a failed request names the URL, key included. + # The message of a failed request names the URL, masked. self.get(requests.ConnectionError("failed for url: " + KEYED_URL)) self.service.update_data_file(self.data_file) self.assertNoKey() self.assertIn( - ("warning", - "An error occurred when connecting to " + MASKED_URL + - " in order to check for data file updates for FakeEngine. " - "Error detail: failed for url: " + MASKED_URL), + ("error", + CHECK_ERROR + "Error accessing data update service at '" + + MASKED_URL + "' for engine 'FakeEngine'. Error detail: " + "failed for url: " + MASKED_URL), self.pipeline.logs) @@ -536,7 +766,7 @@ def testContentThatIsNotCompressedIsUsedAsItIs(self): self.assertEqual([b"plain"], self.data_file.refreshed) self.assertEqual([UpdateStatus.AUTO_UPDATE_SUCCESS], self.statuses) - def testCorruptCompressedContentIsWarnedAndCheckedAgainLater(self): + def testCorruptCompressedContentIsAnErrorAndCheckedAgainLater(self): self.data_file.decompress = True self.data_file.auto_update = True self.hold_timers() @@ -544,14 +774,13 @@ def testCorruptCompressedContentIsWarnedAndCheckedAgainLater(self): self.assertFalse(self.service.update_data_file(self.data_file)) self.assertEqual([], self.data_file.refreshed) self.assertEqual( - [UpdateStatus.AUTO_UPDATE_ERR_READING_STREAM], self.statuses) - warnings = [message for level, message in self.pipeline.logs - if level == "warning"] - self.assertEqual(1, len(warnings), self.pipeline.logs) - self.assertTrue(warnings[0].startswith( - "An error occurred during the integrity check of new data file " - "for FakeEngine. Update will be attempted again later. Error " - "detail: "), warnings[0]) + [UpdateStatus.AUTO_UPDATE_UNKNOWN_ERROR], self.statuses) + errors = [message for level, message in self.pipeline.logs + if level == "error"] + self.assertEqual(1, len(errors), self.pipeline.logs) + self.assertTrue(errors[0].startswith( + "An unhandled error occurred while checking for automatic " + "updates for engine 'FakeEngine'. Error detail: "), errors[0]) self.assertEqual(1, len(self.pending)) def testFailedMd5CheckOnStartUpIsNotRepeatedAtOnce(self): diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py index 6cbe5f29..96dd178f 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py @@ -137,10 +137,14 @@ def testFileSystemWatcherKeepsWatchingAfterFailedRefresh(self): self.run_next_check() self.assertEqual("version 1", data_file.data) self.assertEqual( - [UpdateStatus.AUTO_UPDATE_REFRESH_FAILED], self.statuses) + UpdateStatus.AUTO_UPDATE_REFRESH_FAILED, self.statuses[-1]) errors = [log for log in self.pipeline.logs if log[0] == "error"] self.assertEqual(1, len(errors)) - self.assertIn("The data file cannot be used.", errors[0][1]) + self.assertIn( + "Error during check for data update. Error detail: An error " + "occurred when applying a data update to engine 'StandInDataFile' " + "after 1 tries.", + errors[0][1]) self.assertEqual(1, len(self.pending)) self.write("version 2") @@ -158,7 +162,7 @@ def testFileSystemWatcherRefreshIsRoutedAsPollingIs(self): self.run_next_check() self.assertEqual("version 2", data_file.data) - self.assertEqual([UpdateStatus.AUTO_UPDATE_SUCCESS], self.statuses) + self.assertEqual(UpdateStatus.AUTO_UPDATE_SUCCESS, self.statuses[-1]) attempts = [log for log in self.pipeline.logs if "Attempting to refresh Engine" in log[1]] self.assertEqual(1, len(attempts)) @@ -176,7 +180,9 @@ def testFileSystemWatcherFailedRefreshUsesTheTableMessage(self): errors = [log for log in self.pipeline.logs if log[0] == "error"] self.assertEqual(1, len(errors)) self.assertIn( - "An error occurred while applying a data file update to", + "Error during check for data update. Error detail: An error " + "occurred when applying a data update to engine 'StandInDataFile' " + "after 1 tries.", errors[0][1]) # A file that is missing at a check, for example part way through being From 5b9fd55bcebd3a5bb16ca99002dff81bd1af41cf Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 19:34:47 +0000 Subject: [PATCH 64/68] Take the session id and the sequence from the Sequence Element, and render 1 and no session id without one (#94) * Test that the JavaScript Builder renders sequence 1 and no session id with no Sequence Element * Render sequence 1 and an empty session id with no Sequence Element, as step 5 says * Pin both sides of the branch, and say in the older tests where the no Sequence Element case is covered * Take the session id and the sequence from the Sequence Element Step 5 of the JavaScript Builder's process reads both values from the Sequence Element's Element Data, and renders 1 and an empty session id where there is no Sequence Element. This draft had the second half and read both values from Evidence, which is the half PHP, .NET and Java already have. The Sequence Element now publishes the pair it writes to Evidence as its own Element Data, and the builder reads it from there. The Evidence writes stay, because the JSON Builder and the Sequence Element's own evidence key filter read them. The case-folded Evidence lookup the builder used for these two keys goes with them, having no other reader. A key given in other letters is still read, because Evidence.get folds the key it is given and the Sequence Element reads through it. --- .../javascriptbuilder.py | 54 ++--- .../test_javascriptbuilder_no_sequence.py | 190 ++++++++++++++++++ .../tests/test_javascriptbuilder_sequence.py | 8 +- 3 files changed, 223 insertions(+), 29 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_no_sequence.py diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py index 7f7bc795..8a3d8a0c 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py @@ -51,6 +51,13 @@ # reference for this behaviour. EXCLUDED_PARAMETERS = ["query.session-id", "query.sequence"] +# The data key of the SequenceElement, which is the element that produces the +# session id and the sequence, and the two names its Element Data holds them +# under. A pipeline without that element renders the defaults below. +SEQUENCE_ELEMENT_DATA_KEY = "sequence" +SESSION_ID_NAME = "session-id" +SEQUENCE_NAME = "sequence" + # The object name is written into the script as the name of a global # variable, a session storage key and a property name, all without any # escaping. A name that is not a plain JavaScript identifier would therefore @@ -135,7 +142,7 @@ def parse_sequence(value): A sequence as a positive 32 bit integer. @type value: object - @param value: The query.sequence evidence, or None + @param value: A sequence value, from Evidence or Element Data, or None @rtype: int @return: The sequence, or None when the value is not a whole number from 1 to 2147483647. Any value at all can be passed, including one a page @@ -172,13 +179,11 @@ def get_sequence(value): The template writes the sequence as bare code (var sequence = ...;), so anything other than a whole number would stop the script parsing, or - change what it does. A pipeline without a SequenceElement passes the - query string's value, or nothing at all, straight through. As the - pipeline specification says, a value that is not a positive 32 bit - integer becomes 1. + change what it does. As the pipeline specification says, a value that is + not a positive 32 bit integer becomes 1. @type value: object - @param value: The query.sequence evidence, or None + @param value: The SequenceElement's sequence, or None @rtype: int @return: The sequence to render """ @@ -194,11 +199,10 @@ def get_session_id(value): The template writes the session id inside quotes without any escaping. As the pipeline specification says, a session id that is not 1 to 64 - ASCII letters, digits and hyphens is rendered as an empty string, whether - it came from the SequenceElement or from the query string. + ASCII letters, digits and hyphens is rendered as an empty string. @type value: object - @param value: The query.session-id evidence, or None + @param value: The SequenceElement's session id, or None @rtype: str @return: The session id to render, or an empty string """ @@ -422,22 +426,22 @@ def process_internal(self, flowdata): + self.settings["_objName"] + "' is used.") query_params = self.get_evidence_key_filter().filter_evidence(flowdata.evidence.get_all()) - # Evidence keys ignore letter case, and get_all gives each key in - # the letters it was added with, so a key is matched here in lower - # case and keeps those letters where it is sent on. - folded = {str(key).lower(): key for key in query_params} - - def query_param(name): - key = folded.get(name) - return None if key is None else query_params[key] - - # Both are written into the script, so each always has a safe value. - # The session id is written inside quotes and is empty when absent or - # not safe, and the sequence is written as bare code, so it is always - # a positive number. - variables["_sessionId"] = get_session_id( - query_param("query.session-id")) - variables["_sequence"] = get_sequence(query_param("query.sequence")) + + # Both come from the Sequence Element's Element Data, and with no + # Sequence Element in the pipeline the sequence is 1 and the session + # id empty, whatever the request asked for. Both are written into the + # script, so each always has a safe value. The session id is written + # inside quotes and is empty when absent or not safe, and the sequence + # is written as bare code, so it is always a positive number. + sequence_data = flowdata.data.get(SEQUENCE_ELEMENT_DATA_KEY) + if sequence_data is None: + variables["_sessionId"] = "" + variables["_sequence"] = DEFAULT_SEQUENCE + else: + variables["_sessionId"] = get_session_id( + sequence_data.get(SESSION_ID_NAME)) + variables["_sequence"] = get_sequence( + sequence_data.get(SEQUENCE_NAME)) # The session id and the sequence are left out, because the script # appends both to its own request after it has taken the record of diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_no_sequence.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_no_sequence.py new file mode 100644 index 00000000..82917e2b --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_no_sequence.py @@ -0,0 +1,190 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Step 5 of the JavaScript Builder specification's process takes the session +id and the sequence from the Sequence Element's Element Data, and where there +is no Sequence Element renders the sequence as 1 and the session id as an +empty string, whatever the evidence holds. Both sides of that branch are +tested here, and so is the source the values are read from, because the +builder used to read them from Evidence instead.""" + +import re +import unittest + +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.javascriptbuilder import JavascriptBuilderElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_pipeline_core.sequenceelement import SequenceElement + + +class StandInJsonBuilder(FlowElement): + """Gives the JavaScript Builder its JSON without needing a sequence.""" + + def __init__(self): + super(StandInJsonBuilder, self).__init__() + self.datakey = "jsonbundler" + self.properties = {"json": {"type": "dict"}} + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary( + self, {"json": {"javascriptProperties": []}})) + + +class StandInSequenceElement(FlowElement): + """Takes the place of the SequenceElement, writing one pair into its + Element Data and a different pair into Evidence, so that a test can tell + which of the two the builder read. Element Data of None stands for an + element that is in the pipeline and produced nothing.""" + + def __init__(self, contents): + super(StandInSequenceElement, self).__init__() + self.datakey = "sequence" + self.contents = contents + + def process_internal(self, flowdata): + flowdata.evidence.add("query.session-id", "from-evidence") + flowdata.evidence.add("query.sequence", 99) + if self.contents is not None: + flowdata.set_element_data( + ElementDataDictionary(self, self.contents)) + + +def render(evidence, sequence_element=False, stand_in=None): + """Render the script from a pipeline that differs only in which Sequence + Element it holds, if any. The real JSON bundler refuses a pipeline with no + sequence number, so a stand-in gives the builder its JSON.""" + + builder = PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}) + if sequence_element: + builder = builder.add(SequenceElement()) + if stand_in is not None: + builder = builder.add(stand_in) + pipeline = builder \ + .add(StandInJsonBuilder()) \ + .add(JavascriptBuilderElement({"minify": False})) \ + .build() + flowdata = pipeline.create_flowdata() + for key, value in evidence.items(): + flowdata.evidence.add(key, value) + flowdata.process() + return flowdata.javascriptbuilder.javascript + + +def line(script, name): + return [text.strip() for text in script.splitlines() + if re.match(r"\s*var " + name + r"\s*=", text)] + + +class NoSequenceElementTests(unittest.TestCase): + + def test_the_sequence_is_one_and_the_session_id_empty(self): + script = render({"query.sequence": "5", + "query.session-id": "abc-123"}) + self.assertEqual(["var sequence = 1;"], line(script, "sequence")) + self.assertEqual(['var sessionId = "";'], line(script, "sessionId")) + + def test_an_empty_request_gives_the_same_defaults(self): + script = render({}) + self.assertEqual(["var sequence = 1;"], line(script, "sequence")) + self.assertEqual(['var sessionId = "";'], line(script, "sessionId")) + + def test_a_session_id_the_builder_would_accept_is_still_left_out(self): + """A session id that passes the builder's own check is rendered + empty, so the empty string comes from there being no Sequence + Element rather than from that check.""" + + script = render({"query.session-id": "a" * 64}) + self.assertEqual(['var sessionId = "";'], line(script, "sessionId")) + + def test_with_a_sequence_element_its_values_are_rendered(self): + """The same pipeline with a SequenceElement renders what that element + produced, which is the request's sequence counted on by one and the + session id it was given.""" + + script = render({"query.sequence": "5", + "query.session-id": "abc-123"}, + sequence_element=True) + self.assertEqual(["var sequence = 6;"], line(script, "sequence")) + self.assertEqual( + ['var sessionId = "abc-123";'], line(script, "sessionId")) + + +class WhereTheValuesComeFromTests(unittest.TestCase): + + def test_the_element_data_is_read_and_the_evidence_is_not(self): + """The stand-in puts one pair in its Element Data and a different + pair in Evidence. What is rendered says which of the two was read.""" + + script = render({}, stand_in=StandInSequenceElement( + {"session-id": "from-element-data", "sequence": 7})) + self.assertEqual(["var sequence = 7;"], line(script, "sequence")) + self.assertEqual( + ['var sessionId = "from-element-data";'], + line(script, "sessionId")) + + def test_an_element_that_produced_no_element_data_gives_the_defaults(self): + """A Sequence Element that is in the pipeline but set no Element + Data, because it stopped or failed, is the same as no Sequence + Element at all, so the script still parses.""" + + script = render({}, stand_in=StandInSequenceElement(None)) + self.assertEqual(["var sequence = 1;"], line(script, "sequence")) + self.assertEqual(['var sessionId = "";'], line(script, "sessionId")) + + def test_an_unusable_pair_in_the_element_data_is_rendered_safely(self): + """The session id is written inside quotes and the sequence as bare + code, so a session id that does not match the pattern is rendered + empty and a sequence that is not a positive 32 bit integer is 1.""" + + script = render({}, stand_in=StandInSequenceElement( + {"session-id": 'a" + alert(1) + "', "sequence": "not a number"})) + self.assertEqual(["var sequence = 1;"], line(script, "sequence")) + self.assertEqual(['var sessionId = "";'], line(script, "sessionId")) + + def test_the_sequence_element_publishes_the_pair_it_wrote(self): + """The real SequenceElement's Element Data holds the same session id + and sequence it put in Evidence, which is what the builder reads.""" + + builder = PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}) + pipeline = builder.add(SequenceElement()).build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.session-id", "abc-123") + flowdata.evidence.add("query.sequence", "5") + flowdata.process() + + self.assertEqual("abc-123", flowdata.sequence.get("session-id")) + self.assertEqual(6, flowdata.sequence.get("sequence")) + self.assertEqual( + flowdata.evidence.get("query.session-id"), + flowdata.sequence.get("session-id")) + self.assertEqual( + flowdata.evidence.get("query.sequence"), + flowdata.sequence.get("sequence")) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py index cc7d03e8..8b5e1f18 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py @@ -22,10 +22,10 @@ """ The script must parse whatever the pipeline holds. The sequence is written -into the script as bare code and the session id inside quotes, and a pipeline -built without a SequenceElement, for example from a configuration file that -does not list one, passes the query string's values, or nothing at all, -straight to the builder. +into the script as bare code and the session id inside quotes, and both come +from the SequenceElement, so every pipeline here has one. A pipeline without +a SequenceElement renders sequence 1 and an empty session id, which +test_javascriptbuilder_no_sequence.py covers. The script is checked with "node --check", which needs Node on the path. Every GitHub hosted runner has it. From 126d373304633565c94d41d9c798fefc39d01965 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 19:59:35 +0000 Subject: [PATCH 65/68] Keep reading every renamed environment variable under its old name, deprecated, starting with FOD_CLOUD_API_URL (#108) * Test that the old cloud endpoint variable name is still read The cloud request engine, the 51Did client and the 51Did creator context example read the cloud endpoint from _51DEGREES_CLOUD_ENDPOINT first and, when that is unset or empty, from FOD_CLOUD_API_URL, reporting once that the old name is deprecated. Constants.FOD_CLOUD_API_URL and did_client.ENDPOINT_VARIABLE hold "FOD_CLOUD_API_URL" again and warn that they are deprecated when read. The environment variable name check leaves the old endpoint name alone, so its own tests move those lines from the names it finds to the names it leaves. Tests only. The code follows in the next commit. * Read FOD_CLOUD_API_URL when _51DEGREES_CLOUD_ENDPOINT is not set, and bring back the constants that held the old name The cloud request engine, the 51Did client and the 51Did creator context example read the cloud endpoint from _51DEGREES_CLOUD_ENDPOINT first. When that is unset or empty they read FOD_CLOUD_API_URL, the name the variable had in 4.5.43, and warn once with a FutureWarning that the old name is deprecated and which name replaces it. When both are set the new name wins and nothing is reported. Constants.FOD_CLOUD_API_URL is back in the cloud request engine, holding "FOD_CLOUD_API_URL", and warns with a DeprecationWarning naming Constants.CLOUD_ENDPOINT when read. did_client.ENDPOINT_VARIABLE holds "FOD_CLOUD_API_URL" again and warns in the same way, naming the new did_client.CLOUD_ENDPOINT_VARIABLE. The environment variable name check no longer fails on the old endpoint name, because the name is read again. * Test that the old 51Did endpoint constant is a plain module attribute did_client.ENDPOINT_VARIABLE has to be found every way it was before, so the tests check dir(), the module's dictionary and a star import as well as a read by name, and that the client reads the old variable through the constant, as it did in 4.5.43. Tests only. The code follows in the next commit. * Make the old 51Did endpoint constant a plain module attribute again did_client.ENDPOINT_VARIABLE was served from the module's __getattr__ so that reading it warned, which took it out of the module's dictionary, so dir() and a star import no longer gave it. It is a plain attribute again, holding "FOD_CLOUD_API_URL" and marked deprecated in its comment, which points at CLOUD_ENDPOINT_VARIABLE. The client reads the old variable through the constant, as it did in 4.5.43. * Test that every renamed environment variable is still read under its old name Every environment variable name a package, a test, an example or a CI script reads is a public name, so a name that changed to its _51DEGREES_ form is still read under the old name when the new one is unset or empty, with one message that the old name is deprecated. The tests cover the helper each test folder holds (live_cloud.py), the three example helpers and the constants they hold, the two Location cloud examples, the 51Did creator context example's two keys, the IP Intelligence test data file, and the upper case read in the two key helpers. The name check's tests say an old name is left alone in a deprecated read and in the files that may name one. Tests only. The code follows in the next commit. * Read every renamed environment variable under its old name when the new one is not set resource_key, license_key, cloud_endpoint and cloud_request_origin read as variables, 51DEGREES_DD_PATH, IPINTELLIGENCEDATAFILE and run_performance_tests are read again by the tests and examples that read them in 4.5.43, each only when its _51DEGREES_ name is unset or empty, and each printing once that the old name is deprecated. RESOURCE_KEY and LICENSE_KEY are read again by the 51Did creator context example. KeyUtils.get_named_key in fiftyone_devicedetection_shared and fiftyone_ipintelligence_shared reads the upper case form of a name again when the name as given is not set, with a FutureWarning. The constants in the example helpers that held an old name hold it again, marked deprecated, beside a constant for each new name. Each tox.ini passes the old names to the tests beside the new ones. The environment variable name check leaves an old name alone where it is the argument straight after its _51DEGREES_ name, and in the files that hold the deprecated constants, their tests and the tox lists. * Test the upper case reads in the IP Intelligence example helper, a switch read by presence, and the name check file by file The IP Intelligence example helper read its resource key and its cloud endpoint under the old names in upper case as well, so a test sets only RESOURCE_KEY and only CLOUD_ENDPOINT. A switch such as the one that turns the performance examples on is on when its variable is set, whatever it holds, so the test helper gains tests of that under both names. The name check's tests say a listed file may hold the old names it is listed with and no others, and that any other name starting with a digit still fails there. Tests only. The code follows in the next commit. * Read the IP Intelligence examples' old names in upper case, keep the performance switch a presence test, and allow each file only its own old names The IP Intelligence example helper reads RESOURCE_KEY and CLOUD_ENDPOINT again, after resource_key and cloud_endpoint, as it did in 4.5.43. The test helper live_cloud.py gains is_set, which says whether a variable is set under its new name or its old one, whatever it holds. The performance examples' switch uses it, so setting the variable to an empty value still turns them on. The environment variable name check no longer leaves a listed file out whole. Each file is listed with the old names it may hold, those names alone are left alone there, and every other old name and every other name that starts with a digit still fails. * Test that the engine reads the old endpoint variable through its constant Constants.FOD_CLOUD_API_URL has to be a plain attribute of its class, so every way of reading it gives the old name, and the engine has to read the old variable through it, as it did in 4.5.43, so that code which points the constant at another variable still decides what is read. The tests that reset a record of what has been reported no longer create the attribute when it is missing, so a rename of one would fail the tests. Tests only. The code follows in the next commit. * Make the engine's old endpoint constant a plain attribute that the engine reads Constants.FOD_CLOUD_API_URL was a descriptor that warned when read, and the engine read the old name from a private copy. So the class dictionary held an object where 4.5.43 held the name, and pointing the constant at another variable no longer changed what the engine read. It is a plain attribute again, holding "FOD_CLOUD_API_URL" and marked deprecated in its comment, which points at Constants.CLOUD_ENDPOINT, and the engine reads the old variable through it. The 51Did client already does the same. --- ci/check-environment-variable-names.Tests.ps1 | 88 ++++++++- ci/check-environment-variable-names.ps1 | 149 ++++++++++++-- .../tests/live_cloud.py | 62 +++++- .../fiftyone_devicedetection_cloud/tox.ini | 2 + .../example_utils.py | 82 +++++++- .../onpremise/datafileupdate_console.py | 11 +- .../onpremise/gettingstarted_web/app.py | 4 +- .../tests/live_cloud.py | 62 +++++- .../tests/test_environment_names.py | 162 +++++++++++++++ .../tests/test_examples_without_settings.py | 2 +- .../tests/test_onpremiseexamples.py | 14 +- .../tests/test_onpremisegettingstartedweb.py | 6 +- .../fiftyone_devicedetection_examples/tox.ini | 3 + .../tox.ini | 2 + .../key_utils.py | 25 ++- .../fiftyone_ipintelligence_cloud/tox.ini | 2 + .../cloud/getallproperties_console.py | 2 +- .../cloud/gettingstarted_console.py | 2 +- .../cloud/gettingstarted_web/app.py | 2 +- .../cloud/metadata_console.py | 2 +- .../cloud/mixed_console.py | 2 +- .../example_utils.py | 91 ++++++++- .../onpremise/datafileupdate_console.py | 10 +- .../tests/live_cloud.py | 62 +++++- .../tests/test_environment_names.py | 174 +++++++++++++++++ .../tests/test_examples_without_settings.py | 4 +- .../fiftyone_ipintelligence_examples/tox.ini | 3 + .../tests/constants.py | 8 +- .../tests/live_cloud.py | 139 +++++++++++++ .../tests/test_environment_names.py | 66 +++++++ .../fiftyone_ipintelligence_onpremise/tox.ini | 3 + .../key_utils.py | 25 ++- .../tests/test_shared.py | 18 ++ .../fiftyone_ipintelligence_shared/tox.ini | 2 + .../tox.ini | 2 + location/examples/cloud/gettingstarted.py | 30 ++- location/examples/cloud/web.py | 30 ++- location/tests/live_cloud.py | 139 +++++++++++++ location/tests/test_environment_names.py | 114 +++++++++++ location/tests/test_examples.py | 4 +- location/tests/test_properties51D.py | 6 +- location/tox.ini | 2 + .../example_utils.py | 64 +++++- .../examples/tests/live_cloud.py | 62 +++++- .../examples/tests/test_example_utils.py | 64 ++++++ .../examples/tox.ini | 5 + .../readme.md | 6 +- .../cloudrequestengine.py | 52 ++++- .../constants.py | 15 ++ .../tests/live_cloud.py | 139 +++++++++++++ .../tests/test_cloudengine.py | 5 +- .../tests/test_cloudrequestengine.py | 163 +++++++++++++++- .../tests/test_licensekey_live.py | 5 +- .../tox.ini | 2 + pipeline/fiftyone_pipeline_core/tox.ini | 2 + .../examples/creator_context_web/server.py | 36 +++- pipeline/fiftyone_pipeline_did/readme.md | 4 +- .../src/fiftyone_pipeline_did/did_client.py | 59 +++++- .../fiftyone_pipeline_did/tests/live_cloud.py | 62 +++++- .../tests/test_creator_context_server.py | 122 ++++++++++++ .../tests/test_did_client.py | 135 +++++++++++-- .../tests/test_did_client_live.py | 5 +- .../tests/test_live_cloud.py | 184 ++++++++++++++++++ pipeline/fiftyone_pipeline_did/tox.ini | 3 + pipeline/fiftyone_pipeline_engines/tox.ini | 2 + .../tox.ini | 2 + 66 files changed, 2664 insertions(+), 152 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_examples/tests/test_environment_names.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/tests/test_environment_names.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/live_cloud.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_environment_names.py create mode 100644 location/tests/live_cloud.py create mode 100644 location/tests/test_environment_names.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/live_cloud.py create mode 100644 pipeline/fiftyone_pipeline_did/tests/test_live_cloud.py diff --git a/ci/check-environment-variable-names.Tests.ps1 b/ci/check-environment-variable-names.Tests.ps1 index 7ba15f49..f70f1f79 100644 --- a/ci/check-environment-variable-names.Tests.ps1 +++ b/ci/check-environment-variable-names.Tests.ps1 @@ -31,12 +31,12 @@ Describe 'Find-EnvironmentNameProblem' { @{ Line = 'if "resource_key" in os.environ:' } @{ Line = '$env:resource_key = "key"' } @{ Line = '[Environment]::SetEnvironmentVariable("license_key", $k)' } - @{ Line = 'RESOURCE = env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY")' } - @{ Line = 'LICENCE = env("_51DEGREES_LICENSE_KEY", "license_key")' } @{ Line = 'LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key"' } - @{ Line = 'legacy_license_key_env_var = "x"' } @{ Line = 'self.baseURL = os.environ.get("FOD_CLOUD_API_URL")' } @{ Line = 'FOD_CLOUD_API_URL = "FOD_CLOUD_API_URL"' } + @{ Line = 'saved = os.environ.pop("FOD_CLOUD_API_URL", None)' } + @{ Line = '$env:FOD_CLOUD_API_URL = "https://cloud.51degrees.com/api/v4/"' } + @{ Line = 'key = env("_51DEGREES_RESOURCE_KEY", "resource_key") or os.environ.get("license_key")' } @{ Line = 'RESOURCE_KEY_ENV_VAR = "resource_key"' } @{ Line = 'ENDPOINT_ENV_VAR = "cloud_endpoint"' } @{ Line = 'ORIGIN_ENV_VAR = "cloud_request_origin"' } @@ -49,7 +49,6 @@ Describe 'Find-EnvironmentNameProblem' { @{ Line = 'env = os.environ.get("IPINTELLIGENCEDATAFILE")' } @{ Line = 'os.environ["ipintelligencetestkey"] = "value"' } @{ Line = 'if "run_performance_tests" in os.environ:' } - @{ Line = 'saved = os.environ.pop("FOD_CLOUD_API_URL", None)' } ) { Get-Kind $Line | Should -Contain 'fallback' } @@ -82,4 +81,85 @@ Describe 'Find-EnvironmentNameProblem' { ) { Get-Kind $Line | Should -BeNullOrEmpty } + + # A variable whose name changed is still read under its old name when + # the new one is not set. That read gives the old name as the argument + # straight after the new one, and it is left alone. + It 'leaves the deprecated read alone' -ForEach @( + @{ Line = 'RESOURCE = env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY")' } + @{ Line = 'LICENCE = env("_51DEGREES_LICENSE_KEY", "license_key")' } + @{ Line = 'API = env("_51DEGREES_CLOUD_ENDPOINT", "FOD_CLOUD_API_URL")' } + @{ Line = 'read_environment("_51DEGREES_IPI_PATH", "IPINTELLIGENCEDATAFILE")' } + @{ Line = ' "_51DEGREES_RUN_PERFORMANCE_TESTS", "run_performance_tests")' } + @{ Line = "read_environment('_51DEGREES_DD_PATH', '51DEGREES_DD_PATH')" } + ) { + Get-Kind $Line | Should -BeNullOrEmpty + } +} + +Describe 'Find-EnvironmentNameProblem with the old names a file may hold' { + + It 'leaves alone when the file may hold ' -ForEach @( + @{ Line = 'ENDPOINT_ENV_VAR = "cloud_endpoint"'; Allowed = @('cloud_endpoint') } + @{ Line = 'os.environ["resource_key"] = "old-key"'; Allowed = @('resource_key', 'license_key') } + @{ Line = 'DATA_FILE_ENV_VAR = "51DEGREES_DD_PATH"'; Allowed = @('51DEGREES_DD_PATH') } + @{ Line = ' FOD_CLOUD_API_URL'; Allowed = @('FOD_CLOUD_API_URL') } + @{ Line = 'os.environ["RESOURCE_KEY"] = "old-resource-key"'; Allowed = @('RESOURCE_KEY') } + ) { + @(Find-EnvironmentNameProblem $Line $Allowed) | Should -BeNullOrEmpty + } + + It 'finds in when the file may hold only ' -ForEach @( + @{ Line = 'ENDPOINT_ENV_VAR = "cloud_endpoint"'; Allowed = @('resource_key'); Kind = 'fallback' } + @{ Line = 'os.environ["license_key"] = "old-key"'; Allowed = @('resource_key'); Kind = 'fallback' } + @{ Line = 'os.environ["RESOURCE_KEY"] = "old-key"'; Allowed = @('resource_key'); Kind = 'fallback' } + @{ Line = ' IPINTELLIGENCEDATAFILE'; Allowed = @('FOD_CLOUD_API_URL'); Kind = 'fallback' } + @{ Line = 'DATA_FILE_ENV_VAR = "51DEGREES_IPI_PATH"'; Allowed = @('51DEGREES_DD_PATH'); Kind = 'digit' } + @{ Line = ' 51DEGREES_DD_PATH_OTHER'; Allowed = @('51DEGREES_DD_PATH'); Kind = 'digit' } + @{ Line = 'DATA_FILE_ENV_VAR = "51DEGREES_DD_PATH"'; Allowed = @(); Kind = 'digit' } + ) { + @(Find-EnvironmentNameProblem $Line $Allowed | ForEach-Object { $_.Kind }) | + Should -Contain $Kind + } +} + +Describe 'Get-FallbackName' { + + It 'lets hold ' -ForEach @( + @{ Path = 'pipeline/fiftyone_pipeline_did/tox.ini'; Name = 'FOD_CLOUD_API_URL' } + @{ Path = 'device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py'; Name = '51DEGREES_DD_PATH' } + @{ Path = 'device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py'; Name = 'cloud_endpoint' } + @{ Path = 'ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py'; Name = 'IPINTELLIGENCEDATAFILE' } + @{ Path = 'pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py'; Name = 'FOD_CLOUD_API_URL' } + @{ Path = 'pipeline/fiftyone_pipeline_cloudrequestengine/readme.md'; Name = 'FOD_CLOUD_API_URL' } + @{ Path = 'pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py'; Name = 'FOD_CLOUD_API_URL' } + @{ Path = 'pipeline/fiftyone_pipeline_did/tests/test_live_cloud.py'; Name = 'resource_key' } + ) { + @(Get-FallbackName $Path) | Should -Contain $Name + } + + It 'does not let hold ' -ForEach @( + @{ Path = 'pipeline/fiftyone_pipeline_did/tox.ini'; Name = '51DEGREES_DD_PATH' } + @{ Path = 'pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py'; Name = 'resource_key' } + @{ Path = 'pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py'; Name = '51DEGREES_DD_PATH' } + @{ Path = 'ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py'; Name = '51DEGREES_DD_PATH' } + @{ Path = 'device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py'; Name = 'IPINTELLIGENCEDATAFILE' } + ) { + @(Get-FallbackName $Path) | Should -Not -Contain $Name + } + + It 'lets hold no old name' -ForEach @( + @{ Path = 'ci/run-unit-tests.ps1' } + @{ Path = 'readme.md' } + @{ Path = 'pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py' } + @{ Path = 'pipeline/fiftyone_pipeline_core/readme.md' } + @{ Path = 'device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py' } + @{ Path = 'device-detection/ci/run-integration-tests.ps1' } + @{ Path = 'pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py' } + @{ Path = 'location/examples/cloud/gettingstarted.py' } + @{ Path = 'some/new/package/tox.ini' } + @{ Path = 'some/new/example_utils.py' } + ) { + @(Get-FallbackName $Path) | Should -BeNullOrEmpty + } } diff --git a/ci/check-environment-variable-names.ps1 b/ci/check-environment-variable-names.ps1 index 6c9766d2..5283cd81 100644 --- a/ci/check-environment-variable-names.ps1 +++ b/ci/check-environment-variable-names.ps1 @@ -1,8 +1,15 @@ # Every 51Degrees environment variable name in this repository starts with an # underscore, because a POSIX shell cannot set a name that starts with a digit, -# and each variable has one name with nothing read in its place. This reads -# every file git tracks, which leaves out build output and the submodules, and -# fails on any name that starts with the digit and on any fallback name. +# and everything documents that one name. This reads every file git tracks, +# which leaves out build output and the submodules, and fails on any name +# that starts with the digit and on any old name. +# +# A variable whose name changed is still read under its old name when the new +# one is not set, which is deprecated. So an old name is left alone in the two +# places that read needs it. One is a line where the old name is the argument +# straight after its _51DEGREES_ name, which is how each deprecated read is +# written. The other is the files $FallbackNames lists, each of which may hold +# the old names it is listed with and no others. # # Names are matched in any case, because Windows reads an environment # variable name in any case, so a lower case old name is still the old name @@ -31,9 +38,8 @@ $script:DigitInitialPattern = [regex]::new( "(?") CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +def _report_old_name(name, old_name): + """Prints, once for each old name, that it is deprecated and which + name replaces it.""" + if old_name in _reported_old_names: + return + _reported_old_names.add(old_name) + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + + +def read_environment(name, old_name): + """The value of the environment variable name, or None when it has + none. When name is unset or empty the name the variable had before is + read, and using it prints once that the old name is deprecated.""" + value = os.environ.get(name, "").strip() + if value: + return value + value = os.environ.get(old_name, "").strip() + if not value: + return None + _report_old_name(name, old_name) + return value + + +def is_set(name, old_name): + """Whether the environment variable name is set, whatever it holds, + which is how a switch is read. When it is not set, the name the + variable had before counts, and using it prints once that the old name + is deprecated.""" + if name in os.environ: + return True + if old_name not in os.environ: + return False + _report_old_name(name, old_name) + return True + + def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None.""" + """The first resource key set under any of the names, or None. When + none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" for name in names: value = os.environ.get(name, "").strip() if value: return value + if names is RESOURCE_KEY_NAMES: + return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/device-detection/fiftyone_devicedetection_cloud/tox.ini b/device-detection/fiftyone_devicedetection_cloud/tox.ini index 27672a13..28643c83 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tox.ini +++ b/device-detection/fiftyone_devicedetection_cloud/tox.ini @@ -25,8 +25,10 @@ commands = # which leaves the live tests out of the unit step. pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_LICENSE_KEY + license_key PYTEST_ADDOPTS [testenv:pre-publish] diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py index 873f6568..98f38b8d 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py @@ -26,6 +26,7 @@ import shlex import shutil import subprocess +import sys from pathlib import Path @@ -42,12 +43,35 @@ class ExampleUtils: # resource key variable starts with "_51DEGREES_RESOURCE_KEY". RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" - ENDPOINT_ENV_VAR = "_51DEGREES_CLOUD_ENDPOINT" + # Deprecated. Use RESOURCE_KEY_ENV_VAR. The name the resource key + # variable had before, read when RESOURCE_KEY_ENV_VAR is not set. + LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key" + + # The environment variable the data file update example reads a licence + # key from. + LICENSE_KEY_ENV_VAR = "_51DEGREES_LICENSE_KEY" + + # Deprecated. Use LICENSE_KEY_ENV_VAR. The name the licence key + # variable had before, read when LICENSE_KEY_ENV_VAR is not set. + LEGACY_LICENSE_KEY_ENV_VAR = "license_key" + + # The environment variable naming a cloud service other than the public + # one. + CLOUD_ENDPOINT_ENV_VAR = "_51DEGREES_CLOUD_ENDPOINT" + + # Deprecated. Use CLOUD_ENDPOINT_ENV_VAR. The name that variable had + # before, read when CLOUD_ENDPOINT_ENV_VAR is not set. + ENDPOINT_ENV_VAR = "cloud_endpoint" # The environment variable an on-premise example reads the path of its # device detection data file from, when one is given, in place of the # file named in its configuration. - DATA_FILE_ENV_VAR = "_51DEGREES_DD_PATH" + DD_PATH_ENV_VAR = "_51DEGREES_DD_PATH" + + # Deprecated. Use DD_PATH_ENV_VAR. The name that variable had before, + # read when DD_PATH_ENV_VAR is not set. A POSIX shell cannot set it, + # because it starts with a digit. + DATA_FILE_ENV_VAR = "51DEGREES_DD_PATH" # The data repository's device detection folder, from the root of a # checkout of this repository. Each file the data repository links to @@ -161,7 +185,24 @@ def get_no_profiles_message(): @staticmethod def get_resource_key(): - return ExampleUtils.__get_env_variable(ExampleUtils.RESOURCE_KEY_ENV_VAR) + return ExampleUtils.__get_env_variable( + ExampleUtils.RESOURCE_KEY_ENV_VAR, + ExampleUtils.LEGACY_RESOURCE_KEY_ENV_VAR) + + @staticmethod + def get_license_key(): + """! + The licence key from the environment, or an empty string. The old + name is also read in upper case, as it was before. + """ + + return ( + ExampleUtils.__get_env_variable( + ExampleUtils.LICENSE_KEY_ENV_VAR, + ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR) + or ExampleUtils.__get_env_variable( + ExampleUtils.LICENSE_KEY_ENV_VAR, + ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR.upper())) @staticmethod def get_missing_resource_key_message(): @@ -176,18 +217,41 @@ def get_missing_resource_key_message(): @staticmethod def get_data_file_path(): - return ExampleUtils.__get_env_variable(ExampleUtils.DATA_FILE_ENV_VAR) + return ExampleUtils.__get_env_variable( + ExampleUtils.DD_PATH_ENV_VAR, ExampleUtils.DATA_FILE_ENV_VAR) @staticmethod def get_cloud_endpoint(): - return ExampleUtils.__get_env_variable(ExampleUtils.ENDPOINT_ENV_VAR) + return ExampleUtils.__get_env_variable( + ExampleUtils.CLOUD_ENDPOINT_ENV_VAR, + ExampleUtils.ENDPOINT_ENV_VAR) + + # The old names already reported as deprecated, so each is reported + # once. + __reported_old_names = set() @staticmethod - def __get_env_variable(name): - if name in os.environ: - return os.environ[name] - else: + def __get_env_variable(name, old_name): + """! + The value of the environment variable name, or an empty string. + When name is unset or empty the name the variable had before is + read, and using it prints once that the old name is deprecated. + """ + + value = os.environ.get(name) + if value: + return value + value = os.environ.get(old_name) + if not value: return "" + if old_name not in ExampleUtils.__reported_old_names: + ExampleUtils.__reported_old_names.add(old_name) + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + return value # Find the specified file. A path to a file that exists is returned as # it is. Next, the data repository's folder is checked in the working diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index eeaa8492..109f9fc3 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -171,7 +171,12 @@ from fiftyone_pipeline_engines.datafile_update_service import DataFileUpdateService from fiftyone_pipeline_engines.datafile_update_service import UpdateStatus -UPDATE_EXAMPLE_LICENSE_KEY_NAME = "_51DEGREES_LICENSE_KEY" +# The environment variable checked for the licence key when none is given +# on the command line. +UPDATE_EXAMPLE_LICENSE_KEY_VARIABLE = ExampleUtils.LICENSE_KEY_ENV_VAR +# Deprecated. Use UPDATE_EXAMPLE_LICENSE_KEY_VARIABLE. The name that +# variable had before, read when the new one is not set. +UPDATE_EXAMPLE_LICENSE_KEY_NAME = ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR # Where to ask for a licence key and a copy of the Enterprise data file, tagged with this # example. CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=licence-key" @@ -215,7 +220,7 @@ def explain_license_key(self, data_file, output): output("Updating the data file downloads the Enterprise data file, " "which needs a licence key. Give the key as the first command line " f"argument, or in the environment variable " - f"'{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'. Contact us for a key and a " + f"'{UPDATE_EXAMPLE_LICENSE_KEY_VARIABLE}'. Contact us for a key and a " f"copy at {CONTACT_US}") def run(self, data_file, license_key, interactive, logger, output): @@ -223,7 +228,7 @@ def run(self, data_file, license_key, interactive, logger, output): # try to find a license key if not license_key: - license_key = KeyUtils.get_named_key(UPDATE_EXAMPLE_LICENSE_KEY_NAME) + license_key = ExampleUtils.get_license_key() if not license_key: self.explain_license_key(data_file, output) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index 080b512f..b3b9dcfc 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -199,7 +199,7 @@ def build_config(): if not os.path.exists(envDataFile): raise Exception("The device detection data file " + f"'{envDataFile}' named in the environment variable " + - f"'{ExampleUtils.DATA_FILE_ENV_VAR}' does not exist.") + f"'{ExampleUtils.DD_PATH_ENV_VAR}' does not exist.") ExampleUtils.set_data_file_in_config(config, envDataFile) return config @@ -226,7 +226,7 @@ def build_config(): "`git submodule update --init --recursive`, and run this " + "example again to download the file. Otherwise, ensure that the filename " + "is correct in config.json, or set the environment variable " + - f"'{ExampleUtils.DATA_FILE_ENV_VAR}' to the path of the data file.") + f"'{ExampleUtils.DD_PATH_ENV_VAR}' to the path of the data file.") return config diff --git a/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py b/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py index 23eb9f4f..36bc26ad 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py @@ -22,14 +22,16 @@ """Where the tests that call the live 51Degrees cloud service find their resource key, and what they fail with when there is none or when the -answer lacks what they asked for. +answer lacks what they asked for. Also how a test reads any setting from +an environment variable whose name has changed. -The same file is in each test folder whose tests call the live cloud -service. There is no fallback to a skip, because a cloud test that passes -without calling the cloud has proved nothing. +The same file is in each test folder whose tests read a setting from the +environment. There is no fallback to a skip, because a cloud test that +passes without calling the cloud has proved nothing. """ import os +import sys # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud # 51Did workflow sets it to each key in turn, and each product's @@ -39,18 +41,68 @@ "_51DEGREES_RESOURCE_KEY_BESPOKE", ) +# Deprecated. The name the resource key variable had before, read when +# none of RESOURCE_KEY_NAMES is set. +OLD_RESOURCE_KEY_NAME = "resource_key" + +# The old names already reported as deprecated, so each is reported once. +_reported_old_names = set() + ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +def _report_old_name(name, old_name): + """Prints, once for each old name, that it is deprecated and which + name replaces it.""" + if old_name in _reported_old_names: + return + _reported_old_names.add(old_name) + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + + +def read_environment(name, old_name): + """The value of the environment variable name, or None when it has + none. When name is unset or empty the name the variable had before is + read, and using it prints once that the old name is deprecated.""" + value = os.environ.get(name, "").strip() + if value: + return value + value = os.environ.get(old_name, "").strip() + if not value: + return None + _report_old_name(name, old_name) + return value + + +def is_set(name, old_name): + """Whether the environment variable name is set, whatever it holds, + which is how a switch is read. When it is not set, the name the + variable had before counts, and using it prints once that the old name + is deprecated.""" + if name in os.environ: + return True + if old_name not in os.environ: + return False + _report_old_name(name, old_name) + return True + + def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None.""" + """The first resource key set under any of the names, or None. When + none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" for name in names: value = os.environ.get(name, "").strip() if value: return value + if names is RESOURCE_KEY_NAMES: + return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_environment_names.py b/device-detection/fiftyone_devicedetection_examples/tests/test_environment_names.py new file mode 100644 index 00000000..be74700c --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_environment_names.py @@ -0,0 +1,162 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Checks that the example helper still reads each setting under the name +its environment variable had before, when the new name is not set, and +prints once that the old name is deprecated. These need no resource key. +""" + +import contextlib +import io +import os +import unittest +from unittest import mock + +from fiftyone_devicedetection_examples.example_utils import ExampleUtils +from fiftyone_devicedetection_shared.key_utils import KeyUtils + +# Each reader, with the new name of its variable and the old one. +PAIRS = { + "get_resource_key": ("_51DEGREES_RESOURCE_KEY", "resource_key"), + "get_license_key": ("_51DEGREES_LICENSE_KEY", "license_key"), + "get_cloud_endpoint": ("_51DEGREES_CLOUD_ENDPOINT", "cloud_endpoint"), + "get_data_file_path": ("_51DEGREES_DD_PATH", "51DEGREES_DD_PATH"), +} + +# Each constant that names a variable, with the value it has to hold. The +# ones that held an old name hold it still. +CONSTANTS = { + "RESOURCE_KEY_ENV_VAR": "_51DEGREES_RESOURCE_KEY", + "LEGACY_RESOURCE_KEY_ENV_VAR": "resource_key", + "LICENSE_KEY_ENV_VAR": "_51DEGREES_LICENSE_KEY", + "LEGACY_LICENSE_KEY_ENV_VAR": "license_key", + "CLOUD_ENDPOINT_ENV_VAR": "_51DEGREES_CLOUD_ENDPOINT", + "ENDPOINT_ENV_VAR": "cloud_endpoint", + "DD_PATH_ENV_VAR": "_51DEGREES_DD_PATH", + "DATA_FILE_ENV_VAR": "51DEGREES_DD_PATH", +} + + +class OldNameTests(unittest.TestCase): + + def setUp(self): + # Each test starts with none of the names set and no old name + # reported yet, and what it sets is put back afterwards. + environment = mock.patch.dict(os.environ) + environment.start() + self.addCleanup(environment.stop) + for pair in PAIRS.values(): + for name in pair: + os.environ.pop(name, None) + os.environ.pop(name.upper(), None) + reported = mock.patch.object( + ExampleUtils, "_ExampleUtils__reported_old_names", set()) + reported.start() + self.addCleanup(reported.stop) + + def printed(self, call): + """What a call returned and what it printed to the error stream.""" + errors = io.StringIO() + with contextlib.redirect_stderr(errors): + value = call() + return value, errors.getvalue() + + def test_each_setting_is_read_from_its_old_name(self): + for getter, (name, old_name) in PAIRS.items(): + with self.subTest(getter=getter): + os.environ[old_name] = "old-value" + value, errors = self.printed(getattr(ExampleUtils, getter)) + self.assertEqual("old-value", value) + self.assertIn( + old_name + " environment variable is deprecated", errors) + self.assertIn(name, errors) + del os.environ[old_name] + + def test_new_name_wins_and_nothing_is_printed(self): + for getter, (name, old_name) in PAIRS.items(): + with self.subTest(getter=getter): + os.environ[name] = "new-value" + os.environ[old_name] = "old-value" + value, errors = self.printed(getattr(ExampleUtils, getter)) + self.assertEqual("new-value", value) + self.assertEqual("", errors) + del os.environ[name] + del os.environ[old_name] + + def test_neither_name_gives_nothing(self): + for getter in PAIRS: + with self.subTest(getter=getter): + value, errors = self.printed(getattr(ExampleUtils, getter)) + self.assertFalse(value) + self.assertEqual("", errors) + + def test_old_name_is_reported_once(self): + getter, (name, old_name) = next(iter(PAIRS.items())) + os.environ[old_name] = "old-value" + + def twice(): + return (getattr(ExampleUtils, getter)(), + getattr(ExampleUtils, getter)()) + + value, errors = self.printed(twice) + self.assertEqual(("old-value", "old-value"), value) + self.assertEqual( + 1, errors.count(old_name + " environment variable is deprecated")) + + def test_constants_hold_the_old_and_the_new_names(self): + for constant, value in CONSTANTS.items(): + with self.subTest(constant=constant): + self.assertEqual(value, getattr(ExampleUtils, constant)) + + def test_licence_key_is_read_from_the_old_name_in_upper_case(self): + os.environ.pop("LICENSE_KEY", None) + os.environ["LICENSE_KEY"] = "old-value" + value, _ = self.printed(ExampleUtils.get_license_key) + self.assertEqual("old-value", value) + + +class KeyUtilsTests(unittest.TestCase): + + LOWER = "_51degrees_test_named_key" + UPPER = "_51DEGREES_TEST_NAMED_KEY" + + def setUp(self): + environment = mock.patch.dict(os.environ) + environment.start() + self.addCleanup(environment.stop) + os.environ.pop(self.LOWER, None) + os.environ.pop(self.UPPER, None) + + def test_key_is_read_from_the_name_given(self): + os.environ[self.LOWER] = "value" + self.assertEqual("value", KeyUtils.get_named_key(self.LOWER)) + + def test_key_is_read_from_the_name_in_upper_case(self): + os.environ[self.UPPER] = "value" + self.assertEqual("value", KeyUtils.get_named_key(self.LOWER)) + + def test_missing_key_gives_none(self): + self.assertIsNone(KeyUtils.get_named_key(self.LOWER)) + + +if __name__ == "__main__": + unittest.main() diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py b/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py index 555c0443..6e7fbef4 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py @@ -45,7 +45,7 @@ from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME # The settings the examples read, which a fresh clone does not have. -SETTINGS = ["_51DEGREES_LICENSE_KEY"] +SETTINGS = ["_51DEGREES_LICENSE_KEY", "license_key", "LICENSE_KEY"] # The evidence records the offline processing test reads. The example's own # file has 20,000, which take minutes. diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_onpremiseexamples.py b/device-detection/fiftyone_devicedetection_examples/tests/test_onpremiseexamples.py index aa200bba..7e56aefd 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_onpremiseexamples.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_onpremiseexamples.py @@ -22,7 +22,6 @@ import os import unittest -from fiftyone_devicedetection_shared.key_utils import KeyUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_devicedetection_examples.example_utils import ExampleUtils from fiftyone_devicedetection_examples.onpremise.gettingstarted_console import GettingStartedConsole @@ -30,7 +29,12 @@ from fiftyone_devicedetection_examples.onpremise.offlineprocessing import OfflineProcessing from fiftyone_devicedetection_examples.onpremise.match_metrics import MatchMetricsConsole from fiftyone_devicedetection_examples.onpremise.datafileupdate_console import DataFileUpdateConsole -from fiftyone_devicedetection_examples.onpremise.datafileupdate_console import UPDATE_EXAMPLE_LICENSE_KEY_NAME + +from .live_cloud import is_set + +# Deprecated old name, read when the new one is not set. +RUN_PERFORMANCE_TESTS = ( + "_51DEGREES_RUN_PERFORMANCE_TESTS", "run_performance_tests") class DeviceDetectionExampleTests(unittest.TestCase): @@ -59,12 +63,12 @@ def test_onpremise_match_metrics(self): def test_onpremise_datafileupdate_console(self): example = DataFileUpdateConsole() - license_key = KeyUtils.get_named_key(UPDATE_EXAMPLE_LICENSE_KEY_NAME) + license_key = ExampleUtils.get_license_key() example.run("Enterprise-HashV41.hash", license_key, False, self.logger, print) def test_onpremise_offline_processing(self): # Only run if environment variable set - if "_51DEGREES_RUN_PERFORMANCE_TESTS" in os.environ: + if is_set(*RUN_PERFORMANCE_TESTS): example = OfflineProcessing() with open(self.evidence_file, "r") as input: with open("./offlineprocessing-output.yml", "w") as output: @@ -88,5 +92,5 @@ def test_onpremise_offline_processing_with_tabs(self): def test_onpremise_performance(self): # Only run if environment variable set - if "_51DEGREES_RUN_PERFORMANCE_TESTS" in os.environ: + if is_set(*RUN_PERFORMANCE_TESTS): import fiftyone_devicedetection_examples.onpremise.performance diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py b/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py index a997f5c7..5c8978ee 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py @@ -81,7 +81,7 @@ def test_data_file_from_environment(self): with tempfile.NamedTemporaryFile(suffix=".hash", delete=False) as file: path = file.name try: - with mock.patch.dict(os.environ, {ExampleUtils.DATA_FILE_ENV_VAR: path}): + with mock.patch.dict(os.environ, {ExampleUtils.DD_PATH_ENV_VAR: path}): config = GettingStartedWeb.build_config() self.assertEqual( os.path.abspath(path), @@ -93,7 +93,7 @@ def test_data_file_from_environment(self): # name, rather than the example silently falling back to config.json. def test_missing_data_file_from_environment(self): missing = os.path.join(tempfile.gettempdir(), "no-such-51degrees-file.hash") - with mock.patch.dict(os.environ, {ExampleUtils.DATA_FILE_ENV_VAR: missing}): + with mock.patch.dict(os.environ, {ExampleUtils.DD_PATH_ENV_VAR: missing}): with self.assertRaises(Exception) as context: GettingStartedWeb.build_config() - self.assertIn(ExampleUtils.DATA_FILE_ENV_VAR, str(context.exception)) + self.assertIn(ExampleUtils.DD_PATH_ENV_VAR, str(context.exception)) diff --git a/device-detection/fiftyone_devicedetection_examples/tox.ini b/device-detection/fiftyone_devicedetection_examples/tox.ini index ca26fa15..f7d9f947 100644 --- a/device-detection/fiftyone_devicedetection_examples/tox.ini +++ b/device-detection/fiftyone_devicedetection_examples/tox.ini @@ -33,9 +33,12 @@ commands = {[common]commands} pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_LICENSE_KEY + license_key _51DEGREES_RUN_PERFORMANCE_TESTS + run_performance_tests [testenv:pre-publish] deps = diff --git a/device-detection/fiftyone_devicedetection_onpremise/tox.ini b/device-detection/fiftyone_devicedetection_onpremise/tox.ini index 261fd422..4e3d7d7c 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tox.ini +++ b/device-detection/fiftyone_devicedetection_onpremise/tox.ini @@ -33,7 +33,9 @@ commands = {[common]commands} pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_LICENSE_KEY + license_key TMPDIR TEMP TMP diff --git a/device-detection/fiftyone_devicedetection_shared/src/fiftyone_devicedetection_shared/key_utils.py b/device-detection/fiftyone_devicedetection_shared/src/fiftyone_devicedetection_shared/key_utils.py index e4ea13a4..a6ef6753 100644 --- a/device-detection/fiftyone_devicedetection_shared/src/fiftyone_devicedetection_shared/key_utils.py +++ b/device-detection/fiftyone_devicedetection_shared/src/fiftyone_devicedetection_shared/key_utils.py @@ -21,10 +21,31 @@ # ********************************************************************* import os +import warnings class KeyUtils: - # A key from the environment variable of exactly that name. + # The upper case names already reported as deprecated, so each is + # reported once. + _reported_upper_case = set() + + # A key from the environment variable of that name. When that variable + # is unset or empty, the variable with the same name in upper case is + # read, which is deprecated and reported once with a FutureWarning. @staticmethod def get_named_key(key_name): - return os.environ.get(key_name) + value = os.environ.get(key_name) + upper = key_name.upper() + if value or upper == key_name: + return value + upper_value = os.environ.get(upper) + if not upper_value: + return value or upper_value + if upper not in KeyUtils._reported_upper_case: + KeyUtils._reported_upper_case.add(upper) + warnings.warn( + "Reading a key from the upper case form of its variable name " + "is deprecated. The key was read from {0} because {1} is not " + "set, so set {1} to the same value.".format(upper, key_name), + FutureWarning, stacklevel=2) + return upper_value diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini b/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini index 996a428f..f4a02df3 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini @@ -23,7 +23,9 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_cloud {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_LICENSE_KEY + license_key [testenv:pre-publish] package = skip diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py index 75c8c9d9..fc950959 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py @@ -102,7 +102,7 @@ def main(argv): else: logger.log("error", "No resource key specified in the environment variable " - f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " + f"'{ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " "service is accessed using a 'ResourceKey'. For more information " "see " "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-getallproperties_console.py&utm_term=resource-key-required. " diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py index 7118db82..48bfe83b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py @@ -169,7 +169,7 @@ def main(argv): else: logger.log("error", "No resource key specified in the environment variable " - f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " + f"'{ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " "service is accessed using a 'ResourceKey'. For more " "information see " "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py index 04b2fcf0..59183683 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_web/app.py @@ -91,7 +91,7 @@ def main(resource_key = None): if not resource_key: raise Exception( "No resource key specified in the environment variable " - f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. Obtain one from " + f"'{ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR}'. Obtain one from " "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required") build_pipeline(resource_key) app.run(debug=False) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py index 60db3582..3d6d8f00 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py @@ -101,7 +101,7 @@ def main(argv): else: logger.log("error", "No resource key specified in the environment variable " - f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " + f"'{ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " "service is accessed using a 'ResourceKey'. For more information " "see " "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-metadata_console.py&utm_term=resource-key-required. " diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/mixed_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/mixed_console.py index d4559a10..2c228035 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/mixed_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/mixed_console.py @@ -169,7 +169,7 @@ def main(argv): else: logger.log("error", "No resource key specified in the environment variable " - f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " + f"'{ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " "service is accessed using a 'ResourceKey'. A resource key " "configured with both device detection and IP intelligence " "properties can be created at https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-mixed_console.py&utm_term=resource-key-required.") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py index b973e2a4..bc261253 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py @@ -25,6 +25,7 @@ import shlex import shutil import subprocess +import sys from pathlib import Path @@ -38,13 +39,35 @@ class ExampleUtils: # The default environment variable used to get the resource key # to use when running examples. - RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" + CLOUD_RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" - ENDPOINT_ENV_VAR = "_51DEGREES_CLOUD_ENDPOINT" + # Deprecated. Use CLOUD_RESOURCE_KEY_ENV_VAR. The name the resource key + # variable had before, read when CLOUD_RESOURCE_KEY_ENV_VAR is not set. + RESOURCE_KEY_ENV_VAR = "resource_key" + + # The environment variable the data file update example reads a licence + # key from. + LICENSE_KEY_ENV_VAR = "_51DEGREES_LICENSE_KEY" + + # Deprecated. Use LICENSE_KEY_ENV_VAR. The name the licence key + # variable had before, read when LICENSE_KEY_ENV_VAR is not set. + LEGACY_LICENSE_KEY_ENV_VAR = "license_key" + + # The environment variable naming a cloud service other than the public + # one. + CLOUD_ENDPOINT_ENV_VAR = "_51DEGREES_CLOUD_ENDPOINT" + + # Deprecated. Use CLOUD_ENDPOINT_ENV_VAR. The name that variable had + # before, read when CLOUD_ENDPOINT_ENV_VAR is not set. + ENDPOINT_ENV_VAR = "cloud_endpoint" # The default environment variable used to get the data file path # to use when running examples. - DATA_FILE_ENV_VAR = "_51DEGREES_IPI_PATH" + IPI_PATH_ENV_VAR = "_51DEGREES_IPI_PATH" + + # Deprecated. Use IPI_PATH_ENV_VAR. The name that variable had before, + # read when IPI_PATH_ENV_VAR is not set. + DATA_FILE_ENV_VAR = "IPINTELLIGENCEDATAFILE" # The data repository's IP intelligence folder, from the root of a # checkout of this repository. Each file the data repository links to @@ -123,11 +146,39 @@ def get_value(ip_data, property): @staticmethod def get_resource_key(): - return ExampleUtils.__get_env_variable(ExampleUtils.RESOURCE_KEY_ENV_VAR) + """! + The resource key from the environment, or None. The old name is + also read in upper case, as it was before. + """ + + return ( + ExampleUtils.__get_env_variable( + ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR, + ExampleUtils.RESOURCE_KEY_ENV_VAR) + or ExampleUtils.__get_env_variable( + ExampleUtils.CLOUD_RESOURCE_KEY_ENV_VAR, + ExampleUtils.RESOURCE_KEY_ENV_VAR.upper())) + + @staticmethod + def get_license_key(): + return ExampleUtils.__get_env_variable( + ExampleUtils.LICENSE_KEY_ENV_VAR, + ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR) @staticmethod def get_cloud_endpoint(): - return ExampleUtils.__get_env_variable(ExampleUtils.ENDPOINT_ENV_VAR) + """! + The cloud endpoint from the environment, or None. The old name is + also read in upper case, as it was before. + """ + + return ( + ExampleUtils.__get_env_variable( + ExampleUtils.CLOUD_ENDPOINT_ENV_VAR, + ExampleUtils.ENDPOINT_ENV_VAR) + or ExampleUtils.__get_env_variable( + ExampleUtils.CLOUD_ENDPOINT_ENV_VAR, + ExampleUtils.ENDPOINT_ENV_VAR.upper())) @staticmethod def find_file(file_name): @@ -140,7 +191,8 @@ def find_file(file_name): when it is missing. """ - env = os.environ.get(ExampleUtils.DATA_FILE_ENV_VAR) + env = ExampleUtils.__get_env_variable( + ExampleUtils.IPI_PATH_ENV_VAR, ExampleUtils.DATA_FILE_ENV_VAR) if env and os.path.basename(env) == file_name and os.path.isfile(env): return env @@ -376,6 +428,29 @@ def check_data_file(pipeline, logger): f"or cxx/data/get-data-files.ps1 ip-intelligence -Update " f"on Windows.") + # The old names already reported as deprecated, so each is reported + # once. + __reported_old_names = set() + @staticmethod - def __get_env_variable(name): - return os.environ.get(name) + def __get_env_variable(name, old_name): + """! + The value of the environment variable name, or None. When name is + unset or empty the name the variable had before is read, and using + it prints once that the old name is deprecated. + """ + + value = os.environ.get(name) + if value: + return value + value = os.environ.get(old_name) + if not value: + return None + if old_name not in ExampleUtils.__reported_old_names: + ExampleUtils.__reported_old_names.add(old_name) + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + return value diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py index 1ef6fcf4..809a93ce 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py @@ -76,7 +76,11 @@ # Environment variable checked for the license key if one is not supplied as # a command line argument. -UPDATE_EXAMPLE_LICENSE_KEY_NAME = "_51DEGREES_LICENSE_KEY" +UPDATE_EXAMPLE_LICENSE_KEY_VARIABLE = ExampleUtils.LICENSE_KEY_ENV_VAR + +# Deprecated. Use UPDATE_EXAMPLE_LICENSE_KEY_VARIABLE. The name that +# variable had before, read when the new one is not set. +UPDATE_EXAMPLE_LICENSE_KEY_NAME = ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR # Where to ask for a licence key and a copy of the Enterprise data file, tagged with this # example. @@ -130,7 +134,7 @@ def explain_license_key(self, data_file, output): output("Updating the data file downloads the Enterprise data file, " "which needs a licence key. Give the key as the first command " "line argument, or in the environment variable " - f"'{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'. Contact us for a key " + f"'{UPDATE_EXAMPLE_LICENSE_KEY_VARIABLE}'. Contact us for a key " f"and a copy at {CONTACT_US}") def run(self, data_file, license_key, interactive, logger, output): @@ -138,7 +142,7 @@ def run(self, data_file, license_key, interactive, logger, output): # Try to find a license key in the environment if not supplied. if not license_key: - license_key = os.environ.get(UPDATE_EXAMPLE_LICENSE_KEY_NAME) + license_key = ExampleUtils.get_license_key() if not license_key: self.explain_license_key(data_file, output) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py index 23eb9f4f..36bc26ad 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py @@ -22,14 +22,16 @@ """Where the tests that call the live 51Degrees cloud service find their resource key, and what they fail with when there is none or when the -answer lacks what they asked for. +answer lacks what they asked for. Also how a test reads any setting from +an environment variable whose name has changed. -The same file is in each test folder whose tests call the live cloud -service. There is no fallback to a skip, because a cloud test that passes -without calling the cloud has proved nothing. +The same file is in each test folder whose tests read a setting from the +environment. There is no fallback to a skip, because a cloud test that +passes without calling the cloud has proved nothing. """ import os +import sys # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud # 51Did workflow sets it to each key in turn, and each product's @@ -39,18 +41,68 @@ "_51DEGREES_RESOURCE_KEY_BESPOKE", ) +# Deprecated. The name the resource key variable had before, read when +# none of RESOURCE_KEY_NAMES is set. +OLD_RESOURCE_KEY_NAME = "resource_key" + +# The old names already reported as deprecated, so each is reported once. +_reported_old_names = set() + ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +def _report_old_name(name, old_name): + """Prints, once for each old name, that it is deprecated and which + name replaces it.""" + if old_name in _reported_old_names: + return + _reported_old_names.add(old_name) + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + + +def read_environment(name, old_name): + """The value of the environment variable name, or None when it has + none. When name is unset or empty the name the variable had before is + read, and using it prints once that the old name is deprecated.""" + value = os.environ.get(name, "").strip() + if value: + return value + value = os.environ.get(old_name, "").strip() + if not value: + return None + _report_old_name(name, old_name) + return value + + +def is_set(name, old_name): + """Whether the environment variable name is set, whatever it holds, + which is how a switch is read. When it is not set, the name the + variable had before counts, and using it prints once that the old name + is deprecated.""" + if name in os.environ: + return True + if old_name not in os.environ: + return False + _report_old_name(name, old_name) + return True + + def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None.""" + """The first resource key set under any of the names, or None. When + none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" for name in names: value = os.environ.get(name, "").strip() if value: return value + if names is RESOURCE_KEY_NAMES: + return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_environment_names.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_environment_names.py new file mode 100644 index 00000000..27d3844d --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_environment_names.py @@ -0,0 +1,174 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Checks that the example helper still reads each setting under the name +its environment variable had before, when the new name is not set, and +prints once that the old name is deprecated. These need no resource key. +""" + +import contextlib +import io +import os +import unittest +from unittest import mock + +import tempfile + +from fiftyone_ipintelligence_examples.example_utils import ExampleUtils + +# Each reader, with the new name of its variable and the old one. +PAIRS = { + "get_resource_key": ("_51DEGREES_RESOURCE_KEY", "resource_key"), + "get_license_key": ("_51DEGREES_LICENSE_KEY", "license_key"), + "get_cloud_endpoint": ("_51DEGREES_CLOUD_ENDPOINT", "cloud_endpoint"), +} + +# The data file variable, which find_file reads, under both names. +DATA_FILE = ("_51DEGREES_IPI_PATH", "IPINTELLIGENCEDATAFILE") + +# Each constant that names a variable, with the value it has to hold. The +# ones that held an old name hold it still. +CONSTANTS = { + "CLOUD_RESOURCE_KEY_ENV_VAR": "_51DEGREES_RESOURCE_KEY", + "RESOURCE_KEY_ENV_VAR": "resource_key", + "LICENSE_KEY_ENV_VAR": "_51DEGREES_LICENSE_KEY", + "LEGACY_LICENSE_KEY_ENV_VAR": "license_key", + "CLOUD_ENDPOINT_ENV_VAR": "_51DEGREES_CLOUD_ENDPOINT", + "ENDPOINT_ENV_VAR": "cloud_endpoint", + "IPI_PATH_ENV_VAR": "_51DEGREES_IPI_PATH", + "DATA_FILE_ENV_VAR": "IPINTELLIGENCEDATAFILE", +} + + +class OldNameTests(unittest.TestCase): + + def setUp(self): + # Each test starts with none of the names set and no old name + # reported yet, and what it sets is put back afterwards. + environment = mock.patch.dict(os.environ) + environment.start() + self.addCleanup(environment.stop) + for pair in PAIRS.values(): + for name in pair: + os.environ.pop(name, None) + os.environ.pop(name.upper(), None) + reported = mock.patch.object( + ExampleUtils, "_ExampleUtils__reported_old_names", set()) + reported.start() + self.addCleanup(reported.stop) + + def printed(self, call): + """What a call returned and what it printed to the error stream.""" + errors = io.StringIO() + with contextlib.redirect_stderr(errors): + value = call() + return value, errors.getvalue() + + def test_each_setting_is_read_from_its_old_name(self): + for getter, (name, old_name) in PAIRS.items(): + with self.subTest(getter=getter): + os.environ[old_name] = "old-value" + value, errors = self.printed(getattr(ExampleUtils, getter)) + self.assertEqual("old-value", value) + self.assertIn( + old_name + " environment variable is deprecated", errors) + self.assertIn(name, errors) + del os.environ[old_name] + + def test_new_name_wins_and_nothing_is_printed(self): + for getter, (name, old_name) in PAIRS.items(): + with self.subTest(getter=getter): + os.environ[name] = "new-value" + os.environ[old_name] = "old-value" + value, errors = self.printed(getattr(ExampleUtils, getter)) + self.assertEqual("new-value", value) + self.assertEqual("", errors) + del os.environ[name] + del os.environ[old_name] + + def test_neither_name_gives_nothing(self): + for getter in PAIRS: + with self.subTest(getter=getter): + value, errors = self.printed(getattr(ExampleUtils, getter)) + self.assertFalse(value) + self.assertEqual("", errors) + + def test_old_name_is_reported_once(self): + getter, (name, old_name) = next(iter(PAIRS.items())) + os.environ[old_name] = "old-value" + + def twice(): + return (getattr(ExampleUtils, getter)(), + getattr(ExampleUtils, getter)()) + + value, errors = self.printed(twice) + self.assertEqual(("old-value", "old-value"), value) + self.assertEqual( + 1, errors.count(old_name + " environment variable is deprecated")) + + def test_constants_hold_the_old_and_the_new_names(self): + for constant, value in CONSTANTS.items(): + with self.subTest(constant=constant): + self.assertEqual(value, getattr(ExampleUtils, constant)) + + def test_key_and_endpoint_are_read_from_the_old_names_in_upper_case(self): + # Windows reads a variable name in any case, so the name the + # message gives is compared without regard to case. + for getter, upper_name in ( + ("get_resource_key", "RESOURCE_KEY"), + ("get_cloud_endpoint", "CLOUD_ENDPOINT")): + with self.subTest(getter=getter): + os.environ[upper_name] = "old-value" + value, errors = self.printed(getattr(ExampleUtils, getter)) + self.assertEqual("old-value", value) + self.assertIn( + upper_name.lower() + + " environment variable is deprecated", errors.lower()) + del os.environ[upper_name] + + def test_licence_key_is_not_read_in_upper_case(self): + # The licence key was only ever read under the lower case name. + # Windows reads a name in any case, so this holds elsewhere. + if os.name == "nt": + self.skipTest("Windows reads a variable name in any case") + os.environ["LICENSE_KEY"] = "old-value" + value, errors = self.printed(ExampleUtils.get_license_key) + self.assertFalse(value) + self.assertEqual("", errors) + + def test_data_file_is_found_under_the_old_name(self): + for name in DATA_FILE: + os.environ.pop(name, None) + with tempfile.TemporaryDirectory() as folder: + path = os.path.join(folder, "only-for-this-test.ipi") + with open(path, "wb") as data_file: + data_file.write(b"not a data file") + os.environ[DATA_FILE[1]] = path + value, errors = self.printed( + lambda: ExampleUtils.find_file("only-for-this-test.ipi")) + self.assertEqual(path, value) + self.assertIn( + DATA_FILE[1] + " environment variable is deprecated", errors) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py index 8d4cf2d3..e3cdf702 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py @@ -42,7 +42,9 @@ from fiftyone_ipintelligence_examples.onpremise import performance_console # The settings the examples read, which a fresh clone does not have. -SETTINGS = ["_51DEGREES_LICENSE_KEY", ExampleUtils.DATA_FILE_ENV_VAR] +SETTINGS = [ + "_51DEGREES_LICENSE_KEY", "license_key", + ExampleUtils.IPI_PATH_ENV_VAR, ExampleUtils.DATA_FILE_ENV_VAR] # Options that keep the performance example short. They leave the data file # to the example. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini index c2c0088d..be89d1ca 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini @@ -38,9 +38,12 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_examples {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_LICENSE_KEY + license_key _51DEGREES_IPI_PATH + IPINTELLIGENCEDATAFILE TMPDIR TEMP TMP diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py index dbffc9dd..3e5cdf7b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py @@ -24,6 +24,8 @@ from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME +from .live_cloud import read_environment + class TestConstants: @@ -31,11 +33,13 @@ class TestConstants: IPV6_ADDRESS = "2001:4860:4860::8888" # The data file can be supplied via the _51DEGREES_IPI_PATH - # environment variable. By default the Lite file downloaded to the + # environment variable, or under its deprecated old name. By default + # the Lite file downloaded to the # repository's assets folder by ci/fetch-assets.ps1 is used. @staticmethod def get_data_file_path(): - env = os.environ.get("_51DEGREES_IPI_PATH") + env = read_environment( + "_51DEGREES_IPI_PATH", "IPINTELLIGENCEDATAFILE") if env: return env return os.path.join( diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/live_cloud.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/live_cloud.py new file mode 100644 index 00000000..36bc26ad --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/live_cloud.py @@ -0,0 +1,139 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. Also how a test reads any setting from +an environment variable whose name has changed. + +The same file is in each test folder whose tests read a setting from the +environment. There is no fallback to a skip, because a cloud test that +passes without calling the cloud has proved nothing. +""" + +import os +import sys + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and each product's +# integration test script sets it too. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", +) + +# Deprecated. The name the resource key variable had before, read when +# none of RESOURCE_KEY_NAMES is set. +OLD_RESOURCE_KEY_NAME = "resource_key" + +# The old names already reported as deprecated, so each is reported once. +_reported_old_names = set() + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def _report_old_name(name, old_name): + """Prints, once for each old name, that it is deprecated and which + name replaces it.""" + if old_name in _reported_old_names: + return + _reported_old_names.add(old_name) + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + + +def read_environment(name, old_name): + """The value of the environment variable name, or None when it has + none. When name is unset or empty the name the variable had before is + read, and using it prints once that the old name is deprecated.""" + value = os.environ.get(name, "").strip() + if value: + return value + value = os.environ.get(old_name, "").strip() + if not value: + return None + _report_old_name(name, old_name) + return value + + +def is_set(name, old_name): + """Whether the environment variable name is set, whatever it holds, + which is how a switch is read. When it is not set, the name the + variable had before counts, and using it prints once that the old name + is deprecated.""" + if name in os.environ: + return True + if old_name not in os.environ: + return False + _report_old_name(name, old_name) + return True + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None. When + none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + if names is RESOURCE_KEY_NAMES: + return read_environment(names[0], OLD_RESOURCE_KEY_NAME) + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_environment_names.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_environment_names.py new file mode 100644 index 00000000..5fe9f2e5 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_environment_names.py @@ -0,0 +1,66 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Checks that the tests still read the data file path under the name its +environment variable had before, when the new name is not set. +""" + +import contextlib +import io +import os +import unittest +from unittest import mock + +from .constants import TestConstants + +NEW_NAME = "_51DEGREES_IPI_PATH" +OLD_NAME = "IPINTELLIGENCEDATAFILE" + + +class OldNameTests(unittest.TestCase): + + def setUp(self): + environment = mock.patch.dict(os.environ) + environment.start() + self.addCleanup(environment.stop) + os.environ.pop(NEW_NAME, None) + os.environ.pop(OLD_NAME, None) + + def path(self): + with contextlib.redirect_stderr(io.StringIO()): + return TestConstants.get_data_file_path() + + def test_data_file_path_is_read_from_the_old_name(self): + os.environ[OLD_NAME] = "old.ipi" + self.assertEqual("old.ipi", self.path()) + + def test_new_name_wins(self): + os.environ[NEW_NAME] = "new.ipi" + os.environ[OLD_NAME] = "old.ipi" + self.assertEqual("new.ipi", self.path()) + + def test_neither_name_gives_the_lite_file(self): + self.assertTrue(self.path().endswith("51Degrees-LiteIpiV41.ipi")) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini b/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini index b14580c8..8a13f223 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini @@ -35,8 +35,11 @@ commands = {[common]commands} pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_LICENSE_KEY + license_key _51DEGREES_IPI_PATH + IPINTELLIGENCEDATAFILE TMPDIR TEMP TMP diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/key_utils.py b/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/key_utils.py index e4ea13a4..a6ef6753 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/key_utils.py +++ b/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/key_utils.py @@ -21,10 +21,31 @@ # ********************************************************************* import os +import warnings class KeyUtils: - # A key from the environment variable of exactly that name. + # The upper case names already reported as deprecated, so each is + # reported once. + _reported_upper_case = set() + + # A key from the environment variable of that name. When that variable + # is unset or empty, the variable with the same name in upper case is + # read, which is deprecated and reported once with a FutureWarning. @staticmethod def get_named_key(key_name): - return os.environ.get(key_name) + value = os.environ.get(key_name) + upper = key_name.upper() + if value or upper == key_name: + return value + upper_value = os.environ.get(upper) + if not upper_value: + return value or upper_value + if upper not in KeyUtils._reported_upper_case: + KeyUtils._reported_upper_case.add(upper) + warnings.warn( + "Reading a key from the upper case form of its variable name " + "is deprecated. The key was read from {0} because {1} is not " + "set, so set {1} to the same value.".format(upper, key_name), + FutureWarning, stacklevel=2) + return upper_value diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/tests/test_shared.py b/ip-intelligence/fiftyone_ipintelligence_shared/tests/test_shared.py index 834756dc..f3a5550a 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/tests/test_shared.py +++ b/ip-intelligence/fiftyone_ipintelligence_shared/tests/test_shared.py @@ -46,6 +46,24 @@ def test_get_named_key(self): finally: del os.environ["_51DEGREES_TEST_NAMED_KEY"] + def test_get_named_key_reads_the_name_in_upper_case(self): + # The upper case form of a name is still read when the name as + # given is not set. + import os + os.environ.pop("_51degrees_test_named_key", None) + os.environ["_51DEGREES_TEST_NAMED_KEY"] = "value" + try: + self.assertEqual( + KeyUtils.get_named_key("_51degrees_test_named_key"), "value") + finally: + del os.environ["_51DEGREES_TEST_NAMED_KEY"] + + def test_get_named_key_without_a_key_gives_none(self): + import os + os.environ.pop("_51degrees_test_named_key", None) + os.environ.pop("_51DEGREES_TEST_NAMED_KEY", None) + self.assertIsNone(KeyUtils.get_named_key("_51degrees_test_named_key")) + if __name__ == "__main__": unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini b/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini index c304f959..43f49189 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini @@ -22,7 +22,9 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_shared {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_LICENSE_KEY + license_key [testenv:pre-publish] package = skip diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini b/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini index fabb28ba..e033217e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini @@ -20,7 +20,9 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_translation {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_LICENSE_KEY + license_key [testenv:pre-publish] package = skip diff --git a/location/examples/cloud/gettingstarted.py b/location/examples/cloud/gettingstarted.py index fcd10ddf..143cc373 100644 --- a/location/examples/cloud/gettingstarted.py +++ b/location/examples/cloud/gettingstarted.py @@ -38,11 +38,33 @@ # You need to create a resource key at https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-gettingstarted.py&utm_term=top # and paste it into the code, replacing !!YOUR_RESOURCE_KEY!! below. -# Alternatively, add a _51DEGREES_RESOURCE_KEY environment variable +# Alternatively, add a _51DEGREES_RESOURCE_KEY environment variable. +# resource_key is the old name of that variable. It is deprecated, and is +# read only when _51DEGREES_RESOURCE_KEY is not set. import os -if "_51DEGREES_RESOURCE_KEY" in os.environ: - resource_key = os.environ["_51DEGREES_RESOURCE_KEY"] -else: +import sys + + +def read_environment(name, old_name): + """The value of the environment variable name, or None when it has + none. When name is unset or empty the name the variable had before is + read, and using it prints that the old name is deprecated.""" + value = os.environ.get(name) + if value: + return value + value = os.environ.get(old_name) + if not value: + return None + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + return value + + +resource_key = read_environment("_51DEGREES_RESOURCE_KEY", "resource_key") +if not resource_key: resource_key = "!!YOUR_RESOURCE_KEY!!" if resource_key == "!!YOUR_RESOURCE_KEY!!": diff --git a/location/examples/cloud/web.py b/location/examples/cloud/web.py index a5db8558..f893824f 100644 --- a/location/examples/cloud/web.py +++ b/location/examples/cloud/web.py @@ -38,11 +38,33 @@ # You need to create a resource key at https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=examples-cloud-web.py&utm_term=top # and paste it into the code, replacing !!YOUR_RESOURCE_KEY!! below. -# Alternatively, add a _51DEGREES_RESOURCE_KEY environment variable +# Alternatively, add a _51DEGREES_RESOURCE_KEY environment variable. +# resource_key is the old name of that variable. It is deprecated, and is +# read only when _51DEGREES_RESOURCE_KEY is not set. import os -if "_51DEGREES_RESOURCE_KEY" in os.environ: - resource_key = os.environ["_51DEGREES_RESOURCE_KEY"] -else: +import sys + + +def read_environment(name, old_name): + """The value of the environment variable name, or None when it has + none. When name is unset or empty the name the variable had before is + read, and using it prints that the old name is deprecated.""" + value = os.environ.get(name) + if value: + return value + value = os.environ.get(old_name) + if not value: + return None + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + return value + + +resource_key = read_environment("_51DEGREES_RESOURCE_KEY", "resource_key") +if not resource_key: resource_key = "!!YOUR_RESOURCE_KEY!!" if resource_key == "!!YOUR_RESOURCE_KEY!!": diff --git a/location/tests/live_cloud.py b/location/tests/live_cloud.py new file mode 100644 index 00000000..36bc26ad --- /dev/null +++ b/location/tests/live_cloud.py @@ -0,0 +1,139 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. Also how a test reads any setting from +an environment variable whose name has changed. + +The same file is in each test folder whose tests read a setting from the +environment. There is no fallback to a skip, because a cloud test that +passes without calling the cloud has proved nothing. +""" + +import os +import sys + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and each product's +# integration test script sets it too. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", +) + +# Deprecated. The name the resource key variable had before, read when +# none of RESOURCE_KEY_NAMES is set. +OLD_RESOURCE_KEY_NAME = "resource_key" + +# The old names already reported as deprecated, so each is reported once. +_reported_old_names = set() + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def _report_old_name(name, old_name): + """Prints, once for each old name, that it is deprecated and which + name replaces it.""" + if old_name in _reported_old_names: + return + _reported_old_names.add(old_name) + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + + +def read_environment(name, old_name): + """The value of the environment variable name, or None when it has + none. When name is unset or empty the name the variable had before is + read, and using it prints once that the old name is deprecated.""" + value = os.environ.get(name, "").strip() + if value: + return value + value = os.environ.get(old_name, "").strip() + if not value: + return None + _report_old_name(name, old_name) + return value + + +def is_set(name, old_name): + """Whether the environment variable name is set, whatever it holds, + which is how a switch is read. When it is not set, the name the + variable had before counts, and using it prints once that the old name + is deprecated.""" + if name in os.environ: + return True + if old_name not in os.environ: + return False + _report_old_name(name, old_name) + return True + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None. When + none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + if names is RESOURCE_KEY_NAMES: + return read_environment(names[0], OLD_RESOURCE_KEY_NAME) + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/location/tests/test_environment_names.py b/location/tests/test_environment_names.py new file mode 100644 index 00000000..f3ad6f0a --- /dev/null +++ b/location/tests/test_environment_names.py @@ -0,0 +1,114 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Checks that the two cloud examples still read the resource key under +the name its environment variable had before, when the new name is not set. + +Importing an example runs it, which calls the cloud service, so the +function each example reads its key with is taken out of the example's +source and run on its own. +""" + +import ast +import contextlib +import io +import os +import sys +import unittest +from pathlib import Path +from unittest import mock + +EXAMPLES = Path(__file__).resolve().parent.parent / "examples" / "cloud" + +NEW_NAME = "_51DEGREES_RESOURCE_KEY" +OLD_NAME = "resource_key" + + +def reader(example): + """The read_environment function of an example, or None when the + example has none.""" + source = (EXAMPLES / example).read_text(encoding="utf-8") + for node in ast.parse(source).body: + if isinstance(node, ast.FunctionDef) \ + and node.name == "read_environment": + module = ast.Module(body=[node], type_ignores=[]) + names = {"os": os, "sys": sys} + exec(compile(module, example, "exec"), names) + return names["read_environment"] + return None + + +class OldNameTests(unittest.TestCase): + + def setUp(self): + environment = mock.patch.dict(os.environ) + environment.start() + self.addCleanup(environment.stop) + os.environ.pop(NEW_NAME, None) + os.environ.pop(OLD_NAME, None) + + def read(self, example): + """What the example reads for its resource key, and what it prints + to the error stream.""" + read_environment = reader(example) + self.assertIsNotNone( + read_environment, + example + " has no read_environment function, so it does not " + "read the old name") + # The example has to read the key through the function. + self.assertIn( + 'read_environment("' + NEW_NAME + '", "' + OLD_NAME + '")', + (EXAMPLES / example).read_text(encoding="utf-8")) + errors = io.StringIO() + with contextlib.redirect_stderr(errors): + value = read_environment(NEW_NAME, OLD_NAME) + return value, errors.getvalue() + + def test_resource_key_is_read_from_the_old_name(self): + for example in ("gettingstarted.py", "web.py"): + with self.subTest(example=example): + os.environ[OLD_NAME] = "old-key" + value, errors = self.read(example) + self.assertEqual("old-key", value) + self.assertIn( + OLD_NAME + " environment variable is deprecated", errors) + self.assertIn(NEW_NAME, errors) + + def test_new_name_wins_and_nothing_is_printed(self): + for example in ("gettingstarted.py", "web.py"): + with self.subTest(example=example): + os.environ[NEW_NAME] = "new-key" + os.environ[OLD_NAME] = "old-key" + value, errors = self.read(example) + self.assertEqual("new-key", value) + self.assertEqual("", errors) + + def test_neither_name_gives_none(self): + for example in ("gettingstarted.py", "web.py"): + with self.subTest(example=example): + value, errors = self.read(example) + self.assertIsNone(value) + self.assertEqual("", errors) + + +if __name__ == "__main__": + unittest.main() diff --git a/location/tests/test_examples.py b/location/tests/test_examples.py index c1e6c276..d01a5bee 100644 --- a/location/tests/test_examples.py +++ b/location/tests/test_examples.py @@ -25,10 +25,12 @@ import unittest from pathlib import Path +from .live_cloud import read_environment + class LocationExampleTests(unittest.TestCase): def test_cloud_getting_started(self): - if "_51DEGREES_RESOURCE_KEY" in os.environ: + if read_environment("_51DEGREES_RESOURCE_KEY", "resource_key"): # this is required to import the example code, # since pytest sets sys.path to the test directory sys.path.append(str(Path(__file__).parent.parent.absolute() / "examples")) diff --git a/location/tests/test_properties51D.py b/location/tests/test_properties51D.py index 16c347c8..a31468c2 100644 --- a/location/tests/test_properties51D.py +++ b/location/tests/test_properties51D.py @@ -25,13 +25,13 @@ from fiftyone_location.location_pipelinebuilder import LocationPipelineBuilder from .test_helper import * +from .live_cloud import read_environment location_LAT = '51.4578261'; location_LON = '-0.975922996290084'; -if "_51DEGREES_RESOURCE_KEY" in os.environ: - resource_key = os.environ["_51DEGREES_RESOURCE_KEY"] -else: +resource_key = read_environment("_51DEGREES_RESOURCE_KEY", "resource_key") +if not resource_key: raise Exception("To run the cloud tests, please set a valid 51Degrees " "cloud resource key as the _51DEGREES_RESOURCE_KEY environment variable.") diff --git a/location/tox.ini b/location/tox.ini index e74b3a9f..dbb7b77b 100644 --- a/location/tox.ini +++ b/location/tox.ini @@ -17,7 +17,9 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_location {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_LICENSE_KEY + license_key [testenv:pre-publish] package = skip diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py index 3614ec68..e1534c4d 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/example_utils.py @@ -26,6 +26,7 @@ """ import os +import sys class ExampleUtils: @@ -35,38 +36,89 @@ class ExampleUtils: # "_51DEGREES_RESOURCE_KEY". RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" + # Deprecated. Use RESOURCE_KEY_ENV_VAR. The name the resource key + # variable had before, read when RESOURCE_KEY_ENV_VAR is not set. + LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key" + # The environment variable the examples read a licence key from, to use # in place of a resource key. LICENSE_KEY_ENV_VAR = "_51DEGREES_LICENSE_KEY" + # Deprecated. Use LICENSE_KEY_ENV_VAR. The name the licence key + # variable had before, read when LICENSE_KEY_ENV_VAR is not set. + LEGACY_LICENSE_KEY_ENV_VAR = "license_key" + # The environment variable naming a cloud service other than the public # one, for example one hosted privately. The value includes the api/v4 # path, as in https://cloud.51degrees.com/api/v4/. - ENDPOINT_ENV_VAR = "_51DEGREES_CLOUD_ENDPOINT" + CLOUD_ENDPOINT_ENV_VAR = "_51DEGREES_CLOUD_ENDPOINT" + + # Deprecated. Use CLOUD_ENDPOINT_ENV_VAR. The name that variable had + # before, read when CLOUD_ENDPOINT_ENV_VAR is not set. + ENDPOINT_ENV_VAR = "cloud_endpoint" # The environment variable holding the origin to send with each request # to the cloud service, for a resource key limited to particular # domains, which only answers a request whose origin is one of them. - ORIGIN_ENV_VAR = "_51DEGREES_CLOUD_REQUEST_ORIGIN" + CLOUD_REQUEST_ORIGIN_ENV_VAR = "_51DEGREES_CLOUD_REQUEST_ORIGIN" + + # Deprecated. Use CLOUD_REQUEST_ORIGIN_ENV_VAR. The name that variable + # had before, read when CLOUD_REQUEST_ORIGIN_ENV_VAR is not set. + ORIGIN_ENV_VAR = "cloud_request_origin" # The text shown for a value the results do not hold. UNKNOWN = "Unknown" @staticmethod def get_resource_key(): - return os.environ.get(ExampleUtils.RESOURCE_KEY_ENV_VAR) or "" + return ExampleUtils.__get_env_variable( + ExampleUtils.RESOURCE_KEY_ENV_VAR, + ExampleUtils.LEGACY_RESOURCE_KEY_ENV_VAR) @staticmethod def get_license_key(): - return os.environ.get(ExampleUtils.LICENSE_KEY_ENV_VAR) or "" + return ExampleUtils.__get_env_variable( + ExampleUtils.LICENSE_KEY_ENV_VAR, + ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR) @staticmethod def get_cloud_endpoint(): - return os.environ.get(ExampleUtils.ENDPOINT_ENV_VAR) or "" + return ExampleUtils.__get_env_variable( + ExampleUtils.CLOUD_ENDPOINT_ENV_VAR, + ExampleUtils.ENDPOINT_ENV_VAR) @staticmethod def get_cloud_request_origin(): - return os.environ.get(ExampleUtils.ORIGIN_ENV_VAR) or "" + return ExampleUtils.__get_env_variable( + ExampleUtils.CLOUD_REQUEST_ORIGIN_ENV_VAR, + ExampleUtils.ORIGIN_ENV_VAR) + + # The old names already reported as deprecated, so each is reported + # once. + __reported_old_names = set() + + @staticmethod + def __get_env_variable(name, old_name): + """! + The value of the environment variable name, or an empty string. + When name is unset or empty the name the variable had before is + read, and using it prints once that the old name is deprecated. + """ + + value = os.environ.get(name) + if value: + return value + value = os.environ.get(old_name) + if not value: + return "" + if old_name not in ExampleUtils.__reported_old_names: + ExampleUtils.__reported_old_names.add(old_name) + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + return value @staticmethod def name_values(section, property_names): diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py index 23eb9f4f..36bc26ad 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py @@ -22,14 +22,16 @@ """Where the tests that call the live 51Degrees cloud service find their resource key, and what they fail with when there is none or when the -answer lacks what they asked for. +answer lacks what they asked for. Also how a test reads any setting from +an environment variable whose name has changed. -The same file is in each test folder whose tests call the live cloud -service. There is no fallback to a skip, because a cloud test that passes -without calling the cloud has proved nothing. +The same file is in each test folder whose tests read a setting from the +environment. There is no fallback to a skip, because a cloud test that +passes without calling the cloud has proved nothing. """ import os +import sys # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud # 51Did workflow sets it to each key in turn, and each product's @@ -39,18 +41,68 @@ "_51DEGREES_RESOURCE_KEY_BESPOKE", ) +# Deprecated. The name the resource key variable had before, read when +# none of RESOURCE_KEY_NAMES is set. +OLD_RESOURCE_KEY_NAME = "resource_key" + +# The old names already reported as deprecated, so each is reported once. +_reported_old_names = set() + ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +def _report_old_name(name, old_name): + """Prints, once for each old name, that it is deprecated and which + name replaces it.""" + if old_name in _reported_old_names: + return + _reported_old_names.add(old_name) + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + + +def read_environment(name, old_name): + """The value of the environment variable name, or None when it has + none. When name is unset or empty the name the variable had before is + read, and using it prints once that the old name is deprecated.""" + value = os.environ.get(name, "").strip() + if value: + return value + value = os.environ.get(old_name, "").strip() + if not value: + return None + _report_old_name(name, old_name) + return value + + +def is_set(name, old_name): + """Whether the environment variable name is set, whatever it holds, + which is how a switch is read. When it is not set, the name the + variable had before counts, and using it prints once that the old name + is deprecated.""" + if name in os.environ: + return True + if old_name not in os.environ: + return False + _report_old_name(name, old_name) + return True + + def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None.""" + """The first resource key set under any of the names, or None. When + none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" for name in names: value = os.environ.get(name, "").strip() if value: return value + if names is RESOURCE_KEY_NAMES: + return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_example_utils.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_example_utils.py index c42292e2..1b6cc187 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_example_utils.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_example_utils.py @@ -30,6 +30,70 @@ from fiftyone_pipeline_examples.cloud.engines import WeightedCloudEngine from fiftyone_pipeline_examples.example_utils import ExampleUtils +import pytest + +# Each reader of a setting, with the new name of its variable and the old +# one, which is still read when the new name is not set. +OLD_NAMES = [ + ("get_resource_key", "_51DEGREES_RESOURCE_KEY", "resource_key"), + ("get_license_key", "_51DEGREES_LICENSE_KEY", "license_key"), + ("get_cloud_endpoint", "_51DEGREES_CLOUD_ENDPOINT", "cloud_endpoint"), + ("get_cloud_request_origin", "_51DEGREES_CLOUD_REQUEST_ORIGIN", + "cloud_request_origin"), +] + + +@pytest.fixture +def no_settings(monkeypatch): + """None of the settings in the environment, and no old name reported + as deprecated yet.""" + for _, name, old_name in OLD_NAMES: + monkeypatch.delenv(name, raising=False) + monkeypatch.delenv(old_name, raising=False) + monkeypatch.setattr( + ExampleUtils, "_ExampleUtils__reported_old_names", set()) + return monkeypatch + + +@pytest.mark.parametrize("getter, name, old_name", OLD_NAMES) +def test_setting_is_read_from_its_old_name_and_reported_once( + getter, name, old_name, no_settings, capsys): + no_settings.setenv(old_name, "old-value") + assert getattr(ExampleUtils, getter)() == "old-value" + assert getattr(ExampleUtils, getter)() == "old-value" + errors = capsys.readouterr().err + assert errors.count( + old_name + " environment variable is deprecated") == 1 + assert name in errors + + +@pytest.mark.parametrize("getter, name, old_name", OLD_NAMES) +def test_new_name_wins_and_nothing_is_printed( + getter, name, old_name, no_settings, capsys): + no_settings.setenv(name, "new-value") + no_settings.setenv(old_name, "old-value") + assert getattr(ExampleUtils, getter)() == "new-value" + assert capsys.readouterr().err == "" + + +@pytest.mark.parametrize("getter, name, old_name", OLD_NAMES) +def test_neither_name_gives_an_empty_string( + getter, name, old_name, no_settings, capsys): + assert getattr(ExampleUtils, getter)() == "" + assert capsys.readouterr().err == "" + + +def test_constants_hold_the_old_and_the_new_names(): + assert ExampleUtils.RESOURCE_KEY_ENV_VAR == "_51DEGREES_RESOURCE_KEY" + assert ExampleUtils.LEGACY_RESOURCE_KEY_ENV_VAR == "resource_key" + assert ExampleUtils.LICENSE_KEY_ENV_VAR == "_51DEGREES_LICENSE_KEY" + assert ExampleUtils.LEGACY_LICENSE_KEY_ENV_VAR == "license_key" + assert ExampleUtils.CLOUD_ENDPOINT_ENV_VAR == "_51DEGREES_CLOUD_ENDPOINT" + assert ExampleUtils.ENDPOINT_ENV_VAR == "cloud_endpoint" + assert (ExampleUtils.CLOUD_REQUEST_ORIGIN_ENV_VAR + == "_51DEGREES_CLOUD_REQUEST_ORIGIN") + assert ExampleUtils.ORIGIN_ENV_VAR == "cloud_request_origin" + class Results: """Stands in for an engine's results, holding one value per property.""" diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini index fbd5e095..cd1bd54a 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini @@ -37,10 +37,15 @@ commands = # examples for a resource key limited to particular domains. pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_LICENSE_KEY + license_key _51DEGREES_CLOUD_ENDPOINT + FOD_CLOUD_API_URL + cloud_endpoint _51DEGREES_CLOUD_REQUEST_ORIGIN + cloud_request_origin [testenv:pre-publish] deps = diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md index de740e3b..e123445a 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md @@ -63,7 +63,11 @@ and the rules in The engine calls `https://cloud.51degrees.com/api/v4/` unless told otherwise. Set the `_51DEGREES_CLOUD_ENDPOINT` environment variable, or pass `cloud_endpoint` in the engine settings, to the API base of another -host including the `/api/v4/` segment. A host other than +host including the `/api/v4/` segment. `FOD_CLOUD_API_URL`, the old name +of that variable, is deprecated. The engine reads it only when +`_51DEGREES_CLOUD_ENDPOINT` is not set, and warns once that the old name +was used. `Constants.FOD_CLOUD_API_URL` is deprecated as well, and +`Constants.CLOUD_ENDPOINT` replaces it. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons. This is the private hosting option of the cloud diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py index 5dfc6bc0..ca3146b4 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudrequestengine.py @@ -24,6 +24,7 @@ import json import os +import threading import warnings from functools import cached_property from json import JSONDecodeError @@ -48,6 +49,50 @@ from urllib.parse import urlencode +# Whether the old end point variable has been reported as deprecated, so +# that it is reported once however many engines are built. +_old_endpoint_reported = False +_old_endpoint_lock = threading.Lock() + + +def _endpoint_from_environment(): + + """! + The cloud end point the environment gives, or None when it gives none. + + Constants.CLOUD_ENDPOINT is read first. When it is unset or empty the + variable Constants.FOD_CLOUD_API_URL names is read, which is the old + name, and the first use of it is reported as deprecated. The report is + a FutureWarning, which Python shows by default, because the people who + set the variable are the ones running the application. + + @rtype: string + @return: the end point, or None + """ + + global _old_endpoint_reported + + endpoint = os.environ.get(Constants.CLOUD_ENDPOINT) + if endpoint: + return endpoint + + old_name = Constants.FOD_CLOUD_API_URL + endpoint = os.environ.get(old_name) + if not endpoint: + return None + + with _old_endpoint_lock: + report = not _old_endpoint_reported + _old_endpoint_reported = True + if report: + # The third frame is the code that built the engine. + warnings.warn( + Constants.MESSAGE_OLD_ENDPOINT_VARIABLE.format( + old_name, Constants.CLOUD_ENDPOINT), + FutureWarning, stacklevel=3) + return endpoint + + # Engine that makes a call to the 51Degrees cloud service # Returns raw JSON as a "cloud" property under "cloud" datakey class CloudRequestEngine(Engine): @@ -106,7 +151,8 @@ def __init__(self, settings = {}): self.reported_not_covered = set() # The endpoint is the cloud_endpoint setting, then the - # _51DEGREES_CLOUD_ENDPOINT environment variable, then + # _51DEGREES_CLOUD_ENDPOINT environment variable, then that + # variable's deprecated old name FOD_CLOUD_API_URL, then # cloud.51degrees.com. A host other than cloud.51degrees.com # would be used to (a) use an on premise web server, or (b) use # a privately hosted version of the 51Degrees cloud for @@ -116,8 +162,8 @@ def __init__(self, settings = {}): if "cloud_endpoint" in settings: self.baseURL = settings["cloud_endpoint"] else: - self.baseURL = os.environ.get(Constants.CLOUD_ENDPOINT) - if self.baseURL is None or (self.baseURL is not None and self.baseURL == ""): + self.baseURL = _endpoint_from_environment() + if self.baseURL is None: self.baseURL = Constants.BASE_URL_DEFAULT # Make sure if baseURL does not end with '/', one will be appended diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py index 60bd4d24..fa7c879c 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/constants.py @@ -23,6 +23,21 @@ class Constants: # Environment variable to set cloud end point CLOUD_ENDPOINT = "_51DEGREES_CLOUD_ENDPOINT" + + # @deprecated Use Constants.CLOUD_ENDPOINT. This holds the old name of + # the cloud end point environment variable, which the engine reads + # through this constant when CLOUD_ENDPOINT is not set, so code that + # passes this constant to the environment reads and sets the old + # variable. + FOD_CLOUD_API_URL = "FOD_CLOUD_API_URL" + + # Message when the cloud end point came from the old environment + # variable. The first value is the old name and the second the new one. + MESSAGE_OLD_ENDPOINT_VARIABLE = ( + "The {0} environment variable is deprecated and {1} replaces it. " + "The cloud endpoint was read from {0} because {1} is not set, so " + "set {1} to the same value.") + # Default cloud end point BASE_URL_DEFAULT = "https://cloud.51degrees.com/api/v4/" diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/live_cloud.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/live_cloud.py new file mode 100644 index 00000000..36bc26ad --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/live_cloud.py @@ -0,0 +1,139 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. Also how a test reads any setting from +an environment variable whose name has changed. + +The same file is in each test folder whose tests read a setting from the +environment. There is no fallback to a skip, because a cloud test that +passes without calling the cloud has proved nothing. +""" + +import os +import sys + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and each product's +# integration test script sets it too. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", +) + +# Deprecated. The name the resource key variable had before, read when +# none of RESOURCE_KEY_NAMES is set. +OLD_RESOURCE_KEY_NAME = "resource_key" + +# The old names already reported as deprecated, so each is reported once. +_reported_old_names = set() + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def _report_old_name(name, old_name): + """Prints, once for each old name, that it is deprecated and which + name replaces it.""" + if old_name in _reported_old_names: + return + _reported_old_names.add(old_name) + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + + +def read_environment(name, old_name): + """The value of the environment variable name, or None when it has + none. When name is unset or empty the name the variable had before is + read, and using it prints once that the old name is deprecated.""" + value = os.environ.get(name, "").strip() + if value: + return value + value = os.environ.get(old_name, "").strip() + if not value: + return None + _report_old_name(name, old_name) + return value + + +def is_set(name, old_name): + """Whether the environment variable name is set, whatever it holds, + which is how a switch is read. When it is not set, the name the + variable had before counts, and using it prints once that the old name + is deprecated.""" + if name in os.environ: + return True + if old_name not in os.environ: + return False + _report_old_name(name, old_name) + return True + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None. When + none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + if names is RESOURCE_KEY_NAMES: + return read_environment(names[0], OLD_RESOURCE_KEY_NAME) + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py index dff83016..b88ed79d 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py @@ -30,10 +30,13 @@ from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from .live_cloud import read_environment + class CloudEngineTests(unittest.TestCase): def setUp(self): - self.resource_key = os.environ.get("_51DEGREES_RESOURCE_KEY") + self.resource_key = read_environment( + "_51DEGREES_RESOURCE_KEY", "resource_key") self.assertIsNotNone( self.resource_key, diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py index 8d4bf357..1078da4f 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudrequestengine.py @@ -22,8 +22,10 @@ import os import warnings +from unittest import mock from parameterized import parameterized +from fiftyone_pipeline_cloudrequestengine import cloudrequestengine from fiftyone_pipeline_cloudrequestengine.cloudrequestexception import CloudRequestException from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import CloudRequestEngine from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder @@ -45,10 +47,50 @@ class TestCloudRequestEngine(CloudRequestEngineTestsBase): testResourceKey = "resource_key" testEndPoint = "http://testEndPoint/" testEnvVarEndPoint = "http://testEnvVarEndPoint/" + testOldEnvVarEndPoint = "http://testOldEnvVarEndPoint/" + + # The name the endpoint environment variable had before + # Constants.CLOUD_ENDPOINT, written out so that these tests do not + # depend on the deprecated constant that holds it. + oldEndPointVariable = "FOD_CLOUD_API_URL" def setUp(self): - if os.environ.get(Constants.CLOUD_ENDPOINT) is not None: - del os.environ[Constants.CLOUD_ENDPOINT] + # Each test starts with neither name of the endpoint variable set + # and with the old name not yet reported as deprecated, and what + # it sets is put back afterwards. + environment = mock.patch.dict(os.environ) + environment.start() + self.addCleanup(environment.stop) + os.environ.pop(Constants.CLOUD_ENDPOINT, None) + os.environ.pop(self.oldEndPointVariable, None) + reported = mock.patch.object( + cloudrequestengine, "_old_endpoint_reported", False) + reported.start() + self.addCleanup(reported.stop) + + def new_engine(self, **settings): + """! + A cloud request engine with the test resource key, a mock HTTP + client and any other settings given. + """ + + return CloudRequestEngine({ + "resource_key": TestCloudRequestEngine.testResourceKey, + "http_client": self.mock_http(), + **settings + }) + + def old_name_reports(self, caught): + """! + The warnings in a recorded list that say the old endpoint variable + is deprecated and name the one that replaces it. + """ + + return [ + warning for warning in caught + if issubclass(warning.category, FutureWarning) + and self.oldEndPointVariable in str(warning.message) + and Constants.CLOUD_ENDPOINT in str(warning.message)] def testConfigEndPoint_ExplicitSetting(self): """! @@ -88,6 +130,123 @@ def testConfigEndPoint_DefaultSetting(self): }) self.assertEqual(Constants.BASE_URL_DEFAULT, cloud_request_engine.baseURL) + def testConfigEndPoint_OldEnvironmentVariable(self): + """! + Check that the old name of the endpoint environment variable is + still read when the new name is not set + """ + + os.environ[self.oldEndPointVariable] = self.testOldEnvVarEndPoint + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + engine = self.new_engine() + self.assertEqual(self.testOldEnvVarEndPoint, engine.baseURL) + + def testConfigEndPoint_OldEnvironmentVariable_NewNameEmpty(self): + """! + Check that the old name is read when the new name is set to an + empty value, which counts as not set + """ + + os.environ[Constants.CLOUD_ENDPOINT] = "" + os.environ[self.oldEndPointVariable] = self.testOldEnvVarEndPoint + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + engine = self.new_engine() + self.assertEqual(self.testOldEnvVarEndPoint, engine.baseURL) + + def testConfigEndPoint_OldEnvironmentVariable_Empty(self): + """! + Check that the default is used when the old name is set to an + empty value and the new name is not set, with nothing reported + """ + + os.environ[self.oldEndPointVariable] = "" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + engine = self.new_engine() + self.assertEqual(Constants.BASE_URL_DEFAULT, engine.baseURL) + self.assertEqual([], self.old_name_reports(caught)) + + def testConfigEndPoint_NewEnvironmentVariableWins(self): + """! + Check that the new name wins when both names are set, and that the + old name is not reported because it was not used + """ + + os.environ[Constants.CLOUD_ENDPOINT] = self.testEnvVarEndPoint + os.environ[self.oldEndPointVariable] = self.testOldEnvVarEndPoint + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + engine = self.new_engine() + self.assertEqual(self.testEnvVarEndPoint, engine.baseURL) + self.assertEqual([], self.old_name_reports(caught)) + + def testConfigEndPoint_ExplicitSettingOverOldEnvironmentVariable(self): + """! + Check that the cloud_endpoint setting wins over the old name, and + that the old name is not reported because it was not used + """ + + os.environ[self.oldEndPointVariable] = self.testOldEnvVarEndPoint + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + engine = self.new_engine(cloud_endpoint=self.testEndPoint) + self.assertEqual(self.testEndPoint, engine.baseURL) + self.assertEqual([], self.old_name_reports(caught)) + + def testConfigEndPoint_OldEnvironmentVariableReportedOnce(self): + """! + Check that using the old name is reported as deprecated once, + naming the new name, however many engines are built + """ + + os.environ[self.oldEndPointVariable] = self.testOldEnvVarEndPoint + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + first = self.new_engine() + second = self.new_engine() + self.assertEqual(self.testOldEnvVarEndPoint, first.baseURL) + self.assertEqual(self.testOldEnvVarEndPoint, second.baseURL) + self.assertEqual(1, len(self.old_name_reports(caught))) + + def testOldEndPointConstant(self): + """! + Check that the constant the old name was held in is still there + with its old value, as a plain attribute of its class, so that + every way of reading it gives the old name + """ + + self.assertEqual( + self.oldEndPointVariable, Constants.FOD_CLOUD_API_URL) + self.assertEqual( + self.oldEndPointVariable, + vars(Constants).get("FOD_CLOUD_API_URL")) + self.assertIn("FOD_CLOUD_API_URL", dir(Constants)) + + def testConfigEndPoint_VariableTheOldConstantNames(self): + """! + Check that the engine reads the old name through the constant, as + it did before, so that code which points the constant at another + variable still decides what is read + """ + + os.environ["ANOTHER_VARIABLE"] = self.testOldEnvVarEndPoint + with mock.patch.object( + Constants, "FOD_CLOUD_API_URL", "ANOTHER_VARIABLE"): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + engine = self.new_engine() + self.assertEqual(self.testOldEnvVarEndPoint, engine.baseURL) + + def testEndPointConstant(self): + """! + Check that the constant for the new name holds the new name + """ + + self.assertEqual( + "_51DEGREES_CLOUD_ENDPOINT", Constants.CLOUD_ENDPOINT) + @parameterized.expand([ (False, "query.User-Agent=iPhone", "header.User-Agent=iPhone"), (False, "query.User-Agent=iPhone", "cookie.User-Agent=iPhone"), diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licensekey_live.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licensekey_live.py index 6db15de5..aff8943b 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licensekey_live.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_licensekey_live.py @@ -29,9 +29,12 @@ from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from .live_cloud import read_environment + # The licence key the live tests authenticate with, read from the variable # the examples in this repository read it from. -LICENSE_KEY = (os.environ.get("_51DEGREES_LICENSE_KEY") or "").strip() +LICENSE_KEY = read_environment( + "_51DEGREES_LICENSE_KEY", "license_key") or "" USER_AGENT = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) " diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini index 646020a0..6c759a83 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini @@ -20,7 +20,9 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_cloudrequestengine {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_LICENSE_KEY + license_key [testenv:pre-publish] package = skip diff --git a/pipeline/fiftyone_pipeline_core/tox.ini b/pipeline/fiftyone_pipeline_core/tox.ini index 9a03f3a3..1f52d5d0 100644 --- a/pipeline/fiftyone_pipeline_core/tox.ini +++ b/pipeline/fiftyone_pipeline_core/tox.ini @@ -22,7 +22,9 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_core {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_LICENSE_KEY + license_key [testenv:pre-publish] package = skip diff --git a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py index c3d6dd41..a8b26698 100644 --- a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py +++ b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/server.py @@ -63,10 +63,13 @@ Environment variables. ``_51DEGREES_RESOURCE_KEY`` is required. ``_51DEGREES_LICENSE_KEY`` is optional, and the comment in ``run`` says -why. +why. ``RESOURCE_KEY`` and ``LICENSE_KEY`` are the old names of those two. +They are deprecated, and each is read only when its new name is not set. ``_51DEGREES_CLOUD_ENDPOINT`` is the cloud API base including the ``/api/v4/`` segment, defaulting to ``https://cloud.51degrees.com/api/v4/``, and is the same variable the cloud request engine and the ``DidClient`` honour. +``FOD_CLOUD_API_URL`` is that variable's old name. It is deprecated, and +is read only when ``_51DEGREES_CLOUD_ENDPOINT`` is not set. ``PORT`` is the port to listen on, defaulting to 5100. Standard library plus this package. Run ``python server.py`` then open @@ -79,6 +82,7 @@ import secrets import sys import urllib.parse +import warnings from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path @@ -102,14 +106,27 @@ DEFAULT_API = "https://cloud.51degrees.com/api/v4/" -def env(name): +def env(name, old_name): """The named environment variable when it is set and not empty, - otherwise None.""" - return os.environ.get(name) or None - - -RESOURCE = env("_51DEGREES_RESOURCE_KEY") -LICENCE = env("_51DEGREES_LICENSE_KEY") or "" + otherwise None. When it is unset or empty the name the variable had + before is read, with a warning that the old name is deprecated, as + the ``DidClient`` gives for the cloud endpoint.""" + value = os.environ.get(name) + if value: + return value + value = os.environ.get(old_name) + if not value: + return None + warnings.warn( + "The {0} environment variable is deprecated and {1} replaces it. " + "The value was read from {0} because {1} is not set, so set {1} " + "to the same value.".format(old_name, name), + FutureWarning, stacklevel=2) + return value + + +RESOURCE = env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY") +LICENCE = env("_51DEGREES_LICENSE_KEY", "LICENSE_KEY") or "" # The cloud API base, normalised to end in exactly one slash so that # every URL is the base followed by its path. The page receives the same # value through its __API__ placeholder and builds its two cloud calls @@ -119,7 +136,8 @@ def env(name): # 51Degrees cloud for performance reasons. That is the private hosting # option of the cloud service, and both run the same service, so this # demo works unchanged against either. -API = (env("_51DEGREES_CLOUD_ENDPOINT") or DEFAULT_API).rstrip("/") + "/" +API = (env("_51DEGREES_CLOUD_ENDPOINT", "FOD_CLOUD_API_URL") + or DEFAULT_API).rstrip("/") + "/" PORT = int(os.environ.get("PORT", "5100")) diff --git a/pipeline/fiftyone_pipeline_did/readme.md b/pipeline/fiftyone_pipeline_did/readme.md index 2d1ce4ad..47ca2fa3 100644 --- a/pipeline/fiftyone_pipeline_did/readme.md +++ b/pipeline/fiftyone_pipeline_did/readme.md @@ -424,7 +424,7 @@ client = DidClient(resource_key, licence_key, transport=httpx_transport) | --- | --- | | `resource_key` | Required. The page's resource key, public by nature. It travels in the route of the key and verify requests and in the form body of the redeem request | | `licence_key` | Optional. A licence key of the same account, server side only. Needed to redeem where the account holds licence keys. Sent only in the body of the redeem request, never in a URL | -| `endpoint` | Optional. The API base including the `/api/v4/` segment. Defaults to the `_51DEGREES_CLOUD_ENDPOINT` environment variable, the same variable the cloud request engine honours, then to `https://cloud.51degrees.com/api/v4/`. A value without a trailing slash gains one | +| `endpoint` | Optional. The API base including the `/api/v4/` segment. Defaults to the `_51DEGREES_CLOUD_ENDPOINT` environment variable, the same variable the cloud request engine honours, then to `FOD_CLOUD_API_URL`, the deprecated old name of that variable, then to `https://cloud.51degrees.com/api/v4/`. The client warns once when the old name is the one used. A value without a trailing slash gains one | | `transport` | Optional. The HTTP transport, either an async callable taking the prepared `urllib.request.Request` and answering `(status, body_bytes)`, or an `urllib.request.OpenerDirector`, whose blocking `open` runs on a worker thread. Defaults to `urllib.request.urlopen` on a worker thread. Tests inject one | | `timeout` | Optional. Seconds the default transport waits for the cloud, 30 by default | | `now` | Optional. The clock, returning an aware UTC `datetime`. Tests inject one | @@ -659,7 +659,7 @@ unreachable cloud answers 502 with `{ "error": ... }`. | --- | --- | | `_51DEGREES_RESOURCE_KEY` | Required. The page's resource key, public by nature | | `_51DEGREES_LICENSE_KEY` | Optional. A licence key of the same account, held server side only. Only an account that holds licence keys needs one to redeem, so an account holding none runs without it | -| `_51DEGREES_CLOUD_ENDPOINT` | Optional. The cloud API base including the `/api/v4/` segment, defaulting to `https://cloud.51degrees.com/api/v4/`. This is the same variable the cloud request engine honours. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons, which is the private hosting option of the cloud service. Both run the same service, so the demo works unchanged against either | +| `_51DEGREES_CLOUD_ENDPOINT` | Optional. The cloud API base including the `/api/v4/` segment, defaulting to `https://cloud.51degrees.com/api/v4/`. This is the same variable the cloud request engine honours. `FOD_CLOUD_API_URL` is its old name, which is deprecated and read only when `_51DEGREES_CLOUD_ENDPOINT` is not set. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons, which is the private hosting option of the cloud service. Both run the same service, so the demo works unchanged against either | | `PORT` | The port to listen on, defaulting to `5100` | ### Running diff --git a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py index 72622643..280b770a 100644 --- a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py +++ b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py @@ -53,9 +53,11 @@ import json import os import re +import threading import urllib.error import urllib.parse import urllib.request +import warnings from dataclasses import dataclass from datetime import datetime, timedelta, timezone from enum import Enum @@ -69,12 +71,57 @@ from .id_type import IdType #: The public cloud API base, used when neither the ``endpoint`` argument -#: nor the ``_51DEGREES_CLOUD_ENDPOINT`` environment variable is set. +#: nor the ``_51DEGREES_CLOUD_ENDPOINT`` environment variable, nor that +#: variable's deprecated old name ``FOD_CLOUD_API_URL``, is set. DEFAULT_ENDPOINT = "https://cloud.51degrees.com/api/v4/" #: The environment variable the cloud request engine reads for the API #: base, honoured here when no endpoint argument is given. -ENDPOINT_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT" +CLOUD_ENDPOINT_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT" + +#: Deprecated. Use :data:`CLOUD_ENDPOINT_VARIABLE`. This holds the name that +#: variable had before, which is read when :data:`CLOUD_ENDPOINT_VARIABLE` +#: is not set, so code that passes this constant to the environment reads +#: and sets the old variable. +ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL" + +# Whether the old variable has been reported as deprecated, so that it is +# reported once however many clients are made. +_old_endpoint_reported = False +_old_endpoint_lock = threading.Lock() + + +def _endpoint_from_environment() -> Optional[str]: + """The API base the environment gives, or None when it gives none. + + :data:`CLOUD_ENDPOINT_VARIABLE` is read first. When it is unset or + empty the variable :data:`ENDPOINT_VARIABLE` names is read, which is + the old name, and the first use of it is reported as deprecated. The + report is a :class:`FutureWarning`, which Python shows by default, + because the people who set the variable are the ones running the + application. + """ + global _old_endpoint_reported + endpoint = os.environ.get(CLOUD_ENDPOINT_VARIABLE) + if endpoint: + return endpoint + old_name = ENDPOINT_VARIABLE + endpoint = os.environ.get(old_name) + if not endpoint: + return None + with _old_endpoint_lock: + report = not _old_endpoint_reported + _old_endpoint_reported = True + if report: + # The third frame is the code that made the client. + warnings.warn( + "The {0} environment variable is deprecated and {1} replaces " + "it. The cloud endpoint was read from {0} because {1} is not " + "set, so set {1} to the same value.".format( + old_name, CLOUD_ENDPOINT_VARIABLE), + FutureWarning, stacklevel=3) + return endpoint + _BOUNDARY_TOLERANCE = timedelta(minutes=15) @@ -453,8 +500,10 @@ class DidClient: only. Needed to redeem where the account holds licence keys, and sent only in the body of the redeem request, never in a URL. :param endpoint: the API base including the ``/api/v4/`` segment. - Defaults to the ``_51DEGREES_CLOUD_ENDPOINT`` environment variable, then to - the public cloud. A value without a trailing slash gains one. + Defaults to the ``_51DEGREES_CLOUD_ENDPOINT`` environment variable, + then to that variable's deprecated old name ``FOD_CLOUD_API_URL``, + then to the public cloud. A value without a trailing slash gains + one. :param transport: the HTTP transport, either an async callable taking the prepared :class:`urllib.request.Request` and answering ``(status, body_bytes)``, or an @@ -478,7 +527,7 @@ def __init__(self, resource_key: str, licence_key: Optional[str] = None, raise ValueError("resource_key is required") self._resource_key = resource_key self._licence_key = licence_key if licence_key else None - base = endpoint or os.environ.get(ENDPOINT_VARIABLE) or \ + base = endpoint or _endpoint_from_environment() or \ DEFAULT_ENDPOINT # Normalised to end in exactly one slash so every URL is the base # plus a relative path, as the cloud request engine treats the same diff --git a/pipeline/fiftyone_pipeline_did/tests/live_cloud.py b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py index 23eb9f4f..36bc26ad 100644 --- a/pipeline/fiftyone_pipeline_did/tests/live_cloud.py +++ b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py @@ -22,14 +22,16 @@ """Where the tests that call the live 51Degrees cloud service find their resource key, and what they fail with when there is none or when the -answer lacks what they asked for. +answer lacks what they asked for. Also how a test reads any setting from +an environment variable whose name has changed. -The same file is in each test folder whose tests call the live cloud -service. There is no fallback to a skip, because a cloud test that passes -without calling the cloud has proved nothing. +The same file is in each test folder whose tests read a setting from the +environment. There is no fallback to a skip, because a cloud test that +passes without calling the cloud has proved nothing. """ import os +import sys # Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud # 51Did workflow sets it to each key in turn, and each product's @@ -39,18 +41,68 @@ "_51DEGREES_RESOURCE_KEY_BESPOKE", ) +# Deprecated. The name the resource key variable had before, read when +# none of RESOURCE_KEY_NAMES is set. +OLD_RESOURCE_KEY_NAME = "resource_key" + +# The old names already reported as deprecated, so each is reported once. +_reported_old_names = set() + ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +def _report_old_name(name, old_name): + """Prints, once for each old name, that it is deprecated and which + name replaces it.""" + if old_name in _reported_old_names: + return + _reported_old_names.add(old_name) + print( + f"The {old_name} environment variable is deprecated and {name} " + f"replaces it. The value was read from {old_name} because {name} " + f"is not set, so set {name} to the same value.", + file=sys.stderr) + + +def read_environment(name, old_name): + """The value of the environment variable name, or None when it has + none. When name is unset or empty the name the variable had before is + read, and using it prints once that the old name is deprecated.""" + value = os.environ.get(name, "").strip() + if value: + return value + value = os.environ.get(old_name, "").strip() + if not value: + return None + _report_old_name(name, old_name) + return value + + +def is_set(name, old_name): + """Whether the environment variable name is set, whatever it holds, + which is how a switch is read. When it is not set, the name the + variable had before counts, and using it prints once that the old name + is deprecated.""" + if name in os.environ: + return True + if old_name not in os.environ: + return False + _report_old_name(name, old_name) + return True + + def find_resource_key(names=RESOURCE_KEY_NAMES): - """The first resource key set under any of the names, or None.""" + """The first resource key set under any of the names, or None. When + none of RESOURCE_KEY_NAMES is set, the deprecated old name is read.""" for name in names: value = os.environ.get(name, "").strip() if value: return value + if names is RESOURCE_KEY_NAMES: + return read_environment(names[0], OLD_RESOURCE_KEY_NAME) return None diff --git a/pipeline/fiftyone_pipeline_did/tests/test_creator_context_server.py b/pipeline/fiftyone_pipeline_did/tests/test_creator_context_server.py index cd80f8e3..f0784a29 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_creator_context_server.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_creator_context_server.py @@ -32,9 +32,11 @@ import urllib.error import urllib.parse import urllib.request +import warnings from datetime import timedelta from http.server import HTTPServer from pathlib import Path +from unittest import mock from fiftyone_pipeline_did import DidClient @@ -176,5 +178,125 @@ def test_unreachable_cloud_answers_502_with_an_error(self): self.assertIn("connection refused", json.loads(body)["error"]) +class ServerEndpointVariableTests(unittest.TestCase): + """The demo reads the cloud API base from the environment itself, + because its page needs the same value as its client, so it follows the + same two names as the DidClient. The old name is still read when the + new one is not set.""" + + NEW_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT" + OLD_VARIABLE = "FOD_CLOUD_API_URL" + + def setUp(self): + # Each test starts with neither name set, and what it sets is put + # back afterwards. + environment = mock.patch.dict(os.environ) + environment.start() + self.addCleanup(environment.stop) + os.environ.pop(self.NEW_VARIABLE, None) + os.environ.pop(self.OLD_VARIABLE, None) + + def load_recording_warnings(self): + """The server module, loaded afresh so that it reads the + environment again, with the warnings its import gave that say the + old endpoint variable is deprecated and name the new one.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + module = load_server() + reports = [ + warning for warning in caught + if issubclass(warning.category, FutureWarning) + and self.OLD_VARIABLE in str(warning.message) + and self.NEW_VARIABLE in str(warning.message)] + return module, reports + + def test_api_defaults_to_the_public_cloud(self): + module, reports = self.load_recording_warnings() + self.assertEqual("https://cloud.51degrees.com/api/v4/", module.API) + self.assertEqual([], reports) + + def test_api_read_from_the_new_variable(self): + os.environ[self.NEW_VARIABLE] = "https://private.example/api/v4" + module, reports = self.load_recording_warnings() + self.assertEqual("https://private.example/api/v4/", module.API) + self.assertEqual([], reports) + + def test_api_read_from_the_old_variable_and_reported(self): + os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" + module, reports = self.load_recording_warnings() + self.assertEqual("https://old.example/api/v4/", module.API) + self.assertEqual(1, len(reports)) + + def test_new_variable_wins_over_the_old_one(self): + os.environ[self.NEW_VARIABLE] = "https://private.example/api/v4" + os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" + module, reports = self.load_recording_warnings() + self.assertEqual("https://private.example/api/v4/", module.API) + self.assertEqual([], reports) + + +class ServerKeyVariableTests(unittest.TestCase): + """The demo reads its two keys from the environment, each under its + new name first and then under the name it had before.""" + + NAMES = ( + "_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY", + "_51DEGREES_LICENSE_KEY", "LICENSE_KEY") + + def setUp(self): + # Each test starts with none of the names set, and what it sets is + # put back afterwards. + environment = mock.patch.dict(os.environ) + environment.start() + self.addCleanup(environment.stop) + for name in self.NAMES: + os.environ.pop(name, None) + + def load(self): + """The server module, loaded afresh with the environment as it is, + and the warnings that say an old name is deprecated.""" + spec = importlib.util.spec_from_file_location( + "creator_context_server_keys", SERVER_PY) + module = importlib.util.module_from_spec(spec) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + spec.loader.exec_module(module) + reports = [ + str(warning.message) for warning in caught + if issubclass(warning.category, FutureWarning) + and "is deprecated" in str(warning.message)] + return module, reports + + def test_keys_are_read_from_their_old_names_and_reported(self): + os.environ["RESOURCE_KEY"] = "old-resource-key" + os.environ["LICENSE_KEY"] = "old-licence-key" + module, reports = self.load() + self.assertEqual("old-resource-key", module.RESOURCE) + self.assertEqual("old-licence-key", module.LICENCE) + self.assertEqual(2, len(reports)) + self.assertTrue(any( + "RESOURCE_KEY" in report and "_51DEGREES_RESOURCE_KEY" in report + for report in reports)) + self.assertTrue(any( + "LICENSE_KEY" in report and "_51DEGREES_LICENSE_KEY" in report + for report in reports)) + + def test_new_names_win_and_nothing_is_reported(self): + os.environ["_51DEGREES_RESOURCE_KEY"] = "new-resource-key" + os.environ["_51DEGREES_LICENSE_KEY"] = "new-licence-key" + os.environ["RESOURCE_KEY"] = "old-resource-key" + os.environ["LICENSE_KEY"] = "old-licence-key" + module, reports = self.load() + self.assertEqual("new-resource-key", module.RESOURCE) + self.assertEqual("new-licence-key", module.LICENCE) + self.assertEqual([], reports) + + def test_no_keys_gives_none_and_an_empty_licence(self): + module, reports = self.load() + self.assertIsNone(module.RESOURCE) + self.assertEqual("", module.LICENCE) + self.assertEqual([], reports) + + if __name__ == "__main__": unittest.main() diff --git a/pipeline/fiftyone_pipeline_did/tests/test_did_client.py b/pipeline/fiftyone_pipeline_did/tests/test_did_client.py index 3eb64fce..d3a8f4d3 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_did_client.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_did_client.py @@ -29,6 +29,7 @@ import unittest import urllib.error import urllib.parse +import warnings from datetime import datetime, timedelta, timezone from unittest import mock @@ -50,6 +51,7 @@ SignatureReason, SignatureResult, ) +from fiftyone_pipeline_did import did_client from fiftyone_pipeline_did.did_client import ( KEY_LIST_MAX_AGE, USER_AGENT, parse_iso8601) # The byte layout is not part of the package's public surface. These tests @@ -171,25 +173,85 @@ def test_resource_key_is_required(self): with self.assertRaises(ValueError): DidClient(None) + # The two names of the endpoint environment variable, written out so + # that these tests pin the names themselves. The old one is still read + # when the new one is not set. + NEW_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT" + OLD_VARIABLE = "FOD_CLOUD_API_URL" + + def setUp(self): + # Each test starts with neither name set and with the old name not + # yet reported as deprecated, and what it sets is put back + # afterwards. + environment = mock.patch.dict(os.environ) + environment.start() + self.addCleanup(environment.stop) + os.environ.pop(self.NEW_VARIABLE, None) + os.environ.pop(self.OLD_VARIABLE, None) + reported = mock.patch.object( + did_client, "_old_endpoint_reported", False) + reported.start() + self.addCleanup(reported.stop) + + def old_name_reports(self, caught): + """The warnings in a recorded list that say the old endpoint + variable is deprecated and name the one that replaces it.""" + return [ + warning for warning in caught + if issubclass(warning.category, FutureWarning) + and self.OLD_VARIABLE in str(warning.message) + and self.NEW_VARIABLE in str(warning.message)] + def test_endpoint_defaults_to_the_public_cloud(self): - saved = os.environ.pop("_51DEGREES_CLOUD_ENDPOINT", None) - try: - self.assertEqual(DEFAULT_ENDPOINT, DidClient(RESOURCE).endpoint) - finally: - if saved is not None: - os.environ["_51DEGREES_CLOUD_ENDPOINT"] = saved + self.assertEqual(DEFAULT_ENDPOINT, DidClient(RESOURCE).endpoint) def test_endpoint_read_from_the_environment(self): - saved = os.environ.get("_51DEGREES_CLOUD_ENDPOINT") - os.environ["_51DEGREES_CLOUD_ENDPOINT"] = "https://private.example/api/v4" - try: - self.assertEqual("https://private.example/api/v4/", - DidClient(RESOURCE).endpoint) - finally: - if saved is None: - del os.environ["_51DEGREES_CLOUD_ENDPOINT"] - else: - os.environ["_51DEGREES_CLOUD_ENDPOINT"] = saved + os.environ[self.NEW_VARIABLE] = "https://private.example/api/v4" + self.assertEqual("https://private.example/api/v4/", + DidClient(RESOURCE).endpoint) + + def test_endpoint_read_from_the_old_variable(self): + os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + client = DidClient(RESOURCE) + self.assertEqual("https://old.example/api/v4/", client.endpoint) + + def test_old_variable_read_when_the_new_one_is_empty(self): + os.environ[self.NEW_VARIABLE] = "" + os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + client = DidClient(RESOURCE) + self.assertEqual("https://old.example/api/v4/", client.endpoint) + + def test_empty_old_variable_gives_the_public_cloud(self): + os.environ[self.OLD_VARIABLE] = "" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + client = DidClient(RESOURCE) + self.assertEqual(DEFAULT_ENDPOINT, client.endpoint) + self.assertEqual([], self.old_name_reports(caught)) + + def test_new_variable_wins_over_the_old_one(self): + os.environ[self.NEW_VARIABLE] = "https://private.example/api/v4" + os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + client = DidClient(RESOURCE) + self.assertEqual("https://private.example/api/v4/", client.endpoint) + # The old name was not used, so it is not reported. + self.assertEqual([], self.old_name_reports(caught)) + + def test_old_variable_is_reported_as_deprecated_once(self): + os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + first = DidClient(RESOURCE) + second = DidClient(RESOURCE) + self.assertEqual("https://old.example/api/v4/", first.endpoint) + self.assertEqual("https://old.example/api/v4/", second.endpoint) + self.assertEqual(1, len(self.old_name_reports(caught))) def test_endpoint_argument_wins_and_is_normalised(self): self.assertEqual(ENDPOINT, DidClient( @@ -197,6 +259,47 @@ def test_endpoint_argument_wins_and_is_normalised(self): self.assertEqual(ENDPOINT, DidClient( RESOURCE, endpoint="https://cloud.example/api/v4//").endpoint) + def test_endpoint_argument_wins_over_the_old_variable(self): + os.environ[self.OLD_VARIABLE] = "https://old.example/api/v4" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + client = DidClient(RESOURCE, endpoint=ENDPOINT) + self.assertEqual(ENDPOINT, client.endpoint) + self.assertEqual([], self.old_name_reports(caught)) + + def test_old_constant_keeps_its_old_value(self): + # Code that passes ENDPOINT_VARIABLE to the environment has to keep + # reading the variable it read before. + self.assertEqual(self.OLD_VARIABLE, did_client.ENDPOINT_VARIABLE) + + def test_old_constant_is_given_by_dir_and_a_star_import(self): + # The old constant is a plain attribute of the module, so every + # way of finding it still does. + self.assertIn("ENDPOINT_VARIABLE", dir(did_client)) + self.assertEqual( + self.OLD_VARIABLE, vars(did_client).get("ENDPOINT_VARIABLE")) + names = {} + exec("from fiftyone_pipeline_did.did_client import *", names) + self.assertEqual(self.OLD_VARIABLE, names.get("ENDPOINT_VARIABLE")) + self.assertEqual( + self.NEW_VARIABLE, names.get("CLOUD_ENDPOINT_VARIABLE")) + + def test_client_reads_the_variable_the_old_constant_names(self): + # The client reads the old name through the constant, as it did + # before, so code that points the constant at another variable + # still decides what is read. + os.environ["ANOTHER_VARIABLE"] = "https://other.example/api/v4" + with mock.patch.object( + did_client, "ENDPOINT_VARIABLE", "ANOTHER_VARIABLE"): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + client = DidClient(RESOURCE) + self.assertEqual("https://other.example/api/v4/", client.endpoint) + + def test_new_constant_holds_the_new_name(self): + self.assertEqual( + self.NEW_VARIABLE, did_client.CLOUD_ENDPOINT_VARIABLE) + class PublicKeyTests(unittest.TestCase): """Section 2.1: the key list, its cache and the refetch rules.""" diff --git a/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py b/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py index fdd0f95c..8ef0fc8a 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py @@ -51,12 +51,13 @@ ) from fiftyone_pipeline_did.did_client import USER_AGENT -from .live_cloud import answered_without, find_resource_key, no_resource_key +from .live_cloud import ( + answered_without, find_resource_key, no_resource_key, read_environment) pytestmark = pytest.mark.live RESOURCE_KEY = find_resource_key() -LICENCE_KEY = os.environ.get("_51DEGREES_LICENSE_KEY") +LICENCE_KEY = read_environment("_51DEGREES_LICENSE_KEY", "license_key") FODID = "the FODid product" MARKETING = "the CloudV5FODiD product" diff --git a/pipeline/fiftyone_pipeline_did/tests/test_live_cloud.py b/pipeline/fiftyone_pipeline_did/tests/test_live_cloud.py new file mode 100644 index 00000000..415e24ec --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/test_live_cloud.py @@ -0,0 +1,184 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Checks live_cloud.py, which every test folder that reads a setting from +the environment holds a copy of. A variable whose name changed is still +read under its old name when the new one is not set, and using the old name +prints once that it is deprecated. These need no resource key. +""" + +import contextlib +import io +import os +import unittest +from pathlib import Path +from unittest import mock + +from . import live_cloud + +# Every folder that holds a copy of live_cloud.py, from the root of the +# repository. +COPIES = ( + "device-detection/fiftyone_devicedetection_cloud/tests", + "device-detection/fiftyone_devicedetection_examples/tests", + "ip-intelligence/fiftyone_ipintelligence_examples/tests", + "ip-intelligence/fiftyone_ipintelligence_onpremise/tests", + "location/tests", + "pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests", + "pipeline/fiftyone_pipeline_cloudrequestengine/tests", + "pipeline/fiftyone_pipeline_did/tests", +) + +NAMES = ( + "_51DEGREES_RESOURCE_KEY", "_51DEGREES_RESOURCE_KEY_BESPOKE", + "_51DEGREES_RESOURCE_KEY_BASIC", "resource_key", + "_51DEGREES_LICENSE_KEY", "license_key") + + +class OldNameTests(unittest.TestCase): + + def setUp(self): + # Each test starts with none of the names set and no old name + # reported yet, and what it sets is put back afterwards. + environment = mock.patch.dict(os.environ) + environment.start() + self.addCleanup(environment.stop) + for name in NAMES: + os.environ.pop(name, None) + reported = mock.patch.object( + live_cloud, "_reported_old_names", set()) + reported.start() + self.addCleanup(reported.stop) + + def printed(self, call): + """What a call returned and what it printed to the error stream.""" + errors = io.StringIO() + with contextlib.redirect_stderr(errors): + value = call() + return value, errors.getvalue() + + def test_resource_key_is_read_from_the_old_name(self): + os.environ["resource_key"] = "old-key" + value, errors = self.printed(live_cloud.find_resource_key) + self.assertEqual("old-key", value) + self.assertIn("resource_key environment variable is deprecated", + errors) + self.assertIn("_51DEGREES_RESOURCE_KEY", errors) + + def test_new_resource_key_name_wins_and_nothing_is_printed(self): + os.environ["_51DEGREES_RESOURCE_KEY"] = "new-key" + os.environ["resource_key"] = "old-key" + value, errors = self.printed(live_cloud.find_resource_key) + self.assertEqual("new-key", value) + self.assertEqual("", errors) + + def test_no_resource_key_gives_none(self): + value, errors = self.printed(live_cloud.find_resource_key) + self.assertIsNone(value) + self.assertEqual("", errors) + + def test_other_names_do_not_fall_back_to_the_old_name(self): + # The old name was one key, so a search for a key of another tier + # does not take it. + os.environ["resource_key"] = "old-key" + value, _ = self.printed(lambda: live_cloud.find_resource_key( + ("_51DEGREES_RESOURCE_KEY_BASIC",))) + self.assertIsNone(value) + + def test_old_name_is_read_and_reported_once(self): + os.environ["license_key"] = "old-licence" + + def twice(): + return ( + live_cloud.read_environment( + "_51DEGREES_LICENSE_KEY", "license_key"), + live_cloud.read_environment( + "_51DEGREES_LICENSE_KEY", "license_key")) + + value, errors = self.printed(twice) + self.assertEqual(("old-licence", "old-licence"), value) + self.assertEqual( + 1, errors.count("license_key environment variable is deprecated")) + self.assertIn("_51DEGREES_LICENSE_KEY", errors) + + def test_old_name_is_read_when_the_new_one_is_empty(self): + os.environ["_51DEGREES_LICENSE_KEY"] = "" + os.environ["license_key"] = "old-licence" + value, _ = self.printed(lambda: live_cloud.read_environment( + "_51DEGREES_LICENSE_KEY", "license_key")) + self.assertEqual("old-licence", value) + + def test_new_name_wins_and_nothing_is_printed(self): + os.environ["_51DEGREES_LICENSE_KEY"] = "new-licence" + os.environ["license_key"] = "old-licence" + value, errors = self.printed(lambda: live_cloud.read_environment( + "_51DEGREES_LICENSE_KEY", "license_key")) + self.assertEqual("new-licence", value) + self.assertEqual("", errors) + + def test_neither_name_gives_none(self): + value, errors = self.printed(lambda: live_cloud.read_environment( + "_51DEGREES_LICENSE_KEY", "license_key")) + self.assertIsNone(value) + self.assertEqual("", errors) + + def test_old_name_set_to_nothing_still_counts_as_set(self): + # A switch is on when its variable is set, whatever it holds. + os.environ["license_key"] = "" + value, errors = self.printed(lambda: live_cloud.is_set( + "_51DEGREES_LICENSE_KEY", "license_key")) + self.assertTrue(value) + self.assertEqual( + 1, errors.count("license_key environment variable is deprecated")) + self.assertIn("_51DEGREES_LICENSE_KEY", errors) + + def test_new_name_set_to_nothing_counts_as_set(self): + os.environ["_51DEGREES_LICENSE_KEY"] = "" + value, errors = self.printed(lambda: live_cloud.is_set( + "_51DEGREES_LICENSE_KEY", "license_key")) + self.assertTrue(value) + self.assertEqual("", errors) + + def test_neither_name_is_not_set(self): + value, errors = self.printed(lambda: live_cloud.is_set( + "_51DEGREES_LICENSE_KEY", "license_key")) + self.assertFalse(value) + self.assertEqual("", errors) + + +class CopiesTests(unittest.TestCase): + + def test_every_copy_is_the_same_file(self): + # One copy is tested here, so the others have to be that file. + root = Path(__file__).resolve().parents[3] + if not (root / "ci" / "packages.json").is_file(): + self.skipTest("not run from a checkout of the repository") + expected = Path(__file__).with_name("live_cloud.py").read_bytes() + for folder in COPIES: + with self.subTest(folder=folder): + copy = root / folder / "live_cloud.py" + self.assertTrue(copy.is_file(), str(copy)) + self.assertEqual(expected, copy.read_bytes()) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_did/tox.ini b/pipeline/fiftyone_pipeline_did/tox.ini index 7317c461..f42e48fb 100644 --- a/pipeline/fiftyone_pipeline_did/tox.ini +++ b/pipeline/fiftyone_pipeline_did/tox.ini @@ -16,10 +16,13 @@ wheel_build_env = .pkg # scripts select with. passenv = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_RESOURCE_KEY_BASIC _51DEGREES_LICENSE_KEY + license_key _51DEGREES_CLOUD_ENDPOINT + FOD_CLOUD_API_URL PYTEST_ADDOPTS deps = {[common]deps} diff --git a/pipeline/fiftyone_pipeline_engines/tox.ini b/pipeline/fiftyone_pipeline_engines/tox.ini index 1e8e9619..7ca15783 100644 --- a/pipeline/fiftyone_pipeline_engines/tox.ini +++ b/pipeline/fiftyone_pipeline_engines/tox.ini @@ -19,7 +19,9 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_engines {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_LICENSE_KEY + license_key [testenv:pre-publish] package = skip diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tox.ini b/pipeline/fiftyone_pipeline_engines_fiftyone/tox.ini index cddb3cd4..5e9b48a8 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tox.ini +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tox.ini @@ -20,7 +20,9 @@ commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_engines_fiftyone {tty:--color=yes} {posargs} pass_env = _51DEGREES_RESOURCE_KEY + resource_key _51DEGREES_LICENSE_KEY + license_key [testenv:pre-publish] package = skip From 625ee4ec8a33760e10459539501b1aa802b9cf2e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 21:10:23 +0000 Subject: [PATCH 66/68] Leave the Cloud Request Engine's data out of the JSON Builder's output (#91) * Test that the JSON Builder leaves out the Cloud Request Engine's data * Leave the Cloud Request Engine's data out of the JSON, as the specification says --- .../src/fiftyone_pipeline_core/jsonbundler.py | 3 +- .../tests/test_jsonbundler_excludes.py | 58 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_excludes.py diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index 408a30d0..dbd5216b 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -125,7 +125,8 @@ def process_internal(self, flowdata): if (flow_element.datakey == "jsonbundler" or flow_element.datakey == "javascriptbuilder" or - flow_element.datakey == "set-headers"): + flow_element.datakey == "set-headers" or + flow_element.datakey == "cloud"): continue properties = flow_element.get_properties() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_excludes.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_excludes.py new file mode 100644 index 00000000..204f0b83 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_excludes.py @@ -0,0 +1,58 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The JSON Builder leaves the Cloud Request Engine's Element Data out of the +JSON, as the JSON Builder specification says.""" + +import unittest + +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + + +class StandInCloudRequestEngine(FlowElement): + """Writes the raw answer under the Cloud Request Engine's data key.""" + + def __init__(self): + super(StandInCloudRequestEngine, self).__init__() + self.datakey = "cloud" + self.properties = {"cloud": {"name": "cloud", "type": "string"}} + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary( + self, {"cloud": '{"device":{"ismobile":true}}'})) + + +class JsonBuilderExcludesTests(unittest.TestCase): + + def test_the_cloud_request_engine_is_left_out(self): + pipeline = PipelineBuilder({}).add(StandInCloudRequestEngine()) \ + .build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.process() + self.assertNotIn("cloud", flowdata.jsonbundler.json) + + +if __name__ == "__main__": + unittest.main() From ddbd95db971ebd23fe689dff85f5fde77ece6183 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 21:10:57 +0000 Subject: [PATCH 67/68] Write delayexecution only for a property with a value (#92) * Test that the JSON Builder writes delayexecution only for a property with a value * Write delayexecution only for a property with a value, as the specification says --- .../src/fiftyone_pipeline_core/jsonbundler.py | 13 ++-- .../tests/test_jsonbundler_delayexecution.py | 69 +++++++++++++++++++ 2 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_delayexecution.py diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index dbd5216b..bcb2fe1f 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -176,13 +176,6 @@ def process_internal(self, flowdata): null_reason = "Unknown" no_value = False - # Check if property has delayed execution and set in JSON if yes - - - if propertykey in property_cache["delayExecutionList"]: - output[flow_element.datakey.lower()][propertykey.lower() + "delayexecution"] = True - - # Check if property has any delayed execution evidence properties and set in JSON if yes if propertykey in property_cache["evidenceProperties"]: @@ -248,6 +241,12 @@ def process_internal(self, flowdata): if value is None: output[flow_element.datakey.lower()][propertykey.lower() + "nullreason"] = null_reason + # A Property with delayed execution is marked only when it + # has a value. + if (value is not None and + propertykey in property_cache["delayExecutionList"]): + output[flow_element.datakey.lower()][propertykey.lower() + "delayexecution"] = True + if sequence < 10: if "type" in propertymeta and propertymeta["type"].lower() == "javascript": diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_delayexecution.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_delayexecution.py new file mode 100644 index 00000000..73fff908 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_delayexecution.py @@ -0,0 +1,69 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The JSON Builder writes delayexecution only for a Property that has a +value, as the JSON Builder specification's meta-Property table says.""" + +import unittest + +from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + + +def delayed(name): + return {"name": name, "type": "javascript", "delayexecution": True} + + +class DelayedElement(FlowElement): + + def __init__(self): + super(DelayedElement, self).__init__() + self.datakey = "delayed" + self.properties = { + "withvalue": delayed("WithValue"), + "novalue": delayed("NoValue"), + "plainnull": delayed("PlainNull")} + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary(self, { + "withvalue": AspectPropertyValue(value="script"), + "novalue": AspectPropertyValue(no_value_message="No script"), + "plainnull": None})) + + +class JsonBuilderDelayExecutionTests(unittest.TestCase): + + def test_delayexecution_is_written_only_for_a_property_with_a_value(self): + pipeline = PipelineBuilder({}).add(DelayedElement()).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + flowdata.process() + section = flowdata.jsonbundler.json["delayed"] + self.assertTrue(section["withvaluedelayexecution"]) + self.assertNotIn("novaluedelayexecution", section) + self.assertNotIn("plainnulldelayexecution", section) + + +if __name__ == "__main__": + unittest.main() From c69d00fe75b019018b8791e77689acfdc35e3c6d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 12 Nov 2026 21:29:58 +0000 Subject: [PATCH 68/68] Run the file system watcher on its own setting (#110) * Test that the file system watcher runs on its own setting testTheWatcherRunsOnItsOwnSetting registers a data file in three cases, being the watcher on with automatic updates on, the watcher on with them off, and the watcher off with them on. A watcher is created, logged and loads a replaced file exactly when the watcher's own setting is on. The five watcher tests in FileSystemWatcherTests no longer turn automatic updates on through register_watched, which existed only to get a watcher, and testFileSystemWatcherIsLogged no longer sets auto_update. testNoFileSystemWatcherWithoutAutomaticUpdates, which held the watcher to the automatic update setting, is removed. * Run the file system watcher on its own setting register_data_file creates the watcher when file_system_watcher is on, whether or not auto_update is, as it did on main. Watching a file downloads nothing, so it needs no licence key, and the on-premise engines, which turn auto_update off without a key, keep watching the data file. Polling, the update on start-up and the licence key handling are unchanged. --- .../datafile_update_service.py | 10 ++-- .../tests/test_datafile_update_http.py | 16 ------ .../tests/test_datafile_update_service.py | 53 ++++++++++++++----- 3 files changed, 45 insertions(+), 34 deletions(-) diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py index 7d5c6972..6366e769 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py @@ -542,12 +542,10 @@ def register_data_file(self, data_file): if data_file.update_on_start or data_file.auto_update: self.check_next_update(data_file) - # Watch for changes in file and refresh if - # file_system_watcher is enabled on data_file - - # The watcher is an automatic update, so it only runs with - # automatic updates on. - if data_file.auto_update and data_file.file_system_watcher: + # The file system watcher runs on its own setting, whether or not + # automatic updates are on. It downloads nothing, so it needs no + # licence key. + if data_file.file_system_watcher: self._log("info", "Creating file system watcher") data_file.last_changed_time = os.stat(data_file.path).st_mtime self.track_local_data_file_changes(data_file) diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py index fa30d679..9d3dba02 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_http.py @@ -450,27 +450,11 @@ def testFileSystemWatcherIsLogged(self): with open(data_file.path, "w") as file: file.write("x") data_file.file_system_watcher = True - data_file.auto_update = True self.hold_timers() self.service.register_data_file(data_file) self.assertIn( ("info", "Creating file system watcher"), self.pipeline.logs) - def testNoFileSystemWatcherWithoutAutomaticUpdates(self): - folder = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, folder, ignore_errors=True) - data_file = HttpDataFile(os.path.join(folder, "data.dat")) - with open(data_file.path, "w") as file: - file.write("x") - data_file.file_system_watcher = True - data_file.auto_update = False - self.hold_timers() - self.service.register_data_file(data_file) - self.assertNotIn( - ("info", "Creating file system watcher"), self.pipeline.logs) - self.assertEqual([], self.pending) - self.assertIsNone(getattr(data_file, "watcher_timer", None)) - # A data file kept on disk that cannot put the new file in place, and one # with no temporary file path, carry .NET's message and status. diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py index 96dd178f..b96f9b5f 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py @@ -119,19 +119,48 @@ def run_next_check(self): check = self.pending.pop(0) check.function(*check.args) - def register_watched(self, data_file): - # The watcher runs only with automatic updates on. The update checks - # those would schedule are left out, so only the watcher's are - # pending. - data_file.auto_update = True - with mock.patch.object(self.service, "check_next_update"): - self.service.register_data_file(data_file) + # The watcher runs on its own setting, whether or not automatic updates + # are on. Watching a file downloads nothing, so a data file with no + # licence key, whose automatic updates are off, is still watched. + def testTheWatcherRunsOnItsOwnSetting(self): + for name, automatic, watcher in ( + ("watcher on, automatic updates on", True, True), + ("watcher on, automatic updates off", False, True), + ("watcher off, automatic updates on", True, False)): + with self.subTest(name): + self.write("version 1") + self.pending.clear() + pipeline = RecordingPipeline() + service = DataFileUpdateService(pipeline) + data_file = StandInDataFile(self.path) + data_file.auto_update = automatic + data_file.file_system_watcher = watcher + data_file.polling_interval = 30 + data_file.update_time_maximum_randomisation = 0 + service.register_data_file(data_file) + + watches = [ + check for check in self.pending + if check.function == service.track_local_data_file_changes] + self.assertEqual(1 if watcher else 0, len(watches), name) + self.assertEqual( + watcher, + ("info", "Creating file system watcher") in pipeline.logs, + name) + + self.write("version 2") + for check in watches: + check.function(*check.args) + self.assertEqual( + "version 2" if watcher else "version 1", data_file.data, + name) + service.unregister_data_file(data_file) # A refresh that fails is logged and reported, the data in use is kept, # and a later good file is picked up. def testFileSystemWatcherKeepsWatchingAfterFailedRefresh(self): data_file = StandInDataFile(self.path) - self.register_watched(data_file) + self.service.register_data_file(data_file) self.write("unusable") self.run_next_check() @@ -156,7 +185,7 @@ def testFileSystemWatcherKeepsWatchingAfterFailedRefresh(self): # reports success to the update complete callbacks. def testFileSystemWatcherRefreshIsRoutedAsPollingIs(self): data_file = StandInDataFile(self.path) - self.register_watched(data_file) + self.service.register_data_file(data_file) self.write("version 2") self.run_next_check() @@ -172,7 +201,7 @@ def testFileSystemWatcherRefreshIsRoutedAsPollingIs(self): # the specification's errors table, as the polling route's does. def testFileSystemWatcherFailedRefreshUsesTheTableMessage(self): data_file = StandInDataFile(self.path) - self.register_watched(data_file) + self.service.register_data_file(data_file) self.write("unusable") self.run_next_check() @@ -189,7 +218,7 @@ def testFileSystemWatcherFailedRefreshUsesTheTableMessage(self): # replaced, is looked for again at the next check. def testFileSystemWatcherKeepsWatchingWhileFileIsMissing(self): data_file = StandInDataFile(self.path) - self.register_watched(data_file) + self.service.register_data_file(data_file) os.remove(self.path) self.run_next_check() @@ -206,7 +235,7 @@ def testFileSystemWatcherKeepsWatchingWhileFileIsMissing(self): # nothing is left holding the data file. def testUnregisterStopsFileSystemWatcher(self): data_file = StandInDataFile(self.path) - self.register_watched(data_file) + self.service.register_data_file(data_file) check = self.pending[0] self.service.unregister_data_file(data_file)