diff --git a/.github/images/logo-160.png b/.github/images/logo-160.png new file mode 100644 index 000000000..78952d509 Binary files /dev/null and b/.github/images/logo-160.png differ diff --git a/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml index 69731a8d9..e99cdcb13 100644 --- a/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -3,6 +3,9 @@ name: Cloud 51Did Tests # Runs the live cloud 51Did integration test (pipeline/pipeline.did, # DidClientLiveTests) once for every _51DEGREES_RESOURCE_KEY* key passed in # below. Each run sets the single _51DEGREES_RESOURCE_KEY the test reads. +# Only keys that carry the fodid product are passed in, because the tests ask +# the cloud for a 51Did and fail, saying so, when a key gets an answer with no +# identifier in it. That is entitlement working rather than a fault. # # The secrets are named one by one rather than passed as toJSON(secrets). # GitHub marks a workflow that hands every secret to a step as possibly @@ -10,10 +13,10 @@ name: Cloud 51Did Tests # A new resource key therefore needs a line here as well as the organisation # secret. # -# The test skips itself when no key is set, so a run with no secrets is green -# and proves nothing. That is deliberate, because a pull request from a fork -# has no secrets, and it is the reason the loop below reports which key each -# run used. +# A run with no key available fails. A cloud test that passes without +# calling the cloud reports success having proved nothing, and that is worse +# than a red job. A pull request from a fork gets no secrets, so run this +# workflow from a branch in this repository. on: workflow_dispatch: @@ -51,7 +54,6 @@ jobs: - name: Cloud 51Did test for each resource key secret shell: pwsh env: - _51DEGREES_RESOURCE_KEY_BASIC: ${{ vars._51DEGREES_RESOURCE_KEY_BASIC }} _51DEGREES_RESOURCE_KEY_BESPOKE: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} _51DEGREES_RESOURCE_KEY_51DID: ${{ vars._51DEGREES_RESOURCE_KEY_51DID }} # Without a license key the tests that redeem a sealed verdict skip. @@ -63,8 +65,11 @@ jobs: $keys = Get-ChildItem env: | Where-Object { $_.Name -like '_51DEGREES_RESOURCE_KEY*' -and $_.Value } if (-not $keys) { - Write-Host 'No _51DEGREES_RESOURCE_KEY* secrets available; skipping.' - exit 0 + throw ('No _51DEGREES_RESOURCE_KEY* secret is available, so the ' + + 'live 51Did tests cannot call the cloud service. The keys ' + + 'carry the fodid product these tests need. A pull request ' + + 'from a fork gets no secrets, so run this workflow from a ' + + 'branch in this repository.') } $failed = $false foreach ($k in $keys) { diff --git a/.github/workflows/environment-variable-names.yml b/.github/workflows/environment-variable-names.yml new file mode 100644 index 000000000..f5b3abd55 --- /dev/null +++ b/.github/workflows/environment-variable-names.yml @@ -0,0 +1,30 @@ +name: Environment variable names + +on: + pull_request: + push: + branches: + - main + - rc + workflow_dispatch: + +permissions: + contents: read + +jobs: + environment-variable-names: + name: No tracked file names a variable starting with a digit or an old name + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Test the check + shell: pwsh + run: Invoke-Pester -Path ./ci/check-environment-variable-names.Tests.ps1 -CI + + - name: Check the names + shell: pwsh + run: ./ci/check-environment-variable-names.ps1 diff --git a/.github/workflows/monthly-copyright-update.yml b/.github/workflows/monthly-copyright-update.yml index b9b4f9d4b..fdc6d3383 100644 --- a/.github/workflows/monthly-copyright-update.yml +++ b/.github/workflows/monthly-copyright-update.yml @@ -7,8 +7,8 @@ on: type: boolean default: false schedule: - # * is a special character in YAML so quote this string - - cron: '0 0 1 * *' + # The copyright year only changes once a year, so this runs on 1 January. + - cron: '0 0 1 1 *' jobs: Monthly_Copyright_Update: diff --git a/.gitmodules b/.gitmodules index ed93e738d..f3fd5d50e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,3 +8,6 @@ path = pipeline/owid-java url = https://github.com/SWAN-community/owid-java.git branch = main +[submodule "pipeline/pipeline.core/configuration-validation"] + path = pipeline/pipeline.core/configuration-validation + url = ../configuration-validation diff --git a/AGENTS.md b/AGENTS.md index 971eb5d05..b5125dfc6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,7 +125,7 @@ java -cp examples/ip-intelligence/console/target/ip-intelligence-java-examples.c ``` The example downloads the Lite file first when it is not there. In place of -the argument, the `51DEGREES_IPI_PATH` environment variable or system property +the argument, the `_51DEGREES_IPI_PATH` environment variable or system property can give the absolute path of a data file that is already there. On Java 24 or later, add `--enable-native-access=ALL-UNNAMED` before the class @@ -183,7 +183,7 @@ java -cp examples/device-detection/console/target/device-detection-java-examples It prints the same `Input values:` and `Results:` blocks as the on-premise example. The IP Intelligence cloud examples read their key from -`51DEGREES_RESOURCE_KEY` first and `TestResourceKey` second, and +`_51DEGREES_RESOURCE_KEY` first and `TestResourceKey` second, and `examples/ip-intelligence/README.md` gives the keys to create for them. ### Pitfalls @@ -246,8 +246,11 @@ from wherever Maven is run. Do not add a `.mvn` folder below the root. Those folders are the data repository, a submodule of cxx, where each free data file is a `.link` until `get-data-files.ps1` or `get-data-files.sh` -downloads it. `ci/fetch-assets.ps1` runs the script, and the examples' -`DataFileHelper.findDataFile` runs it for a file that is missing. Look for a +downloads it. In CI, `ci/fetch-assets.ps1` puts every file there from +common-ci's assets, which each run caches once, so no test job downloads a +data file. A file added to the data repository needs an entry in +common-ci's `steps/fetch-assets.ps1` before CI can use it. The examples' +`DataFileHelper.findDataFile` runs the script for a file that is missing. Look for a free file by its folder and name, such as `device-detection/51Degrees-LiteV4.1.hash`, so the search does not stop at a file of the same name elsewhere, such as a Git LFS pointer in an old diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e6c5a12c4..ea856fa88 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,5 @@ # Contributing -If you wish to contribute to this project, please see the shared 51Degrees -[guidance](https://github.com/51Degrees/common-ci/blob/main/CONTRIBUTING.md). +Contributions are welcome. The [51Degrees contributing guide](https://github.com/51Degrees/.github/blob/main/CONTRIBUTING.md) +explains where a change goes and how to propose it. It applies to every +package in this repository. diff --git a/LICENSE b/LICENSE index 4153cd377..a74f04438 100644 --- a/LICENSE +++ b/LICENSE @@ -1,53 +1,44 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 +European Union Public Licence +V. 1.2 -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). +EUPL © the European Union 2007, 2016 + +This European Union Public Licence (the ‘EUPL’) applies to the Work (as +defined below) which is provided under the terms of this Licence. Any use of +the Work, other than as authorised under this Licence is prohibited (to the +extent such use is covered by a right of the copyright holder of the Work). The Work is provided under the terms of this Licence when the Licensor (as defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. +copyright notice for the Work: “Licensed under the EUPL”, or has expressed by +any other means his willingness to license under the EUPL. 1. Definitions In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most +— ‘The Licence’: this Licence. +— ‘The Original Work’: the work or software distributed or communicated by the + ‘Licensor under this Licence, available as Source Code and also as + ‘Executable Code as the case may be. +— ‘Derivative Works’: the works or software that could be created by the + ‘Licensee, based upon the Original Work or modifications thereof. This + ‘Licence does not define the extent of modification or dependence on the + ‘Original Work required in order to classify a work as a Derivative Work; + ‘this extent is determined by copyright law applicable in the country + ‘mentioned in Article 15. +— ‘The Work’: the Original Work or its Derivative Works. +— ‘The Source Code’: the human-readable form of the Work which is the most convenient for people to study and modify. -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates +— ‘The Executable Code’: any code which has generally been compiled and which + is meant to be interpreted by a computer as a program. +— ‘The Licensor’: the natural or legal person that distributes or communicates the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of +— ‘Contributor(s)’: any natural or legal person who modifies the Work under + the Licence, or otherwise contributes to the creation of a Derivative Work. +— ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, +— ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, renting, distributing, communicating, transmitting, or otherwise making available, online or offline, copies of the Work or providing access to its essential functionalities at the disposal of any other natural or legal @@ -56,46 +47,46 @@ In this Licence, the following terms have the following meaning: 2. Scope of the rights granted by the Licence The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. +sublicensable licence to do the following, for the duration of copyright +vested in the Original Work: + +— use the Work in any circumstance and for all usage, +— reproduce the Work, +— modify the Work, and make Derivative Works based upon the Work, +— communicate to the public, including the right to make available or display + the Work or copies thereof to the public and perform publicly, as the case + may be, the Work, +— distribute the Work or copies thereof, +— lend and rent the Work or copies thereof, +— sublicense rights in the Work or copies thereof. Those rights can be exercised on any media, supports and formats, whether now known or later invented, as far as the applicable law permits so. In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. +exercise his moral right to the extent allowed by law in order to make +effective the licence of the economic rights here above listed. -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. +The Licensor grants to the Licensee royalty-free, non-exclusive usage rights +to any patents held by the Licensor, to the extent necessary to make use of +the rights granted on the Work under this Licence. 3. Communication of the Source Code The Licensor may provide the Work either in its Source Code form, or as Executable Code. If the Work is provided as Executable Code, the Licensor provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. +along with each copy of the Work that the Licensor distributes or indicates, +in a notice following the copyright notice attached to the Work, a repository +where the Source Code is easily and freely accessible for as long as the +Licensor continues to distribute or communicate the Work. 4. Limitations on copyright -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. +Nothing in this Licence is intended to deprive the Licensee of the benefits +from any exception or limitation to the exclusive rights of the rights owners +in the Work, of the exhaustion of those rights or of other applicable +limitations thereto. 5. Obligations of the Licensee @@ -104,36 +95,37 @@ obligations imposed on the Licensee. Those obligations are the following: Attribution right: The Licensee shall keep intact all copyright, patent or trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or +disclaimer of warranties. The Licensee must include a copy of such notices and +a copy of the Licence with every copy of the Work he/she distributes or communicates. The Licensee must cause any Derivative Work to carry prominent notices stating that the Work has been modified and the date of modification. Copyleft clause: If the Licensee distributes or communicates copies of the Original Works or Derivative Works, this Distribution or Communication will be done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. +unless the Original Work is expressly distributed only under this version of +the Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee +(becoming Licensor) cannot offer or impose any additional terms or conditions +on the Work or Derivative Work that alter or restrict the terms of the +Licence. Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and +Works or copies thereof based upon both the Work and another work licensed +under a Compatible Licence, this Distribution or Communication can be done +under the terms of this Compatible Licence. For the sake of this clause, +‘Compatible Licence’ refers to the licences listed in the appendix attached to +this Licence. Should the Licensee's obligations under the Compatible Licence +conflict with his/her obligations under this Licence, the obligations of the +Compatible Licence shall prevail. + +Provision of Source Code: When distributing or communicating copies of the +Work, the Licensee will provide a machine-readable copy of the Source Code or +indicate a repository where this Source will be easily and freely available +for as long as the Licensee continues to distribute or communicate the Work. + +Legal Protection: This Licence does not grant permission to use the trade +names, trademarks, service marks, or names of the Licensor, except as required +for reasonable and customary use in describing the origin of the Work and reproducing the content of the copyright notice. 6. Chain of Authorship @@ -142,9 +134,9 @@ The original Licensor warrants that the copyright in the Original Work granted hereunder is owned by him/her or licensed to him/her and that he/she has the power and authority to grant the Licence. -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. +Each Contributor warrants that the copyright in the modifications he/she +brings to the Work are owned by him/her or licensed to him/her and that he/she +has the power and authority to grant the Licence. Each time You accept the Licence, the original Licensor and subsequent Contributors grant You a licence to their contributions to the Work, under the @@ -156,66 +148,66 @@ The Work is a work in progress, which is continuously improved by numerous Contributors. It is not a finished work and may therefore contain defects or ‘bugs’ inherent to this type of development. -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. +For the above reason, the Work is provided under the Licence on an ‘as is’ +basis and without warranties of any kind concerning the Work, including +without limitation merchantability, fitness for a particular purpose, absence +of defects or errors, accuracy, non-infringement of intellectual property +rights other than copyright as stated in Article 6 of this Licence. -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. +This disclaimer of warranty is an essential part of the Licence and a +condition for the grant of any rights to the Work. 8. Disclaimer of Liability Except in the cases of wilful misconduct or damages directly caused to natural persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. +material or moral, damages of any kind, arising out of the Licence or of the +use of the Work, including without limitation, damages for loss of goodwill, +work stoppage, computer failure or malfunction, loss of data or any commercial +damage, even if the Licensor has been advised of the possibility of such +damage. However, the Licensor will be liable under statutory product liability +laws as far such laws apply to the Work. 9. Additional agreements -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. +While distributing the Work, You may choose to conclude an additional +agreement, defining obligations or services consistent with this Licence. +However, if accepting obligations, You may act only on your own behalf and on +your sole responsibility, not on behalf of the original Licensor or any other +Contributor, and only if You agree to indemnify, defend, and hold each +Contributor harmless for any liability incurred by, or claims asserted against +such Contributor by the fact You have accepted any warranty or additional +liability. 10. Acceptance of the Licence -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable +The provisions of this Licence can be accepted by clicking on an icon ‘I +agree’ placed under the bottom of a window displaying the text of this Licence +or by affirming consent in any other similar way, in accordance with the rules +of applicable law. Clicking on that icon indicates your clear and irrevocable acceptance of this Licence and all of its terms and conditions. Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. +conditions by exercising any rights granted to You by Article 2 of this +Licence, such as the use of the Work, the creation by You of a Derivative Work +or the Distribution or Communication by You of the Work or copies thereof. 11. Information to the public -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. +In case of any Distribution or Communication of the Work by means of +electronic communication by You (for example, by offering to download the Work +from a remote location) the distribution channel or media (for example, a +website) must at least provide to the public the information requested by the +applicable law regarding the Licensor, the Licence and the way it may be +accessible, concluded, stored and reproduced by the Licensee. 12. Termination of the Licence The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. +any breach by the Licensee of the terms of the Licence. Such a termination +will not terminate the licences of any person who has received the Work from +the Licensee under the Licence, provided such persons remain in full +compliance with the Licence. 13. Miscellaneous @@ -227,10 +219,11 @@ law, this will not affect the validity or enforceability of the Licence as a whole. Such provision will be construed or reformed so as necessary to make it valid and enforceable. -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. +The European Commission may publish other linguistic versions or new versions +of this Licence or updated versions of the Appendix, so far this is required +and reasonable, without reducing the scope of the rights granted by the +Licence. New versions of the Licence will be published with a unique version +number. All linguistic versions of this Licence, approved by the European Commission, have identical value. Parties can take advantage of the linguistic version of @@ -239,49 +232,43 @@ their choice. 14. Jurisdiction Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising +— any litigation resulting from the interpretation of this License, arising between the European Union institutions, bodies, offices or agencies, as a Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. + of Justice of the European Union, as laid down in article 272 of the Treaty + on the Functioning of the European Union, +— any litigation arising between other parties and resulting from the + interpretation of this License, will be subject to the exclusive + jurisdiction of the competent court where the Licensor resides or conducts + its primary business. 15. Applicable Law Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State +— this Licence shall be governed by the law of the European Union Member State where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, +— this licence shall be governed by Belgian law if the Licensor has no seat, residence or registered office inside a European Union Member State. Appendix ‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for +— GNU General Public License (GPL) v. 2, v. 3 +— GNU Affero General Public License (AGPL) v. 3 +— Open Software License (OSL) v. 2.1, v. 3.0 +— Eclipse Public License (EPL) v. 1.0 +— CeCILL v. 2.0, v. 2.1 +— Mozilla Public Licence (MPL) v. 2 +— GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 +— Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. +— European Union Public Licence (EUPL) v. 1.1, v. 1.2 +— Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or + Strong Reciprocity (LiLiQ-R+) + +— The European Commission may update this Appendix to later versions of the + above licences without producing a new version of the EUPL, as long as they + provide the rights granted in Article 2 of this Licence and protect the + covered Source Code from exclusive appropriation. +— All other changes or additions to this Appendix require the production of a + new EUPL version. diff --git a/README.md b/README.md index 479f1ff36..df1cd6427 100644 --- a/README.md +++ b/README.md @@ -29,10 +29,10 @@ Created from `pipeline-java`, `device-detection-java`, `ip-intelligence-java`, ` | Product | Read first | Documentation | | --- | --- | --- | -| Pipeline API | [pipeline/README.md](pipeline/README.md) | [Java](https://51degrees.com/documentation/apis/pipeline-java/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=pipeline-java) | -| Device Detection | [device-detection/README.md](device-detection/README.md) | [Java](https://51degrees.com/documentation/apis/device-detection-java/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=device-detection-java) | -| IP Intelligence | [ip-intelligence/README.md](ip-intelligence/README.md) | [Java](https://51degrees.com/documentation/apis/ip-intelligence-java/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=ip-intelligence-java) | -| Location | [location/README.md](location/README.md) | [Java](https://51degrees.com/documentation/apis/location-java/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=location-java) | +| Pipeline API | [pipeline/README.md](pipeline/README.md) | [Guide](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=pipeline-guide) and [Java API](https://51degrees.com/documentation/apis/pipeline-java/annotated.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=pipeline-java) | +| Device Detection | [device-detection/README.md](device-detection/README.md) | [Guide](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=device-detection-guide) and [Java API](https://51degrees.com/documentation/apis/device-detection-java/annotated.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=device-detection-java) | +| IP Intelligence | [ip-intelligence/README.md](ip-intelligence/README.md) | [Guide](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=ip-intelligence-guide) and [Java API](https://51degrees.com/documentation/apis/ip-intelligence-java/annotated.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=ip-intelligence-java) | +| Location | [location/README.md](location/README.md) | [Guide](https://51degrees.com/documentation/_reverse_geocoding__quickstart.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=location-guide) and [Java API](https://51degrees.com/documentation/apis/location-java/annotated.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=location-java) | ## What is where diff --git a/ci/README.md b/ci/README.md index bf6b2a62b..b00a5034a 100644 --- a/ci/README.md +++ b/ci/README.md @@ -12,21 +12,22 @@ repositories used to do for one package tree each. generated accessors. The other scripts loop over it rather than naming folders themselves. - The root `pom.xml` builds every tree in one reactor but is not their parent, - so [build-package.ps1](build-package.ps1) sets the version in each tree - before it deploys from the root. The examples are built and tested but left - out of the deploy. + so [build-package.ps1](build-package.ps1) has common-ci set the version in + each tree before it deploys from the root. The examples are built and tested + but left out of the deploy. - [build-project.ps1](build-project.ps1) and [run-unit-tests.ps1](run-unit-tests.ps1) use the `dev` profile, so a pull request tests the examples against the libraries it changes. The publish workflow instead tests the examples against the version it built, through [run-integration-tests.ps1](run-integration-tests.ps1). -- [fetch-assets.ps1](fetch-assets.ps1) links every data file it downloads into +- [fetch-assets.ps1](fetch-assets.ps1) fetches every data file as one of + common-ci's assets, which each run caches once, and links or copies it into `cxx/data/device-detection` or `cxx/data/ip-intelligence`, where the tests and examples find them. Those are the folders of the [data](https://github.com/51Degrees/data) repository, which is a submodule - of cxx. The script then runs the data repository's `get-data-files.ps1`, - which keeps the linked files and downloads the free files that are not - assets, which is the Asn file. + of cxx. No test job downloads a data file, so a file added to the data + repository needs an entry in common-ci's `steps/fetch-assets.ps1` before CI + can use it. - [run-performance-tests.ps1](run-performance-tests.ps1) measures the Pipeline API, Device Detection and IP Intelligence and writes one results file per configuration. The Device Detection and IP Intelligence metrics are prefixed diff --git a/ci/build-package.ps1 b/ci/build-package.ps1 index 38de86fe9..75464f09f 100644 --- a/ci/build-package.ps1 +++ b/ci/build-package.ps1 @@ -21,19 +21,6 @@ foreach ($package in $NativePackages) { } } -# The root pom.xml is not the parent of the package trees, so setting the -# version at the root does not reach them. Set it in each tree, the examples -# included, so the whole repository carries the version being released. -foreach ($package in $Packages) { - Write-Host "Setting the version of '$($package.Path)' to '$Version'" - Push-Location "$RepoName/$($package.Path)" - try { - mvn versions:set @MavenOptions "-DnewVersion=$Version" -DgenerateBackupPoms=false - } finally { - Pop-Location - } -} - # The examples are never published. They are left out of the deploy so that # the last project in the reactor is one the publishing plugin is configured # for. Maven 3 does not exclude a folder's modules with the folder, so each of @@ -45,21 +32,18 @@ $examples = foreach ($package in $Packages | Where-Object { -not $_.Publish }) { ForEach-Object { '!' + [IO.Path]::GetRelativePath($repoPath, $_.DirectoryName).Replace('\', '/') } } -# The publishing plugin writes the one bundle for the whole reactor under the -# first tree that publishes. common-ci copies the bundle from where a single -# tree puts it, below target at the root, so a link at that path leads to the -# bundle. The link is made before the deploy, which Linux allows. -$firstPublishing = ($Packages | Where-Object { $_.Publish } | Select-Object -First 1).Path -New-Item -ItemType Directory -Force -Path "$RepoName/target" | Out-Null -New-Item -ItemType SymbolicLink -Force -Path "$RepoName/target/central-publishing" -Target "../$firstPublishing/target/central-publishing" | Out-Null - -# common-ci sets the version of the root pom.xml, deploys from the root to a -# local staging folder with signing, and copies the result and the publishing -# bundle into the package folder. +# common-ci sets the version, deploys from the root to a local staging folder +# with signing, and copies the result and the publishing bundle into the +# package folder. The root pom.xml is not the parent of the package trees, so +# setting the version at the root would not reach them. VersionPaths names each +# tree, the examples included, so the whole repository carries the version +# being released. The publishing plugin writes the one bundle for the whole +# reactor under the first tree that publishes, and common-ci finds it there. ./java/build-package.ps1 ` -RepoName $RepoName ` -Name $Name ` -Version $Version ` + -VersionPaths $Packages.Path ` -ExtraArgs "-DskipNativeBuild=true", "-pl", ($examples -join ',') ` -JavaGpgKeyPassphrase $Keys['JavaGpgKeyPassphrase'] ` -JavaPGP $Keys['JavaPGP'] ` diff --git a/ci/check-environment-variable-names.Tests.ps1 b/ci/check-environment-variable-names.Tests.ps1 new file mode 100644 index 000000000..84460b9a8 --- /dev/null +++ b/ci/check-environment-variable-names.Tests.ps1 @@ -0,0 +1,151 @@ +# Tests for ci/check-environment-variable-names.ps1. +# +# Each case is one line of text and what the check must make of it. The +# digit-initial name is matched in any case, because Windows reads an +# environment variable name in any case, so a lower case old name is still +# the old name there. The old names keep their case, for the reason the check +# gives. + +BeforeAll { + $script:Check = Join-Path $PSScriptRoot 'check-environment-variable-names.ps1' + . $script:Check + + function Get-Kind([string]$Line) { + @(Find-EnvironmentNameProblem $Line | ForEach-Object { $_.Kind }) + } +} + +Describe 'Find-EnvironmentNameProblem' { + + It 'finds the digit-initial name ' -ForEach @( + @{ Line = 'System.getenv("51DEGREES_IPI_PATH")' } + @{ Line = 'public static final String IPI = "51DEGREES_IPI_PATH";' } + @{ Line = '${51DEGREES_DD_PATH:-lite.hash}' } + @{ Line = "'51DEGREES_DD_PATH' = (Resolve-Path `$file).Path" } + @{ Line = 'System.getenv("51degrees_ipi_path")' } + @{ Line = '$env:51Degrees_Resource_Key = "key"' } + ) { + Get-Kind $Line | Should -Contain 'digit' + } + + It 'finds the old name ' -ForEach @( + @{ Line = 'System.getenv("RESOURCE_KEY")' } + @{ Line = 'env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY");' } + @{ Line = 'env("_51DEGREES_LICENSE_KEY", "LICENSE_KEY");' } + @{ Line = 'public static final String FOD = "FOD_CLOUD_API_URL";' } + @{ Line = '$env:LICENSE_KEY = $Keys.TestResourceKey' } + @{ Line = '${env:LICENSE_KEY} = $Keys.TestResourceKey' } + @{ Line = '${Env:RESOURCE_KEY}' } + @{ Line = '$ENV:RESOURCE_KEY = "key"' } + @{ Line = 'Remove-Item Env:\LICENSE_KEY' } + @{ Line = '${env:FOD_CLOUD_API_URL}' } + @{ Line = ' url: ${{ env.FOD_CLOUD_API_URL }}' } + @{ Line = '${FOD_CLOUD_API_URL}' } + @{ Line = 'Constants.LEGACY_ENV_VARS' } + @{ Line = ' LICENSE_KEY: ${{ secrets.DEVICE_DETECTION_KEY }}' } + @{ Line = '- RESOURCE_KEY names the resource key' } + @{ Line = 'RESOURCE_KEY=my-key mvn test' } + ) { + Get-Kind $Line | Should -Contain 'fallback' + } + + # The cloud request engine's tests name the query string parameter + # resource_key, which is on the wire and not in the environment, so the + # old names keep their case. + It 'leaves alone' -ForEach @( + @{ Line = 'assertEquals("resource_key=" + key, query);' } + @{ Line = ' DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }}' } + @{ Line = ' PAID_RESOURCE_KEY: ${{ secrets.X }}' } + @{ Line = 'System.getenv("_51DEGREES_RESOURCE_KEY")' } + @{ Line = 'System.getenv("_51DEGREES_CLOUD_ENDPOINT")' } + @{ Line = '$env:_51DEGREES_DD_PATH = "file.hash"' } + @{ Line = '$env:_51degrees_dd_path = "file.hash"' } + @{ Line = '${env:_51DEGREES_LICENSE_KEY} = $Keys.TestResourceKey' } + @{ Line = 'assertEquals("${env:user.home}", substitutor.replace(line));' } + @{ Line = ' PAID_RESOURCE_KEY: ${{ env.PAID_RESOURCE_KEY }}' } + @{ Line = 'System.getenv("SUPER_RESOURCE_KEY")' } + @{ Line = 'Set the key with !!YOUR_RESOURCE_KEY!! in the readme.' } + ) { + Get-Kind $Line | Should -BeNullOrEmpty + } +} + +Describe 'the check over a repository' { + + # git quotes a path holding a non-ASCII character unless told not to, + # and a quoted path is not found on disk. + It 'reads a file whose name is not ASCII' { + $repo = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid()) + New-Item -ItemType Directory $repo | Out-Null + try { + git -C $repo init -q + $name = 'Donn' + [char]0xE9 + 'es.java' + Set-Content -LiteralPath (Join-Path $repo $name) ` + -Value 'System.getenv("51DEGREES_DD_PATH");' + git -C $repo add -A + { & $script:Check -RepoRoot $repo } | + Should -Throw '1 names start with a digit*' + } finally { + Remove-Item -Recurse -Force $repo + } + } + + It 'passes an old name in a file that reads it as a deprecated fallback' { + $repo = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid()) + New-Item -ItemType Directory $repo | Out-Null + try { + git -C $repo init -q + $file = Join-Path $repo 'pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java' + New-Item -ItemType Directory (Split-Path $file) -Force | Out-Null + Set-Content -LiteralPath $file ` + -Value 'String ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL";' + git -C $repo add -A + { & $script:Check -RepoRoot $repo } | Should -Not -Throw + } finally { + Remove-Item -Recurse -Force $repo + } + } + + It 'fails on the same old name in a file that is not named for it' { + $repo = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid()) + New-Item -ItemType Directory $repo | Out-Null + try { + git -C $repo init -q + Set-Content -LiteralPath (Join-Path $repo 'Other.java') ` + -Value 'String ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL";' + git -C $repo add -A + { & $script:Check -RepoRoot $repo } | + Should -Throw '*and 1 old names are read*' + } finally { + Remove-Item -Recurse -Force $repo + } + } +} + +Describe 'Test-DeprecatedFallback' { + + # An old name is allowed only in a file that reads it as a deprecated + # fallback or tests that read, and only the names listed for that file. + It 'allows in ' -ForEach @( + @{ Path = 'pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java'; Value = '"FOD_CLOUD_API_URL' } + @{ Path = 'pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java'; Value = '"RESOURCE_KEY' } + @{ Path = 'pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java'; Value = '"LICENSE_KEY' } + @{ Path = 'ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java'; Value = '51DEGREES_IPI_PATH' } + @{ Path = 'examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml'; Value = '51DEGREES_DD_PATH' } + ) { + Test-DeprecatedFallback $Path $Value | Should -BeTrue + } + + It 'refuses in ' -ForEach @( + @{ Path = 'pipeline/pipeline.core/src/main/java/Any.java'; Value = '"FOD_CLOUD_API_URL' } + @{ Path = 'ci/run-unit-tests.ps1'; Value = 'env:RESOURCE_KEY' } + @{ Path = 'pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java'; Value = '"RESOURCE_KEY' } + @{ Path = 'pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java'; Value = 'LEGACY_ENV_VARS' } + @{ Path = 'pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java'; Value = '"FOD_CLOUD_API_URL' } + @{ Path = 'ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java'; Value = '51DEGREES_DD_PATH' } + @{ Path = 'ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java'; Value = '51degrees_ipi_path' } + @{ Path = 'examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml'; Value = '51DEGREES_IPI_PATH' } + ) { + Test-DeprecatedFallback $Path $Value | Should -BeFalse + } +} diff --git a/ci/check-environment-variable-names.ps1 b/ci/check-environment-variable-names.ps1 new file mode 100644 index 000000000..11d75e85c --- /dev/null +++ b/ci/check-environment-variable-names.ps1 @@ -0,0 +1,203 @@ +# Every 51Degrees environment variable name in this repository starts with an +# underscore, because a POSIX shell cannot set a name that starts with a digit, +# and each variable has one name. An old name is still read, as a deprecated +# fallback for when its new name is not set, by the files listed in +# DeprecatedFallbacks below and by nothing else. This reads every file git +# tracks, which leaves out build output and the submodules, and fails on any +# name that starts with the digit and on any old name, outside the files +# listed for that name. +# +# Dot-sourcing the script defines Find-EnvironmentNameProblem and +# Test-DeprecatedFallback without reading the repository, which is how its +# tests use them. + +[CmdletBinding()] +param( + # The root of the checkout to read. The default is the repository this + # script lives in. + [string]$RepoRoot = (Join-Path $PSScriptRoot '..') +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# The start of a name a shell cannot set, split so this file does not match +# itself. +$digitInitial = '5' + '1DEGREES_' + +# A name is not matched when a name character comes before it, so a name with +# a leading underscore is left alone. It is matched in any case, because +# Windows reads an environment variable name in any case, so a lower case +# name that starts with the digit is still one a shell cannot set. +$script:DigitInitialPattern = [regex]::new( + "(?&1 & } finally { Pop-Location } try { - # SELENIUM_TESTS_REF picks a branch of the suite, so a change there - # can be tried here before it is merged. - $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } - if (-not (Test-Path selenium-api-tests)) { - Write-Host "Cloning the Selenium contract tests from '$seleniumRef'" - git clone --depth 1 --branch $seleniumRef https://github.com/51Degrees/selenium-api-tests.git - } - # Wait for the example to come up. - curl -sS -o $(if ($IsWindows) { 'NUL' } else { '/dev/null' }) --retry 20 --retry-delay 3 --retry-max-time 120 --retry-connrefused "http://localhost:$port" - if ($LASTEXITCODE -ne 0) { throw "The cloud web example did not start on port $port." } + curl -sS -o $(if ($IsWindows) { 'NUL' } else { '/dev/null' }) --retry 20 --retry-delay 3 --retry-max-time 120 --retry-connrefused "http://localhost:$Port" + if ($LASTEXITCODE -ne 0) { throw "The $Label web example did not start on port $Port." } - $env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/" - $env:PAID_RESOURCE_KEY = $TestResourceKey - $env:EXAMPLE_URL = "http://localhost:$port" + $env:EXAMPLE_URL = "http://localhost:$Port" $env:EXAMPLE_LANG = 'java' dotnet test selenium-api-tests -c Release --filter TestCategory=Contract # PowerShell does not stop for a native command that returns non-zero, # so without this the browser tests could fail every case and the job # would still be reported as a success. if ($LASTEXITCODE -ne 0) { - throw "The Selenium contract tests failed, returning $LASTEXITCODE. Their output is above." + throw "The Selenium contract tests failed against the $Label example, returning $LASTEXITCODE. Their output is above." } } catch { - if ($example) { Write-Host '>>> example app output >>>'; Receive-Job $example | Out-Host; Write-Host '<<< app output <<<' } + if ($example) { Write-Host ">>> $Label example output >>>"; Receive-Job $example | Out-Host; Write-Host "<<< $Label example output <<<" } throw } finally { if ($example) { Remove-Job -Force $example } + Remove-Item env:PORT, env:EXAMPLE_URL -ErrorAction SilentlyContinue + foreach ($name in $ExampleEnv.Keys) { + Remove-Item "env:$name" -ErrorAction SilentlyContinue + } } } Write-Host "::group::Selenium contract tests" if (-not $TestResourceKey) { - Write-Host "::warning title=Selenium skipped::No resource key, so the Selenium contract tests were not run against the cloud example." + Write-Host "::warning title=Selenium skipped::No resource key, so the Selenium contract tests were not run against the web examples." } elseif ($IsLinux -and [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture -eq [System.Runtime.InteropServices.Architecture]::Arm64) { Write-Host "::warning title=Selenium skipped::Selenium contract skipped on linux-arm64, which has no selenium-manager build. It runs on x64 and macOS arm64." } else { - try { - Invoke-ContractTests - } catch { - Write-Host "::error title=Selenium contract failed::$_" - $failed += 'Selenium contract tests' + # Get the shared contract tests. SELENIUM_TESTS_REF picks a branch of the + # suite, so a change there can be tried here before it is merged. + $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } + if (-not (Test-Path selenium-api-tests)) { + Write-Host "Cloning the Selenium contract tests from '$seleniumRef'" + git clone --depth 1 --branch $seleniumRef https://github.com/51Degrees/selenium-api-tests.git + } + $env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/" + $env:PAID_RESOURCE_KEY = $TestResourceKey + + $contracts = @( + @{ + Label = 'cloud' + Module = 'web/getting-started.cloud' + Port = 8099 + ExampleEnv = @{ + TestCloudEndpoint = "https://cloud.51degrees.com/api/v4" + TestResourceKey = $TestResourceKey + } + } + # The on-premise example, against the TAC data file that + # fetch-assets.ps1 links into the data repository. The Lite data file + # has neither DeviceType nor the JavaScript properties the contract + # tests need. + @{ + Label = 'on-premise' + Module = 'web/getting-started.onprem' + Port = 8098 + ExampleEnv = @{ + '_51DEGREES_DD_PATH' = (Resolve-Path "$RepoName/cxx/data/device-detection/TAC-HashV41.hash").Path + } + } + ) + # Each example is tested whether or not the one before it failed. + foreach ($contract in $contracts) { + try { + Invoke-ContractTests @contract + } catch { + Write-Host "::error title=Selenium contract failed::$_" + $failed += "Selenium contract tests, $($contract.Label) example" + } } } Write-Host "::endgroup::" diff --git a/ci/run-unit-tests.ps1 b/ci/run-unit-tests.ps1 index 562f29486..a54755337 100644 --- a/ci/run-unit-tests.ps1 +++ b/ci/run-unit-tests.ps1 @@ -4,39 +4,21 @@ param( [string]$TestResourceKey, [string]$DeviceDetection ) -$PSNativeCommandUseErrorActionPreference = $false - -. "$PSScriptRoot/packages.ps1" +$ErrorActionPreference = "Stop" # One run of every test in the reactor that build-project.ps1 built, the -# examples included. SuperResourceKey is read by the Location and Device -# Detection examples tests, and LicenseKey by the Device Detection data file -# update example. The IP Intelligence update example's test is ignored, so it -# does not need a key of its own. -Write-Host "Entering '$RepoName'" -Push-Location $RepoName -try { - Write-Host "Testing '$Name'" - mvn surefire:test @MavenOptions -Pdev -DfailIfNoTests=false ` - "-DTestResourceKey=$TestResourceKey" ` - "-DSuperResourceKey=$TestResourceKey" ` - "-DLicenseKey=$DeviceDetection" - # Kept here because the commands below run commands of their own, which - # would replace Maven's exit code. - $MavenExitCode = $LASTEXITCODE -} finally { - Write-Host "Leaving '$RepoName'" - Pop-Location -} - -& "$PSScriptRoot/copy-test-results.ps1" -Path $RepoName -Destination "$RepoName/test-results/unit" -Exclude '*ExampleTests*' - -# A forked JVM that dies leaves why it died only on the runner, so common-ci -# logs it. The collector is only in branches of common-ci that have it, so it -# is called only when present. It does not change the outcome. -$Collector = [IO.Path]::Combine($pwd, "java", "collect-jvm-crash-evidence.ps1") -if ($MavenExitCode -ne 0 -and (Test-Path $Collector) -eq $true) { - & $Collector -RepoName $RepoName -} +# examples included. common-ci runs Surefire from the root, copies the reports +# of every module at any depth into test-results/unit, leaving out the example +# tests the integration tests report, and logs why a forked JVM died when one +# does. The dev profile points the examples at the reactor. SuperResourceKey is +# read by the Location and Device Detection examples tests, and LicenseKey by +# the Device Detection data file update example. The IP Intelligence update +# example's test is ignored, so it does not need a key of its own. +./java/run-unit-tests.ps1 -RepoName $RepoName -Name $Name -ExtraArgs @( + '-Pdev' + "-DTestResourceKey=$TestResourceKey" + "-DSuperResourceKey=$TestResourceKey" + "-DLicenseKey=$DeviceDetection" +) -exit $MavenExitCode +exit $LASTEXITCODE diff --git a/cxx b/cxx index e866f6d3c..7992af701 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit e866f6d3cea936587790f87967eff7863778b30e +Subproject commit 7992af7010b7654c115510938a2f7626204bf936 diff --git a/device-detection/LICENSE b/device-detection/LICENSE deleted file mode 100644 index 4153cd377..000000000 --- a/device-detection/LICENSE +++ /dev/null @@ -1,287 +0,0 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. diff --git a/device-detection/README.md b/device-detection/README.md index 5441e364a..f1e0a9e9e 100644 --- a/device-detection/README.md +++ b/device-detection/README.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=top "Data rewards the curious") **Java Device Detection** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-java/main/.github/images/logo-160.png "Data rewards the curious") **Java Device Detection** -[Developer Documentation](https://51degrees.com/device-detection-java/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=top "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=top "developer documentation") ## Introduction @@ -10,7 +10,6 @@ This repository contains Java implementation of the device detection [specificat ## Dependencies -For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page. The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows the JDK versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. diff --git a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java index b6ea1aa59..400ca1dad 100644 --- a/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java +++ b/device-detection/device-detection.cloud/src/main/java/fiftyone/devicedetection/cloud/flowelements/DeviceDetectionCloudEngine.java @@ -55,9 +55,10 @@ */ public class DeviceDetectionCloudEngine extends CloudAspectEngineBase { - List aspectProperties; + volatile List aspectProperties; private String dataSourceTier; CloudRequestEngine cloudRequestEngine; + private final Object metadataLock = new Object(); /** * Construct a new instance of the {@link DeviceDetectionCloudEngine}. @@ -72,9 +73,37 @@ public DeviceDetectionCloudEngine( this.cloudRequestEngine = null; } + /** + * The property metadata, from the products the Cloud Request Engine + * reports. Until they are known, after a failure at build that can + * pass, this is empty and is read again on the next call. + * @return the property metadata + */ @Override public List getProperties() { - return aspectProperties; + List loaded = aspectProperties; + if (loaded == null && cloudRequestEngine != null) { + synchronized (metadataLock) { + if (aspectProperties == null) { + loadAspectProperties(cloudRequestEngine); + } + loaded = aspectProperties; + } + } + return loaded == null ? + new ArrayList() : + loaded; + } + + /** + * Whether the property metadata is loaded, which reads it if the Cloud + * Request Engine now knows the accessible properties. + * @return true once the property metadata is loaded + */ + @Override + public boolean hasLoadedProperties() { + getProperties(); + return aspectProperties != null; } @Override @@ -111,7 +140,7 @@ protected void processEngine(FlowData data, DeviceDataCloud aspectData) { // Extract data from json to the aspectData instance. JSONObject jsonObj = new JSONObject(json); - JSONObject deviceObj = jsonObj.getJSONObject("device"); + JSONObject deviceObj = getElementJson(jsonObj); Map deviceMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); @@ -188,9 +217,10 @@ public void addPipeline(Pipeline pipeline) { if (cloudRequestEngine == null) { cloudRequestEngine = pipeline.getElement(CloudRequestEngine.class); if (cloudRequestEngine != null) { - if (loadAspectProperties(cloudRequestEngine) == false) { - logger.error("Failed to load aspect properties"); - } + // The products are not known after a failure at build that + // can pass, which the Cloud Request Engine has logged. + // getProperties reads them once they are. + loadAspectProperties(cloudRequestEngine); } } @@ -205,17 +235,15 @@ public void addPipeline(Pipeline pipeline) { * {@link #aspectProperties} */ private boolean loadAspectProperties(CloudRequestEngine engine) { - Map map = - engine.getPublicProperties(); + AccessiblePropertyMetaData.ProductMetaData product = + findProduct(engine.getPublicProperties()); - if (map != null && - map.size() > 0 && - map.containsKey(getElementDataKey())) { - aspectProperties = new ArrayList<>(); - dataSourceTier = map.get(getElementDataKey()).dataTier; + if (product != null) { + List properties = new ArrayList<>(); + dataSourceTier = product.dataTier; for (AccessiblePropertyMetaData.PropertyMetaData item : - map.get(getElementDataKey()).properties) { + product.properties) { AspectPropertyMetaData property = new AspectPropertyMetaDataDefault( item.name, this, @@ -223,13 +251,12 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { item.getPropertyType(), new ArrayList(), true); - aspectProperties.add(property); + properties.add(property); } + aspectProperties = properties; return true; } else { - logger.error("Aspect properties could not be loaded for" + - " the Device Detection cloud engine", this); return false; } } diff --git a/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java new file mode 100644 index 000000000..2cd708eb6 --- /dev/null +++ b/device-detection/device-detection.cloud/src/test/java/fiftyone/devicedetection/cloud/flowelements/ProductMissingTests.java @@ -0,0 +1,255 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.cloud.flowelements; + +import fiftyone.devicedetection.cloud.data.DeviceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.Invocation; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A resource key can be valid and include no product that holds the + * Device Detection properties. The engine then builds with empty property + * metadata, logs the missing product once, and logs or throws nothing on + * each request. + */ +public class ProductMissingTests { + + private Logger logger; + private ILoggerFactory loggerFactory; + + @Before + public void init() { + logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void ProductMissing_BuildsWithEmptyMetadata() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + assertNotNull(engine.getProperties()); + assertEquals(0, engine.getProperties().size()); + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 1, logged.size()); + assertTrue(logged.get(0), logged.get(0).startsWith("warn ")); + assertTrue(logged.get(0), logged.get(0).contains("'device'")); + } + } + + @Test + public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + for (int i = 0; i < 3; i++) { + process(engine); + } + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 1, logged.size()); + } + } + + /** + * With the Device Detection product included, a response that holds no + * part for it leaves the element data without values and throws + * nothing, as the Processing section of the cloud aspect engine + * specification says. + */ + @Test + public void ProductPresent_ResponseWithoutPart_NoValues() + throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine("device", "hardwarevendor")) { + DeviceDataCloud data = process( + engine, + "{ 'ip': { 'registeredname': 'Example' }, " + + "'javascriptProperties': [] }"); + assertFalse(data.getHardwareVendor().hasValue()); + } + } + + /** + * Reading a property behaves as for any property the response does not + * carry. + */ + @Test + public void ProductMissing_PropertyIsMissing() throws Exception { + try (DeviceDetectionCloudEngine engine = buildEngine()) { + DeviceDataCloud device = process(engine); + try { + device.getHardwareVendor(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals( + MissingPropertyReason.ProductNotAccessibleWithResourceKey, + e.getReason()); + } + } + } + + /** + * After a failure at build that can pass, the Cloud Request Engine does + * not know the accessible products yet. The engine builds without + * logging anything, and reads its metadata once the products are known + * rather than keeping none. + */ + @Test + public void DiscoveryNotComplete_LoadsTheMetadataLater() throws Exception { + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("hardwarevendor", "String", "cat", null)); + Map products = new HashMap<>(); + products.put("device", product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()) + .thenReturn(null) + .thenReturn(products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + try (DeviceDetectionCloudEngine engine = + new DeviceDetectionCloudEngineBuilder(loggerFactory).build()) { + engine.addPipeline(pipeline); + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 0, logged.size()); + + assertEquals(1, engine.getProperties().size()); + assertEquals("hardwarevendor", engine.getProperties().get(0).getName()); + } + } + + /** + * An engine whose Cloud Request Engine reports one product, holding + * one string property. + */ + // thenReturn has a varargs overload, so a generic Map is an unchecked array. + @SuppressWarnings("unchecked") + private DeviceDetectionCloudEngine buildEngine( + String productKey, + String property) throws Exception { + DeviceDetectionCloudEngine engine = + new DeviceDetectionCloudEngineBuilder(loggerFactory).build(); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData(property, "String", "cat", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put(productKey, product); + + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + /** + * Process the given cloud response. + */ + private DeviceDataCloud process( + DeviceDetectionCloudEngine engine, + String response) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + + DeviceDataCloud device = new DeviceDataCloudInternal( + loggerFactory.getLogger(DeviceDataCloud.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + engine.processEngine(flowData, device); + return device; + } + + /** + * An engine on a resource key that includes IP Intelligence alone. + */ + private DeviceDetectionCloudEngine buildEngine() throws Exception { + return buildEngine("ip", "registeredname"); + } + + /** + * Process a cloud response that holds nothing for the Device Detection + * product. + */ + private DeviceDataCloud process(DeviceDetectionCloudEngine engine) { + return process( + engine, + "{ 'ip': { 'registeredname': 'Example' }, " + + "'javascriptProperties': [] }"); + } + + /** + * Every warning and error logged, at any overload. + */ + private List warningsAndErrors() { + List logged = new ArrayList<>(); + for (Invocation invocation : + Mockito.mockingDetails(logger).getInvocations()) { + String level = invocation.getMethod().getName(); + if (level.equals("warn") || level.equals("error")) { + logged.add(level + " " + + Arrays.toString(invocation.getArguments())); + } + } + return Collections.unmodifiableList(logged); + } +} diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java index 081205d3d..691761125 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataHashDefault.java @@ -68,6 +68,12 @@ public class DeviceDataHashDefault */ private final List resultsList = new ArrayList<>(); + /** + * The engine which created this instance, used to turn a property name + * into the required property index the engine was built with. + */ + private final DeviceDetectionHashEngine hashEngine; + /** * Constructs a new instance. * @param logger used for logging @@ -83,6 +89,18 @@ public class DeviceDataHashDefault DeviceDetectionHashEngine engine, MissingPropertyService missingPropertyService) { super(logger, flowData, engine, missingPropertyService); + this.hashEngine = engine; + } + + /** + * Get the required property index for a property name, or null when the + * engine has no index for that name, so the read falls back to the name. + * @param propertyName name of the property + * @return required property index, or null + */ + private Integer indexOf(String propertyName) { + return hashEngine == null ? + null : hashEngine.getPropertyIndex(propertyName); } /** @@ -176,13 +194,11 @@ private AspectPropertyValue getDifferenceInternal() { * @return drift */ private AspectPropertyValue getDriftInternal() { - int result = Integer.MAX_VALUE; + int total = 0; for (ResultsHashSwig results : resultsList) { - if (results.getDrift() < result) { - result = results.getDrift(); - } + total += results.getDrift(); } - return new AspectPropertyValueDefault<>(result); + return new AspectPropertyValueDefault<>(total); } /** @@ -258,8 +274,10 @@ public AspectPropertyValue> getValues(String propertyName) { new AspectPropertyValueDefault<>(); ResultsHashSwig results = getSingleResults(propertyName); if (results != null) { - try (VectorStringValuesSwig value = results.getValues( - Swig.asBytes(propertyName))) { + Integer index = indexOf(propertyName); + try (VectorStringValuesSwig value = index == null ? + results.getValues(Swig.asBytes(propertyName)) : + results.getValues(index)) { if (value.hasValue()) { try (VectorStringSwig vector = value.getValue()) { result.setValue(Collections.unmodifiableList( @@ -279,9 +297,10 @@ protected AspectPropertyValue getValueAsString(String propertyName) { AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsHashSwig results = getSingleResults(propertyName); if (results != null) { - - try (StringValueSwig value = results.getValueAsString( - Swig.asBytes(propertyName))) { + Integer index = indexOf(propertyName); + try (StringValueSwig value = index == null ? + results.getValueAsString(Swig.asBytes(propertyName)) : + results.getValueAsString(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } @@ -300,8 +319,10 @@ protected AspectPropertyValue getValueAsJavaScript( new AspectPropertyValueDefault<>(); ResultsHashSwig results = getSingleResults(propertyName); if (results != null) { - try (StringValueSwig value = results.getValueAsString( - Swig.asBytes(propertyName))) { + Integer index = indexOf(propertyName); + try (StringValueSwig value = index == null ? + results.getValueAsString(Swig.asBytes(propertyName)) : + results.getValueAsString(index)) { if (value.hasValue()) { result.setValue(new JavaScript(value.getValue())); } @@ -319,8 +340,10 @@ protected AspectPropertyValue getValueAsInteger( AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsHashSwig results = getSingleResults(propertyName); if (results != null) { - try (IntegerValueSwig value = results.getValueAsInteger( - Swig.asBytes(propertyName))) { + Integer index = indexOf(propertyName); + try (IntegerValueSwig value = index == null ? + results.getValueAsInteger(Swig.asBytes(propertyName)) : + results.getValueAsInteger(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } @@ -337,8 +360,10 @@ protected AspectPropertyValue getValueAsBool(String propertyName) { AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsHashSwig results = getSingleResults(propertyName); if (results != null) { - try (BoolValueSwig value = results.getValueAsBool( - Swig.asBytes(propertyName))) { + Integer index = indexOf(propertyName); + try (BoolValueSwig value = index == null ? + results.getValueAsBool(Swig.asBytes(propertyName)) : + results.getValueAsBool(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } @@ -355,8 +380,10 @@ protected AspectPropertyValue getValueAsDouble(String propertyName) { AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsHashSwig results = getSingleResults(propertyName); if (results != null) { - try (DoubleValueSwig value = results.getValueAsDouble( - Swig.asBytes(propertyName))) { + Integer index = indexOf(propertyName); + try (DoubleValueSwig value = index == null ? + results.getValueAsDouble(Swig.asBytes(propertyName)) : + results.getValueAsDouble(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java index d7b913099..0b5c93fb1 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngine.java @@ -62,6 +62,27 @@ public class DeviceDetectionHashEngine private volatile boolean propertiesPopulated = false; private final Random rand = new Random(); + /** + * Case insensitive map from each property name the engine was built with + * to its required property index, which is the place of the name in that + * list, the first where a name is given more than once, as the native + * engine gives it. Made from the names given at build and never from the + * data set, so an index keeps naming the same property after a reload + * from a data file whose property order differs. When the engine was + * built with no names it is made instead from the names the native engine + * reports after the first data file loads, which are the properties of + * that file sorted by name and which a reload does not change. + */ + private volatile Map propertyIndexes; + + /** + * True once {@link #propertyIndexes} holds the names the indexes are + * fixed on, which is at construction when the engine was built with + * names and after the first data file loads when it was not. A reload + * never makes the map again. + */ + private volatile boolean propertyIndexesFixed; + /** * Construct a new instance of the {@link DeviceDetectionHashEngine}. * @param logger logger instance to use for logging @@ -70,6 +91,8 @@ public class DeviceDetectionHashEngine * @param properties native required properties configuration which define * the properties which the engine should be initialised * with + * @param propertyNames the property names {@code properties} was made + * from, in the same order * @param deviceDataFactory the factory to use when creating a * {@link DeviceDataHash} instance * @param tempDataFileDir the file where a temporary data file copy @@ -80,11 +103,14 @@ public class DeviceDetectionHashEngine AspectEngineDataFile dataFile, ConfigHashSwig config, RequiredPropertiesConfigSwig properties, + List propertyNames, ElementDataFactory deviceDataFactory, String tempDataFileDir) { super(logger, deviceDataFactory, tempDataFileDir); this.config = config; this.propertiesConfigSwig = properties; + this.propertyIndexes = indexesOf(propertyNames); + this.propertyIndexesFixed = this.propertyIndexes.isEmpty() == false; addDataFile(dataFile); } @@ -113,6 +139,55 @@ public MetaDataSwig getMetaData() { return engine.getMetaData(); } + /** + * Get the required property index for a property name, or null when the + * index list has no such name and the read has to go by name. That covers + * a name the engine was not built with, and a property only a data file + * loaded after the first has. + * @param propertyName name of the property + * @return required property index, or null + */ + Integer getPropertyIndex(String propertyName) { + return propertyIndexes.get(propertyName); + } + + /** + * Fix the indexes on the names the native engine reports, which it does + * only for an engine built with no names, and only after its first data + * file has loaded. Those names are the properties of that file sorted by + * name, and the native engine keeps them and their order through every + * later reload, so the map is made once and a reload never changes it. + */ + private void fixPropertyIndexes() { + if (propertyIndexesFixed == false) { + try (VectorStringSwig names = engine.getRequiredProperties()) { + propertyIndexes = indexesOf(new ArrayList<>(names)); + } + propertyIndexesFixed = true; + } + } + + /** + * Map each name to its place in the list, keeping the first place of a + * name given more than once in any letter case. + * @param names the property names in the order the engine is built with, + * or null + * @return unmodifiable case insensitive map from name to index + */ + private static Map indexesOf(List names) { + Map indexes = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + if (names != null) { + for (int i = 0; i < names.size(); i++) { + String name = names.get(i); + if (name != null && name.isEmpty() == false) { + indexes.putIfAbsent(name, i); + } + } + } + return Collections.unmodifiableMap(indexes); + } + @Override public List getProperties() { if (isFalse(propertiesPopulated)) { @@ -349,6 +424,7 @@ protected void unmanagedResourcesCleanup() { } private void setEngineMetaData() { + fixPropertyIndexes(); evidenceKeys = getKeysFromEngine(engine); evidenceKeyFilter = new EvidenceKeyFilterWhitelist( evidenceKeys, @@ -467,45 +543,45 @@ private List getMetricProperties() { Arrays.asList(), new ValueMetaDataDefault("n/a")), new FiftyOneAspectPropertyMetaDataDefault( - Constants.MatchMetrics.METHOD, + Constants.MatchMetrics.ITERATIONS, this, Constants.MatchMetrics.CATEGORY, - String.class, + Integer.class, dataFileList, true, null, (byte) 0, true, - true, false, false, false, - Constants.MatchMetrics.METHOD_DESCRIPTION, + false, + Constants.MatchMetrics.ITERATIONS_DESCRIPTION, null, - Arrays.asList( - new ValueMetaDataDefault("NONE"), - new ValueMetaDataDefault("PERFORMANCE"), - new ValueMetaDataDefault("COMBINED"), - new ValueMetaDataDefault( "PREDICTIVE")), - new ValueMetaDataDefault("NONE")), + Arrays.asList(), + new ValueMetaDataDefault("0")), new FiftyOneAspectPropertyMetaDataDefault( - Constants.MatchMetrics.ITERATIONS, + Constants.MatchMetrics.METHOD, this, Constants.MatchMetrics.CATEGORY, - Integer.class, + String.class, dataFileList, true, null, (byte) 0, true, + true, false, false, false, - false, - Constants.MatchMetrics.ITERATIONS_DESCRIPTION, + Constants.MatchMetrics.METHOD_DESCRIPTION, null, - Arrays.asList(), - new ValueMetaDataDefault("0")) + Arrays.asList( + new ValueMetaDataDefault("NONE"), + new ValueMetaDataDefault("PERFORMANCE"), + new ValueMetaDataDefault("COMBINED"), + new ValueMetaDataDefault( "PREDICTIVE")), + new ValueMetaDataDefault("NONE")) }; return Arrays.asList(metricProperties); } diff --git a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java index 774e1b574..1de52bba8 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/main/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDetectionHashEngineBuilder.java @@ -341,6 +341,7 @@ protected DeviceDetectionHashEngine newEngine(List properties) { dataFile, config, requiredProperties, + properties, new HashDataFactory(loggerFactory), tempDir); } diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataTests.java index 0c5b36069..64dc19f0f 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/DeviceDataTests.java @@ -39,6 +39,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.mockito.Mockito.*; @@ -84,7 +85,32 @@ public void DeviceData_ResultsClose() throws Exception { arg -> data); flowData.close(); verify(results, times(1)).close(); - } + } + } + + /** + * Check that drift adds the drift of every native result, one per + * component, as the specification describes and the native results do. + * @throws Exception + */ + @Test + public void DeviceData_Drift_IsTheSumOfTheResults() throws Exception { + try (FlowData flowData = getWrapper().getPipeline().createFlowData()) { + DeviceDataHashDefault data = new DeviceDataHashDefault( + logger, + flowData, + getWrapper().getEngine(), + missingPropertyService); + + ResultsHashSwig hardware = mock(ResultsHashSwig.class); + when(hardware.getDrift()).thenReturn(3); + ResultsHashSwig browser = mock(ResultsHashSwig.class); + when(browser.getDrift()).thenReturn(5); + data.setResults(hardware); + data.setResults(browser); + + assertEquals(8, data.getDrift().getValue().intValue()); + } } /** diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java index 14b30de62..198e43bb2 100644 --- a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/EngineTests.java @@ -26,6 +26,9 @@ import static org.junit.Assume.assumeTrue; import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.junit.Test; import org.slf4j.ILoggerFactory; @@ -33,6 +36,7 @@ import fiftyone.devicedetection.shared.testhelpers.FileUtils; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; import static fiftyone.devicedetection.shared.testhelpers.FileUtils.TAC_HASH_DATA_FILE_NAME; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -95,4 +99,45 @@ public void Engine_Hash_GetDataSourceTier_Enterprise() throws Exception { testDataSourceTier(TAC_HASH_DATA_FILE_NAME); } + /** + * The match metric Properties follow the data file's Properties, in the + * order of the match metrics table in the specification. + * @throws Exception + */ + @Test + public void Engine_Hash_MatchMetricOrder() throws Exception { + File dataFile = null; + try { + dataFile = getFilePath(LITE_HASH_DATA_FILE_PATH); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + } + List expected = Arrays.asList( + "MatchedNodes", + "Difference", + "Drift", + "DeviceId", + "UserAgents", + "Iterations", + "Method"); + + List names = new ArrayList<>(); + try (DeviceDetectionHashEngine engine = createEngine(dataFile)) { + List properties = + engine.getProperties(); + assertTrue( + "Expected at least " + expected.size() + " Properties, " + + "but there were " + properties.size(), + properties.size() >= expected.size()); + for (FiftyOneAspectPropertyMetaData property : properties.subList( + properties.size() - expected.size(), + properties.size())) { + names.add(property.getName()); + } + } + + assertEquals(expected, names); + } + } diff --git a/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java new file mode 100644 index 000000000..0302a3a95 --- /dev/null +++ b/device-detection/device-detection.hash.engine.on-premise/src/test/java/fiftyone/devicedetection/hash/engine/onpremise/flowelements/PropertyIndexTests.java @@ -0,0 +1,636 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.hash.engine.onpremise.flowelements; + +import fiftyone.devicedetection.hash.engine.onpremise.data.DeviceDataHash; +import fiftyone.devicedetection.hash.engine.onpremise.data.PropertyMetaDataHash; +import fiftyone.devicedetection.hash.engine.onpremise.interop.Swig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.BoolValueSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ConfigHashSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.EngineHashSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.RequiredPropertiesConfigSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.ResultsHashSwig; +import fiftyone.devicedetection.hash.engine.onpremise.interop.swig.VectorStringSwig; +import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.TAC_HASH_DATA_FILE_NAME; +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * Tests for the property index rule, which says that a property's index is + * its place in the list of names the engine was built with, that the index is + * fixed at build, and that every read by name goes through that index. + */ +public class PropertyIndexTests { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * Names in an order that is deliberately not the order the data file + * holds them in, so an index taken from the data file would be wrong. + */ + private static final List NAMES = Arrays.asList( + "ScreenPixelsHeight", + "IsMobile", + "PlatformName", + "BrowserName"); + + private static final String MOBILE = + "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) " + + "Version/15.0 Mobile/15E148 Safari/604.1"; + + private static final int READS = 20000; + + private static final int ROUNDS = 5; + + private static File dataFile(String name) { + try { + return getFilePath(name); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return null; + } + } + + private static DeviceDetectionHashEngine build( + File file, List names) throws Exception { + DeviceDetectionHashEngineBuilder builder = + new DeviceDetectionHashEngineBuilder(loggerFactory, null) + .setPerformanceProfile( + Constants.PerformanceProfiles.HighPerformance) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false); + if (names != null) { + builder.setProperties(names); + } + return builder.build(file.toString(), false); + } + + /** + * Every name the engine was built with has an index, and the index is the + * place of the name in the build time list, not its place in the data + * file. + */ + @Test + public void PropertyIndex_FromBuildTimeNames() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + try (DeviceDetectionHashEngine engine = build(file, NAMES)) { + for (int i = 0; i < NAMES.size(); i++) { + Integer index = engine.getPropertyIndex(NAMES.get(i)); + assertNotNull( + "Expected an index for '" + NAMES.get(i) + "'.", + index); + assertEquals( + "Expected '" + NAMES.get(i) + "' at index " + i + ".", + Integer.valueOf(i), + index); + } + } + } + + /** + * The lookup is case insensitive, because a caller may ask for a property + * in any casing. + */ + @Test + public void PropertyIndex_CaseInsensitive() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + try (DeviceDetectionHashEngine engine = build(file, NAMES)) { + assertEquals( + engine.getPropertyIndex("IsMobile"), + engine.getPropertyIndex("ismobile")); + } + } + + /** + * A name given twice, in any letter case, has the index of its first + * place, as the native engine gives it. + */ + @Test + public void PropertyIndex_NameGivenTwiceHasFirstIndex() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + try (DeviceDetectionHashEngine engine = build(file, + Arrays.asList("IsMobile", "BrowserName", "ismobile"))) { + assertEquals( + Integer.valueOf(0), + engine.getPropertyIndex("IsMobile")); + assertEquals( + Integer.valueOf(1), + engine.getPropertyIndex("BrowserName")); + } + } + + /** + * A name the engine was not built with has no index, so the read goes by + * name. + */ + @Test + public void PropertyIndex_UnknownNameHasNoIndex() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + try (DeviceDetectionHashEngine engine = build(file, NAMES)) { + assertNull(engine.getPropertyIndex("HardwareVendor")); + assertNull(engine.getPropertyIndex("NotAPropertyAtAll")); + } + } + + /** + * An engine built with no names has an index for every property of the + * first data file it loads, no two properties share an index, and the + * native results name that property at that index. + */ + @Test + public void PropertyIndex_NoNamesFromFirstFile() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + List names = dataSetNames(file); + try (DeviceDetectionHashEngine engine = build(file, null); + NativeEngine nativeEngine = new NativeEngine(file, null); + ResultsHashSwig results = nativeEngine.engine.process(MOBILE)) { + Set seen = new HashSet<>(); + for (String name : names) { + Integer index = engine.getPropertyIndex(name); + assertNotNull("Expected an index for '" + name + "'.", index); + assertTrue( + "Index " + index + " given to more than one name.", + seen.add(index)); + assertEquals( + "Expected the native results to name '" + name + + "' at index " + index + ".", + name, + results.getPropertyName(index)); + } + } + } + + /** + * An engine built with no names keeps its first data file's indexes + * through a reload from a data file whose properties are a superset in a + * different order. Each property both files have then reads what an + * engine built on the new file reads, and a property only the new file + * has gets no index. + */ + @Test + public void PropertyIndex_NoNamesSurvivesReload() throws Exception { + File lite = dataFile(LITE_HASH_DATA_FILE_PATH); + File bigger = dataFile(TAC_HASH_DATA_FILE_NAME); + List names = dataSetNames(lite); + names.retainAll(dataSetNames(bigger)); + String added = nameOnlyIn(bigger, lite); + + DeviceDetectionHashEngine engine = build(lite, null); + DeviceDetectionHashEngine fresh = build(bigger, null); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + Pipeline freshPipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(fresh) + .build()) { + + Map before = new HashMap<>(); + for (String name : names) { + Integer index = engine.getPropertyIndex(name); + assertNotNull("Expected an index for '" + name + "'.", index); + before.put(name, index); + } + + engine.refreshData( + engine.getDataFiles().get(0).getIdentifier(), + Files.readAllBytes(bigger.toPath())); + + for (String name : names) { + assertEquals( + "Index of '" + name + "' moved on reload.", + before.get(name), + engine.getPropertyIndex(name)); + } + if (added != null) { + assertNull( + "Expected no index for '" + added + "', which only " + + "the new file has.", + engine.getPropertyIndex(added)); + } + + try (FlowData data = pipeline.createFlowData(); + FlowData expected = freshPipeline.createFlowData()) { + data.addEvidence("header.user-agent", MOBILE); + data.process(); + expected.addEvidence("header.user-agent", MOBILE); + expected.process(); + DeviceData device = data.get(DeviceData.class); + DeviceData want = expected.get(DeviceData.class); + for (String name : names) { + AspectPropertyValue got = + (AspectPropertyValue) device.get(name); + AspectPropertyValue wanted = + (AspectPropertyValue) want.get(name); + assertEquals( + "Expected '" + name + "' to have a value as a " + + "fresh engine gives it.", + wanted.hasValue(), + got.hasValue()); + if (wanted.hasValue()) { + assertEquals( + "Expected '" + name + "' to read as a fresh " + + "engine reads it.", + wanted.getValue(), + got.getValue()); + } + } + } + } + } + + /** + * A read by name and the typed read give the same value, because the + * typed accessor goes through the same path. + */ + @Test + public void PropertyIndex_ByNameMatchesTyped() throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + DeviceDetectionHashEngine engine = build(file, NAMES); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence( + "header.user-agent", + "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) " + + "Version/15.0 Mobile/15E148 Safari/604.1"); + data.process(); + DeviceData device = data.get(DeviceData.class); + AspectPropertyValue typed = device.getIsMobile(); + AspectPropertyValue byName = + (AspectPropertyValue) device.get("IsMobile"); + assertEquals(typed.hasValue(), byName.hasValue()); + if (typed.hasValue()) { + assertEquals(typed.getValue(), byName.getValue()); + } + } + } + + /** + * After a reload from a data file whose properties are a superset and in + * a different order, each index still names the property it named at + * build, so every value read by name is still the right one. + */ + @Test + public void PropertyIndex_SurvivesReload() throws Exception { + File lite = dataFile(LITE_HASH_DATA_FILE_PATH); + File bigger = dataFile(TAC_HASH_DATA_FILE_NAME); + + String userAgent = + "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) " + + "Version/15.0 Mobile/15E148 Safari/604.1"; + + DeviceDetectionHashEngine engine = build(lite, NAMES); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build()) { + + // The indexes the engine was built with. + for (int i = 0; i < NAMES.size(); i++) { + assertEquals( + Integer.valueOf(i), + engine.getPropertyIndex(NAMES.get(i))); + } + + // Reload from the larger data file, whose property order differs. + engine.refreshData( + engine.getDataFiles().get(0).getIdentifier(), + Files.readAllBytes(bigger.toPath())); + + // The indexes have not moved. + for (int i = 0; i < NAMES.size(); i++) { + assertEquals( + "Index of '" + NAMES.get(i) + "' moved on reload.", + Integer.valueOf(i), + engine.getPropertyIndex(NAMES.get(i))); + } + + // Every name still reads its own property, and the typed read + // agrees with the read by name. + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", userAgent); + data.process(); + DeviceData device = data.get(DeviceData.class); + AspectPropertyValue typed = device.getIsMobile(); + AspectPropertyValue byName = + (AspectPropertyValue) device.get("IsMobile"); + assertEquals(typed.hasValue(), byName.hasValue()); + if (typed.hasValue()) { + assertEquals(typed.getValue(), byName.getValue()); + } + for (String name : NAMES) { + assertNotNull( + "Expected a result for '" + name + "'.", + device.get(name)); + } + } + } + } + + /** + * A name the engine was built with keeps its index after a reload from a + * data file that lacks the property. A read by index then gives no value, + * as a read by name does, rather than an error, and the other names still + * read their own properties. + */ + @Test + public void PropertyIndex_NameTheReloadedFileLacks() throws Exception { + File lite = dataFile(LITE_HASH_DATA_FILE_PATH); + File bigger = dataFile(TAC_HASH_DATA_FILE_NAME); + String missing = nameOnlyIn(bigger, lite); + assumeTrue("The TAC file has no property the Lite file lacks.", + missing != null); + List names = Arrays.asList(missing, "IsMobile"); + + DeviceDetectionHashEngine engine = build(bigger, names); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build()) { + engine.refreshData( + engine.getDataFiles().get(0).getIdentifier(), + Files.readAllBytes(lite.toPath())); + + assertEquals( + "Index of '" + missing + "' moved on reload.", + Integer.valueOf(0), + engine.getPropertyIndex(missing)); + assertEquals( + Integer.valueOf(1), + engine.getPropertyIndex("IsMobile")); + + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", MOBILE); + data.process(); + DeviceDataHash device = + (DeviceDataHash) data.get(DeviceData.class); + AspectPropertyValue> values = + device.getValues(missing); + assertFalse( + "'" + missing + "' has a value from a file without it.", + values.hasValue()); + assertNotNull(values.getNoValueMessage()); + assertTrue("IsMobile has no value after the reload.", + device.getIsMobile().hasValue()); + assertEquals(Boolean.TRUE, device.getIsMobile().getValue()); + } + } + } + + /** + * The first property name, in name order, that one data file has and the + * other lacks, or null if there is none. + */ + private static String nameOnlyIn(File has, File lacks) throws Exception { + List lacking = propertyNames(lacks); + for (String name : propertyNames(has)) { + if (lacking.contains(name) == false) { + return name; + } + } + return null; + } + + /** + * The names of the properties the data file holds, sorted, leaving out + * the match metrics the engine adds, which are not in the data file and + * so have no index. + */ + private static List dataSetNames(File file) throws Exception { + List names = new ArrayList<>(); + try (DeviceDetectionHashEngine engine = build(file, null)) { + for (FiftyOneAspectPropertyMetaData property : + engine.getProperties()) { + if (property instanceof PropertyMetaDataHash) { + names.add(property.getName()); + } + } + } + Collections.sort(names); + return names; + } + + private static List propertyNames(File file) throws Exception { + List names = new ArrayList<>(); + try (DeviceDetectionHashEngine engine = build(file, null)) { + for (FiftyOneAspectPropertyMetaData property : + engine.getProperties()) { + names.add(property.getName()); + } + } + Collections.sort(names); + return names; + } + + /** + * Reads IsMobile {@link #READS} times by name and {@link #READS} times by + * index from the same native results, and {@link #READS} times through + * the typed accessor, which is the whole path a caller takes, and prints + * the median nanoseconds of one read over {@link #ROUNDS} rounds after a + * warm-up, for an engine built with {@link #NAMES}. Every read must give + * the same value. The times are printed rather than asserted, because + * runners vary. + */ + @Test + public void PropertyIndex_ReadByIndexAgainstByName() throws Exception { + measureReads(NAMES); + } + + /** + * The same measurement for an engine built with no names, which reads by + * the index of the property in its first data file. + */ + @Test + public void PropertyIndex_NoNamesReadByIndexAgainstByName() + throws Exception { + measureReads(null); + } + + private static void measureReads(List built) throws Exception { + File file = dataFile(LITE_HASH_DATA_FILE_PATH); + DeviceDetectionHashEngine engine = build(file, built); + Integer found = engine.getPropertyIndex("IsMobile"); + assertNotNull("Expected an index for 'IsMobile'.", found); + int index = found; + ConfigHashSwig config = new ConfigHashSwig(); + config.setHighPerformance(); + RequiredPropertiesConfigSwig required; + try (VectorStringSwig names = new VectorStringSwig()) { + if (built != null) { + names.addAll(built); + } + required = new RequiredPropertiesConfigSwig(names); + } + EngineHashSwig nativeEngine = + new EngineHashSwig(file.getAbsolutePath(), config, required); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData(); + ResultsHashSwig results = nativeEngine.process(MOBILE)) { + data.addEvidence("header.user-agent", MOBILE); + data.process(); + DeviceData device = data.get(DeviceData.class); + + boolean expected = readByName(results); + assertEquals(expected, readByIndex(results, index)); + assertEquals(expected, (boolean) device.getIsMobile().getValue()); + + List byName = new ArrayList<>(); + List byIndex = new ArrayList<>(); + List typed = new ArrayList<>(); + int different = 0; + for (int round = 0; round <= ROUNDS; round++) { + long start = System.nanoTime(); + for (int i = 0; i < READS; i++) { + if (readByName(results) != expected) { + different++; + } + } + long nameNanos = System.nanoTime() - start; + start = System.nanoTime(); + for (int i = 0; i < READS; i++) { + if (readByIndex(results, index) != expected) { + different++; + } + } + long indexNanos = System.nanoTime() - start; + start = System.nanoTime(); + for (int i = 0; i < READS; i++) { + if (device.getIsMobile().getValue() != expected) { + different++; + } + } + long typedNanos = System.nanoTime() - start; + if (round > 0) { + byName.add(nameNanos / READS); + byIndex.add(indexNanos / READS); + typed.add(typedNanos / READS); + } + } + assertEquals("Every read gives the same value", 0, different); + System.out.println(String.format( + "READ COST IsMobile names=%s reads=%d rounds=%d " + + "byNameNs=%s byIndexNs=%s typedNs=%s java=%s os=%s", + built == null ? "none" : "given", + READS, ROUNDS, spread(byName), spread(byIndex), + spread(typed), System.getProperty("java.version"), + System.getProperty("os.name"))); + } + finally { + nativeEngine.delete(); + required.delete(); + config.delete(); + } + } + + /** + * A native engine on its own, so a test can read the native results + * directly. Built with no names when the list is null. + */ + private static class NativeEngine implements AutoCloseable { + private final ConfigHashSwig config = new ConfigHashSwig(); + private final RequiredPropertiesConfigSwig required; + private final EngineHashSwig engine; + + NativeEngine(File file, List names) { + config.setHighPerformance(); + RequiredPropertiesConfigSwig made; + try (VectorStringSwig vector = new VectorStringSwig()) { + if (names != null) { + vector.addAll(names); + } + made = new RequiredPropertiesConfigSwig(vector); + } + required = made; + engine = new EngineHashSwig( + file.getAbsolutePath(), config, required); + } + + @Override + public void close() { + engine.delete(); + required.delete(); + config.delete(); + } + } + + /** + * Reads IsMobile by name through the overload that takes the name's + * bytes, which is how a read goes when the engine has no index for it. + */ + private static boolean readByName(ResultsHashSwig results) { + try (BoolValueSwig value = + results.getValueAsBool(Swig.asBytes("IsMobile"))) { + return value.getValue(); + } + } + + private static boolean readByIndex(ResultsHashSwig results, int index) { + try (BoolValueSwig value = results.getValueAsBool(index)) { + return value.getValue(); + } + } + + /** + * The lowest, median and highest, in that order. + */ + private static String spread(List values) { + List sorted = new ArrayList<>(values); + Collections.sort(sorted); + return sorted.get(0) + "/" + sorted.get(sorted.size() / 2) + "/" + + sorted.get(sorted.size() - 1); + } +} diff --git a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java index 21b9e5bd4..aed639665 100644 --- a/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java +++ b/device-detection/device-detection/src/test/java/fiftyone/devicedetection/DeviceDetectionOnPremisePipelineBuilderTest.java @@ -64,6 +64,7 @@ public void setup() throws Exception { when(loggerFactory.getLogger(anyString())).thenReturn(logger); builder = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) .setShareUsage(false); } @@ -122,6 +123,7 @@ public void setEngineData_PropertyPropagatedToDataFileConfiguration() throws Exc try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(new byte[]{}, Enums.DeviceDetectionAlgorithm.Hash) .setEngineData(data) + .setAutoUpdate(false) .setShareUsage(false) .build()) { DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); @@ -144,6 +146,7 @@ public void setEngineData_FilenameIsSet_PropertyNotPropagated() throws Exception public void setShareUsage_False_SharedFlowElementNotAdded() throws Exception { try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) .setShareUsage(false) .build()) { List flowElements = pipeline.getFlowElements(); @@ -156,6 +159,7 @@ public void setShareUsage_False_SharedFlowElementNotAdded() throws Exception { public void setShareUsage_True_SharedFlowElementAdded() throws Exception { try (Pipeline pipeline = new DeviceDetectionPipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) .setShareUsage(true) .build()) { List flowElements = pipeline.getFlowElements(); @@ -168,6 +172,7 @@ public void setShareUsage_True_SharedFlowElementAdded() throws Exception { public void setAutoUpdate_PropertyPropagatedToDataFileConfiguration() throws Exception { try (Pipeline pipeline = builder .setAutoUpdate(true) + .setDataUpdateLicenseKey("licence_key") .build()) { DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); assertTrue(dataFileConfiguration.getAutomaticUpdatesEnabled()); @@ -189,6 +194,7 @@ public void setDataUpdateService_RegisterDataFileTriggered() throws Exception { DataUpdateServiceDefault dataUpdateServiceDefault = mock(DataUpdateServiceDefault.class); try (Pipeline ignored = builder .setAutoUpdate(true) + .setDataUpdateLicenseKey("licence_key") .setDataUpdateService(dataUpdateServiceDefault) .build()) { verify(dataUpdateServiceDefault).registerDataFile(any()); @@ -199,6 +205,8 @@ public void setDataUpdateService_RegisterDataFileTriggered() throws Exception { public void setDataUpdateOnStartup_PropertyPropagatedToDataFileConfiguration() throws Exception { try (Pipeline pipeline = builder .setDataUpdateOnStartup(true) + .setDataUpdateLicenseKey("licence_key") + .setDataUpdateService(mock(DataUpdateServiceDefault.class)) .build()) { DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); assertTrue(dataFileConfiguration.getUpdateOnStartup()); diff --git a/examples/device-detection/LICENSE b/examples/device-detection/LICENSE deleted file mode 100644 index 4153cd377..000000000 --- a/examples/device-detection/LICENSE +++ /dev/null @@ -1,287 +0,0 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. diff --git a/examples/device-detection/README.md b/examples/device-detection/README.md index 4c80193e5..c83d69956 100644 --- a/examples/device-detection/README.md +++ b/examples/device-detection/README.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - Examples -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=51degrees-device-detection-engines-examples "Data rewards the curious") **Java Device Detection** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-java/main/.github/images/logo-160.png "Data rewards the curious") **Java Device Detection** -[Developer Documentation](https://51degrees.com/device-detection-java/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=51degrees-device-detection-engines-examples "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=51degrees-device-detection-engines-examples "developer documentation") ## Introduction @@ -27,7 +27,14 @@ The on-premise examples use the free Lite data file and the evidence files in `cxx/data/device-detection`, a folder of the [data](https://github.com/51Degrees/data) repository. An example that cannot find one of these files runs the data repository's download script for it, -and logs the command it runs. +and logs the command it runs. The examples find `cxx/data` whether they are +run from the root of this repository or from this folder. + +Each example shows the properties that only the Enterprise data file has, +such as HardwareVendor, HardwareName and DeviceType, as needing it, and gives +a link to ask for a copy. [Contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=enterprise-data-file) +for a copy of the Enterprise data file, and pass its path as the first +argument to use it. ## Cloud resource keys @@ -115,7 +122,7 @@ case the example runs until the process is stopped: PORT=8100 java -jar ./web/getting-started.onprem/target/device-detection-java-examples.web.getting-started.onprem-4.4.20-jar-with-dependencies.jar ``` -The on-premise web example takes its data file from the `51DEGREES_DD_PATH` +The on-premise web example takes its data file from the `_51DEGREES_DD_PATH` environment variable, then from `TestDataFile`, and otherwise uses the free Lite file in `cxx/data/device-detection`, which it downloads before the server starts if it is missing. diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java index 44b7227fb..6cb6e8433 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/GettingStartedOnPrem.java @@ -30,6 +30,7 @@ import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.fiftyone.flowelements.FiftyOnePipelineBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +40,7 @@ import java.io.PrintWriter; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.devicedetection.examples.shared.PropertyHelper.asString; @@ -62,11 +64,18 @@ public class GettingStartedOnPrem { cxx/data, and is downloaded if it is missing, or you may specify another file as a command line parameter. - Note that the Lite data file is only used for illustration, and has limited accuracy and - capabilities. Find out about the Enterprise data file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=lite_v_4_1_hash */ + Note that the Lite data file is only used for illustration, and has a small number of + Properties. The example shows the properties only the Enterprise data file has as needing + it. Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=lite_v_4_1_hash */ public static String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; public static String ENTERPRISE_HASH = "TAC-HashV41.hash"; + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-gettingstartedonprem.java&utm_term=enterprise-data-file"; + public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); @@ -115,15 +124,20 @@ public static void run(String dataFile, .buildFromConfiguration(pipelineOptions)) { - // carry out some sample detections - for (Map evidence : evidenceList) { - analyzeEvidence(evidence, pipeline, outputStream); - } - /* Get the 'engine' element within the pipeline that performs device detection. We can use this to get details about the data file as well as meta-data describing things such as the available properties. */ DeviceDetectionHashEngine engine = pipeline.getElement(DeviceDetectionHashEngine.class); + + // carry out some sample detections + boolean needsEnterprise = false; + for (Map evidence : evidenceList) { + needsEnterprise |= analyzeEvidence(evidence, pipeline, engine, outputStream); + } + if (needsEnterprise) { + DataFileHelper.writeEnterpriseNote(new PrintWriter(outputStream), CONTACT_US); + } + DataFileHelper.logDataFileInfo(engine); logger.info("All done"); @@ -135,11 +149,15 @@ public static void run(String dataFile, * supplied and output the device detection results to the output stream. * @param evidence a map representing HTTP headers * @param pipeline a pipeline set up to process the evidence + * @param engine the engine, which says which properties its data file has * @param out somewhere to send the detection results + * @return true if a property needed the Enterprise data file */ - private static void analyzeEvidence(Map evidence, - Pipeline pipeline, - OutputStream out) throws Exception { + private static boolean analyzeEvidence(Map evidence, + Pipeline pipeline, + DeviceDetectionHashEngine engine, + OutputStream out) throws Exception { + boolean needsEnterprise = false; PrintWriter writer = new PrintWriter(out); /* FlowData is a data structure that is used to convey information required for detection and the results of the detection through the pipeline. Information required for @@ -177,9 +195,37 @@ private static void analyzeEvidence(Map evidence, writer.println("\tPlatform Version: " + asString(device.getPlatformVersion())); writer.println("\tBrowser Name: " + asString(device.getBrowserName())); writer.println("\tBrowser Version: " + asString(device.getBrowserVersion())); + // Only the Enterprise data file has these properties. The free + // Lite data file has the ones above. + needsEnterprise |= writeEnterpriseValue(writer, "Hardware Vendor", + engine, "HardwareVendor", device::getHardwareVendor); + needsEnterprise |= writeEnterpriseValue(writer, "Hardware Name", + engine, "HardwareName", device::getHardwareName); + needsEnterprise |= writeEnterpriseValue(writer, "Device Type", + engine, "DeviceType", device::getDeviceType); } writer.println(); writer.flush(); + return needsEnterprise; + } + + /** + * Writes a property that only the Enterprise data file has, or that it + * needs the Enterprise data file when the data file in use does not + * have it. The value is only read when the data file has the property. + * @return true if the property needs the Enterprise data file + */ + private static boolean writeEnterpriseValue(PrintWriter writer, + String name, + DeviceDetectionHashEngine engine, + String property, + Supplier> value) { + if (DataFileHelper.hasProperty(engine, property)) { + writer.println("\t" + name + ": " + asString(value.get())); + return false; + } + writer.println("\t" + name + ": " + DataFileHelper.NEEDS_ENTERPRISE); + return true; } } /*! diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java index 4c0aa55a1..ddabcaed4 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MatchMetrics.java @@ -45,6 +45,7 @@ import java.util.Objects; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static java.util.stream.Collectors.groupingBy; @@ -62,9 +63,18 @@ public class MatchMetrics { private static final Logger logger = LoggerFactory.getLogger(MatchMetrics.class); + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-matchmetrics.java&utm_term=enterprise-data-file"; + public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - String dataFilename = args.length > 0 ? args[0] : null; + // Use the supplied data file, or the free Lite file, which is + // downloaded if it is missing + String dataFilename = DataFileHelper.getDataFileLocation( + args.length > 0 ? args[0] : LITE_HASH_DATA_FILE_PATH); run(dataFilename, EvidenceHelper.setUpEvidence(), false, System.out); } @@ -106,8 +116,9 @@ static void run(String dataFile, List> evidenceList, // Uncomment BrowserName to include Browser component profile ID // in the device ID value. //.setProperty("BrowserName") - // If using the full on-premise data file this property will be - // present in the data file. See https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-matchmetrics.java&utm_term=run + // Only the Enterprise data file has this property, and the + // free Lite data file leaves it out. Contact us for a copy at + // https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-matchmetrics.java&utm_term=run .setProperty("HardwareName") // Only use the predictive graph to better handle variances // between the training data and the target User-Agent string. @@ -215,6 +226,10 @@ static void run(String dataFile, List> evidenceList, }); writer.println(); } + if (DataFileHelper.isLiteDataFile( + pipeline.getElement(DeviceDetectionHashEngine.class))) { + DataFileHelper.writeEnterpriseOffer(writer, CONTACT_US); + } } logger.info("Finished Match Metrics Example"); } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java index 350172405..c85d852b8 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/MetadataOnPrem.java @@ -51,10 +51,16 @@ public class MetadataOnPrem { cxx/data, and is downloaded if it is missing, or you may specify another file as a command line parameter. - Note that the Lite data file is only used for illustration, and has limited accuracy and - capabilities. Find out about the Enterprise data file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=lite_v_4_1_hash */ + Note that the Lite data file is only used for illustration, and has a small number of + Properties. Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=lite_v_4_1_hash */ public static String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-metadataonprem.java&utm_term=enterprise-data-file"; + public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); String dataFile = args.length > 0 ? args[0] : LITE_V_4_1_HASH; @@ -108,6 +114,9 @@ public static void run(String dataFile, OutputStream output) throws Exception { writer.flush(); DataFileHelper.logDataFileInfo(ddEngine); + if (DataFileHelper.isLiteDataFile(ddEngine)) { + DataFileHelper.writeEnterpriseOffer(writer, CONTACT_US); + } } } diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java index 8fc928a56..fbcdee51b 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/OfflineProcessing.java @@ -40,6 +40,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Collectors; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; @@ -71,10 +72,16 @@ public class OfflineProcessing { // cxx/data, and is downloaded if it is missing // // Note that the Lite data file is only used for illustration, and has - // limited accuracy and capabilities. Find out about the Enterprise data - // file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-offlineprocessing.java&utm_term=lite_v_4_1_hash + // a small number of Properties. Contact us for a copy of the + // Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-offlineprocessing.java&utm_term=lite_v_4_1_hash public static final String LITE_V_4_1_HASH = "device-detection/51Degrees-LiteV4.1.hash"; + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-offlineprocessing.java&utm_term=enterprise-data-file"; // This 51degrees file of 20,000 examples is used from the same folder, // and is downloaded if it is missing public static final String HEADER_EVIDENCE_YML = @@ -168,6 +175,7 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws try (Writer writer = new OutputStreamWriter(os)) { // read a batch of device data from the stream int count = 0; + boolean needsEnterprise = false; while (evidenceIterator.hasNext() && count < 20) { // Flow data is the container for inputs and outputs that // flow through the pipeline a flowdata instance is @@ -199,6 +207,17 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws resultMap.put("device.ismobile", asString(device.getIsMobile())); resultMap.put("device.platformname", asString(device.getPlatformName())); resultMap.put("device.platformversion", asString(device.getPlatformVersion())); + // Only the Enterprise data file has these properties. + // The free Lite data file has the ones above. + needsEnterprise |= putEnterpriseValue(resultMap, + "device.hardwarevendor", engine, "HardwareVendor", + device::getHardwareVendor); + needsEnterprise |= putEnterpriseValue(resultMap, + "device.hardwarename", engine, "HardwareName", + device::getHardwareName); + needsEnterprise |= putEnterpriseValue(resultMap, + "device.devicetype", engine, "DeviceType", + device::getDeviceType); // to look at all device detection properties use the following: // resultMap.putAll(getPopulatedProperties(device, "device.")); @@ -211,21 +230,38 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws } count++; } + if (needsEnterprise) { + // A comment keeps the output a valid YAML stream. + writer.write("# Some of these properties need the " + + "Enterprise data file. Contact us for a copy at " + + CONTACT_US + "\n"); + } // finish the last YAML document writer.write("...\n"); writer.flush(); logger.info("Finished processing {} records", count); - - if (engine.getDataSourceTier().equals("Lite")) { - logger.warn("You have used a Lite data file which has " + - "limited properties and is of limited accuracy"); - logger.info("The example requires an Enterprise data file " + - "to work fully. Find out about the Enterprise " + - "data file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-offlineprocessing.java&utm_term=lite-data-file"); - } } } } + + /** + * Records a property that only the Enterprise data file has, or that it + * needs the Enterprise data file when the data file in use does not + * have it. The value is only read when the data file has the property. + * @return true if the property needs the Enterprise data file + */ + private static boolean putEnterpriseValue(Map resultMap, + String key, + DeviceDetectionHashEngine engine, + String property, + Supplier> value) { + if (DataFileHelper.hasProperty(engine, property)) { + resultMap.put(key, asString(value.get())); + return false; + } + resultMap.put(key, DataFileHelper.NEEDS_ENTERPRISE); + return true; + } /** * Filter entries that are not keyed on the required prefix diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java index 8c6f2b0fe..163e9f8da 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/PerformanceBenchmark.java @@ -46,6 +46,7 @@ import static fiftyone.devicedetection.examples.shared.DataFileHelper.getDataFileLocation; import static fiftyone.devicedetection.examples.shared.DataFileHelper.getEvidenceFile; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; import static fiftyone.pipeline.engines.Constants.PerformanceProfiles.*; /** @@ -64,6 +65,15 @@ public class PerformanceBenchmark { public static final Logger logger = LoggerFactory.getLogger(PerformanceBenchmark.class); + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-performancebenchmark.java&utm_term=enterprise-data-file"; + + // true when the benchmarks ran on the free Lite data file + private boolean usedLiteDataFile; + // where the results of the tests are gathered private List> resultList; private int numberOfThreads = DEFAULT_NUMBER_OF_THREADS; @@ -82,7 +92,9 @@ public class PerformanceBenchmark { public static void main(String[] args) throws Exception { LogbackHelper.configureLogback(FileFinder.getFilePath("logback.xml")); - String dataFilename = args.length > 0 ? args[0] : null; + // Use the supplied data file, or the free Lite file, which is + // downloaded if it is missing + String dataFilename = args.length > 0 ? args[0] : LITE_HASH_DATA_FILE_PATH; String evidenceFilename = args.length > 1 ? args[1] : null; int numberOfThreads = DEFAULT_NUMBER_OF_THREADS; if (args.length > 2) { @@ -132,6 +144,9 @@ protected void runBenchmarks(PerformanceConfiguration[] performanceConfiguration for (PerformanceConfiguration config: performanceConfigurations){ executeBenchmark(true, config); } + if (usedLiteDataFile) { + DataFileHelper.writeEnterpriseOffer(writer, CONTACT_US); + } logger.info("Finished Performance example"); } @@ -166,7 +181,10 @@ private void executeBenchmark(boolean configureFromDisk, setPipelinePerformanceProperties(builder, config); pipeline = builder.build(); - DataFileHelper.logDataFileInfo(pipeline.getElement(DeviceDetectionHashEngine.class)); + DeviceDetectionHashEngine engine = + pipeline.getElement(DeviceDetectionHashEngine.class); + DataFileHelper.logDataFileInfo(engine); + usedLiteDataFile = DataFileHelper.isLiteDataFile(engine); } else { logger.info("Load memory from {}", dataFileLocation); byte[] fileContent = Files.readAllBytes(new File(dataFileLocation).toPath()); diff --git a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java index d3babcd27..ebffdef86 100644 --- a/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java +++ b/examples/device-detection/console/src/main/java/fiftyone/devicedetection/examples/console/UpdateDataFile.java @@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.PrintWriter; import java.nio.file.*; import java.text.SimpleDateFormat; import java.util.Objects; @@ -42,6 +43,7 @@ import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.devicedetection.examples.shared.DataFileHelper.getDatafileMetaData; import static fiftyone.devicedetection.shared.testhelpers.FileUtils.ENTERPRISE_HASH_DATA_FILE_NAME; +import static fiftyone.devicedetection.shared.testhelpers.FileUtils.LITE_HASH_DATA_FILE_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; /*! @@ -57,8 +59,9 @@ *
  • Daily auto-update
    • * * ## License Key - * In order to test this example you will need a 51Degrees Enterprise license which can be - * purchased from our [pricing page](https://51degrees.com/pricing/annual?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=updatedatafile). + * Updating the data file needs a 51Degrees licence key for the Enterprise data file. Without + * one, the example shows the free Lite data file and says where to ask for a key. Contact us for + * a key at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=updatedatafile * # Data Files * You can find out more about data files, licenses etc. at our [FAQ page](https://51degrees.com/resources/faqs?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=updatedatafile) * ## Enterprise Data File @@ -72,7 +75,7 @@ * https://distributor.51degrees.com/api/v2/download?LicenseKeys=&Type=27&Download=True&Product=22 * ``` * ## Lite Data File - * Lite data files (free-to-use, limited capabilities, no license key required) are created roughly + * Lite data files (free-to-use, a small number of Properties, no license key required) are created roughly * once a month and cannot be updated using auto-update, they may be downloaded with the * scripts in the [data repository](https://github.com/51Degrees/data) on GitHub, which is * the cxx/data folder of the source. @@ -179,9 +182,9 @@ *
  • Daily auto-update
    • * *

    - * To run this example you must obtain a license key purchased from our - * pricing page. Look for our "Bigger" or - * "Biggest" options. This license key must be supplied as a command line argument or by setting + * To update the data file you need a licence key, which you can ask for on our + * contact page. + * The key must be supplied as a command line argument or by setting * an environment variable or system property called {@link UpdateDataFile#UPDATE_EXAMPLE_LICENSE_KEY_NAME} */ public class UpdateDataFile { @@ -192,6 +195,12 @@ public class UpdateDataFile { private static final Logger logger = LoggerFactory.getLogger(UpdateDataFile.class); + /** + * Where to ask for a licence key and a copy of the Enterprise data file, + * tagged with this example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=licence-key"; + static final UpdateCompletionListener completionListener = new UpdateCompletionListener(); static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss Z"); @@ -221,12 +230,8 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr licenseKey = KeyUtils.getNamedKey(UPDATE_EXAMPLE_LICENSE_KEY_NAME); } if (Objects.isNull(licenseKey) || KeyUtils.isInvalidKey(licenseKey)) { - logger.error("In order to test this example you will need a 51Degrees Enterprise " + - "license which can be obtained on a trial basis or purchased from our\n" + - "pricing page https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=license-key-required. You must supply the license " + - "key as an argument to this program, or as an environment or system variable " + - "named '{}'", UPDATE_EXAMPLE_LICENSE_KEY_NAME); - throw new IllegalArgumentException("No license key available"); + explainLicenseKey(dataFilename, new PrintWriter(System.out, true)); + return; } // work out where the downloaded file will be put, directory must exist @@ -304,7 +309,7 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr // to notify when update complete .setDataUpdateService(dataUpdateService) // For automatic updates to work you will need to provide a license key. - // A license key can be obtained with a subscription from https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=run + // Contact us for a licence key at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-devicedetection-examples-console-updatedatafile.java&utm_term=run .setDataUpdateLicenseKey(licenseKey) // Enable update on startup, the auto update system // will be used to check for an update before the @@ -359,6 +364,40 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr } } + /** + * Without a licence key the example cannot download a data file, so it + * shows the data file it would start from, the free Lite file unless + * another is given, and says what an update needs. + * @param dataFilename the data file given, or null + * @param output where to write + * @throws Exception if the data file cannot be read + */ + static void explainLicenseKey(String dataFilename, PrintWriter output) throws Exception { + File dataFile = null; + try { + dataFile = Objects.isNull(dataFilename) ? + DataFileHelper.findDataFile(LITE_HASH_DATA_FILE_PATH) : + new File(dataFilename); + } catch (IllegalArgumentException e) { + logger.warn(e.getMessage()); + } + if (Objects.nonNull(dataFile) && dataFile.exists()) { + DataFileHelper.DatafileInfo metadata = + getDatafileMetaData(dataFile.getAbsolutePath()); + output.println("The data file in use is the '" + metadata.getTier() + + "' file published on " + simpleDateFormat.format( + metadata.getFileInfo().getDataPublishedDateTime()) + + ", at '" + metadata.getFileInfo().getDataFilePath() + "'."); + } + output.println(); + output.println("Updating the data file downloads the Enterprise data " + + "file, which needs a licence key. Give the key as the first " + + "command line argument, or in the environment variable or " + + "system property '" + UPDATE_EXAMPLE_LICENSE_KEY_NAME + "'. " + + "Contact us for a key and a copy at " + CONTACT_US); + output.flush(); + } + private static void logFileInfo(DataFileHelper.DatafileInfo metadata) { logger.info("Current data file is a {} tier data file, last updated at {}, at " + "location {}. Next update expected {}.", metadata.getTier(), diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java new file mode 100644 index 000000000..aea911082 --- /dev/null +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/ExamplesWithoutSettingsTest.java @@ -0,0 +1,129 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.console; + +import fiftyone.devicedetection.examples.shared.DataFileHelper; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * Runs each on-premise console example's main with no arguments and no + * licence key, as someone who has just cloned the repository runs it, and + * checks that it says what the Enterprise data file adds, with the link to + * the contact page. The examples default to the free Lite data file. CI + * gives the other tests the Enterprise data files, so nothing else tests + * what a fresh clone shows. + */ +public class ExamplesWithoutSettingsTest { + + private static final String CONTACT_US = "https://51degrees.com/contact-us?"; + + private static final String LICENSE_KEY = + UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME; + + /** + * An example's main method. + */ + private interface Main { + void main(String[] args) throws Exception; + } + + /** + * Runs the example's main with no arguments and the licence key system + * property removed, and returns what it wrote to standard output. + */ + private static String runWithoutSettings(Main main) throws Exception { + assumeTrue("Skipping, a licence key is set in the environment", + System.getenv(LICENSE_KEY) == null && + System.getenv(LICENSE_KEY.toUpperCase()) == null); + String licenseKey = System.clearProperty(LICENSE_KEY); + PrintStream out = System.out; + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + System.setOut(new PrintStream(buffer, true, "UTF-8")); + try { + main.main(new String[0]); + } finally { + System.setOut(out); + if (licenseKey != null) { + System.setProperty(LICENSE_KEY, licenseKey); + } + } + String output = buffer.toString("UTF-8"); + out.println(output); + return output; + } + + private static void assertContains(String output, String expected) { + assertTrue("The output should contain '" + expected + "'", + output.contains(expected)); + } + + private static void assertOffer(String output) { + assertContains(output, "This example used the free Lite data file."); + assertContains(output, CONTACT_US); + } + + @Test + public void gettingStarted() throws Exception { + String output = runWithoutSettings(GettingStartedOnPrem::main); + assertContains(output, "Hardware Vendor: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, "Some of these properties need the Enterprise data file."); + assertContains(output, CONTACT_US); + } + + @Test + public void matchMetrics() throws Exception { + assertOffer(runWithoutSettings(MatchMetrics::main)); + } + + @Test + public void metadata() throws Exception { + assertOffer(runWithoutSettings(MetadataOnPrem::main)); + } + + @Test + public void offlineProcessing() throws Exception { + String output = runWithoutSettings(OfflineProcessing::main); + assertContains(output, "device.hardwarevendor: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, "# Some of these properties need the Enterprise " + + "data file. Contact us for a copy at " + CONTACT_US); + } + + @Test + public void performance() throws Exception { + assertOffer(runWithoutSettings(PerformanceBenchmark::main)); + } + + @Test + public void updateDataFile() throws Exception { + String output = runWithoutSettings(UpdateDataFile::main); + assertContains(output, "The data file in use is the 'Lite' file"); + assertContains(output, "which needs a licence key"); + assertContains(output, CONTACT_US); + } +} diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedCloudTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedCloudTest.java index ee38b77f4..c8a85dd39 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedCloudTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/GettingStartedCloudTest.java @@ -24,18 +24,14 @@ import fiftyone.devicedetection.examples.shared.EvidenceHelper; import fiftyone.devicedetection.examples.shared.KeyHelper; - -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class GettingStartedCloudTest { @Test public void gettingStartedCloudTest() throws Exception { - String resourceKey = KeyHelper.getOrSetTestResourceKey(false); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "whether a device is mobile, and its platform and browser", + "device"); GettingStartedCloud.run(resourceKey, EvidenceHelper.setUpEvidence(), System.out); diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataCloudTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataCloudTest.java index cebb3ffed..c818aae69 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataCloudTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MetadataCloudTest.java @@ -23,17 +23,13 @@ package fiftyone.devicedetection.examples.console; import fiftyone.devicedetection.examples.shared.KeyHelper; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class MetadataCloudTest { @Test public void gettingMetaDataCloudTest() throws Exception { - String resourceKey = KeyHelper.getOrSetTestResourceKey(false); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "the device detection properties it offers", "device"); MetadataCloud.run(resourceKey, System.out); } diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MinimalExampleTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MinimalExampleTest.java index f1622ec37..12bd91179 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MinimalExampleTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/MinimalExampleTest.java @@ -23,17 +23,13 @@ package fiftyone.devicedetection.examples.console; import fiftyone.devicedetection.examples.shared.KeyHelper; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class MinimalExampleTest { @Test public void minimalExampleTest() throws Exception { - String resourceKey = KeyHelper.getOrSetTestResourceKey(false); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "whether a device is mobile", "device"); MinimalExample.run(resourceKey); } } \ No newline at end of file diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/NativeModelCloudTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/NativeModelCloudTest.java index 85facd407..9b59297f3 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/NativeModelCloudTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/NativeModelCloudTest.java @@ -23,18 +23,17 @@ package fiftyone.devicedetection.examples.console; import fiftyone.devicedetection.examples.shared.KeyHelper; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; +import java.util.Collections; public class NativeModelCloudTest { @Test public void run() throws Exception { - String resourceKey = - KeyUtils.getNamedKey(NativeModelCloud.NATIVE_MODEL_EXAMPLE_RESOURCE_KEY_NAME); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireResourceKey( + Collections.singletonList( + NativeModelCloud.NATIVE_MODEL_EXAMPLE_RESOURCE_KEY_NAME), + "the devices that match a native model name", "hardware"); NativeModelCloud.run(resourceKey, System.out); } diff --git a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/TacCloudTest.java b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/TacCloudTest.java index a07d2ea14..7d3a356ff 100644 --- a/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/TacCloudTest.java +++ b/examples/device-detection/console/src/test/java/fiftyone/devicedetection/examples/console/TacCloudTest.java @@ -23,18 +23,17 @@ package fiftyone.devicedetection.examples.console; import fiftyone.devicedetection.examples.shared.KeyHelper; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; +import java.util.Collections; public class TacCloudTest { @Test public void run() throws Exception { - String resourceKey = KeyUtils.getNamedKey(TacCloud.TAC_EXAMPLE_RESOURCE_KEY_NAME); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireResourceKey( + Collections.singletonList(TacCloud.TAC_EXAMPLE_RESOURCE_KEY_NAME), + "the devices that match a TAC", "hardware"); TacCloud.run(resourceKey, System.out); } } \ No newline at end of file diff --git a/examples/device-detection/pom.xml b/examples/device-detection/pom.xml index fa55e5534..8e9dd78bc 100644 --- a/examples/device-detection/pom.xml +++ b/examples/device-detection/pom.xml @@ -50,7 +50,7 @@ 4.13.2 true 1.30 - 4.5.128 + 4.5.130 2.22.2 https://oss.sonatype.org diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java index 337f6e165..55a9c44c1 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/DataFileHelper.java @@ -26,6 +26,7 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngineBuilder; import fiftyone.devicedetection.shared.testhelpers.FileUtils; import fiftyone.pipeline.engines.data.AspectEngineDataFile; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFile; import fiftyone.pipeline.util.FileFinder; import org.slf4j.Logger; @@ -33,11 +34,11 @@ import java.io.File; import java.io.IOException; +import java.io.PrintWriter; import java.text.SimpleDateFormat; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Locale; @@ -50,10 +51,30 @@ public class DataFileHelper { /** * Sent with each download, so the Distributor can count the downloads - * made by these examples. + * made by these examples. It is left out inside GitHub Actions, where + * the data repository's script names the build instead, so that test + * runs are not counted as people using the examples. */ private static final String DOWNLOAD_SOURCE = "examples-51degrees-java"; + private static final boolean IN_GITHUB_ACTIONS = + "true".equals(System.getenv("GITHUB_ACTIONS")); + + private static final String WINDOWS_POWERSHELL = "powershell"; + + /** + * Where to ask about the Enterprise data file when the data file in use + * is old or cannot be found. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-datafilehelper.java&utm_term=data-file"; + + /** + * Shown in place of a value for a property that the data file in use + * does not have. Every property the examples ask for is in the + * Enterprise data file. + */ + public static final String NEEDS_ENTERPRISE = "needs the Enterprise data file"; + public static class DatafileInfo { FiftyOneDataFile fileInfo; String tier; @@ -78,6 +99,13 @@ public static DatafileInfo getDatafileMetaData(String dataFileLocation) throws E } } + /** + * Logs the data file in use, and a warning if it is old. An example run + * on the free Lite data file says what the Enterprise data file adds in + * its own output, with {@link #writeEnterpriseNote} or + * {@link #writeEnterpriseOffer}. + * @param engine the engine whose data file is logged + */ public static void logDataFileInfo(DeviceDetectionHashEngine engine) { // Lite or Enterprise String dataTier = engine.getDataSourceTier(); @@ -89,13 +117,6 @@ public static void logDataFileInfo(DeviceDetectionHashEngine engine) { String displayDate = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(fileDate); logger.info("Used a '{}' data file, created {}, {} days ago, from location '{}'", dataTier, displayDate, daysOld, dataFileLocation); - if (dataTier.equals("Lite")) { - logger.warn("This example is using the 'Lite ' data file. " + - "This is used for illustration, and has limited " + - "accuracy and capabilities. Find out about the " + - "Enterprise data file on our pricing page: " + - "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-datafilehelper.java&utm_term=lite-data-file"); - } if (daysOld > 28) { logger.warn("This example is using a data file that is more " + "than {} days old. A more recent data file " + @@ -104,10 +125,9 @@ public static void logDataFileInfo(DeviceDetectionHashEngine engine) { logger.info("Run get-data-files.ps1 -Update or " + "get-data-files.sh -u in cxx/data, which is the data " + "repository https://github.com/51Degrees/data, to get " + - "the latest 'Lite' data file. " + - "Find out about the Enterprise data file, which " + - "includes automatic daily updates, on our pricing " + - "page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-datafilehelper.java&utm_term=data-file-age-warning"); + "the latest 'Lite' data file. Contact us about the " + + "Enterprise data file, which includes automatic daily " + + "updates, at " + CONTACT_US); } } @@ -117,10 +137,67 @@ public static void cantFindDataFile(String dataFile) { logger.info("The free 'Lite' data file and the evidence files " + "are downloaded by get-data-files.ps1 or get-data-files.sh " + "in cxx/data, which is the data repository " + - "https://github.com/51Degrees/data. " + - "Find out about the Enterprise data file, which includes " + - "automatic daily updates, on our pricing " + - "page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-datafilehelper.java&utm_term=data-file-not-found"); + "https://github.com/51Degrees/data. Contact us about the " + + "Enterprise data file, which includes automatic daily " + + "updates, at " + CONTACT_US); + } + + /** + * True when the engine's data file has the property. The Lite data + * file leaves out the properties only the Enterprise data file has, + * such as HardwareVendor, HardwareName and DeviceType. + * @param engine the engine to check + * @param name the name of the property + * @return true if the data file has the property + */ + public static boolean hasProperty(DeviceDetectionHashEngine engine, String name) { + for (FiftyOneAspectPropertyMetaData property : engine.getProperties()) { + if (property.getName().equalsIgnoreCase(name)) { + return true; + } + } + return false; + } + + /** + * True when the engine uses the free Lite data file. + * @param engine the engine to check + * @return true for the Lite data file + */ + public static boolean isLiteDataFile(DeviceDetectionHashEngine engine) { + return "Lite".equals(engine.getDataSourceTier()); + } + + /** + * Writes, after an example has shown properties that the data file in + * use does not have, where to ask for a copy of the Enterprise data + * file. + * @param output where to write + * @param contactUs the link to the contact page, tagged with the + * example's file so an enquiry can be traced to it + */ + public static void writeEnterpriseNote(PrintWriter output, String contactUs) { + output.println(); + output.println("Some of these properties need the Enterprise data " + + "file. Contact us for a copy at " + contactUs); + output.flush(); + } + + /** + * Writes, after an example that shows no properties has run on the + * free Lite data file, what the Enterprise data file adds and where to + * ask for a copy. + * @param output where to write + * @param contactUs the link to the contact page, tagged with the + * example's file so an enquiry can be traced to it + */ + public static void writeEnterpriseOffer(PrintWriter output, String contactUs) { + output.println(); + output.println("This example used the free Lite data file. The " + + "Enterprise data file adds many more properties, such as " + + "HardwareVendor, HardwareName and DeviceType. Contact us " + + "for a copy at " + contactUs); + output.flush(); } /** @@ -173,28 +250,31 @@ public static String getDataFileLocation(String dataFilename) throws Exception { /** * Finds a file below the project root, as - * {@link FileFinder#getFilePath(String)} does. A free data file that is - * missing has a {@code .link} in the data repository, so the data - * repository's script is run to download it, and the file is looked for - * again. + * {@link FileFinder#getFilePath(String)} does, and then in the data + * repository at the root of this repository, which is above the folder + * of the examples that their READMEs run them from. A free data file + * that is missing has a {@code .link} in the data repository, so + * the data repository's script is run to download it, and the file is + * looked for again. * @param file the name of the file, or the end of its path * @return the file * @throws IllegalArgumentException if the file was not found, naming the * command that downloads it when there is one */ public static File findDataFile(String file) { + File repository = findDataRepository(); try { - return getFilePath(file); + return find(file, repository); } catch (IllegalArgumentException notFound) { File link; try { - link = getFilePath(file + ".link"); + link = find(file + ".link", repository); } catch (IllegalArgumentException noLink) { throw notFound; } String command = runDownload(link); try { - return getFilePath(file); + return find(file, repository); } catch (IllegalArgumentException stillNotFound) { String message = "Could not download '" + file + "'. Run this command to download it: " + command; @@ -204,6 +284,37 @@ public static File findDataFile(String file) { } } + /** + * Finds a file below the project root, then below the data repository. + */ + private static File find(String file, File repository) { + try { + return getFilePath(file); + } catch (IllegalArgumentException notFound) { + if (repository == null) { + throw notFound; + } + return getFilePath(file, repository.getPath()); + } + } + + /** + * The data repository, cxx/data, in the nearest folder at or above the + * working directory that has one. + * @return the folder, or null if there is none + */ + private static File findDataRepository() { + File folder = new File(System.getProperty("user.dir")).getAbsoluteFile(); + while (folder != null) { + File data = new File(new File(folder, "cxx"), "data"); + if (new File(data, "get-data-files.sh").isFile()) { + return data; + } + folder = folder.getParentFile(); + } + return null; + } + /** * Runs the data repository's script for the file that a link is for, * and waits for it to finish. The data repository is the folder above @@ -215,17 +326,30 @@ private static String runDownload(File link) { String name = link.getName().substring( 0, link.getName().length() - ".link".length()); File data = link.getAbsoluteFile().getParentFile().getParentFile(); - List command; + List command = new ArrayList<>(); + String powerShell = null; if (System.getProperty("os.name", "") .toLowerCase(Locale.ROOT).startsWith("windows")) { - command = Arrays.asList("powershell", "-NoProfile", - "-ExecutionPolicy", "Bypass", "-File", - new File(data, "get-data-files.ps1").getPath(), name, - "-Source", DOWNLOAD_SOURCE); + powerShell = powerShell(); + command.add(powerShell); + command.add("-NoProfile"); + command.add("-ExecutionPolicy"); + command.add("Bypass"); + command.add("-File"); + command.add(new File(data, "get-data-files.ps1").getPath()); + command.add(name); + if (IN_GITHUB_ACTIONS == false) { + command.add("-Source"); + command.add(DOWNLOAD_SOURCE); + } } else { - command = Arrays.asList("sh", - new File(data, "get-data-files.sh").getPath(), - "-s", DOWNLOAD_SOURCE, name); + command.add("sh"); + command.add(new File(data, "get-data-files.sh").getPath()); + if (IN_GITHUB_ACTIONS == false) { + command.add("-s"); + command.add(DOWNLOAD_SOURCE); + } + command.add(name); } List typed = new ArrayList<>(); for (String argument : command) { @@ -236,7 +360,13 @@ private static String runDownload(File link) { logger.info("'{}' is missing, so it is being downloaded by running {}", name, line); try { - new ProcessBuilder(command).inheritIO().start().waitFor(); + ProcessBuilder builder = new ProcessBuilder(command).inheritIO(); + if (WINDOWS_POWERSHELL.equals(powerShell)) { + // Started from PowerShell 7, Windows PowerShell inherits its + // module path and then cannot load its own commands. + builder.environment().remove("PSModulePath"); + } + builder.start().waitFor(); } catch (IOException e) { logger.error("Could not run {}. {}", line, e.getMessage()); } catch (InterruptedException e) { @@ -244,4 +374,19 @@ private static String runDownload(File link) { } return line; } + + /** + * PowerShell 7 when it is on the path, otherwise Windows PowerShell. + */ + private static String powerShell() { + String path = System.getenv("PATH"); + if (path != null) { + for (String folder : path.split(File.pathSeparator)) { + if (new File(folder.replace("\"", ""), "pwsh.exe").isFile()) { + return "pwsh"; + } + } + } + return WINDOWS_POWERSHELL; + } } diff --git a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java index b6fc1a25f..467a443be 100644 --- a/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java +++ b/examples/device-detection/shared/src/main/java/fiftyone/devicedetection/examples/shared/KeyHelper.java @@ -27,12 +27,127 @@ import org.slf4j.LoggerFactory; import java.util.*; -import java.util.function.Consumer; +import java.util.function.Function; public class KeyHelper { public static final String TEST_RESOURCE_KEY = "TestResourceKey"; static Logger logger = LoggerFactory.getLogger(KeyHelper.class); + /** + * The names every cloud test reads a resource key from first, before + * the names its own example reads. + */ + public static final List SHARED_RESOURCE_KEY_NAMES = + Collections.unmodifiableList(Arrays.asList( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE")); + + static final String CREATE_KEY_URL = + "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-devicedetection-examples-shared-keyhelper.java&utm_term=no-resource-key"; + + /** + * The resource key for a test that calls the live cloud service, from + * {@link #SHARED_RESOURCE_KEY_NAMES} and then {@link #TEST_RESOURCE_KEY}. + * Fails the test when there is none, because a cloud test that passes + * without a key has called nothing. + * @param asksFor what the test asks the service for + * @param products the products the key has to carry + * @return the resource key + */ + public static String requireTestResourceKey(String asksFor, + String... products) { + return requireResourceKey(Collections.singletonList(TEST_RESOURCE_KEY), + asksFor, products); + } + + /** + * The resource key for a test that calls the live cloud service, from + * {@link #SHARED_RESOURCE_KEY_NAMES} and then the given names. + * @param exampleNames the names the example reads, in the order it + * reads them + * @param asksFor what the test asks the service for + * @param products the products the key has to carry + * @return the resource key + */ + public static String requireResourceKey(List exampleNames, + String asksFor, + String... products) { + return requireResourceKey(KeyHelper::readKey, exampleNames, asksFor, + products); + } + + static String requireResourceKey(Function read, + List exampleNames, + String asksFor, + String... products) { + List names = new ArrayList<>(SHARED_RESOURCE_KEY_NAMES); + names.addAll(exampleNames); + for (String name : names) { + String value = read.apply(name); + if (KeyUtils.isInvalidKey(value) == false) { + return value.trim(); + } + } + throw new AssertionError(noResourceKey(names, asksFor, products)); + } + + static String noResourceKey(List names, String asksFor, + String... products) { + return "This test calls the live 51Degrees cloud service, so it " + + "needs a resource key and there is nothing to fall back on. " + + "None of " + String.join(", ", names) + " is set, as an " + + "environment variable or as a system property, tried in that " + + "order. The key has to carry the " + productList(products) + + ", because the test asks the service for " + asksFor + ". To " + + "see the products a key carries, open " + + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=. " + + "Create a key at " + CREATE_KEY_URL + "."; + } + + /** + * The message for a value the cloud answered without. + * @param thing the property or element that has no value + * @param reason the reason the cloud gave, or null + * @param products the products a key needs for the value + * @return the message + */ + public static String answeredWithout(String thing, String reason, + String... products) { + return "The cloud answered without " + thing + " (" + + (reason == null || reason.trim().isEmpty() ? + "no reason given" : reason.trim()) + + "), which is what a key that does not carry the " + + productList(products) + " gets."; + } + + private static String productList(String... products) { + StringBuilder list = new StringBuilder(); + for (int i = 0; i < products.length; i++) { + if (i > 0) { + list.append(i == products.length - 1 ? " and " : ", "); + } + list.append(products[i]); + } + return list + (products.length == 1 ? " product" : " products"); + } + + /** + * A key read as an environment variable, as its upper case name, and + * then as a system property, skipping any that is not a key, such as + * the empty string the example poms pass for a key that is not set. + */ + private static String readKey(String name) { + for (String value : new String[]{ + System.getenv(name), + System.getenv(name.toUpperCase()), + System.getProperty(name)}) { + if (KeyUtils.isInvalidKey(value) == false) { + return value; + } + } + return null; + } + /** * Obtain a resource key either from environment variable or from a property. */ @@ -79,8 +194,11 @@ public static String getOrSetResourceKey(String value, String variableName, } } - // capture the passed parameter for next time called - System.setProperty(variableName, value); + // Keep a valid key for next time. System.setProperty throws on null, + // and an invalid key is not worth keeping. + if (KeyUtils.isInvalidKey(value) == false) { + System.setProperty(variableName, value); + } return value; } public static String getOrSetSuperResourceKey(String value, String variablename) { diff --git a/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/KeyHelperTest.java b/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/KeyHelperTest.java new file mode 100644 index 000000000..1a75b45a0 --- /dev/null +++ b/examples/device-detection/shared/src/test/java/fiftyone/devicedetection/examples/shared/KeyHelperTest.java @@ -0,0 +1,136 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.devicedetection.examples.shared; + +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +public class KeyHelperTest { + /** + * Values of the form KeyUtils accepts as a key. Neither is a real key. + */ + private static final String KEY_A = "AQAAAAAAAAAAAAAAAAAAAAAA"; + private static final String KEY_B = "AgAAAAAAAAAAAAAAAAAAAAAA"; + + /** + * With no key anywhere, the lookup logs that there is none and stores + * nothing, rather than passing null to System.setProperty. + */ + @Test + public void getOrSetTestResourceKey_NoKey_StoresNothing() { + String name = KeyHelper.TEST_RESOURCE_KEY; + // A test can clear a system property but not an environment + // variable. + assumeTrue(name + " is set in the environment", + System.getenv(name) == null && + System.getenv(name.toUpperCase()) == null); + String saved = System.clearProperty(name); + try { + assertNull(KeyHelper.getOrSetTestResourceKey(false)); + assertNull(System.getProperty(name)); + } finally { + if (saved != null) { + System.setProperty(name, saved); + } + } + } + + @Test + public void requireResourceKey_NoKey_FailsNamingEveryName() { + AssertionError error = assertThrows(AssertionError.class, () -> + KeyHelper.requireResourceKey(name -> null, + Collections.singletonList(KeyHelper.TEST_RESOURCE_KEY), + "whether a device is mobile", "device")); + assertEquals("This test calls the live 51Degrees cloud service, so " + + "it needs a resource key and there is nothing to fall back " + + "on. None of _51DEGREES_RESOURCE_KEY, " + + "_51DEGREES_RESOURCE_KEY_BESPOKE, TestResourceKey is set, as " + + "an environment variable or as a system property, tried in " + + "that order. The key has to carry the device product, " + + "because the test asks the service for whether a device is " + + "mobile. To see the products a key carries, open " + + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=. " + + "Create a key at " + KeyHelper.CREATE_KEY_URL + ".", + error.getMessage()); + } + + @Test + public void requireResourceKey_TwoProducts_NamesBoth() { + AssertionError error = assertThrows(AssertionError.class, () -> + KeyHelper.requireResourceKey(name -> null, + Collections.singletonList("SuperResourceKey"), + "the devices that match a TAC", "hardware", "device")); + assertTrue(error.getMessage(), error.getMessage().contains( + "None of _51DEGREES_RESOURCE_KEY, " + + "_51DEGREES_RESOURCE_KEY_BESPOKE, SuperResourceKey is set")); + assertTrue(error.getMessage(), error.getMessage().contains( + "carry the hardware and device products, because")); + } + + /** + * A name holding something that is not a key, such as the empty + * string the example poms pass for a key that is not set, is passed + * over. + */ + @Test + public void requireResourceKey_FirstValidNameWins() { + Map values = new HashMap<>(); + values.put("_51DEGREES_RESOURCE_KEY", ""); + values.put("_51DEGREES_RESOURCE_KEY_BESPOKE", KEY_A); + values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); + assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, + Collections.singletonList(KeyHelper.TEST_RESOURCE_KEY), + "whether a device is mobile", "device")); + } + + @Test + public void requireResourceKey_ExampleNameAfterSharedNames() { + Map values = new HashMap<>(); + values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); + assertEquals(KEY_B, KeyHelper.requireResourceKey(values::get, + Collections.singletonList(KeyHelper.TEST_RESOURCE_KEY), + "whether a device is mobile", "device")); + } + + @Test + public void answeredWithout_NoReason_SaysSo() { + assertEquals("The cloud answered without BrowserName (no reason " + + "given), which is what a key that does not carry the device " + + "product gets.", + KeyHelper.answeredWithout("BrowserName", null, "device")); + assertEquals("The cloud answered without BrowserName (BrowserName " + + "is a paid feature), which is what a key that does not carry " + + "the device product gets.", + KeyHelper.answeredWithout("BrowserName", + "BrowserName is a paid feature", "device")); + } +} diff --git a/examples/device-detection/web/getting-started.cloud/src/main/webapp/WEB-INF/51Degrees-Cloud.xml b/examples/device-detection/web/getting-started.cloud/src/main/webapp/WEB-INF/51Degrees-Cloud.xml index dacfeb006..ced491606 100644 --- a/examples/device-detection/web/getting-started.cloud/src/main/webapp/WEB-INF/51Degrees-Cloud.xml +++ b/examples/device-detection/web/getting-started.cloud/src/main/webapp/WEB-INF/51Degrees-Cloud.xml @@ -42,7 +42,7 @@ JsonBuilderElement diff --git a/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudTest.java b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudTest.java index e2890ac00..11fa1f57a 100644 --- a/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudTest.java +++ b/examples/device-detection/web/getting-started.cloud/src/test/java/fiftyone/devicedetection/web/GettingStartedWebCloudTest.java @@ -24,7 +24,6 @@ import fiftyone.devicedetection.examples.shared.KeyHelper; import fiftyone.devicedetection.examples.web.EmbedJetty; -import fiftyone.devicedetection.shared.testhelpers.KeyUtils; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -33,42 +32,100 @@ import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.Scanner; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static fiftyone.devicedetection.examples.web.GettingStartedWebCloud.resourceBase; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static org.junit.Assert.assertEquals; -import static org.junit.Assume.assumeFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +/** + * Requests the example's page with a known User-Agent and checks that the + * page shows what the cloud detected for it. + */ public class GettingStartedWebCloudTest { + /** + * Chrome 98 on Windows 10. + */ + private static final String USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"; + + /** + * What the page shows in place of a value the cloud did not return, + * followed by the reason. + */ + private static final String UNKNOWN = "Unknown."; + private static Server SERVER = null; @BeforeClass public static void startJetty() throws Exception { + String resourceKey = KeyHelper.requireTestResourceKey( + "the browser and platform of a known User-Agent", "device"); + // The example's configuration file reads the key from here. + System.setProperty(KeyHelper.TEST_RESOURCE_KEY, resourceKey); SERVER = EmbedJetty.startWebApp(getFilePath(resourceBase).getAbsolutePath(), 8081); } @Test public void testWebCloud() throws Exception { - String resourceKey = KeyUtils.getNamedKey("TestResourceKey"); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); - HttpURLConnection connection = (HttpURLConnection) new URL("http://localhost:8081/").openConnection(); + connection.setRequestProperty("User-Agent", USER_AGENT); - InputStream response = connection.getInputStream(); - try (Scanner scanner = new Scanner(response)) { - String responseBody = scanner.useDelimiter("\\A").next(); - System.out.println(responseBody); - } int code = connection.getResponseCode(); + String body = read(code < 400 ? + connection.getInputStream() : connection.getErrorStream()); connection.disconnect(); - assertEquals(200, code); + + assertEquals("The page answered " + code + " with " + body, + 200, code); + assertTrue("The page does not say the cloud engine detected the " + + "values it shows. " + body, + body.contains("cloud device detection engine")); + assertDetected(body, "Browser Name", "BrowserName", "Chrome"); + assertDetected(body, "Platform Name", "PlatformName", "Windows"); + } + + /** + * Checks the value in the page's row for a property, which is the + * reason when the cloud returned no value. + */ + private static void assertDetected(String body, String label, + String property, String expected) { + Matcher row = Pattern.compile(Pattern.quote(label + + "") + "(.*?)") + .matcher(body); + assertTrue("The page has no " + label + " row. " + body, row.find()); + String value = row.group(1).trim(); + if (value.startsWith(UNKNOWN)) { + String reason = value.substring(UNKNOWN.length()).trim(); + fail(KeyHelper.answeredWithout(property, + "null".equals(reason) ? null : reason, "device")); + } + assertEquals(label + " on the page", expected, value); + } + + private static String read(InputStream stream) { + if (stream == null) { + return ""; + } + try (Scanner scanner = new Scanner(stream, + StandardCharsets.UTF_8.name())) { + scanner.useDelimiter("\\A"); + return scanner.hasNext() ? scanner.next() : ""; + } } @AfterClass public static void stopJetty() throws Exception { - SERVER.stop(); + if (SERVER != null) { + SERVER.stop(); + } } } \ No newline at end of file diff --git a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java index 8387720c6..e74dec2b1 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java +++ b/examples/device-detection/web/getting-started.onprem/src/main/java/fiftyone/devicedetection/examples/web/GettingStartedWebOnPrem.java @@ -48,7 +48,7 @@ *

    * The configuration file for the pipeline is at src/main/webapp/WEB-INF/51Degrees-OnPrem.xml *

    - * The data file is taken from the 51DEGREES_DD_PATH environment variable, then + * The data file is taken from the _51DEGREES_DD_PATH environment variable, then * from TestDataFile (an environment variable or system property), and otherwise * the free Lite file in cxx/data/device-detection is used, which is downloaded * before the server starts if it is missing. @@ -62,9 +62,37 @@ public class GettingStartedWebOnPrem extends HttpServlet { public static String resourceBase = "examples/device-detection/web/getting-started.onprem/src/main/webapp"; public static Logger logger = LoggerFactory.getLogger(GettingStartedWebOnPrem.class); + private static final String DD_PATH_VARIABLE = "_51DEGREES_DD_PATH"; + private static final String OLD_DD_PATH_VARIABLE = "51DEGREES_DD_PATH"; + + /** + * Logs a warning when the configuration takes the data file from the + * deprecated name, which it does when that name is set and the name + * that replaces it is not. + */ + private static void warnOfOldDataFileName() { + if (isSet(OLD_DD_PATH_VARIABLE) && isSet(DD_PATH_VARIABLE) == false) { + logger.warn( + "The environment variable " + OLD_DD_PATH_VARIABLE + + " is deprecated and " + DD_PATH_VARIABLE + + " replaces it. Set " + DD_PATH_VARIABLE + + " to the same value, as a later release may no longer " + + "read " + OLD_DD_PATH_VARIABLE + "."); + } + } + + private static boolean isSet(String name) { + String value = System.getenv(name); + if (value == null || value.isEmpty()) { + value = System.getProperty(name); + } + return value != null && value.isEmpty() == false; + } + public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); logger.info("Running Example {}", GettingStartedWebOnPrem.class); + warnOfOldDataFileName(); // If the data file the configuration names is not at that path, it // is searched for, and the free Lite file is downloaded if missing. diff --git a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml index bba6dd60e..f6bc4c25d 100644 --- a/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml +++ b/examples/device-detection/web/getting-started.onprem/src/main/webapp/WEB-INF/51Degrees-OnPrem.xml @@ -28,18 +28,19 @@ false false - ${51DEGREES_DD_PATH:-${TestDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash}} + ${_51DEGREES_DD_PATH:-${51DEGREES_DD_PATH:-${TestDataFile:-cxx/data/device-detection/51Degrees-LiteV4.1.hash}}} LowMemory DeviceDetectionHashEngine JsonBuilderElement diff --git a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java index 108fbd615..7f9bc4df7 100644 --- a/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java +++ b/examples/device-detection/web/getting-started.onprem/src/test/java/fiftyone/devicedetection/examples/web/OnPremDataFileOptionTest.java @@ -35,17 +35,19 @@ /** * Checks the order in which the on-premise web example picks its data file, - * being 51DEGREES_DD_PATH first, then TestDataFile, then the Lite file. + * being _51DEGREES_DD_PATH first, then TestDataFile, then the Lite file. * System properties stand in for the environment variables, because the * lookup reads either and a test cannot set an environment variable. */ public class OnPremDataFileOptionTest { - private static final String DD_PATH = "51DEGREES_DD_PATH"; + private static final String DD_PATH = "_51DEGREES_DD_PATH"; + private static final String OLD_DD_PATH = "51DEGREES_DD_PATH"; private static final String TEST_DATA_FILE = "TestDataFile"; private static final String LITE = "cxx/data/device-detection/51Degrees-LiteV4.1.hash"; private String savedDdPath; + private String savedOldDdPath; private String savedTestDataFile; @Before @@ -53,17 +55,21 @@ public void saveProperties() { // An environment variable of either name would take precedence over // the system properties set here, so the test cannot run. assumeTrue(System.getenv(DD_PATH) == null); + assumeTrue(System.getenv(OLD_DD_PATH) == null); assumeTrue(System.getenv(TEST_DATA_FILE) == null); assumeTrue(System.getenv(TEST_DATA_FILE.toUpperCase()) == null); savedDdPath = System.getProperty(DD_PATH); + savedOldDdPath = System.getProperty(OLD_DD_PATH); savedTestDataFile = System.getProperty(TEST_DATA_FILE); System.clearProperty(DD_PATH); + System.clearProperty(OLD_DD_PATH); System.clearProperty(TEST_DATA_FILE); } @After public void restoreProperties() { restore(DD_PATH, savedDdPath); + restore(OLD_DD_PATH, savedOldDdPath); restore(TEST_DATA_FILE, savedTestDataFile); } @@ -85,6 +91,20 @@ public void prefersDdPathOverTestDataFile() throws Exception { assertEquals("from-dd-path.hash", dataFile()); } + @Test + public void usesOldDdPathWhenNewIsNotSet() throws Exception { + System.setProperty(TEST_DATA_FILE, "from-test-data-file.hash"); + System.setProperty(OLD_DD_PATH, "from-old-dd-path.hash"); + assertEquals("from-old-dd-path.hash", dataFile()); + } + + @Test + public void prefersDdPathOverOldDdPath() throws Exception { + System.setProperty(OLD_DD_PATH, "from-old-dd-path.hash"); + System.setProperty(DD_PATH, "from-dd-path.hash"); + assertEquals("from-dd-path.hash", dataFile()); + } + private static String dataFile() throws Exception { PipelineOptions options = PipelineOptionsFactory.getOptionsFromFile( getFilePath(resourceBase) + "/WEB-INF/51Degrees-OnPrem.xml"); diff --git a/examples/device-detection/web/shared/src/main/java/fiftyone/devicedetection/examples/web/HtmlContentHelper.java b/examples/device-detection/web/shared/src/main/java/fiftyone/devicedetection/examples/web/HtmlContentHelper.java index bf4467038..2df9b35a9 100644 --- a/examples/device-detection/web/shared/src/main/java/fiftyone/devicedetection/examples/web/HtmlContentHelper.java +++ b/examples/device-detection/web/shared/src/main/java/fiftyone/devicedetection/examples/web/HtmlContentHelper.java @@ -22,11 +22,13 @@ package fiftyone.devicedetection.examples.web; +import fiftyone.devicedetection.examples.shared.DataFileHelper; import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.shared.DeviceData; import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.engines.data.AspectPropertyValue; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -40,6 +42,7 @@ import java.util.Date; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import static fiftyone.devicedetection.examples.shared.PropertyHelper.asString; import static fiftyone.devicedetection.examples.shared.PropertyHelper.tryGet; @@ -172,6 +175,9 @@ public static void doDataFileAgeWarning(PrintWriter out, FlowData flowData) { public static void doDeviceData(PrintWriter out, DeviceData device, FlowData flowData, String dataFileLocation) { FlowElement engine = getFlowElement(flowData); + // The on-premise engine says which properties its data file has. + DeviceDetectionHashEngine onPremise = engine instanceof DeviceDetectionHashEngine ? + (DeviceDetectionHashEngine) engine : null; String content = ""; if (engine instanceof CloudRequestEngine) { content = "

    The following values were detected using the cloud " + @@ -200,20 +206,35 @@ public static void doDeviceData(PrintWriter out, DeviceData device, FlowData flo "Key" + "Value\n" + " \n" + - " Hardware Vendor" + asString(tryGet(device::getHardwareVendor)) + "\n" + - " Hardware Name" + asString(tryGet(device::getHardwareName)) + "\n" + - " Device Type" + asString(tryGet(device::getDeviceType)) + "\n" + - " Platform Vendor" + asString(tryGet(device::getPlatformVendor)) + "\n" + - " Platform Name" + asString(tryGet(device::getPlatformName)) + "\n" + - " Platform Version" + asString(tryGet(device::getPlatformVersion)) + "\n" + - " Browser Vendor" + asString(tryGet(device::getBrowserVendor)) + "\n" + - " Browser Name" + asString(tryGet((device::getBrowserName))) + "\n" + - " Browser Version" + asString(tryGet(device::getBrowserVersion)) + "\n" + + " Hardware Vendor" + value(onPremise, "HardwareVendor", device::getHardwareVendor) + "\n" + + " Hardware Name" + value(onPremise, "HardwareName", device::getHardwareName) + "\n" + + " Device Type" + value(onPremise, "DeviceType", device::getDeviceType) + "\n" + + " Platform Vendor" + value(onPremise, "PlatformVendor", device::getPlatformVendor) + "\n" + + " Platform Name" + value(onPremise, "PlatformName", device::getPlatformName) + "\n" + + " Platform Version" + value(onPremise, "PlatformVersion", device::getPlatformVersion) + "\n" + + " Browser Vendor" + value(onPremise, "BrowserVendor", device::getBrowserVendor) + "\n" + + " Browser Name" + value(onPremise, "BrowserName", device::getBrowserName) + "\n" + + " Browser Version" + value(onPremise, "BrowserVersion", device::getBrowserVersion) + "\n" + " \n" + " \n" + "\n"); } + /** + * The value of a property for the page. With the on-premise engine, a + * property its data file does not have needs the Enterprise data file, + * and the value is only read when the data file has the property. + * @param onPremise the on-premise engine, or null for the cloud + */ + private static String value(DeviceDetectionHashEngine onPremise, + String property, + Supplier> value) { + if (onPremise != null && DataFileHelper.hasProperty(onPremise, property) == false) { + return DataFileHelper.NEEDS_ENTERPRISE; + } + return asString(tryGet(value)); + } + public static void doUachInfo(PrintWriter out) { //language=html out.append( diff --git a/examples/ip-intelligence/LICENSE b/examples/ip-intelligence/LICENSE deleted file mode 100644 index 4153cd377..000000000 --- a/examples/ip-intelligence/LICENSE +++ /dev/null @@ -1,287 +0,0 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. diff --git a/examples/ip-intelligence/README.md b/examples/ip-intelligence/README.md index 9344df4e0..e3775778c 100644 --- a/examples/ip-intelligence/README.md +++ b/examples/ip-intelligence/README.md @@ -1,8 +1,8 @@ # 51Degrees IP Intelligence Engines - Examples -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines-examples "Data rewards the curious") **Java IP Intelligence** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-java/main/.github/images/logo-160.png "Data rewards the curious") **Java IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-java/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines-examples "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines-examples "developer documentation") ## Introduction @@ -14,28 +14,44 @@ The data files for on-premise IP intelligence are kept in `cxx/data/ip-intellige folder of the [data](https://github.com/51Degrees/data) repository, which is a submodule of cxx. -**IMPORTANT**: The enterprise IP Intelligence data file must be placed in the -`cxx/data/ip-intelligence` directory of this repository. The expected filename is -`51Degrees-EnterpriseIpiV41.ipi`. - -Two free data files are also usable for many of the examples, the ASN file -(`51Degrees-IPIV4AsnIpiV41.ipi`) and the Lite file (`51Degrees-LiteV41.ipi`). Run -`cxx/data/get-data-files.sh ip-intelligence` from the root of this repository to download -them, or `cxx\data\get-data-files.ps1 ip-intelligence` in PowerShell on Windows. An example -that needs one of them and cannot find it runs the script for that file, and logs the -command it runs. When the enterprise file is -absent, the tests for examples that work with any data file fall back to the ASN file and -then the Lite file. The OfflineProcessing and PerformanceBenchmark examples need the -RegisteredName property, which the ASN file does not contain, so their tests fall back to -the Lite file only. Tests for examples needing enterprise-only properties (Metrics, Compare -and Suspicious) are skipped instead. +The examples use the free data files by default, so they run from a fresh clone with no +settings. The examples that show location and network properties use the Lite file +(`51Degrees-LiteV41.ipi`), which has the registration properties, and the performance and +update examples use the much smaller ASN file (`51Degrees-IPIV4AsnIpiV41.ipi`). An example +that cannot find its file runs the data repository's script to download it, and logs the +command it runs. To download both first, run `cxx/data/get-data-files.sh ip-intelligence` +from the root of this repository, or `cxx\data\get-data-files.ps1 ip-intelligence` in +PowerShell on Windows. + +Each example shows the properties that only the Enterprise data file has as needing it, and +gives a link to ask for a copy. The Compare, Metrics and Suspicious examples need those +properties, so with a free data file they say which they need and stop. +[Contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=enterprise-data-file) for a copy of +the Enterprise data file. To use it, place it in `cxx/data/ip-intelligence` and pass its path +as the first argument, or set the `_51DEGREES_IPI_PATH` environment variable or system +property to its path. + +The Enterprise data file is over 6 GB, and the MaxPerformance profile loads all of it into +memory, so the examples built for it, or that may be given it, use the LowMemory profile, +which reads the file from disk as it is needed. Those are the Compare, Metrics, +OfflineProcessing, Suspicious and UpdateDataFile examples. The Metrics and Compare examples +take a profile as their fourth argument, and `MaxPerformance` there makes them faster on a +machine with memory to hold the whole file. + +The `ExamplesWithoutSettingsTest` tests run each console example as a fresh clone does. The +other tests use the Enterprise data file, `51Degrees-EnterpriseIpiV41.ipi`, when it is in +`cxx/data/ip-intelligence`. When it is absent, the tests for examples that work with any data +file fall back to the ASN file and then the Lite file. The OfflineProcessing and +PerformanceBenchmark tests fall back to the Lite file only, and the tests of the Metrics, +Compare and Suspicious examples with the Enterprise data file are skipped. ``` 51degrees-java/ ├── cxx/ │ └── data/ │ └── ip-intelligence/ -│ └── 51Degrees-EnterpriseIpiV41.ipi +│ ├── 51Degrees-LiteV41.ipi +│ └── 51Degrees-IPIV4AsnIpiV41.ipi └── examples/ └── ip-intelligence/ ├── console/ @@ -45,12 +61,15 @@ and Suspicious) are skipped instead. The examples locate the data file in the following order: -1. The `51DEGREES_IPI_PATH` environment variable or system property, which can - be set to an explicit path to the data file. -2. A search of the project folder hierarchy for the expected file name. -3. The expected location `cxx/data/ip-intelligence/51Degrees-EnterpriseIpiV41.ipi` - (the configuration files for the getting started examples fall back to the - free Lite file `cxx/data/ip-intelligence/51Degrees-LiteV41.ipi`). +1. The `_51DEGREES_IPI_PATH` environment variable or system property, which can + be set to an explicit path to the data file. It is used in place of any + file named without a full path. +2. The path given as the first argument, or the free file the example uses + by default, searched for below the folder the example is run from and then + below `cxx/data`, so that the examples find it whether they are run from + the root of this repository or from this folder. +3. A free file that is still missing is downloaded by the data repository's + script. This project contains sub-modules - **console**, giving examples that are intended to be run from the command line/console and **web**, illustrating use @@ -85,7 +104,7 @@ environment variable or system property. | MetadataCloud | How to access the meta-data that relates to the properties available from the cloud IP intelligence service. | Cloud examples need a resource key. The key is read from the -`51DEGREES_RESOURCE_KEY` environment variable or system property first. The +`_51DEGREES_RESOURCE_KEY` environment variable or system property first. The legacy `TestResourceKey` name is still supported and is checked second. The cloud property tiers changed in May 2026. The examples and this diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java index 4b994e3b9..0fc3c7bbf 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java @@ -54,10 +54,15 @@ * The output CSV file contains the input truth and the result fields for easy * evaluation. * + * The performance profile can be given as the fourth argument, for example + * MaxPerformance, which makes each lookup faster where there is memory to + * hold the whole data file. + * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/CompareOnPrem.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=contact-us). + * This example needs the location properties only the Enterprise IP Intelligence data file has. + * With the free Lite data file, which it uses by default, it says which and stops. To obtain an + * Enterprise data file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=contact-us). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -67,6 +72,7 @@ package fiftyone.ipintelligence.examples.console; +import fiftyone.ipintelligence.IPIntelligenceOnPremisePipelineBuilder; import fiftyone.ipintelligence.IPIntelligencePipelineBuilder; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.examples.console.areas.Calculations; @@ -87,6 +93,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -95,7 +102,7 @@ import java.util.concurrent.Future; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.firstValue; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.tryGet; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -122,6 +129,34 @@ public class CompareOnPrem { */ public static final String DEFAULT_OUTPUT_FILE = "compare-output.csv"; + /** + * The performance profile used unless one is given as the fourth + * argument. This example needs the Enterprise data file, which is over + * 6 GB, and LowMemory reads it from disk as it is needed, so the example + * runs without memory to hold the whole file. MaxPerformance loads the + * whole file into memory, which makes each lookup faster where there is + * memory to hold it. + */ + public static final Constants.PerformanceProfiles + DEFAULT_PERFORMANCE_PROFILE = + Constants.PerformanceProfiles.LowMemory; + + /** + * The properties the comparison asks the engine for, which only the + * Enterprise data file has. + */ + public static final List LOCATION_PROPERTIES = Arrays.asList( + "Latitude", + "Longitude", + "LocationConfidence", + "Areas"); + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=enterprise-data-file"; + /** * Mean radius of the earth in kilometers, used in the haversine distance * calculation. @@ -159,14 +194,33 @@ public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); // Use the supplied path for the data file - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = DataFileHelper.getDataFileLocation( + args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH); // Use the supplied path for the truth CSV file String csvTruthFile = args.length > 1 ? args[1] : GEOIP_COMPARISON_EVIDENCE_REL_PATH; // Get the location for the output file String outputFile = args.length > 2 ? args[2] : DEFAULT_OUTPUT_FILE; + // Get the performance profile named, ignoring case, such as + // MaxPerformance, or use the default + Constants.PerformanceProfiles profile = args.length > 3 ? + DataFileHelper.findPerformanceProfile( + args[3], new PrintWriter(System.out)) : + DEFAULT_PERFORMANCE_PROFILE; + if (profile == null) { + return; + } + + // The free data files have none of the location properties. + if (DataFileHelper.lacksEnterpriseProperties(dataFile, + "Comparing known locations with IP Intelligence", + LOCATION_PROPERTIES, + new PrintWriter(System.out), + CONTACT_US)) { + return; + } try (PrintWriter writer = new PrintWriter(new FileWriter(outputFile))) { - run(dataFile, csvTruthFile, writer); + run(dataFile, csvTruthFile, profile, writer); } logger.info("Output written to '{}'", outputFile); } @@ -180,6 +234,20 @@ public static void main(String[] args) throws Exception { public static void run(String dataFile, String csvTruthFile, PrintWriter output) throws Exception { + run(dataFile, csvTruthFile, DEFAULT_PERFORMANCE_PROFILE, output); + } + + /** + * Run the example + * @param dataFile an IP Intelligence data file + * @param csvTruthFile a CSV file of truth records + * @param profile the performance profile to build the engine with + * @param output somewhere for the result CSV + */ + public static void run(String dataFile, + String csvTruthFile, + Constants.PerformanceProfiles profile, + PrintWriter output) throws Exception { logger.info("Running CompareOnPrem example"); String dataFileLocation; @@ -191,33 +259,33 @@ public static void run(String dataFile, throw e; } - File truthFile = getFilePath(csvTruthFile); + File truthFile = DataFileHelper.findDataFile(csvTruthFile); int processors = Runtime.getRuntime().availableProcessors(); // Build a new pipeline with an on-premise IP Intelligence engine using - // the max performance profile and restricted to the properties needed. - try (Pipeline pipeline = new IPIntelligencePipelineBuilder() + // the performance profile given and restricted to the properties + // needed. + IPIntelligenceOnPremisePipelineBuilder builder = new IPIntelligencePipelineBuilder() .useOnPremise(dataFileLocation, false) - // We use the max performance profile for optimal lookup - // speed in this example. See the documentation for more detail - // on this and other configuration options. + // The profile trades memory for speed, as + // DEFAULT_PERFORMANCE_PROFILE explains. See the documentation + // for more detail on this and other configuration options. // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-compareonprem.java&utm_term=automatic-datafile-updates - .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) + .setPerformanceProfile(profile) // inhibit sharing usage for this example .setShareUsage(false) // inhibit auto-update of the data file for this example .setAutoUpdate(false) .setDataUpdateOnStartup(false) .setDataFileSystemWatcher(false) - // Set to only return from processing the properties needed. - .setProperty("Latitude") - .setProperty("Longitude") - .setProperty("LocationConfidence") - .setProperty("Areas") // Optimize for the expected parallel workload. - .setConcurrency(processors) - .build()) { + .setConcurrency(processors); + // Set to only return from processing the properties needed. + for (String property : LOCATION_PROPERTIES) { + builder.setProperty(property); + } + try (Pipeline pipeline = builder.build()) { // Read the truth records, skipping duplicate IP addresses. List truths = readTruths(truthFile); diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java index 07f33c130..f007e2fb3 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java @@ -35,9 +35,10 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedMixed.java). * - * This example requires: - * - A Device Detection data file (.hash format) - * - An IP Intelligence data file (.ipi format) + * By default the example uses the free Lite Device Detection and IP Intelligence data files, + * which it downloads when they are missing, and shows each property only the Enterprise data + * files have as needing them. To obtain the Enterprise data files, please + * [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedmixed.java&utm_term=header). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -55,6 +56,8 @@ import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.flowelements.FiftyOnePipelineBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,6 +70,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import java.util.function.Supplier; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.*; @@ -84,7 +88,13 @@ public class GettingStartedMixed { private static final Logger logger = LoggerFactory.getLogger(GettingStartedMixed.class); public static final String DD_DATA_FILE_DEFAULT = "device-detection/51Degrees-LiteV4.1.hash"; - public static final String IPI_DATA_FILE_DEFAULT = DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; + public static final String IPI_DATA_FILE_DEFAULT = DataFileHelper.LITE_DATA_FILE_REL_PATH; + + /** + * Where to ask for a copy of the Enterprise data files, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedmixed.java&utm_term=enterprise-data-file"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); @@ -128,7 +138,7 @@ public static void run(String ddDataFile, logger.error("Failed to find IP Intelligence data file at '{}'. " + "Please provide a valid path to an IP Intelligence data file (.ipi). " + "An explicit path can be supplied via the {} environment variable.", - ipiDataFile, DataFileHelper.IPI_PATH_ENV_VAR); + ipiDataFile, DataFileHelper.IPI_PATH_VARIABLE); throw e; } @@ -139,20 +149,27 @@ public static void run(String ddDataFile, try (Pipeline pipeline = new FiftyOnePipelineBuilder() .buildFromConfiguration(pipelineOptions)) { + // The engines say which properties their data files have. + DeviceDetectionHashEngine ddEngine = pipeline.getElement(DeviceDetectionHashEngine.class); + IPIntelligenceOnPremiseEngine ipiEngine = pipeline.getElement(IPIntelligenceOnPremiseEngine.class); + // Process each set of evidence + boolean needsEnterprise = false; for (Map evidence : evidenceList) { - analyzeEvidence(evidence, pipeline, outputStream); + needsEnterprise |= analyzeEvidence( + evidence, pipeline, ddEngine, ipiEngine, outputStream); + } + if (needsEnterprise) { + DataFileHelper.writeEnterpriseNote(new PrintWriter(outputStream), CONTACT_US); } // Log data file info for both engines - DeviceDetectionHashEngine ddEngine = pipeline.getElement(DeviceDetectionHashEngine.class); if (ddEngine != null) { logger.info("Device Detection data file: '{}', tier: '{}'", ddEngine.getDataFileMetaData().getDataFilePath(), ddEngine.getDataSourceTier()); } - IPIntelligenceOnPremiseEngine ipiEngine = pipeline.getElement(IPIntelligenceOnPremiseEngine.class); if (ipiEngine != null) { DataFileHelper.logDataFileInfo(ipiEngine); } @@ -195,9 +212,18 @@ public static List> setUpCombinedEvidence() { return evidence; } - private static void analyzeEvidence(Map evidence, - Pipeline pipeline, - OutputStream out) throws Exception { + /** + * Processes the evidence and writes both engines' results, showing each + * property the data files in use do not have as needing the Enterprise + * data file. + * @return true if a property needed the Enterprise data file + */ + private static boolean analyzeEvidence(Map evidence, + Pipeline pipeline, + DeviceDetectionHashEngine ddEngine, + IPIntelligenceOnPremiseEngine ipiEngine, + OutputStream out) throws Exception { + boolean needsEnterprise = false; PrintWriter writer = new PrintWriter(out); try (FlowData data = pipeline.createFlowData()) { @@ -219,40 +245,99 @@ private static void analyzeEvidence(Map evidence, writer.println("\nDevice Detection Results:"); writer.println("-------------------------"); DeviceData device = data.get(DeviceData.class); - writer.println("\tMobile Device: " + asString(tryGet(device::getIsMobile))); - writer.println("\tPlatform Name: " + asString(tryGet(device::getPlatformName))); - writer.println("\tPlatform Version: " + asString(tryGet(device::getPlatformVersion))); - writer.println("\tBrowser Name: " + asString(tryGet(device::getBrowserName))); - writer.println("\tBrowser Version: " + asString(tryGet(device::getBrowserVersion))); - writer.println("\tHardware Vendor: " + asString(tryGet(device::getHardwareVendor))); - writer.println("\tHardware Name: " + asString(tryGet(device::getHardwareName))); - writer.println("\tDevice Type: " + asString(tryGet(device::getDeviceType))); - writer.println("\tScreen Width: " + asString(tryGet(device::getScreenPixelsWidth))); - writer.println("\tScreen Height: " + asString(tryGet(device::getScreenPixelsHeight))); + needsEnterprise |= writeValue(writer, "Mobile Device", + hasProperty(ddEngine, "IsMobile"), device::getIsMobile); + needsEnterprise |= writeValue(writer, "Platform Name", + hasProperty(ddEngine, "PlatformName"), device::getPlatformName); + needsEnterprise |= writeValue(writer, "Platform Version", + hasProperty(ddEngine, "PlatformVersion"), device::getPlatformVersion); + needsEnterprise |= writeValue(writer, "Browser Name", + hasProperty(ddEngine, "BrowserName"), device::getBrowserName); + needsEnterprise |= writeValue(writer, "Browser Version", + hasProperty(ddEngine, "BrowserVersion"), device::getBrowserVersion); + needsEnterprise |= writeValue(writer, "Hardware Vendor", + hasProperty(ddEngine, "HardwareVendor"), device::getHardwareVendor); + needsEnterprise |= writeValue(writer, "Hardware Name", + hasProperty(ddEngine, "HardwareName"), device::getHardwareName); + needsEnterprise |= writeValue(writer, "Device Type", + hasProperty(ddEngine, "DeviceType"), device::getDeviceType); + needsEnterprise |= writeValue(writer, "Screen Width", + hasProperty(ddEngine, "ScreenPixelsWidth"), device::getScreenPixelsWidth); + needsEnterprise |= writeValue(writer, "Screen Height", + hasProperty(ddEngine, "ScreenPixelsHeight"), device::getScreenPixelsHeight); // Get IP Intelligence results writer.println("\nIP Intelligence Results:"); writer.println("------------------------"); IPIntelligenceData ipData = data.get(IPIntelligenceData.class); - writer.println("\tRegisteredName: " + asStringProperty(tryGet(ipData::getRegisteredName))); - writer.println("\tRegisteredOwner: " + asStringProperty(tryGet(ipData::getRegisteredOwner))); - writer.println("\tRegisteredCountry: " + asStringProperty(tryGet(ipData::getRegisteredCountry))); - writer.println("\tIpRangeStart: " + asIPAddressProperty(tryGet(ipData::getIpRangeStart))); - writer.println("\tIpRangeEnd: " + asIPAddressProperty(tryGet(ipData::getIpRangeEnd))); - writer.println("\tCountry: " + asStringProperty(tryGet(ipData::getCountry))); - writer.println("\tCountryCode: " + asStringProperty(tryGet(ipData::getCountryCode))); - writer.println("\tCountryCode3: " + asStringProperty(tryGet(ipData::getCountryCode3))); - writer.println("\tRegion: " + asStringProperty(tryGet(ipData::getRegion))); - writer.println("\tState: " + asStringProperty(tryGet(ipData::getState))); - writer.println("\tTown: " + asStringProperty(tryGet(ipData::getTown))); - writer.println("\tLatitude: " + asFloatProperty(tryGet(ipData::getLatitude))); - writer.println("\tLongitude: " + asFloatProperty(tryGet(ipData::getLongitude))); - writer.println("\tAreas: " + asWktStringProperty(tryGet(ipData::getAreas))); - writer.println("\tAccuracyRadiusMin: " + asIntegerProperty(tryGet(ipData::getAccuracyRadiusMin))); - writer.println("\tTimeZoneOffset: " + asIntegerProperty(tryGet(ipData::getTimeZoneOffset))); + needsEnterprise |= writeValue(writer, "RegisteredName", + DataFileHelper.hasProperty(ipiEngine, "RegisteredName"), ipData::getRegisteredName); + needsEnterprise |= writeValue(writer, "RegisteredOwner", + DataFileHelper.hasProperty(ipiEngine, "RegisteredOwner"), ipData::getRegisteredOwner); + needsEnterprise |= writeValue(writer, "RegisteredCountry", + DataFileHelper.hasProperty(ipiEngine, "RegisteredCountry"), ipData::getRegisteredCountry); + needsEnterprise |= writeValue(writer, "IpRangeStart", + DataFileHelper.hasProperty(ipiEngine, "IpRangeStart"), ipData::getIpRangeStart); + needsEnterprise |= writeValue(writer, "IpRangeEnd", + DataFileHelper.hasProperty(ipiEngine, "IpRangeEnd"), ipData::getIpRangeEnd); + needsEnterprise |= writeValue(writer, "Country", + DataFileHelper.hasProperty(ipiEngine, "Country"), ipData::getCountry); + needsEnterprise |= writeValue(writer, "CountryCode", + DataFileHelper.hasProperty(ipiEngine, "CountryCode"), ipData::getCountryCode); + needsEnterprise |= writeValue(writer, "CountryCode3", + DataFileHelper.hasProperty(ipiEngine, "CountryCode3"), ipData::getCountryCode3); + needsEnterprise |= writeValue(writer, "Region", + DataFileHelper.hasProperty(ipiEngine, "Region"), ipData::getRegion); + needsEnterprise |= writeValue(writer, "State", + DataFileHelper.hasProperty(ipiEngine, "State"), ipData::getState); + needsEnterprise |= writeValue(writer, "Town", + DataFileHelper.hasProperty(ipiEngine, "Town"), ipData::getTown); + needsEnterprise |= writeValue(writer, "Latitude", + DataFileHelper.hasProperty(ipiEngine, "Latitude"), ipData::getLatitude); + needsEnterprise |= writeValue(writer, "Longitude", + DataFileHelper.hasProperty(ipiEngine, "Longitude"), ipData::getLongitude); + needsEnterprise |= writeValue(writer, "Areas", + DataFileHelper.hasProperty(ipiEngine, "Areas"), ipData::getAreas); + needsEnterprise |= writeValue(writer, "AccuracyRadiusMin", + DataFileHelper.hasProperty(ipiEngine, "AccuracyRadiusMin"), ipData::getAccuracyRadiusMin); + needsEnterprise |= writeValue(writer, "TimeZoneOffset", + DataFileHelper.hasProperty(ipiEngine, "TimeZoneOffset"), ipData::getTimeZoneOffset); } writer.println(); writer.flush(); + return needsEnterprise; + } + + /** + * Writes a property, or that it needs the Enterprise data file when the + * data file in use does not have it. The value is only read when the + * data file has the property. + * @return true if the property needs the Enterprise data file + */ + private static boolean writeValue(PrintWriter writer, + String name, + boolean hasProperty, + Supplier> value) { + if (hasProperty) { + writer.println("\t" + name + ": " + asString(value.get())); + return false; + } + writer.println("\t" + name + ": " + DataFileHelper.NEEDS_ENTERPRISE); + return true; + } + + /** + * True when the Device Detection engine's data file has the property. + * The Lite data file leaves out the properties only the Enterprise data + * file has, such as HardwareVendor, HardwareName and DeviceType. + */ + private static boolean hasProperty(DeviceDetectionHashEngine engine, String name) { + for (FiftyOneAspectPropertyMetaData property : engine.getProperties()) { + if (property.getName().equalsIgnoreCase(name)) { + return true; + } + } + return false; } /** diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java index 41a7712a5..e25f3e109 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java @@ -33,8 +33,10 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/GettingStartedOnPrem.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=header). + * By default the example uses the free Lite IP Intelligence data file, which it downloads when + * it is missing. The Lite data file has the registration properties, and the example shows each + * property only the Enterprise data file has as needing it. To obtain an Enterprise data file, + * please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=header). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -62,16 +64,12 @@ import java.io.PrintWriter; import java.util.List; import java.util.Map; +import java.util.function.Supplier; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asStringProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asIntegerProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asFloatProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asIPAddressProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asWktStringProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.tryGet; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asString; import static fiftyone.pipeline.util.FileFinder.getFilePath; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; /** * Provides an illustration of the fundamental elements of carrying out IP Intelligence using @@ -86,23 +84,23 @@ public class GettingStartedOnPrem { private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); - /* In this example, the 51degrees IP Intelligence data file needs to be specified as a command line parameter. - - To test this example, you need to: - 1. Host an IP Intelligence data file (.ipi) at a custom URL accessible to this application - 2. Provide the path to that hosted data file as a command line parameter - 3. No license key is required when using a custom URL - - For production use, you will eventually need to use a Distributor service and license key - to keep your data file updated. - - To obtain access to enterprise data files for hosting, please contact us: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=main */ + /* In this example, by default, the free 51degrees Lite IP Intelligence data file is used from + the data repository in cxx/data, and is downloaded if it is missing, or you may specify another + file as a command line parameter. + + Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=main */ + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=enterprise-data-file"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); // Use the supplied path for the data file - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH; // prepare 'evidence' for use in pipeline (see below) List> evidence = EvidenceHelper.setUpEvidence(); run(dataFile, evidence, System.out); @@ -131,7 +129,7 @@ public static void run(String dataFile, "An explicit path can be supplied via the {} environment variable. " + "For testing, you can obtain an enterprise data file by contacting us at " + "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=data-file-not-found", - dataFile, DataFileHelper.IPI_PATH_ENV_VAR); + dataFile, DataFileHelper.IPI_PATH_VARIABLE); throw e; } @@ -152,15 +150,20 @@ public static void run(String dataFile, .buildFromConfiguration(pipelineOptions)) { - // carry out some sample detections - for (Map evidence : evidenceList) { - analyzeEvidence(evidence, pipeline, outputStream); - } - /* Get the 'engine' element within the pipeline that performs IP Intelligence. We can use this to get details about the data file as well as meta-data describing things such as the available properties. */ IPIntelligenceOnPremiseEngine engine = pipeline.getElement(IPIntelligenceOnPremiseEngine.class); + + // carry out some sample detections + boolean needsEnterprise = false; + for (Map evidence : evidenceList) { + needsEnterprise |= analyzeEvidence(evidence, pipeline, engine, outputStream); + } + if (needsEnterprise) { + DataFileHelper.writeEnterpriseNote(new PrintWriter(outputStream), CONTACT_US); + } + DataFileHelper.logDataFileInfo(engine); logger.info("All done"); @@ -172,11 +175,15 @@ public static void run(String dataFile, * supplied and output the IP Intelligence results to the output stream. * @param evidence a map representing HTTP headers * @param pipeline a pipeline set up to process the evidence + * @param engine the engine, which says which properties its data file has * @param out somewhere to send the detection results + * @return true if a property needed the Enterprise data file */ - private static void analyzeEvidence(Map evidence, - Pipeline pipeline, - OutputStream out) throws Exception { + private static boolean analyzeEvidence(Map evidence, + Pipeline pipeline, + IPIntelligenceOnPremiseEngine engine, + OutputStream out) throws Exception { + boolean needsEnterprise = false; PrintWriter writer = new PrintWriter(out); /* FlowData is a data structure that is used to convey information required for a lookup and the results of the lookup through the pipeline. Information required for @@ -210,25 +217,45 @@ private static void analyzeEvidence(Map evidence, property dictionary at https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-gettingstartedonprem.java&utm_term=analyzeevidence for details of all available properties. */ - // Output all the properties using shared PropertyHelper methods - writer.println("\tRegisteredName: " + asStringProperty(tryGet(ipData::getRegisteredName))); - writer.println("\tRegisteredOwner: " + asStringProperty(tryGet(ipData::getRegisteredOwner))); - writer.println("\tRegisteredCountry: " + asStringProperty(tryGet(ipData::getRegisteredCountry))); - writer.println("\tIpRangeStart: " + asIPAddressProperty(tryGet(ipData::getIpRangeStart))); - writer.println("\tIpRangeEnd: " + asIPAddressProperty(tryGet(ipData::getIpRangeEnd))); - writer.println("\tCountry: " + asStringProperty(tryGet(ipData::getCountry))); - writer.println("\tCountryCode: " + asStringProperty(tryGet(ipData::getCountryCode))); - writer.println("\tCountryCode3: " + asStringProperty(tryGet(ipData::getCountryCode3))); - writer.println("\tRegion: " + asStringProperty(tryGet(ipData::getRegion))); - writer.println("\tState: " + asStringProperty(tryGet(ipData::getState))); - writer.println("\tTown: " + asStringProperty(tryGet(ipData::getTown))); - writer.println("\tLatitude: " + asFloatProperty(tryGet(ipData::getLatitude))); - writer.println("\tLongitude: " + asFloatProperty(tryGet(ipData::getLongitude))); - writer.println("\tAreas: " + asWktStringProperty(tryGet(ipData::getAreas))); - writer.println("\tAccuracyRadiusMin: " + asIntegerProperty(tryGet(ipData::getAccuracyRadiusMin))); - writer.println("\tTimeZoneOffset: " + asIntegerProperty(tryGet(ipData::getTimeZoneOffset))); + // The free Lite data file has the registration properties, and + // only the Enterprise data file has the others. + needsEnterprise |= writeValue(writer, engine, "RegisteredName", ipData::getRegisteredName); + needsEnterprise |= writeValue(writer, engine, "RegisteredOwner", ipData::getRegisteredOwner); + needsEnterprise |= writeValue(writer, engine, "RegisteredCountry", ipData::getRegisteredCountry); + needsEnterprise |= writeValue(writer, engine, "IpRangeStart", ipData::getIpRangeStart); + needsEnterprise |= writeValue(writer, engine, "IpRangeEnd", ipData::getIpRangeEnd); + needsEnterprise |= writeValue(writer, engine, "Country", ipData::getCountry); + needsEnterprise |= writeValue(writer, engine, "CountryCode", ipData::getCountryCode); + needsEnterprise |= writeValue(writer, engine, "CountryCode3", ipData::getCountryCode3); + needsEnterprise |= writeValue(writer, engine, "Region", ipData::getRegion); + needsEnterprise |= writeValue(writer, engine, "State", ipData::getState); + needsEnterprise |= writeValue(writer, engine, "Town", ipData::getTown); + needsEnterprise |= writeValue(writer, engine, "Latitude", ipData::getLatitude); + needsEnterprise |= writeValue(writer, engine, "Longitude", ipData::getLongitude); + needsEnterprise |= writeValue(writer, engine, "Areas", ipData::getAreas); + needsEnterprise |= writeValue(writer, engine, "AccuracyRadiusMin", ipData::getAccuracyRadiusMin); + needsEnterprise |= writeValue(writer, engine, "TimeZoneOffset", ipData::getTimeZoneOffset); } writer.println(); writer.flush(); + return needsEnterprise; + } + + /** + * Writes a property, or that it needs the Enterprise data file when the + * data file in use does not have it. The value is only read when the + * data file has the property. + * @return true if the property needs the Enterprise data file + */ + private static boolean writeValue(PrintWriter writer, + IPIntelligenceOnPremiseEngine engine, + String property, + Supplier> value) { + if (DataFileHelper.hasProperty(engine, property)) { + writer.println("\t" + property + ": " + asString(value.get())); + return false; + } + writer.println("\t" + property + ": " + DataFileHelper.NEEDS_ENTERPRISE); + return true; } } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java index 6e127722d..223121f1d 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetadataOnPrem.java @@ -42,7 +42,7 @@ import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; import static fiftyone.pipeline.util.FileFinder.getFilePath; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; /** * This example shows how to access metadata about the IP Intelligence properties that are available @@ -56,20 +56,28 @@ * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metadataonprem.java&utm_term=header). + * By default the example uses the free Lite IP Intelligence data file, which it downloads when + * it is missing. To obtain an Enterprise data file, which has many more properties, please + * [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metadataonprem.java&utm_term=header). */ public class MetadataOnPrem { private static final Logger logger = LoggerFactory.getLogger(GettingStartedOnPrem.class); - /* In this example, by default, the 51degrees IP Intelligence data file needs to be somewhere in the project - space, or you may specify another file as a command line parameter. + /* In this example, by default, the free 51degrees Lite IP Intelligence data file is used from + the data repository in cxx/data, and is downloaded if it is missing, or you may specify another + file as a command line parameter. - For testing, contact us to obtain an enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metadataonprem.java&utm_term=main */ + Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metadataonprem.java&utm_term=main */ + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metadataonprem.java&utm_term=enterprise-data-file"; public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH; run(dataFile, System.out); } @@ -83,14 +91,13 @@ public static void run(String dataFile, OutputStream output) throws Exception { logger.error("Failed to find IP Intelligence data file at '{}'. " + "Please provide a valid path to an IP Intelligence data file (.ipi). " + "An explicit path can be supplied via the {} environment variable.", - dataFile, DataFileHelper.IPI_PATH_ENV_VAR); + dataFile, DataFileHelper.IPI_PATH_VARIABLE); throw e; } - // Build a new on-premise IP Intelligence engine with the max performance profile. - // Note that there is no need to construct a complete pipeline in order to access - // the meta-data. - // If you already have a pipeline and just want to get a reference to the engine - // then you can use `var engine = pipeline.GetElement();` + // Build an on-premise IP Intelligence engine with the low memory + // profile. There is no need to build a complete pipeline to read the + // metadata. Code that already has a pipeline can get its engine with + // pipeline.getElement(IPIntelligenceOnPremiseEngine.class). try (IPIntelligenceOnPremiseEngine engine = new IPIntelligenceOnPremiseEngineBuilder(LoggerFactory.getILoggerFactory()) // We use the low memory profile to minimize memory usage for this @@ -119,6 +126,9 @@ public static void run(String dataFile, OutputStream output) throws Exception { writer.flush(); DataFileHelper.logDataFileInfo(engine); + if (DataFileHelper.isFreeDataFile(engine)) { + DataFileHelper.writeEnterpriseOffer(writer, CONTACT_US); + } } } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java index 347c35b60..4730245ec 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java @@ -32,7 +32,9 @@ * to complete. * * The sample of IP addresses used in the metrics can be adjusted as a - * parameter. + * parameter. The performance profile can be given as the fourth argument, + * for example MaxPerformance, which is much faster where there is memory to + * hold the whole data file. * * This example is primarily designed for those who are interested in * verifying the published metrics associated with 51Degrees' @@ -40,8 +42,9 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/MetricsOnPrem.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=contact-us). + * This example needs the location properties only the Enterprise IP Intelligence data file has. + * With the free Lite data file, which it uses by default, it says which and stops. To obtain an + * Enterprise data file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=contact-us). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -72,6 +75,7 @@ import java.io.PrintWriter; import java.net.InetAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -87,7 +91,7 @@ import java.util.function.Predicate; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.firstValue; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.tryGet; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -115,6 +119,24 @@ public class MetricsOnPrem { */ public static final String DEFAULT_OUTPUT_FILE = "metrics-output.csv"; + /** + * The performance profile used unless one is given as the fourth + * argument. This example needs the Enterprise data file, which is over + * 6 GB, and LowMemory reads it from disk as it is needed, so the example + * runs without memory to hold the whole file. MaxPerformance loads the + * whole file into memory and is much faster here, because the example + * reads the values of every profile in the file. + */ + public static final Constants.PerformanceProfiles + DEFAULT_PERFORMANCE_PROFILE = + Constants.PerformanceProfiles.LowMemory; + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=enterprise-data-file"; + /** * Properties needed by the metrics. The first eight form the metric key. */ @@ -289,19 +311,38 @@ public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); // Use the supplied path for the data file - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = DataFileHelper.getDataFileLocation( + args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH); // Get the location for the output file String outputFile = args.length > 1 ? args[1] : DEFAULT_OUTPUT_FILE; // Get the sample percentage or use the default double samplePercentage = args.length > 2 ? Double.parseDouble(args[2]) : DEFAULT_SAMPLE_PERCENTAGE; + // Get the performance profile named, ignoring case, such as + // MaxPerformance, or use the default + Constants.PerformanceProfiles profile = args.length > 3 ? + DataFileHelper.findPerformanceProfile( + args[3], new PrintWriter(System.out)) : + DEFAULT_PERFORMANCE_PROFILE; + if (profile == null) { + return; + } + + // The free data files have none of the location properties. + if (DataFileHelper.lacksEnterpriseProperties(dataFile, + "Measuring the location areas", + Arrays.asList(PROPERTIES), + new PrintWriter(System.out), + CONTACT_US)) { + return; + } try (PrintWriter writer = new PrintWriter(new FileWriter(outputFile))) { // Only include IP addresses with periods in them, i.e. IPv4. // There are too many IPv6 addresses for the metrics example to // complete in a short time frame. Set to null to include all. Predicate condition = range -> range[0].contains("."); - run(dataFile, samplePercentage, 0, condition, writer); + run(dataFile, samplePercentage, 0, condition, profile, writer); } logger.info("Output written to '{}'", outputFile); } @@ -321,6 +362,27 @@ public static void run(String dataFile, int maxRanges, Predicate condition, PrintWriter output) throws Exception { + run(dataFile, samplePercentage, maxRanges, condition, + DEFAULT_PERFORMANCE_PROFILE, output); + } + + /** + * Run the example + * @param dataFile an IP Intelligence data file + * @param samplePercentage percentage of possible IP addresses to include + * in the metrics where 1 is 100% + * @param maxRanges maximum number of IP ranges to process, or 0 for all + * @param condition optional function used to determine if an IP address + * range should be included, or null for all ranges + * @param profile the performance profile to build the engine with + * @param output somewhere for the metrics CSV + */ + public static void run(String dataFile, + double samplePercentage, + int maxRanges, + Predicate condition, + Constants.PerformanceProfiles profile, + PrintWriter output) throws Exception { logger.info("Running MetricsOnPrem example"); String dataFileLocation; @@ -335,16 +397,16 @@ public static void run(String dataFile, int processors = Runtime.getRuntime().availableProcessors(); // Build the pipeline with an on-premise IP Intelligence engine using - // the max performance profile. + // the performance profile given. IPIntelligencePipelineBuilder builder = new IPIntelligencePipelineBuilder(); fiftyone.ipintelligence.IPIntelligenceOnPremisePipelineBuilder onPremiseBuilder = builder.useOnPremise(dataFileLocation, false) - // We use the max performance profile for optimal - // detection speed in this example. See the + // The profile trades memory for speed, as + // DEFAULT_PERFORMANCE_PROFILE explains. See the // documentation for more detail on this and other // configuration options. // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-metricsonprem.java&utm_term=automatic-datafile-updates - .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) + .setPerformanceProfile(profile) // inhibit sharing usage for this example .setShareUsage(false) // inhibit auto-update of the data file for this example diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java index 608989b73..0357b64d8 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java @@ -44,8 +44,9 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/OfflineProcessing.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=header). + * By default the example uses the free Lite IP Intelligence data file, which it downloads when + * it is missing, and records Country, which only the Enterprise data file has, as needing it. To + * obtain an Enterprise data file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=header). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -74,8 +75,9 @@ import java.util.stream.Collectors; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asString; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asStringProperty; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; /** @@ -95,10 +97,10 @@ */ public class OfflineProcessing { static final Logger logger = LoggerFactory.getLogger(OfflineProcessing.class); - // This 51degrees IP Intelligence data file (distributed with the source) needs to - // be somewhere in the project space + // By default the free 51degrees Lite IP Intelligence data file is used from + // the data repository in cxx/data, and is downloaded if it is missing. // - // For testing, contact us to obtain an enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=datadir + // Contact us for a copy of the Enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=datadir private static final String dataDir = "ip-intelligence"; // This 51degrees file of 20,000 examples (distributed with the source) // needs to be somewhere in the project space. Additional evidence files @@ -106,13 +108,19 @@ public class OfflineProcessing { public static final String HEADER_EVIDENCE_YML = dataDir + "/evidence.yml"; + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=enterprise-data-file"; + public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); // Use provided data file argument, or fall back to default data file path - String dataFile = (args.length > 0) ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; - - File evidenceFile = getFilePath(HEADER_EVIDENCE_YML); + String dataFile = (args.length > 0) ? args[0] : LITE_DATA_FILE_REL_PATH; + + File evidenceFile = DataFileHelper.findDataFile(HEADER_EVIDENCE_YML); run(dataFile, Files.newInputStream(evidenceFile.toPath()), System.out); } @@ -135,7 +143,7 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws logger.error("Failed to find IP Intelligence data file at '{}'. " + "Please provide a valid path to an IP Intelligence data file (.ipi). " + "An explicit path can be supplied via the {} environment variable.", - dataFile, DataFileHelper.IPI_PATH_ENV_VAR); + dataFile, DataFileHelper.IPI_PATH_VARIABLE); throw e; } @@ -172,11 +180,14 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws // Performance options for IP Intelligence //.setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) //.setPerformanceProfile(Constants.PerformanceProfiles.HighPerformance) - // Low memory profile has the data streamed from disk on - // demand and is conservative in its use of memory, but - // slower because of disk access - .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) //.setPerformanceProfile(Constants.PerformanceProfiles.Balanced) + // We use the low memory profile, which reads the data file + // from disk as it is needed, because this example may be + // given the Enterprise data file, which is over 6 GB, through + // its first argument or _51DEGREES_IPI_PATH. MaxPerformance + // loads the whole data file into memory, which makes each + // lookup faster where there is memory to hold it. + .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) .build()) { // get the details of the IP Intelligence engine from the pipeline, @@ -192,6 +203,7 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws try (Writer writer = new OutputStreamWriter(os)) { // read a batch of IP evidence from the stream int count = 0; + boolean needsEnterprise = false; while (evidenceIterator.hasNext() && count < 20) { // Flow data is the container for inputs and outputs that // flow through the pipeline a flowdata instance is @@ -229,9 +241,16 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws resultMap.put("ip.RegisteredName", asStringProperty(ipData.getRegisteredName())); resultMap.put("ip.RegisteredOwner", asStringProperty(ipData.getRegisteredOwner())); resultMap.put("ip.RegisteredCountry", asStringProperty(ipData.getRegisteredCountry())); + // Only the Enterprise data file has Country. + if (DataFileHelper.hasProperty(engine, "Country")) { + resultMap.put("ip.Country", asString(ipData.getCountry())); + } else { + resultMap.put("ip.Country", DataFileHelper.NEEDS_ENTERPRISE); + needsEnterprise = true; + } // To output all populated IP Intelligence properties instead of - // the fixed subset above, replace the three puts with the helper: + // the fixed subset above, replace the puts with the helper: // resultMap.putAll(getPopulatedProperties(ipData, "ip.")); // write document to output stream as a YAML document @@ -242,18 +261,16 @@ public static void run(String dataFile, InputStream is, OutputStream os) throws } count++; } + if (needsEnterprise) { + // A comment keeps the output a valid YAML stream. + writer.write("# Some of these properties need the " + + "Enterprise data file. Contact us for a copy at " + + CONTACT_US + "\n"); + } // finish the last YAML document writer.write("...\n"); writer.flush(); logger.info("Finished processing {} records", count); - - if (engine.getDataSourceTier().equals("Lite")) { - logger.warn("You have used a Lite data file which has " + - "limited properties and is of limited accuracy"); - logger.info("The example requires an Enterprise data file " + - "to work fully. Find out about the Enterprise " + - "data file here: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-offlineprocessing.java&utm_term=enterprise-data-file"); - } } } } diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java index 2be3b4599..d3c03c4e2 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java @@ -34,8 +34,11 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmark.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-performancebenchmark.java&utm_term=header). + * By default the example uses the free Asn IP Intelligence data file, which it downloads when it + * is missing, and is far smaller than the Lite file. It asks only for the property it reads, + * being RegisteredName, or AsnName with the free Asn data file, which has no RegisteredName, + * so it runs with any of the data files. To obtain an Enterprise data file, please + * [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-performancebenchmark.java&utm_term=header). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -49,11 +52,13 @@ import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.examples.shared.EvidenceHelper; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; import fiftyone.ipintelligence.shared.IPIntelligenceData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.IWeightedValue; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.util.FileFinder; import org.apache.commons.lang3.BooleanUtils; import org.slf4j.Logger; @@ -83,11 +88,25 @@ public class PerformanceBenchmark { public static final Logger logger = LoggerFactory.getLogger(PerformanceBenchmark.class); + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-performancebenchmark.java&utm_term=enterprise-data-file"; + + // true when the benchmarks ran on a free data file + private boolean usedFreeDataFile; + // true when the data file has RegisteredName, which the free Asn data + // file does not, having AsnName in its place + private boolean readRegisteredName; + // where the results of the tests are gathered private List> resultList; private int numberOfThreads = DEFAULT_NUMBER_OF_THREADS; private List> evidence; private String dataFileLocation; + // the properties asked for when not asking for all of them + private List properties; private PrintWriter writer; // a default set of configurations: (profile, allProperties) @@ -100,7 +119,8 @@ public class PerformanceBenchmark { public static void main(String[] args) throws Exception { LogbackHelper.configureLogback(FileFinder.getFilePath("logback.xml")); - String dataFilename = args.length > 0 ? args[0] : null; + String dataFilename = args.length > 0 ? args[0] : + DataFileHelper.ASN_DATA_FILE_REL_PATH; String evidenceFilename = args.length > 1 ? args[1] : null; int numberOfThreads = DEFAULT_NUMBER_OF_THREADS; if (args.length > 2) { @@ -131,6 +151,7 @@ protected void runBenchmarks(PerformanceConfiguration[] performanceConfiguration logger.info("Running Performance example"); this.dataFileLocation = getDataFileLocation(dataFilename); + this.properties = benchmarkProperties(dataFileLocation); File evidenceFile = getEvidenceFile(evidenceFilename); this.evidence = Collections.unmodifiableList( @@ -147,6 +168,9 @@ protected void runBenchmarks(PerformanceConfiguration[] performanceConfiguration } executeBenchmark(config); } + if (usedFreeDataFile) { + DataFileHelper.writeEnterpriseOffer(writer, CONTACT_US); + } logger.info("Finished Performance example"); } @@ -173,7 +197,11 @@ private void executeBenchmark(PerformanceConfiguration config) throws Exception setPipelinePerformanceProperties(builder, config); pipeline = builder.build(); - DataFileHelper.logDataFileInfo(pipeline.getElement(IPIntelligenceOnPremiseEngine.class)); + IPIntelligenceOnPremiseEngine engine = + pipeline.getElement(IPIntelligenceOnPremiseEngine.class); + DataFileHelper.logDataFileInfo(engine); + usedFreeDataFile = DataFileHelper.isFreeDataFile(engine); + readRegisteredName = DataFileHelper.hasProperty(engine, "RegisteredName"); // run the benchmarks twice, once to warm up the JVM logger.info("Warming up"); @@ -214,9 +242,37 @@ private void setPipelinePerformanceProperties( // Requesting properties from a single component // reduces processing time compared with requesting properties from multiple components. // If you don't specify any properties, then all properties are returned, - // here we choose "all properties" by specifying none, or just "RegisteredName" + // here we choose "all properties" by specifying none, or just the + // one property the benchmark reads, which benchmarkProperties + // chooses for the data file. if (BooleanUtils.isFalse(config.allProperties)) { - builder.setProperty("RegisteredName"); + for (String property : properties) { + builder.setProperty(property); + } + } + } + + /** + * The properties the benchmark asks for when it does not ask for all of + * them, being only the one it reads. That is RegisteredName where the + * data file has it, as the Lite and Enterprise data files do, and + * AsnName for the free Asn data file, which has no RegisteredName. The + * data file is opened with the low memory profile, which reads little + * of it. + * @param dataFileLocation the data file the benchmark uses + * @return the one property to ask for + * @throws Exception if the data file cannot be opened + */ + static List benchmarkProperties(String dataFileLocation) + throws Exception { + try (IPIntelligenceOnPremiseEngine engine = + new IPIntelligenceOnPremiseEngineBuilder() + .setPerformanceProfile(LowMemory) + .setAutoUpdate(false) + .build(dataFileLocation, false)) { + return Collections.singletonList( + DataFileHelper.hasProperty(engine, "RegisteredName") ? + "RegisteredName" : "AsnName"); } } @@ -260,7 +316,7 @@ private long runTests(Pipeline pipeline) throws Exception { // create a list of callables List> callables = new ArrayList<>(); for (int i = 0; i < numberOfThreads; i++) { - callables.add(new BenchmarkRunnable(pipeline, evidence)); + callables.add(new BenchmarkRunnable(pipeline, evidence, readRegisteredName)); } // start multiple threads in a fixed pool ExecutorService service = Executors.newFixedThreadPool(numberOfThreads); @@ -286,11 +342,15 @@ private static class BenchmarkRunnable implements Callable { private final BenchmarkResult result; private final List> testList; private final Pipeline pipeline; + private final boolean readRegisteredName; - BenchmarkRunnable(Pipeline pipeline, List> evidence) { + BenchmarkRunnable(Pipeline pipeline, + List> evidence, + boolean readRegisteredName) { this.testList = evidence; // initialise the benchmark variables this.pipeline = pipeline; + this.readRegisteredName = readRegisteredName; this.result = new BenchmarkResult(); result.elapsedMillis = 0; @@ -318,8 +378,10 @@ public BenchmarkResult call() { // the same data. IPIntelligenceData ipData = flowData.get(IPIntelligenceData.class); if (ipData != null) { - if (ipData.getRegisteredName().hasValue()) { - String value = ipData.getRegisteredName().getValue(); + AspectPropertyValue name = readRegisteredName ? + ipData.getRegisteredName() : ipData.getAsnName(); + if (name.hasValue()) { + String value = name.getValue(); if (value != null) { result.checkSum += value.hashCode(); // for (IWeightedValue weightedValue : value) { diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java index 3df68b732..8f0e5e8f2 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java @@ -35,8 +35,9 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/SuspiciousOnPrem.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=contact-us). + * This example needs properties only the Enterprise IP Intelligence data file has. With the free + * Lite data file, which it uses by default, it says which and stops. To obtain an Enterprise data + * file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=contact-us). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -57,11 +58,12 @@ import java.io.OutputStream; import java.io.PrintWriter; +import java.util.Arrays; import java.util.List; import java.util.Map; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.firstValue; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.tryGet; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -80,16 +82,37 @@ public class SuspiciousOnPrem { private static final Logger logger = LoggerFactory.getLogger(SuspiciousOnPrem.class); - /* In this example, by default, the 51degrees IP Intelligence data file needs to be somewhere - in the project space, or you may specify another file as a command line parameter. + /* In this example, by default, the free 51degrees Lite IP Intelligence data file is used from + the data repository in cxx/data, and is downloaded if it is missing, or you may specify another + file as a command line parameter. The Lite data file has none of the properties the example + needs, so it says so and stops. For testing, contact us to obtain an enterprise data file: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=contact-us */ + /** + * The properties the judgement uses, which only the Enterprise data file + * has. + */ + public static final List ENTERPRISE_PROPERTIES = Arrays.asList( + "IsCellular", + "HardwareDiversity", + "BrowserDiversity", + "LocationConfidence", + "IsHosted", + "CountryCode", + "HumanProbability"); + + /** + * Where to ask for a copy of the Enterprise data file, tagged with this + * example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=enterprise-data-file"; + public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); // Use the supplied path for the data file - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH; // prepare 'evidence' for use in pipeline (see below) List> evidence = EvidenceHelper.setUpEvidence(); run(dataFile, evidence, System.out); @@ -117,6 +140,15 @@ public static void run(String dataFile, throw e; } + // The judgement needs properties only the Enterprise data file has. + if (DataFileHelper.lacksEnterpriseProperties(dataFileLocation, + "Judging whether an IP address is suspicious", + ENTERPRISE_PROPERTIES, + new PrintWriter(outputStream), + CONTACT_US)) { + return; + } + /* In this example, we use the IPIntelligencePipelineBuilder and configure it in code. For more information about builders in general see the documentation at @@ -125,12 +157,15 @@ public static void run(String dataFile, Note that we wrap the creation of a pipeline in a try/resources to control its lifecycle */ try (Pipeline pipeline = new IPIntelligencePipelineBuilder() .useOnPremise(dataFileLocation, false) - // We use the max performance profile for optimal lookup speed in this - // example. See the documentation for more detail on this and other - // configuration options. + // We use the low memory profile, because this example needs + // properties only the Enterprise data file has, and that file + // is over 6 GB. MaxPerformance loads the whole data file into + // memory, which makes each lookup faster where there is memory + // to hold it. See the documentation for more detail on this and + // other configuration options. // https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=automatic-datafile-updates // https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-suspiciousonprem.java&utm_term=usage-sharing - .setPerformanceProfile(Constants.PerformanceProfiles.MaxPerformance) + .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) // inhibit sharing usage for this example, usually this should be set to "true" .setShareUsage(false) // inhibit auto-update of the data file for this example diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java index 144931e2c..a57a9b3e0 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java @@ -26,6 +26,7 @@ import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.services.DataUpdateService; import fiftyone.pipeline.engines.services.DataUpdateServiceDefault; import fiftyone.pipeline.engines.services.OnUpdateComplete; @@ -34,6 +35,7 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.PrintWriter; import java.nio.file.*; import java.text.SimpleDateFormat; import java.util.Objects; @@ -65,12 +67,11 @@ *

  • Filesystem Watcher
    • *
  • Daily auto-update
    • * - * ## Distributor Service and License Key - * For production use, you will eventually need to use a Distributor service and license key - * to keep your data file updated. However, for now this example requires a custom URL where - * an updated data file will be hosted. - * - * To obtain access to enterprise data files, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=header). + * ## License Key + * Updating the data file downloads the Enterprise data file from the Distributor service, which + * needs a 51Degrees licence key. Without one, the example shows the free Asn data file and says + * where to ask for a key. To obtain a key and an Enterprise data file, please + * [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=header). * * ## Configuration * - the pipeline must be configured to use a temp file @@ -153,15 +154,8 @@ * # Location * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/UpdateDataFile.java). * - * To test this example, you need to: - * 1. Host an IP Intelligence data file (.ipi) at a custom URL accessible to this application - * 2. Provide that custom URL using the WithDataUpdateUrl parameter - * 3. No license key is required when using a custom URL - * - * For production use, you will eventually need to use a Distributor service and license key - * to keep your data file updated. - * - * To obtain access to enterprise data files for hosting, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=header-2). + * To test this example you need a licence key for the Enterprise data file. To obtain one, + * please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=header-2). * */ @@ -177,11 +171,8 @@ *
  • Daily auto-update
    • * *

    - * For production use, you will eventually need to use a Distributor service and license key - * to keep your data file updated. However, for now this example requires a custom URL where - * an updated data file will be hosted. - * - * To obtain access to enterprise data files, please contact us. + * Updating the data file needs a licence key for the Enterprise data file. To obtain one, please + * contact us. * This license key must be supplied as a command line argument or by setting * an environment variable or system property called {@link UpdateDataFile#UPDATE_EXAMPLE_LICENSE_KEY_NAME} */ @@ -193,6 +184,12 @@ public class UpdateDataFile { private static final Logger logger = LoggerFactory.getLogger(UpdateDataFile.class); + /** + * Where to ask for a licence key and a copy of the Enterprise data file, + * tagged with this example. + */ + private static final String CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=licence-key"; + static final UpdateCompletionListener completionListener = new UpdateCompletionListener(); static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss Z"); @@ -222,13 +219,8 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr licenseKey = KeyUtils.getNamedKey(UPDATE_EXAMPLE_LICENSE_KEY_NAME); } if (Objects.isNull(licenseKey) || KeyUtils.isInvalidKey(licenseKey)) { - logger.error("For production use, you will eventually need to use a Distributor service and " + - "license key to keep your data file updated. However, for now this example requires " + - "a custom URL where an updated data file will be hosted. To obtain access to enterprise " + - "data files, please contact us at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=license-key-required. You must supply the " + - "license key as an argument to this program, or as an environment or system variable " + - "named '{}'", UPDATE_EXAMPLE_LICENSE_KEY_NAME); - throw new IllegalArgumentException("No license key available"); + explainLicenseKey(dataFilename, new PrintWriter(System.out, true)); + return; } // work out where the downloaded file will be put, directory must exist @@ -258,9 +250,9 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr DataFileHelper.DatafileInfo metadata = getDatafileMetaData(dataFilename); // and output the results logFileInfo(metadata); - if (metadata.getTier().equals("Lite")) { + if (metadata.isFree()) { logger.error("Will not download an 'Enterprise' data file over the top of " + - "a 'Lite' data file, please supply another location."); + "a free data file, please supply another location."); throw new IllegalArgumentException("File supplied has wrong data tier"); } logger.info("Existing data file will be replaced with downloaded data file"); @@ -302,13 +294,15 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr // overwritten the pipeline must be configured to copy the supplied // file to a temporary file (createTempDataCopy parameter == true. .useOnPremise(dataFilename, true) + // Use the low memory profile, because this example + // downloads the Enterprise data file, which is over 6 GB, + // and MaxPerformance would load all of it into memory. + .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) // pass in the update listener which has been configured // to notify when update complete .setDataUpdateService(dataUpdateService) - // For production use, you will eventually need to use a Distributor service and - // license key to keep your data file updated. However, for now this example - // requires a custom URL where an updated data file will be hosted. - // To obtain access to enterprise data files, please contact us at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=run + // Updates from the Distributor service need a licence key for the + // Enterprise data file. To obtain one, please contact us at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=console-src-main-java-fiftyone-ipintelligence-examples-console-updatedatafile.java&utm_term=run .setDataUpdateLicenseKey(licenseKey) // Enable update on startup, the auto update system // will be used to check for an update before the @@ -363,6 +357,40 @@ static void run(String dataFilename, String licenseKey, boolean interactive) thr } } + /** + * Without a licence key the example cannot download a data file, so it + * shows the data file it would start from, the free Asn file unless + * another is given, and says what an update needs. + * @param dataFilename the data file given, or null + * @param output where to write + * @throws Exception if the data file cannot be read + */ + static void explainLicenseKey(String dataFilename, PrintWriter output) throws Exception { + File dataFile = null; + try { + dataFile = Objects.isNull(dataFilename) ? + DataFileHelper.findDataFile(DataFileHelper.ASN_DATA_FILE_REL_PATH) : + new File(dataFilename); + } catch (IllegalArgumentException e) { + logger.warn(e.getMessage()); + } + if (Objects.nonNull(dataFile) && dataFile.exists()) { + DataFileHelper.DatafileInfo metadata = + getDatafileMetaData(dataFile.getAbsolutePath()); + output.println("The data file in use is the '" + metadata.getTier() + + "' file published on " + simpleDateFormat.format( + metadata.getFileInfo().getDataPublishedDateTime()) + + ", at '" + metadata.getFileInfo().getDataFilePath() + "'."); + } + output.println(); + output.println("Updating the data file downloads the Enterprise data " + + "file, which needs a licence key. Give the key as the first " + + "command line argument, or in the environment variable or " + + "system property '" + UPDATE_EXAMPLE_LICENSE_KEY_NAME + "'. " + + "Contact us for a key and a copy at " + CONTACT_US); + output.flush(); + } + private static void logFileInfo(DataFileHelper.DatafileInfo metadata) { logger.info("Current data file is a {} tier data file, last updated at {}, at " + "location {}. Next update expected {}.", metadata.getTier(), diff --git a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/areas/Calculations.java b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/areas/Calculations.java index b3499f480..7f1cc8edb 100644 --- a/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/areas/Calculations.java +++ b/examples/ip-intelligence/console/src/main/java/fiftyone/ipintelligence/examples/console/areas/Calculations.java @@ -259,7 +259,7 @@ private static Geometry transformGeometry( holes); } - private static LinearRing transformRing( + static LinearRing transformRing( LinearRing ring, CoordinateTransform transform) { Coordinate[] source = ring.getCoordinates(); @@ -276,6 +276,14 @@ private static LinearRing transformRing( transformed.y); } + // The first and last points are the same input, but compiled code can + // project them one bit apart, which JTS rejects as an open ring. The + // last point is therefore copied from the first. + if (coordinates.length > 1 && ring.isClosed()) { + coordinates[coordinates.length - 1] = + new Coordinate(coordinates[0]); + } + return ring.getFactory().createLinearRing(coordinates); } diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java new file mode 100644 index 000000000..d14a522f6 --- /dev/null +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/ExamplesWithoutSettingsTest.java @@ -0,0 +1,174 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.console; + +import fiftyone.ipintelligence.examples.shared.DataFileHelper; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * Runs each on-premise console example's main with no arguments, no data + * file setting and no licence key, as someone who has just cloned the + * repository runs it, and checks that it says what the Enterprise data file + * adds, with the link to the contact page. The examples default to the free + * Lite data file, or the free Asn file for performance and updates. CI gives + * the other tests the Enterprise data file, so nothing else tests what a + * fresh clone shows. + */ +public class ExamplesWithoutSettingsTest { + + private static final String CONTACT_US = "https://51degrees.com/contact-us?"; + + private static final String LICENSE_KEY = + UpdateDataFile.UPDATE_EXAMPLE_LICENSE_KEY_NAME; + + /** + * An example's main method. + */ + private interface Main { + void main(String[] args) throws Exception; + } + + /** + * Runs the example's main with no arguments and the data file and + * licence key system properties removed, and returns what it wrote to + * standard output. The deprecated name of the data file variable is + * read when the current one is not set, so both are checked and + * removed. + */ + @SuppressWarnings("deprecation") + private static String runWithoutSettings(Main main) throws Exception { + assumeTrue("Skipping, a data file or licence key is set in the environment", + System.getenv(DataFileHelper.IPI_PATH_VARIABLE) == null && + System.getenv(DataFileHelper.IPI_PATH_ENV_VAR) == null && + System.getenv(LICENSE_KEY) == null && + System.getenv(LICENSE_KEY.toUpperCase()) == null); + String dataFile = System.clearProperty(DataFileHelper.IPI_PATH_VARIABLE); + String oldNameDataFile = + System.clearProperty(DataFileHelper.IPI_PATH_ENV_VAR); + String licenseKey = System.clearProperty(LICENSE_KEY); + PrintStream out = System.out; + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + System.setOut(new PrintStream(buffer, true, "UTF-8")); + try { + main.main(new String[0]); + } finally { + System.setOut(out); + if (dataFile != null) { + System.setProperty(DataFileHelper.IPI_PATH_VARIABLE, dataFile); + } + if (oldNameDataFile != null) { + System.setProperty(DataFileHelper.IPI_PATH_ENV_VAR, + oldNameDataFile); + } + if (licenseKey != null) { + System.setProperty(LICENSE_KEY, licenseKey); + } + } + String output = buffer.toString("UTF-8"); + out.println(output); + return output; + } + + private static void assertContains(String output, String expected) { + assertTrue("The output should contain '" + expected + "'", + output.contains(expected)); + } + + private static void assertStops(String output, String purpose) { + assertContains(output, purpose + " needs the "); + assertContains(output, "so the example stops here."); + assertContains(output, CONTACT_US); + } + + private static void assertOffer(String output) { + assertContains(output, "This example used a free data file."); + assertContains(output, CONTACT_US); + } + + @Test + public void gettingStarted() throws Exception { + String output = runWithoutSettings(GettingStartedOnPrem::main); + assertContains(output, "RegisteredName: "); + assertContains(output, "Country: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, "Some of these properties need the Enterprise data file."); + assertContains(output, CONTACT_US); + } + + @Test + public void suspicious() throws Exception { + assertStops(runWithoutSettings(SuspiciousOnPrem::main), + "Judging whether an IP address is suspicious"); + } + + @Test + public void compare() throws Exception { + assertStops(runWithoutSettings(CompareOnPrem::main), + "Comparing known locations with IP Intelligence"); + } + + @Test + public void metrics() throws Exception { + assertStops(runWithoutSettings(MetricsOnPrem::main), + "Measuring the location areas"); + } + + @Test + public void offlineProcessing() throws Exception { + String output = runWithoutSettings(OfflineProcessing::main); + assertContains(output, "ip.Country: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, "# Some of these properties need the Enterprise " + + "data file. Contact us for a copy at " + CONTACT_US); + } + + @Test + public void metadata() throws Exception { + assertOffer(runWithoutSettings(MetadataOnPrem::main)); + } + + @Test + public void performance() throws Exception { + assertOffer(runWithoutSettings(PerformanceBenchmark::main)); + } + + @Test + public void updateDataFile() throws Exception { + String output = runWithoutSettings(UpdateDataFile::main); + assertContains(output, "The data file in use is the "); + assertContains(output, "which needs a licence key"); + assertContains(output, CONTACT_US); + } + + @Test + public void mixed() throws Exception { + String output = runWithoutSettings(GettingStartedMixed::main); + assertContains(output, "Hardware Vendor: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, "Country: " + DataFileHelper.NEEDS_ENTERPRISE); + assertContains(output, CONTACT_US); + } +} diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloudTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloudTest.java index be4be97b1..e024e7506 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloudTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GetAllPropertiesCloudTest.java @@ -26,15 +26,12 @@ import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class GetAllPropertiesCloudTest { @Test public void getAllPropertiesCloudTest() throws Exception { - String resourceKey = KeyUtils.getNamedKey(KeyHelper.TEST_RESOURCE_KEY); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "every IP Intelligence property the key carries", "ip"); GetAllPropertiesCloud.run(resourceKey, KeyUtils.getNamedKey(GettingStartedCloud.CLOUD_END_POINT_NAME), System.out); diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixedTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixedTest.java index fc0aab3ac..9e43a0078 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixedTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudMixedTest.java @@ -26,15 +26,14 @@ import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class GettingStartedCloudMixedTest { @Test public void gettingStartedCloudMixedTest() throws Exception { - String resourceKey = KeyUtils.getNamedKey(KeyHelper.TEST_RESOURCE_KEY); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "the network and location of an IP address, and whether a " + + "device is mobile", + "ip", "device"); GettingStartedCloudMixed.run(resourceKey, KeyUtils.getNamedKey(GettingStartedCloud.CLOUD_END_POINT_NAME), GettingStartedMixed.setUpCombinedEvidence(), System.out); diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudTest.java index 37dddd104..5030e2f2f 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/GettingStartedCloudTest.java @@ -27,15 +27,12 @@ import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class GettingStartedCloudTest { @Test public void gettingStartedCloudTest() throws Exception { - String resourceKey = KeyUtils.getNamedKey(KeyHelper.TEST_RESOURCE_KEY); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "the network, location and time zone of an IP address", "ip"); GettingStartedCloud.run(resourceKey, KeyUtils.getNamedKey(GettingStartedCloud.CLOUD_END_POINT_NAME), EvidenceHelper.setUpEvidence(), System.out); diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/MetadataCloudTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/MetadataCloudTest.java index c1ac46032..a531deb55 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/MetadataCloudTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/MetadataCloudTest.java @@ -26,15 +26,12 @@ import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import org.junit.Test; -import static org.junit.Assume.assumeFalse; - public class MetadataCloudTest { @Test public void metadataCloudTest() throws Exception { - String resourceKey = KeyUtils.getNamedKey(KeyHelper.TEST_RESOURCE_KEY); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "the IP Intelligence properties it offers", "ip"); MetadataCloud.run(resourceKey, KeyUtils.getNamedKey(GettingStartedCloud.CLOUD_END_POINT_NAME), System.out); diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmarkPropertiesTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmarkPropertiesTest.java new file mode 100644 index 000000000..5490a8c0d --- /dev/null +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/PerformanceBenchmarkPropertiesTest.java @@ -0,0 +1,61 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.console; + +import fiftyone.ipintelligence.examples.shared.DataFileHelper; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; + +/** + * Checks that the performance benchmark asks only for the property it reads, + * being RegisteredName where the data file has it and AsnName for the free + * Asn data file, which has no RegisteredName. + */ +public class PerformanceBenchmarkPropertiesTest { + + @Test + public void registeredNameAloneWhenTheDataFileHasIt() throws Exception { + String dataFile = DataFileHelper.findAvailableDataFile( + DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH, + DataFileHelper.LITE_DATA_FILE_REL_PATH); + assumeTrue("Skipping test, no IP Intelligence data file with the " + + "RegisteredName property found", + dataFile != null); + assertEquals(Collections.singletonList("RegisteredName"), + PerformanceBenchmark.benchmarkProperties(dataFile)); + } + + @Test + public void asnNameAloneWithTheAsnDataFile() throws Exception { + String dataFile = DataFileHelper.findAvailableDataFile( + DataFileHelper.ASN_DATA_FILE_REL_PATH); + assumeTrue("Skipping test, no free Asn IP Intelligence data file found", + dataFile != null); + assertEquals(Collections.singletonList("AsnName"), + PerformanceBenchmark.benchmarkProperties(dataFile)); + } +} diff --git a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/areas/CalculationsTest.java b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/areas/CalculationsTest.java index 8ebeda2d6..6a0b1e307 100644 --- a/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/areas/CalculationsTest.java +++ b/examples/ip-intelligence/console/src/test/java/fiftyone/ipintelligence/examples/console/areas/CalculationsTest.java @@ -27,6 +27,12 @@ import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.LinearRing; import org.locationtech.jts.geom.Polygon; +import org.locationtech.proj4j.CoordinateReferenceSystem; +import org.locationtech.proj4j.CoordinateTransform; +import org.locationtech.proj4j.ProjCoordinate; + +import java.util.HashSet; +import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -168,6 +174,43 @@ public void testContains() throws Exception { false, result2.getContains()); } + /** + * A projection that returns a result one bit different the second time it + * is given the same point, as compiled code can for Math.sin. + */ + private static class DriftingTransform implements CoordinateTransform { + private final Set seen = new HashSet<>(); + + @Override + public CoordinateReferenceSystem getSourceCRS() { + return null; + } + + @Override + public CoordinateReferenceSystem getTargetCRS() { + return null; + } + + @Override + public ProjCoordinate transform(ProjCoordinate src, ProjCoordinate tgt) { + tgt.x = src.x; + tgt.y = src.y; + if (!seen.add(src.x + "," + src.y)) { + tgt.x = Math.nextUp(tgt.x); + } + return tgt; + } + } + + @Test + public void testProjectedRingIsClosedWhenEndsDifferByOneBit() { + LinearRing source = createRectangle(0, 0, 1).getExteriorRing(); + LinearRing projected = Calculations.transformRing( + source, new DriftingTransform()); + assertTrue(projected.isClosed()); + assertEquals(1.0, factory.createPolygon(projected).getArea(), 1e-9); + } + private static Polygon createRectangle(double x, double y, double d) { return factory.createPolygon(new Coordinate[]{ new Coordinate(x, y), diff --git a/examples/ip-intelligence/pom.xml b/examples/ip-intelligence/pom.xml index 2e3624a91..8d7ff14cf 100644 --- a/examples/ip-intelligence/pom.xml +++ b/examples/ip-intelligence/pom.xml @@ -50,8 +50,8 @@ 4.13.2 true 1.30 - 4.5.128 - 4.5.128 + 4.5.130 + 4.5.130 2.22.2 https://oss.sonatype.org diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java index 4cfa22723..e617da672 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/DataFileHelper.java @@ -25,7 +25,9 @@ import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; import fiftyone.ipintelligence.shared.testhelpers.FileUtils; +import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.data.AspectEngineDataFile; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFile; import fiftyone.pipeline.util.FileFinder; import org.slf4j.Logger; @@ -33,6 +35,7 @@ import java.io.File; import java.io.IOException; +import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -40,11 +43,11 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -53,10 +56,24 @@ public class DataFileHelper { /** * Sent with each download, so the Distributor can count the downloads - * made by these examples. + * made by these examples. It is left out inside GitHub Actions, where + * the data repository's script names the build instead, so that test + * runs are not counted as people using the examples. */ private static final String DOWNLOAD_SOURCE = "examples-51degrees-java"; + private static final boolean IN_GITHUB_ACTIONS = + "true".equals(System.getenv("GITHUB_ACTIONS")); + + private static final String WINDOWS_POWERSHELL = "powershell"; + + /** + * Shown in place of a value for a property that the data file in use + * does not have. Every property the examples ask for is in the + * Enterprise data file. + */ + public static final String NEEDS_ENTERPRISE = "needs the Enterprise data file"; + /** * Folder and name of the enterprise IP Intelligence data file, which * belongs in cxx/data/ip-intelligence, a folder of the data repository. @@ -117,15 +134,43 @@ public static String findAvailableDataFile(String... candidates) { * hold an explicit path to the IP Intelligence data file. This name is * checked first, before any search of the project space. */ + public static final String IPI_PATH_VARIABLE = "_51DEGREES_IPI_PATH"; + + /** + * The old name of {@link #IPI_PATH_VARIABLE}, read only when that one + * is not set. + * @deprecated use {@link #IPI_PATH_VARIABLE}, and set the variable it + * names in place of this one. + */ + @Deprecated public static final String IPI_PATH_ENV_VAR = "51DEGREES_IPI_PATH"; + // Set once the use of the deprecated name has been logged. + private static final AtomicBoolean OLD_NAME_LOGGED = + new AtomicBoolean(false); + + /** + * The value of the environment variable of this name, or of the system + * property where that is not set or is empty, or null when neither + * holds a value. + */ + private static String environmentOrProperty(String name) { + String value = System.getenv(name); + if (Objects.isNull(value) || value.isEmpty()) { + value = System.getProperty(name); + } + return Objects.isNull(value) || value.isEmpty() ? null : value; + } + public static class DatafileInfo { FiftyOneDataFile fileInfo; String tier; + boolean free; - public DatafileInfo(FiftyOneDataFile fileInfo, String dataSourceTier) { + public DatafileInfo(FiftyOneDataFile fileInfo, String dataSourceTier, boolean free) { this.fileInfo = fileInfo; this.tier = dataSourceTier; + this.free = free; } public AspectEngineDataFile getFileInfo(){ @@ -134,15 +179,40 @@ public AspectEngineDataFile getFileInfo(){ public String getTier() { return tier; } + + /** + * @return true for the free Lite and Asn data files + */ + public boolean isFree() { + return free; + } } + + /** + * Reads the details of a data file, such as its tier and publish date. + * The data file is opened with the low memory profile, which reads + * little of it. + * @param dataFileLocation the data file to read + * @return the data file's details + * @throws Exception if the data file cannot be opened + */ public static DatafileInfo getDatafileMetaData(String dataFileLocation) throws Exception { try(IPIntelligenceOnPremiseEngine ddhe = new IPIntelligenceOnPremiseEngineBuilder() + .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) .setAutoUpdate(false) .build(dataFileLocation,false)) { - return new DatafileInfo((FiftyOneDataFile) ddhe.getDataFileMetaData(), ddhe.getDataSourceTier()); + return new DatafileInfo((FiftyOneDataFile) ddhe.getDataFileMetaData(), + ddhe.getDataSourceTier(), isFreeDataFile(ddhe)); } } + /** + * Logs the data file in use, and a warning if it is old. An example run + * on a free data file says what the Enterprise data file adds in its + * own output, with {@link #writeEnterpriseNote}, + * {@link #writeEnterpriseOffer} or {@link #lacksEnterpriseProperties}. + * @param engine the engine whose data file is logged + */ public static void logDataFileInfo(IPIntelligenceOnPremiseEngine engine) { // Lite or Enterprise String dataTier = engine.getDataSourceTier(); @@ -154,13 +224,6 @@ public static void logDataFileInfo(IPIntelligenceOnPremiseEngine engine) { String displayDate = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").format(fileDate); logger.info("Used a '{}' data file, created {}, {} days ago, from location '{}'", dataTier, displayDate, daysOld, dataFileLocation); - if (dataTier.equals("Lite")) { - logger.warn("This example is using the 'Lite ' data file. " + - "This is used for illustration, and has limited " + - "accuracy and capabilities. Find out about the " + - "Enterprise data file on our pricing page: " + - "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-datafilehelper.java&utm_term=lite-data-file"); - } if (daysOld > 28) { logger.warn("This example is using a data file that is more " + "than {} days old. A more recent data file " + @@ -168,12 +231,165 @@ public static void logDataFileInfo(IPIntelligenceOnPremiseEngine engine) { } } + /** + * True when the engine's data file has the property. The free Lite data + * file has only RegisteredCountry, RegisteredName and RegisteredOwner, + * and the free Asn data file the Asn properties, such as AsnName. + * @param engine the engine to check + * @param name the name of the property + * @return true if the data file has the property + */ + public static boolean hasProperty(IPIntelligenceOnPremiseEngine engine, String name) { + for (FiftyOneAspectPropertyMetaData property : engine.getProperties()) { + if (property.getName().equalsIgnoreCase(name)) { + return true; + } + } + return false; + } + + /** + * True when the engine uses one of the free data files, Lite or Asn. + * Only the Enterprise data file has the range properties, so a check + * for IpRangeStart tells them apart. + * @param engine the engine to check + * @return true for a free data file + */ + public static boolean isFreeDataFile(IPIntelligenceOnPremiseEngine engine) { + return hasProperty(engine, "IpRangeStart") == false; + } + + /** + * For an example that needs properties only the Enterprise data file + * has, reports when the data file has none of them, and then says so + * and where to ask for a copy of the Enterprise data file. The data + * file is opened with the low memory profile, which reads little of it. + * @param dataFile the data file the example would use + * @param purpose what the example does, as the start of a sentence, for + * example "Comparing known locations with IP Intelligence" + * @param properties the properties the example needs + * @param output where to write + * @param contactUs the link to the contact page, tagged with the + * example's file so an enquiry can be traced to it + * @return true if the data file has none of the properties + * @throws Exception if the data file cannot be opened + */ + public static boolean lacksEnterpriseProperties(String dataFile, + String purpose, + List properties, + PrintWriter output, + String contactUs) throws Exception { + try (IPIntelligenceOnPremiseEngine engine = new IPIntelligenceOnPremiseEngineBuilder() + .setPerformanceProfile(Constants.PerformanceProfiles.LowMemory) + .setAutoUpdate(false) + .build(dataFile, false)) { + return lacksEnterpriseProperties( + engine, purpose, properties, output, contactUs); + } + } + + /** + * As {@link #lacksEnterpriseProperties(String, String, List, PrintWriter, String)} + * for an engine the example has already built. + * @param engine the engine the example uses + * @param purpose what the example does, as the start of a sentence + * @param properties the properties the example needs + * @param output where to write + * @param contactUs the link to the contact page, tagged with the example + * @return true if the data file has none of the properties + */ + public static boolean lacksEnterpriseProperties(IPIntelligenceOnPremiseEngine engine, + String purpose, + List properties, + PrintWriter output, + String contactUs) { + for (String property : properties) { + if (hasProperty(engine, property)) { + return false; + } + } + output.println(purpose + " needs the " + joinNames(properties) + + " properties, which only the Enterprise data file has. The " + + "data file in use has none of them, so the example stops " + + "here. Contact us for a copy of the Enterprise data file at " + + contactUs); + output.flush(); + return true; + } + + /** + * Writes, after an example has shown properties that the data file in + * use does not have, where to ask for a copy of the Enterprise data + * file. + * @param output where to write + * @param contactUs the link to the contact page, tagged with the + * example's file so an enquiry can be traced to it + */ + public static void writeEnterpriseNote(PrintWriter output, String contactUs) { + output.println(); + output.println("Some of these properties need the Enterprise data " + + "file. Contact us for a copy at " + contactUs); + output.flush(); + } + + /** + * Writes, after an example that shows no location properties has run on + * a free data file, what the Enterprise data file adds and where to ask + * for a copy. + * @param output where to write + * @param contactUs the link to the contact page, tagged with the + * example's file so an enquiry can be traced to it + */ + public static void writeEnterpriseOffer(PrintWriter output, String contactUs) { + output.println(); + output.println("This example used a free data file. The Enterprise " + + "data file adds the location, range and connection " + + "properties. Contact us for a copy at " + contactUs); + output.flush(); + } + + /** + * Finds the performance profile an example is given by name, ignoring + * case. When no profile has the name, writes the names accepted. + * @param name the name given, such as MaxPerformance + * @param output where to write when no profile has the name + * @return the profile, or null when no profile has the name + */ + public static Constants.PerformanceProfiles findPerformanceProfile( + String name, + PrintWriter output) { + List names = new ArrayList<>(); + for (Constants.PerformanceProfiles profile : + Constants.PerformanceProfiles.values()) { + if (profile.name().equalsIgnoreCase(name)) { + return profile; + } + names.add(profile.name()); + } + output.println("'" + name + "' is not a performance profile. The " + + "names accepted, in upper or lower case, are " + + joinNames(names) + "."); + output.flush(); + return null; + } + + /** + * Joins names as "A, B and C". + */ + private static String joinNames(List names) { + if (names.size() == 1) { + return names.get(0); + } + return String.join(", ", names.subList(0, names.size() - 1)) + + " and " + names.get(names.size() - 1); + } + public static void cantFindDataFile(String dataFile) { logger.error("Could not find the data file '{}' which must be " + "somewhere in the project space to be found. An explicit " + "path to the data file can be supplied via the {} " + "environment variable or system property.", - dataFile, IPI_PATH_ENV_VAR); + dataFile, IPI_PATH_VARIABLE); } /** @@ -202,9 +418,11 @@ public static File getEvidenceFile(String evidenceFilename) throws Exception { /** * Tries to find the passed file, or if null a default file. Handles both absolute and relative paths. * An absolute path supplied by the caller is used as is. Otherwise an - * explicit path supplied via the {@link #IPI_PATH_ENV_VAR} environment + * explicit path supplied via the {@link #IPI_PATH_VARIABLE} environment * variable or system property is checked before the folder hierarchy is - * searched for the passed filename. + * searched for the passed filename. When neither is set the deprecated + * {@link #IPI_PATH_ENV_VAR} is read, and its first use is logged as a + * warning. * @param dataFilename a filename to find (can be absolute or relative path) * @return a full pathname * @throws Exception if the file was not found @@ -215,21 +433,32 @@ public static String getDataFileLocation(String dataFilename) throws Exception { // system property is checked before any search for the filename if (Objects.isNull(dataFilename) || Paths.get(dataFilename).isAbsolute() == false) { - String envDataFile = System.getenv(IPI_PATH_ENV_VAR); + String envName = IPI_PATH_VARIABLE; + String envDataFile = environmentOrProperty(envName); if (Objects.isNull(envDataFile)) { - envDataFile = System.getProperty(IPI_PATH_ENV_VAR); + envName = IPI_PATH_ENV_VAR; + envDataFile = environmentOrProperty(envName); + if (Objects.nonNull(envDataFile) + && OLD_NAME_LOGGED.compareAndSet(false, true)) { + logger.warn( + "The environment variable " + IPI_PATH_ENV_VAR + + " is deprecated and " + IPI_PATH_VARIABLE + + " replaces it. Set " + IPI_PATH_VARIABLE + + " to the same value, as a later release may no longer " + + "read " + IPI_PATH_ENV_VAR + "."); + } } if (Objects.nonNull(envDataFile)) { if (Files.exists(Paths.get(envDataFile))) { return envDataFile; } logger.warn("Ignoring {} value '{}' as no file exists there", - IPI_PATH_ENV_VAR, envDataFile); + envName, envDataFile); } } if (Objects.isNull(dataFilename)) { - dataFilename = ENTERPRISE_DATA_FILE_REL_PATH; + dataFilename = LITE_DATA_FILE_REL_PATH; } // Check if it's an absolute path @@ -262,7 +491,7 @@ public static String getDataFileLocation(String dataFilename) throws Exception { DataFileHelper.cantFindDataFile(dataFilename); throw new IOException("Data file not found at relative path: " + dataFilename + ". Searched in project directories and current working directory." + - " An explicit path can be supplied via the " + IPI_PATH_ENV_VAR + + " An explicit path can be supplied via the " + IPI_PATH_VARIABLE + " environment variable.", notFound); } } @@ -272,28 +501,31 @@ public static String getDataFileLocation(String dataFilename) throws Exception { /** * Finds a file below the project root, as - * {@link FileFinder#getFilePath(String)} does. A free data file that is - * missing has a {@code .link} in the data repository, so the data - * repository's script is run to download it, and the file is looked for - * again. + * {@link FileFinder#getFilePath(String)} does, and then in the data + * repository at the root of this repository, which is above the folder + * of the examples that their READMEs run them from. A free data file + * that is missing has a {@code .link} in the data repository, so + * the data repository's script is run to download it, and the file is + * looked for again. * @param file the name of the file, or the end of its path * @return the file * @throws IllegalArgumentException if the file was not found, naming the * command that downloads it when there is one */ public static File findDataFile(String file) { + File repository = findDataRepository(); try { - return getFilePath(file); + return find(file, repository); } catch (IllegalArgumentException notFound) { File link; try { - link = getFilePath(file + ".link"); + link = find(file + ".link", repository); } catch (IllegalArgumentException noLink) { throw notFound; } String command = runDownload(link); try { - return getFilePath(file); + return find(file, repository); } catch (IllegalArgumentException stillNotFound) { String message = "Could not download '" + file + "'. Run this command to download it: " + command; @@ -303,6 +535,37 @@ public static File findDataFile(String file) { } } + /** + * Finds a file below the project root, then below the data repository. + */ + private static File find(String file, File repository) { + try { + return getFilePath(file); + } catch (IllegalArgumentException notFound) { + if (repository == null) { + throw notFound; + } + return getFilePath(file, repository.getPath()); + } + } + + /** + * The data repository, cxx/data, in the nearest folder at or above the + * working directory that has one. + * @return the folder, or null if there is none + */ + private static File findDataRepository() { + File folder = new File(System.getProperty("user.dir")).getAbsoluteFile(); + while (folder != null) { + File data = new File(new File(folder, "cxx"), "data"); + if (new File(data, "get-data-files.sh").isFile()) { + return data; + } + folder = folder.getParentFile(); + } + return null; + } + /** * Runs the data repository's script for the file that a link is for, * and waits for it to finish. The data repository is the folder above @@ -314,17 +577,30 @@ private static String runDownload(File link) { String name = link.getName().substring( 0, link.getName().length() - ".link".length()); File data = link.getAbsoluteFile().getParentFile().getParentFile(); - List command; + List command = new ArrayList<>(); + String powerShell = null; if (System.getProperty("os.name", "") .toLowerCase(Locale.ROOT).startsWith("windows")) { - command = Arrays.asList("powershell", "-NoProfile", - "-ExecutionPolicy", "Bypass", "-File", - new File(data, "get-data-files.ps1").getPath(), name, - "-Source", DOWNLOAD_SOURCE); + powerShell = powerShell(); + command.add(powerShell); + command.add("-NoProfile"); + command.add("-ExecutionPolicy"); + command.add("Bypass"); + command.add("-File"); + command.add(new File(data, "get-data-files.ps1").getPath()); + command.add(name); + if (IN_GITHUB_ACTIONS == false) { + command.add("-Source"); + command.add(DOWNLOAD_SOURCE); + } } else { - command = Arrays.asList("sh", - new File(data, "get-data-files.sh").getPath(), - "-s", DOWNLOAD_SOURCE, name); + command.add("sh"); + command.add(new File(data, "get-data-files.sh").getPath()); + if (IN_GITHUB_ACTIONS == false) { + command.add("-s"); + command.add(DOWNLOAD_SOURCE); + } + command.add(name); } List typed = new ArrayList<>(); for (String argument : command) { @@ -335,7 +611,13 @@ private static String runDownload(File link) { logger.info("'{}' is missing, so it is being downloaded by running {}", name, line); try { - new ProcessBuilder(command).inheritIO().start().waitFor(); + ProcessBuilder builder = new ProcessBuilder(command).inheritIO(); + if (WINDOWS_POWERSHELL.equals(powerShell)) { + // Started from PowerShell 7, Windows PowerShell inherits its + // module path and then cannot load its own commands. + builder.environment().remove("PSModulePath"); + } + builder.start().waitFor(); } catch (IOException e) { logger.error("Could not run {}. {}", line, e.getMessage()); } catch (InterruptedException e) { @@ -343,4 +625,19 @@ private static String runDownload(File link) { } return line; } + + /** + * PowerShell 7 when it is on the path, otherwise Windows PowerShell. + */ + private static String powerShell() { + String path = System.getenv("PATH"); + if (path != null) { + for (String folder : path.split(File.pathSeparator)) { + if (new File(folder.replace("\"", ""), "pwsh.exe").isFile()) { + return "pwsh"; + } + } + } + return WINDOWS_POWERSHELL; + } } diff --git a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java index 653ecc7b9..2342f4047 100644 --- a/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java +++ b/examples/ip-intelligence/shared/src/main/java/fiftyone/ipintelligence/examples/shared/KeyHelper.java @@ -27,6 +27,8 @@ import org.slf4j.LoggerFactory; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; public class KeyHelper { /** @@ -34,10 +36,133 @@ public class KeyHelper { * hold the resource key. This name is checked first, before the legacy * {@link #TEST_RESOURCE_KEY} name. */ + public static final String RESOURCE_KEY_VARIABLE = "_51DEGREES_RESOURCE_KEY"; + + /** + * The old name of {@link #RESOURCE_KEY_VARIABLE}, read only when that + * one is not set. + * @deprecated use {@link #RESOURCE_KEY_VARIABLE}, and set the variable + * it names in place of this one. + */ + @Deprecated public static final String RESOURCE_KEY_ENV_VAR = "51DEGREES_RESOURCE_KEY"; + + // Set once the use of the deprecated name has been logged. + private static final AtomicBoolean OLD_NAME_LOGGED = + new AtomicBoolean(false); public static final String TEST_RESOURCE_KEY = "TestResourceKey"; static Logger logger = LoggerFactory.getLogger(KeyHelper.class); + /** + * The names every cloud test reads a resource key from first, before + * the names its own example reads. + */ + public static final List SHARED_RESOURCE_KEY_NAMES = + Collections.unmodifiableList(Arrays.asList( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE")); + + static final String CREATE_KEY_URL = + "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=shared-src-main-java-fiftyone-ipintelligence-examples-shared-keyhelper.java&utm_term=no-resource-key"; + + /** + * The resource key for a test that calls the live cloud service, from + * {@link #SHARED_RESOURCE_KEY_NAMES} and then the names + * {@link #getOrSetTestResourceKey()} reads. Fails the test when there + * is none, because a cloud test that passes without a key has called + * nothing. + * @param asksFor what the test asks the service for + * @param products the products the key has to carry + * @return the resource key + */ + public static String requireTestResourceKey(String asksFor, + String... products) { + return requireResourceKey(KeyHelper::readKey, Arrays.asList( + RESOURCE_KEY_VARIABLE, TEST_RESOURCE_KEY), asksFor, products); + } + + static String requireResourceKey(Function read, + List exampleNames, + String asksFor, + String... products) { + List names = new ArrayList<>(SHARED_RESOURCE_KEY_NAMES); + names.addAll(exampleNames); + for (String name : names) { + String value = read.apply(name); + if (KeyUtils.isInvalidKey(value) + && RESOURCE_KEY_VARIABLE.equals(name)) { + value = oldNameResourceKey(read); + } + if (KeyUtils.isInvalidKey(value) == false) { + return value.trim(); + } + } + throw new AssertionError(noResourceKey(names, asksFor, products)); + } + + /** + * The key held under the deprecated {@link #RESOURCE_KEY_ENV_VAR}, or + * null when there is none. It is read only when + * {@link #RESOURCE_KEY_VARIABLE} holds no key, and its first use is + * logged as a warning. + */ + private static String oldNameResourceKey(Function read) { + String value = read.apply(RESOURCE_KEY_ENV_VAR); + if (KeyUtils.isInvalidKey(value)) { + return null; + } + if (OLD_NAME_LOGGED.compareAndSet(false, true)) { + logger.warn( + "The environment variable " + RESOURCE_KEY_ENV_VAR + + " is deprecated and " + RESOURCE_KEY_VARIABLE + + " replaces it. Set " + RESOURCE_KEY_VARIABLE + + " to the same value, as a later release may no longer " + + "read " + RESOURCE_KEY_ENV_VAR + "."); + } + return value; + } + + static String noResourceKey(List names, String asksFor, + String... products) { + return "This test calls the live 51Degrees cloud service, so it " + + "needs a resource key and there is nothing to fall back on. " + + "None of " + String.join(", ", names) + " is set, as an " + + "environment variable or as a system property, tried in that " + + "order. The key has to carry the " + productList(products) + + ", because the test asks the service for " + asksFor + ". To " + + "see the products a key carries, open " + + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=. " + + "Create a key at " + CREATE_KEY_URL + "."; + } + + private static String productList(String... products) { + StringBuilder list = new StringBuilder(); + for (int i = 0; i < products.length; i++) { + if (i > 0) { + list.append(i == products.length - 1 ? " and " : ", "); + } + list.append(products[i]); + } + return list + (products.length == 1 ? " product" : " products"); + } + + /** + * A key read as an environment variable, as its upper case name, and + * then as a system property, skipping any that is not a key, such as + * the empty string the example poms pass for a key that is not set. + */ + private static String readKey(String name) { + for (String value : new String[]{ + System.getenv(name), + System.getenv(name.toUpperCase()), + System.getProperty(name)}) { + if (KeyUtils.isInvalidKey(value) == false) { + return value; + } + } + return null; + } + /** * Obtain a resource key either from environment variable or from a property. */ @@ -64,16 +189,22 @@ public static String getOrSetTestResourceKey(String value) { /** * Obtain a resource key from the passed argument, * from environment variable or from a property. The aligned - * {@link #RESOURCE_KEY_ENV_VAR} name is checked first, then the - * passed variable name. Store as System Property under the - * passed variable name. + * {@link #RESOURCE_KEY_VARIABLE} name is checked first, then the + * deprecated {@link #RESOURCE_KEY_ENV_VAR}, then the passed variable + * name. Store as System Property under the passed variable name. */ public static String getOrSetResourceKey(String value, String variableName, String errorMessage, boolean shouldThrow) { if (Objects.isNull(value)) { // check the aligned name first, then the legacy name - value = KeyUtils.getNamedKey(RESOURCE_KEY_ENV_VAR); + value = KeyUtils.getNamedKey(RESOURCE_KEY_VARIABLE); + if (Objects.isNull(value) || value.isEmpty()) { + String oldNameKey = oldNameResourceKey(KeyUtils::getNamedKey); + if (Objects.nonNull(oldNameKey)) { + value = oldNameKey; + } + } if (Objects.isNull(value)) { value = KeyUtils.getNamedKey(variableName); } @@ -85,15 +216,18 @@ public static String getOrSetResourceKey(String value, String variableName, "runnable example,\n - as an Environment Variable named \"\u001B[36m{}\u001B[0m\"," + "\n - as a System Property named \"\u001B[36m{}\u001B[0m\" " + "(the legacy name \"\u001B[36m{}\u001B[0m\" is also still accepted).", - RESOURCE_KEY_ENV_VAR, RESOURCE_KEY_ENV_VAR, variableName); + RESOURCE_KEY_VARIABLE, RESOURCE_KEY_VARIABLE, variableName); logger.error(errorMessage); if (shouldThrow) { throw new IllegalStateException("\"" + value + "\" is not a valid resource key"); } } - // capture the passed parameter for next time called - System.setProperty(variableName, value); + // Keep a valid key for next time. System.setProperty throws on null, + // and an invalid key is not worth keeping. + if (KeyUtils.isInvalidKey(value) == false) { + System.setProperty(variableName, value); + } return value; } public static String getOrSetSuperResourceKey(String value, String variablename) { diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java new file mode 100644 index 000000000..e1a3418b3 --- /dev/null +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/DataFileHelperTest.java @@ -0,0 +1,130 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.shared; + +import fiftyone.pipeline.engines.Constants.PerformanceProfiles; +import org.junit.Test; + +import java.io.File; +import java.io.PrintWriter; +import java.io.StringWriter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +public class DataFileHelperTest { + + @Test + public void findPerformanceProfile_IgnoresCase() { + StringWriter written = new StringWriter(); + assertEquals(PerformanceProfiles.MaxPerformance, + DataFileHelper.findPerformanceProfile("maxperformance", + new PrintWriter(written))); + assertEquals(PerformanceProfiles.LowMemory, + DataFileHelper.findPerformanceProfile("LOWMEMORY", + new PrintWriter(written))); + assertEquals("", written.toString()); + } + + @Test + public void findPerformanceProfile_Unknown_WritesNamesAccepted() { + StringWriter written = new StringWriter(); + assertNull(DataFileHelper.findPerformanceProfile("Fastest", + new PrintWriter(written))); + for (PerformanceProfiles profile : PerformanceProfiles.values()) { + assertTrue(written.toString(), + written.toString().contains(profile.name())); + } + } + + private static final String NEW_PATH_NAME = "_51DEGREES_IPI_PATH"; + private static final String OLD_PATH_NAME = "51DEGREES_IPI_PATH"; + + /** + * System properties stand in for the environment variables, because + * the lookup reads either and a test cannot set an environment + * variable. + */ + @Test + public void getDataFileLocation_OldNameWhenNewIsNotSet() + throws Exception { + assumePathNamesNotInEnvironment(); + String savedNew = System.clearProperty(NEW_PATH_NAME); + String savedOld = System.clearProperty(OLD_PATH_NAME); + File file = File.createTempFile("old-path-name", ".ipi"); + try { + System.setProperty(OLD_PATH_NAME, file.getPath()); + assertEquals(file.getPath(), + DataFileHelper.getDataFileLocation(null)); + } finally { + restore(NEW_PATH_NAME, savedNew); + restore(OLD_PATH_NAME, savedOld); + file.delete(); + } + } + + @Test + public void getDataFileLocation_NewNameWinsOverOld() throws Exception { + assumePathNamesNotInEnvironment(); + String savedNew = System.clearProperty(NEW_PATH_NAME); + String savedOld = System.clearProperty(OLD_PATH_NAME); + File newFile = File.createTempFile("new-path-name", ".ipi"); + File oldFile = File.createTempFile("old-path-name", ".ipi"); + try { + System.setProperty(NEW_PATH_NAME, newFile.getPath()); + System.setProperty(OLD_PATH_NAME, oldFile.getPath()); + assertEquals(newFile.getPath(), + DataFileHelper.getDataFileLocation(null)); + } finally { + restore(NEW_PATH_NAME, savedNew); + restore(OLD_PATH_NAME, savedOld); + newFile.delete(); + oldFile.delete(); + } + } + + @Test + @SuppressWarnings("deprecation") + public void oldPathConstantKeepsItsValue() throws Exception { + assertEquals(OLD_PATH_NAME, DataFileHelper.IPI_PATH_ENV_VAR); + assertTrue(DataFileHelper.class.getField("IPI_PATH_ENV_VAR") + .isAnnotationPresent(Deprecated.class)); + assertEquals(NEW_PATH_NAME, + DataFileHelper.class.getField("IPI_PATH_VARIABLE").get(null)); + } + + private static void assumePathNamesNotInEnvironment() { + assumeTrue(System.getenv(NEW_PATH_NAME) == null); + assumeTrue(System.getenv(OLD_PATH_NAME) == null); + } + + private static void restore(String name, String value) { + if (value == null) { + System.clearProperty(name); + } else { + System.setProperty(name, value); + } + } +} diff --git a/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java new file mode 100644 index 000000000..df25ee4b2 --- /dev/null +++ b/examples/ip-intelligence/shared/src/test/java/fiftyone/ipintelligence/examples/shared/KeyHelperTest.java @@ -0,0 +1,168 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.examples.shared; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assume.assumeTrue; + +public class KeyHelperTest { + /** + * Values of the form KeyUtils accepts as a key. Neither is a real key. + */ + private static final String KEY_A = "AQAAAAAAAAAAAAAAAAAAAAAA"; + private static final String KEY_B = "AgAAAAAAAAAAAAAAAAAAAAAA"; + + /** + * The names getOrSetTestResourceKey reads, in its order, apart from + * the deprecated name it reads when the first is not set. + */ + private static final List EXAMPLE_NAMES = Arrays.asList( + KeyHelper.RESOURCE_KEY_VARIABLE, KeyHelper.TEST_RESOURCE_KEY); + + /** + * With no key anywhere, the lookup logs that there is none and stores + * nothing, rather than passing null to System.setProperty. + */ + @Test + public void getOrSetTestResourceKey_NoKey_StoresNothing() { + // The deprecated name is read as well, so it must not be set + // either. + List namesRead = new ArrayList<>(EXAMPLE_NAMES); + namesRead.add(OLD_KEY_NAME); + // A test can clear a system property but not an environment + // variable. + for (String name : namesRead) { + assumeTrue(name + " is set in the environment", + System.getenv(name) == null && + System.getenv(name.toUpperCase()) == null); + } + Map saved = new HashMap<>(); + for (String name : namesRead) { + saved.put(name, System.clearProperty(name)); + } + try { + assertNull(KeyHelper.getOrSetTestResourceKey(false)); + for (String name : namesRead) { + assertNull(name, System.getProperty(name)); + } + } finally { + for (Map.Entry entry : saved.entrySet()) { + if (entry.getValue() != null) { + System.setProperty(entry.getKey(), entry.getValue()); + } + } + } + } + + @Test + public void requireResourceKey_NoKey_FailsNamingEveryName() { + AssertionError error = assertThrows(AssertionError.class, () -> + KeyHelper.requireResourceKey(name -> null, EXAMPLE_NAMES, + "the network and location of an IP address, and whether a " + + "device is mobile", + "ip", "device")); + assertEquals("This test calls the live 51Degrees cloud service, so " + + "it needs a resource key and there is nothing to fall back " + + "on. None of _51DEGREES_RESOURCE_KEY, " + + "_51DEGREES_RESOURCE_KEY_BESPOKE, _51DEGREES_RESOURCE_KEY, " + + "TestResourceKey is set, as an environment variable or as a " + + "system property, tried in that order. The key has to carry " + + "the ip and device products, because the test asks the " + + "service for the network and location of an IP address, and " + + "whether a device is mobile. To see the products a key " + + "carries, open " + + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=. " + + "Create a key at " + KeyHelper.CREATE_KEY_URL + ".", + error.getMessage()); + } + + /** + * A name holding something that is not a key, such as the empty + * string the example poms pass for a key that is not set, is passed + * over. + */ + @Test + public void requireResourceKey_FirstValidNameWins() { + Map values = new HashMap<>(); + values.put("_51DEGREES_RESOURCE_KEY", ""); + values.put("_51DEGREES_RESOURCE_KEY_BESPOKE", KEY_A); + values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); + assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, + EXAMPLE_NAMES, "every IP Intelligence property", "ip")); + } + + @Test + public void requireResourceKey_ExampleNamesInTheirOrder() { + Map values = new HashMap<>(); + values.put(KeyHelper.RESOURCE_KEY_VARIABLE, KEY_A); + values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); + assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, + EXAMPLE_NAMES, "every IP Intelligence property", "ip")); + values.remove(KeyHelper.RESOURCE_KEY_VARIABLE); + assertEquals(KEY_B, KeyHelper.requireResourceKey(values::get, + EXAMPLE_NAMES, "every IP Intelligence property", "ip")); + } + + private static final String NEW_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; + private static final String OLD_KEY_NAME = "51DEGREES_RESOURCE_KEY"; + + @Test + public void requireResourceKey_OldNameWhenNewIsNotSet() { + Map values = new HashMap<>(); + values.put(OLD_KEY_NAME, KEY_A); + values.put(KeyHelper.TEST_RESOURCE_KEY, KEY_B); + assertEquals(KEY_A, KeyHelper.requireResourceKey(values::get, + Arrays.asList(NEW_KEY_NAME, KeyHelper.TEST_RESOURCE_KEY), + "every IP Intelligence property", "ip")); + } + + @Test + public void requireResourceKey_NewNameWinsOverOld() { + Map values = new HashMap<>(); + values.put(NEW_KEY_NAME, KEY_B); + values.put(OLD_KEY_NAME, KEY_A); + assertEquals(KEY_B, KeyHelper.requireResourceKey(values::get, + Arrays.asList(NEW_KEY_NAME, KeyHelper.TEST_RESOURCE_KEY), + "every IP Intelligence property", "ip")); + } + + @Test + @SuppressWarnings("deprecation") + public void oldKeyConstantKeepsItsValue() throws Exception { + assertEquals(OLD_KEY_NAME, KeyHelper.RESOURCE_KEY_ENV_VAR); + assertEquals(true, KeyHelper.class.getField("RESOURCE_KEY_ENV_VAR") + .isAnnotationPresent(Deprecated.class)); + assertEquals(NEW_KEY_NAME, + KeyHelper.class.getField("RESOURCE_KEY_VARIABLE").get(null)); + } +} diff --git a/examples/ip-intelligence/web/getting-started.api/pom.xml b/examples/ip-intelligence/web/getting-started.api/pom.xml index 5cb1db7d1..ef29feee4 100644 --- a/examples/ip-intelligence/web/getting-started.api/pom.xml +++ b/examples/ip-intelligence/web/getting-started.api/pom.xml @@ -37,7 +37,7 @@ 2.0.20 1.3.16 - 4.5.128 + 4.5.130 20231013 diff --git a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java index 9887afc57..34b00b036 100644 --- a/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java +++ b/examples/ip-intelligence/web/getting-started.api/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedApi.java @@ -85,7 +85,7 @@ import java.util.zip.GZIPOutputStream; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; import static fiftyone.pipeline.util.FileFinder.getFilePath; /** @@ -110,7 +110,7 @@ public class GettingStartedApi extends HttpServlet { public static void main(String[] args) throws Exception { configureLogback(getFilePath("logback.xml")); - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH; String dataFileLocation = DataFileHelper.getDataFileLocation(dataFile); logger.info("Running GettingStartedApi server using data file '{}'", dataFileLocation); diff --git a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java index ca1b1ecfc..ff49ca9e7 100644 --- a/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java +++ b/examples/ip-intelligence/web/getting-started.cloud.mixed/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudMixedTest.java @@ -23,7 +23,6 @@ package fiftyone.ipintelligence.examples.web; import fiftyone.ipintelligence.examples.shared.KeyHelper; -import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -36,7 +35,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeFalse; public class GettingStartedWebCloudMixedTest { private static Server SERVER; @@ -44,9 +42,10 @@ public class GettingStartedWebCloudMixedTest { @BeforeClass public static void startJetty() throws Exception { - String resourceKey = KeyUtils.getNamedKey(KeyHelper.TEST_RESOURCE_KEY); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "the network and location of the address a request comes from, " + + "and the device that sent it", + "ip", "device"); // Make the resource key available to the pipeline configuration file. // The resource key must include both device detection and IP // Intelligence properties as this example uses both engines. diff --git a/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java b/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java index ea87c40a7..06a7b8b48 100644 --- a/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java +++ b/examples/ip-intelligence/web/getting-started.cloud/src/test/java/fiftyone/ipintelligence/examples/web/GettingStartedWebCloudTest.java @@ -23,7 +23,6 @@ package fiftyone.ipintelligence.examples.web; import fiftyone.ipintelligence.examples.shared.KeyHelper; -import fiftyone.ipintelligence.shared.testhelpers.KeyUtils; import org.eclipse.jetty.server.Server; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -38,7 +37,6 @@ import static fiftyone.pipeline.util.FileFinder.getFilePath; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeFalse; public class GettingStartedWebCloudTest { private static Server SERVER; @@ -46,9 +44,9 @@ public class GettingStartedWebCloudTest { @BeforeClass public static void startJetty() throws Exception { - String resourceKey = KeyUtils.getNamedKey(KeyHelper.TEST_RESOURCE_KEY); - assumeFalse("Skipping test, no resource key found", - KeyUtils.isInvalidKey(resourceKey)); + String resourceKey = KeyHelper.requireTestResourceKey( + "the network and location of the address a request comes from", + "ip"); // Make the resource key available to the pipeline configuration file System.setProperty(KeyHelper.TEST_RESOURCE_KEY, resourceKey); diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java index b6addc8d7..18f5bc05b 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java @@ -34,9 +34,10 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.mixed/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebMixed.java). * - * This example requires: - * - A Device Detection data file (.hash format) - * - An IP Intelligence data file (.ipi format) + * By default the example uses the free Lite Device Detection and IP Intelligence data files, + * which it downloads when they are missing, and shows each property only the Enterprise data + * files have as needing them. To obtain the Enterprise data files, please + * [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.mixed-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebmixed.java&utm_term=header). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -48,9 +49,13 @@ import fiftyone.devicedetection.hash.engine.onpremise.flowelements.DeviceDetectionHashEngine; import fiftyone.devicedetection.shared.DeviceData; +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.shared.IPIntelligenceData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; import fiftyone.pipeline.web.services.FlowDataProviderCore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,10 +70,11 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Map; +import java.util.function.Supplier; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.IPI_PATH_ENV_VAR; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.IPI_PATH_VARIABLE; import static fiftyone.ipintelligence.examples.shared.PropertyHelper.*; import static fiftyone.pipeline.util.FileFinder.getFilePath; @@ -79,9 +85,7 @@ *

    * The configuration file for the pipeline is at src/main/webapp/WEB-INF/51Degrees-Mixed.xml *

    - * This example requires: - * - A Device Detection data file (.hash format) - * - An IP Intelligence data file (.ipi format) + * By default the free Lite data files are used. */ public class GettingStartedWebMixed extends HttpServlet { private static final long serialVersionUID = 1734154705981153541L; @@ -95,7 +99,7 @@ public static void main(String[] args) throws Exception { // Accept two command line arguments: device detection data file and IP intelligence data file String ddDataFile = args.length > 0 ? args[0] : DD_DATA_FILE_DEFAULT; - String ipiDataFile = args.length > 1 ? args[1] : ENTERPRISE_DATA_FILE_REL_PATH; + String ipiDataFile = args.length > 1 ? args[1] : LITE_DATA_FILE_REL_PATH; // Set Device Detection data file location try { @@ -110,13 +114,13 @@ public static void main(String[] args) throws Exception { // Set IP Intelligence data file location try { - String ipiPath = fiftyone.ipintelligence.examples.shared.DataFileHelper.getDataFileLocation(ipiDataFile); + String ipiPath = DataFileHelper.getDataFileLocation(ipiDataFile); System.setProperty("TestDataFile", ipiPath); logger.info("Using IP Intelligence data file: {}", ipiPath); } catch (Exception e) { logger.warn("IP Intelligence data file not found at: {}. " + "An explicit path can be supplied via the {} environment variable.", - ipiDataFile, IPI_PATH_ENV_VAR); + ipiDataFile, IPI_PATH_VARIABLE); logger.warn("Will attempt to use default path from XML configuration"); logger.debug("Error finding IP Intelligence data file", e); } @@ -184,61 +188,99 @@ private String loadTemplate(String templatePath) throws IOException { private String substituteTemplateValues(String template, DeviceData deviceData, IPIntelligenceData ipiData, String inputIp, FlowData flowData) { + // The engines say which properties their data files have. + DeviceDetectionHashEngine ddEngine = + flowData.getPipeline().getElement(DeviceDetectionHashEngine.class); + IPIntelligenceOnPremiseEngine ipiEngine = + flowData.getPipeline().getElement(IPIntelligenceOnPremiseEngine.class); return template .replace("${DATA_FILE_WARNING}", HtmlContentHelper.getDataFileAgeWarning(flowData)) .replace("${INPUT_IP_ADDRESS}", inputIp != null ? inputIp : "") // Device Detection properties - .replace("${HARDWARE_VENDOR}", asString(tryGet(deviceData::getHardwareVendor))) - .replace("${HARDWARE_NAME}", asString(tryGet(deviceData::getHardwareName))) - .replace("${DEVICE_TYPE}", asString(tryGet(deviceData::getDeviceType))) - .replace("${PLATFORM_NAME}", asString(tryGet(deviceData::getPlatformName))) - .replace("${PLATFORM_VERSION}", asString(tryGet(deviceData::getPlatformVersion))) - .replace("${BROWSER_NAME}", asString(tryGet(deviceData::getBrowserName))) - .replace("${BROWSER_VERSION}", asString(tryGet(deviceData::getBrowserVersion))) - .replace("${SCREEN_WIDTH}", asString(tryGet(deviceData::getScreenPixelsWidth))) - .replace("${SCREEN_HEIGHT}", asString(tryGet(deviceData::getScreenPixelsHeight))) + .replace("${HARDWARE_VENDOR}", value(ddEngine, "HardwareVendor", deviceData::getHardwareVendor)) + .replace("${HARDWARE_NAME}", value(ddEngine, "HardwareName", deviceData::getHardwareName)) + .replace("${DEVICE_TYPE}", value(ddEngine, "DeviceType", deviceData::getDeviceType)) + .replace("${PLATFORM_NAME}", value(ddEngine, "PlatformName", deviceData::getPlatformName)) + .replace("${PLATFORM_VERSION}", value(ddEngine, "PlatformVersion", deviceData::getPlatformVersion)) + .replace("${BROWSER_NAME}", value(ddEngine, "BrowserName", deviceData::getBrowserName)) + .replace("${BROWSER_VERSION}", value(ddEngine, "BrowserVersion", deviceData::getBrowserVersion)) + .replace("${SCREEN_WIDTH}", value(ddEngine, "ScreenPixelsWidth", deviceData::getScreenPixelsWidth)) + .replace("${SCREEN_HEIGHT}", value(ddEngine, "ScreenPixelsHeight", deviceData::getScreenPixelsHeight)) + // DeviceId is a match metric, which every data file gives. .replace("${DEVICE_ID}", asString(tryGet(deviceData::getDeviceId))) // IP Intelligence properties - .replace("${REGISTERED_NAME}", asStringProperty(tryGet(ipiData::getRegisteredName))) - .replace("${REGISTERED_OWNER}", asStringProperty(tryGet(ipiData::getRegisteredOwner))) - .replace("${REGISTERED_COUNTRY}", asStringProperty(tryGet(ipiData::getRegisteredCountry))) - .replace("${IP_RANGE_START}", asIPAddressProperty(tryGet(ipiData::getIpRangeStart))) - .replace("${IP_RANGE_END}", asIPAddressProperty(tryGet(ipiData::getIpRangeEnd))) - .replace("${COUNTRY}", asStringProperty(tryGet(ipiData::getCountry))) - .replace("${COUNTRY_CODE}", asStringProperty(tryGet(ipiData::getCountryCode))) - .replace("${COUNTRY_CODE3}", asStringProperty(tryGet(ipiData::getCountryCode3))) - .replace("${REGION}", asStringProperty(tryGet(ipiData::getRegion))) - .replace("${STATE}", asStringProperty(tryGet(ipiData::getState))) - .replace("${TOWN}", asStringProperty(tryGet(ipiData::getTown))) - .replace("${LATITUDE}", asFloatProperty(tryGet(ipiData::getLatitude))) - .replace("${LONGITUDE}", asFloatProperty(tryGet(ipiData::getLongitude))) - .replace("${AREAS}", asWktStringProperty(tryGet(ipiData::getAreas))) - .replace("${AREAS_JS}", escapeForJs(asWktStringProperty(tryGet(ipiData::getAreas)))) - .replace("${ACCURACY_RADIUS}", asIntegerProperty(tryGet(ipiData::getAccuracyRadiusMin))) - .replace("${TIME_ZONE_OFFSET}", asIntegerProperty(tryGet(ipiData::getTimeZoneOffset))) + .replace("${REGISTERED_NAME}", value(ipiEngine, "RegisteredName", ipiData::getRegisteredName)) + .replace("${REGISTERED_OWNER}", value(ipiEngine, "RegisteredOwner", ipiData::getRegisteredOwner)) + .replace("${REGISTERED_COUNTRY}", value(ipiEngine, "RegisteredCountry", ipiData::getRegisteredCountry)) + .replace("${IP_RANGE_START}", value(ipiEngine, "IpRangeStart", ipiData::getIpRangeStart)) + .replace("${IP_RANGE_END}", value(ipiEngine, "IpRangeEnd", ipiData::getIpRangeEnd)) + .replace("${COUNTRY}", value(ipiEngine, "Country", ipiData::getCountry)) + .replace("${COUNTRY_CODE}", value(ipiEngine, "CountryCode", ipiData::getCountryCode)) + .replace("${COUNTRY_CODE3}", value(ipiEngine, "CountryCode3", ipiData::getCountryCode3)) + .replace("${REGION}", value(ipiEngine, "Region", ipiData::getRegion)) + .replace("${STATE}", value(ipiEngine, "State", ipiData::getState)) + .replace("${TOWN}", value(ipiEngine, "Town", ipiData::getTown)) + .replace("${LATITUDE}", value(ipiEngine, "Latitude", ipiData::getLatitude)) + .replace("${LONGITUDE}", value(ipiEngine, "Longitude", ipiData::getLongitude)) + .replace("${AREAS}", value(ipiEngine, "Areas", ipiData::getAreas)) + // The map shows nothing when the data file has no areas. + .replace("${AREAS_JS}", DataFileHelper.hasProperty(ipiEngine, "Areas") ? + escapeForJs(asString(ipiData.getAreas())) : "") + .replace("${ACCURACY_RADIUS}", value(ipiEngine, "AccuracyRadiusMin", ipiData::getAccuracyRadiusMin)) + .replace("${TIME_ZONE_OFFSET}", value(ipiEngine, "TimeZoneOffset", ipiData::getTimeZoneOffset)) .replace("${EVIDENCE_ROWS}", buildEvidenceRows(flowData)) - //.replace("${HumanProbability}",asStringProperty(tryGet(ipiData::getHumanProbability))) - .replace("${LocationConfidence}",asStringProperty(tryGet(ipiData::getLocationConfidence))) - .replace("${ConnectionType}",asStringProperty(tryGet(ipiData::getConnectionType))) - .replace("${HumanProbability}",asIntegerProperty(tryGet(ipiData::getHumanProbability))) - .replace("${HardwareDiversity}",asIntegerProperty(tryGet(ipiData::getHardwareDiversity))) - .replace("${BrowserDiversity}",asIntegerProperty(tryGet(ipiData::getBrowserDiversity))) - .replace("${PlatformDiversity}",asIntegerProperty(tryGet(ipiData::getPlatformDiversity))) - // On-premise engines: only invite the user to contact us about more - // properties and features when running against the free Lite tier. + .replace("${LocationConfidence}", value(ipiEngine, "LocationConfidence", ipiData::getLocationConfidence)) + .replace("${ConnectionType}", value(ipiEngine, "ConnectionType", ipiData::getConnectionType)) + .replace("${HumanProbability}", value(ipiEngine, "HumanProbability", ipiData::getHumanProbability)) + .replace("${HardwareDiversity}", value(ipiEngine, "HardwareDiversity", ipiData::getHardwareDiversity)) + .replace("${BrowserDiversity}", value(ipiEngine, "BrowserDiversity", ipiData::getBrowserDiversity)) + .replace("${PlatformDiversity}", value(ipiEngine, "PlatformDiversity", ipiData::getPlatformDiversity)) + // On-premise engines: only invite the user to contact us about the + // Enterprise data files when running on a free data file. .replace("${CONTACT_MESSAGE}", HtmlContentHelper.getContactMessage( - HtmlContentHelper.ContactMessageVariant.ON_PREMISE, isLiteDataFile(flowData))); + HtmlContentHelper.ContactMessageVariant.ON_PREMISE, + "Lite".equals(ddEngine.getDataSourceTier()) || + DataFileHelper.isFreeDataFile(ipiEngine))); } /** - * Determine whether the on-premise IP Intelligence engine is using the free Lite tier data file. + * The value of a property for the page, or that it needs the Enterprise + * data file when the data file in use does not have it. The value is + * only read when the data file has the property. */ - private boolean isLiteDataFile(FlowData flowData) { - fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine engine = - flowData.getPipeline().getElement( - fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine.class); - return engine != null && engine.getDataSourceTier().equals("Lite"); + private static String value(IPIntelligenceOnPremiseEngine engine, + String property, + Supplier> value) { + return DataFileHelper.hasProperty(engine, property) ? + asString(value.get()) : + DataFileHelper.NEEDS_ENTERPRISE; + } + + /** + * As {@link #value(IPIntelligenceOnPremiseEngine, String, Supplier)} for + * a Device Detection property. + */ + private static String value(DeviceDetectionHashEngine engine, + String property, + Supplier> value) { + return hasProperty(engine, property) ? + asString(value.get()) : + DataFileHelper.NEEDS_ENTERPRISE; + } + + /** + * True when the Device Detection engine's data file has the property. + * The Lite data file leaves out the properties only the Enterprise data + * file has, such as HardwareVendor, HardwareName and DeviceType. + */ + private static boolean hasProperty(DeviceDetectionHashEngine engine, String name) { + for (FiftyOneAspectPropertyMetaData property : engine.getProperties()) { + if (property.getName().equalsIgnoreCase(name)) { + return true; + } + } + return false; } private String buildEvidenceRows(FlowData flowData) { diff --git a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml index 127b4665d..83f577d3e 100644 --- a/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml +++ b/examples/ip-intelligence/web/getting-started.mixed/src/main/webapp/WEB-INF/51Degrees-Mixed.xml @@ -42,7 +42,7 @@ IPIntelligenceOnPremiseEngine - + JsonBuilderElement diff --git a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java index ec48e2df9..d87b20d5d 100644 --- a/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java +++ b/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java @@ -33,8 +33,9 @@ * * This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-java/blob/main/examples/ip-intelligence/web/getting-started.onprem/src/main/java/fiftyone/ipintelligence/examples/web/GettingStartedWebOnPrem.java). * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.onprem-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebonprem.java&utm_term=header). + * By default the example uses the free Lite IP Intelligence data file, which it downloads when + * it is missing, and shows each property only the Enterprise data file has as needing it. To + * obtain an Enterprise data file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.onprem-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebonprem.java&utm_term=header). * * Required Maven Dependencies: * - [com.51degrees:ip-intelligence](https://central.sonatype.com/artifact/com.51degrees/ip-intelligence) @@ -60,9 +61,12 @@ package fiftyone.ipintelligence.examples.web; +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.examples.shared.DataFileHelper; import fiftyone.ipintelligence.shared.IPIntelligenceData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.web.services.FlowDataProviderCore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,16 +80,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; +import java.util.function.Supplier; import static fiftyone.common.testhelpers.LogbackHelper.configureLogback; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.ENTERPRISE_DATA_FILE_REL_PATH; -import static fiftyone.ipintelligence.examples.shared.DataFileHelper.IPI_PATH_ENV_VAR; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asStringProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asIntegerProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asFloatProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asIPAddressProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asWktStringProperty; -import static fiftyone.ipintelligence.examples.shared.PropertyHelper.tryGet; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.LITE_DATA_FILE_REL_PATH; +import static fiftyone.ipintelligence.examples.shared.DataFileHelper.IPI_PATH_VARIABLE; +import static fiftyone.ipintelligence.examples.shared.PropertyHelper.asString; import static fiftyone.pipeline.util.FileFinder.getFilePath; /** @@ -102,8 +102,9 @@ * * The configuration file for the pipeline is at src/main/webapp/WEB-INF/51Degrees-OnPrem.xml * - * This example requires an enterprise IP Intelligence data file (.ipi). - * To obtain an enterprise data file for testing, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.onprem-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebonprem.java&utm_term=gettingstartedwebonprem). + * By default the example uses the free Lite IP Intelligence data file, which it downloads when + * it is missing, and shows each property only the Enterprise data file has as needing it. To + * obtain an Enterprise data file, please [contact us](https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-java&utm_content=web-getting-started.onprem-src-main-java-fiftyone-ipintelligence-examples-web-gettingstartedwebonprem.java&utm_term=gettingstartedwebonprem). */ public class GettingStartedWebOnPrem extends HttpServlet { private static final long serialVersionUID = 1734154705981153540L; @@ -114,17 +115,17 @@ public static void main(String[] args) throws Exception { logger.info("Running Example {}", GettingStartedWebOnPrem.class); // Use the supplied path for the data file - String dataFile = args.length > 0 ? args[0] : ENTERPRISE_DATA_FILE_REL_PATH; + String dataFile = args.length > 0 ? args[0] : LITE_DATA_FILE_REL_PATH; // Set data file location using DataFileHelper try { - String dataFilePath = fiftyone.ipintelligence.examples.shared.DataFileHelper.getDataFileLocation(dataFile); + String dataFilePath = DataFileHelper.getDataFileLocation(dataFile); System.setProperty("TestDataFile", dataFilePath); logger.info("Using data file: {}", dataFilePath); } catch (Exception e) { logger.warn("Data file not found at expected location: {}. " + "An explicit path can be supplied via the {} environment variable.", - dataFile, IPI_PATH_ENV_VAR); + dataFile, IPI_PATH_VARIABLE); logger.warn("Will attempt to use default path from XML configuration"); logger.debug("Error finding data file", e); } @@ -201,41 +202,50 @@ private String loadTemplate(String templatePath) throws IOException { * Replace template variables with IP Intelligence data */ private String substituteTemplateValues(String template, IPIntelligenceData ipiData, String inputIp, FlowData flowData) { + // The engine says which properties its data file has. + IPIntelligenceOnPremiseEngine ipiEngine = + flowData.getPipeline().getElement(IPIntelligenceOnPremiseEngine.class); return template .replace("${DATA_FILE_WARNING}", HtmlContentHelper.getDataFileAgeWarning(flowData)) .replace("${INPUT_IP_ADDRESS}", inputIp != null ? inputIp : "") - .replace("${REGISTERED_NAME}", asStringProperty(tryGet(ipiData::getRegisteredName))) - .replace("${REGISTERED_OWNER}", asStringProperty(tryGet(ipiData::getRegisteredOwner))) - .replace("${REGISTERED_COUNTRY}", asStringProperty(tryGet(ipiData::getRegisteredCountry))) - .replace("${IP_RANGE_START}", asIPAddressProperty(tryGet(ipiData::getIpRangeStart))) - .replace("${IP_RANGE_END}", asIPAddressProperty(tryGet(ipiData::getIpRangeEnd))) - .replace("${COUNTRY}", asStringProperty(tryGet(ipiData::getCountry))) - .replace("${COUNTRY_CODE}", asStringProperty(tryGet(ipiData::getCountryCode))) - .replace("${COUNTRY_CODE3}", asStringProperty(tryGet(ipiData::getCountryCode3))) - .replace("${REGION}", asStringProperty(tryGet(ipiData::getRegion))) - .replace("${STATE}", asStringProperty(tryGet(ipiData::getState))) - .replace("${TOWN}", asStringProperty(tryGet(ipiData::getTown))) - .replace("${LATITUDE}", asFloatProperty(tryGet(ipiData::getLatitude))) - .replace("${LONGITUDE}", asFloatProperty(tryGet(ipiData::getLongitude))) - .replace("${AREAS}", asWktStringProperty(tryGet(ipiData::getAreas))) - .replace("${AREAS_JS}", escapeForJs(asWktStringProperty(tryGet(ipiData::getAreas)))) - .replace("${ACCURACY_RADIUS}", asIntegerProperty(tryGet(ipiData::getAccuracyRadiusMin))) - .replace("${TIME_ZONE_OFFSET}", asIntegerProperty(tryGet(ipiData::getTimeZoneOffset))) + .replace("${REGISTERED_NAME}", value(ipiEngine, "RegisteredName", ipiData::getRegisteredName)) + .replace("${REGISTERED_OWNER}", value(ipiEngine, "RegisteredOwner", ipiData::getRegisteredOwner)) + .replace("${REGISTERED_COUNTRY}", value(ipiEngine, "RegisteredCountry", ipiData::getRegisteredCountry)) + .replace("${IP_RANGE_START}", value(ipiEngine, "IpRangeStart", ipiData::getIpRangeStart)) + .replace("${IP_RANGE_END}", value(ipiEngine, "IpRangeEnd", ipiData::getIpRangeEnd)) + .replace("${COUNTRY}", value(ipiEngine, "Country", ipiData::getCountry)) + .replace("${COUNTRY_CODE}", value(ipiEngine, "CountryCode", ipiData::getCountryCode)) + .replace("${COUNTRY_CODE3}", value(ipiEngine, "CountryCode3", ipiData::getCountryCode3)) + .replace("${REGION}", value(ipiEngine, "Region", ipiData::getRegion)) + .replace("${STATE}", value(ipiEngine, "State", ipiData::getState)) + .replace("${TOWN}", value(ipiEngine, "Town", ipiData::getTown)) + .replace("${LATITUDE}", value(ipiEngine, "Latitude", ipiData::getLatitude)) + .replace("${LONGITUDE}", value(ipiEngine, "Longitude", ipiData::getLongitude)) + .replace("${AREAS}", value(ipiEngine, "Areas", ipiData::getAreas)) + // The map shows nothing when the data file has no areas. + .replace("${AREAS_JS}", DataFileHelper.hasProperty(ipiEngine, "Areas") ? + escapeForJs(asString(ipiData.getAreas())) : "") + .replace("${ACCURACY_RADIUS}", value(ipiEngine, "AccuracyRadiusMin", ipiData::getAccuracyRadiusMin)) + .replace("${TIME_ZONE_OFFSET}", value(ipiEngine, "TimeZoneOffset", ipiData::getTimeZoneOffset)) .replace("${EVIDENCE_ROWS}", buildEvidenceRows(flowData)) - // On-premise engine: only invite the user to contact us about more - // properties and features when running against the free Lite tier. + // On-premise engine: only invite the user to contact us about the + // Enterprise data file when running on a free data file. .replace("${CONTACT_MESSAGE}", HtmlContentHelper.getContactMessage( - HtmlContentHelper.ContactMessageVariant.ON_PREMISE, isLiteDataFile(flowData))); + HtmlContentHelper.ContactMessageVariant.ON_PREMISE, + DataFileHelper.isFreeDataFile(ipiEngine))); } /** - * Determine whether the on-premise engine is using the free Lite tier data file. + * The value of a property for the page, or that it needs the Enterprise + * data file when the data file in use does not have it. The value is + * only read when the data file has the property. */ - private boolean isLiteDataFile(FlowData flowData) { - fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine engine = - flowData.getPipeline().getElement( - fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine.class); - return engine != null && engine.getDataSourceTier().equals("Lite"); + private static String value(IPIntelligenceOnPremiseEngine engine, + String property, + Supplier> value) { + return DataFileHelper.hasProperty(engine, property) ? + asString(value.get()) : + DataFileHelper.NEEDS_ENTERPRISE; } /** diff --git a/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java b/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java index 0d70be0f1..2c4266b3f 100644 --- a/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java +++ b/examples/ip-intelligence/web/shared/src/main/java/fiftyone/ipintelligence/examples/web/HtmlContentHelper.java @@ -80,9 +80,8 @@ public static String getDataFileAgeWarning(FlowData flowData) { "A more recent data file may be needed for the most accurate IP " + "intelligence results. The latest Lite data file is downloaded by the " + "scripts in the data " + - "repository on GitHub. Find out about the full Enterprise data file, " + - "which includes automatic daily updates, on our " + - "pricing page.\n" + + "repository on GitHub. Contact us " + + "about the Enterprise data file, which includes automatic daily updates.\n" + ""; } @@ -90,8 +89,8 @@ public static String getDataFileAgeWarning(FlowData flowData) { * Build the contact-us message banner shown at the bottom of the web example * pages. Cloud examples are free by design, so the cloud variant is shown * unconditionally. On-premise examples only show the banner when running - * against the free Lite tier data file, so the {@code show} flag should be - * wired to {@code engine.getDataSourceTier().equals("Lite")}. + * against a free data file, so the {@code show} flag should be wired to + * {@code DataFileHelper.isFreeDataFile(engine)}. * * @param variant which message wording to use * @param show whether the banner should be rendered at all diff --git a/ip-intelligence/LICENSE b/ip-intelligence/LICENSE deleted file mode 100644 index 4153cd377..000000000 --- a/ip-intelligence/LICENSE +++ /dev/null @@ -1,287 +0,0 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. diff --git a/ip-intelligence/README.md b/ip-intelligence/README.md index 8f4534ed9..e5c19598b 100644 --- a/ip-intelligence/README.md +++ b/ip-intelligence/README.md @@ -1,8 +1,8 @@ # 51Degrees IP Intelligence Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines "Data rewards the curious") **Java IP Intelligence** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-java/main/.github/images/logo-160.png "Data rewards the curious") **Java IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-java/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines "developer documentation") ## Introduction @@ -10,7 +10,6 @@ This repository contains Java implementation of the IP Intelligence [specificati ## Dependencies -For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page. The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=dependencies) page shows the JDK versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. @@ -36,7 +35,7 @@ data file. The free 'Lite' and Asn data files come from the [data](https://github.com/51Degrees/data) repository, which is the `cxx/data` folder of this repository. Run `cxx/data/get-data-files.sh ip-intelligence` from the root of this repository to download them, or `cxx\data\get-data-files.ps1 ip-intelligence` in PowerShell on Windows. Otherwise [contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=on-premise) to obtain an 'Enterprise' data file. For tests, the data file can be supplied explicitly by setting the -`51DEGREES_IPI_PATH` environment variable or system property to the path of +`_51DEGREES_IPI_PATH` environment variable or system property to the path of the .ipi file. When it is not set, the project space is searched for the expected file name, with the free 'Lite' file expected in `cxx/data/ip-intelligence`. @@ -113,7 +112,7 @@ You will need [resource keys](https://51degrees.com/documentation/_services__clo The tests look for a resource key in the following order: -1. The aligned `51DEGREES_RESOURCE_KEY` name, set as an environment variable +1. The aligned `_51DEGREES_RESOURCE_KEY` name, set as an environment variable or as a system property. 2. The legacy `TestResourceKey` name, which is still supported and is checked after the aligned name. diff --git a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java index ce84df4d8..99b84cbb3 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java +++ b/ip-intelligence/ip-intelligence.cloud/src/main/java/fiftyone/ipintelligence/cloud/flowelements/IPIntelligenceCloudEngine.java @@ -63,8 +63,9 @@ */ public class IPIntelligenceCloudEngine extends CloudAspectEngineBase { - List aspectProperties; + volatile List aspectProperties; private String dataSourceTier; + private final Object metadataLock = new Object(); // Lower-cased names of properties the cloud returns as weighted lists // (cloud type "Weighted..."), so processEngine can read their value/weight // pairs rather than treating them as a plain list. @@ -84,9 +85,37 @@ public IPIntelligenceCloudEngine( this.cloudRequestEngine = null; } + /** + * The property metadata, from the products the Cloud Request Engine + * reports. Until they are known, after a failure at build that can + * pass, this is empty and is read again on the next call. + * @return the property metadata + */ @Override public List getProperties() { - return aspectProperties; + List loaded = aspectProperties; + if (loaded == null && cloudRequestEngine != null) { + synchronized (metadataLock) { + if (aspectProperties == null) { + loadAspectProperties(cloudRequestEngine); + } + loaded = aspectProperties; + } + } + return loaded == null ? + new ArrayList() : + loaded; + } + + /** + * Whether the property metadata is loaded, which reads it if the Cloud + * Request Engine now knows the accessible properties. + * @return true once the property metadata is loaded + */ + @Override + public boolean hasLoadedProperties() { + getProperties(); + return aspectProperties != null; } @Override @@ -123,7 +152,7 @@ protected void processEngine(FlowData data, IPIntelligenceDataCloud aspectData) // Extract data from json to the aspectData instance. JSONObject jsonObj = new JSONObject(json); - JSONObject ipiObj = jsonObj.getJSONObject("ip"); + JSONObject ipiObj = getElementJson(jsonObj); Map propertyMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); @@ -222,9 +251,10 @@ public void addPipeline(Pipeline pipeline) { if (cloudRequestEngine == null) { cloudRequestEngine = pipeline.getElement(CloudRequestEngine.class); if (cloudRequestEngine != null) { - if (loadAspectProperties(cloudRequestEngine) == false) { - logger.error("Failed to load aspect properties"); - } + // The products are not known after a failure at build that + // can pass, which the Cloud Request Engine has logged. + // getProperties reads them once they are. + loadAspectProperties(cloudRequestEngine); } } @@ -239,18 +269,16 @@ public void addPipeline(Pipeline pipeline) { * {@link #aspectProperties} */ private boolean loadAspectProperties(CloudRequestEngine engine) { - Map map = - engine.getPublicProperties(); + AccessiblePropertyMetaData.ProductMetaData product = + findProduct(engine.getPublicProperties()); - if (map != null && - map.size() > 0 && - map.containsKey(getElementDataKey())) { - aspectProperties = new ArrayList<>(); + if (product != null) { + List properties = new ArrayList<>(); weightedPropertyNames.clear(); - dataSourceTier = map.get(getElementDataKey()).dataTier; + dataSourceTier = product.dataTier; for (AccessiblePropertyMetaData.PropertyMetaData item : - map.get(getElementDataKey()).properties) { + product.properties) { if (item.type != null && item.type.startsWith("Weighted")) { weightedPropertyNames.add(item.name.toLowerCase()); } @@ -261,13 +289,12 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { resolvePropertyType(item), new ArrayList(), true); - aspectProperties.add(property); + properties.add(property); } + aspectProperties = properties; return true; } else { - logger.error("Aspect properties could not be loaded for" + - " the IP Intelligence cloud engine", this); return false; } } diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/ValueTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/ValueTests.java index 3b7ee8bb1..80be99739 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/ValueTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/ValueTests.java @@ -70,12 +70,25 @@ public static void valueTypes(WrapperCloud wrapper) throws Exception { assertNotNull("Value of " + property.getName() + " is null. ", value); assertTrue(AspectPropertyValue.class.isAssignableFrom(value.getClass())); + AspectPropertyValue propertyValue = + (AspectPropertyValue) value; + // A property the resource key is not entitled to, or one + // the client script populates, is listed but carries no + // value and says why instead. Asking such a value for its + // type throws, so what is checked here is that the reason + // was given, which is what a caller reads. + if (propertyValue.hasValue() == false) { + assertNotNull("Property '" + property.getName() + + "' has no value and no reason was given.", + propertyValue.getNoValueMessage()); + continue; + } assertTrue("Value of '" + property.getName() + - "' was of type " + ((AspectPropertyValue) value).getValue().getClass().getSimpleName() + + "' was of type " + propertyValue.getValue().getClass().getSimpleName() + " but should have been " + expectedType.getSimpleName() + ".", expectedType.isAssignableFrom( - ((AspectPropertyValue) value).getValue().getClass())); + propertyValue.getValue().getClass())); } } } diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/data/ValueCloudTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/data/ValueCloudTests.java index 83cb2b82d..fc703b82a 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/data/ValueCloudTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/data/ValueCloudTests.java @@ -27,10 +27,8 @@ import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; -@Ignore public class ValueCloudTests extends TestsBase { @Before diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java index 2e860d039..e212f527f 100644 --- a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/MissingPropertyHandlingTests.java @@ -33,7 +33,6 @@ import fiftyone.pipeline.engines.services.MissingPropertyReason; import fiftyone.pipeline.engines.services.MissingPropertyResult; import fiftyone.pipeline.engines.services.MissingPropertyService; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; @@ -46,7 +45,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -@Ignore public class MissingPropertyHandlingTests { protected static final ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); @@ -136,7 +134,7 @@ public void PropertyNotInResource() throws Exception { private static String nullValueJson = "{\n" + - " 'ip-intelligence': {\n" + + " 'ip': {\n" + " 'registeredname': 'Example Network',\n" + " 'registeredcountry': 'US',\n" + " 'latitude': null,\n" + diff --git a/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java new file mode 100644 index 000000000..a597f4662 --- /dev/null +++ b/ip-intelligence/ip-intelligence.cloud/src/test/java/fiftyone/ipintelligence/cloud/flowelements/ProductMissingTests.java @@ -0,0 +1,256 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.cloud.flowelements; + +import fiftyone.ipintelligence.cloud.data.IPIntelligenceDataCloud; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.Invocation; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A resource key can be valid and include no product that holds the + * IP Intelligence properties. The engine then builds with empty property + * metadata, logs the missing product once, and logs or throws nothing on + * each request. + */ +public class ProductMissingTests { + + private Logger logger; + private ILoggerFactory loggerFactory; + + @Before + public void init() { + logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void ProductMissing_BuildsWithEmptyMetadata() throws Exception { + try (IPIntelligenceCloudEngine engine = buildEngine()) { + assertNotNull(engine.getProperties()); + assertEquals(0, engine.getProperties().size()); + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 1, logged.size()); + assertTrue(logged.get(0), logged.get(0).startsWith("warn ")); + assertTrue(logged.get(0), logged.get(0).contains("'ip'")); + } + } + + @Test + public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { + try (IPIntelligenceCloudEngine engine = buildEngine()) { + for (int i = 0; i < 3; i++) { + process(engine); + } + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 1, logged.size()); + } + } + + /** + * With the IP Intelligence product included, a response that holds no + * part for it leaves the element data without values and throws + * nothing, as the Processing section of the cloud aspect engine + * specification says. + */ + @Test + public void ProductPresent_ResponseWithoutPart_NoValues() + throws Exception { + try (IPIntelligenceCloudEngine engine = buildEngine("ip", "registeredname")) { + IPIntelligenceDataCloud data = process( + engine, + "{ 'device': { 'hardwarevendor': 'Example' }, " + + "'javascriptProperties': [] }"); + assertFalse(data.getRegisteredName().hasValue()); + } + } + + /** + * Reading a property behaves as for any property the response does not + * carry. + */ + @Test + public void ProductMissing_PropertyIsMissing() throws Exception { + try (IPIntelligenceCloudEngine engine = buildEngine()) { + IPIntelligenceDataCloud data = process(engine); + try { + data.getRegisteredName(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals( + MissingPropertyReason.ProductNotAccessibleWithResourceKey, + e.getReason()); + } + } + } + + /** + * After a failure at build that can pass, the Cloud Request Engine does + * not know the accessible products yet. The engine builds without + * logging anything, and reads its metadata once the products are known + * rather than keeping none. + */ + @Test + public void DiscoveryNotComplete_LoadsTheMetadataLater() throws Exception { + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("registeredname", "String", "cat", null)); + Map products = new HashMap<>(); + products.put("ip", product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()) + .thenReturn(null) + .thenReturn(products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + try (IPIntelligenceCloudEngine engine = + new IPIntelligenceCloudEngineBuilder(loggerFactory).build()) { + engine.addPipeline(pipeline); + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 0, logged.size()); + + assertEquals(1, engine.getProperties().size()); + assertEquals("registeredname", engine.getProperties().get(0).getName()); + } + } + + /** + * An engine whose Cloud Request Engine reports one product, holding + * one string property. + */ + // thenReturn has a varargs overload, so a generic Map is an unchecked array. + @SuppressWarnings("unchecked") + private IPIntelligenceCloudEngine buildEngine( + String productKey, + String property) throws Exception { + IPIntelligenceCloudEngine engine = + new IPIntelligenceCloudEngineBuilder(loggerFactory).build(); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData(property, "String", "cat", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put(productKey, product); + + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + /** + * Process the given cloud response. + */ + private IPIntelligenceDataCloud process( + IPIntelligenceCloudEngine engine, + String response) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + + IPIntelligenceDataCloud data = new IPIntelligenceDataCloudInternal( + loggerFactory.getLogger( + IPIntelligenceDataCloud.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + engine.processEngine(flowData, data); + return data; + } + + /** + * An engine on a resource key that includes Device Detection alone. + */ + private IPIntelligenceCloudEngine buildEngine() throws Exception { + return buildEngine("device", "hardwarevendor"); + } + + /** + * Process a cloud response that holds nothing for the IP Intelligence + * product. + */ + private IPIntelligenceDataCloud process(IPIntelligenceCloudEngine engine) { + return process( + engine, + "{ 'device': { 'hardwarevendor': 'Example' }, " + + "'javascriptProperties': [] }"); + } + + /** + * Every warning and error logged, at any overload. + */ + private List warningsAndErrors() { + List logged = new ArrayList<>(); + for (Invocation invocation : + Mockito.mockingDetails(logger).getInvocations()) { + String level = invocation.getMethod().getName(); + if (level.equals("warn") || level.equals("error")) { + logged.add(level + " " + + Arrays.toString(invocation.getArguments())); + } + } + return Collections.unmodifiableList(logged); + } +} diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java index 6baf8af0e..6d1300fd8 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceDataHashDefault.java @@ -68,6 +68,12 @@ public class IPIntelligenceDataHashDefault */ private final List resultsList = new ArrayList<>(); + /** + * The engine which created this instance, used to turn a property name + * into the required property index the engine was built with. + */ + private final IPIntelligenceOnPremiseEngine ipiEngine; + /** * Constructs a new instance. * @param logger used for logging @@ -83,6 +89,18 @@ public class IPIntelligenceDataHashDefault IPIntelligenceOnPremiseEngine engine, MissingPropertyService missingPropertyService) { super(logger, flowData, engine, missingPropertyService); + this.ipiEngine = engine; + } + + /** + * Get the required property index for a property name, or null when the + * engine has no index for that name, so the read falls back to the name. + * @param propertyName name of the property + * @return required property index, or null + */ + private Integer indexOf(String propertyName) { + return ipiEngine == null ? + null : ipiEngine.getPropertyIndex(propertyName); } /** @@ -184,10 +202,13 @@ public AspectPropertyValue> getValues(String propertyName, return null; } Class valueType = parametrizedTypes[2]; + Integer index = indexOf(propertyName); if (valueType.equals(Boolean.class)) { populateWeightedValues( result, - () -> results.getValuesAsWeightedBoolList(propertyName), + () -> index == null ? + results.getValuesAsWeightedBoolList(propertyName) : + results.getValuesAsWeightedBoolList(index), WeightedBoolListValueSwig::hasValue, WeightedBoolListValueSwig::getNoValueMessage, WeightedBoolListValueSwig::getValue, @@ -196,7 +217,9 @@ public AspectPropertyValue> getValues(String propertyName, } else if (valueType.equals(Double.class)) { populateWeightedValues( result, - () -> results.getValuesAsWeightedDoubleList(propertyName), + () -> index == null ? + results.getValuesAsWeightedDoubleList(propertyName) : + results.getValuesAsWeightedDoubleList(index), WeightedDoubleListValueSwig::hasValue, WeightedDoubleListValueSwig::getNoValueMessage, WeightedDoubleListValueSwig::getValue, @@ -205,7 +228,9 @@ public AspectPropertyValue> getValues(String propertyName, } else if (valueType.equals(Float.class)) { populateWeightedValues( result, - () -> results.getValuesAsWeightedDoubleList(propertyName), + () -> index == null ? + results.getValuesAsWeightedDoubleList(propertyName) : + results.getValuesAsWeightedDoubleList(index), WeightedDoubleListValueSwig::hasValue, WeightedDoubleListValueSwig::getNoValueMessage, WeightedDoubleListValueSwig::getValue, @@ -214,7 +239,9 @@ public AspectPropertyValue> getValues(String propertyName, } else if (valueType.equals(Integer.class)) { populateWeightedValues( result, - () -> results.getValuesAsWeightedIntegerList(propertyName), + () -> index == null ? + results.getValuesAsWeightedIntegerList(propertyName) : + results.getValuesAsWeightedIntegerList(index), WeightedIntListValueSwig::hasValue, WeightedIntListValueSwig::getNoValueMessage, WeightedIntListValueSwig::getValue, @@ -225,7 +252,9 @@ public AspectPropertyValue> getValues(String propertyName, || valueType.equals(Inet6Address.class)) { populateWeightedValues( result, - () -> results.getValuesAsWeightedStringList(propertyName), + () -> index == null ? + results.getValuesAsWeightedStringList(propertyName) : + results.getValuesAsWeightedStringList(index), WeightedStringListValueSwig::hasValue, WeightedStringListValueSwig::getNoValueMessage, WeightedStringListValueSwig::getValue, @@ -240,7 +269,9 @@ public AspectPropertyValue> getValues(String propertyName, } else if (valueType.equals(String.class)) { populateWeightedValues( result, - () -> results.getValuesAsWeightedUTF8StringList(propertyName), + () -> index == null ? + results.getValuesAsWeightedUTF8StringList(propertyName) : + results.getValuesAsWeightedUTF8StringList(index), WeightedUTF8StringListValueSwig::hasValue, WeightedUTF8StringListValueSwig::getNoValueMessage, WeightedUTF8StringListValueSwig::getValue, @@ -267,9 +298,10 @@ protected AspectPropertyValue getValueAsString(String propertyName) { AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); if (results != null) { - - try (StringValueSwig value = results.getValueAsString( - propertyName)) { + Integer index = indexOf(propertyName); + try (StringValueSwig value = index == null ? + results.getValueAsString(propertyName) : + results.getValueAsString(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } @@ -288,8 +320,10 @@ protected AspectPropertyValue getValueAsJavaScript( new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); if (results != null) { - try (StringValueSwig value = results.getValueAsString( - propertyName)) { + Integer index = indexOf(propertyName); + try (StringValueSwig value = index == null ? + results.getValueAsString(propertyName) : + results.getValueAsString(index)) { if (value.hasValue()) { result.setValue(new JavaScript(value.getValue())); } @@ -307,8 +341,10 @@ protected AspectPropertyValue getValueAsInteger( AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); if (results != null) { - try (IntegerValueSwig value = results.getValueAsInteger( - propertyName)) { + Integer index = indexOf(propertyName); + try (IntegerValueSwig value = index == null ? + results.getValueAsInteger(propertyName) : + results.getValueAsInteger(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } @@ -325,8 +361,10 @@ protected AspectPropertyValue getValueAsBool(String propertyName) { AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); if (results != null) { - try (BoolValueSwig value = results.getValueAsBool( - propertyName)) { + Integer index = indexOf(propertyName); + try (BoolValueSwig value = index == null ? + results.getValueAsBool(propertyName) : + results.getValueAsBool(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } @@ -343,8 +381,10 @@ protected AspectPropertyValue getValueAsDouble(String propertyName) { AspectPropertyValue result = new AspectPropertyValueDefault<>(); ResultsIpiSwig results = getSingleResults(propertyName); if (results != null) { - try (DoubleValueSwig value = results.getValueAsDouble( - propertyName)) { + Integer index = indexOf(propertyName); + try (DoubleValueSwig value = index == null ? + results.getValueAsDouble(propertyName) : + results.getValueAsDouble(index)) { if (value.hasValue()) { result.setValue(value.getValue()); } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java index a0779d7dc..6955ef9e9 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngine.java @@ -62,6 +62,27 @@ public class IPIntelligenceOnPremiseEngine private volatile boolean propertiesPopulated = false; private final Random rand = new Random(); + /** + * Case insensitive map from each property name the engine was built with + * to its required property index, which is the place of the name in that + * list, the first where a name is given more than once, as the native + * engine gives it. Made from the names given at build and never from the + * data set, so an index keeps naming the same property after a reload + * from a data file whose property order differs. When the engine was + * built with no names it is made instead from the names the native engine + * reports after the first data file loads, which are the properties of + * that file sorted by name and which a reload does not change. + */ + private volatile Map propertyIndexes; + + /** + * True once {@link #propertyIndexes} holds the names the indexes are + * fixed on, which is at construction when the engine was built with + * names and after the first data file loads when it was not. A reload + * never makes the map again. + */ + private volatile boolean propertyIndexesFixed; + /** * Construct a new instance of the {@link IPIntelligenceOnPremiseEngine}. * @param logger logger instance to use for logging @@ -70,6 +91,8 @@ public class IPIntelligenceOnPremiseEngine * @param properties native required properties configuration which define * the properties which the engine should be initialised * with + * @param propertyNames the property names {@code properties} was made + * from, in the same order * @param ipiDataFactory the factory to use when creating a * {@link IPIntelligenceDataHash} instance * @param tempDataFileDir the file where a temporary data file copy @@ -80,11 +103,14 @@ public class IPIntelligenceOnPremiseEngine AspectEngineDataFile dataFile, ConfigIpiSwig config, RequiredPropertiesConfigSwig properties, + List propertyNames, ElementDataFactory ipiDataFactory, String tempDataFileDir) { super(logger, ipiDataFactory, tempDataFileDir); this.config = config; this.propertiesConfigSwig = properties; + this.propertyIndexes = indexesOf(propertyNames); + this.propertyIndexesFixed = this.propertyIndexes.isEmpty() == false; addDataFile(dataFile); } @@ -113,6 +139,55 @@ public MetaDataSwig getMetaData() { return engine.getMetaData(); } + /** + * Get the required property index for a property name, or null when the + * index list has no such name and the read has to go by name. That covers + * a name the engine was not built with, and a property only a data file + * loaded after the first has. + * @param propertyName name of the property + * @return required property index, or null + */ + Integer getPropertyIndex(String propertyName) { + return propertyIndexes.get(propertyName); + } + + /** + * Fix the indexes on the names the native engine reports, which it does + * only for an engine built with no names, and only after its first data + * file has loaded. Those names are the properties of that file sorted by + * name, and the native engine keeps them and their order through every + * later reload, so the map is made once and a reload never changes it. + */ + private void fixPropertyIndexes() { + if (propertyIndexesFixed == false) { + try (VectorStringSwig names = engine.getRequiredProperties()) { + propertyIndexes = indexesOf(new ArrayList<>(names)); + } + propertyIndexesFixed = true; + } + } + + /** + * Map each name to its place in the list, keeping the first place of a + * name given more than once in any letter case. + * @param names the property names in the order the engine is built with, + * or null + * @return unmodifiable case insensitive map from name to index + */ + private static Map indexesOf(List names) { + Map indexes = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + if (names != null) { + for (int i = 0; i < names.size(); i++) { + String name = names.get(i); + if (name != null && name.isEmpty() == false) { + indexes.putIfAbsent(name, i); + } + } + } + return Collections.unmodifiableMap(indexes); + } + @Override public List getProperties() { if (isFalse(propertiesPopulated)) { @@ -339,6 +414,7 @@ protected void unmanagedResourcesCleanup() { } private void setEngineMetaData() { + fixPropertyIndexes(); evidenceKeys = getKeysFromEngine(engine); evidenceKeyFilter = new EvidenceKeyFilterWhitelist( evidenceKeys, diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java index c48e88184..cf0b071fe 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/flowelements/IPIntelligenceOnPremiseEngineBuilder.java @@ -62,9 +62,27 @@ public class IPIntelligenceOnPremiseEngineBuilder private final String dataDownloadType = "IPIV41"; /** - * Native configuration instance for this engine. + * Native configuration instance for this engine, which starts from the + * Balanced performance profile. */ - private final ConfigIpiSwig config = new ConfigIpiSwig(); + private final ConfigIpiSwig config = balancedConfig(); + + /** + * A native configuration with the Balanced profile applied. The profile + * is applied here rather than left to the native library, so that a + * builder whose caller sets no profile is Balanced whatever + * configuration the native library starts from. A field initializer + * runs before the constructor sets the concurrency, which matters + * because applying a profile replaces the concurrency of every + * collection. + * + * @return a new Balanced configuration + */ + private static ConfigIpiSwig balancedConfig() { + ConfigIpiSwig config = new ConfigIpiSwig(); + config.setBalanced(); + return config; + } /** * Default constructor which uses the {@link ILoggerFactory} implementation @@ -239,6 +257,7 @@ protected IPIntelligenceOnPremiseEngine newEngine(List properties) { dataFile, config, requiredProperties, + properties, new IpiDataFactory(loggerFactory), tempDir); } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/swig/ConfigIpiSwig.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/swig/ConfigIpiSwig.java index e86fa537f..522943998 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/swig/ConfigIpiSwig.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/main/java/fiftyone/ipintelligence/engine/onpremise/interop/swig/ConfigIpiSwig.java @@ -81,7 +81,7 @@ public CollectionConfigSwig getStrings() { } public CollectionConfigSwig getComponents() { - return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getComponents(swigCPtr, this), true); + return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getComponents(swigCPtr, this), false); } public CollectionConfigSwig getMaps() { @@ -89,15 +89,15 @@ public CollectionConfigSwig getMaps() { } public CollectionConfigSwig getProperties() { - return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getProperties(swigCPtr, this), true); + return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getProperties(swigCPtr, this), false); } public CollectionConfigSwig getValues() { - return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getValues(swigCPtr, this), true); + return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getValues(swigCPtr, this), false); } public CollectionConfigSwig getProfiles() { - return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getProfiles(swigCPtr, this), true); + return new CollectionConfigSwig(IpIntelligenceOnPremiseEngineModuleJNI.ConfigIpiSwig_getProfiles(swigCPtr, this), false); } public CollectionConfigSwig getGraphs() { diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/BuilderTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/BuilderTests.java index 4d825688c..405a4a227 100644 --- a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/BuilderTests.java +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/BuilderTests.java @@ -22,12 +22,21 @@ package fiftyone.ipintelligence.engine.onpremise.flowelements; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.CollectionConfigSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ConfigIpiSwig; +import fiftyone.pipeline.engines.Constants.PerformanceProfiles; import fiftyone.pipeline.engines.configuration.CacheConfiguration; import org.junit.Before; import org.junit.Test; import org.slf4j.ILoggerFactory; import org.slf4j.LoggerFactory; +import java.lang.reflect.Field; +import java.util.Arrays; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertFalse; + public class BuilderTests { protected static final ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); @@ -46,4 +55,137 @@ public void init() { public void Builder_CacheDisabled() { builder.setCache(new CacheConfiguration(1000)); } + + /** + * Check that a builder whose caller sets no performance profile gives + * the engine the collection settings of the Balanced profile, and not + * the in-memory settings of MaxPerformance. + */ + @Test + public void Builder_NoProfile_CollectionsAreBalanced() throws Exception { + long[] given = collections(configOf(builder)); + + assertArrayEquals( + collections(PerformanceProfiles.Balanced), given); + assertFalse(Arrays.equals( + collections(PerformanceProfiles.MaxPerformance), given)); + } + + /** + * Check that a profile the caller names, in code or by name as a + * configuration file gives it, replaces the Balanced default. + * MaxPerformance and LowMemory give every collection the same settings, + * so this cannot tell those two apart. + */ + @Test + public void Builder_MaxPerformance_ReplacesBalanced() throws Exception { + long[] inMemory = collections(PerformanceProfiles.MaxPerformance); + + builder.setPerformanceProfile(PerformanceProfiles.MaxPerformance); + assertArrayEquals(inMemory, collections(configOf(builder))); + + IPIntelligenceOnPremiseEngineBuilder named = + new IPIntelligenceOnPremiseEngineBuilder(loggerFactory) + .setPerformanceProfile("MaxPerformance"); + assertArrayEquals(inMemory, collections(configOf(named))); + assertFalse(Arrays.equals( + collections(PerformanceProfiles.Balanced), inMemory)); + } + + /** + * Check that every collection the configuration hands out is borrowed. + * The collections belong to the configuration, so a proxy that owned + * one would free it a second time when the proxy is deleted. + */ + @Test + public void Config_CollectionsAreBorrowed() throws Exception { + ConfigIpiSwig config = new ConfigIpiSwig(); + try { + config.setBalanced(); + long[] before = collections(config); + CollectionConfigSwig[] all = { + config.getStrings(), + config.getComponents(), + config.getMaps(), + config.getProperties(), + config.getValues(), + config.getProfiles(), + config.getGraphs(), + config.getProfileGroups(), + config.getProfileOffsets(), + config.getPropertyTypes(), + config.getGraph() + }; + Field own = CollectionConfigSwig.class + .getDeclaredField("swigCMemOwn"); + own.setAccessible(true); + for (CollectionConfigSwig collection : all) { + assertFalse(own.getBoolean(collection)); + collection.delete(); + } + assertArrayEquals(before, collections(config)); + } finally { + config.delete(); + } + } + + /** + * The native configuration the builder hands to the engine it builds. + */ + private static ConfigIpiSwig configOf( + IPIntelligenceOnPremiseEngineBuilder builder) throws Exception { + Field field = IPIntelligenceOnPremiseEngineBuilder.class + .getDeclaredField("config"); + field.setAccessible(true); + return (ConfigIpiSwig) field.get(builder); + } + + /** + * The collection settings the native library gives the profile. + */ + private static long[] collections(PerformanceProfiles profile) { + ConfigIpiSwig config = new ConfigIpiSwig(); + try { + switch (profile) { + case Balanced: + config.setBalanced(); + break; + case MaxPerformance: + config.setMaxPerformance(); + break; + default: + throw new IllegalArgumentException(profile.name()); + } + return collections(config); + } finally { + config.delete(); + } + } + + /** + * The number of entries loaded at the start and the cache capacity of + * every collection in the configuration, in a fixed order. Concurrency + * is left out, because the builder sets its own. + */ + private static long[] collections(ConfigIpiSwig config) { + CollectionConfigSwig[] all = { + config.getStrings(), + config.getComponents(), + config.getMaps(), + config.getProperties(), + config.getValues(), + config.getProfiles(), + config.getGraphs(), + config.getProfileGroups(), + config.getProfileOffsets(), + config.getPropertyTypes(), + config.getGraph() + }; + long[] settings = new long[all.length * 2]; + for (int i = 0; i < all.length; i++) { + settings[i * 2] = all[i].getLoaded(); + settings[i * 2 + 1] = all[i].getCapacity(); + } + return settings; + } } diff --git a/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexTests.java b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexTests.java new file mode 100644 index 000000000..9ae3290c2 --- /dev/null +++ b/ip-intelligence/ip-intelligence.engine.on-premise/src/test/java/fiftyone/ipintelligence/engine/onpremise/flowelements/PropertyIndexTests.java @@ -0,0 +1,258 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.engine.onpremise.flowelements; + +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ConfigIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.EngineIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.RequiredPropertiesConfigSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ResultsIpiSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.VectorStringSwig; +import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.fiftyone.data.FiftyOneAspectPropertyMetaData; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static fiftyone.ipintelligence.shared.testhelpers.FileUtils.LITE_IPI_DATA_FILE_NAME; +import static fiftyone.pipeline.util.FileFinder.getFilePath; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * Tests for the property index rule on the IP Intelligence on-premise + * engine, which says that a property's index is its place in the list of + * names the engine was built with, that the index is fixed at build, and that + * every read by name goes through that index. + */ +public class PropertyIndexTests { + + private static final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + /** + * Names in an order that is deliberately not the order the data file + * holds them in, so an index taken from the data file would be wrong. + */ + private static final List NAMES = Arrays.asList( + "RegisteredCountry", + "AccuracyRadiusMin", + "RegisteredName", + "Latitude"); + + private static final String ADDRESS = "185.28.167.77"; + + private static File dataFile(String name) { + try { + return getFilePath(name); + } + catch (IllegalArgumentException e) { + assumeTrue(e.getMessage(), false); + return null; + } + } + + private static IPIntelligenceOnPremiseEngine build( + File file, List names) throws Exception { + IPIntelligenceOnPremiseEngineBuilder builder = + new IPIntelligenceOnPremiseEngineBuilder(loggerFactory, null) + .setPerformanceProfile( + Constants.PerformanceProfiles.MaxPerformance) + .setAutoUpdate(false) + .setDataFileSystemWatcher(false); + if (names != null) { + builder.setProperties(names); + } + return builder.build(file.toString(), false); + } + + /** + * Every name the engine was built with has an index, and the index is the + * place of the name in the build time list, not its place in the data + * file. + */ + @Test + public void PropertyIndex_FromBuildTimeNames() throws Exception { + File file = dataFile(LITE_IPI_DATA_FILE_NAME); + try (IPIntelligenceOnPremiseEngine engine = build(file, NAMES)) { + for (int i = 0; i < NAMES.size(); i++) { + Integer index = engine.getPropertyIndex(NAMES.get(i)); + assertNotNull( + "Expected an index for '" + NAMES.get(i) + "'.", + index); + assertEquals( + "Expected '" + NAMES.get(i) + "' at index " + i + ".", + Integer.valueOf(i), + index); + } + } + } + + /** + * The lookup is case insensitive, because a caller may ask for a property + * in any casing. + */ + @Test + public void PropertyIndex_CaseInsensitive() throws Exception { + File file = dataFile(LITE_IPI_DATA_FILE_NAME); + try (IPIntelligenceOnPremiseEngine engine = build(file, NAMES)) { + assertEquals( + engine.getPropertyIndex("RegisteredName"), + engine.getPropertyIndex("registeredname")); + } + } + + /** + * A name given twice, in any letter case, has the index of its first + * place, as the native engine gives it. + */ + @Test + public void PropertyIndex_NameGivenTwiceHasFirstIndex() throws Exception { + File file = dataFile(LITE_IPI_DATA_FILE_NAME); + try (IPIntelligenceOnPremiseEngine engine = build(file, + Arrays.asList("RegisteredName", "Latitude", "registeredname"))) { + assertEquals( + Integer.valueOf(0), + engine.getPropertyIndex("RegisteredName")); + assertEquals( + Integer.valueOf(1), + engine.getPropertyIndex("Latitude")); + } + } + + /** + * A name the engine was not built with has no index, so the read goes by + * name. + */ + @Test + public void PropertyIndex_UnknownNameHasNoIndex() throws Exception { + File file = dataFile(LITE_IPI_DATA_FILE_NAME); + try (IPIntelligenceOnPremiseEngine engine = build(file, NAMES)) { + assertNull(engine.getPropertyIndex("NotAPropertyAtAll")); + } + } + + /** + * An engine built with no names has an index for every property of the + * first data file it loads, no two properties share an index, and the + * native results name that property at that index. + */ + @Test + public void PropertyIndex_NoNamesFromFirstFile() throws Exception { + File file = dataFile(LITE_IPI_DATA_FILE_NAME); + try (IPIntelligenceOnPremiseEngine engine = build(file, null); + NativeEngine nativeEngine = new NativeEngine(file); + ResultsIpiSwig results = nativeEngine.engine.process(ADDRESS)) { + List names = propertyNames(engine); + assertTrue("Expected the data file to have properties.", + names.isEmpty() == false); + Set seen = new HashSet<>(); + for (String name : names) { + Integer index = engine.getPropertyIndex(name); + assertNotNull("Expected an index for '" + name + "'.", index); + assertTrue( + "Index " + index + " given to more than one name.", + seen.add(index)); + assertEquals( + "Expected the native results to name '" + name + + "' at index " + index + ".", + name, + results.getPropertyName(index)); + } + } + } + + /** + * An engine built with no names keeps the indexes it took from its first + * data file through a reload, because they are made once and not from + * the reloaded data set. + */ + @Test + public void PropertyIndex_NoNamesSurvivesReload() throws Exception { + File file = dataFile(LITE_IPI_DATA_FILE_NAME); + try (IPIntelligenceOnPremiseEngine engine = build(file, null)) { + Map before = new HashMap<>(); + for (String name : propertyNames(engine)) { + Integer index = engine.getPropertyIndex(name); + assertNotNull("Expected an index for '" + name + "'.", index); + before.put(name, index); + } + engine.refreshData(engine.getDataFiles().get(0).getIdentifier()); + for (Map.Entry entry : before.entrySet()) { + assertEquals( + "Index of '" + entry.getKey() + "' moved on reload.", + entry.getValue(), + engine.getPropertyIndex(entry.getKey())); + } + } + } + + private static List propertyNames( + IPIntelligenceOnPremiseEngine engine) { + List names = new ArrayList<>(); + for (FiftyOneAspectPropertyMetaData property : + engine.getProperties()) { + names.add(property.getName()); + } + return names; + } + + /** + * A native engine on its own with no names, so a test can read the + * native results directly. + */ + private static class NativeEngine implements AutoCloseable { + private final ConfigIpiSwig config = new ConfigIpiSwig(); + private final RequiredPropertiesConfigSwig required; + private final EngineIpiSwig engine; + + NativeEngine(File file) { + config.setMaxPerformance(); + RequiredPropertiesConfigSwig made; + try (VectorStringSwig vector = new VectorStringSwig()) { + made = new RequiredPropertiesConfigSwig(vector); + } + required = made; + engine = new EngineIpiSwig( + file.getAbsolutePath(), config, required); + } + + @Override + public void close() { + engine.delete(); + required.delete(); + config.delete(); + } + } +} diff --git a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java index a39dd6b80..ded023de3 100644 --- a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java +++ b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/FileUtils.java @@ -32,6 +32,7 @@ import java.nio.file.Path; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import static fiftyone.pipeline.util.FileFinder.getFilePath; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; @@ -43,8 +44,21 @@ public class FileUtils { * supply an explicit path to the IPI data file. When set and the file * exists it is used in preference to searching the project space. */ + public static final String IPI_PATH_VARIABLE = "_51DEGREES_IPI_PATH"; + + /** + * The old name of {@link #IPI_PATH_VARIABLE}, read only when that one + * is not set. + * @deprecated use {@link #IPI_PATH_VARIABLE}, and set the variable it + * names in place of this one. + */ + @Deprecated public static final String IPI_PATH_ENV_VAR = "51DEGREES_IPI_PATH"; + // Set once the use of the deprecated name has been reported. + private static final AtomicBoolean OLD_NAME_REPORTED = + new AtomicBoolean(false); + public static final String ENTERPRISE_IPI_DATA_FILE_NAME = "Enterprise-IpIntelligenceV41.ipi"; public static final String ENTERPRISE_IPI_V41_DATA_FILE_NAME = "51Degrees-EnterpriseIpiV41.ipi"; @@ -99,15 +113,26 @@ public static File getHashFile() { } /** - * Check the aligned {@link #IPI_PATH_ENV_VAR} environment variable and - * system property for an explicit path to the IPI data file. + * Check the aligned {@link #IPI_PATH_VARIABLE} environment variable and + * system property for an explicit path to the IPI data file, and when + * neither is set the deprecated {@link #IPI_PATH_ENV_VAR}, whose first + * use is reported on standard error. * * @return a file if the path is set and the file exists, otherwise null */ private static File getExplicitHashFile() { - String explicitPath = System.getenv(IPI_PATH_ENV_VAR); + String explicitPath = environmentOrProperty(IPI_PATH_VARIABLE); if (Objects.isNull(explicitPath)) { - explicitPath = System.getProperty(IPI_PATH_ENV_VAR); + explicitPath = environmentOrProperty(IPI_PATH_ENV_VAR); + if (Objects.nonNull(explicitPath) + && OLD_NAME_REPORTED.compareAndSet(false, true)) { + System.err.println( + "The environment variable " + IPI_PATH_ENV_VAR + + " is deprecated and " + IPI_PATH_VARIABLE + + " replaces it. Set " + IPI_PATH_VARIABLE + + " to the same value, as a later release may no longer " + + "read " + IPI_PATH_ENV_VAR + "."); + } } if (Objects.nonNull(explicitPath)) { File explicitFile = new File(explicitPath); @@ -118,6 +143,19 @@ private static File getExplicitHashFile() { return null; } + /** + * The value of the environment variable of this name, or of the system + * property where that is not set or is empty, or null when neither + * holds a value. + */ + private static String environmentOrProperty(String name) { + String value = System.getenv(name); + if (Objects.isNull(value) || value.isEmpty()) { + value = System.getProperty(name); + } + return Objects.isNull(value) || value.isEmpty() ? null : value; + } + public static File getEvidenceFile() { if (Objects.nonNull(EVIDENCE_FILE)) { return EVIDENCE_FILE.orElse(null); diff --git a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java index cabb6648c..75533a5ef 100644 --- a/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java +++ b/ip-intelligence/ip-intelligence.shared/src/main/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtils.java @@ -24,6 +24,7 @@ import java.util.Base64; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; /** * Helpers to obtain keys from the environment @@ -33,19 +34,48 @@ public class KeyUtils { * Name of the aligned environment variable or system property which * supplies the resource key. This name is checked before any legacy name. */ + public static final String RESOURCE_KEY_VARIABLE = "_51DEGREES_RESOURCE_KEY"; + + /** + * The old name of {@link #RESOURCE_KEY_VARIABLE}, read only when that + * one is not set. + * @deprecated use {@link #RESOURCE_KEY_VARIABLE}, and set the variable + * it names in place of this one. + */ + @Deprecated public static final String RESOURCE_KEY_ENV_VAR = "51DEGREES_RESOURCE_KEY"; + // Set once the use of the deprecated name has been reported. + private static final AtomicBoolean OLD_NAME_REPORTED = + new AtomicBoolean(false); + /** * Obtain a resource key from an environment variable or a system property. *

    - * The aligned name {@link #RESOURCE_KEY_ENV_VAR} is checked first and the - * supplied legacy name is checked second. Each name is looked up using + * The aligned name {@link #RESOURCE_KEY_VARIABLE} is checked first. When + * it is not set or is empty the deprecated {@link #RESOURCE_KEY_ENV_VAR} + * is checked, and its first use is reported on standard error. The + * supplied legacy name is checked last. Each name is looked up using * {@link #getNamedKey(String)}. * @param legacyKeyName legacy name to try when the aligned name is not set - * @return the resource key, or null if neither name is set + * @return the resource key, or null if no name is set */ public static String getResourceKey(String legacyKeyName) { - String resourceKey = getNamedKey(RESOURCE_KEY_ENV_VAR); + String resourceKey = getNamedKey(RESOURCE_KEY_VARIABLE); + if (Objects.isNull(resourceKey) || resourceKey.isEmpty()) { + String oldNameKey = getNamedKey(RESOURCE_KEY_ENV_VAR); + if (Objects.nonNull(oldNameKey) && oldNameKey.isEmpty() == false) { + if (OLD_NAME_REPORTED.compareAndSet(false, true)) { + System.err.println( + "The environment variable " + RESOURCE_KEY_ENV_VAR + + " is deprecated and " + RESOURCE_KEY_VARIABLE + + " replaces it. Set " + RESOURCE_KEY_VARIABLE + + " to the same value, as a later release may no longer " + + "read " + RESOURCE_KEY_ENV_VAR + "."); + } + return oldNameKey; + } + } if (Objects.isNull(resourceKey)) { resourceKey = getNamedKey(legacyKeyName); } diff --git a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/FileUtilsTest.java b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/FileUtilsTest.java index bb1b302b7..b1478996b 100644 --- a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/FileUtilsTest.java +++ b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/FileUtilsTest.java @@ -30,6 +30,7 @@ import java.io.IOException; import static org.junit.Assert.*; +import static org.junit.Assume.assumeTrue; public class FileUtilsTest { @@ -47,5 +48,69 @@ public void testUserDir() throws IOException { assertEquals(". and user dir should be the same", f.getCanonicalPath(), g.getCanonicalPath()); } + private static final String NEW_PATH_NAME = "_51DEGREES_IPI_PATH"; + private static final String OLD_PATH_NAME = "51DEGREES_IPI_PATH"; -} \ No newline at end of file + /** + * System properties stand in for the environment variables, because + * the lookup reads either and a test cannot set an environment + * variable. + */ + @Test + public void oldPathNameIsReadWhenNewIsNotSet() throws IOException { + assumePathNamesNotInEnvironment(); + String savedNew = System.clearProperty(NEW_PATH_NAME); + String savedOld = System.clearProperty(OLD_PATH_NAME); + File file = File.createTempFile("old-path-name", ".ipi"); + try { + System.setProperty(OLD_PATH_NAME, file.getPath()); + assertEquals(file, FileUtils.getHashFile()); + } finally { + restore(NEW_PATH_NAME, savedNew); + restore(OLD_PATH_NAME, savedOld); + file.delete(); + } + } + + @Test + public void newPathNameWinsOverOld() throws IOException { + assumePathNamesNotInEnvironment(); + String savedNew = System.clearProperty(NEW_PATH_NAME); + String savedOld = System.clearProperty(OLD_PATH_NAME); + File newFile = File.createTempFile("new-path-name", ".ipi"); + File oldFile = File.createTempFile("old-path-name", ".ipi"); + try { + System.setProperty(NEW_PATH_NAME, newFile.getPath()); + System.setProperty(OLD_PATH_NAME, oldFile.getPath()); + assertEquals(newFile, FileUtils.getHashFile()); + } finally { + restore(NEW_PATH_NAME, savedNew); + restore(OLD_PATH_NAME, savedOld); + newFile.delete(); + oldFile.delete(); + } + } + + @Test + @SuppressWarnings("deprecation") + public void oldPathConstantKeepsItsValue() throws Exception { + assertEquals(OLD_PATH_NAME, FileUtils.IPI_PATH_ENV_VAR); + assertTrue(FileUtils.class.getField("IPI_PATH_ENV_VAR") + .isAnnotationPresent(Deprecated.class)); + assertEquals(NEW_PATH_NAME, + FileUtils.class.getField("IPI_PATH_VARIABLE").get(null)); + } + + private static void assumePathNamesNotInEnvironment() { + assumeTrue(System.getenv(NEW_PATH_NAME) == null); + assumeTrue(System.getenv(OLD_PATH_NAME) == null); + } + + private static void restore(String name, String value) { + if (value == null) { + System.clearProperty(name); + } else { + System.setProperty(name, value); + } + } +} diff --git a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtilsTest.java b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtilsTest.java index f102146b9..03a7d5341 100644 --- a/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtilsTest.java +++ b/ip-intelligence/ip-intelligence.shared/src/test/java/fiftyone/ipintelligence/shared/testhelpers/KeyUtilsTest.java @@ -25,6 +25,7 @@ import org.junit.Test; import static org.junit.Assert.*; +import static org.junit.Assume.assumeTrue; public class KeyUtilsTest { @Test @@ -33,4 +34,65 @@ public void testGetEnv(){ System.out.println(KeyUtils.getNamedKey("path")); } -} \ No newline at end of file + private static final String NEW_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; + private static final String OLD_KEY_NAME = "51DEGREES_RESOURCE_KEY"; + + /** + * System properties stand in for the environment variables, because + * the lookup reads either and a test cannot set an environment + * variable. + */ + @Test + public void oldKeyNameIsReadWhenNewIsNotSet() { + assumeKeyNamesNotInEnvironment(); + String savedNew = System.clearProperty(NEW_KEY_NAME); + String savedOld = System.clearProperty(OLD_KEY_NAME); + try { + System.setProperty(OLD_KEY_NAME, "old-name-key"); + assertEquals("old-name-key", + KeyUtils.getResourceKey("NoSuchLegacyName")); + } finally { + restore(NEW_KEY_NAME, savedNew); + restore(OLD_KEY_NAME, savedOld); + } + } + + @Test + public void newKeyNameWinsOverOld() { + assumeKeyNamesNotInEnvironment(); + String savedNew = System.clearProperty(NEW_KEY_NAME); + String savedOld = System.clearProperty(OLD_KEY_NAME); + try { + System.setProperty(NEW_KEY_NAME, "new-name-key"); + System.setProperty(OLD_KEY_NAME, "old-name-key"); + assertEquals("new-name-key", + KeyUtils.getResourceKey("NoSuchLegacyName")); + } finally { + restore(NEW_KEY_NAME, savedNew); + restore(OLD_KEY_NAME, savedOld); + } + } + + @Test + @SuppressWarnings("deprecation") + public void oldKeyConstantKeepsItsValue() throws Exception { + assertEquals(OLD_KEY_NAME, KeyUtils.RESOURCE_KEY_ENV_VAR); + assertTrue(KeyUtils.class.getField("RESOURCE_KEY_ENV_VAR") + .isAnnotationPresent(Deprecated.class)); + assertEquals(NEW_KEY_NAME, + KeyUtils.class.getField("RESOURCE_KEY_VARIABLE").get(null)); + } + + private static void assumeKeyNamesNotInEnvironment() { + assumeTrue(System.getenv(NEW_KEY_NAME) == null); + assumeTrue(System.getenv(OLD_KEY_NAME) == null); + } + + private static void restore(String name, String value) { + if (value == null) { + System.clearProperty(name); + } else { + System.setProperty(name, value); + } + } +} diff --git a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryBuildersConfigurationTest.java b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryBuildersConfigurationTest.java new file mode 100644 index 000000000..b959c506c --- /dev/null +++ b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryBuildersConfigurationTest.java @@ -0,0 +1,95 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence.translation; + +import fiftyone.ipintelligence.translation.flowelements.CountriesTranslationEngine; +import fiftyone.ipintelligence.translation.flowelements.CountryCodeTranslationEngine; +import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.translation.flowelements.TranslationEngine; +import fiftyone.pipeline.translation.flowelements.TranslationEngineBuilder; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests that a configuration file still builds the country translation + * engines with their own builders when the general Translation Engine + * builder can be named too, and a prepared one is given. + */ +public class CountryBuildersConfigurationTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + @SuppressWarnings("rawtypes") + public void countryBuilders_AreUnaffectedByAPreparedGeneralBuilder() + throws Exception { + String xml = "" + + "" + + "CountryCodeTranslationEngine" + + "" + + "" + + "CountriesTranslationEngine" + + "" + + "" + + "TranslationEngine" + + "" + + "ip" + + "" + + "" + + ""; + File file = folder.newFile("pipeline.xml"); + Files.write(file.toPath(), xml.getBytes(StandardCharsets.UTF_8)); + TranslationEngineBuilder general = + new TranslationEngineBuilder(LoggerFactory.getILoggerFactory()) + .addSource("countries.fr_FR.yml", "England: Angleterre") + .addTranslation("Country", "CountryTranslated"); + + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(general) + .buildFromConfiguration( + PipelineOptionsFactory.getOptionsFromFile(file))) { + List elements = pipeline.getFlowElements(); + assertEquals(3, elements.size()); + assertTrue(elements.get(0).getClass().getName(), + elements.get(0) instanceof CountryCodeTranslationEngine); + assertTrue(elements.get(1).getClass().getName(), + elements.get(1) instanceof CountriesTranslationEngine); + assertTrue(elements.get(2).getClass().getName(), + elements.get(2) instanceof TranslationEngine); + } + } +} diff --git a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java index a5eb58df6..3f5ab406b 100644 --- a/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java +++ b/ip-intelligence/ip-intelligence.translation/src/test/java/fiftyone/ipintelligence/translation/CountryTranslationIntegrationTest.java @@ -50,7 +50,8 @@ * complete, localized, ordered country dropdown is produced. *

    * The data file is resolved from the {@code TestDataFile} system property, the - * {@code 51DEGREES_IPI_PATH} environment variable, or a few well known + * {@code _51DEGREES_IPI_PATH} environment variable (or its deprecated + * name {@code 51DEGREES_IPI_PATH} when that is not set), or a few well known * locations, and the test is skipped if none is found. *

    * The IP Intelligence element data key has differed between engine versions @@ -63,10 +64,38 @@ public class CountryTranslationIntegrationTest { private static final String SAMPLE_IP = "8.8.8.8"; + private static final String IPI_PATH_VARIABLE = "_51DEGREES_IPI_PATH"; + private static final String OLD_IPI_PATH_VARIABLE = "51DEGREES_IPI_PATH"; + + private static boolean oldNameReported = false; + + /** + * The path the environment gives, from the deprecated name when the + * name that replaces it is not set, saying so on standard error the + * first time. + */ + private static String pathFromEnvironment() { + String path = System.getenv(IPI_PATH_VARIABLE); + if (path == null || path.isEmpty()) { + path = System.getenv(OLD_IPI_PATH_VARIABLE); + if (path != null && path.isEmpty() == false + && oldNameReported == false) { + oldNameReported = true; + System.err.println( + "The environment variable " + OLD_IPI_PATH_VARIABLE + + " is deprecated and " + IPI_PATH_VARIABLE + + " replaces it. Set " + IPI_PATH_VARIABLE + + " to the same value, as a later release may no longer " + + "read " + OLD_IPI_PATH_VARIABLE + "."); + } + } + return path; + } + private static String resolveDataFile() { String[] candidates = { System.getProperty("TestDataFile"), - System.getenv("51DEGREES_IPI_PATH"), + pathFromEnvironment(), "51Degrees-IPIV4EnterpriseIpiV41.ipi", "../51Degrees-IPIV4EnterpriseIpiV41.ipi", "../../51Degrees-IPIV4EnterpriseIpiV41.ipi", diff --git a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java index 0ebff7a84..210fa80dd 100644 --- a/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java +++ b/ip-intelligence/ip-intelligence/src/main/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilder.java @@ -61,8 +61,11 @@ public class IPIntelligenceOnPremisePipelineBuilder private String dataUpdateUrl = null; private Boolean dataUpdateVerifyMd5 = null; private DataUpdateUrlFormatter dataUpdateUrlFormatter = null; - private Constants.PerformanceProfiles performanceProfile = - Constants.PerformanceProfiles.MaxPerformance; + /** + * The profile the caller chose, or null for the engine builder's + * default, which is Balanced. + */ + private Constants.PerformanceProfiles performanceProfile = null; private DataUpdateService dataUpdateService; @@ -240,7 +243,8 @@ public IPIntelligenceOnPremisePipelineBuilder setDataUpdateLicenseKey( /** * Set the performance profile for the IP Intelligence engine. - * Defaults to MaxPerformance. + * Defaults to Balanced, which reads the data file as lookups need it, + * through caches. MaxPerformance loads the whole data file into memory. * @param profile The performance profile to use. * @return This builder instance. */ @@ -386,8 +390,13 @@ private IPIntelligenceOnPremiseEngine configureAndBuild( builder.setProperty(property); } } - // Configure performance profile - builder.setPerformanceProfile(performanceProfile); + // Configure the performance profile only when the caller chose one. + // The engine builder already starts from Balanced, and applying + // Balanced again here would also replace the concurrency the engine + // builder set. + if (performanceProfile != null) { + builder.setPerformanceProfile(performanceProfile); + } // Configure the concurrency if (concurrency > 1) { diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java index d220eb776..9e37ada63 100644 --- a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/IPIntelligenceOnPremisePipelineBuilderTest.java @@ -65,6 +65,7 @@ public void setup() throws Exception { when(loggerFactory.getLogger(anyString())).thenReturn(logger); builder = new IPIntelligencePipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) .setShareUsage(false); } @@ -121,6 +122,7 @@ public void setFilename_OtherExtension_ExceptionThrown() { public void setShareUsage_False_SharedFlowElementNotAdded() throws Exception { try (Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) .setShareUsage(false) .build()) { List flowElements = pipeline.getFlowElements(); @@ -133,6 +135,7 @@ public void setShareUsage_False_SharedFlowElementNotAdded() throws Exception { public void setShareUsage_True_SharedFlowElementAdded() throws Exception { try (Pipeline pipeline = new IPIntelligencePipelineBuilder(loggerFactory) .useOnPremise(datafile.getAbsolutePath(), false) + .setAutoUpdate(false) .setShareUsage(true) .build()) { List flowElements = pipeline.getFlowElements(); @@ -145,6 +148,7 @@ public void setShareUsage_True_SharedFlowElementAdded() throws Exception { public void setAutoUpdate_PropertyPropagatedToDataFileConfiguration() throws Exception { try (Pipeline pipeline = builder .setAutoUpdate(true) + .setDataUpdateLicenseKey("licence_key") .build()) { DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); assertTrue(dataFileConfiguration.getAutomaticUpdatesEnabled()); @@ -166,6 +170,7 @@ public void setDataUpdateService_RegisterDataFileTriggered() throws Exception { DataUpdateServiceDefault dataUpdateServiceDefault = mock(DataUpdateServiceDefault.class); try (Pipeline ignored = builder .setAutoUpdate(true) + .setDataUpdateLicenseKey("licence_key") .setDataUpdateService(dataUpdateServiceDefault) .build()) { verify(dataUpdateServiceDefault).registerDataFile(any()); @@ -176,6 +181,8 @@ public void setDataUpdateService_RegisterDataFileTriggered() throws Exception { public void setDataUpdateOnStartup_PropertyPropagatedToDataFileConfiguration() throws Exception { try (Pipeline pipeline = builder .setDataUpdateOnStartup(true) + .setDataUpdateLicenseKey("licence_key") + .setDataUpdateService(mock(DataUpdateServiceDefault.class)) .build()) { DataFileConfiguration dataFileConfiguration = getDataFileConfiguration(pipeline); assertTrue(dataFileConfiguration.getUpdateOnStartup()); diff --git a/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java new file mode 100644 index 000000000..c7fb46d09 --- /dev/null +++ b/ip-intelligence/ip-intelligence/src/test/java/fiftyone/ipintelligence/OnPremisePerformanceProfileTests.java @@ -0,0 +1,193 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.ipintelligence; + +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngine; +import fiftyone.ipintelligence.engine.onpremise.flowelements.IPIntelligenceOnPremiseEngineBuilder; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.CollectionConfigSwig; +import fiftyone.ipintelligence.engine.onpremise.interop.swig.ConfigIpiSwig; +import fiftyone.ipintelligence.shared.testhelpers.FileUtils; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.Constants.PerformanceProfiles; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.Arrays; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * Tests that the on-premise pipeline gives its engine the Balanced + * performance profile unless the caller names another, by reading the + * native configuration the engine was built with. + */ +public class OnPremisePerformanceProfileTests { + + private static File dataFile; + + @BeforeClass + public static void findDataFile() { + dataFile = FileUtils.getHashFile(); + } + + @Before + public void requireDataFile() { + Assume.assumeTrue( + "No IP Intelligence data file was found, so this test is " + + "skipped. Set " + FileUtils.IPI_PATH_VARIABLE + " to the path " + + "of an .ipi file, or download the Lite file with " + + "cxx/data/get-data-files.", + dataFile != null); + } + + @Test + public void build_NoProfile_EngineIsGivenBalanced() throws Exception { + try (Pipeline pipeline = builder().build()) { + ConfigIpiSwig given = configOf(pipeline); + + assertArrayEquals( + collections(PerformanceProfiles.Balanced), + collections(given)); + assertFalse(Arrays.equals( + collections(PerformanceProfiles.MaxPerformance), + collections(given))); + // Applying a profile replaces the concurrency, so this checks + // the pipeline did not apply one over the engine builder's. + assertEquals( + configOf(new IPIntelligenceOnPremiseEngineBuilder()) + .getConcurrency(), + given.getConcurrency()); + } + } + + /** + * MaxPerformance and LowMemory give every collection the same settings, + * so this checks only that the profile named replaced Balanced. + */ + @Test + public void build_LowMemory_ReplacesBalanced() throws Exception { + try (Pipeline pipeline = builder() + .setPerformanceProfile(PerformanceProfiles.LowMemory) + .build()) { + assertArrayEquals( + collections(PerformanceProfiles.LowMemory), + collections(configOf(pipeline))); + } + } + + /** + * A pipeline builder with the profile and the concurrency left at their + * defaults. Data updates and usage sharing are switched off, so that + * the test needs no licence key and no network, and neither touches + * the profile. + */ + private static IPIntelligenceOnPremisePipelineBuilder builder() + throws Exception { + return new IPIntelligencePipelineBuilder( + LoggerFactory.getILoggerFactory()) + .useOnPremise(dataFile.getAbsolutePath(), false) + .setAutoUpdate(false) + .setDataUpdateOnStartup(false) + .setShareUsage(false); + } + + /** The native configuration the pipeline's engine was built with. */ + private static ConfigIpiSwig configOf(Pipeline pipeline) + throws Exception { + return (ConfigIpiSwig) field( + IPIntelligenceOnPremiseEngine.class, "config") + .get(pipeline.getElement(IPIntelligenceOnPremiseEngine.class)); + } + + /** The native configuration an engine builder would hand on. */ + private static ConfigIpiSwig configOf( + IPIntelligenceOnPremiseEngineBuilder builder) throws Exception { + return (ConfigIpiSwig) field( + IPIntelligenceOnPremiseEngineBuilder.class, "config") + .get(builder); + } + + private static Field field(Class type, String name) + throws NoSuchFieldException { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field; + } + + /** The collection settings the native library gives the profile. */ + private static long[] collections(PerformanceProfiles profile) { + ConfigIpiSwig config = new ConfigIpiSwig(); + try { + switch (profile) { + case Balanced: + config.setBalanced(); + break; + case MaxPerformance: + config.setMaxPerformance(); + break; + case LowMemory: + config.setLowMemory(); + break; + default: + throw new IllegalArgumentException(profile.name()); + } + return collections(config); + } finally { + config.delete(); + } + } + + /** + * The number of entries loaded at the start and the cache capacity of + * every collection in the configuration, in a fixed order. Concurrency + * is left out, because the engine builder sets its own. + */ + private static long[] collections(ConfigIpiSwig config) { + CollectionConfigSwig[] all = { + config.getStrings(), + config.getComponents(), + config.getMaps(), + config.getProperties(), + config.getValues(), + config.getProfiles(), + config.getGraphs(), + config.getProfileGroups(), + config.getProfileOffsets(), + config.getPropertyTypes(), + config.getGraph() + }; + long[] settings = new long[all.length * 2]; + for (int i = 0; i < all.length; i++) { + settings[i * 2] = all[i].getLoaded(); + settings[i * 2 + 1] = all[i].getCapacity(); + } + return settings; + } +} diff --git a/location/LICENSE b/location/LICENSE deleted file mode 100644 index 43159c60b..000000000 --- a/location/LICENSE +++ /dev/null @@ -1,298 +0,0 @@ -/* ********************************************************************* -* This Original Work is copyright of 51 Degrees Mobile Experts Limited. -* Copyright 2019 51 Degrees Mobile Experts Limited, 5 Charlotte Close, -* Caversham, Reading, Berkshire, United Kingdom RG4 7BY. -* -* This Original Work is licensed under the European Union Public Licence (EUPL) -* v.1.2 and is subject to its terms as set out below. -* -* If a copy of the EUPL was not distributed with this file, You can obtain -* one at https://opensource.org/licenses/EUPL-1.2. -* ********************************************************************* */ - -European Union Public Licence -V. 1.2 - -EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as -defined below) which is provided under the terms of this Licence. Any use of -the Work, other than as authorised under this Licence is prohibited (to the -extent such use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: “Licensed under the EUPL”, or has expressed by -any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: -— ‘The Licence’: this Licence. -— ‘The Original Work’: the work or software distributed or communicated by the - ‘Licensor under this Licence, available as Source Code and also as - ‘Executable Code as the case may be. -— ‘Derivative Works’: the works or software that could be created by the - ‘Licensee, based upon the Original Work or modifications thereof. This - ‘Licence does not define the extent of modification or dependence on the - ‘Original Work required in order to classify a work as a Derivative Work; - ‘this extent is determined by copyright law applicable in the country - ‘mentioned in Article 15. -— ‘The Work’: the Original Work or its Derivative Works. -— ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -— ‘The Executable Code’: any code which has generally been compiled and which - is meant to be interpreted by a computer as a program. -— ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. -— ‘Contributor(s)’: any natural or legal person who modifies the Work under - the Licence, or otherwise contributes to the creation of a Derivative Work. -— ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. -— ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright -vested in the Original Work: - -— use the Work in any circumstance and for all usage, -— reproduce the Work, -— modify the Work, and make Derivative Works based upon the Work, -— communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case - may be, the Work, -— distribute the Work or copies thereof, -— lend and rent the Work or copies thereof, -— sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make -effective the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights -to any patents held by the Licensor, to the extent necessary to make use of -the rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, -in a notice following the copyright notice attached to the Work, a repository -where the Source Code is easily and freely accessible for as long as the -Licensor continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits -from any exception or limitation to the exclusive rights of the rights owners -in the Work, of the exhaustion of those rights or of other applicable -limitations thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and -a copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of -the Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions -on the Work or Derivative Work that alter or restrict the terms of the -Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed -under a Compatible Licence, this Distribution or Communication can be done -under the terms of this Compatible Licence. For the sake of this clause, -‘Compatible Licence’ refers to the licences listed in the appendix attached to -this Licence. Should the Licensee's obligations under the Compatible Licence -conflict with his/her obligations under this Licence, the obligations of the -Compatible Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the -Work, the Licensee will provide a machine-readable copy of the Source Code or -indicate a repository where this Source will be easily and freely available -for as long as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade -names, trademarks, service marks, or names of the Licensor, except as required -for reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she -brings to the Work are owned by him/her or licensed to him/her and that he/she -has the power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ -basis and without warranties of any kind concerning the Work, including -without limitation merchantability, fitness for a particular purpose, absence -of defects or errors, accuracy, non-infringement of intellectual property -rights other than copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a -condition for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the -use of the Work, including without limitation, damages for loss of goodwill, -work stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such -damage. However, the Licensor will be liable under statutory product liability -laws as far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional -agreement, defining obligations or services consistent with this Licence. -However, if accepting obligations, You may act only on your own behalf and on -your sole responsibility, not on behalf of the original Licensor or any other -Contributor, and only if You agree to indemnify, defend, and hold each -Contributor harmless for any liability incurred by, or claims asserted against -such Contributor by the fact You have accepted any warranty or additional -liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I -agree’ placed under the bottom of a window displaying the text of this Licence -or by affirming consent in any other similar way, in accordance with the rules -of applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this -Licence, such as the use of the Work, the creation by You of a Derivative Work -or the Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of -electronic communication by You (for example, by offering to download the Work -from a remote location) the distribution channel or media (for example, a -website) must at least provide to the public the information requested by the -applicable law regarding the Licensor, the Licence and the way it may be -accessible, concluded, stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. Such a termination -will not terminate the licences of any person who has received the Work from -the Licensee under the Licence, provided such persons remain in full -compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions -of this Licence or updated versions of the Appendix, so far this is required -and reasonable, without reducing the scope of the rights granted by the -Licence. New versions of the Licence will be published with a unique version -number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, -— any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty - on the Functioning of the European Union, -— any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive - jurisdiction of the competent court where the Licensor resides or conducts - its primary business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, -— this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, -— this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: -— GNU General Public License (GPL) v. 2, v. 3 -— GNU Affero General Public License (AGPL) v. 3 -— Open Software License (OSL) v. 2.1, v. 3.0 -— Eclipse Public License (EPL) v. 1.0 -— CeCILL v. 2.0, v. 2.1 -— Mozilla Public Licence (MPL) v. 2 -— GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -— Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -— European Union Public Licence (EUPL) v. 1.1, v. 1.2 -— Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or - Strong Reciprocity (LiLiQ-R+) - -— The European Commission may update this Appendix to later versions of the - above licences without producing a new version of the EUPL, as long as they - provide the rights granted in Article 2 of this Licence and protect the - covered Source Code from exclusive appropriation. -— All other changes or additions to this Appendix require the production of a - new EUPL version. - -/* ********************************************************************* -* The 'Compatible Licences' set out in the Appendix to the EUPL (as may be -* amended by the European Commission) shall be deemed incompatible for -* the purposes of the Work and the provisions of the compatibility -* clause in Article 5 of the EUPL shall not apply. -* -* If using the Work as, or as part of, a network application, by -* including the attribution notice(s) required under Article 5 of the EUPL -* in the end user terms of the application under an appropriate heading, -* such notice(s) shall fulfil the requirements of that article. -* ********************************************************************* */ diff --git a/location/README.md b/location/README.md index 031f81f12..b3f8f3db7 100644 --- a/location/README.md +++ b/location/README.md @@ -1,8 +1,8 @@ # 51Degrees Geo-Location Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=51degrees-geo-location-engines "Data rewards the curious") **Pipeline API** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-java/main/.github/images/logo-160.png "Data rewards the curious") **Pipeline API** -[Developer Documentation](https://51degrees.com/location-java/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=51degrees-geo-location-engines "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_reverse_geocoding__quickstart.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=51degrees-geo-location-engines "developer documentation") ## Introduction @@ -75,5 +75,5 @@ See below for a list of the examples available. For complete documentation on the Pipeline API and associated engines, see the [51Degrees documentation site][Documentation]. [Documentation]: https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=project-documentation -[maven]: https://search.maven.org/artifact/com.51degrees/geo-location +[maven]: https://central.sonatype.com/artifact/com.51degrees/geo-location diff --git a/location/geo-location.cloud/pom.xml b/location/geo-location.cloud/pom.xml index 4195dafd5..17679707e 100644 --- a/location/geo-location.cloud/pom.xml +++ b/location/geo-location.cloud/pom.xml @@ -55,5 +55,9 @@ junit junit + + org.mockito + mockito-core + diff --git a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoData.java b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoData.java index 181e740f4..5ca54ae3b 100644 --- a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoData.java +++ b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoData.java @@ -26,9 +26,16 @@ import java.util.Map; /** - * - * @author joseph + * The Location Element Data from the cloud. */ public interface CloudGeoData extends GeoData { - public void setNoValueReasons(Map value); + + /** + * Has no effect. + * @param value the messages for properties with no value, by name + * @deprecated The cloud engine sets the message of each property with no + * value on the value itself. + */ + @Deprecated + void setNoValueReasons(Map value); } diff --git a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoDataDefault.java b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoDataDefault.java index 4a217422d..f6080b94c 100644 --- a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoDataDefault.java +++ b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/data/CloudGeoDataDefault.java @@ -24,145 +24,32 @@ import fiftyone.geolocation.core.data.GeoDataDefault; import fiftyone.pipeline.core.data.FlowData; -import fiftyone.pipeline.core.data.TryGetResult; -import fiftyone.pipeline.core.data.types.JavaScript; import fiftyone.pipeline.engines.data.AspectData; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; -import fiftyone.pipeline.engines.data.AspectPropertyValue; -import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; import fiftyone.pipeline.engines.flowelements.AspectEngine; import fiftyone.pipeline.engines.services.MissingPropertyService; import java.util.Map; -import java.util.Objects; -import org.json.JSONObject; import org.slf4j.Logger; +/** + * The Location Element Data from the cloud. The cloud engine stores each + * property the response holds as an AspectPropertyValue, so the getters of + * GeoDataDefault return them as they are. + */ public class CloudGeoDataDefault extends GeoDataDefault implements CloudGeoData { - private Map noValueReasons; - public CloudGeoDataDefault(Logger logger, FlowData flowData, AspectEngine engine, MissingPropertyService missingPropertyService) { super(logger, flowData, engine, missingPropertyService); } - public void setNoValueReasons(Map value){ - noValueReasons = value; - } - - @SuppressWarnings("unchecked") - @Override - public AspectPropertyValue getJavaScript() { - return getAs("javascript", AspectPropertyValue.class, JavaScript.class); - } - - @Override - public AspectPropertyValue getRoad() { - return getValueAsString("road"); - } - - @Override - public AspectPropertyValue getTown() { - return getValueAsString("town"); - } - - @Override - public AspectPropertyValue getSuburb() { - return getValueAsString("suburb"); - } - - @Override - public AspectPropertyValue getCounty() { - return getValueAsString("county"); - } - - @Override - public AspectPropertyValue getRegion() { - return getValueAsString("region"); - } - - @Override - public AspectPropertyValue getState() { - return getValueAsString("state"); - } - - @Override - public AspectPropertyValue getZipCode() { - return getValueAsString("zipcode"); - } - - @Override - public AspectPropertyValue getCountry() { - return getValueAsString("country"); - } - - @Override - public AspectPropertyValue getCountryCode() { - return getValueAsString("countrycode"); - } - - @Override - public AspectPropertyValue getCountryCode3() { - return getValueAsString("countrycode3"); - } - - @Override - public AspectPropertyValue getTimeZoneOffset() { - return getValueAs("timezoneoffset"); - } - + /** + * Has no effect. + * @param value the messages for properties with no value, by name + * @deprecated The cloud engine sets the message of each property with no + * value on the value itself. + */ + @Deprecated @Override - public AspectPropertyValue getAddress() { - return getValueAsString("address"); + public void setNoValueReasons(Map value) { } - - private AspectPropertyValue getValueAsString(String key) { - return getValueAs(key); - } - - @SuppressWarnings("unchecked") - private AspectPropertyValue getValueAs(String key) { - AspectPropertyValue value = getAs(key, AspectPropertyValue.class); - - if(noValueReasons.containsKey(key)) - value.setNoValueMessage(noValueReasons.get(key)); - - return value; - } - - @Override - protected TryGetResult tryGetValue(String key, Class type, Class... parameterisedTypes) { - if (AspectPropertyValue.class.isAssignableFrom(type)) { - TryGetResult result = new TryGetResult<>(); - Map map = asKeyMap(); - if (map.containsKey(key)) { - Object obj = map.get(key); - - try { - Object temp = setValue(key, obj); - T value = type.cast(temp); - result.setValue(value); - } catch (ClassCastException e) { - throw new ClassCastException("Expected property '" + key + - "' to be of type '" + type.getSimpleName() + - "' but it is '" + obj.getClass().getSimpleName() + "'"); - } - } - return result; - } - else { - return super.tryGetValue(key, type, parameterisedTypes); - } - } - - private AspectPropertyValue setValue(String key, Object obj) { - AspectPropertyValue temp = new AspectPropertyValueDefault(); - if (Objects.isNull(obj) || obj == JSONObject.NULL) { - temp.setNoValueMessage(noValueReasons.get(key)); - } - else { - temp.setValue(obj); - } - return temp; - } - } diff --git a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java index 098faac53..6c8eae436 100644 --- a/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java +++ b/location/geo-location.cloud/src/main/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngine.java @@ -37,6 +37,8 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; import fiftyone.pipeline.engines.data.AspectPropertyMetaDataDefault; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; import org.json.JSONObject; import org.slf4j.Logger; @@ -47,15 +49,44 @@ public class GeoLocationCloudEngine extends CloudAspectEngineBase { private CloudRequestEngine cloudRequestEngine = null; - private List properties; + private volatile List properties; + private final Object metadataLock = new Object(); private String dataSourceTier; private String elementDataKey; private EvidenceKeyFilter evidenceKeyFilter = new EvidenceKeyFilterWhitelist(Collections.emptyList()); private String dataProviderPrefix; + /** + * The property metadata, from the products the Cloud Request Engine + * reports. Until they are known, after a failure at build that can + * pass, this is empty and is read again on the next call. + * @return the property metadata + */ @Override public List getProperties() { - return properties; + List loaded = properties; + if (loaded == null && cloudRequestEngine != null) { + synchronized (metadataLock) { + if (properties == null) { + loadAspectProperties(cloudRequestEngine); + } + loaded = properties; + } + } + return loaded == null ? + new ArrayList() : + loaded; + } + + /** + * Whether the property metadata is loaded, which reads it if the Cloud + * Request Engine now knows the accessible properties. + * @return true once the property metadata is loaded + */ + @Override + public boolean hasLoadedProperties() { + getProperties(); + return properties != null; } @Override @@ -107,44 +138,39 @@ protected void processEngine(FlowData data, CloudGeoData aspectData) { String json = requestData.getJsonResponse(); JSONObject jsonObj = new JSONObject(json); - JSONObject geoObj = jsonObj.getJSONObject(getElementDataKey()); + JSONObject geoObj = getElementJson(jsonObj); Map methodData = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); for (AspectPropertyMetaData property : getProperties()) { + String key = property.getName().toLowerCase(); + // A property the response leaves out has no entry, so + // reading it throws a PropertyMissingException. + if (geoObj.has(key) == false) { + continue; + } + Object value; switch(property.getType().getSimpleName()) { case ("List"): - JSONArray jsonArray = geoObj.optJSONArray(property.getName().toLowerCase()); - methodData.put( - property.getName(), - jsonArray == null ? null : jsonArray.toList()); + JSONArray jsonArray = geoObj.optJSONArray(key); + value = jsonArray == null ? null : jsonArray.toList(); break; case ("JavaScript"): - String javascript = geoObj.optString(property.getName().toLowerCase(), null); - methodData.put( - property.getName(), - javascript == null ? null : new JavaScript(javascript)); + String javascript = geoObj.optString(key, null); + value = javascript == null ? + null : + new JavaScript(javascript); break; default: - methodData.put( - property.getName(), - geoObj.opt(property.getName().toLowerCase())); - break; + value = geoObj.opt(key); + break; } + methodData.put( + property.getName(), + toAspectPropertyValue(geoObj, key, value)); } aspectData.populateFromMap(methodData); - - Map noValueReasonsMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - for(Map.Entry entry : methodData.entrySet()){ - if("null".equals(String.valueOf(entry.getValue()))){ - String key = entry.getKey().toLowerCase(); - Object reason = tryToGet(geoObj, key); - noValueReasonsMap.put(entry.getKey().toLowerCase(), (reason == null ? "" : reason.toString())); - } - } - - aspectData.setNoValueReasons(noValueReasonsMap); } else { throw new PipelineConfigurationException( @@ -158,6 +184,41 @@ public static Object tryToGet(JSONObject jsonObj, String key) { return jsonObj.opt(key + "nullreason"); } + /** + * The message for a property with no value when the response gives no + * reason. + */ + private static final String UNKNOWN_NULL_REASON = "Unknown"; + + /** + * Wrap a value read from the response. A JSON null is no value, with the + * reason the response gives under the property's name followed by + * nullreason. + * @param geoObj the response's part for this engine + * @param key the lower case property name + * @param value the value read, null or JSONObject.NULL where the + * response has none + * @return the property value + */ + private static AspectPropertyValue toAspectPropertyValue( + JSONObject geoObj, + String key, + Object value) { + AspectPropertyValue result = + new AspectPropertyValueDefault<>(); + if (value == null || value == JSONObject.NULL) { + Object reason = tryToGet(geoObj, key); + result.setNoValueMessage( + reason == null || reason == JSONObject.NULL ? + UNKNOWN_NULL_REASON : + reason.toString()); + } + else { + result.setValue(value); + } + return result; + } + @Override protected void unmanagedResourcesCleanup() { } @@ -167,9 +228,10 @@ public void addPipeline(Pipeline pipeline) { if (cloudRequestEngine == null) { cloudRequestEngine = pipeline.getElement(CloudRequestEngine.class); if (cloudRequestEngine != null) { - if (loadAspectProperties(cloudRequestEngine) == false) { - logger.error("Failed to load aspect properties"); - } + // The products are not known after a failure at build that + // can pass, which the Cloud Request Engine has logged. + // getProperties reads them once they are. + loadAspectProperties(cloudRequestEngine); } } @@ -177,16 +239,14 @@ public void addPipeline(Pipeline pipeline) { } private boolean loadAspectProperties(CloudRequestEngine engine) { - Map map = - engine.getPublicProperties(); + AccessiblePropertyMetaData.ProductMetaData product = + findProduct(engine.getPublicProperties()); - if (map != null && - map.size() > 0 && - map.containsKey(getElementDataKey())) { - properties = new ArrayList<>(); - dataSourceTier = map.get(getElementDataKey()).dataTier; + if (product != null) { + List loaded = new ArrayList<>(); + dataSourceTier = product.dataTier; - for (AccessiblePropertyMetaData.PropertyMetaData item : map.get(getElementDataKey()).properties) + for (AccessiblePropertyMetaData.PropertyMetaData item : product.properties) { AspectPropertyMetaData property = new AspectPropertyMetaDataDefault( item.name, @@ -199,13 +259,12 @@ private boolean loadAspectProperties(CloudRequestEngine engine) { item.delayExecution, item.evidenceProperties); - properties.add(property); + loaded.add(property); } + properties = loaded; return true; } else { - logger.error("Aspect properties could not be loaded for" + - " the Device Detection cloud engine", this); return false; } } diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineNullValueTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineNullValueTests.java new file mode 100644 index 000000000..8bfe09c1b --- /dev/null +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineNullValueTests.java @@ -0,0 +1,230 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.flowelements; + +import fiftyone.geolocation.core.Enums; +import fiftyone.geolocation.data.CloudGeoData; +import fiftyone.geolocation.data.CloudGeoDataDefault; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.data.AspectPropertyValue; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A property the cloud response gives as null has no value, and its message + * is the text under the property's name followed by nullreason, as the + * Processing section of the cloud aspect engine specification says. A + * property the response leaves out is missing, so reading it throws, as the + * Missing Properties section of the properties specification says. The tests + * read each property by name, from the map the JSON Builder reads, and from + * the typed getter. + */ +public class GeoLocationCloudEngineNullValueTests { + + private static final String SUBURB_NULL_REASON = + "No suburb was found at this location."; + + private static final String NULL_SUBURB_RESPONSE = + "{ 'location': { " + + "'suburb': null, " + + "'suburbnullreason': '" + SUBURB_NULL_REASON + "', " + + "'town': 'Reading' }, " + + "'javascriptProperties': [] }"; + + private static final String NULL_SUBURB_WITHOUT_REASON_RESPONSE = + "{ 'location': { 'suburb': null, 'town': 'Reading' }, " + + "'javascriptProperties': [] }"; + + private static final String NO_SUBURB_RESPONSE = + "{ 'location': { 'town': 'Reading' }, " + + "'javascriptProperties': [] }"; + + private final ILoggerFactory loggerFactory = + LoggerFactory.getILoggerFactory(); + + @Test + public void NullValue_NoValueWithNullReason() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process(engine, NULL_SUBURB_RESPONSE); + + assertNoValue(SUBURB_NULL_REASON, data.get("Suburb")); + assertNoValue(SUBURB_NULL_REASON, data.asKeyMap().get("Suburb")); + assertNoValue(SUBURB_NULL_REASON, data.getSuburb()); + } + } + + @Test + public void Value_HasValue() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process(engine, NULL_SUBURB_RESPONSE); + + assertValue("Reading", data.get("Town")); + assertValue("Reading", data.asKeyMap().get("Town")); + assertValue("Reading", data.getTown()); + } + } + + @Test + public void NullValueWithoutReason_NoValueUnknown() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process( + engine, + NULL_SUBURB_WITHOUT_REASON_RESPONSE); + + assertNoValue("Unknown", data.get("Suburb")); + assertNoValue("Unknown", data.asKeyMap().get("Suburb")); + assertNoValue("Unknown", data.getSuburb()); + } + } + + @Test + public void NoEntry_PropertyIsMissing() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process(engine, NO_SUBURB_RESPONSE); + + assertFalse(data.asKeyMap().containsKey("Suburb")); + assertMissing(() -> data.get("Suburb")); + assertMissing(() -> data.getSuburb()); + } + } + + @Test + @SuppressWarnings("deprecation") + public void SetNoValueReasons_ChangesNothing() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process(engine, NULL_SUBURB_RESPONSE); + Map reasons = new HashMap<>(); + reasons.put("suburb", "Another reason"); + data.setNoValueReasons(reasons); + + assertNoValue(SUBURB_NULL_REASON, data.get("Suburb")); + assertNoValue(SUBURB_NULL_REASON, data.getSuburb()); + } + } + + private static void assertMissing(Runnable read) { + try { + read.run(); + fail("Expected a PropertyMissingException"); + } + catch (PropertyMissingException e) { + // The property is not in the result, as expected. + } + } + + private static AspectPropertyValue asAspectPropertyValue(Object value) { + assertTrue( + "Expected an AspectPropertyValue but found " + + (value == null ? "null" : value.getClass().getName()), + value instanceof AspectPropertyValue); + return (AspectPropertyValue) value; + } + + private static void assertNoValue(String reason, Object value) { + AspectPropertyValue propertyValue = asAspectPropertyValue(value); + assertFalse( + "Expected no value but found " + propertyValue, + propertyValue.hasValue()); + assertEquals(reason, propertyValue.getNoValueMessage()); + } + + private static void assertValue(Object expected, Object value) + throws Exception { + AspectPropertyValue propertyValue = asAspectPropertyValue(value); + assertTrue( + "Expected a value but found none, because " + + propertyValue.getNoValueMessage(), + propertyValue.hasValue()); + assertEquals(expected, propertyValue.getValue()); + } + + /** + * An engine whose Cloud Request Engine reports the Location product with + * the Suburb and Town properties. + */ + private GeoLocationCloudEngine buildEngine() throws Exception { + GeoLocationCloudEngine engine = + new GeoLocationCloudEngineBuilder(loggerFactory) + .build(Enums.GeoLocationProvider.FiftyOneDegrees); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("Suburb", "String", "Location", null), + new PropertyMetaData("Town", "String", "Location", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put("location", product); + + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + /** + * Process the given cloud response. + */ + private CloudGeoData process( + GeoLocationCloudEngine engine, + String response) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + + CloudGeoData data = new CloudGeoDataDefault( + loggerFactory.getLogger(CloudGeoData.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + engine.processEngine(flowData, data); + return data; + } +} diff --git a/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java new file mode 100644 index 000000000..34a070f5f --- /dev/null +++ b/location/geo-location.cloud/src/test/java/fiftyone/geolocation/flowelements/GeoLocationCloudEngineTests.java @@ -0,0 +1,264 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.geolocation.flowelements; + +import fiftyone.geolocation.core.Enums; +import fiftyone.geolocation.data.CloudGeoData; +import fiftyone.geolocation.data.CloudGeoDataDefault; +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.cloudrequestengine.flowelements.CloudRequestEngine; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.ProductMetaData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData.PropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.services.MissingPropertyReason; +import fiftyone.pipeline.engines.services.MissingPropertyServiceDefault; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.Invocation; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * A resource key can be valid and include no product that holds the + * Location properties. The engine then builds with empty property + * metadata, logs the missing product once, and logs or throws nothing on + * each request. + */ +public class GeoLocationCloudEngineTests { + + private Logger logger; + private ILoggerFactory loggerFactory; + + @Before + public void init() { + logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + @Test + public void ProductMissing_BuildsWithEmptyMetadata() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + assertNotNull(engine.getProperties()); + assertEquals(0, engine.getProperties().size()); + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 1, logged.size()); + assertTrue(logged.get(0), logged.get(0).startsWith("warn ")); + assertTrue(logged.get(0), logged.get(0).contains("'location'")); + } + } + + @Test + public void ProductMissing_ProcessLogsAndThrowsNothing() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + for (int i = 0; i < 3; i++) { + process(engine); + } + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 1, logged.size()); + } + } + + /** + * With the Location product included, a response that holds no + * part for it leaves the element data without values and throws + * nothing, as the Processing section of the cloud aspect engine + * specification says. Reading a property then throws, as for any + * property the response leaves out. + */ + @Test + public void ProductPresent_ResponseWithoutPart_PropertyIsMissing() + throws Exception { + try (GeoLocationCloudEngine engine = buildEngine("location", "town")) { + CloudGeoData data = process( + engine, + "{ 'ip': { 'registeredname': 'Example' }, " + + "'javascriptProperties': [] }"); + try { + data.getTown(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + // The response held nothing for the Location product. + } + } + } + + /** + * Reading a property behaves as for any property the response does not + * carry. + */ + @Test + public void ProductMissing_PropertyIsMissing() throws Exception { + try (GeoLocationCloudEngine engine = buildEngine()) { + CloudGeoData data = process(engine); + try { + data.getTown(); + fail("A PropertyMissingException should have been thrown"); + } catch (PropertyMissingException e) { + assertEquals( + MissingPropertyReason.ProductNotAccessibleWithResourceKey, + e.getReason()); + } + } + } + + /** + * After a failure at build that can pass, the Cloud Request Engine does + * not know the accessible products yet. The engine builds without + * logging anything, and reads its metadata once the products are known + * rather than keeping none. + */ + @Test + public void DiscoveryNotComplete_LoadsTheMetadataLater() throws Exception { + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData("town", "String", "cat", null)); + Map products = new HashMap<>(); + products.put("location", product); + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()) + .thenReturn(null) + .thenReturn(products); + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + try (GeoLocationCloudEngine engine = + new GeoLocationCloudEngineBuilder(loggerFactory) + .build(Enums.GeoLocationProvider.FiftyOneDegrees)) { + engine.addPipeline(pipeline); + List logged = warningsAndErrors(); + assertEquals(logged.toString(), 0, logged.size()); + + assertEquals(1, engine.getProperties().size()); + assertEquals("town", engine.getProperties().get(0).getName()); + } + } + + /** + * An engine whose Cloud Request Engine reports one product, holding + * one string property. + */ + // thenReturn has a varargs overload, so a generic Map is an unchecked array. + @SuppressWarnings("unchecked") + private GeoLocationCloudEngine buildEngine( + String productKey, + String property) throws Exception { + GeoLocationCloudEngine engine = + new GeoLocationCloudEngineBuilder(loggerFactory) + .build(Enums.GeoLocationProvider.FiftyOneDegrees); + + ProductMetaData product = new ProductMetaData(); + product.dataTier = "test"; + product.properties = Arrays.asList( + new PropertyMetaData(property, "String", "cat", null)); + Map publicProperties = new HashMap<>(); + publicProperties.put(productKey, product); + + CloudRequestEngine requestEngine = mock(CloudRequestEngine.class); + when(requestEngine.getPublicProperties()).thenReturn(publicProperties); + + Pipeline pipeline = mock(Pipeline.class); + when(pipeline.getElement(CloudRequestEngine.class)) + .thenReturn(requestEngine); + + engine.addPipeline(pipeline); + return engine; + } + + /** + * Process the given cloud response. + */ + private CloudGeoData process( + GeoLocationCloudEngine engine, + String response) { + CloudRequestData cloudData = mock(CloudRequestData.class); + when(cloudData.getJsonResponse()).thenReturn(response); + + FlowData flowData = mock(FlowData.class); + when(flowData.getFromElement(any(CloudRequestEngine.class))) + .thenReturn(cloudData); + + CloudGeoData data = new CloudGeoDataDefault( + loggerFactory.getLogger(CloudGeoData.class.getSimpleName()), + flowData, + engine, + MissingPropertyServiceDefault.getInstance()); + engine.processEngine(flowData, data); + return data; + } + + /** + * An engine on a resource key that includes IP Intelligence alone. + */ + private GeoLocationCloudEngine buildEngine() throws Exception { + return buildEngine("ip", "registeredname"); + } + + /** + * Process a cloud response that holds nothing for the Location + * product. + */ + private CloudGeoData process(GeoLocationCloudEngine engine) { + return process( + engine, + "{ 'ip': { 'registeredname': 'Example' }, " + + "'javascriptProperties': [] }"); + } + + /** + * Every warning and error logged, at any overload. + */ + private List warningsAndErrors() { + List logged = new ArrayList<>(); + for (Invocation invocation : + Mockito.mockingDetails(logger).getInvocations()) { + String level = invocation.getMethod().getName(); + if (level.equals("warn") || level.equals("error")) { + logged.add(level + " " + + Arrays.toString(invocation.getArguments())); + } + } + return Collections.unmodifiableList(logged); + } +} diff --git a/location/geo-location/src/test/java/fiftyone/geolocation/data/tests/ValueTests.java b/location/geo-location/src/test/java/fiftyone/geolocation/data/tests/ValueTests.java index 060e6625c..ce3edd363 100644 --- a/location/geo-location/src/test/java/fiftyone/geolocation/data/tests/ValueTests.java +++ b/location/geo-location/src/test/java/fiftyone/geolocation/data/tests/ValueTests.java @@ -67,9 +67,26 @@ public static void valueTypes(Wrapper wrapper) throws Exception { assertNotNull("Value of " + property.getName() + " is null. ", value); assertTrue("Value of '" + property.getName() + "' was of type " + value.getClass().getSimpleName() + + " but should have been AspectPropertyValue.", + value instanceof AspectPropertyValue); + AspectPropertyValue propertyValue = + (AspectPropertyValue) value; + // A property can have no value at these coordinates, such + // as Suburb. Its type cannot be read then, so only that it + // carries a message is checked. + if (propertyValue.hasValue() == false) { + assertNotNull("Property '" + property.getName() + + "' has no value and no message.", + propertyValue.getNoValueMessage()); + continue; + } + assertTrue("Value of '" + property.getName() + + "' was of type " + + propertyValue.getValue().getClass().getSimpleName() + " but should have been " + expectedType.getSimpleName() + ".", - expectedType.isAssignableFrom(value.getClass())); + expectedType.isAssignableFrom( + propertyValue.getValue().getClass())); } } } diff --git a/location/pom.xml b/location/pom.xml index 930a853f5..9974783aa 100644 --- a/location/pom.xml +++ b/location/pom.xml @@ -60,7 +60,7 @@ 1.8 ${project.version} - 4.5.128 + 4.5.130 4.13.2 1.7.36 diff --git a/pipeline/LICENSE b/pipeline/LICENSE deleted file mode 100644 index 4153cd377..000000000 --- a/pipeline/LICENSE +++ /dev/null @@ -1,287 +0,0 @@ - EUROPEAN UNION PUBLIC LICENCE v. 1.2 - EUPL © the European Union 2007, 2016 - -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined -below) which is provided under the terms of this Licence. Any use of the Work, -other than as authorised under this Licence is prohibited (to the extent such -use is covered by a right of the copyright holder of the Work). - -The Work is provided under the terms of this Licence when the Licensor (as -defined below) has placed the following notice immediately following the -copyright notice for the Work: - - Licensed under the EUPL - -or has expressed by any other means his willingness to license under the EUPL. - -1. Definitions - -In this Licence, the following terms have the following meaning: - -- ‘The Licence’: this Licence. - -- ‘The Original Work’: the work or software distributed or communicated by the - Licensor under this Licence, available as Source Code and also as Executable - Code as the case may be. - -- ‘Derivative Works’: the works or software that could be created by the - Licensee, based upon the Original Work or modifications thereof. This Licence - does not define the extent of modification or dependence on the Original Work - required in order to classify a work as a Derivative Work; this extent is - determined by copyright law applicable in the country mentioned in Article 15. - -- ‘The Work’: the Original Work or its Derivative Works. - -- ‘The Source Code’: the human-readable form of the Work which is the most - convenient for people to study and modify. - -- ‘The Executable Code’: any code which has generally been compiled and which is - meant to be interpreted by a computer as a program. - -- ‘The Licensor’: the natural or legal person that distributes or communicates - the Work under the Licence. - -- ‘Contributor(s)’: any natural or legal person who modifies the Work under the - Licence, or otherwise contributes to the creation of a Derivative Work. - -- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of - the Work under the terms of the Licence. - -- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, - renting, distributing, communicating, transmitting, or otherwise making - available, online or offline, copies of the Work or providing access to its - essential functionalities at the disposal of any other natural or legal - person. - -2. Scope of the rights granted by the Licence - -The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -sublicensable licence to do the following, for the duration of copyright vested -in the Original Work: - -- use the Work in any circumstance and for all usage, -- reproduce the Work, -- modify the Work, and make Derivative Works based upon the Work, -- communicate to the public, including the right to make available or display - the Work or copies thereof to the public and perform publicly, as the case may - be, the Work, -- distribute the Work or copies thereof, -- lend and rent the Work or copies thereof, -- sublicense rights in the Work or copies thereof. - -Those rights can be exercised on any media, supports and formats, whether now -known or later invented, as far as the applicable law permits so. - -In the countries where moral rights apply, the Licensor waives his right to -exercise his moral right to the extent allowed by law in order to make effective -the licence of the economic rights here above listed. - -The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to -any patents held by the Licensor, to the extent necessary to make use of the -rights granted on the Work under this Licence. - -3. Communication of the Source Code - -The Licensor may provide the Work either in its Source Code form, or as -Executable Code. If the Work is provided as Executable Code, the Licensor -provides in addition a machine-readable copy of the Source Code of the Work -along with each copy of the Work that the Licensor distributes or indicates, in -a notice following the copyright notice attached to the Work, a repository where -the Source Code is easily and freely accessible for as long as the Licensor -continues to distribute or communicate the Work. - -4. Limitations on copyright - -Nothing in this Licence is intended to deprive the Licensee of the benefits from -any exception or limitation to the exclusive rights of the rights owners in the -Work, of the exhaustion of those rights or of other applicable limitations -thereto. - -5. Obligations of the Licensee - -The grant of the rights mentioned above is subject to some restrictions and -obligations imposed on the Licensee. Those obligations are the following: - -Attribution right: The Licensee shall keep intact all copyright, patent or -trademarks notices and all notices that refer to the Licence and to the -disclaimer of warranties. The Licensee must include a copy of such notices and a -copy of the Licence with every copy of the Work he/she distributes or -communicates. The Licensee must cause any Derivative Work to carry prominent -notices stating that the Work has been modified and the date of modification. - -Copyleft clause: If the Licensee distributes or communicates copies of the -Original Works or Derivative Works, this Distribution or Communication will be -done under the terms of this Licence or of a later version of this Licence -unless the Original Work is expressly distributed only under this version of the -Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee -(becoming Licensor) cannot offer or impose any additional terms or conditions on -the Work or Derivative Work that alter or restrict the terms of the Licence. - -Compatibility clause: If the Licensee Distributes or Communicates Derivative -Works or copies thereof based upon both the Work and another work licensed under -a Compatible Licence, this Distribution or Communication can be done under the -terms of this Compatible Licence. For the sake of this clause, ‘Compatible -Licence’ refers to the licences listed in the appendix attached to this Licence. -Should the Licensee's obligations under the Compatible Licence conflict with -his/her obligations under this Licence, the obligations of the Compatible -Licence shall prevail. - -Provision of Source Code: When distributing or communicating copies of the Work, -the Licensee will provide a machine-readable copy of the Source Code or indicate -a repository where this Source will be easily and freely available for as long -as the Licensee continues to distribute or communicate the Work. - -Legal Protection: This Licence does not grant permission to use the trade names, -trademarks, service marks, or names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the copyright notice. - -6. Chain of Authorship - -The original Licensor warrants that the copyright in the Original Work granted -hereunder is owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each Contributor warrants that the copyright in the modifications he/she brings -to the Work are owned by him/her or licensed to him/her and that he/she has the -power and authority to grant the Licence. - -Each time You accept the Licence, the original Licensor and subsequent -Contributors grant You a licence to their contributions to the Work, under the -terms of this Licence. - -7. Disclaimer of Warranty - -The Work is a work in progress, which is continuously improved by numerous -Contributors. It is not a finished work and may therefore contain defects or -‘bugs’ inherent to this type of development. - -For the above reason, the Work is provided under the Licence on an ‘as is’ basis -and without warranties of any kind concerning the Work, including without -limitation merchantability, fitness for a particular purpose, absence of defects -or errors, accuracy, non-infringement of intellectual property rights other than -copyright as stated in Article 6 of this Licence. - -This disclaimer of warranty is an essential part of the Licence and a condition -for the grant of any rights to the Work. - -8. Disclaimer of Liability - -Except in the cases of wilful misconduct or damages directly caused to natural -persons, the Licensor will in no event be liable for any direct or indirect, -material or moral, damages of any kind, arising out of the Licence or of the use -of the Work, including without limitation, damages for loss of goodwill, work -stoppage, computer failure or malfunction, loss of data or any commercial -damage, even if the Licensor has been advised of the possibility of such damage. -However, the Licensor will be liable under statutory product liability laws as -far such laws apply to the Work. - -9. Additional agreements - -While distributing the Work, You may choose to conclude an additional agreement, -defining obligations or services consistent with this Licence. However, if -accepting obligations, You may act only on your own behalf and on your sole -responsibility, not on behalf of the original Licensor or any other Contributor, -and only if You agree to indemnify, defend, and hold each Contributor harmless -for any liability incurred by, or claims asserted against such Contributor by -the fact You have accepted any warranty or additional liability. - -10. Acceptance of the Licence - -The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ -placed under the bottom of a window displaying the text of this Licence or by -affirming consent in any other similar way, in accordance with the rules of -applicable law. Clicking on that icon indicates your clear and irrevocable -acceptance of this Licence and all of its terms and conditions. - -Similarly, you irrevocably accept this Licence and all of its terms and -conditions by exercising any rights granted to You by Article 2 of this Licence, -such as the use of the Work, the creation by You of a Derivative Work or the -Distribution or Communication by You of the Work or copies thereof. - -11. Information to the public - -In case of any Distribution or Communication of the Work by means of electronic -communication by You (for example, by offering to download the Work from a -remote location) the distribution channel or media (for example, a website) must -at least provide to the public the information requested by the applicable law -regarding the Licensor, the Licence and the way it may be accessible, concluded, -stored and reproduced by the Licensee. - -12. Termination of the Licence - -The Licence and the rights granted hereunder will terminate automatically upon -any breach by the Licensee of the terms of the Licence. - -Such a termination will not terminate the licences of any person who has -received the Work from the Licensee under the Licence, provided such persons -remain in full compliance with the Licence. - -13. Miscellaneous - -Without prejudice of Article 9 above, the Licence represents the complete -agreement between the Parties as to the Work. - -If any provision of the Licence is invalid or unenforceable under applicable -law, this will not affect the validity or enforceability of the Licence as a -whole. Such provision will be construed or reformed so as necessary to make it -valid and enforceable. - -The European Commission may publish other linguistic versions or new versions of -this Licence or updated versions of the Appendix, so far this is required and -reasonable, without reducing the scope of the rights granted by the Licence. New -versions of the Licence will be published with a unique version number. - -All linguistic versions of this Licence, approved by the European Commission, -have identical value. Parties can take advantage of the linguistic version of -their choice. - -14. Jurisdiction - -Without prejudice to specific agreement between parties, - -- any litigation resulting from the interpretation of this License, arising - between the European Union institutions, bodies, offices or agencies, as a - Licensor, and any Licensee, will be subject to the jurisdiction of the Court - of Justice of the European Union, as laid down in article 272 of the Treaty on - the Functioning of the European Union, - -- any litigation arising between other parties and resulting from the - interpretation of this License, will be subject to the exclusive jurisdiction - of the competent court where the Licensor resides or conducts its primary - business. - -15. Applicable Law - -Without prejudice to specific agreement between parties, - -- this Licence shall be governed by the law of the European Union Member State - where the Licensor has his seat, resides or has his registered office, - -- this licence shall be governed by Belgian law if the Licensor has no seat, - residence or registered office inside a European Union Member State. - -Appendix - -‘Compatible Licences’ according to Article 5 EUPL are: - -- GNU General Public License (GPL) v. 2, v. 3 -- GNU Affero General Public License (AGPL) v. 3 -- Open Software License (OSL) v. 2.1, v. 3.0 -- Eclipse Public License (EPL) v. 1.0 -- CeCILL v. 2.0, v. 2.1 -- Mozilla Public Licence (MPL) v. 2 -- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 -- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for - works other than software -- European Union Public Licence (EUPL) v. 1.1, v. 1.2 -- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong - Reciprocity (LiLiQ-R+). - -The European Commission may update this Appendix to later versions of the above -licences without producing a new version of the EUPL, as long as they provide -the rights granted in Article 2 of this Licence and protect the covered Source -Code from exclusive appropriation. - -All other changes or additions to this Appendix require the production of a new -EUPL version. diff --git a/pipeline/README.md b/pipeline/README.md index 8370cc09b..66b7d319d 100644 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -1,8 +1,8 @@ # 51Degrees Pipeline -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=top "Data rewards the curious") **Java Pipeline** +![51Degrees](https://raw.githubusercontent.com/51Degrees/51degrees-java/main/.github/images/logo-160.png "Data rewards the curious") **Java Pipeline** -[Developer Documentation](https://51degrees.com/pipeline-java/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=top "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=top "developer documentation") ## Introduction This repository contains all the projects required to build the Java implementation of the 51Degrees Pipeline API. @@ -12,7 +12,7 @@ is also available on GitHub and is recommended reading if you wish to understand the concepts and design of this API. Reference documentation for the Java implementation can be found on the -[Java API documentation](https://51degrees.com/pipeline-java/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=introduction) page. +[Java API documentation](https://51degrees.com/documentation/apis/pipeline-java/annotated.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-java&utm_content=readme.md&utm_term=introduction) page. ## Pre-requisites @@ -131,13 +131,15 @@ and on our [website](https://51degrees.com/documentation/_device_detection__exam | pipeline.developer-examples.onpremise-engine | Shows how to modify SimpleFlowElement to make use of the 'engine' functionality and use a custom data file to map dates to star signs rather than relying on hard coded data. | | pipeline.developer-examples.clientside-element | Shows how to modify SimpleFlowElement to request the data of birth from the user using client-side JavaScript. | | pipeline.developer-examples.clientside-element-mvc | An example project showing how to use the code from SimpleClientSideElement in a Java web application using the Model-View-Controller Pattern. | -| pipeline.developer-examples.cloud-engine | Shows how to modify SimpleFlowElement to perform the star sign lookup via a cloud service rather than locally. The endpoint and resource key come from `FOD_CLOUD_API_URL` and `_51DEGREES_RESOURCE_KEY` when set, otherwise the star sign service the example was written for. | +| pipeline.developer-examples.cloud-engine | Shows how to modify SimpleFlowElement to perform the star sign lookup via a cloud service rather than locally. The endpoint and resource key come from `_51DEGREES_CLOUD_ENDPOINT` and `_51DEGREES_RESOURCE_KEY` when set, otherwise the star sign service the example was written for. | | pipeline.developer-examples.usage-sharing | Shows how to share usage with 51Degrees. This helps us to keep our products up to date and accurate. | | pipeline.developer-examples.fodid | Shows how to read a 51Did, and, in a web demo, how a browser creates and verifies one against the 51Degrees cloud and the server redeems the encrypted result, so that the creator context proves the identifier is being presented by the browser it was created on. The module README describes the flow, the environment variables, what a run costs and the redeem call to copy into your own server. | ### Pointing the cloud examples at another host -Every example that calls the 51Degrees cloud takes its endpoint from the `FOD_CLOUD_API_URL` environment variable, which is the cloud API base including the `/api/v4/` segment and defaults to `https://cloud.51degrees.com/api/v4/` (the cloud-engine example defaults to the star sign service it was written for, as its row above says). This is the same variable the cloud request engine in this repository honours, so setting it once points every example, and any pipeline built with the engine, at the same place. The resource key comes from `_51DEGREES_RESOURCE_KEY`, or the older `RESOURCE_KEY`. +Every example that calls the 51Degrees cloud takes its endpoint from the `_51DEGREES_CLOUD_ENDPOINT` environment variable, which is the cloud API base including the `/api/v4/` segment and defaults to `https://cloud.51degrees.com/api/v4/` (the cloud-engine example defaults to the star sign service it was written for, as its row above says). This is the same variable the cloud request engine in this repository honours, so setting it once points every example, and any pipeline built with the engine, at the same place. The resource key comes from `_51DEGREES_RESOURCE_KEY`. + +The variable was called `FOD_CLOUD_API_URL` before, and that name is deprecated. The cloud request engine, the 51Did client and the cloud-engine example still read `FOD_CLOUD_API_URL` when `_51DEGREES_CLOUD_ENDPOINT` is not set, and each logs a warning the first time it does. When both are set, `_51DEGREES_CLOUD_ENDPOINT` is used. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons. This is the private hosting option of the cloud service. Both run the same service, so the examples work unchanged. diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/CloudDiscoveryState.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/CloudDiscoveryState.java new file mode 100644 index 000000000..0731e46d8 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/CloudDiscoveryState.java @@ -0,0 +1,115 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * A copy of what the Cloud Request Engine discovered for a resource key, + * being the accepted Evidence keys and the accessible properties, which + * depend only on the resource key. Read it from the builder after a + * successful build, keep it, for example as {@link #toJson()}, and give it + * to a later builder, which then makes no discovery request. A Pipeline can + * so be built offline, and a host that starts often need not call the + * cloud service each time. + * @see Specification + */ +public final class CloudDiscoveryState { + + private static final String EVIDENCE_KEYS = "evidenceKeys"; + + private static final String ACCESSIBLE_PROPERTIES = "accessibleProperties"; + + private final List evidenceKeys; + + private final String accessibleProperties; + + /** + * Construct a new instance. + * @param evidenceKeys the evidence keys the cloud service accepts + * @param accessibleProperties the answer to the accessibleproperties + * request, as JSON + * @throws org.json.JSONException if the accessible properties are not + * a JSON object with a Products member + */ + public CloudDiscoveryState( + List evidenceKeys, + String accessibleProperties) { + Objects.requireNonNull(evidenceKeys, "evidenceKeys"); + Objects.requireNonNull(accessibleProperties, "accessibleProperties"); + new JSONObject(accessibleProperties).getJSONObject("Products"); + this.evidenceKeys = Collections.unmodifiableList( + new ArrayList<>(evidenceKeys)); + this.accessibleProperties = accessibleProperties; + } + + /** + * @return the evidence keys the cloud service accepts + */ + public List getEvidenceKeys() { + return evidenceKeys; + } + + /** + * @return the answer to the accessibleproperties request, as JSON + */ + public String getAccessibleProperties() { + return accessibleProperties; + } + + /** + * @return this copy as one JSON document, which {@link #fromJson} + * reads back + */ + public String toJson() { + JSONObject json = new JSONObject(); + json.put(EVIDENCE_KEYS, new JSONArray(evidenceKeys)); + json.put(ACCESSIBLE_PROPERTIES, new JSONObject(accessibleProperties)); + return json.toString(); + } + + /** + * Read a copy written by {@link #toJson()}. + * @param json the JSON document + * @return the copy + * @throws org.json.JSONException if the document is not one + * {@link #toJson()} writes + */ + public static CloudDiscoveryState fromJson(String json) { + JSONObject document = new JSONObject(json); + JSONArray keys = document.getJSONArray(EVIDENCE_KEYS); + List evidenceKeys = new ArrayList<>(); + for (int i = 0; i < keys.length(); i++) { + evidenceKeys.add(keys.getString(i)); + } + return new CloudDiscoveryState( + evidenceKeys, + document.getJSONObject(ACCESSIBLE_PROPERTIES).toString()); + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java index 593c618d6..eebb82eb4 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/Constants.java @@ -27,6 +27,35 @@ public class Constants { public static final String OriginHeaderName = "Origin"; public class Messages { + public static final String NoCredential = + "The Cloud Request Engine needs a resource key or a license key " + + "to authenticate with, as the cloud service answers 401 to a " + + "request that carries neither. Use setResourceKey(String) with " + + "a resource key created for free at " + + "https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=no-credential"; + + public static final String LicenseKeyNeedsProperties = + "The Cloud Request Engine has a license key and no requested " + + "properties. A license key names no properties of its own, so " + + "the cloud service answers 400 to every request made with one " + + "that does not list the properties it wants. Use " + + "setRequestedProperties to name the fully qualified " + + "properties wanted, for example \"device.ismobile\" and " + + "\"device.iscrawler\"."; + + public static final String PropertiesNotCovered = + "The cloud service did not return the requested properties %s " + + "in its answer. They are not covered by the keys in use, so " + + "they will be missing from every result. The other requested " + + "properties came back and are usable."; + + public static final String RequestedNamesNotProperties = + "The cloud service did not return the requested properties %s " + + "in its answer. Every property is answered, with a reason " + + "when it has no value, so these names are not properties. " + + "Check the spelling of each one. The other requested " + + "properties came back and are usable."; + public static final String EvidenceConflict = "'%s:%s' evidence conflicts with %s"; public static final String MessageNoDataInResponse = @@ -40,6 +69,10 @@ public class Messages { public static final String ExceptionCloudError = "Error returned from 51Degrees cloud service: %s"; + + public static final String ExceptionResponseNotJson = + "Cloud service returned status code '%d' with content that is " + + "not JSON"; public static final String ExceptionFailedToLoadProperties = "Failed to load aspect properties for element '%s'. This is " @@ -48,6 +81,18 @@ public class Messages { + "see our explainer: " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-constants.java&utm_term=exceptionfailedtoloadproperties"; + public static final String MessageDiscoveryDeferred = + "The cloud request engine could not fetch its accessible " + + "properties or accepted evidence keys when it was built, " + + "because of a failure that can pass. It will fetch them " + + "again when it is first used, and sends no data request " + + "until it has both."; + + public static final String MessageProductNotInResourceKey = + "The resource key does not include the '%s' product, so " + + "%s has no properties and its results hold no values. " + + "Use a resource key that includes the product to get them."; + public static final String ProcessCloudEngineNotImplemented = "This method should be overridden in the class derived from " + "CloudAspectEngineBase class. The implementation should use " diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java index 70f10ea2a..80b39e9a6 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBase.java @@ -22,7 +22,7 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; -import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.ExceptionFailedToLoadProperties; +import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.MessageProductNotInResourceKey; import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.ProcessCloudEngineNotImplemented; import fiftyone.pipeline.cloudrequestengine.NotImplementedException; @@ -43,12 +43,14 @@ import fiftyone.pipeline.engines.flowelements.CloudAspectEngine; import fiftyone.pipeline.util.Types; +import org.json.JSONObject; import org.slf4j.Logger; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; /** * Base class for 51Degrees Cloud Aspect Engines @@ -109,6 +111,7 @@ public CloudRequestEngine getInstance() throws PipelineConfigurationException { private volatile List aspectProperties; private String dataSourceTier; private RequestEngineAccessor requestEngine; + private final AtomicBoolean productMissingLogged = new AtomicBoolean(false); /** * Construct a new instance of the {@link CloudAspectEngineBase}. @@ -141,6 +144,13 @@ protected void setRequestEngine(RequestEngineAccessor requestEngine) { this.requestEngine = requestEngine; } + /** + * The property metadata of this engine, from the products the Cloud + * Request Engine reports. After a failure at build that can pass the + * products are not known yet, and an empty list is returned without + * being kept, so the metadata is read again on the next call. + * @return the property metadata + */ @Override public List getProperties() { List localRef = aspectProperties; @@ -149,17 +159,24 @@ public List getProperties() { localRef = aspectProperties; if (localRef == null) { if(loadAspectProperties() == false) { - throw new RuntimeException( - String.format( - ExceptionFailedToLoadProperties, - this.getElementDataKey(), - this.getElementDataKey()) - ); + return new ArrayList<>(); } + localRef = aspectProperties; } } } - return aspectProperties; + return localRef; + } + + /** + * Whether the property metadata is loaded, which reads it if the Cloud + * Request Engine now knows the accessible properties. + * @return true once the property metadata is loaded + */ + @Override + public boolean hasLoadedProperties() { + getProperties(); + return aspectProperties != null; } @Override @@ -192,17 +209,15 @@ public TypedKey getTypedDataKey() { */ private boolean loadAspectProperties() { CloudRequestEngine requestEngine = getRequestEngine().getInstance(); - Map map = - requestEngine.getPublicProperties(); + AccessiblePropertyMetaData.ProductMetaData product = + findProduct(requestEngine.getPublicProperties()); - if(map != null && - map.size() > 0 && - map.containsKey(this.getElementDataKey())) { + if(product != null) { List properties = new ArrayList<>(); - dataSourceTier = map.get(this.getElementDataKey()).dataTier; + dataSourceTier = product.dataTier; for (AccessiblePropertyMetaData.PropertyMetaData item : - map.get(this.getElementDataKey()).properties) { + product.properties) { AspectPropertyMetaData property = new AspectPropertyMetaDataDefault( item.name, this, @@ -218,12 +233,60 @@ private boolean loadAspectProperties() { aspectProperties = properties; return true; } else { - logger.error("Aspect properties could not be loaded for " + - this.getClass().getName(), this); + // The Cloud Request Engine has logged why the products are not + // known yet. return false; } } + /** + * Find this engine's product among the products the Cloud Request Engine + * reports for the resource key. A valid resource key can include no product + * that holds this engine's properties. That is logged once for this engine, + * as a warning naming its Data Key, and an empty product is returned, so + * the engine builds with empty property metadata and logs nothing on each + * request. + * @param products the products from + * {@link CloudRequestEngine#getPublicProperties()}, or null if they have + * not been loaded + * @return the product, an empty product if the resource key does not + * include it, or null if the products have not been loaded + */ + protected AccessiblePropertyMetaData.ProductMetaData findProduct( + Map products) { + if (products == null) { + return null; + } + AccessiblePropertyMetaData.ProductMetaData product = + products.get(getElementDataKey()); + if (product != null) { + return product; + } + if (productMissingLogged.compareAndSet(false, true)) { + logger.warn(String.format( + MessageProductNotInResourceKey, + getElementDataKey(), + getClass().getSimpleName())); + } + AccessiblePropertyMetaData.ProductMetaData empty = + new AccessiblePropertyMetaData.ProductMetaData(); + empty.properties = new ArrayList<>(); + return empty; + } + + /** + * This engine's part of the cloud response. Where the response has no + * part for this engine, as when the resource key does not include its + * product, an empty object is returned, so the element data holds no + * values and no error is raised. + * @param response the whole cloud response + * @return this engine's part of the response + */ + protected JSONObject getElementJson(JSONObject response) { + JSONObject element = response.optJSONObject(getElementDataKey()); + return element == null ? new JSONObject() : element; + } + private List loadElementProperties( List itemProperties) { List result = null; diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudEvidenceKeyFilter.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudEvidenceKeyFilter.java new file mode 100644 index 000000000..bfcac3997 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudEvidenceKeyFilter.java @@ -0,0 +1,94 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * The Evidence keys the cloud service accepts, from its evidencekeys + * request. It includes nothing until the keys are known, and reads them on + * each call, so a filter that holds this one sees the keys as soon as they + * arrive. It derives from {@link EvidenceKeyFilterWhitelist} so that the + * Pipeline's filter keeps it rather than copying its keys once, and so + * that the web integration lists its headers in the Vary header. + */ +class CloudEvidenceKeyFilter extends EvidenceKeyFilterWhitelist { + + private volatile EvidenceKeyFilterWhitelist keys = null; + + CloudEvidenceKeyFilter() { + super(new ArrayList(), String.CASE_INSENSITIVE_ORDER); + } + + /** + * @return true once the keys are known + */ + boolean isKnown() { + return keys != null; + } + + /** + * Set the keys the cloud service accepts. + * @param evidenceKeys the keys + */ + void setKeys(List evidenceKeys) { + keys = new EvidenceKeyFilterWhitelist( + evidenceKeys, + String.CASE_INSENSITIVE_ORDER); + } + + /** + * @return the keys, or null if they are not known + */ + List getKeys() { + EvidenceKeyFilterWhitelist current = keys; + return current == null ? + null : + new ArrayList<>(current.getWhitelist().keySet()); + } + + @Override + public boolean include(String key) { + EvidenceKeyFilterWhitelist current = keys; + return current != null && current.include(key); + } + + @Override + public Integer order(String key) { + EvidenceKeyFilterWhitelist current = keys; + return current == null ? null : current.order(key); + } + + @Override + public Map getWhitelist() { + EvidenceKeyFilterWhitelist current = keys; + return current == null ? + Collections.emptyMap() : + current.getWhitelist(); + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngine.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngine.java index 1025b5799..d3027f2d0 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngine.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngine.java @@ -41,7 +41,8 @@ public interface CloudRequestEngine /** * A collection of the properties that the cloud service can populate in the * JSON response. Keyed on property name. - * @return public properties + * @return public properties, or null if they are not known yet, after a + * failure when the engine was built that can pass */ Map getPublicProperties(); } diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java index bc72731bf..db9f39443 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilder.java @@ -25,6 +25,8 @@ import fiftyone.pipeline.annotations.DefaultValue; import fiftyone.pipeline.annotations.ElementBuilder; +import fiftyone.pipeline.cloudrequestengine.Constants.Messages; +import fiftyone.pipeline.cloudrequestengine.CloudDiscoveryState; import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.factories.ElementDataFactory; @@ -36,7 +38,12 @@ import fiftyone.pipeline.engines.services.HttpClientDefault; import org.slf4j.ILoggerFactory; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; /** @@ -54,14 +61,22 @@ public class CloudRequestEngineBuilder extends private String evidenceKeysEndpoint = null; private String resourceKey = null; private String licenseKey = null; + private final List requestedProperties = new ArrayList<>(); private String cloudRequestOrigin = null; private int timeoutMillis = Constants.DEFAULT_TIMEOUT_MILLIS; + private CloudDiscoveryState discoveryState = null; + private CloudRequestEngineDefault built = null; // Function to get environment variable value. This enable testable code. private Function getEnvVar = (name) -> { return System.getenv(name); }; + // Set once the use of the deprecated end point variable has been + // logged, so that a process building many engines logs it once. + private static final AtomicBoolean oldEndPointVariableLogged = + new AtomicBoolean(false); + public CloudRequestEngineBuilder(ILoggerFactory loggerFactory) { this(loggerFactory, new HttpClientDefault()); } @@ -75,21 +90,19 @@ public CloudRequestEngineBuilder( @Override protected CloudRequestEngine newEngine(List properties) throws Exception { - if(resourceKey == null || resourceKey.isEmpty()){ - throw new PipelineConfigurationException("A resource key is " + - "required to access the cloud server. Please use the " + - "'setResourceKey(String) method to supply your resource " + - "key obtained from https://configure.51degrees.com?utm_source=code&utm_medium=comment&utm_campaign=51degrees-java&utm_content=pipeline.cloudrequestengine-src-main-java-fiftyone-pipeline-cloudrequestengine-flowelements-cloudrequestenginebuilder.java&utm_term=resource-key-required"); - } + String trimmedResourceKey = trimmed(resourceKey); + String trimmedLicenseKey = trimmed(licenseKey); + List requested = normalized(requestedProperties); + checkCredentials(trimmedResourceKey, trimmedLicenseKey, requested); /* * Check if endPoint has been explicitly set via setEndpoint. - * If not check for environment variable FOD_CLOUD_API_URL. + * If not check for environment variable _51DEGREES_CLOUD_ENDPOINT, + * and then for its deprecated name FOD_CLOUD_API_URL. * If nothing else is set, use default value. */ if (endPoint == null || endPoint.isEmpty()) { - String envVarEndPoint = getEnvVar.apply( - Constants.FOD_CLOUD_API_URL); + String envVarEndPoint = endPointFromEnvironment(); if (envVarEndPoint == null || envVarEndPoint.isEmpty()) { setEndpoint(Constants.END_POINT_DEFAULT); } @@ -98,17 +111,127 @@ protected CloudRequestEngine newEngine(List properties) throws Exception } } - return new CloudRequestEngineDefault( + CloudRequestEngineDefault engine = new CloudRequestEngineDefault( loggerFactory.getLogger(CloudRequestEngine.class.getName()), new CloudRequestDataFactory(loggerFactory), httpClient, dataEndpoint, - resourceKey, - licenseKey, + trimmedResourceKey, + trimmedLicenseKey, propertiesEndpoint, evidenceKeysEndpoint, - timeoutMillis, - cloudRequestOrigin); + timeoutMillis, + cloudRequestOrigin, + requested); + if (discoveryState != null) { + engine.useDiscoveryState(discoveryState); + } + else { + // Fetch the accessible properties and the accepted evidence + // keys now, so that the built engine is ready. A 4xx answer + // fails the build, and any other failure is retried on first + // use. + engine.discover(); + } + built = engine; + return engine; + } + + /** + * Build the engine from a copy of what an earlier engine discovered for + * the same resource key, so that the build makes no discovery request. + * The copy comes from {@link #getDiscoveryState()} after a successful + * build, and can be kept with {@link CloudDiscoveryState#toJson()}. + * @param state the copy, or null to discover when building + * @return this builder + */ + public CloudRequestEngineBuilder setDiscoveryState( + CloudDiscoveryState state) { + this.discoveryState = state; + return this; + } + + /** + * A copy of what the engine this builder last built discovered, to give + * to a later build with {@link #setDiscoveryState(CloudDiscoveryState)}. + * @return the copy, or null if no engine has been built or it does not + * yet know both the accepted evidence keys and the accessible properties + */ + public CloudDiscoveryState getDiscoveryState() { + CloudRequestEngineDefault engine = built; + return engine == null ? null : engine.getDiscoveryState(); + } + + /** + * The end point the environment gives, or null or empty when it gives + * none. {@link Constants#CLOUD_ENDPOINT_VARIABLE} is read first. Only + * when that is not set or is empty is the deprecated + * {@link Constants#FOD_CLOUD_API_URL} read, and the first use of it in + * the process is logged as a warning. + */ + @SuppressWarnings("deprecation") + private String endPointFromEnvironment() { + String value = getEnvVar.apply(Constants.CLOUD_ENDPOINT_VARIABLE); + if (value != null && value.isEmpty() == false) { + return value; + } + value = getEnvVar.apply(Constants.FOD_CLOUD_API_URL); + if (value != null && value.isEmpty() == false + && oldEndPointVariableLogged.compareAndSet(false, true)) { + loggerFactory.getLogger(CloudRequestEngineBuilder.class.getName()) + .warn("The environment variable " + + Constants.FOD_CLOUD_API_URL + " is deprecated and " + + Constants.CLOUD_ENDPOINT_VARIABLE + " replaces it. " + + "Set " + Constants.CLOUD_ENDPOINT_VARIABLE + + " to the same value, as a later release may no " + + "longer read " + Constants.FOD_CLOUD_API_URL + "."); + } + return value; + } + + private static String trimmed(String value) { + return value == null ? "" : value.trim(); + } + + /** + * The property names to send as values, trimmed and sent as given, + * with blank names and repeats left out. A repeat differs only in + * letter case, as the cloud service reads names without regard to + * case, and the first spelling given is kept. + */ + private static List normalized(List names) { + List result = new ArrayList<>(); + Set seen = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (String name : names) { + String normal = trimmed(name); + if (normal.isEmpty() == false && seen.add(normal)) { + result.add(normal); + } + } + return result; + } + + /** + * Refuse the two combinations of keys and requested properties that the + * cloud service refuses on every request, so that a deployment sees one + * configuration error when the engine is built. A blank value counts as + * absent. + */ + private static void checkCredentials( + String resourceKey, + String licenseKey, + List requestedProperties) { + boolean hasResourceKey = resourceKey.isEmpty() == false; + boolean hasLicenseKey = licenseKey.isEmpty() == false; + + if (hasResourceKey == false && hasLicenseKey == false) { + throw new PipelineConfigurationException( + Messages.NoCredential); + } + if (hasResourceKey == false && requestedProperties.isEmpty()) { + throw new PipelineConfigurationException( + Messages.LicenseKeyNeedsProperties); + } } private String getEnvironmentVariable(String name) { @@ -133,7 +256,8 @@ public CloudRequestEngineBuilder setEndpoint(String uri) { uri += '/'; } endPoint = uri; - return setDataEndpoint(uri + (resourceKey != null ? resourceKey + "." : "") + "json") + String key = trimmed(resourceKey); + return setDataEndpoint(uri + (key.isEmpty() ? "" : key + ".") + "json") .setPropertiesEndpoint(uri + "accessibleproperties") .setEvidenceKeysEndpoint(uri + "evidencekeys"); @@ -142,7 +266,8 @@ public CloudRequestEngineBuilder setEndpoint(String uri) { /** * The endpoint the CloudRequestEngine will query to get a processing result. *

    - * By default, this is the endpoint value suffixed with the resourcekey and ".json" + * By default, this is the endpoint value suffixed with the resourcekey and + * ".json", or with "json" alone when no resource key is set * @param uri data endpoint * @return this builder */ @@ -183,11 +308,11 @@ public CloudRequestEngineBuilder setEvidenceKeysEndpoint(String uri) { /** * The resource key to query the endpoint with. *

    - * No default, a value must be supplied + * No default. Either a resource key or a license key must be supplied. * @param resourceKey resource key * @return this builder */ - @DefaultValue("No default - a resource key must be supplied") + @DefaultValue("No default - a resource key or a license key must be supplied") public CloudRequestEngineBuilder setResourceKey(String resourceKey) { this.resourceKey = resourceKey; return this; @@ -196,6 +321,11 @@ public CloudRequestEngineBuilder setResourceKey(String resourceKey) { /** * The license key to query the endpoint with. *

    + * A license key used without a resource key needs the properties + * wanted, named with {@link #setRequestedProperties(List)}. Beside a + * resource key it adds the requested properties the license grants to + * what the resource key carries. + *

    * There is no default * @param licenseKey license key * @return this builder @@ -207,9 +337,51 @@ public CloudRequestEngineBuilder setLicenseKey(String licenseKey) { } /** - * Timeout in seconds for the request to the endpoint. + * The properties each request asks the cloud service for, as fully + * qualified names such as {@code device.ismobile}, sent as values. + * Required with a license key alone. Beside a resource key the cloud + * service adds them to the properties the resource key carries. Names + * are trimmed and sent as given, and a name given again in another + * letter case is sent once. The properties set with + * {@link #setProperty(String)} and {@link #setProperties(List)} are not + * sent. + *

    + * By default no properties are requested + * @param properties the fully qualified property names + * @return this builder + */ + @DefaultValue("None - required with a license key alone") + public CloudRequestEngineBuilder setRequestedProperties( + List properties) { + if (properties == null) { + throw new IllegalArgumentException( + "properties must not be null"); + } + requestedProperties.addAll(properties); + return this; + } + + /** + * As {@link #setRequestedProperties(List)}, taking the names as one + * comma separated string, as a configuration file gives them. + * @param properties comma separated fully qualified property names + * @return this builder + */ + @DefaultValue("None - required with a license key alone") + public CloudRequestEngineBuilder setRequestedProperties( + String properties) { + if (properties == null) { + throw new IllegalArgumentException( + "properties must not be null"); + } + return setRequestedProperties(Arrays.asList(properties.split(","))); + } + + /** + * Timeout in seconds for each request to the cloud service, the two + * discovery requests when the engine is built and each data request. *

    - * Default value is 100 seconds + * Default value is 2 seconds * @param timeout in seconds * @return this builder */ diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java index f3e00e7e7..b95d7ec5d 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDefault.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; +import fiftyone.pipeline.cloudrequestengine.CloudDiscoveryState; import fiftyone.pipeline.cloudrequestengine.CloudRequestException; import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; import fiftyone.pipeline.core.Constants; @@ -36,7 +37,9 @@ import fiftyone.pipeline.engines.flowelements.AspectEngineBase; import fiftyone.pipeline.engines.services.HttpClient; import fiftyone.pipeline.exceptions.AggregateException; +import fiftyone.pipeline.util.LicenseKeyMask; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; @@ -50,6 +53,9 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -63,6 +69,26 @@ public class CloudRequestEngineDefault extends AspectEngineBase implements CloudRequestEngine { + /** + * The Evidence key of the whole web request, which the web integration + * adds for usage sharing to read. It holds every header and cookie of + * the request, so it is never sent to the cloud service. The usage + * sharing package defines the same key, and this package does not + * depend on that one. + */ + private static final String HTTP_REQUEST_KEY = "server.http-request"; + + /** + * Form fields the request reserves for the credentials and the list of + * Properties, with licence, which the cloud service reads as the + * license key where license is absent. Evidence whose key strips to one + * of them is never sent, as the Processing section of the + * specification requires. + */ + private static final Set ENGINE_FIELDS = + Collections.unmodifiableSet(new HashSet<>( + Arrays.asList("resource", "license", "licence", "values"))); + private HttpClient httpClient; private String endPoint; @@ -72,13 +98,55 @@ public class CloudRequestEngineDefault private String evidenceKeysEndpoint; private String cloudRequestOrigin; private Integer timeoutMillis; + private final List requestedProperties; + private final AtomicBoolean requestedPropertiesChecked = + new AtomicBoolean(false); + + /** + * The lowest HTTP status of a client error. A discovery request answered + * with a status from this up to {@link #HTTP_SERVER_ERROR_MIN} is a + * definitive configuration error. + */ + private static final int HTTP_CLIENT_ERROR_MIN = 400; + + /** + * The lowest HTTP status of a server error. + */ + private static final int HTTP_SERVER_ERROR_MIN = 500; private List propertyMetaData; + + /** + * The products the resource key gives access to, from the + * accessibleproperties request, or null until that request succeeds. + */ private volatile Map publicProperties; - private final Object publicPropertiesLock = new Object(); - private volatile EvidenceKeyFilter evidenceKeyFilter; - private final Object evidenceKeyFilterLock = new Object(); + /** + * The answer to the accessibleproperties request that gave + * {@link #publicProperties}, kept for {@link #getDiscoveryState()}. + */ + private volatile String accessibleProperties; + + /** + * The accepted Evidence, from the evidencekeys request. It is the same + * instance before and after the request succeeds. + */ + private final CloudEvidenceKeyFilter evidenceKeyFilter = + new CloudEvidenceKeyFilter(); + + /** + * The prefixes after query, header and cookie, highest precedence + * first, as evidence.md lists them. + */ + private static final String[] OTHER_PREFIX_PRECEDENCE = + { "server", "fiftyone", "location" }; + + /** + * Held while the discovery requests are made, so that concurrent first + * uses make them once. + */ + private final Object discoveryLock = new Object(); public CloudRequestEngineDefault( Logger logger, @@ -124,7 +192,6 @@ public CloudRequestEngineDefault( cloudRequestOrigin); } - @SuppressWarnings("this-escape") public CloudRequestEngineDefault( Logger logger, ElementDataFactory aspectDataFactory, @@ -136,7 +203,42 @@ public CloudRequestEngineDefault( String evidenceKeysEndpoint, int timeoutMillis, String cloudRequestOrigin) { + this( + logger, + aspectDataFactory, + httpClient, + endPoint, + resourceKey, + licenseKey, + propertiesEndpoint, + evidenceKeysEndpoint, + timeoutMillis, + cloudRequestOrigin, + Collections.emptyList()); + } + + /** + * Constructor. + * @param requestedProperties the fully qualified names of the + * properties to ask for, sent as {@code values} + * when the list is not empty + */ + @SuppressWarnings("this-escape") + public CloudRequestEngineDefault( + Logger logger, + ElementDataFactory aspectDataFactory, + HttpClient httpClient, + String endPoint, + String resourceKey, + String licenseKey, + String propertiesEndpoint, + String evidenceKeysEndpoint, + int timeoutMillis, + String cloudRequestOrigin, + List requestedProperties) { super(logger, aspectDataFactory); + this.requestedProperties = Collections.unmodifiableList( + new ArrayList<>(requestedProperties)); this.endPoint = endPoint; this.resourceKey = resourceKey; @@ -184,40 +286,185 @@ public String getElementDataKey() { return "cloud-response"; } + /** + * The accepted Evidence keys from the evidencekeys request. Until that + * request succeeds the filter includes nothing, and it makes no request + * itself. A filter that holds this one, such as the Pipeline's, sees + * the keys as soon as a retry fetches them. + * @return the accepted Evidence + */ @Override - public EvidenceKeyFilter getEvidenceKeyFilter() throws CloudRequestException, AggregateException, PropertyNotLoadedException { - if (evidenceKeyFilter == null) { - synchronized (evidenceKeyFilterLock) { - if (evidenceKeyFilter == null) { - getCloudEvidenceKeys(); + public EvidenceKeyFilter getEvidenceKeyFilter() { + return evidenceKeyFilter; + } + + /** + * The products the resource key gives access to, from the + * accessibleproperties request, which the builder makes when it builds + * the engine. After a failure at build that can pass this is null until + * processing fetches them. It makes no request itself. + * @return the products, or null if they are not known yet + */ + @Override + public Map + getPublicProperties() { + return publicProperties; + } + + /** + * Fetch the accepted Evidence keys and the accessible properties that + * are not known yet. The builder calls this when it builds the engine, + * so in the normal case a built engine is ready to process. Both + * requests are made. A 4xx answer to either is a definitive + * configuration error, and its {@link CloudRequestException} is thrown, + * because retrying with the same configuration can never succeed. Any + * other failure, such as the service being unreachable, a timeout, a + * 5xx answer or a body that cannot be read, is logged as one warning + * and the list that did arrive is kept. Processing fetches what is + * still missing. + * @throws CloudRequestException for a 4xx answer to either request + */ + public void discover() { + List failures = new ArrayList<>(); + synchronized (discoveryLock) { + // The two requests are made at once, so a service that does not + // answer delays the build by one timeout rather than two. + CompletableFuture evidenceKeys = + evidenceKeyFilter.isKnown() ? + CompletableFuture.completedFuture(null) : + CompletableFuture.runAsync(this::fetchEvidenceKeys); + if (publicProperties == null) { + try { + fetchPublicProperties(); + } catch (RuntimeException e) { + failures.add(e); } } + try { + evidenceKeys.join(); + } catch (CompletionException e) { + failures.add(e.getCause() instanceof RuntimeException ? + (RuntimeException) e.getCause() : + e); + } } - return evidenceKeyFilter; + if (failures.isEmpty()) { + return; + } + for (RuntimeException failure : failures) { + CloudRequestException definitive = definitiveError(failure); + if (definitive != null) { + throw definitive; + } + } + RuntimeException first = failures.get(0); + for (int i = 1; i < failures.size(); i++) { + first.addSuppressed(failures.get(i)); + } + logger.warn(MessageDiscoveryDeferred, first); } + /** + * Whether discovery has fetched both the accepted evidence keys and the + * accessible properties, so that what reads them, such as the + * Pipeline's map of available properties, can keep what it reads. + * @return true once both are known + */ @Override - public Map - getPublicProperties() throws CloudRequestException, AggregateException, PropertyNotLoadedException { - if (publicProperties == null) { - synchronized (publicPropertiesLock) { + public boolean hasLoadedProperties() { + return evidenceKeyFilter.isKnown() && publicProperties != null; + } + + /** + * A copy of what discovery fetched, which a later builder can use in + * place of the discovery requests. + * @return the copy, or null unless both the accepted evidence keys and + * the accessible properties are known + */ + public CloudDiscoveryState getDiscoveryState() { + synchronized (discoveryLock) { + List keys = evidenceKeyFilter.getKeys(); + return keys == null || accessibleProperties == null ? + null : + new CloudDiscoveryState(keys, accessibleProperties); + } + } + + /** + * Use a copy of what an earlier discovery fetched, in place of the + * discovery requests. + * @param state the copy + */ + public void useDiscoveryState(CloudDiscoveryState state) { + synchronized (discoveryLock) { + evidenceKeyFilter.setKeys(state.getEvidenceKeys()); + setPublicProperties(state.getAccessibleProperties()); + } + } + + /** + * The failure of a discovery request that the cloud service received + * and rejected with a 4xx status, so that retrying with the same + * configuration can never succeed. + * @param failure the failure of a discovery request + * @return the exception with the 4xx status, or null if there is none + */ + private static CloudRequestException definitiveError(Throwable failure) { + if (failure instanceof CloudRequestException) { + int status = ((CloudRequestException) failure).getHttpStatusCode(); + return status >= HTTP_CLIENT_ERROR_MIN && + status < HTTP_SERVER_ERROR_MIN ? + (CloudRequestException) failure : + null; + } + if (failure instanceof AggregateException) { + for (Throwable inner : failure.getSuppressed()) { + CloudRequestException definitive = definitiveError(inner); + if (definitive != null) { + return definitive; + } + } + } + return null; + } + + /** + * Fetch what discovery could not when the engine was built, evidence + * keys first, stopping at the first failure, which is a process-time + * failure. No data request is sent until both are known. + * @throws IOException if a discovery request could not be made + */ + private void completeDiscovery() throws IOException { + if (evidenceKeyFilter.isKnown() && publicProperties != null) { + return; + } + synchronized (discoveryLock) { + try { + if (evidenceKeyFilter.isKnown() == false) { + fetchEvidenceKeys(); + } if (publicProperties == null) { - getCloudProperties(); + fetchPublicProperties(); + } + } catch (PropertyNotLoadedException e) { + // A request that could not be made is reported as the + // IOException it was, as for the data request. + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); } + throw e; } } - return publicProperties; } @Override protected void processEngine(FlowData data, CloudRequestData aspectData) throws IOException { + ((CloudRequestDataInternal)aspectData).setProcessStarted(true); + completeDiscovery(); + byte[] content = getContent(data); HttpURLConnection connection = httpClient.connect(toUrl(endPoint.trim())); - if (timeoutMillis != null) { - connection.setConnectTimeout(timeoutMillis); - connection.setReadTimeout(timeoutMillis); - } - ((CloudRequestDataInternal)aspectData).setProcessStarted(true); + setTimeouts(connection); Map headers = new HashMap<>(); setCommonHeaders(headers); @@ -229,6 +476,61 @@ protected void processEngine(FlowData data, CloudRequestData aspectData) throws ((CloudRequestDataInternal)aspectData).setJsonResponse(response); validateResponse(response, connection); + reportPropertiesNotCovered(response); + } + + /** + * Compare the properties asked for with the ones the answer carries, and + * warn once about any left out. The service leaves out a requested + * property it does not answer without naming it, and lower-cases the + * names it answers with. A property that is present but null counts as + * answered, as it carries a null reason beside it. The request is never + * failed, because the properties that did come back are usable. With no + * license key, a property the resource key does not carry is answered + * as null, so a name left out is not a property, and the message says + * so. + */ + private void reportPropertiesNotCovered(String response) { + if (requestedProperties.isEmpty() || + requestedPropertiesChecked.compareAndSet(false, true) == false) { + return; + } + JSONObject answer; + try { + answer = new JSONObject(response); + } catch (JSONException e) { + return; + } + List notCovered = new ArrayList<>(); + for (String name : requestedProperties) { + String[] parts = name.split( + Pattern.quote(Constants.EVIDENCE_SEPERATOR), 2); + if (parts.length < 2 || hasKeyIgnoringCase( + answer.optJSONObject(keyIgnoringCase(answer, parts[0])), + parts[1]) == false) { + notCovered.add(name); + } + } + if (notCovered.isEmpty() == false) { + logger.warn(String.format( + hasLicenseKey() ? PropertiesNotCovered : + RequestedNamesNotProperties, + stringJoin(notCovered, ", "))); + } + } + + private static String keyIgnoringCase(JSONObject object, String name) { + for (String key : object.keySet()) { + if (key.equalsIgnoreCase(name)) { + return key; + } + } + return name; + } + + private static boolean hasKeyIgnoringCase(JSONObject object, String name) { + return object != null && + object.has(keyIgnoringCase(object, name)); } /** @@ -248,9 +550,10 @@ private byte[] getContent(FlowData data) throws UnsupportedEncodingException { Map formData = getFormData(data); List formItems = new ArrayList<>(); - formItems.add("resource=" + resourceKey); - if(licenseKey != null && licenseKey.isEmpty() == false){ - formItems.add("license=" + licenseKey); + formItems.addAll(credentialItems()); + if (requestedProperties.isEmpty() == false) { + formItems.add("values=" + URLEncoder.encode( + stringJoin(requestedProperties, ","), "UTF-8")); } List formKeys = Arrays.asList(formData.keySet().toArray(new String[0])); @@ -265,11 +568,53 @@ private byte[] getContent(FlowData data) throws UnsupportedEncodingException { return string.getBytes(StandardCharsets.UTF_8); } + private boolean hasResourceKey() { + return resourceKey != null && resourceKey.isEmpty() == false; + } + + private boolean hasLicenseKey() { + return licenseKey != null && licenseKey.isEmpty() == false; + } + + /** The resource and license keys that are set, as form items. */ + private List credentialItems() throws UnsupportedEncodingException { + List items = new ArrayList<>(); + if (hasResourceKey()) { + items.add("resource=" + URLEncoder.encode(resourceKey, "UTF-8")); + } + if (hasLicenseKey()) { + items.add("license=" + URLEncoder.encode(licenseKey, "UTF-8")); + } + return items; + } + + /** + * The Evidence to send, keyed by the name the cloud service expects. + * Only Evidence that {@link #isSent(String)} allows is included. Keys + * ignore letter case, so one name under two prefixes is one field, + * sent with the name and value of the key that wins by the precedence + * in evidence.md, being query, header, cookie, server, fiftyone and + * location, then any other prefix alphabetically. + * Evidence with no prefix ranks with the other prefixes, unless the + * whole key is query, header or cookie, as {@link #keyPrefix(String)} + * gives the whole key as its prefix. + * @param flowData the FlowData + * @return the form fields to send + */ Map getFormData(FlowData flowData) { - Map evidence = flowData.getEvidence().asKeyMap(); - Map formData = new HashMap<>(); + Map evidence = new HashMap<>(); + for (Map.Entry entry : + flowData.getEvidence().asKeyMap().entrySet()) { + if (isSent(entry.getKey())) { + evidence.put(entry.getKey(), entry.getValue()); + } + } + Map formData = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - // Add evidence in reverse alphabetical order, excluding special keys. + // Add the other evidence lowest precedence first, being other + // prefixes in reverse alphabetical order, then location, fiftyone + // and server. addFormData(formData, evidence, getSelectedEvidence(evidence, "other")); // Add cookie evidence. addFormData(formData, evidence, getSelectedEvidence(evidence, "cookie")); @@ -281,10 +626,50 @@ Map getFormData(FlowData flowData) { return formData; } + /** + * Whether Evidence is sent to the cloud service. Every key is sent, + * including keys the service does not list as accepted and keys with no + * prefix, apart from the web request and a key whose + * {@link #keySuffix(String)} is in {@link #ENGINE_FIELDS}, compared + * without regard to case. + * @param key the Evidence key + * @return true if the Evidence is sent + */ + private boolean isSent(String key) { + return HTTP_REQUEST_KEY.equalsIgnoreCase(key) == false && + ENGINE_FIELDS.contains( + keySuffix(key).toLowerCase(Locale.ROOT)) == false; + } + + /** + * The prefix of an Evidence key, being the part before its first + * separator, or the whole key where it has none. + * @param key the Evidence key + * @return the prefix + */ + private static String keyPrefix(String key) { + int separator = key.indexOf(Constants.EVIDENCE_SEPERATOR); + return separator >= 0 ? key.substring(0, separator) : key; + } + + /** + * The name an Evidence key is sent under, being the part after its + * first separator, or the whole key where it has none. The name can + * hold a dot itself, as id.usage does in query.id.usage. + * @param key the Evidence key + * @return the name to send + */ + private static String keySuffix(String key) { + int separator = key.indexOf(Constants.EVIDENCE_SEPERATOR); + return separator >= 0 ? + key.substring(separator + Constants.EVIDENCE_SEPERATOR.length()) : + key; + } + /** * Add form data to the evidence. * @param formData the destination map to add the data to - * @param allEvidence all evidence in the FlowData. This is used to + * @param allEvidence all the Evidence being sent. This is used to * report which evidence keys are conflicting * @param evidence evidence to add to the form data */ @@ -293,17 +678,20 @@ private void addFormData( Map allEvidence, Map evidence) { - List evidenceKeys = Arrays.asList(evidence.keySet().toArray(new String[0])); - Collections.sort(evidenceKeys, Collections.reverseOrder()); + // Lowest precedence first, so a later key replaces an earlier one. + // Other prefixes are compared as prefixes, in lower case, then the + // whole key, so that the order is the same whatever the map gives. + List evidenceKeys = new ArrayList<>(evidence.keySet()); + evidenceKeys.sort(Comparator + .comparingInt((String key) -> otherPrefixRank(keyPrefix(key))) + .thenComparing((String key) -> + keyPrefix(key).toLowerCase(Locale.ROOT)) + .thenComparing((String key) -> key.toLowerCase(Locale.ROOT)) + .reversed()); for (String evidenceKey : evidenceKeys) { - // Split at the first separator only. Everything after the prefix - // is the name the cloud service expects, and that name can hold - // a dot itself, as id.usage does in query.id.usage. - String[] evidenceKeyParts = evidenceKey.split( - Pattern.quote(Constants.EVIDENCE_SEPERATOR), 2); - String prefix = evidenceKeyParts[0]; - String suffix = evidenceKeyParts[1]; + String prefix = keyPrefix(evidenceKey); + String suffix = keySuffix(evidenceKey); // Check and add the evidence to the query parameters. if (formData.containsKey(suffix) == false) { @@ -315,10 +703,12 @@ private void addFormData( // warning, if the evidence prefix is not query. Otherwise a // warning is not needed as query evidence is expected // to overwrite any existing evidence with the same suffix. - if (prefix.equals("query") == false) { + if (prefix.equalsIgnoreCase("query") == false) { Map conflicts = new HashMap<>(); for (String key : allEvidence.keySet()) { - if (key.equals(evidenceKey) == false && key.contains(suffix)) { + if (key.equals(evidenceKey) == false && + key.toLowerCase(Locale.ROOT).contains( + suffix.toLowerCase(Locale.ROOT))) { conflicts.put(key, allEvidence.get(key)); } } @@ -338,15 +728,33 @@ private void addFormData( conflictStr.toString()); logger.warn(warningMessage); } - // Overwrite the existing queryParameter value. + // Replace the existing field, name and value, so the name + // sent is the winning key's. + formData.remove(suffix); formData.put(suffix, evidence.get(evidenceKey)); } } } + /** + * The precedence of a prefix that is not query, header or cookie, + * lower first, being server, fiftyone and location as evidence.md + * lists them, then any other prefix. + * @param prefix the prefix + * @return the rank, where a lower rank wins + */ + private static int otherPrefixRank(String prefix) { + for (int i = 0; i < OTHER_PREFIX_PRECEDENCE.length; i++) { + if (OTHER_PREFIX_PRECEDENCE[i].equalsIgnoreCase(prefix)) { + return i; + } + } + return OTHER_PREFIX_PRECEDENCE.length; + } + /** * Get evidence with specified prefix. - * @param evidence all evidence in the FlowData + * @param evidence the Evidence being sent * @param type required evidence key prefix * @return evidence with the required key prefix */ @@ -373,13 +781,15 @@ Map getSelectedEvidence(Map evidence, String typ } /** - * Check that the key of a KeyValuePair has the given prefix. + * Check that an Evidence key has the given prefix, being the part + * before its first separator, or the whole key where it has none, + * compared without regard to case. * @param itemKey key to check * @param prefix the prefix to check for * @return true if the key has the prefix */ private boolean hasKeyPrefix(String itemKey, String prefix) { - return itemKey.startsWith(prefix + "."); + return keyPrefix(itemKey).equalsIgnoreCase(prefix); } @Override @@ -414,27 +824,59 @@ private static URL toUrl(String spec) throws MalformedURLException { } } - private void getCloudProperties() throws CloudRequestException, AggregateException, PropertyNotLoadedException { + /** + * Apply the configured timeout to a request. + */ + private void setTimeouts(HttpURLConnection connection) { + if (timeoutMillis != null) { + connection.setConnectTimeout(timeoutMillis); + connection.setReadTimeout(timeoutMillis); + } + } + + /** + * The license key sent with each request, and each key in it when it + * holds several separated by '|', to mask in messages. + */ + private List getLicenseKeys() { + List keys = new ArrayList<>(); + if (licenseKey != null && licenseKey.isEmpty() == false) { + keys.add(licenseKey); + keys.addAll(Arrays.asList(licenseKey.split(Pattern.quote("|")))); + } + return keys; + } + + private void fetchPublicProperties() throws CloudRequestException, AggregateException, PropertyNotLoadedException { try { String jsonResult; Map headers = new HashMap<>(); setCommonHeaders(headers); - HttpURLConnection connection = httpClient.connect(toUrl(propertiesEndpoint.trim() + (resourceKey != null ? "?Resource=" + resourceKey : ""))); - jsonResult = httpClient.getResponseString(connection, headers); - validateResponse(jsonResult, connection); - - JSONObject jsonObj = null; - if (jsonResult.isEmpty() == false) { - jsonObj = new JSONObject(jsonResult); + if (hasLicenseKey()) { + // The credentials go in the body, not the URL, so a license + // key stays out of request logs. + byte[] content = stringJoin(credentialItems(), "&") + .getBytes(StandardCharsets.UTF_8); + headers.put( + "Content-Type", "application/x-www-form-urlencoded"); + headers.put("Content-Length", Integer.toString(content.length)); + HttpURLConnection connection = + httpClient.connect(toUrl(propertiesEndpoint.trim())); + setTimeouts(connection); + jsonResult = httpClient.postData(connection, headers, content); + validateResponse(jsonResult, connection); + } else { + HttpURLConnection connection = httpClient.connect(toUrl( + propertiesEndpoint.trim() + "?Resource=" + resourceKey)); + setTimeouts(connection); + jsonResult = httpClient.getResponseString(connection, headers); + validateResponse(jsonResult, connection); } - if (jsonObj != null) { - AccessiblePropertyMetaData.LicencedProducts accessiblePropertyData = - new AccessiblePropertyMetaData.LicencedProducts(jsonObj.getJSONObject("Products")); - - publicProperties = accessiblePropertyData.products; + if (jsonResult.isEmpty() == false) { + setPublicProperties(jsonResult); } else { throw new RuntimeException("Failed to retrieve available properties " + "from cloud service at " + propertiesEndpoint + "."); @@ -444,7 +886,19 @@ private void getCloudProperties() throws CloudRequestException, AggregateExcepti } } - private void getCloudEvidenceKeys() throws CloudRequestException, AggregateException, PropertyNotLoadedException { + /** + * Read the answer to the accessibleproperties request. + * @param json the answer + */ + private void setPublicProperties(String json) { + AccessiblePropertyMetaData.LicencedProducts accessiblePropertyData = + new AccessiblePropertyMetaData.LicencedProducts( + new JSONObject(json).getJSONObject("Products")); + accessibleProperties = json; + publicProperties = accessiblePropertyData.products; + } + + private void fetchEvidenceKeys() throws CloudRequestException, AggregateException, PropertyNotLoadedException { try { String jsonResult; @@ -452,6 +906,7 @@ private void getCloudEvidenceKeys() throws CloudRequestException, AggregateExcep setCommonHeaders(headers); HttpURLConnection connection = httpClient.connect(toUrl(evidenceKeysEndpoint.trim())); + setTimeouts(connection); jsonResult = httpClient.getResponseString(connection, headers); validateResponse(jsonResult, connection, false); @@ -461,8 +916,7 @@ private void getCloudEvidenceKeys() throws CloudRequestException, AggregateExcep for (int i = 0; i < jsonArray.length(); i++) { keys.add(jsonArray.get(i).toString()); } - evidenceKeyFilter = new EvidenceKeyFilterWhitelist(keys, - String.CASE_INSENSITIVE_ORDER); + evidenceKeyFilter.setKeys(keys); } } catch (IOException e) { throw new PropertyNotLoadedException(e); @@ -496,7 +950,19 @@ private void validateResponse(String jsonResult, List messages = new ArrayList<>(); if (hasData && checkForErrorMessages) { - JSONObject jObj = new JSONObject(jsonResult); + JSONObject jObj; + try { + jObj = new JSONObject(jsonResult); + } catch (JSONException e) { + // The status says whether the failure is definitive, so it + // is kept when the body cannot be read. + CloudRequestException error = new CloudRequestException( + String.format(ExceptionResponseNotJson, code), + code, + connection.getHeaderFields()); + error.initCause(e); + throw error; + } boolean hasErrors = jObj.keySet().contains("errors"); hasData = hasErrors ? jObj.keySet().size() > 1 : @@ -533,6 +999,9 @@ else if (messages.size() == 0 && code != 200) messages.add(message); } + // The cloud's errors can name the license key it was sent. + messages.replaceAll(m -> LicenseKeyMask.maskIn(m, getLicenseKeys())); + Map> headers = null; if (messages.size() > 0){ headers = connection.getHeaderFields(); diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java index f6300acfb..84b3e024e 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/Constants.java @@ -27,7 +27,17 @@ public class Constants { public static final String END_POINT_DEFAULT = "https://cloud.51degrees.com/api/v4"; // Environment variable name to set end point + public static final String CLOUD_ENDPOINT_VARIABLE = "_51DEGREES_CLOUD_ENDPOINT"; + /** + * The old name of the environment variable that sets the end point, + * read only when {@link #CLOUD_ENDPOINT_VARIABLE} is not set. + * @deprecated use {@link #CLOUD_ENDPOINT_VARIABLE}, and set the + * environment variable it names in place of this one. + */ + @Deprecated public static final String FOD_CLOUD_API_URL = "FOD_CLOUD_API_URL"; - public static final int DEFAULT_TIMEOUT_MILLIS = 100000; + // Default timeout for each request to the cloud service, as the + // specification's Configuration options table gives. + public static final int DEFAULT_TIMEOUT_MILLIS = 2000; } diff --git a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java index 414743370..7d8a19821 100644 --- a/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/main/java/fiftyone/pipeline/cloudrequestengine/flowelements/PropertyKeyedCloudEngineBase.java @@ -33,6 +33,7 @@ import fiftyone.pipeline.engines.data.AspectPropertyMetaData; import fiftyone.pipeline.engines.data.MultiProfileData; import fiftyone.pipeline.util.Types; +import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; @@ -72,9 +73,14 @@ protected void processCloudEngine(FlowData data, TData aspectData, String json) // Extract data from json to the aspectData instance. JSONObject map = new JSONObject(json); // Access the data relating to this engine. - JSONObject propertyKeyed = map.getJSONObject(getElementDataKey()); - // Access the 'Profiles' property - for (Object entry : propertyKeyed.getJSONArray("profiles")) { + JSONObject propertyKeyed = getElementJson(map); + // Access the 'Profiles' property. A response without it, as for a + // product the resource key does not include, gives no profiles. + JSONArray profiles = propertyKeyed.optJSONArray("profiles"); + if (profiles == null) { + return; + } + for (Object entry : profiles) { // Iterate through the devices, parsing each one and // adding it to the result. diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java index 514a9369e..4f073192e 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudAspectEngineBaseTests.java @@ -22,7 +22,6 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; -import static fiftyone.pipeline.cloudrequestengine.Constants.Messages.ExceptionFailedToLoadProperties; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -30,10 +29,12 @@ import static org.mockito.Mockito.verify; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import fiftyone.common.testhelpers.TestLogger; import fiftyone.common.testhelpers.TestLoggerFactory; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -57,6 +58,7 @@ import fiftyone.pipeline.engines.data.AspectData; import fiftyone.pipeline.engines.data.AspectDataBase; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.data.MultiProfileData; import fiftyone.pipeline.engines.flowelements.AspectEngine; import fiftyone.pipeline.engines.flowelements.AspectEngineBase; @@ -107,6 +109,55 @@ protected void processCloudEngine(FlowData data, TestData aspectData, String jso } } + public class TestProfiles extends AspectDataBase + implements MultiProfileData { + private final List profiles = new ArrayList<>(); + + public TestProfiles( + Logger logger, + FlowData flowData, + AspectEngine engine) { + super(logger, flowData, engine); + } + + @Override + public List getProfiles() { + return profiles; + } + + @Override + public void addProfile(TestData profile) { + profiles.add(profile); + } + } + + private class TestKeyedInstance + extends PropertyKeyedCloudEngineBase { + public TestKeyedInstance() { + super(loggerFactory.getLogger("TestKeyedInstance"), new ElementDataFactory() { + @Override + public TestProfiles create(FlowData flowData, FlowElement flowElement) { + return new TestProfiles(null, flowData, (AspectEngine) flowElement); + } + }); + } + + @Override + public String getElementDataKey() { + return "test"; + } + + @Override + protected TestData createProfileData(FlowData flowData) { + return new TestData(null, flowData, this); + } + + @Override + protected void unmanagedResourcesCleanup() { + + } + } + private class TestRequestEngine extends AspectEngineBase implements CloudRequestEngine { @@ -227,31 +278,26 @@ public void loadProperties() throws Exception { } /** - * Test that an exception is thrown by the Properties auto property if - * the cloud request engine returns no properties. - * @throws Exception + * A resource key that includes no products at all includes none for this + * engine, so the Pipeline builds, the engine has empty property metadata + * and the missing product is logged once. + * @throws Exception */ @Test public void loadProperties_noProperties() throws Exception { - try { - createPipeline(); - fail("RuntimeException should be thrown"); - } catch (RuntimeException ex){ - assertEquals( - String.format( - ExceptionFailedToLoadProperties, - "test", - "test"), - ex.getMessage() - ); - } - maxErrors = 1; + createPipeline(); + + assertEquals(0, engine.getProperties().size()); + assertProductMissingLoggedOnce(); + maxWarnings = 1; } - + /** - * Test that an exception is thrown by the Properties auto property if - * the cloud engine only returns properties for other engines. - * @throws Exception + * A resource key whose products hold only other engines' properties + * does not include this engine's product. The Pipeline builds, the engine + * has empty property metadata, and the missing product is logged once + * however often the metadata is read. + * @throws Exception */ @Test public void loadProperties_wrongProperties() throws Exception { @@ -263,20 +309,154 @@ public void loadProperties_wrongProperties() throws Exception { devicePropertyData.properties = properties; propertiesReturnedByRequestEngine.put("test2", devicePropertyData); - try { - createPipeline(); - fail("RuntimeException should be thrown"); - } catch (RuntimeException ex){ - assertEquals( - String.format( - ExceptionFailedToLoadProperties, - "test", - "test"), - ex.getMessage() - ); + createPipeline(); + + for (int i = 0; i < 3; i++) { + assertEquals(0, engine.getProperties().size()); } + assertProductMissingLoggedOnce(); + maxWarnings = 1; + } - maxErrors = 1; + /** + * Accessible properties the Cloud Request Engine has not fetched yet, + * after a failure at build that can pass, are not a missing product. + * The Pipeline builds, the metadata is empty and nothing is logged, and + * the metadata is read again once the properties are known rather than + * kept empty. + * @throws Exception + */ + @Test + public void loadProperties_notLoaded() throws Exception { + propertiesReturnedByRequestEngine = null; + createPipeline(); + + assertEquals(0, engine.getProperties().size()); + + List properties = new ArrayList<>(); + properties.add(new AccessiblePropertyMetaData.PropertyMetaData("ismobile", "Boolean", null, null)); + AccessiblePropertyMetaData.ProductMetaData product = new AccessiblePropertyMetaData.ProductMetaData(); + product.properties = properties; + Map products = new HashMap<>(); + products.put("test", product); + requestEngine.setPublicProperties(products); + + assertEquals(1, engine.getProperties().size()); + assertTrue(propertiesContainName(engine.getProperties(), "ismobile")); + } + + /** + * The Pipeline's map of available properties, built before the Cloud + * Request Engine knows the accessible properties, holds the engine's + * properties once they are known rather than keeping what it first saw. + * @throws Exception + */ + @Test + public void elementAvailableProperties_ReadAgainOnceKnown() throws Exception { + propertiesReturnedByRequestEngine = null; + createPipeline(); + + assertTrue( + pipeline.getElementAvailableProperties().get("test") == null || + pipeline.getElementAvailableProperties().get("test").isEmpty()); + + List properties = new ArrayList<>(); + properties.add(new AccessiblePropertyMetaData.PropertyMetaData("ismobile", "Boolean", null, null)); + AccessiblePropertyMetaData.ProductMetaData product = new AccessiblePropertyMetaData.ProductMetaData(); + product.properties = properties; + Map products = new HashMap<>(); + products.put("test", product); + requestEngine.setPublicProperties(products); + + assertTrue(pipeline.getElementAvailableProperties().get("test") + .containsKey("ismobile")); + } + + /** + * A property keyed engine, such as the hardware profile engine, whose + * product the resource key does not include builds, gives no profiles + * for a response that holds nothing for it, and logs nothing on each + * request. + * @throws Exception + */ + @Test + public void propertyKeyed_productMissing_noProfiles() throws Exception { + List properties = new ArrayList<>(); + properties.add(new AccessiblePropertyMetaData.PropertyMetaData("ismobile", "Boolean", null, null)); + AccessiblePropertyMetaData.ProductMetaData otherProduct = new AccessiblePropertyMetaData.ProductMetaData(); + otherProduct.properties = properties; + propertiesReturnedByRequestEngine.put("test2", otherProduct); + TestKeyedInstance keyed = new TestKeyedInstance(); + requestEngine = new TestRequestEngine(); + requestEngine.setPublicProperties(propertiesReturnedByRequestEngine); + pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(requestEngine) + .addFlowElement(keyed) + .build(); + + for (int i = 0; i < 3; i++) { + TestProfiles profiles = new TestProfiles( + null, + Mockito.mock(FlowData.class), + keyed); + keyed.processCloudEngine( + Mockito.mock(FlowData.class), + profiles, + "{ \"test2\": { \"ismobile\": true } }"); + assertEquals(0, profiles.getProfiles().size()); + } + assertEquals(0, keyed.getProperties().size()); + assertProductMissingLoggedOnce(); + maxWarnings = 1; + } + + /** + * A property keyed engine whose product the resource key includes gives + * no profiles, and throws nothing, for a response that holds no part + * for it or a part without profiles, as the Processing section of the + * cloud aspect engine specification says. + * @throws Exception + */ + @Test + public void propertyKeyed_productPresent_noProfiles() throws Exception { + List properties = new ArrayList<>(); + properties.add(new AccessiblePropertyMetaData.PropertyMetaData("ismobile", "Boolean", null, null)); + AccessiblePropertyMetaData.ProductMetaData product = new AccessiblePropertyMetaData.ProductMetaData(); + product.properties = properties; + propertiesReturnedByRequestEngine.put("test", product); + TestKeyedInstance keyed = new TestKeyedInstance(); + requestEngine = new TestRequestEngine(); + requestEngine.setPublicProperties(propertiesReturnedByRequestEngine); + pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(requestEngine) + .addFlowElement(keyed) + .build(); + + for (String response : Arrays.asList( + "{ \"test2\": { \"ismobile\": true } }", + "{ \"test\": { } }")) { + TestProfiles profiles = new TestProfiles( + null, + Mockito.mock(FlowData.class), + keyed); + keyed.processCloudEngine( + Mockito.mock(FlowData.class), + profiles, + response); + assertEquals(0, profiles.getProfiles().size(), response); + } + } + + private void assertProductMissingLoggedOnce() { + List errors = new ArrayList<>(); + List warnings = new ArrayList<>(); + for (TestLogger logger : loggerFactory.loggers) { + errors.addAll(logger.errorsLogged); + warnings.addAll(logger.warningsLogged); + } + assertEquals(1, warnings.size(), warnings.toString()); + assertTrue(warnings.get(0).contains("'test'"), warnings.get(0)); + assertEquals(0, errors.size(), errors.toString()); } /** diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java index 87851ceb0..8159049a0 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineBuilderTests.java @@ -22,9 +22,11 @@ package fiftyone.pipeline.cloudrequestengine.flowelements; +import fiftyone.common.testhelpers.TestLogger; import fiftyone.pipeline.cloudrequestengine.CloudRequestException; import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.engines.services.HttpClientDefault; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.LoggerFactory; @@ -32,6 +34,10 @@ import java.io.IOException; import java.lang.reflect.Field; import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.function.Function; import static org.junit.jupiter.api.Assertions.*; @@ -40,6 +46,17 @@ public class CloudRequestEngineBuilderTests extends CloudRequestEngineTestsBase private final String testResourceKey = "resource_key"; private final String testEndpoint = "https://testEndpoint/"; private final String testEnvVarEndpoint = "https://testEnvVarEndpoint/"; + private final String testOldEnvVarEndpoint = + "https://testOldEnvVarEndpoint/"; + + // The deprecated name of the end point variable, which is also the + // name of the constant that holds it. + private static final String OLD_ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL"; + + // The old name is logged as deprecated once in a process, so what was + // logged is carried from one test of this class to the next. + private static final List deprecationsInEarlierTests = + new ArrayList<>(); public CloudRequestEngineBuilderTests() throws MalformedURLException { super(); @@ -49,6 +66,11 @@ public CloudRequestEngineBuilderTests() throws MalformedURLException { public void setUp() throws IOException { configureMockedClient(); } + + @AfterEach + public void countDeprecations() { + deprecationsInEarlierTests.addAll(deprecationsLogged()); + } @Test @SuppressWarnings("unused") @@ -122,4 +144,126 @@ public void CloudEndPoint_Default_Setting() throws Exception { assertEquals( Constants.END_POINT_DEFAULT + "/", getEndpointValue(builder)); } + + /** + * Gives the builder an environment holding only the variables in the + * map, so that a test chooses which of the two names is set. + */ + private void setEnvironment( + CloudRequestEngineBuilder builder, + Map environment) throws Exception { + Field getEnvVarField = builder.getClass().getDeclaredField("getEnvVar"); + getEnvVarField.setAccessible(true); + getEnvVarField.set( + builder, + (Function)(name) -> environment.get(name)); + } + + /** + * A builder that finds the old name in the environment, and the new + * name only where a value is given for it. + */ + private CloudRequestEngineBuilder builderWithOldName( + String newNameValue) throws Exception { + Map environment = new HashMap<>(); + environment.put(OLD_ENDPOINT_VARIABLE, testOldEnvVarEndpoint); + if (newNameValue != null) { + environment.put(Constants.CLOUD_ENDPOINT_VARIABLE, newNameValue); + } + CloudRequestEngineBuilder builder = + new CloudRequestEngineBuilder(loggerFactory, httpClient); + setEnvironment(builder, environment); + return builder.setResourceKey(testResourceKey); + } + + /** + * The warnings this test's loggers hold that name the old variable. + */ + private List deprecationsLogged() { + List result = new ArrayList<>(); + for (TestLogger logger : loggerFactory.loggers) { + for (String warning : logger.warningsLogged) { + if (warning.contains(OLD_ENDPOINT_VARIABLE)) { + result.add(warning); + } + } + } + return result; + } + + @Test + public void CloudEndPoint_Old_Environment_Name() throws Exception { + CloudRequestEngineBuilder builder = builderWithOldName(null); + + builder.build(); + + assertEquals(testOldEnvVarEndpoint, getEndpointValue(builder)); + assertEquals( + testOldEnvVarEndpoint + "accessibleproperties", + getPropertiesEndpointValue(builder)); + } + + @Test + public void CloudEndPoint_Old_Environment_Name_When_New_Is_Empty() + throws Exception { + CloudRequestEngineBuilder builder = builderWithOldName(""); + + builder.build(); + + assertEquals(testOldEnvVarEndpoint, getEndpointValue(builder)); + } + + @Test + public void CloudEndPoint_New_Environment_Name_Wins() throws Exception { + CloudRequestEngineBuilder builder = + builderWithOldName(testEnvVarEndpoint); + + builder.build(); + + assertEquals(testEnvVarEndpoint, getEndpointValue(builder)); + assertEquals( + testEnvVarEndpoint + "accessibleproperties", + getPropertiesEndpointValue(builder)); + assertEquals(0, deprecationsLogged().size()); + } + + @Test + public void CloudEndPoint_Explicit_Setting_Beats_Old_Name() + throws Exception { + CloudRequestEngineBuilder builder = builderWithOldName(null); + + builder.setEndpoint(testEndpoint).build(); + + assertEquals(testEndpoint, getEndpointValue(builder)); + assertEquals(0, deprecationsLogged().size()); + } + + /** + * Two engines built from the old name log one warning between them, + * however many of this class's tests read the old name before this one. + */ + @Test + public void CloudEndPoint_Old_Environment_Name_Logged_Once() + throws Exception { + builderWithOldName(null).build(); + builderWithOldName(null).build(); + + List logged = new ArrayList<>(deprecationsInEarlierTests); + logged.addAll(deprecationsLogged()); + assertEquals(1, logged.size(), logged.toString()); + String warning = logged.get(0); + assertTrue( + warning.contains(Constants.CLOUD_ENDPOINT_VARIABLE), warning); + assertTrue(warning.contains("deprecated"), warning); + } + + @Test + public void CloudEndPoint_Old_Constant_Is_Kept() throws Exception { + Field old = Constants.class.getField(OLD_ENDPOINT_VARIABLE); + + assertEquals(OLD_ENDPOINT_VARIABLE, old.get(null)); + assertTrue(old.isAnnotationPresent(Deprecated.class)); + assertEquals( + "_51DEGREES_CLOUD_ENDPOINT", Constants.CLOUD_ENDPOINT_VARIABLE); + } } diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java new file mode 100644 index 000000000..aeadfb1ae --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineDiscoveryTests.java @@ -0,0 +1,545 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.common.testhelpers.TestLogger; +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.cloudrequestengine.CloudDiscoveryState; +import fiftyone.pipeline.cloudrequestengine.CloudRequestException; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.io.IOException; +import java.net.ConnectException; +import java.net.HttpURLConnection; +import java.net.SocketTimeoutException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The Cloud Request Engine fetches its accessible properties and accepted + * Evidence keys when it is built. A 4xx answer to either fails the build. + * Any other failure is logged as one warning, the build completes, and + * processing fetches what is missing before it sends a data request. + */ +public class CloudRequestEngineDiscoveryTests { + + private static final String PROPERTIES = "accessibleproperties"; + private static final String EVIDENCE_KEYS = "evidencekeys"; + private static final String DATA = "json"; + + private static final String PROPERTIES_BODY = + "{\"Products\": {\"device\": {\"DataTier\": \"tier\", " + + "\"Properties\": [{\"Name\": \"IsMobile\", \"Type\": \"Boolean\"," + + " \"Category\": \"Device\"}]}}}"; + private static final String EVIDENCE_KEYS_BODY = + "[\"query.User-Agent\", \"header.User-Agent\"]"; + private static final String DATA_BODY = "{\"device\": {\"ismobile\": true}}"; + + /** + * A failure that is not an HTTP status, such as a refused connection. + */ + private static final int UNREACHABLE = -1; + + /** + * A failure where the request hangs until the timeout. + */ + private static final int TIMES_OUT = -2; + + /** + * A 200 answer whose body cannot be read. + */ + private static final int UNREADABLE = -3; + + private static final int TIMEOUT_SECONDS = 3; + + /** + * How long a discovery request waits for the other one to start. + */ + private static final int WAIT_SECONDS = 5; + + /** + * The timeout the page's Configuration options table and .NET give. + */ + private static final int DEFAULT_TIMEOUT_SECONDS = 2; + + private TestLoggerFactory loggerFactory; + private HttpClient httpClient; + private final Map requests = new ConcurrentHashMap<>(); + private final Map answers = new HashMap<>(); + + /** + * A body to answer with in place of the usual one, by endpoint. + */ + private final Map bodies = new HashMap<>(); + private final List connections = + Collections.synchronizedList(new ArrayList()); + + /** + * When set, each discovery request waits until the other has started, + * so the two only complete if they are made at once. + */ + private volatile CountDownLatch bothDiscoveryRequests = null; + + @BeforeEach + public void init() throws IOException { + ILoggerFactory internal = mock(ILoggerFactory.class); + when(internal.getLogger(anyString())).thenReturn(mock(Logger.class)); + loggerFactory = new TestLoggerFactory(internal); + answers.put(PROPERTIES, 200); + answers.put(EVIDENCE_KEYS, 200); + answers.put(DATA, 200); + + httpClient = mock(HttpClient.class); + when(httpClient.connect(any(URL.class))).thenAnswer(invocation -> { + URL url = invocation.getArgument(0); + String name = endpoint(url); + requests.merge(name, 1, Integer::sum); + CountDownLatch latch = bothDiscoveryRequests; + if (latch != null && name.equals(DATA) == false) { + latch.countDown(); + if (latch.await(WAIT_SECONDS, TimeUnit.SECONDS) == false) { + throw new SocketTimeoutException( + "The other discovery request was not made"); + } + } + int answer = answers.get(name); + if (answer == UNREACHABLE) { + throw new ConnectException("Connection refused"); + } + HttpURLConnection connection = mock(HttpURLConnection.class); + when(connection.getURL()).thenReturn(url); + when(connection.getResponseCode()) + .thenReturn(answer < 0 ? 200 : answer); + when(connection.getHeaderFields()) + .thenReturn(new HashMap>()); + connections.add(connection); + return connection; + }); + when(httpClient.getResponseString( + any(HttpURLConnection.class), anyMap())) + .thenAnswer(invocation -> body(invocation.getArgument(0))); + when(httpClient.getResponseString(any(HttpURLConnection.class))) + .thenAnswer(invocation -> body(invocation.getArgument(0))); + when(httpClient.postData( + any(HttpURLConnection.class), anyMap(), any(byte[].class))) + .thenAnswer(invocation -> body(invocation.getArgument(0))); + } + + /** + * In the normal case the built engine is ready. Both lists are fetched + * once, at build, and processing sends only the data request. + */ + @Test + public void Build_FetchesBothLists() throws Exception { + CloudRequestEngine engine = build(); + + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); + assertEquals(0, requests(DATA), "json"); + assertNotNull(engine.getPublicProperties()); + assertTrue(engine.getPublicProperties().containsKey("device")); + assertTrue(engine.getEvidenceKeyFilter().include("header.user-agent")); + assertEquals(0, warnings().size(), warnings().toString()); + + process(engine, false); + + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); + assertEquals(1, requests(DATA), "json"); + } + + private static Stream definitiveFailures() { + List arguments = new ArrayList<>(); + for (int status : new int[] { 400, 401, 403, 404, 408, 429 }) { + arguments.add(Arguments.of(PROPERTIES, status)); + arguments.add(Arguments.of(EVIDENCE_KEYS, status)); + } + return arguments.stream(); + } + + /** + * A 4xx answer to either request is a definitive configuration error, + * which fails the build once both requests have been made. + */ + @ParameterizedTest(name = "{1} from {0}") + @MethodSource("definitiveFailures") + public void Build_4xx_FailsTheBuild(String endpoint, int status) { + answers.put(endpoint, status); + + CloudRequestException error = + assertThrows(CloudRequestException.class, this::build); + + assertEquals(status, error.getHttpStatusCode()); + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); + } + + /** + * A 4xx answer fails the build even when its body is not JSON, such as + * an error page from a proxy, because the status alone makes it + * definitive. + */ + @ParameterizedTest(name = "{0}") + @ValueSource(strings = { PROPERTIES, EVIDENCE_KEYS }) + public void Build_4xxWithBodyNotJson_FailsTheBuild(String endpoint) { + answers.put(endpoint, 403); + bodies.put(endpoint, "Forbidden"); + + CloudRequestException error = + assertThrows(CloudRequestException.class, this::build); + + assertEquals(403, error.getHttpStatusCode()); + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); + } + + /** + * A 4xx answer fails the build even when the other request failed in a + * way that can pass. + */ + @Test + public void Build_4xxAndTransient_FailsTheBuild() { + answers.put(PROPERTIES, 503); + answers.put(EVIDENCE_KEYS, 403); + + CloudRequestException error = + assertThrows(CloudRequestException.class, this::build); + + assertEquals(403, error.getHttpStatusCode()); + } + + private static Stream transientFailures() { + return Stream.of( + Arguments.of("500 from accessibleproperties", 500, 200), + Arguments.of("503 from evidencekeys", 200, 503), + Arguments.of("502 from both", 502, 502), + Arguments.of("unreadable accessibleproperties", UNREADABLE, 200), + Arguments.of("unreadable evidencekeys", 200, UNREADABLE), + Arguments.of("connection refused to both", UNREACHABLE, UNREACHABLE), + Arguments.of("timeout on evidencekeys", 200, TIMES_OUT)); + } + + /** + * Any other failure, the service unreachable, a timeout, a 5xx answer or + * a body that cannot be read, is logged as one warning and the build + * completes. + */ + @ParameterizedTest(name = "{0}") + @MethodSource("transientFailures") + public void Build_TransientFailure_CompletesWithOneWarning( + String name, + int propertiesAnswer, + int evidenceKeysAnswer) throws Exception { + answers.put(PROPERTIES, propertiesAnswer); + answers.put(EVIDENCE_KEYS, evidenceKeysAnswer); + + CloudRequestEngine engine = build(); + + assertNotNull(engine); + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); + assertEquals(1, warnings().size(), warnings().toString()); + } + + /** + * A list that arrived at build is kept, and the other is empty until it + * is fetched. Reading them makes no request. + */ + @Test + public void Build_PartialFailure_KeepsWhatArrived() throws Exception { + answers.put(EVIDENCE_KEYS, 503); + + CloudRequestEngine engine = build(); + + assertNotNull(engine.getPublicProperties()); + assertTrue(engine.getPublicProperties().containsKey("device")); + assertFalse(engine.getEvidenceKeyFilter().include("header.user-agent")); + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(1, requests(EVIDENCE_KEYS), "evidencekeys"); + } + + /** + * Processing fetches only what the build could not, then sends the data + * request. The Pipeline's Evidence filter, built before the retry, sees + * the keys the retry fetched. + */ + @Test + public void Process_FetchesOnlyWhatIsMissing() throws Exception { + answers.put(EVIDENCE_KEYS, 503); + CloudRequestEngine engine = build(); + answers.put(EVIDENCE_KEYS, 200); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build()) { + assertFalse(pipeline.getEvidenceKeyFilter() + .include("header.user-agent")); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "iPhone"); + data.process(); + } + assertTrue(pipeline.getEvidenceKeyFilter() + .include("header.user-agent")); + } + + assertEquals(1, requests(PROPERTIES), "accessibleproperties"); + assertEquals(2, requests(EVIDENCE_KEYS), "evidencekeys"); + assertEquals(1, requests(DATA), "json"); + assertTrue(engine.getEvidenceKeyFilter().include("header.user-agent")); + } + + /** + * While discovery still fails no data request is sent, and the failure + * is recorded as SuppressProcessExceptions says. + */ + @Test + public void Process_DiscoveryStillFailing_Suppressed() throws Exception { + answers.put(PROPERTIES, 503); + CloudRequestEngine engine = build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "iPhone"); + data.process(); + + assertFalse(data.getErrors().isEmpty()); + } + + assertEquals(2, requests(PROPERTIES), "accessibleproperties"); + assertEquals(0, requests(DATA), "json"); + } + + @Test + public void Process_DiscoveryStillFailing_Throws() throws Exception { + answers.put(PROPERTIES, UNREACHABLE); + answers.put(EVIDENCE_KEYS, UNREACHABLE); + CloudRequestEngine engine = build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "iPhone"); + assertThrows(RuntimeException.class, data::process); + } + + assertEquals(0, requests(DATA), "json"); + } + + /** + * The configured timeout applies to the discovery requests, so a + * service that does not answer delays the build by the timeout alone. + */ + @Test + public void Build_AppliesTheTimeoutToDiscovery() throws Exception { + build(); + + assertEquals(2, connections.size()); + for (HttpURLConnection connection : connections) { + verify(connection).setConnectTimeout(TIMEOUT_SECONDS * 1000); + verify(connection).setReadTimeout(TIMEOUT_SECONDS * 1000); + } + } + + /** + * A copy of what discovery fetched is read from the builder after a + * build, kept as JSON, and given to a later build, which makes no + * discovery request and is ready to process. + */ + @Test + public void DiscoveryState_KeptAndUsedWithNoRequest() throws Exception { + CloudRequestEngineBuilder first = builder(); + first.build(); + CloudDiscoveryState state = first.getDiscoveryState(); + assertNotNull(state); + String kept = state.toJson(); + + answers.put(PROPERTIES, UNREACHABLE); + answers.put(EVIDENCE_KEYS, UNREACHABLE); + requests.clear(); + CloudRequestEngine engine = builder() + .setDiscoveryState(CloudDiscoveryState.fromJson(kept)) + .build(); + + assertEquals(0, requests(PROPERTIES), "accessibleproperties"); + assertEquals(0, requests(EVIDENCE_KEYS), "evidencekeys"); + assertTrue(engine.getPublicProperties().containsKey("device")); + assertTrue(engine.getEvidenceKeyFilter().include("header.user-agent")); + assertEquals(0, warnings().size(), warnings().toString()); + + process(engine, false); + + assertEquals(0, requests(PROPERTIES), "accessibleproperties"); + assertEquals(0, requests(EVIDENCE_KEYS), "evidencekeys"); + assertEquals(1, requests(DATA), "json"); + } + + /** + * A build that does not know both lists gives no copy. + */ + @Test + public void DiscoveryState_NoneWithoutBothLists() throws Exception { + answers.put(EVIDENCE_KEYS, 503); + CloudRequestEngineBuilder builder = builder(); + builder.build(); + + assertNull(builder.getDiscoveryState()); + } + + private CloudRequestEngineBuilder builder() { + return new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .setTimeOutSeconds(TIMEOUT_SECONDS); + } + + /** + * The two discovery requests are made at once, as the page allows, so + * a service that does not answer delays the build by one timeout. Each + * request here waits for the other to start, so made one after the + * other they would both fail. + */ + @Test + public void Build_MakesBothRequestsAtOnce() throws Exception { + bothDiscoveryRequests = new CountDownLatch(2); + + CloudRequestEngine engine = build(); + + assertEquals(0, warnings().size(), warnings().toString()); + assertNotNull(engine.getPublicProperties()); + assertTrue(engine.getEvidenceKeyFilter().include("header.user-agent")); + } + + /** + * With no timeout set, the requests time out after 2 seconds. + */ + @Test + public void Build_DefaultTimeoutIsTwoSeconds() throws Exception { + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .build(); + + assertEquals(2, connections.size()); + for (HttpURLConnection connection : connections) { + verify(connection).setConnectTimeout( + DEFAULT_TIMEOUT_SECONDS * 1000); + verify(connection).setReadTimeout(DEFAULT_TIMEOUT_SECONDS * 1000); + } + } + + private CloudRequestEngine build() throws Exception { + return new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resource_key") + .setTimeOutSeconds(TIMEOUT_SECONDS) + .build(); + } + + private void process(CloudRequestEngine engine, boolean suppress) + throws Exception { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(suppress) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "iPhone"); + data.process(); + } + } + + private int requests(String endpoint) { + return requests.getOrDefault(endpoint, 0); + } + + private List warnings() { + List warnings = new ArrayList<>(); + for (TestLogger logger : loggerFactory.loggers) { + warnings.addAll(logger.warningsLogged); + } + return warnings; + } + + private static String endpoint(URL url) { + String path = url.getPath().toLowerCase(); + if (path.endsWith(PROPERTIES)) { + return PROPERTIES; + } + if (path.endsWith(EVIDENCE_KEYS)) { + return EVIDENCE_KEYS; + } + return DATA; + } + + private String body(HttpURLConnection connection) throws IOException { + String name = endpoint(connection.getURL()); + int answer = answers.get(name); + if (answer == TIMES_OUT) { + throw new SocketTimeoutException("Read timed out"); + } + if (answer == UNREADABLE) { + return "proxy error"; + } + if (bodies.containsKey(name)) { + return bodies.get(name); + } + if (answer != 200) { + return "{\"errors\": [\"" + name + " answered " + answer + "\"]}"; + } + switch (name) { + case PROPERTIES: + return PROPERTIES_BODY; + case EVIDENCE_KEYS: + return EVIDENCE_KEYS_BODY; + default: + return DATA_BODY; + } + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java new file mode 100644 index 000000000..e05e9bc09 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLicenseKeyTests.java @@ -0,0 +1,640 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +/** + * The credentials a Cloud Request Engine accepts, what it sends for each + * and that a license key stays out of URLs, logs and messages. + */ +public class CloudRequestEngineLicenseKeyTests { + + private static final String BASE = "https://cloud.example/api/v4/"; + private static final String LICENSE_KEY = "abcdefghijkl1234"; + private static final String RESOURCE_KEY = "resourcekey"; + private static final List PROPERTY_LIST = + Arrays.asList("device.ismobile", "device.iscrawler"); + + /** A request the engine made. */ + private static class Call { + final String method; + final URL url; + final String body; + + Call(String method, URL url, String body) { + this.method = method; + this.url = url; + this.body = body; + } + } + + /** Records every request and answers from fixed responses. */ + private static class FakeCloud implements HttpClient { + // Discovery makes its two requests at once, so connections and + // calls are recorded from two threads. + final List calls = + Collections.synchronizedList(new ArrayList<>()); + String dataResponse = "{\"device\":{\"ismobile\":true," + + "\"iscrawler\":false}}"; + int dataStatus = 200; + String propertiesResponse = + "{\"Products\":{\"device\":{\"DataTier\":\"t\",\"Properties\":" + + "[{\"Name\":\"IsMobile\",\"Type\":\"Boolean\"," + + "\"Category\":\"Device\"}]}}}"; + String evidenceKeysResponse = "[\"query.User-Agent\"]"; + private final Map urls = + new ConcurrentHashMap<>(); + + @Override + public HttpURLConnection connect(URL url) { + HttpURLConnection connection = mock(HttpURLConnection.class); + try { + when(connection.getURL()).thenReturn(url); + when(connection.getResponseCode()).thenReturn( + url.getPath().endsWith("json") ? dataStatus : 200); + } catch (IOException e) { + throw new IllegalStateException(e); + } + when(connection.getHeaderFields()).thenReturn( + Collections.emptyMap()); + urls.put(connection, url); + return connection; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + URL url = urls.get(connection); + calls.add(new Call( + "POST", + url, + data == null ? "" : + new String(data, StandardCharsets.UTF_8))); + return answer(url); + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return getResponseString(connection, null); + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + URL url = urls.get(connection); + calls.add(new Call("GET", url, "")); + return answer(url); + } + + private String answer(URL url) { + if (url.getPath().endsWith("accessibleproperties")) { + return propertiesResponse; + } + if (url.getPath().endsWith("evidencekeys")) { + return evidenceKeysResponse; + } + return dataResponse; + } + + List callsTo(String pathEnd) { + List found = new ArrayList<>(); + for (Call call : calls) { + if (call.url.getPath().endsWith(pathEnd)) { + found.add(call); + } + } + return found; + } + } + + private FakeCloud cloud; + private Logger logger; + private ILoggerFactory loggerFactory; + + @BeforeEach + public void setUp() { + cloud = new FakeCloud(); + logger = mock(Logger.class); + loggerFactory = mock(ILoggerFactory.class); + when(loggerFactory.getLogger(anyString())).thenReturn(logger); + } + + private CloudRequestEngineBuilder builder() { + return new CloudRequestEngineBuilder(loggerFactory, cloud); + } + + /** + * Build with the test endpoint set last, as the default endpoint is when + * no endpoint is set, so the data endpoint's route reflects the keys. + */ + private CloudRequestEngine build(CloudRequestEngineBuilder builder) + throws Exception { + return builder.setEndpoint(BASE).build(); + } + + private String process(CloudRequestEngine engine) throws Exception { + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + StringBuilder errors = new StringBuilder(); + for (FlowError error : data.getErrors()) { + for (Throwable t = error.getThrowable(); t != null; + t = t.getCause()) { + errors.append(t).append('\n'); + } + } + return errors.toString(); + } + } + + private static Map form(String body) + throws UnsupportedEncodingException { + Map fields = new HashMap<>(); + for (String pair : body.split("&")) { + if (pair.isEmpty()) { + continue; + } + String[] parts = pair.split("=", 2); + fields.put( + URLDecoder.decode(parts[0], "UTF-8"), + URLDecoder.decode(parts.length > 1 ? parts[1] : "", "UTF-8")); + } + return fields; + } + + private Map dataRequest(String pathEnd) + throws UnsupportedEncodingException { + List posts = cloud.callsTo(pathEnd); + assertEquals(1, posts.size()); + assertEquals("POST", posts.get(0).method); + return form(posts.get(0).body); + } + + private void assertRefused( + CloudRequestEngineBuilder builder, + String... expectedInMessage) { + int sent = cloud.calls.size(); + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + () -> build(builder)); + for (String expected : expectedInMessage) { + assertTrue( + e.getMessage().contains(expected), + "Message should contain '" + expected + "' but was: " + + e.getMessage()); + } + assertEquals(sent, cloud.calls.size(), "A refused build sends nothing."); + } + + // The rows of the specification's table of accepted combinations. + + @Test + public void Build_ResourceKeyOnly_Accepted() throws Exception { + assertNotNull(build(builder().setResourceKey(RESOURCE_KEY))); + } + + @Test + public void Build_ResourceKeyAndLicenseKey_Accepted() throws Exception { + assertNotNull(build(builder() + .setResourceKey(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY))); + } + + @Test + public void Build_AllThree_Accepted() throws Exception { + assertNotNull(build(builder() + .setResourceKey(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(PROPERTY_LIST))); + } + + @Test + public void Build_LicenseKeyAndProperties_Accepted() throws Exception { + assertNotNull(build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(PROPERTY_LIST))); + } + + @Test + public void Build_LicenseKeyWithoutProperties_Refused() { + assertRefused( + builder().setLicenseKey(LICENSE_KEY), + "answers 400", + "setRequestedProperties"); + } + + @Test + public void Build_ResourceKeyWithProperties_Accepted() throws Exception { + assertNotNull(build(builder() + .setResourceKey(RESOURCE_KEY) + .setRequestedProperties("device.hardwarevendor"))); + } + + @Test + public void Build_NoCredential_Refused() { + assertRefused(builder(), "401", "resource key"); + assertRefused(builder().setRequestedProperties(PROPERTY_LIST), "401"); + } + + @Test + public void Build_BlankValuesCountAsAbsent() throws Exception { + assertRefused( + builder().setResourceKey(" ").setLicenseKey("\n"), + "401"); + assertRefused( + builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList(" ", "")), + "answers 400"); + assertNotNull(build(builder() + .setResourceKey(RESOURCE_KEY) + .setLicenseKey(" ") + .setRequestedProperties(" "))); + } + + @Test + public void Build_RefusalsDoNotContainTheLicenseKey() { + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + () -> build(builder().setLicenseKey(LICENSE_KEY))); + assertFalse(e.getMessage().contains(LICENSE_KEY)); + } + + // Requests. + + @Test + public void Process_LicenseKeyAlone_SendsLicenseAndValues() + throws Exception { + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY + "\n") + .setRequestedProperties( + Arrays.asList(" Device.IsMobile", "device.iscrawler "))); + + assertEquals("", process(engine)); + + Map fields = dataRequest("/api/v4/json"); + assertEquals(LICENSE_KEY, fields.get("license")); + assertEquals("Device.IsMobile,device.iscrawler", fields.get("values")); + assertFalse(fields.containsKey("resource")); + assertFalse(fields.containsKey("licence")); + assertEquals("iPhone", fields.get("User-Agent")); + } + + @Test + public void Process_ResourceKey_SendsResourceOnly() throws Exception { + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY)); + + assertEquals("", process(engine)); + + Map fields = dataRequest(RESOURCE_KEY + ".json"); + assertEquals(RESOURCE_KEY, fields.get("resource")); + assertFalse(fields.containsKey("license")); + assertFalse(fields.containsKey("values")); + } + + @Test + public void Process_ResourceKeyAndLicenseKey_SendsBoth() + throws Exception { + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY)); + + assertEquals("", process(engine)); + + Map fields = dataRequest(RESOURCE_KEY + ".json"); + assertEquals(RESOURCE_KEY, fields.get("resource")); + assertEquals(LICENSE_KEY, fields.get("license")); + assertFalse(fields.containsKey("values")); + } + + @Test + public void Process_ResourceKeyAndProperties_SendsResourceAndValues() + throws Exception { + cloud.dataResponse = "{\"device\":{\"ismobile\":true," + + "\"hardwarevendor\":null," + + "\"hardwarevendornullreason\":\"Paid feature\"}}"; + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setRequestedProperties("Device.HardwareVendor")); + + assertEquals("", process(engine)); + + Map fields = dataRequest(RESOURCE_KEY + ".json"); + assertEquals(RESOURCE_KEY, fields.get("resource")); + assertEquals("Device.HardwareVendor", fields.get("values")); + assertFalse(fields.containsKey("license")); + verify(logger, never()).warn(anyString()); + } + + @Test + public void Process_AllThree_SendsResourceLicenseAndValues() + throws Exception { + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties("IP.RegisteredName")); + cloud.dataResponse = "{\"device\":{\"ismobile\":true}," + + "\"ip\":{\"registeredname\":\"Example\"}}"; + + assertEquals("", process(engine)); + + Map fields = dataRequest(RESOURCE_KEY + ".json"); + assertEquals(RESOURCE_KEY, fields.get("resource")); + assertEquals(LICENSE_KEY, fields.get("license")); + assertEquals("IP.RegisteredName", fields.get("values")); + for (Call call : cloud.calls) { + assertFalse(call.url.toString().contains(LICENSE_KEY)); + } + } + + @Test + public void Process_RequestedPropertiesAsText_Sent() throws Exception { + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties("Device.IsMobile, device.iscrawler ,")); + + assertEquals("", process(engine)); + + Map fields = dataRequest("/api/v4/json"); + assertEquals("Device.IsMobile,device.iscrawler", fields.get("values")); + } + + /** + * Properties set with setProperty or setProperties are not sent, as + * before, so a configuration that lists them keeps the answer it had. A + * license key with only those properties is refused. + */ + @Test + public void Process_SetProperties_NotSent() throws Exception { + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setProperties(Arrays.asList("IsMobile", "DeviceId"))); + + assertEquals("", process(engine)); + + Map fields = dataRequest(RESOURCE_KEY + ".json"); + assertEquals(RESOURCE_KEY, fields.get("resource")); + assertFalse(fields.containsKey("values")); + verify(logger, never()).warn(anyString()); + + assertRefused( + builder() + .setLicenseKey(LICENSE_KEY) + .setProperty("device.ismobile"), + "answers 400"); + } + + // Discovery. + + @Test + public void Discovery_LicenseKeyAlone_PostsCredentialsInTheBody() + throws Exception { + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(PROPERTY_LIST)); + + Map products = + engine.getPublicProperties(); + assertTrue(products.containsKey("device")); + assertNotNull(engine.getEvidenceKeyFilter()); + + List properties = cloud.callsTo("accessibleproperties"); + assertEquals(1, properties.size()); + assertEquals("POST", properties.get(0).method); + assertNull(properties.get(0).url.getQuery()); + assertFalse(properties.get(0).url.toString().contains(LICENSE_KEY)); + Map fields = form(properties.get(0).body); + assertEquals(LICENSE_KEY, fields.get("license")); + assertFalse(fields.containsKey("resource")); + + List evidence = cloud.callsTo("evidencekeys"); + assertEquals(1, evidence.size()); + assertEquals("GET", evidence.get(0).method); + assertNull(evidence.get(0).url.getQuery()); + } + + @Test + public void Discovery_ResourceKeyAndProperties_GetsWithTheKeyInTheQuery() + throws Exception { + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setRequestedProperties("device.hardwarevendor")); + + engine.getPublicProperties(); + + List properties = cloud.callsTo("accessibleproperties"); + assertEquals(1, properties.size()); + assertEquals("GET", properties.get(0).method); + assertEquals( + "Resource=" + RESOURCE_KEY, + properties.get(0).url.getQuery()); + } + + @Test + public void Discovery_BothKeys_PostsBothInTheBody() throws Exception { + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setLicenseKey(LICENSE_KEY)); + + engine.getPublicProperties(); + + Call call = cloud.callsTo("accessibleproperties").get(0); + assertEquals("POST", call.method); + assertNull(call.url.getQuery()); + Map fields = form(call.body); + assertEquals(RESOURCE_KEY, fields.get("resource")); + assertEquals(LICENSE_KEY, fields.get("license")); + } + + // Keys in URLs, logs and messages. + + @Test + public void Process_FailedRequest_LicenseKeyInNoUrlLogOrMessage() + throws Exception { + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList( + "device.ismobile", "device.other"))); + assertEquals("", process(engine)); + + cloud.dataStatus = 500; + cloud.dataResponse = "{\"status\":\"failure\"}"; + String errors = process(engine); + + assertTrue(errors.contains("500"), errors); + assertFalse(errors.contains(LICENSE_KEY), errors); + for (Call call : cloud.calls) { + assertFalse(call.url.toString().contains(LICENSE_KEY)); + } + assertFalse(mockingDetails(logger).getInvocations().isEmpty()); + mockingDetails(logger).getInvocations().forEach(i -> + assertFalse( + Arrays.deepToString(i.getArguments()).contains(LICENSE_KEY), + i.toString())); + } + + // Properties the service did not return. + + @Test + public void Process_PropertyDropped_ReportedOnceAndRequestSucceeds() + throws Exception { + cloud.dataResponse = "{\"device\":{\"ismobile\":true}}"; + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties( + Arrays.asList("Device.IsMobile", "Device.IsCrawler"))); + + assertEquals("", process(engine)); + assertEquals("", process(engine)); + + verify(logger, times(1)).warn(argThat( + (String m) -> m.contains("Device.IsCrawler") && + m.contains("not covered by the keys") && + !m.contains("Device.IsMobile"))); + verify(logger, times(1)).warn(anyString()); + } + + /** + * With a resource key, requested properties and no license key, a + * property no licence grants is answered as null, so a name the + * answer leaves out is not a property. The report says so, and names + * no license key, because none was given. + */ + @Test + public void Process_ResourceKeyNameNotAnswered_ReportedAsNotAProperty() + throws Exception { + cloud.dataResponse = "{\"device\":{\"ismobile\":true}}"; + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY) + .setRequestedProperties( + Arrays.asList("Device.IsMobile", "device.notaproperty"))); + + assertEquals("", process(engine)); + assertEquals("", process(engine)); + + verify(logger, times(1)).warn(argThat( + (String m) -> m.contains("device.notaproperty") && + m.contains("not properties") && + !m.contains("license") && + !m.contains("not covered"))); + verify(logger, times(1)).warn(anyString()); + } + + /** + * A name given twice in different letter cases is sent once, as it + * was first given, because the cloud service reads property names + * without regard to case. + */ + @Test + public void Process_NameGivenTwice_SentOnceAsFirstGiven() + throws Exception { + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(Arrays.asList( + "Device.IsMobile", "device.ismobile", "device.iscrawler"))); + + assertEquals("", process(engine)); + + assertEquals("Device.IsMobile,device.iscrawler", + dataRequest("/api/v4/json").get("values")); + } + + /** + * A null list of requested properties is refused with the name of + * the parameter, as .NET refuses it, rather than failing inside the + * list. + */ + @Test + public void SetRequestedProperties_Null_Refused() { + IllegalArgumentException list = assertThrows( + IllegalArgumentException.class, + () -> builder().setRequestedProperties((List) null)); + assertTrue(list.getMessage().contains("properties")); + IllegalArgumentException text = assertThrows( + IllegalArgumentException.class, + () -> builder().setRequestedProperties((String) null)); + assertTrue(text.getMessage().contains("properties")); + } + + @Test + public void Process_PropertyPresentButNull_CountsAsAnswered() + throws Exception { + cloud.dataResponse = "{\"device\":{\"ismobile\":true," + + "\"iscrawler\":null,\"iscrawlernullreason\":\"No data\"}}"; + CloudRequestEngine engine = build(builder() + .setLicenseKey(LICENSE_KEY) + .setRequestedProperties(PROPERTY_LIST)); + + assertEquals("", process(engine)); + + verify(logger, never()).warn(anyString()); + } + + @Test + public void Process_ResourceKey_NothingToCompare() throws Exception { + cloud.dataResponse = "{\"device\":{\"ismobile\":true}}"; + CloudRequestEngine engine = build(builder() + .setResourceKey(RESOURCE_KEY)); + + assertEquals("", process(engine)); + + verify(logger, never()).warn(anyString()); + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java new file mode 100644 index 000000000..8f8487633 --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineLiveTests.java @@ -0,0 +1,139 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +import fiftyone.pipeline.cloudrequestengine.data.CloudRequestData; +import fiftyone.pipeline.core.data.AccessiblePropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.HttpClientDefault; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Calls the live 51Degrees cloud service through the cloud request + * engine, end to end. + *

    + * Every other test in this module answers the engine from a mock, which + * checks the request the engine builds and the way it reads a reply, and + * proves nothing about the service on the other end. A field the service + * stops writing, a header it starts refusing or an endpoint that moves + * all leave those tests green. This one makes the three calls the engine + * really makes, which are accessible properties and evidence keys when it + * is built, and the JSON request when it processes, so a change in the + * service is seen here. + *

    + * {@code _51DEGREES_CLOUD_ENDPOINT} points the engine at another host, + * which the builder reads for itself. + */ +public class CloudRequestEngineLiveTests { + + /** + * A real browser User-Agent, so the service has something to detect + * rather than a string it will report as unknown. + */ + private static final String USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"; + + @Test + public void build_Process_ReadsDeviceDataFromTheLiveService() + throws Exception { + String resourceKey = + LiveCloudKeys.find(LiveCloudKeys.RESOURCE_KEY_NAMES); + if (resourceKey == null) { + fail(LiveCloudKeys.noResourceKey()); + } + + CloudRequestEngineBuilder builder = new CloudRequestEngineBuilder( + LoggerFactory.getILoggerFactory(), new HttpClientDefault()) + .setResourceKey(resourceKey); + String licenceKey = + LiveCloudKeys.find(LiveCloudKeys.LICENCE_KEY_NAMES); + if (licenceKey != null) { + builder.setLicenseKey(licenceKey); + } + + // Building asks the service for its accessible properties and its + // evidence keys, so a service that cannot be reached, or a key it + // refuses, fails here. + CloudRequestEngine engine = builder.build(); + + Map products = + engine.getPublicProperties(); + assertNotNull(products, + "The live cloud service returned no accessible properties at " + + "all, which is the service or the key rather than this " + + "package."); + assertFalse(products.isEmpty(), + "The live cloud service listed no products for this resource " + + "key. A key carries the products its account holds, so this " + + "is an entitlement problem and not a fault in the service. " + + "Create a key at https://configure.51degrees.com."); + assertTrue(products.containsKey("device"), + "The live cloud service listed the products " + + String.join(", ", products.keySet()) + " for this resource " + + "key, and none of them is the device product this test asks " + + "for. That is the key not carrying the device product rather " + + "than a fault in the service."); + + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(engine) + .build()) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", USER_AGENT).process(); + + CloudRequestData requestData = data.getFromElement(engine); + assertNotNull(requestData, + "The cloud request engine put no data in the flow, so " + + "the request to the live cloud service produced " + + "nothing to read."); + String json = requestData.getJsonResponse(); + assertNotNull(json, + "The live cloud service returned no JSON body for a " + + "request carrying a User-Agent."); + + JSONObject answer = new JSONObject(json); + assertTrue(answer.has("device"), + "The live cloud service answered with " + json + + ", which carries no device element. The resource key " + + "lists the device product, so this is the service " + + "rather than entitlement."); + JSONObject device = answer.getJSONObject("device"); + assertTrue(device.length() > 0, + "The live cloud service returned an empty device " + + "element for a real browser User-Agent."); + } + } + } +} diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java index 01b82a180..f7b274895 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTests.java @@ -33,20 +33,28 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.engines.services.HttpClientDefault; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.NullSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; import java.io.IOException; import java.net.MalformedURLException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -101,6 +109,66 @@ public void Process() throws Exception { } } + /** + * The cloud answers a refused license key with an error that names it. + * The exception the engine raises, and every log entry, show the key + * only by its last 4 characters. + */ + @Test + public void Process_LicenseKeyInCloudError_Masked() throws Exception { + final String licenseKey = "TESTKEYNOTREALABCDEFGHIJ1234"; + final String masked = "************************1234"; + jsonResponse = "{\"errors\":[\"Invalid license key(s) supplied: " + + licenseKey + "\"]}"; + configureMockedClient(); + Logger logger = mock(Logger.class); + when(logger.isDebugEnabled()).thenReturn(true); + when(logger.isInfoEnabled()).thenReturn(true); + when(logger.isWarnEnabled()).thenReturn(true); + when(logger.isErrorEnabled()).thenReturn(true); + ILoggerFactory capturingLoggerFactory = mock(ILoggerFactory.class); + when(capturingLoggerFactory.getLogger(anyString())).thenReturn(logger); + + CloudRequestEngine engine = new CloudRequestEngineBuilder( + capturingLoggerFactory, httpClient) + .setResourceKey("resource_key") + .setLicenseKey(licenseKey) + .build(); + + try (Pipeline pipeline = new PipelineBuilder(capturingLoggerFactory) + .setSuppressProcessException(true) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.process(); + + assertFalse(data.getErrors().isEmpty()); + StringBuilder reported = new StringBuilder(); + for (FlowError error : data.getErrors()) { + reported.append(printed(error.getThrowable())); + } + for (org.mockito.invocation.Invocation invocation : + mockingDetails(logger).getInvocations()) { + for (Object argument : invocation.getArguments()) { + reported.append(argument instanceof Throwable ? + printed((Throwable) argument) : + String.valueOf(argument)); + } + } + assertFalse(reported.toString().contains(licenseKey), + reported.toString()); + assertTrue(reported.toString().contains(masked), + reported.toString()); + } + } + + private static String printed(Throwable throwable) { + java.io.StringWriter text = new java.io.StringWriter(); + throwable.printStackTrace(new java.io.PrintWriter(text)); + return text.toString(); + } + /** * Test errors thrown by cloud request engine are added to flow.errors when SuppressProcessExceptions == true */ @@ -263,14 +331,11 @@ private static Stream getValidateErrorArgs() { } /** - * Test cloud request engine handles errors from the cloud service - * as expected. - * An AggregateException should be thrown by the cloud request engine - * containing the errors from the cloud service - * and the pipeline is configured to throw any exceptions up - * the stack in an AggregateException. - * We also check that the exception message includes the content - * from the JSON response. + * Errors from the cloud service when the engine is built. A 4xx answer + * is a definitive configuration error, which fails the build with a + * CloudRequestException that carries the cloud's message. Any other + * answer that holds errors is logged as one warning, the build + * completes, and the accessible properties are unknown until a retry. */ @ParameterizedTest(name = "response code {1} - {0}") @MethodSource("getValidateErrorArgs") @@ -284,24 +349,33 @@ public void validateErrorHandling(String name, int responseCode, boolean include configureMockedClient(); - Exception exception = null; - - try { - new CloudRequestEngineBuilder(loggerFactory, httpClient) - .setResourceKey(resourceKey) - .build() - .getPublicProperties(); - } catch (Exception ex) { - exception = ex; + if (responseCode >= 400 && responseCode < 500) { + CloudRequestException cloudEx = assertThrows( + CloudRequestException.class, + () -> new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey(resourceKey) + .build()); + assertEquals(responseCode, cloudEx.getHttpStatusCode()); + if (includeMessage) { + assertTrue(cloudEx.getMessage().contains(errorMessage), + "Exception message did not contain the expected text."); + } + } else { + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey(resourceKey) + .build(); + assertNull(engine.getPublicProperties()); + assertEquals(1, warningCount()); } + } - assertNotNull(exception, "Expected exception to occur"); - assertTrue(exception instanceof CloudRequestException); - CloudRequestException cloudEx = (CloudRequestException) exception; - if (includeMessage) { - assertTrue(cloudEx.getMessage().contains(errorMessage), - "Exception message did not contain the expected text."); + private int warningCount() { + int count = 0; + for (TestLogger logger : loggerFactory.loggers) { + count += logger.warningsLogged.size(); } + return count; } private static Stream getPrecidenceArgs() { @@ -326,6 +400,7 @@ public void evidencePrecidence(String name, boolean shouldWarn, String evidence1 loggerFactory.loggers.clear(); String[] evidence1Parts = evidence1.split("="); String[] evidence2Parts = evidence2.split("="); + acceptEvidence(Arrays.asList(evidence1Parts[0], evidence2Parts[0])); configureMockedClient(); CloudRequestEngine engine = new CloudRequestEngineBuilder(loggerFactory, httpClient) @@ -446,6 +521,7 @@ private static Stream getFormDataArgs() { @ParameterizedTest(name = "{0}") @MethodSource("getFormDataArgs") public void getFormData(String name, Map evidence, String expectedValue) throws Exception { + acceptEvidence(evidence.keySet()); configureMockedClient(); CloudRequestEngineDefault engine = (CloudRequestEngineDefault) new CloudRequestEngineBuilder(loggerFactory, httpClient) .setResourceKey("resourcekey") @@ -464,6 +540,94 @@ public void getFormData(String name, Map evidence, String expect } } + private static Stream oneNameInTwoCasesArgs() { + return Stream.of( + Arguments.of("header.User-Agent", "query.user-agent", "user-agent", "B"), + Arguments.of("query.User-Agent", "header.user-agent", "User-Agent", "A"), + Arguments.of("cookie.X-Thing", "header.x-thing", "x-thing", "B"), + Arguments.of("location.Value", "server.value", "value", "B"), + Arguments.of("location.Value", "fiftyone.value", "value", "B"), + Arguments.of("fiftyone.Value", "server.value", "value", "B"), + Arguments.of("server.Value", "a.value", "Value", "A"), + Arguments.of("b.Value", "a.value", "value", "B"), + Arguments.of("ab-c.Value", "ab.value", "value", "B"), + Arguments.of("a.Value", "_x.value", "value", "B")); + } + + /** + * Evidence keys ignore letter case, so one name under two prefixes in + * two letter cases is one field. It is sent once, with the name and + * value of the key that wins by the precedence in evidence.md, being + * query, header, cookie, server, fiftyone and location, then any other + * prefix alphabetically. Two other prefixes are compared as prefixes, + * in lower case, so ab ranks before ab-c and _x before a. + */ + @ParameterizedTest(name = "{0} and {1}") + @MethodSource("oneNameInTwoCasesArgs") + public void getFormData_OneNameInTwoCases_SentOnceByPrecedence( + String first, + String second, + String expectedName, + String expectedValue) throws Exception { + acceptEvidence(Arrays.asList(first, second)); + configureMockedClient(); + CloudRequestEngineDefault engine = (CloudRequestEngineDefault) + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence(first, "A"); + data.addEvidence(second, "B"); + + Map result = engine.getFormData(data); + List names = new ArrayList<>(); + for (String key : result.keySet()) { + if (key.equalsIgnoreCase(expectedName)) { + names.add(key); + } + } + assertEquals(Collections.singletonList(expectedName), names, + "The name must be sent once: " + result); + assertEquals(expectedValue, result.get(expectedName)); + } + } + + /** + * The web integration adds the whole web request as server.http-request + * for usage sharing to read. It holds every header and cookie, so it is + * never sent to the cloud service, even where the service lists its key. + */ + @Test + public void getFormData_LeavesOutTheWebRequest() throws Exception { + acceptEvidence(Arrays.asList( + "header.User-Agent", "server.client-ip", "server.http-request")); + configureMockedClient(); + CloudRequestEngineDefault engine = (CloudRequestEngineDefault) new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .build(); + + Map request = new HashMap<>(); + request.put("header.Cookie", "session=secret"); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine) + .build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.User-Agent", "iPhone"); + data.addEvidence("server.client-ip", "1.2.3.4"); + data.addEvidence("server.http-request", request); + + Map result = engine.getFormData(data); + assertEquals("iPhone", result.get("User-Agent")); + assertEquals("1.2.3.4", result.get("client-ip")); + assertFalse(result.containsKey("http-request"), + "The web request was sent to the cloud service."); + } + } + /** * An evidence key whose name itself holds a dot, such as * {@code query.id.usage}, must reach the cloud service under its whole @@ -473,6 +637,7 @@ public void getFormData(String name, Map evidence, String expect */ @Test public void getFormData_KeepsDotsInEvidenceName() throws Exception { + acceptEvidence(Arrays.asList("query.id.usage", "header.id.email")); configureMockedClient(); CloudRequestEngineDefault engine = (CloudRequestEngineDefault) new CloudRequestEngineBuilder(loggerFactory, httpClient) @@ -500,6 +665,7 @@ public void getFormData_KeepsDotsInEvidenceName() throws Exception { */ @Test public void process_PostsWholeEvidenceName() throws Exception { + acceptEvidence(Arrays.asList("query.id.usage")); configureMockedClient(); CloudRequestEngine engine = new CloudRequestEngineBuilder(loggerFactory, httpClient) @@ -514,13 +680,288 @@ public void process_PostsWholeEvidenceName() throws Exception { } ArgumentCaptor body = ArgumentCaptor.forClass(byte[].class); - verify(httpClient, times(1)).postData(any(), any(), body.capture()); + verify(httpClient, times(1)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false), + any(), + body.capture()); List items = Arrays.asList( new String(body.getValue(), StandardCharsets.UTF_8).split("&")); assertTrue(items.contains("id.usage=non-marketing"), "The posted body does not carry id.usage: " + items); } + /** + * Every Evidence value is sent, including Evidence the cloud service + * does not list as accepted, apart from the reserved names under any + * prefix. + * See the Processing section of the cloud request engine specification. + */ + @Test + public void process_SendsEveryEvidenceValueButTheReservedNames() + throws Exception { + acceptEvidence(Arrays.asList("query.User-Agent")); + Map evidence = new HashMap<>(); + evidence.put("query.User-Agent", "iPhone"); + evidence.put("header.accept-language", "en-GB"); + evidence.put("header.X-Other", "other"); + evidence.put("cookie.session", "abc"); + evidence.put("server.client-ip", "1.2.3.4"); + evidence.put("other.VALUES", "ismobile"); + evidence.put("server.licence", "evidence-licence"); + + List items = postedItems(null, evidence); + + assertEquals( + new HashSet<>(Arrays.asList( + "resource=resourcekey", + "User-Agent=iPhone", + "accept-language=en-GB", + "X-Other=other", + "session=abc", + "client-ip=1.2.3.4")), + new HashSet<>(items), + "The request did not carry every Evidence value apart from " + + "the reserved names: " + items); + } + + /** + * Evidence whose key has no prefix is sent under its whole key, unless + * the key is a reserved name in any letter case. Keys only usually + * carry a prefix, as the Overview of the Evidence specification says. + */ + @Test + public void process_SendsEvidenceWithNoPrefixUnderItsWholeKey() + throws Exception { + acceptEvidence(Arrays.asList("query.User-Agent")); + Map evidence = new HashMap<>(); + evidence.put("query.User-Agent", "iPhone"); + evidence.put("bare", "value"); + evidence.put("Values", "ismobile"); + evidence.put("license", "evidence-license"); + evidence.put("RESOURCE", "evidence-resource"); + evidence.put("licence", "evidence-licence"); + + List items = postedItems(null, evidence); + + assertEquals( + new HashSet<>(Arrays.asList( + "resource=resourcekey", + "User-Agent=iPhone", + "bare=value")), + new HashSet<>(items), + "The request did not carry the Evidence with no prefix, or " + + "carried a reserved name: " + items); + } + + /** + * Prefixes are compared without regard to case, so query Evidence + * takes precedence over header Evidence of the same name, with no + * warning, whatever the case of its prefix. Keys are case-insensitive, + * as the Overview of the Evidence specification says. + */ + @Test + public void process_ComparesPrefixesWithoutRegardToCase() + throws Exception { + loggerFactory.loggers.clear(); + acceptEvidence(Arrays.asList("query.User-Agent")); + Map evidence = new HashMap<>(); + evidence.put("Query.User-Agent", "query-value"); + evidence.put("header.User-Agent", "header-value"); + + List items = postedItems(null, evidence); + + assertEquals( + new HashSet<>(Arrays.asList( + "resource=resourcekey", + "User-Agent=query-value")), + new HashSet<>(items), + "The query Evidence did not take precedence: " + items); + loggerFactory.assertMaxWarnings(0); + } + + /** + * Evidence with no prefix ranks with Evidence of any prefix other than + * query, header and cookie, so it has the lowest precedence, unless the + * whole key is one of those prefixes, which puts it with that prefix's + * Evidence. The precedence order of the Evidence specification orders + * prefixes only, so this follows .NET. + */ + @Test + public void process_RanksEvidenceWithNoPrefix() throws Exception { + acceptEvidence(Arrays.asList("query.User-Agent")); + Map evidence = new HashMap<>(); + evidence.put("X-Thing", "bare"); + evidence.put("header.X-Thing", "header"); + evidence.put("query", "bare-query"); + evidence.put("header.query", "header-query"); + + List items = postedItems(null, evidence); + + assertEquals( + new HashSet<>(Arrays.asList( + "resource=resourcekey", + "X-Thing=header", + "query=bare-query")), + new HashSet<>(items), + "Evidence with no prefix was not ranked as .NET ranks it: " + + items); + } + + /** + * Evidence whose key strips to resource, license or values is never + * sent, whatever its case and even where the cloud service lists the + * key, so the request carries the configured resource key, and the + * license key where one is configured, once each. + * See the Processing section of the cloud request engine specification. + */ + @ParameterizedTest(name = "license key {0}") + @NullSource + @ValueSource(strings = {"ABCDEFG"}) + public void process_LeavesOutEvidenceNamedResourceLicenseOrValues( + String licenseKey) throws Exception { + acceptEvidence(Arrays.asList( + "query.User-Agent", + "query.resource", + "header.Resource", + "query.license", + "cookie.LICENSE", + "query.values")); + Map evidence = new HashMap<>(); + evidence.put("query.User-Agent", "iPhone"); + evidence.put("query.resource", "evidence-resource"); + evidence.put("header.Resource", "header-resource"); + evidence.put("query.license", "evidence-license"); + evidence.put("cookie.LICENSE", "cookie-license"); + evidence.put("query.values", "ismobile"); + + List items = postedItems(licenseKey, evidence); + + List expected = new ArrayList<>(); + expected.add("resource=resourcekey"); + if (licenseKey != null) { + expected.add("license=" + licenseKey); + } + expected.add("User-Agent=iPhone"); + assertEquals(expected, items); + } + + /** + * Evidence whose key strips to licence, whatever its case, is never + * sent, because the cloud service reads the license key from licence + * where license is absent. + * See the Processing section of the cloud request engine specification. + */ + @ParameterizedTest(name = "license key {0}") + @NullSource + @ValueSource(strings = {"ABCDEFG"}) + public void process_LeavesOutEvidenceNamedLicence( + String licenseKey) throws Exception { + acceptEvidence(Arrays.asList( + "query.User-Agent", + "query.licence", + "cookie.LICENCE")); + Map evidence = new HashMap<>(); + evidence.put("query.User-Agent", "iPhone"); + evidence.put("query.licence", "evidence-licence"); + evidence.put("cookie.LICENCE", "cookie-licence"); + + List items = postedItems(licenseKey, evidence); + + List expected = new ArrayList<>(); + expected.add("resource=resourcekey"); + if (licenseKey != null) { + expected.add("license=" + licenseKey); + } + expected.add("User-Agent=iPhone"); + assertEquals(expected, items); + } + + /** + * A cache key is built from the Evidence the engine advertises, which is + * still the keys the cloud service lists, so a second request that + * differs only in Evidence the service does not list finds the entry + * the first one wrote, and one request is posted. That request carries + * the Evidence the service does not list all the same. + * See the Generation of keys section of the caching specification. + */ + @Test + public void process_RepeatedRequestHitsTheCache() throws Exception { + acceptEvidence(Arrays.asList("query.User-Agent")); + configureMockedClient(); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setCacheSize(10) + .build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build()) { + for (String platform : Arrays.asList("Windows", "macOS")) { + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence("query.User-Agent", "iPhone"); + data.addEvidence("header.sec-ch-ua-platform", platform); + data.process(); + } + } + } + + assertFalse( + engine.getEvidenceKeyFilter().include("header.sec-ch-ua-platform"), + "The engine advertises a key the cloud service does not list."); + ArgumentCaptor body = ArgumentCaptor.forClass(byte[].class); + verify(httpClient, times(1)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false), + any(), + body.capture()); + List items = Arrays.asList( + new String(body.getValue(), StandardCharsets.UTF_8).split("&")); + assertTrue(items.contains("sec-ch-ua-platform=Windows"), + "The request did not carry the Evidence the cloud service does " + + "not list: " + items); + } + + /** + * Answer the evidencekeys request with the given keys. + * @param keys the Evidence keys the cloud service accepts + */ + private void acceptEvidence(Collection keys) { + evidenceKeysResponse = new JSONArray(keys).toString(); + } + + /** + * Process the Evidence with an engine on the resource key + * {@code resourcekey} and return the fields of the one request it + * posted, in order. + * @param licenseKey the license key, or null for none + * @param evidence the Evidence to process + * @return the posted fields + */ + private List postedItems( + String licenseKey, + Map evidence) throws Exception { + configureMockedClient(); + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey("resourcekey") + .setLicenseKey(licenseKey) + .build(); + + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(engine).build(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + data.process(); + } + + ArgumentCaptor body = ArgumentCaptor.forClass(byte[].class); + verify(httpClient, times(1)).postData( + argThat(c -> c.getURL().getPath().endsWith("properties") == false), + any(), + body.capture()); + return Arrays.asList( + new String(body.getValue(), StandardCharsets.UTF_8).split("&")); + } + /** * Verify that the request to the cloud service will contain * the configured origin header value. @@ -626,22 +1067,22 @@ public void getPublicProperties() throws Exception { } /** - * Verify that PropertyNotLoadedException is thrown on getPublicProperties if remote server is unavailable + * An unreachable server when the engine is built is a failure that can + * pass. The build completes with one warning, and the accessible + * properties are unknown until processing fetches them. */ @Test - public void getPublicProperties_Throw_Exception_When_Server_Unavailable() throws IOException { + public void getPublicProperties_Server_Unavailable_At_Build() throws Exception { final String resourceKey = "resource_key"; configureFailingMockClient(); - try { - new CloudRequestEngineBuilder(loggerFactory, httpClient) - .setResourceKey(resourceKey) - .build() - .getPublicProperties(); - fail("Expected exception was not thrown"); - } catch (Exception e) { - assertInstanceOf(PropertyNotLoadedException.class, e); - } + CloudRequestEngine engine = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey(resourceKey) + .build(); + + assertNull(engine.getPublicProperties()); + assertEquals(1, warningCount()); } /** @@ -659,21 +1100,23 @@ public void getEvidenceKeyFilter() throws Exception { } /** - * Verify that PropertyNotLoadedException is thrown on getEvidenceKeyFilter if remote server is unavailable + * An unreachable server when the engine is built leaves the accepted + * Evidence empty, rather than failing whatever reads it, until + * processing fetches it. */ @Test - public void getEvidenceKeyFilter_Throw_Exception_When_Server_Unavailable() throws IOException { + public void getEvidenceKeyFilter_Server_Unavailable_At_Build() throws Exception { final String resourceKey = "resource_key"; configureFailingMockClient(); - try { - new CloudRequestEngineBuilder(loggerFactory, httpClient) - .setResourceKey(resourceKey) - .build() - .getEvidenceKeyFilter(); - fail("Expected exception was not thrown"); - } catch (Exception e) { - assertInstanceOf(PropertyNotLoadedException.class, e); - } + EvidenceKeyFilter filter = + new CloudRequestEngineBuilder(loggerFactory, httpClient) + .setResourceKey(resourceKey) + .build() + .getEvidenceKeyFilter(); + + assertNotNull(filter); + assertFalse(filter.include("query.User-Agent")); + assertEquals(1, warningCount()); } } diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTestsBase.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTestsBase.java index 0320d94e5..dfbd106b4 100644 --- a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTestsBase.java +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/CloudRequestEngineTestsBase.java @@ -69,24 +69,26 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { invocationOnMock.getMethod().getName() + "' should not have been called."); } }); - final HttpURLConnection connection = mock(HttpURLConnection.class, new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - throw new Exception("The method '" + - invocationOnMock.getMethod().getName() + "' should not have been called."); - } - }); - doNothing().when(connection).setConnectTimeout(anyInt()); - doNothing().when(connection).setReadTimeout(anyInt()); - Map> responseHeaders = new HashMap>(); - doReturn(responseHeaders).when(connection).getHeaderFields(); + // Each request gets a connection of its own, because the engine + // makes its two discovery requests at once. doAnswer(new Answer() { @Override public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - URL url = (URL)invocationOnMock.getArgument(0); - doReturn(url).when(connection).getURL(); + URL url = (URL)invocationOnMock.getArgument(0); + HttpURLConnection connection = mock(HttpURLConnection.class, new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + throw new Exception("The method '" + + invocation.getMethod().getName() + "' should not have been called."); + } + }); + doNothing().when(connection).setConnectTimeout(anyInt()); + doNothing().when(connection).setReadTimeout(anyInt()); + Map> responseHeaders = new HashMap>(); + doReturn(responseHeaders).when(connection).getHeaderFields(); + doReturn(url).when(connection).getURL(); if (url.getPath().endsWith("properties")) { - propertiesEndPoint = url.getQuery(); + propertiesEndPoint = url.getQuery(); doReturn(accessiblePropertiesResponseCode).when(connection).getResponseCode(); } else { @@ -120,7 +122,21 @@ else if (url.getPath().endsWith("evidencekeys")) { .when(httpClient) .getResponseString(any(HttpURLConnection.class), anyMap()); - doReturn(jsonResponse) + // A license key makes the discovery requests POSTs, so a POST is + // answered by its URL in the same way. + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocationOnMock) throws Throwable { + URL url = ((HttpURLConnection)invocationOnMock.getArgument(0)).getURL(); + if (url.getPath().endsWith("properties")) { + return accessiblePropertiesResponse; + } + if (url.getPath().endsWith("evidencekeys")) { + return evidenceKeysResponse; + } + return jsonResponse; + } + }) .when(httpClient) .postData( any(HttpURLConnection.class), diff --git a/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java new file mode 100644 index 000000000..6bb87c5fe --- /dev/null +++ b/pipeline/pipeline.cloudrequestengine/src/test/java/fiftyone/pipeline/cloudrequestengine/flowelements/LiveCloudKeys.java @@ -0,0 +1,93 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.cloudrequestengine.flowelements; + +/** + * Where the tests that call the live 51Degrees cloud service read their + * resource key and licence key from, and what they say when there is + * none. + *

    + * Every name is tried as an environment variable first and then as a + * system property, so a developer can export one and a build can pass one + * on the command line with {@code -D}. There is no fallback to a skip, + * because a cloud test that quietly passes with no key reports success + * having called nothing. + */ +final class LiveCloudKeys { + + /** + * The resource key names, in the order they are tried. + */ + static final String[] RESOURCE_KEY_NAMES = { + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "TestResourceKey", + "SUPER_RESOURCE_KEY", + }; + + /** + * The licence key names, in the order they are tried. A licence key + * is optional, because an account can hold its products on the + * resource key alone. + */ + static final String[] LICENCE_KEY_NAMES = { + "_51DEGREES_LICENSE_KEY", + "_51DEGREES_LICENSE_KEY_BESPOKE", + "_51DEGREES_LICENSE_KEY_CLOUDV5BESPOKE", + "TestLicenseKey", + }; + + private LiveCloudKeys() { + } + + /** + * The first value set under any of the given names, or null when none + * of them is set anywhere. + */ + static String find(String[] names) { + for (String name : names) { + String value = System.getenv(name); + if (value == null || value.trim().isEmpty()) { + value = System.getProperty(name); + } + if (value != null && value.trim().isEmpty() == false) { + return value.trim(); + } + } + return null; + } + + /** + * What to fail with when no resource key is set, naming every place + * that was looked and what the key has to be entitled to. + */ + static String noResourceKey() { + return "This test calls the live 51Degrees cloud service, so it " + + "needs a resource key and there is nothing to fall back on. " + + "None of " + String.join(", ", RESOURCE_KEY_NAMES) + " is set, " + + "as an environment variable or as a system property, tried in " + + "that order. The key has to carry the device product, because " + + "the test asks the service for device detection properties. " + + "Create a key at https://configure.51degrees.com."; + } +} diff --git a/pipeline/pipeline.core/configuration-validation b/pipeline/pipeline.core/configuration-validation new file mode 160000 index 000000000..b2740eddf --- /dev/null +++ b/pipeline/pipeline.core/configuration-validation @@ -0,0 +1 @@ +Subproject commit b2740eddf2824290157cca5f1f7d00f150162f87 diff --git a/pipeline/pipeline.core/pom.xml b/pipeline/pipeline.core/pom.xml index 3e30f3f64..39a8335ff 100644 --- a/pipeline/pipeline.core/pom.xml +++ b/pipeline/pipeline.core/pom.xml @@ -88,6 +88,19 @@ + + + src/main/resources + + + + configuration-validation + + pipelineOptions.xsd + + + org.apache.maven.plugins diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementData.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementData.java index 6f164c05d..da33f9074 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementData.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementData.java @@ -28,7 +28,13 @@ /** * Represents property values that have be determined by a specific * {@link FlowElement} based on the supplied evidence. + *

    + * An instance is written only by the {@link FlowElement} that creates + * it, while that element processes. It is not intended to be written + * from several threads at once, nor read while it is being written. + *

    * @see Specification + * @see Thread safety */ public interface ElementData extends Data { diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementDataBase.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementDataBase.java index fc67b9759..937b3c590 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementDataBase.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/data/ElementDataBase.java @@ -33,7 +33,13 @@ * Base implementation of {@link ElementData}. * Represents property values that have be determined by a specific * {@link FlowElement} based on the supplied evidence. + *

    + * An instance is written only by the {@link FlowElement} that creates + * it, while that element processes. It is not intended to be written + * from several threads at once, nor read while it is being written. + *

    * @see Specification + * @see Thread safety */ public abstract class ElementDataBase extends DataBase implements ElementData { diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowElement.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowElement.java index c27716d21..b03878a83 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowElement.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/FlowElement.java @@ -103,6 +103,19 @@ public interface FlowElement< */ boolean isConcurrent(); + /** + * Whether the element knows all the property metadata and accepted + * Evidence it will expose. An element that gets them from a remote + * service returns false until it has them, and anything that keeps what + * it reads from the element, such as the Pipeline's map of available + * properties, reads it again until then. + * @return true if the property metadata and accepted Evidence are + * complete + */ + default boolean hasLoadedProperties() { + return true; + } + /** * Indicates whether the element has been closed using the {@link #close()} * method, either explicitly or as a result of a 'try with resource'. diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/ParallelElements.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/ParallelElements.java index 79b8e4148..b1a479b75 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/ParallelElements.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/ParallelElements.java @@ -117,6 +117,19 @@ public boolean isConcurrent() { return true; } + /** + * @return true if every child element has loaded its properties + */ + @Override + public boolean hasLoadedProperties() { + for (FlowElement element : flowElements) { + if (element.hasLoadedProperties() == false) { + return false; + } + } + return true; + } + @Override protected void managedResourcesCleanup() { for (FlowElement element : flowElements) { diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java index fbebda96e..f10d49d1c 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineBuilder.java @@ -62,6 +62,13 @@ public class PipelineBuilder private final Map, Class> primitiveTypes = getPrimitiveTypeMap(); private final Set> elementBuilders; + /** + * Element builders prepared in code, which + * {@link #buildFromConfiguration(PipelineOptions)} uses in preference to + * constructing new ones. + */ + private final List preparedBuilders = new ArrayList<>(); + /** * Construct a new builder. */ @@ -80,6 +87,27 @@ public PipelineBuilder(ILoggerFactory loggerFactory) { elementBuilders = getAvailableElementBuilders(); } + /** + * Add an element builder prepared in code, which + * {@link #buildFromConfiguration(PipelineOptions)} uses for any element + * whose builder name resolves to the builder's type, in place of + * constructing a new builder. Code can then set the options a + * configuration file cannot express, such as the sources and + * translations of a Translation Engine, and the configuration sets the + * remaining options and builds the element. + *

    + * The builder's type must still be one the configuration can name, being + * annotated with {@link ElementBuilder}. When more than one prepared + * builder is of that type, the first added is used, and every element + * of that type in the configuration uses the same instance. + * @param builder the prepared builder + * @return this builder + */ + public PipelineBuilder addElementBuilder(Object builder) { + preparedBuilders.add(getNotNull(builder, "builder cannot be null")); + return this; + } + @Override public Pipeline buildFromConfiguration(PipelineOptions options) throws Exception { @@ -244,10 +272,13 @@ private void addElementToList( } - // A service collection does not exist in the builder, so try - // to construct a builder instance from the assemblies - // currently loaded. - Object builderInstance = getBuilder(builderType); + // Use a builder prepared in code if there is one, because it can + // carry options a configuration file cannot express. Otherwise + // construct a new builder. + Object builderInstance = getPreparedBuilder(builderType); + if (builderInstance == null) { + builderInstance = getBuilder(builderType); + } if (builderInstance == null) { throw new PipelineConfigurationException( "Builder '" + builderType.getName() + "' for " + @@ -353,10 +384,14 @@ private void addElementToList( try { result = buildMethod.invoke(builderInstance, parameters.toArray()); } catch (Exception e) { + // Keep the build method's own message, which says why the + // element could not be built. + Throwable cause = e.getCause() != null ? e.getCause() : e; throw new PipelineConfigurationException( "Failed to build " + elementLocation + " using " + - "'" + builderType.getName() + "'.", - e); + "'" + builderType.getName() + "'." + + (cause.getMessage() == null ? "" : " " + cause.getMessage()), + cause); } if (result == null) { throw new PipelineConfigurationException( @@ -530,6 +565,21 @@ private Object callConstructorWithServices(Constructor constructor, Class } } + /** + * Get the first builder added with {@link #addElementBuilder(Object)} + * that is an instance of the builder type provided. + * @param builderType type of builder to find + * @return the prepared builder, or null if none is of the type + */ + private Object getPreparedBuilder(Class builderType) { + for (Object builder : preparedBuilders) { + if (builderType.isInstance(builder)) { + return builder; + } + } + return null; + } + /** * Get a new instance of the builder type provided. This tries to call the * constructor with the most arguments which can be fulfilled. Potential @@ -981,10 +1031,15 @@ private Object parseToType( try { parsedValue = parse.invoke(new Object(), value); } catch (Exception e) { + // Keep the parse method's own message, which says what is wrong + // with the value. + Throwable cause = e.getCause() != null ? e.getCause() : e; throw new PipelineConfigurationException( errorTextPrefix + ". Failed to parse value " + "'" + value + "' using the static " + - "'parse' method."); + "'parse' method." + + (cause.getMessage() == null ? "" : " " + cause.getMessage()), + cause); } return parsedValue; } diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java index f53cda4ad..302b51d94 100644 --- a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/core/flowelements/PipelineDefault.java @@ -71,9 +71,15 @@ class PipelineDefault implements PipelineInternal { * Map of {@link ElementPropertyMetaData}s keyed first on the data key of * the element which they belong to, then by the property name. */ - private final Map> + private volatile Map> elementAvailableProperties; + /** + * Held while the map of available properties is built after an element + * had not loaded its properties when the Pipeline was built. + */ + private final Object availablePropertiesLock = new Object(); + /** * Lock used for synchronised blocks. */ @@ -155,8 +161,7 @@ class PipelineDefault implements PipelineInternal { for (FlowElement element : this.flowElements) { isConcurrent = isConcurrent || element.isConcurrent(); } - this.elementAvailableProperties = - getElementAvailableProperties(flowElements); + getElementAvailableProperties(); logger.debug("Pipeline '" + hashCode() + "' created."); @@ -360,10 +365,41 @@ public List getFlowElements() { return Collections.unmodifiableList(flowElements); } + /** + * The available properties of each element. The map is kept once every + * element has loaded its properties. Until then, as after a cloud + * element could not reach its service when it was built, it is built + * again on each call, so it gains the properties once they are known. + * @return the available properties, keyed on element data key then + * property name + */ @Override public Map> getElementAvailableProperties() { - return Collections.unmodifiableMap(elementAvailableProperties); + Map> properties = + elementAvailableProperties; + if (properties == null) { + synchronized (availablePropertiesLock) { + properties = elementAvailableProperties; + if (properties == null) { + boolean loaded = allPropertiesLoaded(); + properties = getElementAvailableProperties(flowElements); + if (loaded) { + elementAvailableProperties = properties; + } + } + } + } + return Collections.unmodifiableMap(properties); + } + + private boolean allPropertiesLoaded() { + for (FlowElement element : flowElements) { + if (element.hasLoadedProperties() == false) { + return false; + } + } + return true; } public void close(boolean closing) throws Exception { diff --git a/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java new file mode 100644 index 000000000..6f323b5ba --- /dev/null +++ b/pipeline/pipeline.core/src/main/java/fiftyone/pipeline/util/LicenseKeyMask.java @@ -0,0 +1,234 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.util; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; + +/** + * Masks license keys in text written to a log entry or an error message. A + * masked key shows only its last 4 characters, with every character before + * them replaced by '*', so a reader can tell two keys apart but cannot use + * either. A key of 4 characters or fewer is replaced in full, because its + * last 4 characters would be the whole key. + * @see Specification + */ +public final class LicenseKeyMask { + + /** + * How many characters at the end of a key stay visible. + */ + public static final int VISIBLE_CHARACTERS = 4; + + /** + * The character that replaces each hidden character of a key. + */ + public static final char MASK_CHARACTER = '*'; + + private LicenseKeyMask() { + } + + /** + * The key with every character before its last 4 replaced. + * @param key a license key, or null + * @return the masked key, or null if the key is null + */ + public static String mask(String key) { + if (key == null) { + return null; + } + int hidden = key.length() > VISIBLE_CHARACTERS ? + key.length() - VISIBLE_CHARACTERS : + key.length(); + StringBuilder masked = new StringBuilder(key.length()); + for (int i = 0; i < hidden; i++) { + masked.append(MASK_CHARACTER); + } + return masked.append(key, hidden, key.length()).toString(); + } + + /** + * The text with every occurrence of each key masked, both as the key is + * written and as it is encoded in a URL, with upper or lower case + * hexadecimal digits. An encoded key is replaced by the encoded form of + * its masked plain form, so it shows the same last 4 characters. Every + * occurrence of a key's text is masked, so a key of only a few + * characters also masks those characters wherever else they appear. + * @param text the text, such as a URL or an error message, or null + * @param keys the license keys, or null + * @return the masked text, or null if the text is null + */ + public static String maskIn(String text, Iterable keys) { + if (text == null || keys == null) { + return text; + } + String result = text; + for (String key : longestFirst(keys)) { + String masked = mask(key); + result = result.replace(key, masked); + String encoded = urlEncode(key); + if (encoded.equals(key) == false) { + String maskedEncoded = urlEncode(masked); + result = result.replace(encoded, maskedEncoded); + result = result.replace( + lowerCaseEscapes(encoded), + lowerCaseEscapes(maskedEncoded)); + } + } + return result; + } + + /** + * The throwable to log or report in place of the one given, so that no + * message in it, its causes or its suppressed throwables holds a key. + * When none does, the throwable given is returned. Otherwise a copy is + * returned whose messages are masked, each starting with the name of the + * class it stands for, and whose stack traces are the originals. + * @param throwable the throwable, or null + * @param keys the license keys, or null + * @return a throwable that holds no key + */ + public static Throwable maskIn(Throwable throwable, Iterable keys) { + if (throwable == null || + keys == null || + holdsKey(throwable, keys, new IdentityHashMap()) + == false) { + return throwable; + } + return copy(throwable, keys, new IdentityHashMap()); + } + + private static boolean holdsKey( + Throwable throwable, + Iterable keys, + Map seen) { + if (throwable == null || seen.put(throwable, Boolean.TRUE) != null) { + return false; + } + String message = throwable.getMessage(); + if (message != null && message.equals(maskIn(message, keys)) == false) { + return true; + } + if (holdsKey(throwable.getCause(), keys, seen)) { + return true; + } + for (Throwable suppressed : throwable.getSuppressed()) { + if (holdsKey(suppressed, keys, seen)) { + return true; + } + } + return false; + } + + private static Throwable copy( + Throwable throwable, + Iterable keys, + Map copies) { + Throwable existing = copies.get(throwable); + if (existing != null) { + return existing; + } + String message = maskIn(throwable.getMessage(), keys); + MaskedThrowable masked = new MaskedThrowable( + throwable.getClass().getName() + + (message == null ? "" : ": " + message)); + copies.put(throwable, masked); + masked.setStackTrace(throwable.getStackTrace()); + Throwable cause = throwable.getCause(); + if (cause != null) { + Throwable maskedCause = copy(cause, keys, copies); + if (maskedCause != masked) { + masked.initCause(maskedCause); + } + } + for (Throwable suppressed : throwable.getSuppressed()) { + Throwable maskedSuppressed = copy(suppressed, keys, copies); + if (maskedSuppressed != masked) { + masked.addSuppressed(maskedSuppressed); + } + } + return masked; + } + + /** + * The keys that are not null or empty, longest first, so that a key + * that holds another is masked whole. + */ + private static List longestFirst(Iterable keys) { + List result = new ArrayList<>(); + for (String key : keys) { + if (key != null && key.isEmpty() == false) { + result.add(key); + } + } + Collections.sort(result, (a, b) -> b.length() - a.length()); + return result; + } + + /** + * The text with the two hexadecimal digits after each '%' in lower + * case. + */ + private static String lowerCaseEscapes(String text) { + StringBuilder result = new StringBuilder(text); + for (int i = text.indexOf('%'); + i >= 0 && i + 2 < text.length(); + i = text.indexOf('%', i + 1)) { + result.setCharAt(i + 1, Character.toLowerCase(text.charAt(i + 1))); + result.setCharAt(i + 2, Character.toLowerCase(text.charAt(i + 2))); + } + return result.toString(); + } + + private static String urlEncode(String key) { + try { + return URLEncoder.encode(key, "UTF-8"); + } catch (UnsupportedEncodingException e) { + // Every Java platform supports UTF-8. + throw new IllegalStateException(e); + } + } + + /** + * Stands for a throwable whose message held a license key. Its message + * starts with the name of the class it stands for, and it prints as + * that message alone. + */ + private static final class MaskedThrowable extends Exception { + private static final long serialVersionUID = 1L; + + MaskedThrowable(String message) { + super(message); + } + + @Override + public String toString() { + return getMessage(); + } + } +} diff --git a/pipeline/pipeline.core/src/main/resources/pipelineOptions.xsd b/pipeline/pipeline.core/src/main/resources/pipelineOptions.xsd deleted file mode 100644 index e04f6dc41..000000000 --- a/pipeline/pipeline.core/src/main/resources/pipelineOptions.xsd +++ /dev/null @@ -1,42 +0,0 @@ - - - - This is an XML schema for 51Degrees pipeline options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/configuration/PipelineBuilderPreparedBuilderTest.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/configuration/PipelineBuilderPreparedBuilderTest.java new file mode 100644 index 000000000..93b18675e --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/core/configuration/PipelineBuilderPreparedBuilderTest.java @@ -0,0 +1,114 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.core.configuration; + +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.core.testclasses.flowelements.ListSplitterElement; +import fiftyone.pipeline.core.testclasses.flowelements.ListSplitterElementBuilder; +import fiftyone.pipeline.core.testclasses.flowelements.MultiplyByElementBuilder; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +/** + * Tests for element builders prepared in code and given to the pipeline + * builder, which building from configuration uses in place of new ones. + */ +public class PipelineBuilderPreparedBuilderTest { + + /** + * Options naming the list splitter and setting only its maximum length, + * so the delimiter is whatever the builder already holds. + */ + private static PipelineOptions listSplitterOptions(String maxLength) { + ElementOptions element = new ElementOptions(); + element.builderName = "ListSplitterElement"; + element.buildParameters.put("MaxLength", maxLength); + PipelineOptions options = new PipelineOptions(); + options.elements.add(element); + return options; + } + + private static ListSplitterElementBuilder splitter(String delimiter) { + ListSplitterElementBuilder builder = new ListSplitterElementBuilder(); + builder.setDelimiter(delimiter); + return builder; + } + + @Test + public void preparedBuilder_KeepsItsOptionsAndTakesTheConfiguredOnes() + throws Exception { + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(splitter("|")) + .buildFromConfiguration(listSplitterOptions("50"))) { + ListSplitterElement element = + pipeline.getElement(ListSplitterElement.class); + assertEquals("|", element.getDelimiter()); + assertEquals(50, element.getMaxLength()); + } + } + + @Test + public void twoPreparedBuildersOfTheSameType_TheFirstIsUsed() + throws Exception { + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(splitter("|")) + .addElementBuilder(splitter(";")) + .buildFromConfiguration(listSplitterOptions("50"))) { + assertEquals("|", pipeline.getElement(ListSplitterElement.class) + .getDelimiter()); + } + } + + @Test + public void preparedBuilderOfAnotherType_ANewBuilderIsConstructed() + throws Exception { + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(new MultiplyByElementBuilder()) + .buildFromConfiguration(listSplitterOptions("50"))) { + // A new list splitter builder starts with a comma. + assertEquals(",", pipeline.getElement(ListSplitterElement.class) + .getDelimiter()); + } + } + + @Test + public void nullBuilder_IsRefused() { + assertThrows(IllegalArgumentException.class, + () -> new PipelineBuilder().addElementBuilder(null)); + } + + @Test + public void valueTheParseMethodRefuses_ErrorKeepsTheParseMessage() { + PipelineConfigurationException thrown = assertThrows( + PipelineConfigurationException.class, + () -> new PipelineBuilder().buildFromConfiguration( + listSplitterOptions("fifty"))); + assertTrue(thrown.getMessage(), + thrown.getMessage().contains("For input string")); + } +} diff --git a/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java new file mode 100644 index 000000000..312e2c08e --- /dev/null +++ b/pipeline/pipeline.core/src/test/java/fiftyone/pipeline/util/LicenseKeyMaskTests.java @@ -0,0 +1,191 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.util; + +import org.junit.Test; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * A license key in a log entry or an error message shows only its last 4 + * characters, with every character before them replaced. + */ +public class LicenseKeyMaskTests { + + private static final String KEY = "TESTKEYNOTREALABCDEFGHIJ1234"; + private static final String MASKED = "************************1234"; + private static final String OTHER_KEY = "OTHERTESTKEYNOTREALXYZ5678"; + private static final String OTHER_MASKED = "**********************5678"; + + @Test + public void mask_ShowsTheLast4Characters() { + assertEquals(MASKED, LicenseKeyMask.mask(KEY)); + assertEquals(KEY.length(), LicenseKeyMask.mask(KEY).length()); + } + + /** + * A key of 4 characters or fewer would be shown whole, so all of it is + * replaced. + */ + @Test + public void mask_ShortKeyIsReplacedInFull() { + assertEquals("****", LicenseKeyMask.mask("1234")); + assertEquals("**", LicenseKeyMask.mask("12")); + assertEquals("*2345", LicenseKeyMask.mask("12345")); + assertEquals("", LicenseKeyMask.mask("")); + assertNull(LicenseKeyMask.mask(null)); + } + + @Test + public void maskIn_MasksEveryKeyInAUrl() { + String url = "https://distributor.example/api/v2/download?" + + "LicenseKeys=" + KEY + "|" + OTHER_KEY + "&Download=True"; + + String masked = LicenseKeyMask.maskIn( + url, + Arrays.asList(KEY, OTHER_KEY)); + + assertEquals( + "https://distributor.example/api/v2/download?" + + "LicenseKeys=" + MASKED + "|" + OTHER_MASKED + + "&Download=True", + masked); + } + + /** + * A key that a URL holds encoded is masked in its encoded form. + */ + @Test + public void maskIn_MasksAKeyEncodedInAUrl() { + String key = "KEY+WITH/CHARS=ENCODED9876"; + String url = "https://example.test/?license=" + + "KEY%2BWITH%2FCHARS%3DENCODED9876"; + + String masked = LicenseKeyMask.maskIn(url, Collections.singletonList(key)); + + assertFalse(masked, masked.contains("KEY%2BWITH")); + assertTrue(masked, masked.endsWith("9876")); + } + + /** + * A key whose last characters are encoded in a URL shows the same last + * 4 characters as its plain form, encoded, rather than the last 4 + * characters of its encoded form. + */ + @Test + public void maskIn_EncodedKeyShowsThePlainLast4() { + String key = "ABCDEFGHIJKLMNOPQRST12+="; + String url = "https://example.test/?license=" + + "ABCDEFGHIJKLMNOPQRST12%2B%3D"; + + String masked = LicenseKeyMask.maskIn(url, Collections.singletonList(key)); + + assertEquals( + "https://example.test/?license=********************12%2B%3D", + masked); + } + + /** + * A key encoded with lower case hexadecimal digits, as some libraries + * write them, is masked as well. + */ + @Test + public void maskIn_MasksAKeyEncodedInLowerCase() { + String key = "ABCDEFGHIJKLMNOPQRST12+="; + String url = "https://example.test/?license=" + + "ABCDEFGHIJKLMNOPQRST12%2b%3d"; + + String masked = LicenseKeyMask.maskIn(url, Collections.singletonList(key)); + + assertEquals( + "https://example.test/?license=********************12%2b%3d", + masked); + } + + @Test + public void maskIn_LeavesTextWithoutAKeyAlone() { + String text = "No key here."; + assertEquals(text, LicenseKeyMask.maskIn(text, Arrays.asList(KEY))); + assertNull(LicenseKeyMask.maskIn((String) null, Arrays.asList(KEY))); + assertEquals(text, LicenseKeyMask.maskIn(text, null)); + assertEquals( + text, + LicenseKeyMask.maskIn(text, Arrays.asList(null, ""))); + } + + /** + * The message of an exception an HTTP library raises often holds the + * URL it requested, and so the key. The throwable to log holds the key + * in none of its messages, its causes' or its suppressed throwables', + * and keeps the name of each class it stands for. + */ + @Test + public void maskIn_MasksAThrowableAndItsCauses() { + IOException cause = new IOException( + "Server returned HTTP response code: 403 for URL: " + + "https://distributor.example/?LicenseKeys=" + KEY); + RuntimeException thrown = new RuntimeException( + "Download failed for " + KEY, cause); + thrown.addSuppressed(new IllegalStateException(OTHER_KEY)); + List keys = Arrays.asList(KEY, OTHER_KEY); + + Throwable masked = LicenseKeyMask.maskIn(thrown, keys); + String printed = printed(masked); + + assertFalse(printed, printed.contains(KEY)); + assertFalse(printed, printed.contains(OTHER_KEY)); + assertTrue(printed, printed.contains(MASKED)); + assertTrue(printed, printed.contains(OTHER_MASKED)); + assertTrue(printed, printed.contains("java.lang.RuntimeException")); + assertTrue(printed, printed.contains("java.io.IOException")); + assertTrue(printed, + printed.contains("java.lang.IllegalStateException")); + assertEquals( + thrown.getStackTrace().length, + masked.getStackTrace().length); + } + + @Test + public void maskIn_ThrowableWithoutAKeyIsReturnedAsItIs() { + IOException thrown = new IOException("Connection refused"); + assertSame(thrown, LicenseKeyMask.maskIn(thrown, Arrays.asList(KEY))); + assertNull(LicenseKeyMask.maskIn((Throwable) null, Arrays.asList(KEY))); + } + + private static String printed(Throwable throwable) { + StringWriter text = new StringWriter(); + throwable.printStackTrace(new PrintWriter(text)); + return text.toString(); + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java index 5b7c1b325..963827bdb 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/main/java/pipeline/developerexamples/cloudengine/Main.java @@ -34,6 +34,8 @@ import pipeline.developerexamples.cloudengine.flowelements.SimpleCloudEngine; import pipeline.developerexamples.cloudengine.flowelements.SimpleCloudEngineBuilder; +import java.util.concurrent.atomic.AtomicBoolean; + public class Main { private static final ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); private static final HttpClient httpClient = new HttpClientDefault(); @@ -47,21 +49,69 @@ public class Main { private static final String STAR_SIGN_RESOURCE_KEY = "cloudexample"; /** - * The cloud endpoint, taken from FOD_CLOUD_API_URL when that is set, - * which is the variable every 51Degrees cloud example honours and the - * one the cloud request engine builder reads by itself when no + * The 51Degrees cloud service, used when a 51Degrees resource key is + * set and no other endpoint is given. + */ + private static final String FIFTYONE_CLOUD_ENDPOINT = + "https://cloud.51degrees.com/api/v4/"; + + /** + * The name the cloud endpoint is read from, and the deprecated name + * read when that one is not set. + */ + private static final String ENDPOINT_VARIABLE = + "_51DEGREES_CLOUD_ENDPOINT"; + private static final String OLD_ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL"; + + private static final AtomicBoolean oldEndpointVariableLogged = + new AtomicBoolean(false); + + /** + * The deprecated name of the first of the resource key names, read + * when that one is not set. + */ + private static final String OLD_RESOURCE_KEY_VARIABLE = "RESOURCE_KEY"; + + private static final AtomicBoolean oldResourceKeyVariableLogged = + new AtomicBoolean(false); + + /** + * The cloud endpoint, taken from _51DEGREES_CLOUD_ENDPOINT when that is + * set, which is the variable every 51Degrees cloud example honours and + * the one the cloud request engine builder reads by itself when no * endpoint is given. A host other than cloud.51degrees.com would be * used to (a) use an on premise web server, or (b) use a privately * hosted version of the 51Degrees cloud for performance reasons, * which is the private hosting option of the cloud service. Both run * the same service, so the example works unchanged. Normalised to * end in one slash so the builder appends its three paths to it - * directly. + * directly. The deprecated name FOD_CLOUD_API_URL is read when + * _51DEGREES_CLOUD_ENDPOINT is not set, and the first use of it is + * logged as a warning. */ static String endpoint() { - String value = env("FOD_CLOUD_API_URL"); + String value = env(ENDPOINT_VARIABLE); + if (value == null) { + value = env(OLD_ENDPOINT_VARIABLE); + if (value != null + && oldEndpointVariableLogged.compareAndSet(false, true)) { + loggerFactory.getLogger(Main.class.getName()).warn( + "The environment variable " + OLD_ENDPOINT_VARIABLE + + " is deprecated and " + ENDPOINT_VARIABLE + + " replaces it. Set " + ENDPOINT_VARIABLE + + " to the same value, as a later release may no " + + "longer read " + OLD_ENDPOINT_VARIABLE + "."); + } + } if (value == null) { - return STAR_SIGN_ENDPOINT; + // A resource key and an endpoint travel together, because a + // key is only known to the service that issued it. So the + // 51Degrees cloud is used whenever a 51Degrees resource key + // was found, and the star sign service, with the star sign + // key, only when there was none. + return configuredResourceKey() == null + ? STAR_SIGN_ENDPOINT + : FIFTYONE_CLOUD_ENDPOINT; } while (value.endsWith("/")) { value = value.substring(0, value.length() - 1); @@ -70,30 +120,72 @@ static String endpoint() { } /** - * The resource key from _51DEGREES_RESOURCE_KEY, or the older - * RESOURCE_KEY, so the example can be pointed at a host whose keys - * are its own. Otherwise the key of the star sign service. + * The names a 51Degrees resource key is read from, in the order they + * are tried. Each is read as an environment variable first and then + * as a system property, so a developer can export one and a build + * can pass one on the command line with -D. */ - static String resourceKey() { - String value = env("_51DEGREES_RESOURCE_KEY"); - if (value == null) { - value = env("RESOURCE_KEY"); + static final String[] RESOURCE_KEY_NAMES = { + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "TestResourceKey", + "SUPER_RESOURCE_KEY", + }; + + /** + * The 51Degrees resource key that was found, or null when none of the + * names above is set anywhere. When the first name is not set its + * deprecated name RESOURCE_KEY is read before the others, and the + * first use of it is logged as a warning. + */ + static String configuredResourceKey() { + for (String name : RESOURCE_KEY_NAMES) { + String value = env(name); + if (value == null && name.equals(RESOURCE_KEY_NAMES[0])) { + value = env(OLD_RESOURCE_KEY_VARIABLE); + if (value != null + && oldResourceKeyVariableLogged.compareAndSet( + false, true)) { + loggerFactory.getLogger(Main.class.getName()).warn( + "The environment variable " + OLD_RESOURCE_KEY_VARIABLE + + " is deprecated and " + name + + " replaces it. Set " + name + + " to the same value, as a later release may no longer " + + "read " + OLD_RESOURCE_KEY_VARIABLE + "."); + } + } + if (value != null) { + return value; + } } + return null; + } + + /** + * The resource key the example runs with, which is a 51Degrees key + * where one was found and the key of the star sign service where + * none was. + */ + static String resourceKey() { + String value = configuredResourceKey(); return value == null ? STAR_SIGN_RESOURCE_KEY : value; } static String env(String name) { String value = System.getenv(name); - return value == null || value.trim().isEmpty() ? null : value; + if (value == null || value.trim().isEmpty()) { + value = System.getProperty(name); + } + return value == null || value.trim().isEmpty() ? null : value.trim(); } public static class Example { public void run() throws Exception { //! [usage] - // The endpoint comes from FOD_CLOUD_API_URL when it is set, so - // the example can be pointed at an on premise web server or a - // privately hosted version of the 51Degrees cloud, and - // otherwise at the star sign service it was written for. + // The endpoint comes from _51DEGREES_CLOUD_ENDPOINT when it is + // set, so the example can be pointed at an on premise web + // server or a privately hosted version of the 51Degrees cloud, + // and otherwise at the star sign service it was written for. CloudRequestEngine cloudRequestEngine = new CloudRequestEngineBuilder(loggerFactory, httpClient) .setEndpoint(endpoint()) diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java new file mode 100644 index 000000000..c6969a90e --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/EndpointTests.java @@ -0,0 +1,119 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.cloudengine; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Tests which names the example takes its endpoint and its resource key + * from. The example reads each name as an environment variable and then as + * a system property, so these tests set system properties, and are skipped + * where the environment already sets a name they use, because the + * environment would answer first. + */ +public class EndpointTests { + + private static final String NEW_NAME = "_51DEGREES_CLOUD_ENDPOINT"; + private static final String OLD_NAME = "FOD_CLOUD_API_URL"; + + @BeforeEach + public void setUp() { + assumeTrue(System.getenv(NEW_NAME) == null); + assumeTrue(System.getenv(OLD_NAME) == null); + clear(); + } + + @AfterEach + public void clear() { + System.clearProperty(NEW_NAME); + System.clearProperty(OLD_NAME); + } + + @Test + public void Endpoint_Old_Name_Alone() { + System.setProperty(OLD_NAME, "https://old.example/api/v4"); + + assertEquals("https://old.example/api/v4/", Main.endpoint()); + } + + @Test + public void Endpoint_New_Name_Wins() { + System.setProperty(NEW_NAME, "https://new.example/api/v4/"); + System.setProperty(OLD_NAME, "https://old.example/api/v4/"); + + assertEquals("https://new.example/api/v4/", Main.endpoint()); + } + + @Test + public void Endpoint_Neither_Name() { + String expected = Main.configuredResourceKey() == null + ? "https://51degrees.com/starsign/api/" + : "https://cloud.51degrees.com/api/v4/"; + + assertEquals(expected, Main.endpoint()); + } + + private static final String NEW_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; + private static final String OLD_KEY_NAME = "RESOURCE_KEY"; + + @Test + public void ResourceKey_Old_Name_Alone() { + assumeTrue(System.getenv(NEW_KEY_NAME) == null); + assumeTrue(System.getenv(OLD_KEY_NAME) == null); + String savedNew = System.clearProperty(NEW_KEY_NAME); + try { + System.setProperty(OLD_KEY_NAME, "old-name-key"); + + assertEquals("old-name-key", Main.configuredResourceKey()); + } finally { + System.clearProperty(OLD_KEY_NAME); + if (savedNew != null) { + System.setProperty(NEW_KEY_NAME, savedNew); + } + } + } + + @Test + public void ResourceKey_New_Name_Wins() { + assumeTrue(System.getenv(NEW_KEY_NAME) == null); + assumeTrue(System.getenv(OLD_KEY_NAME) == null); + String savedNew = System.clearProperty(NEW_KEY_NAME); + try { + System.setProperty(NEW_KEY_NAME, "new-name-key"); + System.setProperty(OLD_KEY_NAME, "old-name-key"); + + assertEquals("new-name-key", Main.configuredResourceKey()); + } finally { + System.clearProperty(OLD_KEY_NAME); + System.clearProperty(NEW_KEY_NAME); + if (savedNew != null) { + System.setProperty(NEW_KEY_NAME, savedNew); + } + } + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/ExampleTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/ExampleTests.java index 36c491b54..626622038 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/ExampleTests.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.cloud-engine/src/test/java/pipeline/developerexamples/cloudengine/ExampleTests.java @@ -25,19 +25,32 @@ import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.Assertions.fail; +/** + * Runs the cloud engine example against the live cloud service. + *

    + * Nothing is caught here. The example builds a cloud request engine, + * which asks the service for its accessible properties and its evidence + * keys, and then processes evidence through it, so any failure to reach + * the service, or any answer the pipeline cannot use, has to be seen. + * Catching and skipping would report success for a run that never + * reached the service at all. + */ public class ExampleTests { @Test public void SimpleCloudEngine_Test() throws Exception { - try { - new Main.Example().run(); - } - catch (Exception e) { - // Only assume, as this example will fail if there is no internet - // connection. - assumeFalse(true, "Exception thrown"); + if (Main.configuredResourceKey() == null) { + fail("This test runs the cloud engine example against the live " + + "51Degrees cloud service, so it needs a resource key. None " + + "of " + String.join(", ", Main.RESOURCE_KEY_NAMES) + " is " + + "set, as an environment variable or as a system property, " + + "tried in that order. Any key works here, because the " + + "example reports a product the key does not carry rather " + + "than failing on it. Create a key at " + + "https://configure.51degrees.com."); } + new Main.Example().run(); } } diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/README.md b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/README.md index 586556167..73ab80af9 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/README.md +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/README.md @@ -51,7 +51,7 @@ The only server-side part of the flow is the redeem call, which adds the licence key the browser never sees. The `redeem` method in `CreatorContextDemoServer.java` is that call, and these are its essential lines. `DidClient` is built once at start-up from the -resource key and the licence key (it reads `FOD_CLOUD_API_URL` itself +resource key and the licence key (it reads `_51DEGREES_CLOUD_ENDPOINT` itself and falls back to the public cloud), and `did`, `result` and `challenge` are what the page passed on from `verify-full`: @@ -90,9 +90,9 @@ scope. | Variable | Meaning | | --- | --- | -| `_51DEGREES_RESOURCE_KEY` | Required. The resource key of the page, public by nature. The older name `RESOURCE_KEY` is read when this one is not set. | -| `_51DEGREES_LICENSE_KEY` | Optional. A licence key of the same account, server side only. The older name `LICENSE_KEY` is read when this one is not set. Only an account that holds licence keys needs one to redeem, because the licence key is what keeps redemption to the acting party's own servers, so an account holding none redeems without one. | -| `FOD_CLOUD_API_URL` | Optional. The cloud API base including the `/api/v4/` segment, defaulting to `https://cloud.51degrees.com/api/v4/`. This is the same variable the cloud request engine in this repository honours, so setting it once points every 51Degrees example at the same place. A trailing slash is added where missing. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons, which is the private hosting option of the cloud service. Both run the same service, so the examples work unchanged. | +| `_51DEGREES_RESOURCE_KEY` | Required. The resource key of the page, public by nature. | +| `_51DEGREES_LICENSE_KEY` | Optional. A licence key of the same account, server side only. Only an account that holds licence keys needs one to redeem, because the licence key is what keeps redemption to the acting party's own servers, so an account holding none redeems without one. | +| `_51DEGREES_CLOUD_ENDPOINT` | Optional. The cloud API base including the `/api/v4/` segment, defaulting to `https://cloud.51degrees.com/api/v4/`. This is the same variable the cloud request engine in this repository honours, so setting it once points every 51Degrees example at the same place. A trailing slash is added where missing. A host other than cloud.51degrees.com would be used to (a) use an on premise web server, or (b) use a privately hosted version of the 51Degrees cloud for performance reasons, which is the private hosting option of the cloud service. Both run the same service, so the examples work unchanged. | | `PORT` | The port to listen on, defaulting to `5100`. | ### How to run diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java index 02c7e8326..3eaf3c223 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/java/pipeline/developerexamples/fodid/CreatorContextDemoServer.java @@ -101,11 +101,13 @@ * {@code fodid/creator-context/} in this module's resources. Environment * variables: *

      - *
    • {@code _51DEGREES_RESOURCE_KEY}, or the older {@code RESOURCE_KEY}, - * required. The resource key of the page, public by nature.
      • - *
    • {@code _51DEGREES_LICENSE_KEY}, or the older {@code LICENSE_KEY}, - * optional. A licence key of the same account, server side only.
      • - *
    • {@code FOD_CLOUD_API_URL}, optional. The cloud API base including + *
    • {@code _51DEGREES_RESOURCE_KEY}, required. The resource key of the + * page, public by nature. Its deprecated name {@code RESOURCE_KEY} is read + * when it is not set.
      • + *
    • {@code _51DEGREES_LICENSE_KEY}, optional. A licence key of the same + * account, server side only. Its deprecated name {@code LICENSE_KEY} is + * read when it is not set.
      • + *
    • {@code _51DEGREES_CLOUD_ENDPOINT}, optional. The cloud API base including * the {@code /api/v4/} segment, defaulting to * {@code https://cloud.51degrees.com/api/v4/}. This is the same variable * the cloud request engine of this package honours.
      • @@ -119,9 +121,9 @@ public class CreatorContextDemoServer { static final String RESOURCES = "/fodid/creator-context/"; static final String RESOURCE = - env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY"); + key("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY"); static final String LICENCE = - env("_51DEGREES_LICENSE_KEY", "LICENSE_KEY"); + key("_51DEGREES_LICENSE_KEY", "LICENSE_KEY"); /** * The one client, built at start-up and shared by every request, @@ -131,8 +133,8 @@ public class CreatorContextDemoServer { public static void main(String[] args) throws Exception { if (RESOURCE == null) { - System.err.println("Set _51DEGREES_RESOURCE_KEY (or the older " - + "RESOURCE_KEY) to the resource key of the page."); + System.err.println( + "Set _51DEGREES_RESOURCE_KEY to the resource key of the page."); System.exit(1); } if (LICENCE == null) { @@ -149,7 +151,7 @@ public static void main(String[] args) throws Exception { + "will report the context unreadable where it holds " + "some."); } - // The client reads FOD_CLOUD_API_URL itself when no endpoint is + // The client reads _51DEGREES_CLOUD_ENDPOINT itself when no endpoint is // given, and falls back to the public cloud. A host other than // cloud.51degrees.com would be used to (a) use an on premise web // server, or (b) use a privately hosted version of the 51Degrees @@ -462,6 +464,27 @@ static String env(String... names) { return null; } + /** + * The value of the named environment variable, or of its deprecated + * old name when the named one is not set, saying on standard error + * that the old name is deprecated when it is the one used. + */ + static String key(String name, String oldName) { + String value = env(name); + if (value == null) { + value = env(oldName); + if (value != null) { + System.err.println( + "The environment variable " + oldName + + " is deprecated and " + name + + " replaces it. Set " + name + + " to the same value, as a later release may no longer " + + "read " + oldName + "."); + } + } + return value; + } + static String envOr(String name, String fallback) { String value = env(name); return value == null ? fallback : value; diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/examples-main.min.css b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/examples-main.min.css index 8007642a3..42a9f40d7 100644 --- a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/examples-main.min.css +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/main/resources/fodid/creator-context/examples-main.min.css @@ -1 +1 @@ -/*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */html{line-height:1.15;-webkit-text-size-adjust:100%}body{margin:0}main{display:block}h1{font-size:2em;margin:.67em 0}hr{box-sizing:content-box;height:0;overflow:visible}pre{font-family:monospace,monospace;font-size:1em}a{background-color:transparent}abbr[title]{border-bottom:none;text-decoration:underline;text-decoration:underline dotted}b,strong{font-weight:bolder}code,kbd,samp{font-family:monospace,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}img{border-style:none}button,input,optgroup,select,textarea{font-family:inherit;font-size:100%;line-height:1.15;margin:0}button,input{overflow:visible}button,select{text-transform:none}button,[type=button],[type=reset],[type=submit]{-webkit-appearance:button}button::-moz-focus-inner,[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner{border-style:none;padding:0}button:-moz-focusring,[type=button]:-moz-focusring,[type=reset]:-moz-focusring,[type=submit]:-moz-focusring{outline:1px dotted ButtonText}fieldset{padding:.35em .75em .625em}legend{box-sizing:border-box;color:inherit;display:table;max-width:100%;padding:0;white-space:normal}progress{vertical-align:baseline}textarea{overflow:auto}[type=checkbox],[type=radio]{box-sizing:border-box;padding:0}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}details{display:block}summary{display:list-item}template{display:none}[hidden]{display:none}.a-size--10{width:.1615055829rem;height:.1615055829rem}.a-size--9{width:.1938066995rem;height:.1938066995rem}.a-size--8{width:.2325680394rem;height:.2325680394rem}.a-size--7{width:.2790816472rem;height:.2790816472rem}.a-size--6{width:.3348979767rem;height:.3348979767rem}.a-size--5{width:.401877572rem;height:.401877572rem}.a-size--4{width:.4822530864rem;height:.4822530864rem}.a-size--3{width:.5787037037rem;height:.5787037037rem}.a-size--2{width:.6944444444rem;height:.6944444444rem}.a-size--1{width:.8333333333rem;height:.8333333333rem}.a-size-0{width:1rem;height:1rem}.a-size-1{width:1.2rem;height:1.2rem}.a-size-2{width:1.44rem;height:1.44rem}.a-size-3{width:1.728rem;height:1.728rem}.a-size-4{width:2.0736rem;height:2.0736rem}.a-size-5{width:2.48832rem;height:2.48832rem}.a-size-6{width:2.985984rem;height:2.985984rem}.a-size-7{width:3.5831808rem;height:3.5831808rem}.a-size-8{width:4.29981696rem;height:4.29981696rem}.a-size-9{width:5.159780352rem;height:5.159780352rem}.a-size-10{width:6.1917364224rem;height:6.1917364224rem}.a-size-11{width:7.4300837069rem;height:7.4300837069rem}.a-size-12{width:8.9161004483rem;height:8.9161004483rem}.a-size-13{width:10.6993205379rem;height:10.6993205379rem}.a-size-14{width:12.8391846455rem;height:12.8391846455rem}.a-size-15{width:15.4070215746rem;height:15.4070215746rem}.a-size-16{width:18.4884258895rem;height:18.4884258895rem}.a-size-17{width:22.1861110674rem;height:22.1861110674rem}.a-size-18{width:26.6233332809rem;height:26.6233332809rem}.a-size-19{width:31.9479999371rem;height:31.9479999371rem}.a-size-20{width:38.3375999245rem;height:38.3375999245rem}.a-font-size--10{font-size:.1615055829rem}.a-font-size--9{font-size:.1938066995rem}.a-font-size--8{font-size:.2325680394rem}.a-font-size--7{font-size:.2790816472rem}.a-font-size--6{font-size:.3348979767rem}.a-font-size--5{font-size:.401877572rem}.a-font-size--4{font-size:.4822530864rem}.a-font-size--3{font-size:.5787037037rem}.a-font-size--2{font-size:.6944444444rem}.a-font-size--1{font-size:.8333333333rem}.a-font-size-0{font-size:1rem}.a-font-size-1{font-size:1.2rem}.a-font-size-2{font-size:1.44rem}.a-font-size-3{font-size:1.728rem}.a-font-size-4{font-size:2.0736rem}.a-font-size-5{font-size:2.48832rem}.a-font-size-6{font-size:2.985984rem}.a-font-size-7{font-size:3.5831808rem}.a-font-size-8{font-size:4.29981696rem}.a-font-size-9{font-size:5.159780352rem}.a-font-size-10{font-size:6.1917364224rem}.a-font-size-11{font-size:7.4300837069rem}.a-font-size-12{font-size:8.9161004483rem}.a-font-size-13{font-size:10.6993205379rem}.a-font-size-14{font-size:12.8391846455rem}.a-font-size-15{font-size:15.4070215746rem}.a-font-size-16{font-size:18.4884258895rem}.a-font-size-17{font-size:22.1861110674rem}.a-font-size-18{font-size:26.6233332809rem}.a-font-size-19{font-size:31.9479999371rem}.a-font-size-20{font-size:38.3375999245rem}.a-font--default{font-family:Arial,sans-serif}.a-font--code{font-family:Consolas,monospace}.b-text--hidden{position:absolute;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.b-text--default{font-family:Arial,sans-serif;font-size:1rem;color:#252525}.b-text--count{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#9b9b9b}.b-text--count-pill{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#9b9b9b;border:1px solid #BAC82A;border-radius:28px;padding:.093463879rem .4822530864rem;color:#252525;background-color:#fff;display:inline}.b-text--property-pill{font-family:Arial,sans-serif;font-size:.8333333333rem;color:#646060}.b-text--heading-1{font-family:Arial,sans-serif;font-weight:700;font-size:1.44rem;color:#252525;margin:0 0 1.728rem}.b-text--heading-2{font-family:Arial,sans-serif;font-weight:700;font-size:1.2rem;color:#252525;margin:0 0 1.44rem}.b-text--heading-3{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:0 0 1.2rem}.b-text--code{font-family:Consolas,monospace;font-size:.8333333333rem;color:#646060}.b-text--code-snip{font-family:Consolas,monospace;font-size:.8333333333rem;color:#646060;background-color:#fff;border-radius:5px;border:1px solid #EAEAEA;padding:0 .2325680394rem}.b-text--heading-caps{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#252525;font-weight:400;text-transform:uppercase;letter-spacing:.5px}.b-text--copyright{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#252525}.b-text--progress{font-family:Arial,sans-serif;font-size:.8333333333rem}@media(min-width:769px){.b-text--progress{font-size:1rem}}.b-text--progress{color:#252525;letter-spacing:.7px;text-transform:uppercase;font-weight:700}.b-text--red{color:#cc2b27}body{font-family:Arial,sans-serif;font-size:1rem;color:#252525;margin:0;line-height:1.5}*{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;box-sizing:border-box}.b-link{transition:color .3s ease;color:#252525;text-decoration:underline}.b-link:active,.b-link:hover,.b-link:focus{color:#f5841f;text-decoration:underline}.b-link{display:inline-flex;align-items:center}.b-link img{margin:0 .5787037037rem}.b-link--property{font-family:Arial,sans-serif;font-size:.8333333333rem;color:#646060;color:#9b9b9b;border-bottom:2px dotted #EAEAEA;text-decoration:none}.b-link--property:active,.b-link--property:hover,.b-link--property:focus{color:#646060;border-bottom:2px dotted #646060;text-decoration:none}.b-link--dotted{color:#9b9b9b;border-bottom:2px dotted #EAEAEA;text-decoration:none}.b-link--dotted:active,.b-link--dotted:hover,.b-link--dotted:focus{color:#646060;border-bottom:2px dotted #646060;text-decoration:none}.b-link--unstyled{color:inherit;text-decoration:none}.b-link--unstyled:active,.b-link--unstyled:hover,.b-link--unstyled:focus{color:inherit;text-decoration:none}.b-link--docs{transition:color .3s ease;color:#00aeef;text-decoration:none;font-weight:700}.b-link--docs:active,.b-link--docs:hover,.b-link--docs:focus{text-decoration:underline}a.anchor{scroll-margin-top:128px}.b-btn{font-family:Arial,sans-serif;font-size:1rem;color:#252525;color:inherit;text-decoration:none}.b-btn:active,.b-btn:hover,.b-btn:focus{color:inherit;text-decoration:none}.b-btn{display:inline-flex;justify-content:center;align-items:center;padding:.8333333333rem 1.728rem;border:1px solid #BAC82A;background-color:#cbdb2a;cursor:pointer}.b-btn:active,.b-btn:hover,.b-btn:focus{background-color:#daed1a;border:1px solid #CBDB2A;text-decoration:none}.b-btn.b-btn--disabled,.b-btn:disabled{background-color:#f8f8f8;border:1px solid #EAEAEA;color:#646060;cursor:not-allowed;pointer-events:all!important}@media(min-width:769px){.b-btn--large{padding:1.2rem 2.48832rem}}.b-btn--secondary{background-color:#fff}.b-btn--secondary:active,.b-btn--secondary:hover,.b-btn--secondary:focus{background-color:#fff}.b-btn--block{width:100%}.b-btn__icon{margin-right:1rem}.b-btn__icon--after{margin-right:0;margin-left:1rem}input[type=text]{-webkit-appearance:none;-moz-appearance:none;appearance:none}.b-label{color:#646060;font-size:.8333333333rem;margin-bottom:.4822530864rem;width:100%}.b-input-group{display:flex;flex-direction:column}@media(min-width:769px){.b-input-group{flex-direction:row}}.b-input{font-family:Arial,sans-serif;font-size:1rem;color:#252525;border:solid 1px #646060;padding:.8333333333rem;line-height:1.2}.b-form-group--with-icon .b-input{padding-left:46px}.b-input::placeholder{color:#bfbfbf;opacity:1}.b-input,.b-input--block{width:100%}.b-form-group{display:flex;flex-direction:column}.b-form-group .b-btn{margin-top:1rem}@media(min-width:769px){.b-form-group .b-btn{margin-top:0;margin-left:.8333333333rem}}.b-form-group--with-icon{position:relative}.b-form-group--with-icon .b-icon{position:absolute;width:20px;height:20px;top:14px;left:10px}.c-eg-page{max-width:800px;margin:0 auto;padding:1.44rem 1.2rem}.c-eg-page__title{font-family:Arial,sans-serif;font-weight:700;font-size:1.2rem;color:#252525;margin:0 0 1.44rem}.c-eg-page__heading{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:1.728rem 0 1.2rem}.c-eg-page__lead{margin-bottom:1.44rem}.c-eg-section{margin-bottom:1.728rem}.c-eg-alert{font-family:Arial,sans-serif;font-size:1rem;color:#252525;border:1px solid #CC2B27;border-radius:5px;background-color:#fef1f9;padding:.8333333333rem 1rem;margin:1.2rem 0}.c-eg-alert a{color:#cc2b27;font-weight:700}.c-eg-table{font-family:Arial,sans-serif;font-size:1rem;color:#252525;width:100%;max-width:800px;font-size:.8333333333rem;background-color:#fff;border-collapse:collapse;margin-bottom:1.2rem}.c-eg-table__head .c-eg-table__cell{font-weight:700;border-bottom:3px solid #EAEAEA}.c-eg-table__cell{padding:.5787037037rem .6944444444rem;text-align:left;vertical-align:top;border-bottom:1px solid #EAEAEA}.c-eg-table__cell--key{font-weight:700;white-space:nowrap}.c-eg-table__cell ul{margin:0;padding-left:1rem}.c-eg-table__row--used{background-color:#f9fbe9}.c-eg-table__row--present,.c-eg-table__row--alt{background-color:#f8f8f8}.c-eg-table__action{padding:.4822530864rem .6944444444rem;font-size:.6944444444rem;white-space:nowrap;width:auto}.c-eg-legend{font-size:.6944444444rem;color:#646060;margin-bottom:.6944444444rem}.c-eg-legend__swatch{padding:0 .4822530864rem;border-radius:5px}.c-eg-legend__swatch--used{background-color:#f9fbe9}.c-eg-legend__swatch--present{background-color:#f8f8f8}.c-eg-form{margin-bottom:1.44rem}.c-eg-form__row{display:flex;flex-direction:column}@media(min-width:769px){.c-eg-form__row{flex-direction:row;align-items:flex-end}}.c-eg-form .b-btn{margin-top:.8333333333rem}@media(min-width:769px){.c-eg-form .b-btn{margin-top:0;margin-left:.8333333333rem}}.c-eg-map{margin:1.44rem 0}.c-eg-map__title{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:0 0 1.2rem}.c-eg-map__canvas{height:400px;width:100%;border:1px solid #EAEAEA;border-radius:5px}.c-eg-columns{display:grid;grid-template-columns:1fr;gap:1.44rem}@media(min-width:993px){.c-eg-columns{grid-template-columns:1fr 1fr}}.c-eg-details{margin:1.2rem 0 1.44rem;border:1px solid #EAEAEA;border-radius:5px;padding:0 1rem}.c-eg-details>summary{cursor:pointer;padding:.6944444444rem 0;font-weight:700}.c-eg-button-row{margin:1.2rem 0 1.44rem}.c-eg-message{display:flex;flex-direction:column;align-items:flex-start;gap:.8333333333rem;margin-top:2.0736rem;padding:1rem 1.2rem;background:#f8f8f8;border:1px solid #EAEAEA;border-radius:5px}@media(min-width:769px){.c-eg-message{flex-direction:row;align-items:center;justify-content:space-between;gap:1.2rem}}.c-eg-message__text{margin:0}.c-eg-message__cta{flex-shrink:0}.c-eg-status{font-family:Arial,sans-serif;font-size:1rem;color:#252525;font-weight:700}.c-eg-status--pass{color:#9ba90d}.c-eg-status--fail{color:#cc2b27}.c-eg-status--part{color:#f5841f}.c-eg-status--none{color:#646060;font-weight:400}.c-eg-value{font-family:Consolas,monospace;font-size:.8333333333rem;color:#646060;word-break:break-all} +/*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */html{line-height:1.15;-webkit-text-size-adjust:100%}body{margin:0}main{display:block}h1{font-size:2em;margin:.67em 0}hr{box-sizing:content-box;height:0;overflow:visible}pre{font-family:monospace,monospace;font-size:1em}a{background-color:transparent}abbr[title]{border-bottom:none;text-decoration:underline;text-decoration:underline dotted}b,strong{font-weight:bolder}code,kbd,samp{font-family:monospace,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}img{border-style:none}button,input,optgroup,select,textarea{font-family:inherit;font-size:100%;line-height:1.15;margin:0}button,input{overflow:visible}button,select{text-transform:none}button,[type=button],[type=reset],[type=submit]{-webkit-appearance:button}button::-moz-focus-inner,[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner{border-style:none;padding:0}button:-moz-focusring,[type=button]:-moz-focusring,[type=reset]:-moz-focusring,[type=submit]:-moz-focusring{outline:1px dotted ButtonText}fieldset{padding:.35em .75em .625em}legend{box-sizing:border-box;color:inherit;display:table;max-width:100%;padding:0;white-space:normal}progress{vertical-align:baseline}textarea{overflow:auto}[type=checkbox],[type=radio]{box-sizing:border-box;padding:0}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}details{display:block}summary{display:list-item}template{display:none}[hidden]{display:none}.a-size--10{width:.1615055829rem;height:.1615055829rem}.a-size--9{width:.1938066995rem;height:.1938066995rem}.a-size--8{width:.2325680394rem;height:.2325680394rem}.a-size--7{width:.2790816472rem;height:.2790816472rem}.a-size--6{width:.3348979767rem;height:.3348979767rem}.a-size--5{width:.401877572rem;height:.401877572rem}.a-size--4{width:.4822530864rem;height:.4822530864rem}.a-size--3{width:.5787037037rem;height:.5787037037rem}.a-size--2{width:.6944444444rem;height:.6944444444rem}.a-size--1{width:.8333333333rem;height:.8333333333rem}.a-size-0{width:1rem;height:1rem}.a-size-1{width:1.2rem;height:1.2rem}.a-size-2{width:1.44rem;height:1.44rem}.a-size-3{width:1.728rem;height:1.728rem}.a-size-4{width:2.0736rem;height:2.0736rem}.a-size-5{width:2.48832rem;height:2.48832rem}.a-size-6{width:2.985984rem;height:2.985984rem}.a-size-7{width:3.5831808rem;height:3.5831808rem}.a-size-8{width:4.29981696rem;height:4.29981696rem}.a-size-9{width:5.159780352rem;height:5.159780352rem}.a-size-10{width:6.1917364224rem;height:6.1917364224rem}.a-size-11{width:7.4300837069rem;height:7.4300837069rem}.a-size-12{width:8.9161004483rem;height:8.9161004483rem}.a-size-13{width:10.6993205379rem;height:10.6993205379rem}.a-size-14{width:12.8391846455rem;height:12.8391846455rem}.a-size-15{width:15.4070215746rem;height:15.4070215746rem}.a-size-16{width:18.4884258895rem;height:18.4884258895rem}.a-size-17{width:22.1861110674rem;height:22.1861110674rem}.a-size-18{width:26.6233332809rem;height:26.6233332809rem}.a-size-19{width:31.9479999371rem;height:31.9479999371rem}.a-size-20{width:38.3375999245rem;height:38.3375999245rem}.a-font-size--10{font-size:.1615055829rem}.a-font-size--9{font-size:.1938066995rem}.a-font-size--8{font-size:.2325680394rem}.a-font-size--7{font-size:.2790816472rem}.a-font-size--6{font-size:.3348979767rem}.a-font-size--5{font-size:.401877572rem}.a-font-size--4{font-size:.4822530864rem}.a-font-size--3{font-size:.5787037037rem}.a-font-size--2{font-size:.6944444444rem}.a-font-size--1{font-size:.8333333333rem}.a-font-size-0{font-size:1rem}.a-font-size-1{font-size:1.2rem}.a-font-size-2{font-size:1.44rem}.a-font-size-3{font-size:1.728rem}.a-font-size-4{font-size:2.0736rem}.a-font-size-5{font-size:2.48832rem}.a-font-size-6{font-size:2.985984rem}.a-font-size-7{font-size:3.5831808rem}.a-font-size-8{font-size:4.29981696rem}.a-font-size-9{font-size:5.159780352rem}.a-font-size-10{font-size:6.1917364224rem}.a-font-size-11{font-size:7.4300837069rem}.a-font-size-12{font-size:8.9161004483rem}.a-font-size-13{font-size:10.6993205379rem}.a-font-size-14{font-size:12.8391846455rem}.a-font-size-15{font-size:15.4070215746rem}.a-font-size-16{font-size:18.4884258895rem}.a-font-size-17{font-size:22.1861110674rem}.a-font-size-18{font-size:26.6233332809rem}.a-font-size-19{font-size:31.9479999371rem}.a-font-size-20{font-size:38.3375999245rem}.a-font--default{font-family:Arial,sans-serif}.a-font--code{font-family:Consolas,monospace}.b-text--hidden{position:absolute;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.b-text--default{font-family:Arial,sans-serif;font-size:1rem;color:#252525}.b-text--count{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#9b9b9b}.b-text--count-pill{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#9b9b9b;border:1px solid #BAC82A;border-radius:28px;padding:.093463879rem .4822530864rem;color:#252525;background-color:#fff;display:inline}.b-text--property-pill{font-family:Arial,sans-serif;font-size:.8333333333rem;color:#646060}.b-text--heading-1{font-family:Arial,sans-serif;font-weight:700;font-size:1.44rem;color:#252525;margin:0 0 1.728rem}.b-text--heading-2{font-family:Arial,sans-serif;font-weight:700;font-size:1.2rem;color:#252525;margin:0 0 1.44rem}.b-text--heading-3{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:0 0 1.2rem}.b-text--code{font-family:Consolas,monospace;font-size:.8333333333rem;color:#646060}.b-text--code-snip{font-family:Consolas,monospace;font-size:.8333333333rem;color:#646060;background-color:#fff;border-radius:5px;border:1px solid #EAEAEA;padding:0 .2325680394rem}.b-text--heading-caps{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#252525;font-weight:400;text-transform:uppercase;letter-spacing:.5px}.b-text--copyright{font-family:Arial,sans-serif;font-size:.6944444444rem;color:#252525}.b-text--progress{font-family:Arial,sans-serif;font-size:.8333333333rem}@media(min-width:769px){.b-text--progress{font-size:1rem}}.b-text--progress{color:#252525;letter-spacing:.7px;text-transform:uppercase;font-weight:700}.b-text--red{color:#cc2b27}body{font-family:Arial,sans-serif;font-size:1rem;color:#252525;margin:0;line-height:1.5}*{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;box-sizing:border-box}.b-link{transition:color .3s ease;color:#252525;text-decoration:underline}.b-link:active,.b-link:hover,.b-link:focus{color:#f5841f;text-decoration:underline}.b-link{display:inline-flex;align-items:center}.b-link img{margin:0 .5787037037rem}.b-link--property{font-family:Arial,sans-serif;font-size:.8333333333rem;color:#646060;color:#9b9b9b;border-bottom:2px dotted #EAEAEA;text-decoration:none}.b-link--property:active,.b-link--property:hover,.b-link--property:focus{color:#646060;border-bottom:2px dotted #646060;text-decoration:none}.b-link--dotted{color:#9b9b9b;border-bottom:2px dotted #EAEAEA;text-decoration:none}.b-link--dotted:active,.b-link--dotted:hover,.b-link--dotted:focus{color:#646060;border-bottom:2px dotted #646060;text-decoration:none}.b-link--unstyled{color:inherit;text-decoration:none}.b-link--unstyled:active,.b-link--unstyled:hover,.b-link--unstyled:focus{color:inherit;text-decoration:none}.b-link--docs{transition:color .3s ease;color:#00aeef;text-decoration:none;font-weight:700}.b-link--docs:active,.b-link--docs:hover,.b-link--docs:focus{text-decoration:underline}a.anchor{scroll-margin-top:128px}.b-btn{font-family:Arial,sans-serif;font-size:1rem;color:#252525;color:inherit;text-decoration:none}.b-btn:active,.b-btn:hover,.b-btn:focus{color:inherit;text-decoration:none}.b-btn{display:inline-flex;justify-content:center;align-items:center;padding:.8333333333rem 1.728rem;border:1px solid #BAC82A;background-color:#cbdb2a;cursor:pointer}.b-btn:active,.b-btn:hover,.b-btn:focus{background-color:#daed1a;border:1px solid #CBDB2A;text-decoration:none}.b-btn.b-btn--disabled,.b-btn:disabled{background-color:#f8f8f8;border:1px solid #EAEAEA;color:#646060;cursor:not-allowed;pointer-events:all!important}@media(min-width:769px){.b-btn--large{padding:1.2rem 2.48832rem}}.b-btn--secondary{background-color:#fff}.b-btn--secondary:active,.b-btn--secondary:hover,.b-btn--secondary:focus{background-color:#fff}.b-btn--block{width:100%}.b-btn__icon{margin-right:1rem}.b-btn__icon--after{margin-right:0;margin-left:1rem}input[type=text]{-webkit-appearance:none;-moz-appearance:none;appearance:none}.b-label{color:#646060;font-size:.8333333333rem;margin-bottom:.4822530864rem;width:100%}.b-input-group{display:flex;flex-direction:column}@media(min-width:769px){.b-input-group{flex-direction:row}}.b-input{font-family:Arial,sans-serif;font-size:1rem;color:#252525;border:solid 1px #646060;padding:.8333333333rem;line-height:1.2}.b-form-group--with-icon .b-input{padding-left:46px}.b-input::placeholder{color:#bfbfbf;opacity:1}.b-input,.b-input--block{width:100%}.b-form-group{display:flex;flex-direction:column}.b-form-group .b-btn{margin-top:1rem}@media(min-width:769px){.b-form-group .b-btn{margin-top:0;margin-left:.8333333333rem}}.b-form-group--with-icon{position:relative}.b-form-group--with-icon .b-icon{position:absolute;width:20px;height:20px;top:14px;left:10px}.c-eg-page{max-width:800px;margin:0 auto;padding:1.44rem 1.2rem}.c-eg-page__title{font-family:Arial,sans-serif;font-weight:700;font-size:1.2rem;color:#252525;margin:0 0 1.44rem}.c-eg-page__heading{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:1.728rem 0 1.2rem}.c-eg-page__lead{margin-bottom:1.44rem}.c-eg-section{margin-bottom:1.728rem}.c-eg-alert{font-family:Arial,sans-serif;font-size:1rem;color:#252525;border:1px solid #CC2B27;border-radius:5px;background-color:#fef1f9;padding:.8333333333rem 1rem;margin:1.2rem 0}.c-eg-alert a{color:#cc2b27;font-weight:700}.c-eg-table{font-family:Arial,sans-serif;font-size:1rem;color:#252525;width:100%;max-width:800px;font-size:.8333333333rem;background-color:#fff;border-collapse:collapse;margin-bottom:1.2rem}.c-eg-table__head .c-eg-table__cell{font-weight:700;border-bottom:3px solid #EAEAEA}.c-eg-table__cell{padding:.5787037037rem .6944444444rem;text-align:left;vertical-align:top;border-bottom:1px solid #EAEAEA}.c-eg-table__cell--key{font-weight:700;white-space:nowrap}.c-eg-table__cell ul{margin:0;padding-left:1rem}.c-eg-table__row--used{background-color:#f9fbe9}.c-eg-table__row--present,.c-eg-table__row--alt{background-color:#f8f8f8}.c-eg-legend{font-size:.6944444444rem;color:#646060;margin-bottom:.6944444444rem}.c-eg-legend__swatch{padding:0 .4822530864rem;border-radius:5px}.c-eg-legend__swatch--used{background-color:#f9fbe9}.c-eg-legend__swatch--present{background-color:#f8f8f8}.c-eg-form{margin-bottom:1.44rem}.c-eg-form__row{display:flex;flex-direction:column}@media(min-width:769px){.c-eg-form__row{flex-direction:row;align-items:flex-end}}.c-eg-form .b-btn{margin-top:.8333333333rem}@media(min-width:769px){.c-eg-form .b-btn{margin-top:0;margin-left:.8333333333rem}}.c-eg-map{margin:1.44rem 0}.c-eg-map__title{font-family:Arial,sans-serif;font-weight:700;font-size:1rem;color:#252525;margin:0 0 1.2rem}.c-eg-map__canvas{height:400px;width:100%;border:1px solid #EAEAEA;border-radius:5px}.c-eg-columns{display:grid;grid-template-columns:1fr;gap:1.44rem}@media(min-width:993px){.c-eg-columns{grid-template-columns:1fr 1fr}}.c-eg-details{margin:1.2rem 0 1.44rem;border:1px solid #EAEAEA;border-radius:5px;padding:0 1rem}.c-eg-details>summary{cursor:pointer;padding:.6944444444rem 0;font-weight:700}.c-eg-button-row{margin:1.2rem 0 1.44rem}.c-eg-message{display:flex;flex-direction:column;align-items:flex-start;gap:.8333333333rem;margin-top:2.0736rem;padding:1rem 1.2rem;background:#f8f8f8;border:1px solid #EAEAEA;border-radius:5px}@media(min-width:769px){.c-eg-message{flex-direction:row;align-items:center;justify-content:space-between;gap:1.2rem}}.c-eg-message__text{margin:0}.c-eg-message__cta{flex-shrink:0} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/DeprecatedKeyNameTests.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/DeprecatedKeyNameTests.java new file mode 100644 index 000000000..9aab1e55c --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/DeprecatedKeyNameTests.java @@ -0,0 +1,196 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.fodid; + +import fiftyone.pipeline.did.DidClient; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests which environment variables the example takes its resource key and + * its licence key from. Each test starts {@link KeyProbe} in a Java process + * of its own with the variables it needs, so the example reads a real + * environment. + */ +public class DeprecatedKeyNameTests { + + private static final String RESOURCE_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; + private static final String OLD_RESOURCE_KEY_NAME = "RESOURCE_KEY"; + private static final String LICENCE_KEY_NAME = "_51DEGREES_LICENSE_KEY"; + private static final String OLD_LICENCE_KEY_NAME = "LICENSE_KEY"; + + private static final int PROBE_TIMEOUT_SECONDS = 120; + + @Test + public void oldNamesAlone_GiveTheirKeys() throws Exception { + List output = + probe(null, "old-name-key", null, "old-name-licence"); + + assertEquals("old-name-key", + value(output, KeyProbe.RESOURCE_PREFIX), output.toString()); + assertEquals("old-name-licence", + value(output, KeyProbe.LICENCE_PREFIX), output.toString()); + } + + @Test + public void oldNamesAlone_AreEachReportedAsDeprecatedOnce() + throws Exception { + List output = + probe(null, "old-name-key", null, "old-name-licence"); + + assertEquals(1, deprecations(output, OLD_RESOURCE_KEY_NAME, + RESOURCE_KEY_NAME), output.toString()); + assertEquals(1, deprecations(output, OLD_LICENCE_KEY_NAME, + LICENCE_KEY_NAME), output.toString()); + } + + @Test + public void bothNames_NewNamesWin() throws Exception { + List output = probe( + "new-name-key", "old-name-key", + "new-name-licence", "old-name-licence"); + + assertEquals("new-name-key", + value(output, KeyProbe.RESOURCE_PREFIX), output.toString()); + assertEquals("new-name-licence", + value(output, KeyProbe.LICENCE_PREFIX), output.toString()); + assertEquals(0, deprecations(output, OLD_RESOURCE_KEY_NAME, + RESOURCE_KEY_NAME), output.toString()); + assertEquals(0, deprecations(output, OLD_LICENCE_KEY_NAME, + LICENCE_KEY_NAME), output.toString()); + } + + @Test + public void noNames_GiveNoKeys() throws Exception { + List output = probe(null, null, null, null); + + assertEquals("null", + value(output, KeyProbe.RESOURCE_PREFIX), output.toString()); + assertEquals("null", + value(output, KeyProbe.LICENCE_PREFIX), output.toString()); + } + + /** + * Runs the probe with each variable set to the value given, or absent + * where the value is null, and returns the lines it printed. + */ + private static List probe( + String resourceKey, + String oldResourceKey, + String licenceKey, + String oldLicenceKey) throws Exception { + String java = Paths.get( + System.getProperty("java.home"), "bin", "java").toString(); + ProcessBuilder builder = + new ProcessBuilder(java, KeyProbe.class.getName()); + Map environment = builder.environment(); + environment.put("CLASSPATH", classPath()); + set(environment, RESOURCE_KEY_NAME, resourceKey); + set(environment, OLD_RESOURCE_KEY_NAME, oldResourceKey); + set(environment, LICENCE_KEY_NAME, licenceKey); + set(environment, OLD_LICENCE_KEY_NAME, oldLicenceKey); + builder.redirectErrorStream(true); + + Process process = builder.start(); + List output = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader( + process.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + output.add(line); + } + } + assertTrue( + process.waitFor(PROBE_TIMEOUT_SECONDS, TimeUnit.SECONDS), + "The probe did not finish."); + assertEquals(0, process.exitValue(), output.toString()); + return output; + } + + private static void set( + Map environment, String name, String value) { + if (value == null) { + environment.remove(name); + } else { + environment.put(name, value); + } + } + + /** + * The places the probe loads its classes from, being the classes of + * the example, its test classes, the 51Did package and the JSON + * library. + */ + private static String classPath() throws Exception { + return location(CreatorContextDemoServer.class) + File.pathSeparator + + location(KeyProbe.class) + File.pathSeparator + + location(DidClient.class) + File.pathSeparator + + location(JSONObject.class); + } + + private static String location(Class type) throws Exception { + return Paths.get(type.getProtectionDomain().getCodeSource() + .getLocation().toURI()).toString(); + } + + /** The text after the prefix on the one line that starts with it. */ + private static String value(List output, String prefix) { + String found = null; + for (String line : output) { + if (line.startsWith(prefix)) { + assertNull(found, output.toString()); + found = line.substring(prefix.length()); + } + } + return found; + } + + /** + * How many lines say the old name is deprecated and name the new one. + */ + private static int deprecations( + List output, String oldName, String newName) { + int count = 0; + for (String line : output) { + if (line.contains(" " + oldName + " is deprecated") + && line.contains(newName)) { + count++; + } + } + return count; + } +} diff --git a/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/KeyProbe.java b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/KeyProbe.java new file mode 100644 index 000000000..074504798 --- /dev/null +++ b/pipeline/pipeline.developer-examples/pipeline.developer-examples.fodid/src/test/java/pipeline/developerexamples/fodid/KeyProbe.java @@ -0,0 +1,48 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package pipeline.developerexamples.fodid; + +/** + * Prints the resource key and the licence key the example read from the + * environment of the process. {@link DeprecatedKeyNameTests} runs this in + * a Java process of its own, because the example reads both keys once, + * when its class loads, and a process cannot change its own environment + * variables. + */ +public final class KeyProbe { + + static final String RESOURCE_PREFIX = "RESOURCE "; + static final String LICENCE_PREFIX = "LICENCE "; + + private KeyProbe() { + } + + public static void main(String[] args) { + System.out.println( + RESOURCE_PREFIX + CreatorContextDemoServer.RESOURCE); + System.out.println( + LICENCE_PREFIX + CreatorContextDemoServer.LICENCE); + System.out.flush(); + System.exit(0); + } +} diff --git a/pipeline/pipeline.did/README.md b/pipeline/pipeline.did/README.md index f6bb2f1b5..2b5fdea00 100644 --- a/pipeline/pipeline.did/README.md +++ b/pipeline/pipeline.did/README.md @@ -154,9 +154,11 @@ status in the message, a payload shorter than its type's minimum throws Both string readers accept the standard alphabet the cloud issues (`+`, `/`, padded) and the URL-safe alphabet a page puts in a link (`-`, `_`, padding -optional), with or without whitespace around the value. `asBase64Url()` gives -the URL-safe form back, so an identifier can go into a URL without any -conversion by the caller. +optional), with or without white space around the value. A value with white +space inside it, or with characters from both alphabets, is refused as +`INVALID_BASE64`, because the issuer writes neither and either is a sign of a +damaged value. `asBase64Url()` gives the URL-safe form back, so an identifier +can go into a URL without any conversion by the caller. ### What the status means @@ -167,8 +169,8 @@ found it, and adds its own for the payload rules. | Status | Meaning | Layer | |---|---|---| | `PARSED` | A structurally valid 51Did. Says nothing about the signature. | | -| `MISSING_INPUT` | Null or empty input. | OWID | -| `INVALID_BASE64` | The string is not base64. | OWID | +| `MISSING_INPUT` | Null or empty input, or white space alone. | OWID | +| `INVALID_BASE64` | The string is not base64 in one alphabet. White space inside it, and characters from both alphabets, report this. | OWID, or 51Did for white space inside the text and mixed alphabets | | `UNSUPPORTED_VERSION` | The first byte names an envelope version the reader does not know. | OWID | | `UNEXPECTED_END` | The data stopped in the middle of an envelope field. | OWID | | `INVALID_DOMAIN_ENCODING` | The creator domain is unterminated or longer than a domain name can be. | OWID | @@ -355,7 +357,7 @@ import fiftyone.pipeline.did.RedeemResult; // The resource key is the page's and public by nature. The licence key is // server side only and is needed to redeem where the account holds licence // keys. The endpoint defaults to https://cloud.51degrees.com/api/v4/, or -// the FOD_CLOUD_API_URL environment variable where that is set. +// the _51DEGREES_CLOUD_ENDPOINT environment variable where that is set. DidClient client = new DidClient(resourceKey, licenceKey); ``` diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java index f0d2c4010..9c2d067d2 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidClient.java @@ -59,7 +59,9 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; /** * The client for everything a server does with a 51Did against the @@ -114,6 +116,17 @@ public final class DidClient { * The environment variable read for the API base when the constructor * is given none, the same one the cloud request engine honours. */ + public static final String CLOUD_ENDPOINT_VARIABLE = + "_51DEGREES_CLOUD_ENDPOINT"; + + /** + * The old name of the environment variable read for the API base, read + * only when {@link #CLOUD_ENDPOINT_VARIABLE} is not set. + * + * @deprecated use {@link #CLOUD_ENDPOINT_VARIABLE}, and set the + * environment variable it names in place of this one. + */ + @Deprecated public static final String ENDPOINT_VARIABLE = "FOD_CLOUD_API_URL"; /** @@ -136,6 +149,13 @@ public final class DidClient { private static final String USER_AGENT = "pipeline.did/" + version(); + /** + * Set once the use of {@link #ENDPOINT_VARIABLE} has been logged, so + * that a process creating many clients logs it once. + */ + private static final AtomicBoolean OLD_VARIABLE_LOGGED = + new AtomicBoolean(false); + /** * Longest encoded identifier this client will take from a caller. The * figure is arbitrary and deliberately generous, far above anything the @@ -180,7 +200,7 @@ public enum SignatureCheck { /** * A client for the public cloud, or the host named by - * {@value #ENDPOINT_VARIABLE}, with no licence key. + * {@value #CLOUD_ENDPOINT_VARIABLE}, with no licence key. * * @param resourceKey the page's resource key, public by nature */ @@ -190,7 +210,7 @@ public DidClient(String resourceKey) { /** * A client for the public cloud, or the host named by - * {@value #ENDPOINT_VARIABLE}, with a licence key. + * {@value #CLOUD_ENDPOINT_VARIABLE}, with a licence key. * * @param resourceKey the page's resource key, public by nature * @param licenceKey a licence key of the same account, server side @@ -207,9 +227,9 @@ public DidClient(String resourceKey, String licenceKey) { * @param resourceKey the page's resource key, public by nature * @param licenceKey a licence key of the same account, or null * @param endpoint the API base including {@code /api/v4/}, or null - * to read {@value #ENDPOINT_VARIABLE} and fall back - * to {@link #DEFAULT_ENDPOINT}. A trailing slash is - * added where missing. + * to read {@value #CLOUD_ENDPOINT_VARIABLE} and fall + * back to {@link #DEFAULT_ENDPOINT}. A trailing + * slash is added where missing. */ public DidClient(String resourceKey, String licenceKey, String endpoint) { this(builder(resourceKey).licenceKey(licenceKey).endpoint(endpoint)); @@ -268,8 +288,8 @@ public Builder licenceKey(String licenceKey) { /** * @param endpoint the API base including {@code /api/v4/}, or null - * to read {@value #ENDPOINT_VARIABLE} and fall back - * to {@link #DEFAULT_ENDPOINT} + * to read {@value #CLOUD_ENDPOINT_VARIABLE} and + * fall back to {@link #DEFAULT_ENDPOINT} * @return this builder */ public Builder endpoint(String endpoint) { @@ -940,7 +960,11 @@ public CompletableFuture redeem( } } - /** The typed result a redeem answer carries. */ + /** + * The typed result a redeem answer carries. The request sent the + * licence key, and the answer can name it, so the messages of the + * errors show the key masked. + */ private RedeemResult readRedeem(HttpTransport.Response response) { int status = response.getStatusCode(); JSONObject json = asObject(response.getBody()); @@ -949,21 +973,22 @@ private RedeemResult readRedeem(HttpTransport.Response response) { case 503: if (json == null) { throw new CompletionException( - httpError("Redemption", response)); + httpError("Redemption", response, licenceKey)); } return RedeemResult.parse(status, json, response.getBody()); case 400: if (json != null && json.has("errors")) { - throw new IllegalArgumentException(errorsText(json)); + throw new IllegalArgumentException( + maskLicence(errorsText(json), licenceKey)); } throw new CompletionException( - httpError("Redemption", response)); + httpError("Redemption", response, licenceKey)); case 404: throw new CompletionException(new DidNotSupportedException( endpoint, response.getBody())); default: throw new CompletionException( - httpError("Redemption", response)); + httpError("Redemption", response, licenceKey)); } } @@ -1045,13 +1070,54 @@ private static void ensureReadsAs51Did(String fodId) { private static DidHttpException httpError( String what, HttpTransport.Response response) { + return httpError(what, response, null); + } + + /** + * An error whose message summarises the body, with the licence key the + * request sent masked in it. + */ + private static DidHttpException httpError( + String what, HttpTransport.Response response, String licence) { String body = response.getBody(); - String summary = body.length() > 200 ? body.substring(0, 200) : body; + String masked = maskLicence(body, licence); + String summary = masked.length() > 200 ? + masked.substring(0, 200) : + masked; return new DidHttpException( what + " answered HTTP " + response.getStatusCode() + ": " + summary, response.getStatusCode(), body); } + /** + * The text with the licence key shown only by its last 4 characters, + * every character before them replaced by '*', or replaced in full when + * it has 4 characters or fewer. This is how every 51Degrees package + * writes a licence key in a message, and it matches + * fiftyone.pipeline.util.LicenseKeyMask, which this package cannot use + * because it depends on nothing in the Pipeline. + */ + private static String maskLicence(String text, String licence) { + if (text == null || licence == null || licence.isEmpty()) { + return text; + } + String result = text.replace(licence, maskKey(licence)); + String encoded = encode(licence); + if (encoded.equals(licence) == false) { + result = result.replace(encoded, maskKey(encoded)); + } + return result; + } + + private static String maskKey(String key) { + int hidden = key.length() > 4 ? key.length() - 4 : key.length(); + StringBuilder masked = new StringBuilder(key.length()); + for (int i = 0; i < hidden; i++) { + masked.append('*'); + } + return masked.append(key, hidden, key.length()).toString(); + } + /** The body as a JSON object, or null when it is not one. */ private static JSONObject asObject(String body) { try { @@ -1106,14 +1172,29 @@ static String encode(String value) { } /** - * The API base to use: the argument, else {@value #ENDPOINT_VARIABLE}, - * else {@link #DEFAULT_ENDPOINT}, ending in exactly one slash so every - * route can be appended directly. + * The API base to use: the argument, else + * {@value #CLOUD_ENDPOINT_VARIABLE}, else the deprecated + * {@value #ENDPOINT_VARIABLE}, else {@link #DEFAULT_ENDPOINT}, ending + * in exactly one slash so every route can be appended directly. The + * first use of the deprecated variable in the process is logged as a + * warning. */ static String resolveEndpoint(String endpoint) { String value = blankToNull(endpoint); + if (value == null) { + value = blankToNull(System.getenv(CLOUD_ENDPOINT_VARIABLE)); + } if (value == null) { value = blankToNull(System.getenv(ENDPOINT_VARIABLE)); + if (value != null + && OLD_VARIABLE_LOGGED.compareAndSet(false, true)) { + Logger.getLogger(DidClient.class.getName()).warning( + "The environment variable " + ENDPOINT_VARIABLE + + " is deprecated and " + CLOUD_ENDPOINT_VARIABLE + + " replaces it. Set " + CLOUD_ENDPOINT_VARIABLE + + " to the same value, as a later release may no " + + "longer read " + ENDPOINT_VARIABLE + "."); + } } if (value == null) { value = DEFAULT_ENDPOINT; diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java index 49d6d41cb..5a1747925 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/DidHttpException.java @@ -54,7 +54,13 @@ public int getStatusCode() { return statusCode; } - /** @return the body the service answered with, never null */ + /** + * The body the service answered with, as it was sent. It is not masked, + * and the answer to a request that sent a license key can hold that + * key, so a caller that logs the body or puts it in a message must + * mask any license key in it first. + * @return the body, never null + */ public String getBody() { return body; } diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java index 2ff930fac..e94ef24d2 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodId.java @@ -87,7 +87,9 @@ *

      * The cloud issues a 51Did in standard base64 with padding, and a page that * puts one in a link converts it to the URL-safe alphabet without padding. - * Both readers of a string accept either form. + * Both readers of a string accept either form, and ignore white space + * around it. They refuse a text with white space inside it, or with + * characters from both alphabets, because the issuer writes neither. *

      * Java's {@link Owid} is {@code final}, so this type composes an OWID * rather than inheriting from it: it holds the envelope and delegates @@ -195,15 +197,21 @@ private FodId( * Reads a 51Did from its base64 form without throwing, in either the * standard alphabet ({@code +} and {@code /}, as the cloud issues it) or * the URL-safe alphabet ({@code -} and {@code _}, as a page puts it in a - * link), with or without padding, and with or without whitespace around - * it. + * link), with or without padding, and with or without white space around + * it. White space inside the text, or characters from both alphabets, + * give {@link FodIdParseStatus#INVALID_BASE64}, and a text of white space + * alone gives {@link FodIdParseStatus#MISSING_INPUT}. White space is + * every character with the Unicode White_Space property. *

      * The value may be anything at all, because it is external data and * failing to be a 51Did is an ordinary outcome rather than an error. The * result reports whether the read worked, the 51Did only when it did, * and a named reason either way. An envelope failure carries the OWID * library's own status unchanged, and a payload failure is one of the - * 51Did payload statuses. See {@link FodIdParseStatus}. + * 51Did payload statuses. White space inside the text and characters + * from both alphabets are refused by this reader itself, before the OWID + * library sees the text, under the same + * {@link FodIdParseStatus#INVALID_BASE64}. See {@link FodIdParseStatus}. *

      * Success means the input is structurally a 51Did. The signature has * not been checked. @@ -216,7 +224,11 @@ public static FodIdParseResult tryFromBase64(String base64) { if (base64 == null) { return FodIdParseResult.failed(FodIdParseStatus.MISSING_INPUT); } - return read(Owid.parse(toStandardBase64(base64))); + String standard = toStandardBase64(base64); + if (standard == null) { + return FodIdParseResult.failed(FodIdParseStatus.INVALID_BASE64); + } + return read(Owid.parse(standard)); } /** @@ -331,8 +343,9 @@ private static FodIdParseResult read(Owid owid) { * @param base64 base64 of the full OWID envelope * @return the parsed 51Did * @throws NullPointerException if {@code base64} is null - * @throws OwidException if the string is not valid base64 or not a - * valid OWID envelope, with the + * @throws OwidException if the string is not valid base64 in one + * alphabet, has white space inside it, or + * is not a valid OWID envelope, with the * {@link FodIdParseStatus} in the message * @throws IllegalArgumentException if the payload is shorter than the * minimum for its identifier type, names a @@ -347,22 +360,57 @@ public static FodId fromBase64(String base64) throws OwidException { /** * Restores a base64 string that may use the URL-safe alphabet, with or * without padding, to the standard alphabet with padding, which is the - * only alphabet the envelope library reads. Leading and trailing - * whitespace is removed first, because a value read from a header, a + * only alphabet the envelope library reads. White space before and + * after the text is removed first, being every character with the + * Unicode White_Space property, because a value read from a header, a * file or a form field often carries a newline or a space around it and * neither belongs to the identifier. Then {@code -} becomes {@code +}, * {@code _} becomes {@code /}, and {@code ==} or {@code =} is appended * when the length modulo 4 is 2 or 3. That padding is worked out from - * the trimmed length, so whitespace cannot push the value into the wrong - * case. A value already in the standard padded form with no whitespace - * around it is returned unchanged. Nothing here decides whether the - * result is base64 at all, which is the envelope library's answer. + * the trimmed length, so white space cannot push the value into the + * wrong case. A value already in the standard padded form with no white + * space around it is returned unchanged. + *

      + * A text with white space inside it, or with a character from each + * alphabet such as both {@code +} and {@code -}, is refused, as the + * 51Did specification requires. The issuer writes neither, so either is + * a sign of a damaged value, such as a {@code +} a form decoder turned + * into a space or a value converted only in part. Any other character + * outside the alphabet is left for the envelope library to refuse. * * @param value the base64 text in either alphabet - * @return the same value in the standard alphabet with padding + * @return the same value in the standard alphabet with padding, or null + * when the text is refused */ static String toStandardBase64(String value) { - String standard = value.trim().replace('-', '+').replace('_', '/'); + int start = 0; + int end = value.length(); + while (start < end && isWhiteSpace(value.charAt(start))) { + start++; + } + while (end > start && isWhiteSpace(value.charAt(end - 1))) { + end--; + } + String text = value.substring(start, end); + // Whether the text holds a character that only the standard + // alphabet uses, and one that only the URL-safe alphabet uses. + boolean hasStandard = false; + boolean hasUrlSafe = false; + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + // The envelope library skips spaces and line breaks inside a + // text, so they are refused here, before it sees them. + if (isWhiteSpace(c)) { + return null; + } + hasStandard |= c == '+' || c == '/'; + hasUrlSafe |= c == '-' || c == '_'; + } + // Checked before the mapping below, which would hide a mix. + if (hasStandard && hasUrlSafe) { + return null; + } + String standard = text.replace('-', '+').replace('_', '/'); switch (standard.length() % 4) { case 2: return standard + "=="; @@ -373,6 +421,27 @@ static String toStandardBase64(String value) { } } + /** + * Whether the character has the Unicode White_Space property. The 25 + * code points are listed rather than taken from the runtime, because + * {@link String#trim()} and {@link Character#isWhitespace(char)} each + * differ from the property, and the Unicode version behind the regular + * expression classes changes with the Java release. + */ + private static boolean isWhiteSpace(char c) { + return (c >= '\t' && c <= '\r') + || c == ' ' + || c == 0x0085 + || c == 0x00A0 + || c == 0x1680 + || (c >= 0x2000 && c <= 0x200A) + || c == 0x2028 + || c == 0x2029 + || c == 0x202F + || c == 0x205F + || c == 0x3000; + } + /** * Parses a 51Did from the raw bytes of an OWID envelope, accepting the * same inputs as {@link #tryFromByteArray(byte[])}, and throws when the diff --git a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodIdParseStatus.java b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodIdParseStatus.java index cbb98bdcd..2ce6d06fc 100644 --- a/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodIdParseStatus.java +++ b/pipeline/pipeline.did/src/main/java/fiftyone/pipeline/did/FodIdParseStatus.java @@ -49,7 +49,10 @@ public enum FodIdParseStatus { */ PARSED, - /** Nothing was supplied to read, being a null or empty value. */ + /** + * Nothing was supplied to read, being a null or empty value, or a text + * of white space alone. + */ MISSING_INPUT, /** @@ -59,7 +62,11 @@ public enum FodIdParseStatus { */ INVALID_INPUT_TYPE, - /** The string is not valid base64, so there are no bytes to read. */ + /** + * The string is not valid base64 in one alphabet, so there are no bytes + * to read. White space inside the string, and characters from both + * alphabets, report this. + */ INVALID_BASE64, /** The first byte names an envelope version this reader does not know. */ diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java new file mode 100644 index 000000000..562bbddb2 --- /dev/null +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientEnvironmentTests.java @@ -0,0 +1,190 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.did; + +import org.json.JSONObject; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests which environment variable {@link DidClient} takes its endpoint + * from. Each test starts {@link EndpointProbe} in a Java process of its own + * with the variables it needs, so the client reads a real environment. + */ +public class DidClientEnvironmentTests { + + private static final String NEW_NAME = "_51DEGREES_CLOUD_ENDPOINT"; + private static final String OLD_NAME = "FOD_CLOUD_API_URL"; + + private static final String NEW_ENDPOINT = "https://new.example/api/v4/"; + private static final String OLD_ENDPOINT = "https://old.example/api/v4/"; + + private static final int PROBE_TIMEOUT_SECONDS = 120; + + @Test + public void oldNameAlone_GivesItsEndpoint() throws Exception { + List output = probe(null, OLD_ENDPOINT); + + assertEquals( + Arrays.asList(OLD_ENDPOINT, OLD_ENDPOINT), endpoints(output)); + } + + @Test + public void oldNameAlone_IsLoggedAsDeprecatedOnce() throws Exception { + List logged = logged(probe(null, OLD_ENDPOINT)); + + assertEquals(logged.toString(), 1, logged.size()); + String record = logged.get(0); + assertTrue(record, record.startsWith("WARNING ")); + assertTrue(record, record.contains(OLD_NAME)); + assertTrue(record, record.contains(NEW_NAME)); + assertTrue(record, record.contains("deprecated")); + } + + @Test + public void newNameBlank_OldNameIsRead() throws Exception { + List output = probe(" ", OLD_ENDPOINT); + + assertEquals( + Arrays.asList(OLD_ENDPOINT, OLD_ENDPOINT), endpoints(output)); + } + + @Test + public void bothNames_NewNameWins() throws Exception { + List output = probe(NEW_ENDPOINT, OLD_ENDPOINT); + + assertEquals( + Arrays.asList(NEW_ENDPOINT, NEW_ENDPOINT), endpoints(output)); + assertEquals(output.toString(), 0, logged(output).size()); + } + + @Test + public void neitherName_GivesTheDefault() throws Exception { + List output = probe(null, null); + + assertEquals( + Arrays.asList( + DidClient.DEFAULT_ENDPOINT, DidClient.DEFAULT_ENDPOINT), + endpoints(output)); + assertEquals(output.toString(), 0, logged(output).size()); + } + + @Test + @SuppressWarnings("deprecation") + public void oldConstant_KeepsItsNameAndValue() throws Exception { + assertEquals(OLD_NAME, DidClient.ENDPOINT_VARIABLE); + assertTrue(DidClient.class.getField("ENDPOINT_VARIABLE") + .isAnnotationPresent(Deprecated.class)); + assertEquals( + NEW_NAME, + DidClient.class.getField("CLOUD_ENDPOINT_VARIABLE").get(null)); + } + + /** + * Runs the probe with each variable set to the value given, or absent + * where the value is null, and returns the lines it printed. + */ + private static List probe(String newValue, String oldValue) + throws Exception { + String java = Paths.get( + System.getProperty("java.home"), "bin", "java").toString(); + ProcessBuilder builder = + new ProcessBuilder(java, EndpointProbe.class.getName()); + Map environment = builder.environment(); + environment.put("CLASSPATH", classPath()); + set(environment, NEW_NAME, newValue); + set(environment, OLD_NAME, oldValue); + builder.redirectErrorStream(true); + + Process process = builder.start(); + List output = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader( + process.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + output.add(line); + } + } + assertTrue( + "The probe did not finish.", + process.waitFor(PROBE_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + assertEquals(output.toString(), 0, process.exitValue()); + return output; + } + + private static void set( + Map environment, String name, String value) { + if (value == null) { + environment.remove(name); + } else { + environment.put(name, value); + } + } + + /** + * The places the probe loads its classes from, being the classes of + * this module, its test classes and the JSON library the client uses. + */ + private static String classPath() throws Exception { + return location(DidClient.class) + File.pathSeparator + + location(EndpointProbe.class) + File.pathSeparator + + location(JSONObject.class); + } + + private static String location(Class type) throws Exception { + return Paths.get(type.getProtectionDomain().getCodeSource() + .getLocation().toURI()).toString(); + } + + private static List endpoints(List output) { + return withPrefix(output, EndpointProbe.ENDPOINT_PREFIX); + } + + private static List logged(List output) { + return withPrefix(output, EndpointProbe.LOG_PREFIX); + } + + private static List withPrefix( + List output, String prefix) { + List result = new ArrayList<>(); + for (String line : output) { + if (line.startsWith(prefix)) { + result.add(line.substring(prefix.length())); + } + } + return result; + } +} diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java index 6ea863aa2..6794f4ee8 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientLiveTests.java @@ -30,6 +30,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URI; import java.nio.charset.StandardCharsets; @@ -41,15 +42,21 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** - * Tests against the live cloud, skipped unless {@code _51DEGREES_RESOURCE_KEY} - * (or the older {@code RESOURCE_KEY}) is set. {@code FOD_CLOUD_API_URL} - * points them at another host, and {@code _51DEGREES_LICENSE_KEY} (or - * {@code LICENSE_KEY}) supplies the licence key where the account holds - * one. Each test creates a 51Did through the cloud {@code json} endpoint, - * which is one use against the resource key, plus one for each cloud call - * it then makes. + * Tests against the live cloud service. They need a resource key, read + * from the names {@link LiveCloudKeys} lists, and they fail rather than + * skip when there is none, because a run that called nothing must not + * report success. {@code _51DEGREES_CLOUD_ENDPOINT} points them at another + * host, and a licence key is taken from the names {@link LiveCloudKeys} lists + * where the account holds one. Each test creates a 51Did through the + * cloud {@code json} endpoint, which is one use against the resource key, + * plus one for each cloud call it then makes. + *

      + * The resource key has to carry the fodid product. A key without it gets + * an answer from the service with no identifier in it, which these tests + * report as an entitlement failure naming the product. */ public class DidClientLiveTests { @@ -59,11 +66,13 @@ public class DidClientLiveTests { @Before public void init() { - resourceKey = env("_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY"); - Assume.assumeTrue( - "Set _51DEGREES_RESOURCE_KEY to run the live 51Did cloud tests.", - resourceKey != null); - licenceKey = env("_51DEGREES_LICENSE_KEY", "LICENSE_KEY"); + resourceKey = LiveCloudKeys.find(LiveCloudKeys.RESOURCE_KEY_NAMES); + if (resourceKey == null) { + fail(LiveCloudKeys.noResourceKey()); + } + // One lookup serves both, so the key the form body sends and the + // one hasLicenceKey() answers for can never disagree. + licenceKey = LiveCloudKeys.find(LiveCloudKeys.LICENCE_KEY_NAMES); client = new DidClient(resourceKey, licenceKey); } @@ -88,10 +97,17 @@ public void redeem_GarbageResult_IsUnreadable() throws Exception { .join(); } catch (CompletionException reported) { if (reported.getCause() instanceof DidNotSupportedException) { - Assume.assumeNoException( - "The host does not offer the creator context.", - reported.getCause()); - return; + // Reported rather than skipped. The redeem endpoint is + // part of the cloud service these tests exist to check, + // so a host that does not offer it is a real result and + // has to be seen. + throw new AssertionError( + "The cloud service at " + client.getEndpoint() + + " answered that it does not offer the creator " + + "context, so the redeem endpoint could not be " + + "called. Point _51DEGREES_CLOUD_ENDPOINT at a " + + "service that offers it, or leave it unset for the " + + "51Degrees cloud.", reported.getCause()); } throw reported; } @@ -173,13 +189,14 @@ public void verifyFullThenRedeemWithTheLicence_ReadsTheVerdict() /** * Every factor the service compares, read by name. The identifier is - * created for a stated address rather than for the connection that - * asked, so when the same connection then presents it the address the - * identifier carries is not the one being checked and the verdict is a - * mismatch. That is the one mismatch a test can arrange without a - * browser, and a mismatch is the only verdict that carries the - * breakdown, since a verified verdict matched everywhere and has - * nothing to diagnose. + * created by a licensed caller that states another address, so when + * this connection then presents it as a page the address it carries is + * not the one being checked and the verdict is a mismatch. That is the + * one mismatch a test can arrange without a browser. + *

      + * The address has to come from a licensed caller. The service takes + * the address of a caller that sends a resource key alone from the + * connection, whatever address that caller states. *

      * The names are written out rather than taken from the package, * because a list the package supplied would agree with itself whatever @@ -192,7 +209,7 @@ public void anIdentifierForAnotherAddress_ReportsEveryFactor() "Set _51DEGREES_LICENSE_KEY to redeem a sealed result.", client.hasLicenceKey()); - List identifiers = identifiersFor( + List identifiers = identifiersCreatedFor( "id.usage", "standard", CREATED_FOR); Assume.assumeTrue( "This resource key returned no marketing 51Did, so there was " @@ -273,9 +290,7 @@ private String verifyFull(String fodId) throws Exception { */ private FodId create() throws Exception { // id.usage is required. Without it the service takes the caller - // as not having asked for a 51Did at all and creates none, so this - // skipped on every run and the job reported success having tested - // nothing. + // as not having asked for a 51Did at all and creates none. String url = client.getEndpoint() + "json?resource=" + DidClient.encode(resourceKey) + "&id.usage=non-marketing&values=FODiD.IdProbGlobal"; @@ -291,9 +306,11 @@ private FodId create() throws Exception { String value = fodid == null ? null : fodid.optString("idprobglobal", null); - Assume.assumeTrue( - "The resource key does not return FODiD.IdProbGlobal.", - value != null); + if (value == null) { + fail(LiveCloudKeys.notEntitled( + "a request for FODiD.IdProbGlobal with " + + "id.usage=non-marketing")); + } return FodId.fromBase64(value); } @@ -314,15 +331,6 @@ private static String readAll(InputStream stream) throws IOException { } } - private static String env(String... names) { - for (String name : names) { - String value = System.getenv(name); - if (value != null && value.trim().isEmpty() == false) { - return value.trim(); - } - } - return null; - } /** * The versioned Model Terms for Marketing document a marketing 51Did is * created under. @@ -339,33 +347,53 @@ private static String env(String... names) { * Asks the {@code json} endpoint for a 51Did with the given query * parameter and returns every identifier it answered with. An empty * list means the resource key is not entitled to that usage, which the - * caller reports rather than fails. + * caller fails on, naming the product the key is missing. */ private List identifiersFor(String name, String value) throws Exception { - return identifiersFor(name, value, null); + String url = client.getEndpoint() + "json?resource=" + + DidClient.encode(resourceKey) + + "&" + name + "=" + DidClient.encode(value) + + "&values=FODiD.IdProbGlobal&values=FODiD.IdProbLic"; + HttpURLConnection connection = (HttpURLConnection) + URI.create(url).toURL().openConnection(); + connection.setRequestProperty("User-Agent", "pipeline.did live test"); + return readIdentifiers(connection); } /** - * As above, creating from the given address where one is given, so a - * test can create from one address and present the identifier from - * another. The service takes a supplied address only from a caller - * presenting a licence key, a server acting for a visitor, so the - * licence key goes with it. + * As above, asked by a licensed caller that states the address the + * identifier is created for, so a test can create for one address and + * present the identifier from another. Both keys travel in the form + * body, because a query string is written to access logs. */ - private List identifiersFor( + private List identifiersCreatedFor( String name, String value, String clientIp) throws Exception { - String url = client.getEndpoint() + "json?resource=" - + DidClient.encode(resourceKey) + String form = "resource=" + DidClient.encode(resourceKey) + + "&license=" + DidClient.encode(licenceKey) + "&" + name + "=" + DidClient.encode(value) - + (clientIp == null - ? "" - : "&client-ip=" + DidClient.encode(clientIp) - + "&license=" + DidClient.encode(licenceKey)) + + "&client-ip=" + DidClient.encode(clientIp) + "&values=FODiD.IdProbGlobal&values=FODiD.IdProbLic"; HttpURLConnection connection = (HttpURLConnection) - URI.create(url).toURL().openConnection(); + URI.create(client.getEndpoint() + "json").toURL() + .openConnection(); + connection.setRequestMethod("POST"); connection.setRequestProperty("User-Agent", "pipeline.did live test"); + connection.setRequestProperty("Content-Type", + "application/x-www-form-urlencoded; charset=utf-8"); + connection.setDoOutput(true); + try (OutputStream out = connection.getOutputStream()) { + out.write(form.getBytes(StandardCharsets.UTF_8)); + } + return readIdentifiers(connection); + } + + /** + * Every identifier in the answer to a request sent to the {@code json} + * endpoint. + */ + private static List readIdentifiers(HttpURLConnection connection) + throws Exception { int status = connection.getResponseCode(); String body = readAll(status >= 400 ? connection.getErrorStream() @@ -437,9 +465,7 @@ public void everyUsageReadsBackTheTermsAndFlagsTheServiceWrote() List identifiers = identifiersFor("id.usage", name); if (identifiers.isEmpty()) { - System.err.println("id.usage=" + name + ": no identifier " - + "returned, so this key is not entitled to that usage."); - continue; + fail(LiveCloudKeys.notEntitled("id.usage=" + name)); } for (int i = 0; i < identifiers.size(); i++) { assertAligned(name + "[" + i + "]", identifiers.get(i), @@ -450,14 +476,13 @@ public void everyUsageReadsBackTheTermsAndFlagsTheServiceWrote() } } - // Reported as a skip rather than written to stderr, because - // surefire captures test output and prints only the counts, so a - // run that proved nothing would otherwise look identical to one - // that proved everything. - Assume.assumeTrue( - "This resource key returned no marketing 51Did, so no terms " - + "address was read and this run did not prove it. Use a key " - + "entitled to the standard or personalized usage.", + // A run that read no marketing identifier read no terms address + // either, so it proved nothing and must not report success. + assertTrue( + "The live cloud service returned no marketing 51Did, so no " + + "terms address was read. The resource key has to carry the " + + "fodid product and be entitled to the standard or " + + "personalized usage.", checked > 0); } @@ -495,10 +520,8 @@ public void consentStringSetsTheUsageIsIndirectBit() throws Exception { // would prove the opposite of what it says. List identifiers = identifiersFor("tcstring", one[0]); if (identifiers.isEmpty()) { - System.err.println("consent string granting " + usage - + ": no identifier returned, so this key is not entitled " - + "to that marketing usage."); - continue; + fail(LiveCloudKeys.notEntitled( + "a consent string granting " + usage)); } for (int i = 0; i < identifiers.size(); i++) { // A consent string granting a marketing usage produces a @@ -511,10 +534,10 @@ public void consentStringSetsTheUsageIsIndirectBit() throws Exception { } // Same reasoning as the usage test above. - Assume.assumeTrue( - "This resource key returned no identifier for either consent " - + "string, so the usage is indirect bit was never read and " - + "this run did not prove it.", + assertTrue( + "The live cloud service returned no identifier for either " + + "consent string, so the usage-from-consent bit was never " + + "read and this run proved nothing.", proven > 0); } } diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java index 71969a6ac..501a2cabf 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/DidClientTests.java @@ -67,6 +67,9 @@ public class DidClientTests { private static final String ENDPOINT = "https://example.test/api/v4/"; + private static final String TEST_LICENCE = "TESTKEYNOTREALABCDEFGHIJ1234"; + private static final String TEST_LICENCE_MASKED = + "************************1234"; private static final Instant WEEK1 = Instant.parse("2026-08-03T00:00:00Z"); private static final Instant WEEK2 = WEEK1.plus(Duration.ofDays(7)); private static final Instant WEEK3 = WEEK2.plus(Duration.ofDays(7)); @@ -126,8 +129,13 @@ public void endpoint_IsNormalisedToOneTrailingSlash() { } @Test + @SuppressWarnings("deprecation") public void endpoint_DefaultsToTheCloudOrTheEnvironment() { - String fromEnvironment = System.getenv(DidClient.ENDPOINT_VARIABLE); + String fromEnvironment = + System.getenv(DidClient.CLOUD_ENDPOINT_VARIABLE); + if (fromEnvironment == null || fromEnvironment.trim().isEmpty()) { + fromEnvironment = System.getenv(DidClient.ENDPOINT_VARIABLE); + } String expected = fromEnvironment == null || fromEnvironment.trim().isEmpty() ? DidClient.DEFAULT_ENDPOINT @@ -1273,6 +1281,71 @@ public void redeem_TransportFailureFailsWithIoException() { client.redeem(validDid, "sealed", "abc")); } + /** + * The service answers a refused licence key with a body that names it. + * The error message shows the key only by its last 4 characters, with + * every character before them replaced, as every 51Degrees package + * writes a licence key in a message. + */ + @Test + public void redeem_RefusedLicenceIsMaskedInTheMessage() { + transport.queue(401, + "Invalid license key(s) supplied: " + TEST_LICENCE); + + DidHttpException error = failure(DidHttpException.class, + licensedClient().redeem(validDid, "sealed", "abc")); + + assertEquals(401, error.getStatusCode()); + assertFalse(error.getMessage(), + error.getMessage().contains(TEST_LICENCE)); + assertTrue(error.getMessage(), + error.getMessage().contains(TEST_LICENCE_MASKED)); + } + + @Test + public void redeem_LicenceInErrorsIsMaskedInTheMessage() { + transport.queue(400, "{\"errors\":[\"The license key " + + TEST_LICENCE + " does not include this product.\"]}"); + + IllegalArgumentException error = failure( + IllegalArgumentException.class, + licensedClient().redeem(validDid, "sealed", "abc")); + + assertFalse(error.getMessage(), + error.getMessage().contains(TEST_LICENCE)); + assertTrue(error.getMessage(), + error.getMessage().contains(TEST_LICENCE_MASKED)); + } + + /** + * A key of 4 characters or fewer is replaced in full, because its last + * 4 characters would be all of it. + */ + @Test + public void redeem_ShortLicenceIsMaskedInFull() { + DidClient shortLicence = DidClient.builder("resource") + .licenceKey("1234") + .endpoint(ENDPOINT) + .transport(transport) + .build(); + transport.queue(401, "Invalid license key(s) supplied: 1234"); + + DidHttpException error = failure(DidHttpException.class, + shortLicence.redeem(validDid, "sealed", "abc")); + + assertFalse(error.getMessage(), error.getMessage().contains("1234")); + assertTrue(error.getMessage(), error.getMessage().endsWith("****")); + } + + private DidClient licensedClient() { + return DidClient.builder("resource") + .licenceKey(TEST_LICENCE) + .endpoint(ENDPOINT) + .transport(transport) + .clock(clock) + .build(); + } + @Test public void redeem_WithoutLicenceKeyOmitsTheField() throws Exception { DidClient noLicence = DidClient.builder("resource") diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/EndpointProbe.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/EndpointProbe.java new file mode 100644 index 000000000..016ee2eeb --- /dev/null +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/EndpointProbe.java @@ -0,0 +1,71 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.did; + +import java.util.logging.Handler; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +/** + * Prints the endpoint two {@link DidClient} instances resolve from the + * environment of the process, and every record logged while they do. + * {@link DidClientEnvironmentTests} runs this in a Java process of its own, + * because a process cannot change its own environment variables. + */ +public final class EndpointProbe { + + static final String ENDPOINT_PREFIX = "ENDPOINT "; + static final String LOG_PREFIX = "LOG "; + + private EndpointProbe() { + } + + public static void main(String[] args) { + Logger root = Logger.getLogger(""); + for (Handler handler : root.getHandlers()) { + root.removeHandler(handler); + } + root.addHandler(new Handler() { + @Override + public void publish(LogRecord record) { + System.out.println(LOG_PREFIX + record.getLevel().getName() + + " " + record.getMessage()); + } + + @Override + public void flush() { + } + + @Override + public void close() { + } + }); + + for (int i = 0; i < 2; i++) { + System.out.println( + ENDPOINT_PREFIX + new DidClient("resource").getEndpoint()); + } + System.out.flush(); + System.exit(0); + } +} diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdTests.java index adf06d49b..f30d70811 100644 --- a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdTests.java +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/FodIdTests.java @@ -46,6 +46,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -456,7 +457,7 @@ public void roundtrip_ThroughBytesConstructor_PreservesAllFields() @Test public void fromBase64_AcceptsStandardUrlSafeAndUnpadded() throws Exception { - String standard = factory.signedOwidBase64(canonicalPayload()); + String standard = standardWithBothSpecials(); String urlSafe = standard.replace('+', '-').replace('/', '_'); String unpadded = urlSafe.replace("=", ""); // The envelope is 124 bytes, so the standard form always ends in @@ -511,6 +512,111 @@ public void toStandardBase64_RestoresAlphabetAndPadding() { assertEquals("+/8=", FodId.toStandardBase64(" -_8 ")); assertEquals("+/==", FodId.toStandardBase64("-_\r\n")); assertEquals("abcd", FodId.toStandardBase64(" abcd ")); + assertEquals("YQ==", FodId.toStandardBase64(" YQ\n")); + assertEquals("+/==", FodId.toStandardBase64("\u3000-_\u3000")); + } + + // ----- White space and the two alphabets ----- + + @Test + public void toStandardBase64_WhiteSpaceInsideOrBothAlphabets_GivesNull() { + String[] refused = { + "YW Jj", "YW\tJj", "YW\r\nJj", "YW\u00A0Jj", "YW\u3000Jj", + "+_", "-/", "+-", "/_" + }; + for (String text : refused) { + assertNull(show(text), FodId.toStandardBase64(text)); + } + } + + @Test + public void toStandardBase64_EachWhiteSpace_TrimmedAroundRefusedInside() { + assertEquals(25, WHITE_SPACE.length()); + for (char space : WHITE_SPACE.toCharArray()) { + String shown = show(String.valueOf(space)); + assertEquals(shown, + "YQ==", FodId.toStandardBase64(space + "YQ" + space)); + assertNull(shown, FodId.toStandardBase64("YW" + space + "Jj")); + } + } + + @Test + public void tryFromBase64_WhiteSpaceAroundTheText_Parses() throws Exception { + String standard = standardWithBothSpecials(); + String[][] around = { + { " ", "\n" }, + { "\t", "\r\n" }, + { "\u00A0", "\u00A0" }, + { "\u3000", "\u3000" }, + { WHITE_SPACE, WHITE_SPACE } + }; + for (String text : forms(standard)) { + for (String[] pair : around) { + assertParsesAs(standard, pair[0] + text + pair[1]); + } + for (char space : WHITE_SPACE.toCharArray()) { + assertParsesAs(standard, space + text + space); + } + } + } + + @Test + public void tryFromBase64_WhiteSpaceInsideTheText_InvalidBase64() + throws Exception { + String[] spaces = { " ", " ", " ", "\t", "\r\n", "\u00A0", "\u3000" }; + int[] counts = { 1, 2, 3, 4, 2, 4, 1 }; + for (String text : forms(standardWithBothSpecials())) { + for (int i = 0; i < spaces.length; i++) { + assertRefusedBeforeDecoding( + spaced(text, spaces[i], counts[i])); + } + for (char space : WHITE_SPACE.toCharArray()) { + assertRefusedBeforeDecoding( + spaced(text, String.valueOf(space), 1)); + } + } + } + + @Test + public void tryFromBase64_BothAlphabets_InvalidBase64() throws Exception { + String[] forms = forms(standardWithBothSpecials()); + // The standard text with one + or / changed to its URL-safe partner. + assertRefusedBeforeDecoding(replaceFirst(forms[0], '+', '-')); + assertRefusedBeforeDecoding(replaceFirst(forms[0], '/', '_')); + // Both URL-safe texts with one - or _ changed to its standard + // partner. + for (String urlSafe : new String[] { forms[1], forms[2] }) { + assertRefusedBeforeDecoding(replaceFirst(urlSafe, '-', '+')); + assertRefusedBeforeDecoding(replaceFirst(urlSafe, '_', '/')); + } + } + + @Test + public void tryFromBase64_OnlyWhiteSpace_MissingInput() { + String[] texts = { " ", "\r\n", "\u00A0", "\u3000", WHITE_SPACE }; + for (String text : texts) { + FodIdParseResult read = FodId.tryFromBase64(text); + assertFalse(show(text), read.isSuccess()); + assertEquals(show(text), + FodIdParseStatus.MISSING_INPUT, read.getStatus()); + } + } + + @Test + public void tryFromBase64_OtherCharacterAroundTheText_InvalidBase64() + throws Exception { + // Characters without the Unicode White_Space property that are + // easy to take for it, being controls String.trim removes, U+180E, + // which had the property before Unicode 6.3, and two zero width + // characters. + char[] others = { + '\u0000', '\u001C', '\u001F', '\u180E', '\u200B', '\uFEFF' + }; + for (String text : forms(standardWithBothSpecials())) { + for (char other : others) { + assertReadersRefuse(other + text + other); + } + } } @Test @@ -569,4 +675,108 @@ public void factory_SignedAtChosenDate_VerifiesWithItsKey() throws Exception { assertTrue(fodId.verify(factory.publicPem)); assertFalse(fodId.verify(Crypto.generate().publicKeyPem())); } + + // ----- Helpers ----- + + /** + * Every character with the Unicode White_Space property, as listed in + * the Unicode Character Database file PropList.txt. + */ + private static final String WHITE_SPACE = + "\t\n\u000B\f\r \u0085\u00A0\u1680" + + "\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009" + + "\u200A\u2028\u2029\u202F\u205F\u3000"; + + /** + * A signed envelope whose standard base64 holds both characters that + * differ between the alphabets, so every conversion is exercised. The + * signature is random, so envelopes are signed until one qualifies, + * which usually takes a few tries. + */ + private String standardWithBothSpecials() throws OwidException { + for (int attempt = 0; attempt < 100; attempt++) { + String base64 = factory.signedOwidBase64(canonicalPayload()); + if (base64.indexOf('+') >= 0 && base64.indexOf('/') >= 0) { + return base64; + } + } + throw new IllegalStateException( + "No signed envelope held both + and / in 100 tries."); + } + + /** + * The standard text, and the URL-safe text with and without its + * padding. + */ + private static String[] forms(String standard) { + String urlSafe = standard.replace('+', '-').replace('/', '_'); + return new String[] { standard, urlSafe, urlSafe.replace("=", "") }; + } + + /** + * The text with the white space put in at a number of points spread + * evenly through it. + */ + private static String spaced(String text, String space, int count) { + StringBuilder spaced = new StringBuilder(text); + for (int point = count; point >= 1; point--) { + spaced.insert(text.length() * point / (count + 1), space); + } + return spaced.toString(); + } + + /** The text with its first {@code from} changed to {@code to}. */ + private static String replaceFirst(String text, char from, char to) { + int index = text.indexOf(from); + assertTrue(show(text) + " holds no " + from, index >= 0); + return text.substring(0, index) + to + text.substring(index + 1); + } + + /** + * The text with every character outside printable ASCII written as its + * code, so a failure message shows where it is. + */ + private static String show(String text) { + StringBuilder shown = new StringBuilder(); + for (char c : text.toCharArray()) { + if (c > ' ' && c < 0x7F) { + shown.append(c); + } else { + shown.append(String.format("\\u%04X", (int) c)); + } + } + return shown.toString(); + } + + private static void assertParsesAs(String standard, String text) + throws OwidException { + FodIdParseResult read = FodId.tryFromBase64(text); + assertTrue(read.getStatus() + " for " + show(text), read.isSuccess()); + assertEquals(show(text), standard, read.getValue().asBase64()); + assertEquals(show(text), standard, FodId.fromBase64(text).asBase64()); + } + + /** + * Checks that both readers refuse the text as invalid base64, the one + * that throws naming the status. + */ + private static void assertReadersRefuse(String text) { + FodIdParseResult read = FodId.tryFromBase64(text); + assertFalse("Parsed " + show(text), read.isSuccess()); + assertNull(show(text), read.getValue()); + assertEquals(show(text), + FodIdParseStatus.INVALID_BASE64, read.getStatus()); + OwidException thrown = assertThrows(show(text), + OwidException.class, () -> FodId.fromBase64(text)); + assertTrue(thrown.getMessage().contains("INVALID_BASE64")); + } + + /** + * Checks that both readers refuse the text, and that the step they + * share refuses it before the envelope library decodes anything. + */ + private static void assertRefusedBeforeDecoding(String text) { + assertReadersRefuse(text); + assertNull(show(text), FodId.toStandardBase64(text)); + } } diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java new file mode 100644 index 000000000..88b51e27e --- /dev/null +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeys.java @@ -0,0 +1,163 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.did; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * Where the tests that call the live 51Degrees cloud service read their + * resource key and licence key from, and what they say when there is + * none. + *

      + * Every name is tried as an environment variable first and then as a + * system property, so a developer can export one and a build can pass one + * on the command line with {@code -D}. There is no fallback to a skip, + * because a cloud test that quietly passes with no key reports success + * having called nothing. + */ +final class LiveCloudKeys { + + /** + * The resource key names, in the order they are tried. + *

      + * {@code _51DEGREES_RESOURCE_KEY} is first because the workflow that + * runs these tests once for every key the organisation holds sets + * that single name to each key in turn, and a name later in this list + * winning would make every one of those runs use the same key. + */ + static final String[] RESOURCE_KEY_NAMES = { + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "TestResourceKey", + "SUPER_RESOURCE_KEY", + }; + + /** + * The licence key names, in the order they are tried. A licence key + * is optional, because an account can hold its products on the + * resource key alone. + */ + static final String[] LICENCE_KEY_NAMES = { + "_51DEGREES_LICENSE_KEY", + "_51DEGREES_LICENSE_KEY_BESPOKE", + "_51DEGREES_LICENSE_KEY_CLOUDV5BESPOKE", + "TestLicenseKey", + }; + + private LiveCloudKeys() { + } + + /** + * The names that have a deprecated old name, each followed by it. The + * old name is read only when its new name is not set. + */ + private static final String[][] DEPRECATED_NAMES = { + {"_51DEGREES_RESOURCE_KEY", "RESOURCE_KEY"}, + {"_51DEGREES_LICENSE_KEY", "LICENSE_KEY"}, + }; + + /** The old names whose use has been reported, each reported once. */ + private static final Set REPORTED_OLD_NAMES = + Collections.synchronizedSet(new HashSet()); + + /** + * The first value set under any of the given names, or null when none + * of them is set anywhere. A name with a deprecated old name falls + * back to it, saying so on standard error the first time. + */ + static String find(String[] names) { + for (String name : names) { + String value = read(name); + if (value == null) { + String oldName = oldNameOf(name); + if (oldName != null) { + value = read(oldName); + if (value != null && REPORTED_OLD_NAMES.add(oldName)) { + System.err.println( + "The environment variable " + oldName + + " is deprecated and " + name + + " replaces it. Set " + name + + " to the same value, as a later release may no longer " + + "read " + oldName + "."); + } + } + } + if (value != null) { + return value; + } + } + return null; + } + + /** + * The trimmed value of the environment variable of this name, or of + * the system property where that is not set or is blank, or null when + * neither holds a value. + */ + private static String read(String name) { + String value = System.getenv(name); + if (value == null || value.trim().isEmpty()) { + value = System.getProperty(name); + } + return value == null || value.trim().isEmpty() ? null : value.trim(); + } + + private static String oldNameOf(String name) { + for (String[] pair : DEPRECATED_NAMES) { + if (pair[0].equals(name)) { + return pair[1]; + } + } + return null; + } + + /** + * What to fail with when no resource key is set, naming every place + * that was looked and what the key has to be entitled to. + */ + static String noResourceKey() { + return "These tests call the live 51Degrees cloud service, so they " + + "need a resource key and there is nothing to fall back on. " + + "None of " + String.join(", ", RESOURCE_KEY_NAMES) + " is set, " + + "as an environment variable or as a system property, tried in " + + "that order. The key has to carry the fodid product, because " + + "these tests ask the cloud for FODiD.IdProbGlobal and " + + "FODiD.IdProbLic. Create a key at " + + "https://configure.51degrees.com."; + } + + /** + * What to fail with when the cloud answered but returned no 51Did, + * which is the resource key not carrying the product rather than the + * service being broken. + */ + static String notEntitled(String detail) { + return "The live 51Degrees cloud service answered, but returned no " + + "51Did for " + detail + ". That is the resource key not " + + "carrying the fodid product rather than a fault in the " + + "service or in this package. Use a key entitled to fodid, " + + "from https://configure.51degrees.com."; + } +} diff --git a/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java new file mode 100644 index 000000000..bf0f5a370 --- /dev/null +++ b/pipeline/pipeline.did/src/test/java/fiftyone/pipeline/did/LiveCloudKeysTests.java @@ -0,0 +1,105 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.did; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assume.assumeTrue; + +/** + * Tests which names {@link LiveCloudKeys} reads a key from. System + * properties stand in for the environment variables, because the lookup + * reads either and a test cannot set an environment variable. + */ +public class LiveCloudKeysTests { + + private static final String RESOURCE_KEY_NAME = "_51DEGREES_RESOURCE_KEY"; + private static final String OLD_RESOURCE_KEY_NAME = "RESOURCE_KEY"; + private static final String LICENCE_KEY_NAME = "_51DEGREES_LICENSE_KEY"; + private static final String OLD_LICENCE_KEY_NAME = "LICENSE_KEY"; + private static final String OTHER_NAME = "TestResourceKey"; + + private static final String[] NAMES = { + RESOURCE_KEY_NAME, OLD_RESOURCE_KEY_NAME, + LICENCE_KEY_NAME, OLD_LICENCE_KEY_NAME, OTHER_NAME, + }; + + private final String[] saved = new String[NAMES.length]; + + @Before + public void clearNames() { + for (int i = 0; i < NAMES.length; i++) { + // An environment variable would answer before the system + // properties set here, so the test cannot run. + assumeTrue(System.getenv(NAMES[i]) == null); + saved[i] = System.clearProperty(NAMES[i]); + } + } + + @After + public void restoreNames() { + for (int i = 0; i < NAMES.length; i++) { + if (saved[i] == null) { + System.clearProperty(NAMES[i]); + } else { + System.setProperty(NAMES[i], saved[i]); + } + } + } + + @Test + public void resourceKey_OldNameWhenNewIsNotSet() { + System.setProperty(OLD_RESOURCE_KEY_NAME, "old-name-key"); + + assertEquals("old-name-key", + LiveCloudKeys.find(new String[] {RESOURCE_KEY_NAME})); + } + + @Test + public void licenceKey_OldNameWhenNewIsNotSet() { + System.setProperty(OLD_LICENCE_KEY_NAME, "old-name-licence"); + + assertEquals("old-name-licence", + LiveCloudKeys.find(new String[] {LICENCE_KEY_NAME})); + } + + @Test + public void newNameWinsOverOld() { + System.setProperty(RESOURCE_KEY_NAME, "new-name-key"); + System.setProperty(OLD_RESOURCE_KEY_NAME, "old-name-key"); + + assertEquals("new-name-key", + LiveCloudKeys.find(new String[] {RESOURCE_KEY_NAME})); + } + + @Test + public void oldNameIsReadOnlyForItsNewName() { + System.setProperty(OLD_RESOURCE_KEY_NAME, "old-name-key"); + + assertNull(LiveCloudKeys.find(new String[] {OTHER_NAME})); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java index 3df2733df..3bfd42e8f 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsage.java @@ -34,18 +34,25 @@ import static fiftyone.pipeline.core.Constants.*; /** - * This filter is used by the {@link ShareUsageElement}. + * This filter is used by the {@link ShareUsageElement} to decide which of the + * values {@link ShareUsageEvidence#find} finds are shared. It is not the + * element's accepted Evidence, which is empty. * It will include anything that is: - * 1) An HTTP header that is not blocked by the constructor parameter. + * 1) An HTTP header that is not blocked by the constructor parameter, other + * than Cookie. * 2) A cookie that starts with {@link Constants#FIFTYONE_COOKIE_PREFIX} or is * the session cookie (if configured in the constructor). - * 3) An query string parameters that have been configured to be shared - * using the constructor parameter. - * 4) Not a header, cookie or query string parameter. - * - * As this filter is generally inclusive, it will often cause far more - * evidence to be passed into a pipeline than the engine-specific - * filters, which tend to be based on a white list + * 3) A session value that starts with {@link Constants#FIFTYONE_COOKIE_PREFIX}. + * 4) A query string parameter that starts with + * {@link Constants#FIFTYONE_COOKIE_PREFIX} or has been configured to be + * shared using the constructor parameter. + * 5) Not a header, cookie, session value or query string parameter. + * When all Evidence is shared, everything is included. When all query string + * parameters are shared, every query string parameter is included. + * The prefix of a key is everything before its first '.', and the field + * everything after it. An email address, the salt for hashed email + * identifiers, the query string, the path and the Cookie header are never + * included, whatever the configuration. */ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { @@ -61,6 +68,17 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { */ private final String sessionCookieName; + /** + * If true then every key is included, apart from those that are never + * shared. + */ + private final boolean shareAllEvidence; + + /** + * If true then every query string parameter is included. + */ + private final boolean shareAllQueryStringParameters; + /** * The content of HTTP headers in this array will not be included in the * request information sent to 51degrees. Any header names added here are @@ -75,6 +93,23 @@ public class EvidenceKeyFilterShareUsage implements EvidenceKeyFilter { */ private final Set includedQueryStringParams = new HashSet<>(); + /** + * Evidence keys that are never shared, matched without regard to case as + * the whole key or after a '.'. 'id.email' and 'id.salt' are a raw email + * address and the salt used to make hashed email identifiers. + * 'header.cookie' is the Cookie header, which carries every cookie. + * 'request-query' is the whole query string, which would undo the rules + * for its parameters. 'request-path' would make each page a visitor opens + * look like a new session to the tracker. + */ + private static final String[] NEVER_SHARED_EVIDENCE_KEYS = { + "id.email", + "id.salt", + "header.cookie", + "request-query", + "request-path" + }; + /** * Constructor. * @param blockedHttpHeaders a list of the names of the HTTP headers that @@ -92,9 +127,47 @@ public EvidenceKeyFilterShareUsage( List includedQueryStringParams, boolean includeSession, String sessionCookieName) { + this( + blockedHttpHeaders, + includedQueryStringParams, + includeSession, + sessionCookieName, + false, + false); + } + + /** + * Constructor. + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParams a list of the names of query string + * parameters that share usage should send + * to 51Degrees + * @param includeSession if true then the session cookie will be included in + * the filter + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param shareAllEvidence if true then all Evidence is included, apart from + * the keys that are never shared + * @param shareAllQueryStringParameters if true then all query string + * parameters are included + */ + public EvidenceKeyFilterShareUsage( + List blockedHttpHeaders, + List includedQueryStringParams, + boolean includeSession, + String sessionCookieName, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { + this.shareAllEvidence = shareAllEvidence; + this.shareAllQueryStringParameters = shareAllQueryStringParameters; + // The Cookie header carries every cookie, including the ones the + // cookie rule leaves out. this.blockedHttpHeaders.add("cookies"); + this.blockedHttpHeaders.add("cookie"); this.includeSession = includeSession; - this.sessionCookieName = sessionCookieName; + this.sessionCookieName = sessionCookieName == null ? + null : sessionCookieName.toLowerCase(); for (String header : blockedHttpHeaders) { String lowerHeader = header.toLowerCase(); if (!this.blockedHttpHeaders.contains(lowerHeader)) { @@ -111,30 +184,52 @@ public EvidenceKeyFilterShareUsage( @Override public boolean include(String key) { + // The Usage Sharing Element reads the whole web request itself, and + // the request is never a value to share. + if (ShareUsageEvidence.HTTP_REQUEST_KEY.equalsIgnoreCase(key)) { + return false; + } + String lowerKey = key.toLowerCase(); + for (String neverShared : NEVER_SHARED_EVIDENCE_KEYS) { + if (lowerKey.equals(neverShared) || + lowerKey.endsWith(EVIDENCE_SEPERATOR + neverShared)) { + return false; + } + } + + if (shareAllEvidence) { + return true; + } + boolean result; - String[] parts = key.toLowerCase().split("\\."); - if (parts.length == 2) { - switch (parts[0]) { + int separator = lowerKey.indexOf(EVIDENCE_SEPERATOR); + if (separator > 0) { + String prefix = lowerKey.substring(0, separator); + String field = lowerKey.substring(separator + 1); + switch (prefix) { case EVIDENCE_HTTPHEADER_PREFIX: // Add the header to the list if the header name does not // appear in the list of blocked headers. - result = blockedHttpHeaders.contains(parts[1]) == false; + result = blockedHttpHeaders.contains(field) == false; break; case EVIDENCE_COOKIE_PREFIX: // Only add cookies that start with the 51Degrees cookie // prefix. - result = parts[1].startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || - (includeSession && parts[1].equals(sessionCookieName)); + result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || + (includeSession && field.equals(sessionCookieName)); break; case EVIDENCE_SESSION_PREFIX: // Only add session values that start with the 51Degrees cookie // prefix. - result = parts[1].startsWith(Constants.FIFTYONE_COOKIE_PREFIX); + result = field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX); break; case EVIDENCE_QUERY_PREFIX: - // Only include query string parameters that have been - // specified in the constructor. - result = includedQueryStringParams.contains(parts[1]); + // Only include query string parameters that start with the + // 51Degrees prefix or have been specified in the + // constructor, unless all of them are shared. + result = shareAllQueryStringParameters || + field.startsWith(Constants.FIFTYONE_COOKIE_PREFIX) || + includedQueryStringParams.contains(field); break; default: // Add anything that is not a cookie or a header. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java index 9240b3267..61c2c348c 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTracker.java @@ -57,6 +57,38 @@ public EvidenceKeyFilterShareUsageTracker( sessionCookieName); } + /** + * Constructor. + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParams a list of the names of query string + * parameters that share usage should send + * to 51Degrees + * @param includeSession if true then the session cookie will be included in + * the filter + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param shareAllEvidence if true then all Evidence is included, apart from + * the keys that are never shared + * @param shareAllQueryStringParameters if true then all query string + * parameters are included + */ + public EvidenceKeyFilterShareUsageTracker( + List blockedHttpHeaders, + List includedQueryStringParams, + boolean includeSession, + String sessionCookieName, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { + super( + blockedHttpHeaders, + includedQueryStringParams, + includeSession, + sessionCookieName, + shareAllEvidence, + shareAllQueryStringParameters); + } + @Override public boolean include(String key) { if(key.equalsIgnoreCase(EVIDENCE_SESSIONID) || diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SequenceData.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SequenceData.java new file mode 100644 index 000000000..e6f18024d --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/SequenceData.java @@ -0,0 +1,86 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.data; + +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; +import org.slf4j.Logger; + +/** + * Data containing the result of {@link SequenceElement}. + * @see Specification + */ +public class SequenceData extends ElementDataBase { + + /** + * Name of the property holding the session id. + */ + public static final String SESSION_ID_PROPERTY_NAME = "session-id"; + + /** + * Name of the property holding the sequence. + */ + public static final String SEQUENCE_PROPERTY_NAME = "sequence"; + + /** + * Construct a new instance. + * @param logger the logger to use + * @param flowData the flow data the data belongs to + */ + public SequenceData(Logger logger, FlowData flowData) { + super(logger, flowData); + } + + /** + * Get the identifier of the session. + * @return the session id + */ + public String getSessionId() { + return getAs(SESSION_ID_PROPERTY_NAME, String.class); + } + + /** + * Set the identifier of the session. + * @param sessionId the session id + */ + public void setSessionId(String sessionId) { + put(SESSION_ID_PROPERTY_NAME, sessionId); + } + + /** + * Get the number of consecutive callbacks made in the session. + * @return the sequence + */ + public Integer getSequence() { + return getAs(SEQUENCE_PROPERTY_NAME, Integer.class); + } + + /** + * Set the number of consecutive callbacks made in the session. + * @param sequence the sequence + */ + public void setSequence(int sequence) { + put(SEQUENCE_PROPERTY_NAME, sequence); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java new file mode 100644 index 000000000..d6e7d22d4 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidence.java @@ -0,0 +1,84 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.data; + +import fiftyone.pipeline.core.data.Evidence; + +import java.util.Map; +import java.util.TreeMap; + +/** + * The values usage sharing finds for a request. The usage sharing element + * accepts no Evidence, so that cache keys, the ETag and the Vary header never + * depend on it, and finds what to share here instead. The sharing rules of + * {@link EvidenceKeyFilterShareUsage} then decide which of these are shared. + * @see Specification + */ +public final class ShareUsageEvidence { + + /** + * The Evidence key under which the web integration adds the web request, + * as a read-only map keyed the way Evidence is keyed, such as + * header.user-agent or cookie.51D_Profile. No element accepts it. + */ + public static final String HTTP_REQUEST_KEY = "server.http-request"; + + private ShareUsageEvidence() { + } + + /** + * Every Evidence value, and every value of the web request when the + * Evidence holds it, keyed without regard to case. A key in both keeps + * the Evidence value. The web request itself is not one of the values. + * Without a web request the values found are the Evidence's own + * read-only map, as Evidence is already keyed without regard to case and + * holds no null value, so nothing is copied. + * @param evidence the Evidence of the request + * @return the values found, which the caller must not change + */ + public static Map find(Evidence evidence) { + Map values = evidence.asKeyMap(); + if (values.containsKey(HTTP_REQUEST_KEY) == false) { + return values; + } + Map found = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + Object request = null; + for (Map.Entry entry : values.entrySet()) { + if (HTTP_REQUEST_KEY.equalsIgnoreCase(entry.getKey())) { + request = entry.getValue(); + } else if (entry.getValue() != null) { + found.put(entry.getKey(), entry.getValue()); + } + } + if (request instanceof Map) { + for (Map.Entry entry : ((Map) request).entrySet()) { + if (entry.getKey() instanceof String && + entry.getValue() != null) { + found.putIfAbsent((String) entry.getKey(), entry.getValue()); + } + } + } + return found; + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java index f53a7bf29..48e1e1b88 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/Constants.java @@ -28,10 +28,10 @@ public class Constants { /** - * The maximum length of a piece of evidence's value which can be added to - * the usage data being sent. + * The maximum length, in characters, of a piece of evidence's value which + * can be added to the usage data being sent. */ - public static final int SHARE_USAGE_MAX_EVIDENCE_LENGTH = 512; + public static final int SHARE_USAGE_MAX_EVIDENCE_LENGTH = 10000; /** * Session id evidence suffix. This is prefixed by the query constant. @@ -93,6 +93,13 @@ public class Constants { * Default lost data count for share usage before it wraps */ static final int LOST_DATA_RESET_DEFAULT = 5000000; + + /** + * While the queue is full, a message is logged for the first entry + * discarded and then once for every this many more, never one for each + * entry. + */ + static final int QUEUE_FULL_LOG_EVERY = 1000; /** * The default URL to send usage data to. */ @@ -104,4 +111,43 @@ public class Constants { */ static final String EVIDENCE_HTTPHEADER_COOKIE_SUFFIX = "cookie"; + /** + * The default number of failed sends within the failures window that + * pauses sending. + */ + public static final int SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT = 10; + + /** + * The fewest failed sends that can be set to pause sending. + */ + public static final int SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN = 1; + + /** + * The most failed sends that can be set to pause sending. + */ + public static final int SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX = 100; + + /** + * The default time in seconds over which failed sends are counted. + */ + public static final int SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT = 100; + + /** + * The default length in seconds of the first pause after failed sends. + */ + public static final double + SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT = 2.0; + + /** + * The default longest pause in seconds. + */ + public static final double + SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT = 300.0; + + /** + * The default multiplier applied to the pause each time the send tried + * when a pause ends fails. + */ + public static final double + SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT = 2.0; } diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneOnPremiseAspectEngineBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneOnPremiseAspectEngineBuilderBase.java index bcd634359..b6ea858ec 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneOnPremiseAspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneOnPremiseAspectEngineBuilderBase.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.engines.fiftyone.flowelements; +import fiftyone.pipeline.engines.configuration.DataFileConfiguration; import fiftyone.pipeline.engines.data.AspectEngineDataFile; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFile; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFileDefault; @@ -80,6 +81,17 @@ public FiftyOneOnPremiseAspectEngineBuilderBase( setDataUpdateUrlFormatter(new FiftyOneUrlFormatter()); } + /** + * The Distributor serves nothing without a license key, so a key is + * required while the data update URL is the Distributor's. + * @param dataFileConfig the data file configuration to check + * @return true if the data update URL is the Distributor's + */ + @Override + protected boolean isLicenseKeyRequired(DataFileConfiguration dataFileConfig) { + return DATA_UPDATE_URL.equals(dataFileConfig.getDataUpdateUrl()); + } + /** * Create a new empty data file instance to be populated with the details of * the data file to be used. diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java index 39ca21a6f..588f1947c 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElement.java @@ -23,12 +23,17 @@ package fiftyone.pipeline.engines.fiftyone.flowelements; import fiftyone.pipeline.core.data.*; +import fiftyone.pipeline.core.data.factories.ElementDataFactory; import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.FlowElementBase; +import fiftyone.pipeline.engines.fiftyone.data.SequenceData; +import java.math.BigInteger; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.regex.Pattern; import org.slf4j.Logger; /** @@ -38,46 +43,129 @@ public class SequenceElement extends FlowElementBase { + private static final Pattern DIGITS = Pattern.compile("[0-9]+"); + + private final List properties; + + /** + * The session id and sequence of a callback, which the specification + * lists as this element's accepted Evidence. Without them the web + * integration would not add them, and every callback would start a new + * session. + */ + private final EvidenceKeyFilter acceptedEvidence = + new EvidenceKeyFilterWhitelist(Arrays.asList( + Constants.EVIDENCE_SESSIONID, + Constants.EVIDENCE_SEQUENCE), + String.CASE_INSENSITIVE_ORDER); + /** * Construct a new instance of the {@link FlowElement}. * @param logger logger instance to use for logging */ - public SequenceElement(Logger logger) { - super(logger, null); + @SuppressWarnings("this-escape") + public SequenceElement(final Logger logger) { + super(logger, new ElementDataFactory() { + @Override + public ElementData create( + FlowData flowData, + FlowElement flowElement) { + return new SequenceData(logger, flowData); + } + }); + properties = new ArrayList<>(); + properties.add(new ElementPropertyMetaDataDefault( + SequenceData.SESSION_ID_PROPERTY_NAME, + this, + "", + String.class, + true)); + properties.add(new ElementPropertyMetaDataDefault( + SequenceData.SEQUENCE_PROPERTY_NAME, + this, + "", + Integer.class, + true)); } @Override protected void processInternal(FlowData data) throws Exception { Map evidence = data.getEvidence().asKeyMap(); - - // If the evidence does not contain a session id then create a new one. - if (evidence.containsKey(Constants.EVIDENCE_SESSIONID) == false) { - data.addEvidence(Constants.EVIDENCE_SESSIONID, getNewSessionId()); - } - // If the evidence does not have a sequence then add one. - if (evidence.containsKey(Constants.EVIDENCE_SEQUENCE) == false) { - data.addEvidence(Constants.EVIDENCE_SEQUENCE, 1); + // The session id and the sequence are each used only when they are + // valid, and otherwise count as absent. + Object givenSession = evidence.get(Constants.EVIDENCE_SESSIONID); + String sessionId = + givenSession instanceof String && + ((String) givenSession).isEmpty() == false ? + (String) givenSession : getNewSessionId(); + Integer previous = toSequence(evidence.get(Constants.EVIDENCE_SEQUENCE)); + int sequence = previous == null ? 1 : previous + 1; + + data.addEvidence(Constants.EVIDENCE_SESSIONID, sessionId); + data.addEvidence(Constants.EVIDENCE_SEQUENCE, sequence); + + SequenceData sequenceData = (SequenceData) data.getOrAdd( + getElementDataKey(), + getDataFactory()); + sequenceData.setSessionId(sessionId); + sequenceData.setSequence(sequence); + } + + /** + * The sequence in Evidence, which is a whole number from 0 up, or text + * made only of decimal digits. A value that is anything else, is + * negative or has no larger int after it counts as no sequence, so that + * adding one never overflows. + * @return the sequence, or null if there is none + */ + private static Integer toSequence(Object sequence) { + long value; + if (sequence instanceof Integer || sequence instanceof Long || + sequence instanceof Short || sequence instanceof Byte) { + value = ((Number) sequence).longValue(); + } else if (sequence instanceof BigInteger) { + BigInteger big = (BigInteger) sequence; + if (big.bitLength() > 31) { + return null; + } + value = big.longValue(); + } else if (sequence instanceof String && + DIGITS.matcher((String) sequence).matches()) { + // More digits than any int has cannot fit, however many zeros + // come first, so only the value of a short string is read. + String digits = ((String) sequence).replaceFirst("^0+(?=.)", ""); + if (digits.length() > 10) { + return null; + } + value = Long.parseLong(digits); } else { - Object sequence = evidence.get(Constants.EVIDENCE_SEQUENCE); - int seq = (int) sequence; - data.addEvidence(Constants.EVIDENCE_SEQUENCE, seq + 1); + return null; } + return value < Integer.MAX_VALUE && value >= 0 ? + Integer.valueOf((int) value) : null; } + /** + * The key the Element Data is stored under in the Flow Data, unless a + * subclass gives another from {@link #getElementDataKey()}. + * @see Specification + */ + public static final String DEFAULT_ELEMENT_DATA_KEY = "sequence"; + @Override public String getElementDataKey() { - return "sequence-element"; + return DEFAULT_ELEMENT_DATA_KEY; } @Override public EvidenceKeyFilter getEvidenceKeyFilter() { - return new EvidenceKeyFilterWhitelist(new ArrayList()); + return acceptedEvidence; } @Override public List getProperties() { - return new ArrayList<>(); + return properties; } @Override diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElement.java index 78d4c29ba..9bb05c6ea 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElement.java @@ -38,6 +38,7 @@ import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.factories.ElementDataFactory; +import fiftyone.pipeline.core.flowelements.FlowElement; import fiftyone.pipeline.core.flowelements.FlowElementBase; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.data.AspectPropertyValue; @@ -101,8 +102,20 @@ protected void processInternal(FlowData data) throws Exception { PipelineConfig config; Pipeline pipeline = data.getPipeline(); if ((config = pipelineConfigs.get(pipeline)) == null) { + // The configuration is kept once every element has loaded its + // properties. Until then, as after a cloud element could not + // reach its service when the Pipeline was built, it is built + // again on each request, so it gains the SetHeader properties + // once they are known. + boolean loaded = allPropertiesLoaded(pipeline); config = populateConfig(pipeline); - config = pipelineConfigs.getOrDefault(pipeline, config); + if (loaded) { + PipelineConfig kept = + pipelineConfigs.putIfAbsent(pipeline, config); + if (kept != null) { + config = kept; + } + } } SetHeadersData elementData = data.getOrAdd( @@ -139,6 +152,21 @@ protected void unmanagedResourcesCleanup() { // Nothing to clean up here } + /** + * @param pipeline the Pipeline + * @return true if every element of the Pipeline has loaded its + * properties + */ + @SuppressWarnings("rawtypes") + private static boolean allPropertiesLoaded(Pipeline pipeline) { + for (FlowElement element : pipeline.getFlowElements()) { + if (element.hasLoadedProperties() == false) { + return false; + } + } + return true; + } + /** * Build the response headers from the SetHeader properties in the FlowData. * @param data flowData object diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java new file mode 100644 index 000000000..d0bb4d0c3 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOff.java @@ -0,0 +1,169 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import java.util.ArrayDeque; +import java.util.Deque; + +/** + * Decides when usage sharing may send. Sending pauses for the initial delay + * once the set number of sends fail within the window. A failed send after a + * pause multiplies the delay, up to the maximum, and a successful send ends + * the back-off. Only sending pauses, so evidence is still queued while + * paused. Each method takes the time from the caller, in milliseconds on + * the clock that {@link #now()} reads, so a change to the system time cannot + * lengthen or skip a pause. + */ +class ShareUsageBackOff { + + private final int failuresToEnterRecovery; + private final long failuresWindowMillis; + private final double initialDelaySeconds; + private final double maxDelaySeconds; + private final double multiplier; + + /** + * When each recent failed send happened, oldest first. Empty while + * backing off. + */ + private final Deque failures = new ArrayDeque<>(); + + /** + * The length of the current pause in seconds, or zero when not backing + * off. + */ + private double delaySeconds = 0; + + /** + * When sending may start again. + */ + private long resumeAt = Long.MIN_VALUE; + + /** + * @return the time in milliseconds on a monotonic clock, for the caller to + * pass to the other methods. Its zero is not a calendar time. + */ + static long now() { + return System.nanoTime() / 1000000; + } + + ShareUsageBackOff( + int failuresToEnterRecovery, + int failuresWindowSeconds, + double initialDelaySeconds, + double maxDelaySeconds, + double multiplier) { + if (failuresToEnterRecovery < + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN || + failuresToEnterRecovery > + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX) { + throw new IllegalArgumentException( + "failuresToEnterRecovery must be from " + + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN + + " to " + + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX + "."); + } + if (failuresWindowSeconds < 1) { + throw new IllegalArgumentException( + "failuresWindowSeconds must be 1 or more."); + } + if ((initialDelaySeconds > 0) == false) { + throw new IllegalArgumentException( + "initialDelaySeconds must be more than 0."); + } + if ((maxDelaySeconds > 0) == false) { + throw new IllegalArgumentException( + "maxDelaySeconds must be more than 0."); + } + if ((multiplier >= 1) == false) { + throw new IllegalArgumentException( + "multiplier must be 1 or more."); + } + this.failuresToEnterRecovery = failuresToEnterRecovery; + this.failuresWindowMillis = failuresWindowSeconds * 1000L; + this.initialDelaySeconds = initialDelaySeconds; + this.maxDelaySeconds = maxDelaySeconds; + this.multiplier = multiplier; + } + + /** + * @return true from the start of a pause until a send succeeds + */ + synchronized boolean isBackingOff() { + return delaySeconds > 0; + } + + /** + * @param now the time now in milliseconds + * @return the milliseconds left before sending may start again, or zero + * if it may start now + */ + synchronized long remaining(long now) { + return resumeAt > now ? resumeAt - now : 0; + } + + /** + * Records a failed send. + * @param now the time now in milliseconds + * @return the new pause in milliseconds if this failure started or + * lengthened one, otherwise zero + */ + synchronized long failed(long now) { + if (delaySeconds > 0) { + // The send tried when the last pause ended failed. + delaySeconds = Math.min( + delaySeconds * multiplier, + maxDelaySeconds); + } else { + failures.addLast(now); + // The emptiness check comes first so that nothing is unboxed + // from an empty deque. The constructor refuses a window below + // one second, so the deque always keeps the failure just added. + while (failures.isEmpty() == false && + failures.peekFirst() < now - failuresWindowMillis) { + failures.removeFirst(); + } + if (failures.size() < failuresToEnterRecovery) { + return 0; + } + failures.clear(); + delaySeconds = Math.min(initialDelaySeconds, maxDelaySeconds); + } + long pause = Math.max(1, Math.round(delaySeconds * 1000)); + resumeAt = now + pause; + return pause; + } + + /** + * Records a successful send. + * @return true if this ended a back-off + */ + synchronized boolean succeeded() { + if (delaySeconds == 0) { + return false; + } + delaySeconds = 0; + resumeAt = Long.MIN_VALUE; + return true; + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java index 58a46d731..1fd5bf088 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBase.java @@ -29,6 +29,8 @@ import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.engines.configuration.CacheConfiguration; import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsage; +import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsageTracker; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; import fiftyone.pipeline.engines.fiftyone.exceptions.HttpException; import fiftyone.pipeline.engines.fiftyone.trackers.ShareUsageTracker; import fiftyone.pipeline.engines.trackers.Tracker; @@ -105,12 +107,15 @@ public abstract class ShareUsageBase protected final String shareUsageUrl; /** - * Evidence key filter including the session id. + * The Evidence this element accepts, which is none, so that cache keys, + * the ETag and the Vary header never depend on it. It finds what to share + * with {@link ShareUsageEvidence#find} instead. */ - private final EvidenceKeyFilter evidenceKeyFilter; + private final EvidenceKeyFilter acceptedEvidence = + new EvidenceKeyFilterWhitelist(new ArrayList()); /** - * Evidence key filter excluding the session id + * Which of the values found are shared, leaving out the session cookie. */ private final EvidenceKeyFilter evidenceKeyFilterExclSession; @@ -223,6 +228,23 @@ public abstract class ShareUsageBase */ protected long lostDataReset = LOST_DATA_RESET_DEFAULT; + /** + * True from the first entry discarded because the queue is full until an + * entry is next queued. While true, a message is logged only for every + * {@link Constants#QUEUE_FULL_LOG_EVERY} entries discarded. + */ + private volatile boolean queueFullReported = false; + + /** + * Decides when sending may happen after failed sends. + */ + private final ShareUsageBackOff backOff; + + /** + * Released when the element starts closing, to end any pause at once. + */ + private final CountDownLatch closing = new CountDownLatch(1); + /** * Constructor * @param logger the logger to use @@ -391,6 +413,98 @@ protected ShareUsageBase( List> ignoreDataEvidenceFilter, String sessionCookieName, Tracker tracker) { + this( + logger, + sharePercentage, + minimumEntriesPerMessage, + maximumQueueSize, + addTimeout, + takeTimeout, + repeatEvidenceIntervalMinutes, + trackSession, + shareUsageUrl, + blockedHttpHeaders, + includedQueryStringParameters, + ignoreDataEvidenceFilter, + sessionCookieName, + tracker, + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT, + Constants.SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT, + Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT, + Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT, + Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT, + false, + false); + } + + /** + * Constructor + * @param logger the logger to use + * @param sharePercentage the approximate proportion of requests to share. + * 1 = 100%, 0.5 = 50%, etc. + * @param minimumEntriesPerMessage the minimum number of request entries per + * message sent to 51Degrees + * @param maximumQueueSize the maximum number of items to hold in the queue + * at one time. This must be larger than minimum + * entries + * @param addTimeout the timeout in milliseconds to allow when attempting to + * add an item to the queue. If this timeout is exceeded + * then the item is not shared + * @param takeTimeout the timeout in milliseconds to allow when attempting + * to take an item to the queue + * @param repeatEvidenceIntervalMinutes the interval (in minutes) which is + * used to decide if repeat evidence is + * old enough to consider a new session + * @param trackSession set if the tracker should consider sessions in share + * usage + * @param shareUsageUrl the URL to send data to + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParameters a list of the names of query string + * parameters that share usage should + * send to 51Degrees + * @param ignoreDataEvidenceFilter the filter used to determine if an item + * of evidence should be ignored or not + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param tracker the {@link Tracker} to use to determine if a given + * {@link FlowData} instance should be shared or not + * @param failuresToEnterRecovery the number of failed sends within the + * failures window that pauses sending + * @param failuresWindowSeconds the time in seconds over which failed sends + * are counted + * @param initialRecoveryDelaySeconds the length in seconds of the first + * pause + * @param maxRecoveryDelaySeconds the longest a pause can be, in seconds + * @param recoveryMultiplier what the pause is multiplied by each time the + * send tried when a pause ends fails + * @param shareAllEvidence if true then all Evidence is shared, apart from + * the keys that are never shared + * @param shareAllQueryStringParameters if true then all query string + * parameters are shared + */ + protected ShareUsageBase( + Logger logger, + double sharePercentage, + int minimumEntriesPerMessage, + int maximumQueueSize, + int addTimeout, + int takeTimeout, + int repeatEvidenceIntervalMinutes, + boolean trackSession, + String shareUsageUrl, + List blockedHttpHeaders, + List includedQueryStringParameters, + List> ignoreDataEvidenceFilter, + String sessionCookieName, + Tracker tracker, + int failuresToEnterRecovery, + int failuresWindowSeconds, + double initialRecoveryDelaySeconds, + double maxRecoveryDelaySeconds, + double recoveryMultiplier, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { super(logger, null); if (minimumEntriesPerMessage > maximumQueueSize) { @@ -405,6 +519,12 @@ protected ShareUsageBase( } evidenceCollection = new LinkedBlockingQueue<>(maximumQueueSize); + backOff = new ShareUsageBackOff( + failuresToEnterRecovery, + failuresWindowSeconds, + initialRecoveryDelaySeconds, + maxRecoveryDelaySeconds, + recoveryMultiplier); this.addTimeout = addTimeout; this.takeTimeout = takeTimeout; @@ -424,10 +544,13 @@ protected ShareUsageBase( includedQueryStringParameters.add(Constants.EVIDENCE_SESSIONID_SUFFIX); includedQueryStringParameters.add(Constants.EVIDENCE_SEQUENCE_SUFIX); - evidenceKeyFilter = new EvidenceKeyFilterShareUsage( - blockedHttpHeaders, includedQueryStringParameters, true, sessionCookieName); evidenceKeyFilterExclSession = new EvidenceKeyFilterShareUsage( - blockedHttpHeaders, includedQueryStringParameters, false, sessionCookieName); + blockedHttpHeaders, + includedQueryStringParameters, + false, + sessionCookieName, + shareAllEvidence, + shareAllQueryStringParameters); this.ignoreDataEvidenceFilter = ignoreDataEvidenceFilter; @@ -440,8 +563,13 @@ protected ShareUsageBase( this.tracker = new ShareUsageTracker( cacheConfiguration, intervalMillis, - new EvidenceKeyFilterShareUsage( - blockedHttpHeaders, includedQueryStringParameters, trackSession, sessionCookieName)); + new EvidenceKeyFilterShareUsageTracker( + blockedHttpHeaders, + includedQueryStringParameters, + trackSession, + sessionCookieName, + shareAllEvidence, + shareAllQueryStringParameters)); } properties = new ArrayList<>(); @@ -517,15 +645,20 @@ public void addPipeline(Pipeline pipeline) { @Override protected void processInternal(FlowData flowData){ boolean ignoreData = false; - Map evidence = flowData.getEvidence().asKeyMap(); + // The values are found here only for the ignore filter. Otherwise + // they are found once the tracker allows the call to be shared, as + // most calls repeat one already shared. + Map found = null; logger.debug("processInternal()"); - if (ignoreDataEvidenceFilter != null) { + if (ignoreDataEvidenceFilter != null && + ignoreDataEvidenceFilter.isEmpty() == false) { + found = ShareUsageEvidence.find(flowData.getEvidence()); for (Map.Entry entry : ignoreDataEvidenceFilter) { String key = entry.getKey(); - if (evidence.containsKey(key)) { - if (evidence.get(key).toString().equals(entry.getValue())) { + if (found.containsKey(key)) { + if (found.get(key).toString().equals(entry.getValue())) { ignoreData = true; break; } @@ -534,13 +667,13 @@ protected void processInternal(FlowData flowData){ } if (isCanceled() == false && ignoreData == false) { - processData(flowData); + processData(flowData, found); } } @Override public EvidenceKeyFilter getEvidenceKeyFilter() { - return evidenceKeyFilter; + return acceptedEvidence; } @Override @@ -605,14 +738,15 @@ protected void managedResourcesCleanup() { // This is set before anything is submitted, so a consumer can never // observe it as false and then exit leaving a partial batch behind. stopping = true; - // Queue a final consumer without consulting isRunning(). That check is - // subject to a time-of-check/time-of-use race: a consumer that has - // finished its loop with a sub-threshold remainder still reports as - // running until its Future flips to done, which used to stop close from - // queueing the drain that would have sent the remainder. The executor is - // single threaded, so the task queued here cannot run alongside a - // consumer that is already in flight - it runs after it - and shutdown() - // still allows an already submitted task to run to completion. + // Ends a pause at once, so closing never waits for one. Nothing is + // sent while paused. + closing.countDown(); + // Queue a final consumer without consulting isRunning(). A consumer + // that has finished its loop with a sub-threshold remainder still + // reports as running until its Future is done, so checking would skip + // the drain that sends the remainder. The executor is single threaded, + // so the task queued here runs after any consumer already in flight, + // and shutdown() still allows an already submitted task to finish. submitSendData(); try { executor.shutdown(); @@ -626,6 +760,13 @@ protected void managedResourcesCleanup() { // abandon its own wait immediately. logger.warn("Interrupted while awaiting close down"); } + if (evidenceCollection.isEmpty() == false && + backOff.remaining(ShareUsageBackOff.now()) > 0) { + logger.info( + "{} usage sharing entries were not sent, as sending was " + + "paused after failed sends when usage sharing closed.", + evidenceCollection.size()); + } } @Override @@ -652,19 +793,24 @@ private boolean isLocalHost(String address) throws UnknownHostException { /** * Process the supplied request data * @param data the {@link FlowData} instance that provides the evidence + * @param found the values {@link ShareUsageEvidence#find} found for it, + * or null if they have not been found yet */ - private void processData(FlowData data) { + private void processData(FlowData data, Map found) { logger.debug("processData()"); if (random.nextDouble() <= sharePercentage) { // Check if the tracker will allow sharing of this data if (tracker.track(data)) { + Map shared = found != null ? found : + ShareUsageEvidence.find(data.getEvidence()); // Extract the data we want from the evidence and add // it to the collection. try { if (evidenceCollection.offer( - getDataFromEvidence(data.getEvidence()), + getDataFromEvidence(shared), addTimeout, TimeUnit.MILLISECONDS)) { + queueFullReported = false; logger.debug("Queued {} entries", evidenceCollection.size()); // If the collection has enough entries then start // taking data from it to be sent. @@ -672,11 +818,17 @@ private void processData(FlowData data) { trySendData(); } } else { - if (++lostData >= lostDataReset) { - logger.warn("Could not queue data for Share Usage. This could mean that" + - " the queue is too small, that the proportion of data shared is" + - " too high - or that the sending process has encountered" + - " problems sending."); + long lost = ++lostData; + if (queueFullReported == false || + lost % Constants.QUEUE_FULL_LOG_EVERY == 0) { + queueFullReported = true; + logger.warn("The usage sharing queue is full, so usage data is " + + "being discarded. This could mean that the queue is too " + + "small, that the proportion of data shared is too high, " + + "or that sending is failing. {} entries have been " + + "discarded.", lost); + } + if (lost >= lostDataReset) { lostData = 0; } } @@ -688,24 +840,27 @@ private void processData(FlowData data) { } /** - * Extract the desired data from the evidence. + * Extract the desired data from the values found. * In order to avoid problems with the evidence data being disposed before * it is sent, the data placed into a new object rather than being a * reference to the existing evidence instance. * - * @param evidence an {@link Evidence} instance that contains the data to be extracted - * @return a {@link ShareUsageData} instance populated with data from the evidence + * @param found the values {@link ShareUsageEvidence#find} found + * @return a {@link ShareUsageData} instance populated with the values + * the sharing rules allow */ - private ShareUsageData getDataFromEvidence(Evidence evidence) { + private ShareUsageData getDataFromEvidence(Map found) { ShareUsageData shareUsageData = new ShareUsageData(); - for (Map.Entry entry : evidence.asKeyMap().entrySet()) { + for (Map.Entry entry : found.entrySet()) { // Check if we can send this piece of evidence if (evidenceKeyFilterExclSession.include(entry.getKey()) == false) { continue; } - switch (entry.getKey()) { + // The keys are compared without regard to case, and the constants + // are lower case. + switch (entry.getKey().toLowerCase(Locale.ROOT)) { case fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY: // The client IP is dealt with separately for backwards // compatibility purposes. @@ -783,6 +938,51 @@ private void submitSendData() { protected abstract void sendUsageData() throws HttpException; + /** + * Waits until sending may start again after failed sends. A consumer + * calls this before each message it sends. + * @return false if the element started closing while sending was paused, + * in which case nothing more should be sent + */ + protected boolean waitWhilePaused() { + long pause; + while ((pause = backOff.remaining(ShareUsageBackOff.now())) > 0) { + try { + if (closing.await(pause, TimeUnit.MILLISECONDS)) { + return false; + } + } catch (InterruptedException e) { + // Not re-interrupting, for the reason given in + // managedResourcesCleanup. The interrupt comes from closing. + return false; + } + } + return true; + } + + /** + * Records whether the remote service accepted a message, so that repeated + * failures pause sending and a success ends the pause. + * @param accepted true if the message was accepted + */ + protected void recordSendResult(boolean accepted) { + if (accepted) { + if (backOff.succeeded()) { + logger.info("Usage sharing resumed after a send succeeded."); + } + } else { + long pause = backOff.failed(ShareUsageBackOff.now()); + if (pause > 0) { + logger.warn( + "Usage sharing paused for {} seconds after failed " + + "sends. Usage data is still queued, up to the " + + "maximum queue size, and sending starts again when " + + "the pause ends.", + pause / 1000.0); + } + } + } + /** * Get engine Version No. * @return engine version no diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java index 56a96e9ac..c73a86326 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilder.java @@ -129,7 +129,14 @@ public ShareUsageElement build() throws IOException { includedQueryStringParameters, ignoreDataEvidenceFilter, sessionCookieName, - null); + null, + failuresToEnterRecovery, + failuresWindowSeconds, + initialRecoveryDelaySeconds, + maxRecoveryDelaySeconds, + recoveryMultiplier, + shareAllEvidence, + shareAllQueryStringParameters); if (Objects.nonNull(this.httpClient)) { sue.httpClient = httpClient; } else if (Objects.nonNull(dataUploader)){ diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java index bad9c1bbd..6e07f6dd3 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBuilderBase.java @@ -59,6 +59,13 @@ public abstract class ShareUsageBuilderBase { protected final List includedQueryStringParameters = new ArrayList<>(); protected final List> ignoreDataEvidenceFilter = new ArrayList<>(); protected boolean trackSession; + protected int failuresToEnterRecovery = Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT; + protected int failuresWindowSeconds = Constants.SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT; + protected double initialRecoveryDelaySeconds = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT; + protected double maxRecoveryDelaySeconds = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT; + protected double recoveryMultiplier = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT; + protected boolean shareAllEvidence; + protected boolean shareAllQueryStringParameters; /** * Constructor @@ -83,7 +90,8 @@ public ShareUsageBuilderBase(ILoggerFactory loggerFactory, Logger logger) { } /** - * By default, query string parameters are not shared. + * By default, only query string parameters that start with 51D_ are + * shared. *

      * This setter replaces the current list, if any *

      @@ -95,7 +103,7 @@ public ShareUsageBuilderBase(ILoggerFactory loggerFactory, Logger logger) { * @return this builder */ - @DefaultValue("No sharing") + @DefaultValue("No sharing unless the parameter starts with 51D_") public ShareUsageBuilderBase setIncludedQueryStringParameters( List queryStringParameterNames) { includedQueryStringParameters.clear(); @@ -104,7 +112,8 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( } /** - * By default, query string parameters are not shared. + * By default, only query string parameters that start with 51D_ are + * shared. *

      * This setter replaces the current list, if any *

      @@ -115,7 +124,7 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( * query string parameters to include * @return this builder */ - @DefaultValue("No sharing") + @DefaultValue("No sharing unless the parameter starts with 51D_") public ShareUsageBuilderBase setIncludedQueryStringParameters( String queryStringParameterNames) { includedQueryStringParameters.clear(); @@ -125,7 +134,8 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( } /** - * By default, query string parameters are not shared. + * By default, only query string parameters that start with 51D_ are + * shared. *

      * This setting adds the name of a parameter to share with 51Degrees * if needed. @@ -133,13 +143,63 @@ public ShareUsageBuilderBase setIncludedQueryStringParameters( * query string parameter to include * @return this builder */ - @DefaultValue("No sharing") + @DefaultValue("No sharing unless the parameter starts with 51D_") public ShareUsageBuilderBase setIncludedQueryStringParameter( String queryStringParameterName) { includedQueryStringParameters.add(queryStringParameterName.trim()); return this; } + /** + * If set to true, every query string and HTTP form parameter is shared, + * rather than only those that start with 51D_ or are named in the + * included parameters. The whole query string and the parameters that are + * never shared are still not shared. + *

      + * Default value is false + * @param shareAll true to share all query string parameters + * @return this builder + */ + @DefaultValue(booleanValue = false) + public ShareUsageBuilderBase setShareAllQueryStringParameters( + boolean shareAll) { + this.shareAllQueryStringParameters = shareAll; + return this; + } + + /** + * Whether every query string parameter is shared. + * @return true if all query string parameters are shared + */ + public boolean isShareAllQueryStringParameters() { + return shareAllQueryStringParameters; + } + + /** + * If set to true, all Evidence is shared, which overrides the blocked + * HTTP headers and the rules for cookies, session values and query string + * parameters. The keys that are never shared (an email address, the salt + * for hashed email identifiers, the whole query string, the path and the + * Cookie header) are still not shared. + *

      + * Default value is false + * @param shareAll true to share all Evidence + * @return this builder + */ + @DefaultValue(booleanValue = false) + public ShareUsageBuilderBase setShareAllEvidence(boolean shareAll) { + this.shareAllEvidence = shareAll; + return this; + } + + /** + * Whether all Evidence is shared. + * @return true if all Evidence is shared + */ + public boolean isShareAllEvidence() { + return shareAllEvidence; + } + /** * By default, all HTTP headers (excluding a few such as 'cookies', if they * don't start with 51D_) are shared. Individual headers can be excluded from sharing by @@ -417,6 +477,92 @@ public ShareUsageBuilderBase setTrackSession(boolean track) { return this; } + /** + * Set the number of failed sends within the failures window that pauses + * sending. Usage data is still queued while sending is paused, up to the + * maximum queue size. + * @param failuresToEnterRecovery the number of failed sends, from 1 to 100 + * @return this builder + */ + @DefaultValue(intValue = Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT) + public ShareUsageBuilderBase setFailuresToEnterRecovery( + int failuresToEnterRecovery) { + if (failuresToEnterRecovery < Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN || + failuresToEnterRecovery > Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX) { + throw new IllegalArgumentException( + "failuresToEnterRecovery must be from " + + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MIN + " to " + + Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_MAX + + ". Received: " + failuresToEnterRecovery + "."); + } + this.failuresToEnterRecovery = failuresToEnterRecovery; + return this; + } + + /** + * Set the time in seconds over which failed sends are counted. + * @param failuresWindowSeconds the time in seconds, 1 or more + * @return this builder + */ + @DefaultValue(intValue = Constants.SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT) + public ShareUsageBuilderBase setFailuresWindowSeconds( + int failuresWindowSeconds) { + if (failuresWindowSeconds < 1) { + throw new IllegalArgumentException( + "failuresWindowSeconds must be 1 or more."); + } + this.failuresWindowSeconds = failuresWindowSeconds; + return this; + } + + /** + * Set the length in seconds of the first pause after failed sends. + * @param initialDelaySeconds the length in seconds, more than 0 + * @return this builder + */ + @DefaultValue(doubleValue = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT) + public ShareUsageBuilderBase setInitialRecoveryDelaySeconds( + double initialDelaySeconds) { + if ((initialDelaySeconds > 0) == false) { + throw new IllegalArgumentException( + "initialDelaySeconds must be more than 0."); + } + this.initialRecoveryDelaySeconds = initialDelaySeconds; + return this; + } + + /** + * Set the longest a pause can be, in seconds. + * @param maxDelaySeconds the length in seconds, more than 0 + * @return this builder + */ + @DefaultValue(doubleValue = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT) + public ShareUsageBuilderBase setMaxRecoveryDelaySeconds( + double maxDelaySeconds) { + if ((maxDelaySeconds > 0) == false) { + throw new IllegalArgumentException( + "maxDelaySeconds must be more than 0."); + } + this.maxRecoveryDelaySeconds = maxDelaySeconds; + return this; + } + + /** + * Set what the pause is multiplied by each time the send tried when a + * pause ends fails. 2 doubles the pause each time. + * @param multiplier the multiplier, 1 or more + * @return this builder + */ + @DefaultValue(doubleValue = Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT) + public ShareUsageBuilderBase setRecoveryMultiplier(double multiplier) { + if ((multiplier >= 1) == false) { + throw new IllegalArgumentException( + "multiplier must be 1 or more."); + } + this.recoveryMultiplier = multiplier; + return this; + } + /** * Create the {@link ShareUsageElement}. * @return the newly created target instance diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java index c56e032f8..58109cb2a 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElement.java @@ -40,12 +40,14 @@ import java.net.URI; import java.util.*; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; import java.util.zip.GZIPOutputStream; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_DEFAULT_HTTP_POST_TIMEOUT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT; +import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT; import static fiftyone.pipeline.engines.fiftyone.flowelements.Constants.SHARE_USAGE_MAX_EVIDENCE_LENGTH; /** @@ -309,6 +311,98 @@ public class ShareUsageElement extends ShareUsageBase { List> ignoreDataEvidenceFilter, String sessionCookieName, Tracker tracker) { + this( + logger, + sharePercentage, + minimumEntriesPerMessage, + maximumQueueSize, + addTimeout, + takeTimeout, + repeatEvidenceIntervalMinutes, + trackSession, + shareUsageUrl, + blockedHttpHeaders, + includedQueryStringParameters, + ignoreDataEvidenceFilter, + sessionCookieName, + tracker, + SHARE_USAGE_FAILURES_TO_ENTER_RECOVERY_DEFAULT, + SHARE_USAGE_FAILURES_WINDOW_SECONDS_DEFAULT, + SHARE_USAGE_EXPONENTIAL_BACKOFF_INITIAL_DELAY_SECONDS_DEFAULT, + SHARE_USAGE_EXPONENTIAL_BACKOFF_MAX_DELAY_SECONDS_DEFAULT, + SHARE_USAGE_EXPONENTIAL_BACKOFF_MULTIPLIER_DEFAULT, + false, + false); + } + + /** + * Constructor + * + * @param logger the logger to use + * @param sharePercentage the approximate proportion of requests to share. + * 1 = 100%, 0.5 = 50%, etc. + * @param minimumEntriesPerMessage the minimum number of request entries per + * message sent to 51Degrees + * @param maximumQueueSize the maximum number of items to hold in the queue + * at one time. This must be larger than minimum + * entries + * @param addTimeout the timeout in milliseconds to allow when attempting to + * add an item to the queue. If this timeout is exceeded + * then the item is not shared + * @param takeTimeout the timeout in milliseconds to allow when attempting + * to take an item to the queue + * @param repeatEvidenceIntervalMinutes the interval (in minutes) which is + * used to decide if repeat evidence is + * old enough to consider a new session + * @param trackSession set if the tracker should consider sessions in share + * usage + * @param shareUsageUrl the URL to send data to + * @param blockedHttpHeaders a list of the names of the HTTP headers that + * share usage should not send to 51Degrees + * @param includedQueryStringParameters a list of the names of query string + * parameters that share usage should + * send to 51Degrees + * @param ignoreDataEvidenceFilter the filter used to determine if an item + * of evidence should be ignored or not + * @param sessionCookieName the name of the cookie that contains the session + * id + * @param tracker the {@link Tracker} to use to determine if a given + * {@link FlowData} instance should be shared or not + * @param failuresToEnterRecovery the number of failed sends within the failures + * window that pauses sending + * @param failuresWindowSeconds the time in seconds over which failed sends are + * counted + * @param initialRecoveryDelaySeconds the length in seconds of the first pause + * @param maxRecoveryDelaySeconds the longest a pause can be, in seconds + * @param recoveryMultiplier what the pause is multiplied by each time the send + * tried when a pause ends fails + * @param shareAllEvidence if true then all Evidence is shared, apart from + * the keys that are never shared + * @param shareAllQueryStringParameters if true then all query string + * parameters are shared + */ + ShareUsageElement( + Logger logger, + double sharePercentage, + int minimumEntriesPerMessage, + int maximumQueueSize, + int addTimeout, + int takeTimeout, + int repeatEvidenceIntervalMinutes, + boolean trackSession, + String shareUsageUrl, + List blockedHttpHeaders, + List includedQueryStringParameters, + List> ignoreDataEvidenceFilter, + String sessionCookieName, + Tracker tracker, + int failuresToEnterRecovery, + int failuresWindowSeconds, + double initialRecoveryDelaySeconds, + double maxRecoveryDelaySeconds, + double recoveryMultiplier, + boolean shareAllEvidence, + boolean shareAllQueryStringParameters) { super( logger, sharePercentage, @@ -323,7 +417,14 @@ public class ShareUsageElement extends ShareUsageBase { includedQueryStringParameters, ignoreDataEvidenceFilter, sessionCookieName, - tracker); + tracker, + failuresToEnterRecovery, + failuresWindowSeconds, + initialRecoveryDelaySeconds, + maxRecoveryDelaySeconds, + recoveryMultiplier, + shareAllEvidence, + shareAllQueryStringParameters); headers.put("Content-Type", "text/xml; charset=utf-8"); // assume default GZIP connection @@ -336,7 +437,8 @@ public class ShareUsageElement extends ShareUsageBase { } /** - * Take data from the queue until there are fewer entries than the minimum batch size + * Take data from the queue until there are fewer entries than the minimum + * batch size, waiting out any pause after failed sends before each message */ @Override protected void sendUsageData() { @@ -345,10 +447,14 @@ protected void sendUsageData() { List allData = new ArrayList<>(); // drain queue while minimum entries in it do { + // Closing while paused ends the wait, and nothing more is sent. + if (waitWhilePaused() == false) { + break; + } logger.debug(threadMarker, "Queue size is {}", evidenceCollection.size()); try { allData.clear(); - // for historical reasons max entries set to 2* + // A message holds at most twice the minimum entries. while (allData.size() < minEntriesPerMessage * 2) { ShareUsageData currentData = evidenceCollection.poll(takeTimeout, TimeUnit.MILLISECONDS); // no data available @@ -362,7 +468,13 @@ protected void sendUsageData() { // drained everything - and there is no point posting a document // with no entries in it. if (allData.isEmpty() == false) { - sendAsXML(allData); + boolean accepted = false; + try { + sendAsXML(allData); + accepted = true; + } finally { + recordSendResult(accepted); + } } } catch (InterruptedException interruptedException) { // Deliberately not re-interrupting: the interrupt comes from @@ -394,7 +506,10 @@ protected void sendAsXML(List allData) throws Exception { // check for completion int code = dataUploader.getResponseCode(); if (code != 200) { - throw new Exception(String.format("Share Usage response code was %d ", code)); + throw new Exception(String.format( + "Share Usage response code was %d with content '%s'", + code, + dataUploader.getResponseContent())); } logger.debug(threadMarker, "Send elements done"); } @@ -422,7 +537,13 @@ private void streamXml(List allData, OutputStream os) throws XML */ protected void writeXmlData(XMLStreamWriter writer, ShareUsageData data) throws XMLStreamException { writer.writeStartElement("Device"); - // --- write invariant data + // The elements follow the order of the specification's example. + // The SessionID used to track a series of requests + writeXmlElement(writer, "SessionId", data.sessionId); + // The sequence number of the request in a series of requests. + writeXmlElement(writer, "Sequence", String.valueOf(data.sequence)); + // The UTC date/time this entry was written + writeXmlElement(writer, "DateSent", DATE_FMT.format(new Date())); // The version number of the Pipeline API writeXmlElement(writer, "Version", coreVersion); // Write Pipeline information @@ -435,21 +556,13 @@ protected void writeXmlData(XMLStreamWriter writer, ShareUsageData data) throws writeXmlElement(writer, "Language", "java"); // The software language version writeXmlElement(writer, "LanguageVersion", languageVersion); + // The client IP of the request + writeXmlElement(writer, "ClientIP", data.clientIP); // The IP of this server writeXmlElement(writer, "ServerIP", getHostAddress()); // The OS name and version writeXmlElement(writer, "Platform", osVersion); - // --- write variable data - // The SessionID used to track a series of requests - writeXmlElement(writer, "SessionId", data.sessionId); - // The sequence number of the request in a series of requests. - writeXmlElement(writer, "Sequence", String.valueOf(data.sequence)); - // The UTC date/time this entry was written - writeXmlElement(writer, "DateSent", DATE_FMT.format(new Date())); - // The client IP of the request - writeXmlElement(writer, "ClientIP", data.clientIP); - // Write all other evidence data that has been included. for (Map.Entry> category : data.evidenceData.entrySet()) { for (Map.Entry entry : category.getValue().entrySet()) { @@ -461,7 +574,7 @@ protected void writeXmlData(XMLStreamWriter writer, ShareUsageData data) throws writer.writeStartElement(entry.getKey()); } if (replacedString.isReplaced()) { - writer.writeAttribute("replaced", "true"); + writer.writeAttribute("escaped", "true"); } if (replacedString.isTruncated()) { writer.writeAttribute("truncated", "true"); @@ -488,45 +601,55 @@ protected void writeXmlElement(XMLStreamWriter writer, * replace characters that cause problems in XML with the "Replacement character" */ public static class ReplacedString { - // a set of valid XML character values (ignoring valid controls x09, x0a, x0d, x85) - static final Set VALID_XML_CHARS = - Stream.of( IntStream.range(0x20,0x7F), - IntStream.range(0xA0, 0x100)) - .flatMapToInt(i -> i) - .boxed() - .collect(Collectors.toCollection(HashSet::new)); - // an array describing whether a character value is valid - static final Boolean[] IS_VALID_XML_CHAR = IntStream - .range(0, 0x100) - .mapToObj(VALID_XML_CHARS::contains) - .toArray(Boolean[]::new); - - // maximum string length + // maximum string length, in characters static final int MAX_LENGTH = SHARE_USAGE_MAX_EVIDENCE_LENGTH; private boolean replaced; private boolean truncated; private final StringBuilder builder; + /** + * Whether the character is allowed in an XML 1.0 document, as the + * Char production in section 2.2 of the XML 1.0 specification says. + * @param codePoint the character to check + * @return true if the character is allowed in XML 1.0 + */ + static boolean isXmlChar(int codePoint) { + return codePoint == 0x9 || codePoint == 0xA || codePoint == 0xD || + (codePoint >= 0x20 && codePoint <= 0xD7FF) || + (codePoint >= 0xE000 && codePoint <= 0xFFFD) || + (codePoint >= 0x10000 && codePoint <= 0x10FFFF); + } + + /** + * Keep the first {@link #MAX_LENGTH} characters of the text, and write + * each character XML 1.0 does not allow as a backslash, an x and the + * four lower-case hexadecimal digits of its code. The limit counts + * characters, so a pair of surrogates is never split. + * @param text the text to prepare + */ public ReplacedString(String text) { if (Objects.isNull(text) || text.length() == 0) { builder = new StringBuilder(0); - // Using 'return' statement in a constructor throws compilation - // error in Jdk 8. - // return; } else { - builder = new StringBuilder(Math.min(text.length(), MAX_LENGTH)); - truncated = text.length() > MAX_LENGTH; - text.chars() - .limit(MAX_LENGTH) - .forEach(c -> { - if (c < IS_VALID_XML_CHAR.length && IS_VALID_XML_CHAR[c]) { - builder.append((char) c); - } else { - builder.append((char) 0xFFFD); - replaced = true; - } - }); + int end = text.length(); + if (text.codePointCount(0, end) > MAX_LENGTH) { + end = text.offsetByCodePoints(0, MAX_LENGTH); + truncated = true; + } + builder = new StringBuilder(end); + int index = 0; + while (index < end) { + int codePoint = text.codePointAt(index); + if (isXmlChar(codePoint)) { + builder.appendCodePoint(codePoint); + } else { + builder.append("\\x").append( + String.format("%04x", codePoint)); + replaced = true; + } + index += Character.charCount(codePoint); + } } } @@ -555,6 +678,9 @@ protected void legacySendAsXML(List allData) throws Exception { } HttpURLConnection connection = httpClient.connect(new URI(shareUsageUrl.trim()).toURL()); + // A send that is never answered must end, or it stops all sending. + connection.setConnectTimeout(httpSendTimeout); + connection.setReadTimeout(httpSendTimeout); String response = httpClient.postData(connection, headers, baos.toByteArray()); int responseCode = connection.getResponseCode(); String responseMessage = connection.getResponseMessage() + "data: '" + response + "'"; diff --git a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java index 33514287d..8bfa52d3b 100644 --- a/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java +++ b/pipeline/pipeline.engines.fiftyone/src/main/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTracker.java @@ -22,16 +22,23 @@ package fiftyone.pipeline.engines.fiftyone.trackers; +import fiftyone.pipeline.core.data.DataKey; +import fiftyone.pipeline.core.data.DataKeyBuilder; +import fiftyone.pipeline.core.data.DataKeyBuilderDefault; import fiftyone.pipeline.core.data.EvidenceKeyFilter; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.engines.configuration.CacheConfiguration; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; import fiftyone.pipeline.engines.trackers.TrackerBase; import java.util.Date; +import java.util.Map; /** * A tracker used by share usage to attempt to avoid repeatedly sending data - * relating to the same user session. + * relating to the same user session. Evidence that matches an entry is shared + * again once the interval has passed since it was last shared, so a repeat is + * shared at most once in each interval. * @see Specification */ public class ShareUsageTracker extends TrackerBase { @@ -50,7 +57,8 @@ public class ShareUsageTracker extends TrackerBase { * Construct a new instance. * @param configuration the cache configuration to use when building the * internal cache used by this tracker - * @param intervalMillis interval between matched in millis + * @param intervalMillis how long after evidence is shared a repeat of it is + * not shared, in milliseconds * @param filter the {@link EvidenceKeyFilter} that defines the evidence * values to use when creating a key from an {@link FlowData} */ @@ -65,18 +73,47 @@ public ShareUsageTracker( @Override protected Date newValue(FlowData data) { - // Stored meta-data values for this tracker are always true. + // The time the new evidence is shared. return new Date(); } @Override protected boolean match(FlowData data, Date value) { - Date now = new Date(); - return value.getTime() <= (now.getTime() - interval); + // Sharing a repeat stores the time in the cached value, so the next + // interval starts from this share. + long now = System.currentTimeMillis(); + synchronized (value) { + if (value.getTime() <= now - interval) { + value.setTime(now); + return true; + } + return false; + } } @Override protected EvidenceKeyFilter getFilter() { return filter; } + + /** + * The key is made from the values {@link ShareUsageEvidence#find} finds + * that the filter includes, which take in the web request's values, + * as the session tracking hash is of the values that would be shared. + * The Evidence alone does not hold them. + */ + @Override + protected DataKey generateKey(FlowData data) { + DataKeyBuilder builder = new DataKeyBuilderDefault(); + for (Map.Entry entry : + ShareUsageEvidence.find(data.getEvidence()).entrySet()) { + if (filter.include(entry.getKey())) { + builder.add( + filter.order(entry.getKey()), + entry.getKey(), + entry.getValue()); + } + } + return builder.build(); + } } diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java new file mode 100644 index 000000000..33eacefc6 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/EvidenceKeyFilterShareUsageTests.java @@ -0,0 +1,237 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.data; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests which evidence {@link EvidenceKeyFilterShareUsage} lets usage sharing + * keep and send. + */ +public class EvidenceKeyFilterShareUsageTests { + + private static EvidenceKeyFilterShareUsage filter( + List blockedHttpHeaders, + List includedQueryStringParams, + boolean includeSession) { + return new EvidenceKeyFilterShareUsage( + blockedHttpHeaders, + includedQueryStringParams, + includeSession, + DEFAULT_SESSION_COOKIE_NAME); + } + + private static EvidenceKeyFilterShareUsage defaults() { + return filter( + Collections.emptyList(), + Collections.emptyList(), + false); + } + + /** + * By default headers other than Cookie are shared, cookies, session + * values and query string parameters only when their names start with + * 51D_, and any other evidence. + */ + /** + * The whole web request is read by the Usage Sharing Element and never + * shared as a value, so the filter does not include it. + */ + @Test + public void webRequestIsNeverIncluded() { + EvidenceKeyFilterShareUsage filter = defaults(); + + assertFalse(filter.include("server.http-request")); + assertFalse(filter.include("Server.Http-Request")); + } + + @Test + public void defaultRules() { + EvidenceKeyFilterShareUsage filter = defaults(); + + assertTrue(filter.include("header.user-agent")); + assertFalse(filter.include("header.cookie")); + assertTrue(filter.include("cookie.51d_screenpixelswidth")); + assertTrue(filter.include("cookie.51D_ScreenPixelsWidth")); + assertFalse(filter.include("cookie.sessionid")); + assertFalse(filter.include("cookie.a.b")); + assertTrue(filter.include("session.51d_value")); + assertFalse(filter.include("session.org.example.attribute")); + assertTrue(filter.include("query.51d_gethighentropyvalues")); + assertTrue(filter.include("query.51D_Test")); + assertFalse(filter.include("query.password")); + assertFalse(filter.include("query.id.email")); + assertTrue(filter.include("server.client-ip")); + assertTrue(filter.include("nodot")); + } + + /** + * A name containing a dot is matched as a whole against the parameters to + * share. + */ + @Test + public void fieldIsEverythingAfterTheFirstDot() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.emptyList(), + Collections.singletonList("valid.email"), + false); + + assertTrue(filter.include("query.valid.email")); + assertFalse(filter.include("query.valid")); + assertFalse(filter.include("query.other.name")); + } + + /** + * The headers not to share and the parameters to share are matched + * without regard to case. + */ + @Test + public void listsIgnoreCase() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.singletonList("X-Secret"), + Collections.singletonList("Campaign"), + false); + + assertFalse(filter.include("header.x-secret")); + assertTrue(filter.include("query.campaign")); + assertTrue(filter.include("header.user-agent")); + } + + /** + * The session cookie is included only when asked for, whatever the case of + * its name, and no other ordinary cookie is. + */ + @Test + public void sessionCookie() { + EvidenceKeyFilterShareUsage kept = filter( + Collections.emptyList(), + Collections.emptyList(), + true); + EvidenceKeyFilterShareUsage sent = defaults(); + + assertTrue(kept.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertTrue(kept.include("cookie.jsessionid")); + assertFalse(sent.include("cookie." + DEFAULT_SESSION_COOKIE_NAME)); + assertFalse(kept.include("cookie._ga")); + assertFalse(sent.include("cookie._ga")); + } + + /** + * The email address, the salt for hashed email identifiers, the query + * string and the path are never shared, whether bare or after a prefix, + * and without regard to case, even when the rules for their prefixes + * would share them. + */ + @Test + public void neverShared() { + List filters = Arrays.asList( + defaults(), + filter( + Collections.emptyList(), + Arrays.asList("id.email", "id.salt"), + true)); + List keys = Arrays.asList( + "id.email", "id.salt", + "query.id.email", "QUERY.ID.EMAIL", + "header.id.email", "header.id.salt", "HEADER.ID.SALT", + "server.request-query", "SERVER.REQUEST-QUERY", + "server.request-path", "SERVER.REQUEST-PATH"); + for (EvidenceKeyFilterShareUsage filter : filters) { + for (String key : keys) { + assertFalse(filter.include(key), key); + } + } + } + + /** + * The Cookie header carries every cookie, so it is never shared, whether + * bare or after a prefix, and without regard to case. + */ + @Test + public void cookieHeaderNeverShared() { + EvidenceKeyFilterShareUsage filter = defaults(); + for (String key : Arrays.asList( + "header.cookie", "HEADER.COOKIE", "x.header.cookie", + "server.header.cookie")) { + assertFalse(filter.include(key), key); + } + assertTrue(filter.include("header.cookies-allowed")); + assertTrue(filter.include("server.request-method")); + } + + /** + * A key that only ends with the same letters as a key never shared is + * treated as an ordinary key. + */ + @Test + public void neverSharedKeysMatchWholeParts() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.emptyList(), + Collections.singletonList("valid.email"), + false); + + assertTrue(filter.include("query.valid.email")); + } + + /** + * Sharing one parameter does not share the whole query string, which + * carries every parameter. + */ + @Test + public void namingAParameterDoesNotShareTheQueryString() { + EvidenceKeyFilterShareUsage filter = filter( + Collections.emptyList(), + Collections.singletonList("utm_source"), + false); + + assertTrue(filter.include("query.utm_source")); + assertFalse(filter.include("server.request-query")); + } + + /** + * The tracker leaves out the session id and sequence number, which differ + * on every request, and follows the same rules otherwise. + */ + @Test + public void trackerLeavesOutSessionIdAndSequence() { + EvidenceKeyFilterShareUsageTracker tracker = + new EvidenceKeyFilterShareUsageTracker( + Collections.emptyList(), + Arrays.asList("session-id", "sequence"), + false, + DEFAULT_SESSION_COOKIE_NAME); + + assertFalse(tracker.include("query.session-id")); + assertFalse(tracker.include("query.sequence")); + assertTrue(tracker.include("header.user-agent")); + assertFalse(tracker.include("query.id.email")); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidenceTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidenceTests.java new file mode 100644 index 000000000..fee9b7269 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/data/ShareUsageEvidenceTests.java @@ -0,0 +1,117 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.data; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +/** + * The usage sharing element finds the values it shares on every request, so + * with no web request finding them reads the Evidence rather than copying + * every value of it. + */ +public class ShareUsageEvidenceTests { + + private static Pipeline createPipeline() throws Exception { + return new PipelineBuilder(LoggerFactory.getILoggerFactory()) + .addFlowElement(new EmptyEngineBuilder( + LoggerFactory.getILoggerFactory()).build()) + .setAutoCloseElements(true) + .build(); + } + + /** + * With no web request the values found are the Evidence itself, so a + * value added afterwards is among them. A copy would not hold it. + */ + @Test + public void Find_NoWebRequest_ReadsTheEvidenceWithoutCopying() + throws Exception { + try (Pipeline pipeline = createPipeline(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "evidence-ua"); + Map found = + ShareUsageEvidence.find(data.getEvidence()); + data.addEvidence("header.x-later", "later"); + + assertEquals("later", found.get("header.x-later"), + "The Evidence was copied, so every request pays for a copy " + + "of every Evidence value."); + } + } + + /** + * With no web request the values found are still keyed without regard + * to case. + */ + @Test + public void Find_NoWebRequest_KeyedWithoutRegardToCase() + throws Exception { + try (Pipeline pipeline = createPipeline(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.User-Agent", "evidence-ua"); + Map found = + ShareUsageEvidence.find(data.getEvidence()); + + assertEquals("evidence-ua", found.get("HEADER.USER-AGENT")); + assertEquals(1, found.size()); + } + } + + /** + * With a web request its values are added under their own keys, the + * Evidence value is kept for a key in both, and the web request itself + * is not one of the values. + */ + @Test + public void Find_WebRequest_AddsItsValuesAndKeepsTheEvidenceValue() + throws Exception { + Map request = new HashMap<>(); + request.put("header.User-Agent", "request-ua"); + request.put("header.X-Test", "shared"); + try (Pipeline pipeline = createPipeline(); + FlowData data = pipeline.createFlowData()) { + data.addEvidence("header.user-agent", "evidence-ua"); + data.addEvidence(ShareUsageEvidence.HTTP_REQUEST_KEY, + Collections.unmodifiableMap(request)); + Map found = + ShareUsageEvidence.find(data.getEvidence()); + + assertEquals("evidence-ua", found.get("HEADER.USER-AGENT")); + assertEquals("shared", found.get("header.x-test")); + assertFalse(found.containsKey(ShareUsageEvidence.HTTP_REQUEST_KEY)); + assertEquals(2, found.size()); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneDegreesOnPremiseAspectEngineBuilderBaseTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneDegreesOnPremiseAspectEngineBuilderBaseTests.java index c1ff832dc..bb7231527 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneDegreesOnPremiseAspectEngineBuilderBaseTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/FiftyOneDegreesOnPremiseAspectEngineBuilderBaseTests.java @@ -22,12 +22,15 @@ package fiftyone.pipeline.engines.fiftyone.flowelements; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.data.AspectData; import fiftyone.pipeline.engines.data.AspectEngineDataFile; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; import fiftyone.pipeline.engines.data.AspectPropertyMetaData; import fiftyone.pipeline.engines.fiftyone.data.FiftyOneDataFile; import fiftyone.pipeline.engines.services.DataUpdateService; +import org.mockito.ArgumentCaptor; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.invocation.InvocationOnMock; @@ -38,7 +41,10 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; @@ -118,6 +124,7 @@ public void DataFile_SetEngine() throws Exception { ImplementedBuilder builder = new ImplementedBuilder( LoggerFactory.getILoggerFactory(), updateService); + builder.setAutoUpdate(false); builder.build("", false); assertNotNull(engine.getDataFileMetaData()); @@ -145,4 +152,124 @@ public void DataFile_UrlFormatter() throws Exception { "https://distributor.51degrees.com/api/v2/download?LicenseKeys="+licenseKey+"&Download=True&Type="+downloadType, engine.getDataFileMetaData().getFormattedUrl()); } + + private ImplementedBuilder newBuilder() { + return new ImplementedBuilder( + LoggerFactory.getILoggerFactory(), + updateService); + } + + /** + * The data file is registered with the update service when update on + * start-up is on, even though auto update is off. + */ + @Test + public void UpdateOnStartup_AutoUpdateOff_DataFileRegistered() + throws Exception { + ImplementedBuilder builder = newBuilder() + .setAutoUpdate(false) + .setDataUpdateOnStartup(true) + .setDataUpdateLicenseKey(licenseKey); + + builder.build("", false); + + verify(updateService, times(1)).registerDataFile(any()); + } + + /** + * The engine does not exist when the data file is registered, so the + * data file carries the engine's type name for the update service's log. + */ + @Test + public void UpdateOnStartup_DataFileCarriesTheEngineTypeName() + throws Exception { + ImplementedBuilder builder = newBuilder() + .setAutoUpdate(false) + .setDataUpdateOnStartup(true) + .setDataUpdateLicenseKey(licenseKey); + + builder.build("", false); + + ArgumentCaptor registered = + ArgumentCaptor.forClass(AspectEngineDataFile.class); + verify(updateService).registerDataFile(registered.capture()); + assertEquals("Implemented", + ((AspectEngineDataFileDefault) registered.getValue()) + .getEngineTypeName()); + } + + @Test + public void AutoUpdateAndUpdateOnStartupOff_DataFileNotRegistered() + throws Exception { + ImplementedBuilder builder = newBuilder() + .setAutoUpdate(false) + .setDataUpdateOnStartup(false); + + builder.build("", false); + + verify(updateService, never()).registerDataFile(any()); + } + + /** + * The default update URL is the Distributor's, which serves nothing + * without a license key. + */ + @Test + public void AutoUpdate_DefaultUrlNoLicenseKey_BuildRefused() { + ImplementedBuilder builder = newBuilder().setAutoUpdate(true); + + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + () -> builder.build("", false)); + + assertTrue(e.getMessage().contains("setAutoUpdate(false)")); + assertFalse(e.getMessage().contains("setDataUpdateOnStartup(false)")); + verify(updateService, never()).registerDataFile(any()); + } + + @Test + public void UpdateOnStartup_DefaultUrlNoLicenseKey_BuildRefused() { + ImplementedBuilder builder = newBuilder() + .setAutoUpdate(false) + .setDataUpdateOnStartup(true); + + PipelineConfigurationException e = assertThrows( + PipelineConfigurationException.class, + () -> builder.build("", false)); + + assertTrue(e.getMessage().contains("setDataUpdateOnStartup(false)")); + assertFalse(e.getMessage().contains("setAutoUpdate(false)")); + } + + @Test + public void AutoUpdate_LicenseKeySet_BuildAllowed() throws Exception { + ImplementedBuilder builder = newBuilder() + .setAutoUpdate(true) + .setDataUpdateLicenseKey(licenseKey); + + builder.build("", false); + + verify(updateService, times(1)).registerDataFile(any()); + } + + @Test + public void AutoUpdate_OtherUrlNoLicenseKey_BuildAllowed() + throws Exception { + ImplementedBuilder builder = newBuilder() + .setAutoUpdate(true) + .setDataUpdateUrl("https://example.com/data.file"); + + builder.build("", false); + + verify(updateService, times(1)).registerDataFile(any()); + } + + @Test + public void AutoUpdateOff_NoLicenseKey_BuildAllowed() throws Exception { + ImplementedBuilder builder = newBuilder().setAutoUpdate(false); + + builder.build("", false); + + verify(updateService, never()).registerDataFile(any()); + } } diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ReplacedStringTest.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ReplacedStringTest.java index c98691cca..df4d534bb 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ReplacedStringTest.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ReplacedStringTest.java @@ -39,7 +39,7 @@ */ class ReplacedStringTest { - // is transparent to legal characters + // is transparent to the three control characters XML 1.0 allows @Test public void testNormal(){ String aString = IntStream.of(0x9, 0xA, 0xD) @@ -47,8 +47,8 @@ public void testNormal(){ .collect(joining()); ReplacedString replacedString = new ReplacedString(aString); assertFalse(replacedString.isTruncated()); - assertTrue(replacedString.isReplaced()); - assertEquals(3, replacedString.toString().length()); + assertFalse(replacedString.isReplaced()); + assertEquals(aString, replacedString.toString()); } // is transparent to legal characters except \ @Test @@ -83,14 +83,74 @@ public void testNull(){ assertEquals(0, escapedString.toString().length()); assertEquals("", escapedString.toString()); } - // truncates longer than 512 + // truncates longer than 10,000 characters @Test public void testTruncate(){ - String aString = IntStream.range(0,513).mapToObj(i -> "a").collect(joining()); + String aString = repeat("a", 10001); + ReplacedString replacedString = new ReplacedString(aString); + assertTrue(replacedString.isTruncated()); + assertFalse(replacedString.isReplaced()); + assertEquals(10000, replacedString.toString().length()); + } + // keeps a value of exactly 10,000 characters whole + @Test + public void testNotTruncatedAtTheLimit(){ + String aString = repeat("a", 10000); + ReplacedString replacedString = new ReplacedString(aString); + assertFalse(replacedString.isTruncated()); + assertEquals(aString, replacedString.toString()); + } + // the limit counts characters, so a character made of a pair of + // surrogates at the limit is kept whole and the next one is cut + @Test + public void testTruncateNeverSplitsASurrogatePair(){ + String emoji = "\uD83D\uDE00"; + String aString = repeat("a", 9999) + emoji + "b"; ReplacedString replacedString = new ReplacedString(aString); assertTrue(replacedString.isTruncated()); assertFalse(replacedString.isReplaced()); - assertEquals(512, replacedString.toString().length()); + assertEquals(repeat("a", 9999) + emoji, replacedString.toString()); + } + // a pair that would start after the limit is cut with the rest + @Test + public void testTruncateBeforeASurrogatePair(){ + String emoji = "\uD83D\uDE00"; + String aString = repeat("a", 10000) + emoji; + ReplacedString replacedString = new ReplacedString(aString); + assertTrue(replacedString.isTruncated()); + assertEquals(repeat("a", 10000), replacedString.toString()); + } + // counts characters and not UTF-16 code units + @Test + public void testLimitCountsCharactersNotCodeUnits(){ + String emoji = "\uD83D\uDE00"; + ReplacedString atLimit = new ReplacedString(repeat(emoji, 10000)); + assertFalse(atLimit.isTruncated()); + assertEquals(20000, atLimit.toString().length()); + ReplacedString over = new ReplacedString(repeat(emoji, 10001)); + assertTrue(over.isTruncated()); + assertEquals(repeat(emoji, 10000), over.toString()); + } + // the limit counts the value as supplied, so the escapes written for + // invalid characters do not count towards it + @Test + public void testEscapesDoNotCountTowardsTheLimit(){ + String aString = "a" + (char) 0x1 + repeat("b", 9998); + ReplacedString replacedString = new ReplacedString(aString); + assertFalse(replacedString.isTruncated()); + assertTrue(replacedString.isReplaced()); + assertEquals(10000 - 1 + 6, replacedString.toString().length()); + // One character more is cut, whatever the escapes add. + ReplacedString over = new ReplacedString(aString + "c"); + assertTrue(over.isTruncated()); + assertEquals(10000 - 1 + 6, over.toString().length()); + } + private static String repeat(String text, int times) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < times; i++) { + builder.append(text); + } + return builder.toString(); } // escapes single digit Hex @Test @@ -99,8 +159,7 @@ public void testEscapedSmall(){ ReplacedString replacedString = new ReplacedString(aString); assertFalse(replacedString.isTruncated()); assertTrue(replacedString.isReplaced()); - System.out.println(replacedString); - assertEquals(5, replacedString.toString().length()); + assertEquals(repeat("\\x0001", 5), replacedString.toString()); } // escapes 2 digit hex @Test @@ -109,8 +168,7 @@ public void testEscapedBigger(){ ReplacedString replacedString = new ReplacedString(aString); assertFalse(replacedString.isTruncated()); assertTrue(replacedString.isReplaced()); - System.out.println(replacedString); - assertEquals(5, replacedString.toString().length()); + assertEquals(repeat("\\x001f", 5), replacedString.toString()); } // does not escape valid chars @Test @@ -123,15 +181,50 @@ public void testEscapedBiggerStill(){ assertEquals(5, replacedString.toString().length()); } - @Test () - public void testEscapedEvenBigger(){ - String aString = IntStream.range(0,5).mapToObj(i -> "\u0100").collect(joining()); + // does not replace characters above U+00FF, which XML 1.0 allows + @Test + public void testNonLatinKept(){ + String aString = "\u0100 \u0416 \u6F22\u5B57 \uD7FF \uE000 \uFFFD"; ReplacedString replacedString = new ReplacedString(aString); assertFalse(replacedString.isTruncated()); - assertTrue(replacedString.isReplaced()); - System.out.println(replacedString); - assertEquals(5, replacedString.toString().length()); - + assertFalse(replacedString.isReplaced()); + assertEquals(aString, replacedString.toString()); + } + // does not replace the controls from U+007F to U+009F, which XML 1.0 allows + @Test + public void testC1ControlsKept(){ + String aString = IntStream.range(0x7F, 0xA0) + .mapToObj(i -> Character.toString((char)i)) + .collect(joining()); + ReplacedString replacedString = new ReplacedString(aString); + assertFalse(replacedString.isReplaced()); + assertEquals(aString, replacedString.toString()); + } + // keeps a pair of surrogates, which is one character above U+FFFF + @Test + public void testSurrogatePairKept(){ + String aString = "a\uD83D\uDE00b"; + ReplacedString replacedString = new ReplacedString(aString); + assertFalse(replacedString.isReplaced()); + assertEquals(aString, replacedString.toString()); + } + // writes a surrogate on its own, U+FFFE and U+FFFF as \x and four + // lower-case hexadecimal digits + @Test + public void testForbiddenCharactersEscaped(){ + String[][] cases = { + {"\uD83D", "\\xd83d"}, {"\uDE00", "\\xde00"}, + {"\uFFFE", "\\xfffe"}, {"\uFFFF", "\\xffff"}, + {"\u0000", "\\x0000"}, {"\u000B", "\\x000b"}}; + for (String[] c : cases) { + ReplacedString replacedString = new ReplacedString("a" + c[0] + "b"); + assertTrue(replacedString.isReplaced()); + assertEquals("a" + c[1] + "b", replacedString.toString()); + } + // A low surrogate before a high one is two surrogates on their own. + ReplacedString reversed = new ReplacedString("\uDE00\uD83D"); + assertTrue(reversed.isReplaced()); + assertEquals("\\xde00\\xd83d", reversed.toString()); } private final XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance(); @@ -140,12 +233,12 @@ public void testEscapedEvenBigger(){ @Test public void testXmlSerialise() throws XMLStreamException { xmlOutputFactory.setProperty("escapeCharacters", true); - String text = ReplacedString.VALID_XML_CHARS.stream() - .map(i->Character.toString((char)i.intValue())) - .collect(joining()); + String text = "\t\n a\u00E9\u0100\u0416\u6F22\u5B57\uD83D\uDE00" + + "\u007F\u0080\u009F\uD7FF\uE000\uFFFD <&>"; + assertEquals(text, new ReplacedString(text).toString()); ByteArrayOutputStream os = new ByteArrayOutputStream(); XMLStreamWriter writer = xmlOutputFactory.createXMLStreamWriter(os, "UTF-8"); - writer.writeStartDocument("UTF-8", "1.1"); + writer.writeStartDocument("UTF-8", "1.0"); writer.writeStartElement("test"); writer.writeCharacters(text); writer.writeEndElement(); @@ -155,8 +248,6 @@ public void testXmlSerialise() throws XMLStreamException { XMLStreamReader reader = xmlInputFactory.createXMLStreamReader(is); reader.next(); String readText = reader.getElementText(); - System.out.println(text); - System.out.println(readText); reader.close(); assertEquals(readText, text); diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElementTests.java new file mode 100644 index 000000000..9243c966a --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SequenceElementTests.java @@ -0,0 +1,256 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The session id and sequence are written to Element Data as well as + * Evidence. + * @see Specification + */ +public class SequenceElementTests { + + private static final String SESSION_ID_EVIDENCE = "query.session-id"; + private static final String SEQUENCE_EVIDENCE = "query.sequence"; + + private Pipeline newPipeline() throws Exception { + ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory(); + return new PipelineBuilder(loggerFactory) + .addFlowElement(new SequenceElementBuilder(loggerFactory).build()) + .build(); + } + + private ElementData sequenceData(FlowData flowData) throws Exception { + ElementData data = flowData.get("sequence"); + assertNotNull(data, "Expected Element Data under 'sequence'"); + return data; + } + + @Test + public void NoEvidence_NewSessionAndSequenceOne() throws Exception { + try (Pipeline pipeline = newPipeline(); + FlowData flowData = pipeline.createFlowData()) { + flowData.process(); + + ElementData data = sequenceData(flowData); + assertNotNull(data.get("session-id")); + assertEquals(1, data.get("sequence")); + assertEquals( + data.get("session-id"), + flowData.getEvidence().asKeyMap().get(SESSION_ID_EVIDENCE)); + assertEquals( + 1, + flowData.getEvidence().asKeyMap().get(SEQUENCE_EVIDENCE)); + } + } + + @Test + public void EvidenceGiven_SessionKeptAndSequenceIncreased() + throws Exception { + try (Pipeline pipeline = newPipeline(); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence(SESSION_ID_EVIDENCE, "abc"); + flowData.addEvidence(SEQUENCE_EVIDENCE, 2); + flowData.process(); + + ElementData data = sequenceData(flowData); + assertEquals("abc", data.get("session-id")); + assertEquals(3, data.get("sequence")); + assertEquals( + 3, + flowData.getEvidence().asKeyMap().get(SEQUENCE_EVIDENCE)); + } + } + + @Test + public void Properties_AreSessionIdAndSequence() throws Exception { + try (Pipeline pipeline = newPipeline()) { + SequenceElement element = + (SequenceElement) pipeline.getFlowElements().get(0); + List names = new ArrayList<>(); + for (ElementPropertyMetaData property : element.getProperties()) { + names.add(property.getName()); + } + assertTrue(names.contains("session-id")); + assertTrue(names.contains("sequence")); + assertEquals(2, names.size()); + } + } + + /** + * Process with the given Evidence values, where null means none, and + * check that the Evidence and the Element Data hold the same values. + * @return the session id and the sequence + */ + private Object[] run(Object sessionId, Object sequence) throws Exception { + try (Pipeline pipeline = newPipeline(); + FlowData flowData = pipeline.createFlowData()) { + if (sessionId != null) { + flowData.addEvidence(SESSION_ID_EVIDENCE, sessionId); + } + if (sequence != null) { + flowData.addEvidence(SEQUENCE_EVIDENCE, sequence); + } + flowData.process(); + + ElementData data = sequenceData(flowData); + Object evidenceSession = + flowData.getEvidence().asKeyMap().get(SESSION_ID_EVIDENCE); + Object evidenceSequence = + flowData.getEvidence().asKeyMap().get(SEQUENCE_EVIDENCE); + assertEquals(evidenceSession, data.get("session-id")); + assertEquals(evidenceSequence, data.get("sequence")); + return new Object[] {evidenceSession, evidenceSequence}; + } + } + + private void assertNewSession(Object sessionId, Object sequence, + int expectedSequence) throws Exception { + Object[] result = run(sessionId, sequence); + assertTrue(result[0] instanceof String); + assertFalse(((String) result[0]).isEmpty()); + if (sessionId != null) { + assertNotEquals(sessionId, result[0]); + } + assertEquals(expectedSequence, result[1]); + } + + private void assertSequence(Object sequence, int expected) + throws Exception { + Object[] result = run("abc", sequence); + assertEquals("abc", result[0]); + assertEquals(expected, result[1]); + } + + @Test + public void SequenceAsText_Digits_CountsUp() throws Exception { + assertSequence("2", 3); + assertSequence("0", 1); + assertSequence("00012", 13); + } + + @Test + public void SequenceAsOtherWholeNumbers_CountsUp() throws Exception { + assertSequence(2L, 3); + assertSequence((short) 2, 3); + assertSequence(new java.math.BigInteger("2"), 3); + } + + @Test + public void SequenceNotAWholeNumber_CountsAsAbsent() throws Exception { + assertSequence("abc", 1); + assertSequence("", 1); + assertSequence(" 2", 1); + assertSequence("2 ", 1); + assertSequence("-2", 1); + assertSequence("+2", 1); + assertSequence("2.5", 1); + assertSequence(2.5, 1); + assertSequence(new Object(), 1); + } + + /** + * A sequence runs from 0 up, and as text is the digits 0 to 9 alone, + * so a negative number, a sign or white space counts as absent. + */ + @Test + public void SequenceNegativeSignedOrSpaced_CountsAsAbsent() + throws Exception { + assertSequence(-1, 1); + assertSequence(-1L, 1); + assertSequence((short) -1, 1); + assertSequence((byte) -1, 1); + assertSequence(new java.math.BigInteger("-1"), 1); + assertSequence(Integer.MIN_VALUE, 1); + assertSequence("-1", 1); + assertSequence("+5", 1); + assertSequence(" 5", 1); + assertSequence("5 ", 1); + assertSequence("\t5", 1); + assertSequence("5\n", 1); + assertSequence("5 6", 1); + } + + @Test + public void SequenceZero_GivesOne() throws Exception { + assertSequence(0, 1); + assertSequence(0L, 1); + assertSequence("0", 1); + assertSequence("000", 1); + } + + @Test + public void SequenceOutOfRange_CountsAsAbsent() throws Exception { + assertSequence("99999999999999999999", 1); + assertSequence("2147483648", 1); + assertSequence(2147483648L, 1); + // One more would not fit in an int either. + assertSequence("2147483647", 1); + assertSequence(2147483647, 1); + assertSequence("2147483646", 2147483647); + } + + @Test + public void SessionIdNotANonEmptyString_CountsAsAbsent() throws Exception { + assertNewSession("", 2, 3); + assertNewSession(5, 2, 3); + assertNewSession(new Object(), null, 1); + } + + @Test + public void SessionIdZero_Kept() throws Exception { + Object[] result = run("0", 4); + assertEquals("0", result[0]); + assertEquals(5, result[1]); + } + + @Test + public void SequenceWithoutSessionId_CountsUpWithNewSession() + throws Exception { + assertNewSession(null, 4, 5); + } + + @Test + public void SessionIdWithoutSequence_GivesOne() throws Exception { + Object[] result = run("abc", null); + assertEquals("abc", result[0]); + assertEquals(1, result[1]); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java index cd25d63ac..93029030b 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/SetHeadersElementTests.java @@ -49,8 +49,11 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** @@ -234,6 +237,79 @@ public void setHeadersElement_MultipleProperties_SameHeader() throws Exception { } } + /** + * Once every element of the Pipeline has loaded its properties, the + * configuration built from the Pipeline's available properties is kept, + * so later requests do not build it again. + */ + @Test + public void setHeadersElement_ConfigKeptOnceLoaded() throws Exception { + FlowElement source = mock(FlowElement.class); + when(source.hasLoadedProperties()).thenReturn(true); + Pipeline mockPipeline = mockPipeline(source); + createElement(); + + for (int i = 0; i < 3; i++) { + element.processInternal(mockFlowData(mockPipeline)); + } + + verify(mockPipeline, times(1)).getElementAvailableProperties(); + } + + /** + * While an element has not loaded its properties, as a cloud element + * after a failure to reach its service when the Pipeline was built, the + * configuration is built again on each request, and kept once every + * element has loaded. + */ + @Test + public void setHeadersElement_ConfigBuiltAgainUntilLoaded() throws Exception { + FlowElement source = mock(FlowElement.class); + when(source.hasLoadedProperties()).thenReturn(false, true); + Pipeline mockPipeline = mockPipeline(source); + createElement(); + + for (int i = 0; i < 4; i++) { + element.processInternal(mockFlowData(mockPipeline)); + } + + verify(mockPipeline, times(2)).getElementAvailableProperties(); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static Pipeline mockPipeline(FlowElement source) { + Pipeline mockPipeline = mock(Pipeline.class); + List elements = new ArrayList<>(); + elements.add(source); + when(mockPipeline.getFlowElements()).thenReturn(elements); + when(mockPipeline.getElementAvailableProperties()).thenReturn( + new HashMap>()); + return mockPipeline; + } + + private static FlowData mockFlowData(Pipeline mockPipeline) { + FlowData data = mock(FlowData.class); + when(data.getPipeline()).thenReturn(mockPipeline); + when(data.getOrAdd(anyString(), any())).thenReturn(mock(SetHeadersData.class)); + return data; + } + + private void createElement() { + element = new SetHeadersElement( + loggerFactory.getLogger(SetHeadersElement.class.getName()), + new ElementDataFactory() { + @Override + public SetHeadersData create( + FlowData flowData, + FlowElement flowElement) { + return new SetHeadersData( + loggerFactory.getLogger( + SetHeadersData.class.getName()), + flowData); + } + }); + } + private void createPipeline( Map propertyNameValues) throws Exception { sourceElement = new ActivePropertySourceElement( diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageAcceptedEvidenceTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageAcceptedEvidenceTests.java new file mode 100644 index 000000000..6fae6d5ae --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageAcceptedEvidenceTests.java @@ -0,0 +1,74 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngine; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The usage sharing element accepts no Evidence, so the Pipeline's accepted + * Evidence, from which cache keys, the ETag and the Vary header are made, is + * that of its other elements alone. + */ +public class ShareUsageAcceptedEvidenceTests { + + @Test + public void Pipeline_WithShareUsage_AcceptsOnlyTheOtherElementsEvidence() + throws Exception { + ShareUsageElement shareUsage = new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()) + .setSharePercentage(0) + .build(); + EmptyEngine engine = new EmptyEngineBuilder( + LoggerFactory.getILoggerFactory()) + .build(); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(engine) + .addFlowElement(shareUsage) + .setAutoCloseElements(true) + .build()) { + EvidenceKeyFilter accepted = pipeline.getEvidenceKeyFilter(); + + assertTrue(accepted.include("test.value"), + "The other element's Evidence is not accepted."); + assertFalse(accepted.include("header.if-none-match"), + "If-None-Match is accepted, so a revalidation's ETag differs " + + "from the one it sends."); + assertFalse(accepted.include("header.x-test"), + "A header no element uses is accepted."); + assertFalse(accepted.include("server.http-request"), + "The web request is accepted."); + assertFalse(shareUsage.getEvidenceKeyFilter().include("header.user-agent"), + "The usage sharing element accepts Evidence."); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java new file mode 100644 index 000000000..1a628235c --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageBackOffTests.java @@ -0,0 +1,199 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The rules that decide when usage sharing may send after failed sends, + * with each test setting the time, and the builder settings for them. + */ +public class ShareUsageBackOffTests { + + private static final long START = 1767225600000L; + + private static long at(double seconds) { + return START + Math.round(seconds * 1000); + } + + /** + * Three failures within a ten second window start a two second pause, + * rising by two each time up to five. + */ + private static ShareUsageBackOff create( + int failuresToEnterRecovery, + double initialDelaySeconds, + double maxDelaySeconds) { + return new ShareUsageBackOff( + failuresToEnterRecovery, + 10, + initialDelaySeconds, + maxDelaySeconds, + 2); + } + + private static ShareUsageBackOff create() { + return create(3, 2, 5); + } + + /** + * The time the back-off is given comes from a clock that a change to the + * system time does not move. + */ + @Test + public void ShareUsageBackOff_Now_FollowsTheMonotonicClock() { + long before = System.nanoTime() / 1000000; + long now = ShareUsageBackOff.now(); + long after = System.nanoTime() / 1000000; + + assertTrue(before <= now && now <= after); + } + + @Test + public void ShareUsageBackOff_PausesAfterFailuresWithinWindow() { + ShareUsageBackOff backOff = create(); + + assertEquals(0, backOff.failed(at(0))); + assertEquals(0, backOff.failed(at(1))); + assertEquals(0, backOff.remaining(at(1)), + "Sending paused before enough sends failed."); + assertFalse(backOff.isBackingOff()); + + assertEquals(2000, backOff.failed(at(2))); + assertTrue(backOff.isBackingOff()); + assertEquals(1000, backOff.remaining(at(3))); + assertEquals(0, backOff.remaining(at(4))); + } + + @Test + public void ShareUsageBackOff_FailuresOutsideWindowDoNotCount() { + ShareUsageBackOff backOff = create(); + + backOff.failed(at(0)); + backOff.failed(at(1)); + + assertEquals(0, backOff.failed(at(11.5)), + "Failures older than the window were counted."); + assertFalse(backOff.isBackingOff()); + } + + /** + * The window counts failures, not failures in a row, so a high failure + * rate pauses sending even when some sends succeed. + */ + @Test + public void ShareUsageBackOff_SuccessesBetweenFailuresDoNotReset() { + ShareUsageBackOff backOff = create(); + + backOff.failed(at(0)); + assertFalse(backOff.succeeded()); + backOff.failed(at(1)); + assertFalse(backOff.succeeded()); + + assertEquals(2000, backOff.failed(at(2))); + } + + @Test + public void ShareUsageBackOff_FailedSendAfterPauseMultipliesDelay() { + ShareUsageBackOff backOff = create(1, 2, 5); + + assertEquals(2000, backOff.failed(at(0))); + assertEquals(4000, backOff.failed(at(2))); + assertEquals(5000, backOff.failed(at(6)), + "The delay grew past the maximum."); + assertEquals(5000, backOff.failed(at(11))); + assertEquals(5000, backOff.remaining(at(11))); + } + + @Test + public void ShareUsageBackOff_SuccessEndsBackOff() { + ShareUsageBackOff backOff = create(2, 2, 5); + backOff.failed(at(0)); + backOff.failed(at(1)); + backOff.failed(at(3)); + + assertTrue(backOff.succeeded(), + "A success did not end the back-off."); + assertFalse(backOff.isBackingOff()); + assertEquals(0, backOff.remaining(at(4))); + + // The next pause needs the full number of failures again, and starts + // from the initial delay. + assertEquals(0, backOff.failed(at(8))); + assertEquals(2000, backOff.failed(at(9))); + } + + @Test + public void ShareUsageBackOff_MaximumBelowInitialLimitsFirstPause() { + ShareUsageBackOff backOff = create(1, 10, 3); + + assertEquals(3000, backOff.failed(at(0))); + } + + @Test + public void ShareUsageBackOff_InvalidSettingsAreRejected() { + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(0, 10, 2, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(101, 10, 2, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 0, 2, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 10, 0, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 10, Double.NaN, 5, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 10, 2, 0, 2)); + assertThrows(IllegalArgumentException.class, + () -> new ShareUsageBackOff(1, 10, 2, 5, 0.5)); + } + + /** + * The builder settings that pause sending after failed sends reject + * values the pause cannot work with. + */ + @Test + public void ShareUsageBuilder_RecoverySettings_RejectValuesOutOfRange() { + ShareUsageBuilder builder = + new ShareUsageBuilder(LoggerFactory.getILoggerFactory()); + + assertThrows(IllegalArgumentException.class, + () -> builder.setFailuresToEnterRecovery(0)); + assertThrows(IllegalArgumentException.class, + () -> builder.setFailuresToEnterRecovery(101)); + assertThrows(IllegalArgumentException.class, + () -> builder.setFailuresWindowSeconds(0)); + assertThrows(IllegalArgumentException.class, + () -> builder.setInitialRecoveryDelaySeconds(0)); + assertThrows(IllegalArgumentException.class, + () -> builder.setMaxRecoveryDelaySeconds(-1)); + assertThrows(IllegalArgumentException.class, + () -> builder.setRecoveryMultiplier(0.5)); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java new file mode 100644 index 000000000..0a6405558 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementBackOffTests.java @@ -0,0 +1,436 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUploader; +import fiftyone.pipeline.engines.trackers.Tracker; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.Map; +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static fiftyone.pipeline.core.Constants.EVIDENCE_CLIENTIP_KEY; +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Usage sharing pauses sending after repeated failed sends, keeps queueing + * evidence while paused, and drains the queue once a send succeeds. These + * tests use the default settings, which pause sending for two seconds after + * ten sends fail within 100 seconds. + */ +public class ShareUsageElementBackOffTests { + + /** + * The longest a test waits for something it expects to happen. + */ + private static final long WAIT_LIMIT_MILLIS = 30000; + + private static final int DEFAULT_FAILURES = 10; + + private static final long DEFAULT_PAUSE_MILLIS = 2000; + + private static final Pattern CLIENT_IP = + Pattern.compile("([^<]*)"); + + /** + * Answers each send with the status the test sets, and records when each + * send was answered and the client IPs of the sends it accepted. + */ + private static class RecordingUploader implements DataUploader { + private final long start = System.nanoTime(); + private final List times = new ArrayList<>(); + private final List accepted = new ArrayList<>(); + private volatile int code = 500; + private volatile String content = ""; + private ByteArrayOutputStream body; + + @Override + public synchronized OutputStream getOutputStream() { + body = new ByteArrayOutputStream(); + return body; + } + + @Override + public synchronized int getResponseCode() { + int answer = code; + times.add(now()); + if (answer == 200) { + Matcher matcher = CLIENT_IP.matcher( + new String(body.toByteArray(), StandardCharsets.UTF_8)); + while (matcher.find()) { + accepted.add(matcher.group(1)); + } + } + return answer; + } + + public String getResponseContent() { + return content; + } + + long now() { + return (System.nanoTime() - start) / 1000000; + } + + synchronized int count() { + return times.size(); + } + + synchronized List times() { + return new ArrayList<>(times); + } + + synchronized List accepted() { + return new ArrayList<>(accepted); + } + } + + /** + * Shares every request. + */ + private static class ShareAll implements Tracker { + @Override + public boolean track(FlowData flowData) { + return true; + } + } + + private Pipeline pipeline; + private Logger logger = LoggerFactory.getLogger(ShareUsageElement.class); + private RecordingUploader uploader; + private ShareUsageElement element; + private int requests; + + /** + * Each entry is sent on its own, as the minimum entries per message is + * one. + */ + @BeforeEach + public void init() throws Exception { + pipeline = new PipelineBuilder().build(); + uploader = new RecordingUploader(); + requests = 0; + element = new ShareUsageElement( + LoggerFactory.getLogger(ShareUsageElement.class), + 1, + 1, + 100, + 100, + 100, + 1, + true, + "http://localhost/usage", + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + DEFAULT_SESSION_COOKIE_NAME, + new ShareAll()); + element.addPipeline(pipeline); + element.dataUploader = uploader; + } + + /** + * Replaces the element with one that has its own back-off settings. The + * failures window is long enough that every failure in a test counts. + */ + private void useBackOff( + int failuresToEnterRecovery, + double initialDelaySeconds, + double maxDelaySeconds, + int maximumQueueSize) throws Exception { + element.close(); + element = new ShareUsageElement( + logger, + 1, + 1, + maximumQueueSize, + 100, + 100, + 1, + true, + "http://localhost/usage", + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + DEFAULT_SESSION_COOKIE_NAME, + new ShareAll(), + failuresToEnterRecovery, + 100, + initialDelaySeconds, + maxDelaySeconds, + 2, + false, + false); + element.addPipeline(pipeline); + element.dataUploader = uploader; + } + + @AfterEach + public void cleanUp() throws Exception { + element.close(); + pipeline.close(); + } + + /** + * Processes a request whose evidence no other request has, and returns + * its client IP. + */ + private String process() throws Exception { + int number = requests++; + String ip = "10.0." + (number / 250) + "." + (number % 250 + 1); + Map evidence = new HashMap<>(); + evidence.put(EVIDENCE_CLIENTIP_KEY, ip); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + element.process(data); + } + return ip; + } + + private static void waitFor( + BooleanSupplier condition, + Supplier message) throws InterruptedException { + long end = System.currentTimeMillis() + WAIT_LIMIT_MILLIS; + while (condition.getAsBoolean() == false) { + if (System.currentTimeMillis() > end) { + fail(message.get()); + } + Thread.sleep(20); + } + } + + /** + * Processes requests one at a time, waiting for each to be sent, so each + * is a separate failed send. + */ + private void failSends(int count) throws Exception { + for (int i = 1; i <= count; i++) { + process(); + final int expected = i; + waitFor( + () -> uploader.count() == expected && + element.isRunning() == false, + () -> "Send " + expected + " was not made."); + } + } + + /** + * Ten failed sends pause sending. Nothing is sent during the pause, the + * evidence processed during it is queued, and all of it is sent once the + * pause ends and the service answers, with no more requests processed to + * start the sending. + */ + @Test + public void ShareUsageElement_BackOff_PausesThenDrainsQueue() + throws Exception { + failSends(DEFAULT_FAILURES); + long pausedAt = uploader.now(); + + List queued = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + queued.add(process()); + } + Thread.sleep(700); + assertEquals(DEFAULT_FAILURES, uploader.count(), + "Sends were made during the pause that ten failed sends should " + + "have started."); + + uploader.code = 200; + waitFor( + () -> uploader.accepted().containsAll(queued), + () -> "The evidence processed during the pause was not all " + + "sent once the service answered. Sent: " + uploader.accepted()); + long waited = uploader.times().get(DEFAULT_FAILURES) - pausedAt; + assertTrue(waited >= DEFAULT_PAUSE_MILLIS - 500, + "Sending resumed " + waited + " ms after the pause started, " + + "before the two second pause ended."); + } + + /** + * Closing while paused returns at once rather than waiting for the pause + * to end, and sends nothing. + */ + @Test + public void ShareUsageElement_BackOff_CloseWhilePausedDoesNotWaitOrSend() + throws Exception { + failSends(DEFAULT_FAILURES); + process(); + process(); + + long start = System.currentTimeMillis(); + element.close(); + long took = System.currentTimeMillis() - start; + + assertTrue(took < 1500, + "Closing took " + took + " ms, as it waited for the pause."); + assertEquals(DEFAULT_FAILURES, uploader.count(), + "A send was made during the pause or when closing."); + } + + /** + * Each failed send after a pause multiplies the delay, and the delay + * stops growing at the maximum. + */ + @Test + public void ShareUsageElement_BackOff_DelayGrowsToMaximum() + throws Exception { + useBackOff(1, 0.5, 1.0, 100); + for (int i = 0; i < 40; i++) { + process(); + } + + waitFor( + () -> uploader.count() >= 4, + () -> "Fewer than four sends were made."); + List times = uploader.times(); + long first = times.get(1) - times.get(0); + long second = times.get(2) - times.get(1); + long third = times.get(3) - times.get(2); + String shown = first + ", " + second + ", " + third + " ms"; + assertTrue(first >= 400, + "The first pause was not the initial delay. Gaps: " + shown); + assertTrue(second >= 900, + "The second pause was not doubled. Gaps: " + shown); + assertTrue(third >= 900 && third < 1900, + "The third pause was not held at the maximum. Gaps: " + shown); + } + + /** + * When the queue fills during a pause, new evidence is dropped and what + * was queued first is sent once sending resumes. + */ + @Test + public void ShareUsageElement_BackOff_FullQueueDropsNewEvidence() + throws Exception { + useBackOff(1, 2, 2, 4); + failSends(1); + + List kept = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + kept.add(process()); + } + List dropped = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + dropped.add(process()); + } + uploader.code = 200; + + waitFor( + () -> uploader.accepted().containsAll(kept), + () -> "The evidence queued during the pause was not all sent. " + + "Sent: " + uploader.accepted()); + Thread.sleep(500); + for (String ip : dropped) { + assertTrue(uploader.accepted().contains(ip) == false, + "Evidence processed when the queue was full was sent."); + } + } + + /** + * A logger that records the warnings it is given. + */ + private static Logger recordingLogger(List warnings) { + return (Logger) Proxy.newProxyInstance( + Logger.class.getClassLoader(), + new Class[] {Logger.class}, + (proxy, method, args) -> { + if (method.getName().equals("warn")) { + warnings.add(Arrays.deepToString(args)); + return null; + } + if (method.getReturnType() == boolean.class) { + return false; + } + if (method.getReturnType() == int.class) { + return 0; + } + if (method.getReturnType() == String.class) { + return "test"; + } + return null; + }); + } + + /** + * Evidence discarded because the queue is full is logged, but not with a + * message for each entry discarded. + */ + @Test + public void ShareUsageElement_FullQueue_LogsWithoutAMessagePerEntry() + throws Exception { + List warnings = new CopyOnWriteArrayList<>(); + logger = recordingLogger(warnings); + useBackOff(1, 2, 2, 4); + failSends(1); + warnings.clear(); + + for (int i = 0; i < 10; i++) { + process(); + } + + long full = warnings.stream() + .filter(warning -> warning.contains("full")) + .count(); + assertTrue(full >= 1, "Discarding entries was not logged."); + assertTrue(full < 6, "A message was logged for each entry discarded."); + } + + /** + * A send answered with anything other than 200 reports the status code + * and the content of the answer. + */ + @Test + public void ShareUsageElement_FailedAnswer_ReportsStatusAndContent() { + uploader.content = "the service says no"; + + Exception failure = assertThrows( + Exception.class, + () -> element.sendAsXML(new ArrayList<>())); + + assertTrue(failure.getMessage().contains("500")); + assertTrue(failure.getMessage().contains("the service says no")); + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java index 176a2a156..a7ecfecbe 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageElementTests.java @@ -40,6 +40,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; +import org.w3c.dom.Node; import org.w3c.dom.NodeList; import javax.xml.parsers.DocumentBuilder; @@ -487,6 +488,91 @@ public void ShareUsageElement_RestrictedHeaders() throws Exception { } } + /** + * The element accepts no Evidence, so a header no other element accepts + * reaches it only through the web request the web integration adds as + * server.http-request. It shares that header under the key it would have + * as Evidence, keeps the Evidence value for a key in both places, and + * still leaves out the cookie header, a blocked header and the web + * request itself. + */ + @Test + public void ShareUsageElement_WebRequest_SharesWhatNoOtherElementAccepts() + throws Exception { + createShareUsage( + 1, + 1, + 1, + new ArrayList<>(Collections.singletonList("x-blocked")), + new ArrayList<>(), + new ArrayList<>()); + + Map request = new HashMap<>(); + request.put("header.User-Agent", "request-ua"); + request.put("header.X-Test", "shared"); + request.put("header.Cookie", "a=1; 51D_Profile=123456"); + request.put("header.X-Blocked", "blocked"); + request.put("cookie.51D_Profile", "123456"); + request.put("cookie.other", "not-shared"); + request.put("query.secret", "not-shared"); + request.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + request.put("server.request-method", "GET"); + request.put("server.request-path", "/a%2Fb/c"); + + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence( + EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR + "user-agent", + "evidence-ua"); + data.addEvidence("server.http-request", + Collections.unmodifiableMap(request)); + shareUsageElement.process(data); + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + + Document doc = docBuilderFactory.newDocumentBuilder().parse( + new ByteArrayInputStream(connector.getBaos().toByteArray())); + + assertEquals("shared", xpath.evaluate( + "//header[@Name='X-Test']", doc), + "A header only the web request holds was not shared."); + NodeList userAgents = (NodeList) xpath.evaluate( + "//header[translate(@Name,'USERAGNT','useragnt')='user-agent']", + doc, XPathConstants.NODESET); + assertEquals(1, userAgents.getLength(), + "The User-Agent in both places was not shared once."); + assertEquals("evidence-ua", userAgents.item(0).getTextContent(), + "The User-Agent in both places did not keep the Evidence value."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//header[@Name='Cookie' or @Name='cookie']", doc, + XPathConstants.NODESET)).getLength(), + "The cookie header was shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//header[@Name='X-Blocked']", doc, + XPathConstants.NODESET)).getLength(), + "A blocked header was shared."); + assertEquals("123456", xpath.evaluate( + "//cookie[@Name='51D_Profile']", doc), + "A 51D_ cookie from the web request was not shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//cookie[@Name='other'] | //query[@Name='secret']", doc, + XPathConstants.NODESET)).getLength(), + "A cookie or query string parameter the rules leave out was shared."); + assertEquals("1.2.3.4", xpath.evaluate("//ClientIP", doc), + "The client IP from the web request was not shared."); + assertEquals("GET", xpath.evaluate( + "//server[@Name='request-method']", doc), + "The method of the request line was not shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//*[@Name='request-path']", doc, + XPathConstants.NODESET)).getLength(), + "The path of the request line was shared."); + assertEquals(0, ((NodeList) xpath.evaluate( + "//*[@Name='http-request']", doc, + XPathConstants.NODESET)).getLength(), + "The web request itself was shared."); + } + } + @Test public void ShareUsageElement_LowPercentage() throws Exception { // Arrange @@ -712,13 +798,109 @@ public void ShareUsageElement_BadSchema() throws Exception { // Check that the expected values are populated. NodeList f = (NodeList) xpath.compile("//header[@Name='user-agent']").evaluate(doc, XPathConstants.NODESET); assertEquals(1, f.getLength(), "1 user-agent header expected"); - assertEquals("", f.item(0).getTextContent()); - assertEquals("true", f.item(0).getAttributes().getNamedItem("replaced").getTextContent()); + assertEquals("", f.item(0).getTextContent()); + assertEquals("true", f.item(0).getAttributes().getNamedItem("escaped").getTextContent()); NodeList b = (NodeList) xpath.compile("//BadSchema").evaluate(doc, XPathConstants.NODESET); assertEquals(0, b.getLength()); } } + /** + * Check that the invalid characters in a value are marked with the + * attribute the specification names, and that a value that is not too + * long is not marked as truncated. + */ + @Test + public void ShareUsageElement_InvalidCharacter_EscapedAttributeOnly() + throws Exception { + Document doc = shareUserAgent(""); + + Node userAgent = doc.getElementsByTagName("header").item(0); + assertEquals("true", + userAgent.getAttributes().getNamedItem("escaped").getTextContent()); + assertNull(userAgent.getAttributes().getNamedItem("replaced")); + assertNull(userAgent.getAttributes().getNamedItem("truncated")); + assertEquals("", userAgent.getTextContent()); + } + + /** + * Check that a value that is too long is cut and marked as truncated, + * without the escaped attribute. + */ + @Test + public void ShareUsageElement_LongValue_TruncatedAttributeOnly() + throws Exception { + StringBuilder longValue = new StringBuilder(); + for (int i = 0; i < 10001; i++) { + longValue.append('a'); + } + Document doc = shareUserAgent(longValue.toString()); + + Node userAgent = doc.getElementsByTagName("header").item(0); + assertEquals("true", + userAgent.getAttributes().getNamedItem("truncated").getTextContent()); + assertNull(userAgent.getAttributes().getNamedItem("escaped")); + assertEquals(10000, userAgent.getTextContent().length()); + } + + /** + * Check that the values are written in the order the specification's + * example gives, and that no CDATA section or BadSchema element is used. + */ + @Test + public void ShareUsageElement_RecordFollowsTheSpecificationExample() + throws Exception { + Document doc = shareUserAgent(""); + + Node device = doc.getElementsByTagName("Device").item(0); + List names = new ArrayList<>(); + for (int i = 0; i < device.getChildNodes().getLength(); i++) { + Node child = device.getChildNodes().item(i); + assertNotEquals(Node.CDATA_SECTION_NODE, child.getNodeType()); + if (child.getNodeType() == Node.ELEMENT_NODE) { + names.add(child.getNodeName()); + } + } + String[] expected = {"SessionId", "Sequence", "DateSent", "Version", + "Product", "FlowElement", "Language", "LanguageVersion", + "ClientIP", "ServerIP", "Platform"}; + List required = Arrays.asList("SessionId", "Sequence", + "DateSent", "Product", "Language", "LanguageVersion"); + int last = -1; + for (String name : expected) { + int index = names.indexOf(name); + if (!required.contains(name)) { + // Only present when the Pipeline, the request and the + // platform give one. + if (index < 0) { + continue; + } + } + assertTrue(index >= 0, name + " missing from " + names); + assertTrue(index > last, name + " out of order in " + names); + last = index; + } + assertEquals(0, doc.getElementsByTagName("BadSchema").getLength()); + assertEquals(1, doc.getElementsByTagName("header").getLength()); + } + + private Document shareUserAgent(String userAgent) throws Exception { + createShareUsage(1, 1, 1, new ArrayList<>(), + new ArrayList<>(), new ArrayList<>()); + Map evidenceData = new HashMap<>(); + evidenceData.put(EVIDENCE_HEADER_USERAGENT_KEY, userAgent); + evidenceData.put("query.session-id", "session"); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + shareUsageElement.process(data); + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + byte[] output = connector.getBaos().toByteArray(); + return docBuilderFactory.newDocumentBuilder() + .parse(new ByteArrayInputStream(output)); + } + } + /** * Test that the ShareUsageElement generates a session id if one is not * contained in the evidence and adds it to the results. @@ -1076,4 +1258,144 @@ public void CheckForContinueAndResumeWhenQueueFull() throws Exception { } } + + /** + * Default settings share headers other than Cookie, the 51Degrees cookies + * and query string parameters, and other evidence, and leave out ordinary + * cookies, other parameters, session values and the evidence that is + * never shared. + */ + @Test + public void ShareUsageElement_DefaultSharesOnlyAcceptedEvidence() throws Exception { + // Arrange + createShareUsage( + 1, + 1, + 1, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + + Map evidenceData = new HashMap<>(); + evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + evidenceData.put("header.user-agent", "some user agent"); + evidenceData.put("cookie.51d_screenpixelswidth", "screen width"); + evidenceData.put("query.51D_Test", "fiftyone parameter"); + evidenceData.put("cookie.sessionid", "sessionid cookie"); + evidenceData.put("header.cookie", "cookie header"); + evidenceData.put("query.password", "password parameter"); + evidenceData.put("query.id.email", "email parameter"); + evidenceData.put("header.id.email", "email header"); + evidenceData.put("id.salt", "salt value"); + evidenceData.put("server.request-query", "whole query string"); + evidenceData.put("server.request-path", "request path"); + evidenceData.put("session.org.example.attribute", "session attribute"); + + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + + // Act + shareUsageElement.process(data); + assertNotNull(shareUsageElement.getSendDataFuture()); + shareUsageElement.getSendDataFuture().get(); + } + + // Assert + String output = new String( + connector.getBaos().toByteArray(), StandardCharsets.UTF_8); + for (String shared : Arrays.asList( + "some user agent", "screen width", "fiftyone parameter")) { + assertTrue(output.contains(shared), shared); + } + for (String withheld : Arrays.asList( + "sessionid cookie", "cookie header", "password parameter", + "email parameter", "email header", "salt value", + "whole query string", "request path", "session attribute")) { + assertFalse(output.contains(withheld), withheld); + } + } + + /** + * The default tracker does not share again a request that differs from + * one already shared only in its session id and sequence number. + */ + @Test + public void ShareUsageElement_RepeatWithNewSequence_NotShared() throws Exception { + // Arrange + tracker = null; + createShareUsage( + 1, + 3, + 1, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + + // Act + for (int sequence = 1; sequence <= 2; sequence++) { + Map evidenceData = new HashMap<>(); + evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + evidenceData.put("header.user-agent", "repeated user agent"); + evidenceData.put(Constants.EVIDENCE_SESSIONID, "session " + sequence); + evidenceData.put(Constants.EVIDENCE_SEQUENCE, sequence); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + shareUsageElement.process(data); + } + } + shareUsageElement.close(); + + // Assert + Document doc = docBuilderFactory.newDocumentBuilder().parse( + new ByteArrayInputStream(connector.getBaos().toByteArray())); + NodeList devices = (NodeList) xpath.compile("//Device") + .evaluate(doc, XPathConstants.NODESET); + assertEquals(1, devices.getLength(), "1 device expected"); + } + + /** + * The session cookie the element names is read by the tracker, whatever + * the case of its name, and never sent. With sessions tracked, the same + * evidence from a new session is shared again. The element accepts no + * Evidence, so the cookie reaches the tracker from what it finds rather + * than through its accepted Evidence. + */ + @Test + public void ShareUsageElement_TrackSession_SessionCookie() throws Exception { + // Arrange + tracker = null; + createShareUsage( + 1, + 3, + 1, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + String sessionCookie = EVIDENCE_COOKIE_PREFIX + EVIDENCE_SEPERATOR + + DEFAULT_SESSION_COOKIE_NAME; + + // Act + for (int session = 1; session <= 2; session++) { + Map evidenceData = new HashMap<>(); + evidenceData.put(EVIDENCE_CLIENTIP_KEY, "1.2.3.4"); + evidenceData.put("header.user-agent", "repeated user agent"); + evidenceData.put(sessionCookie, "session cookie " + session); + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidenceData); + shareUsageElement.process(data); + } + } + shareUsageElement.close(); + + // Assert + assertFalse(shareUsageElement.getEvidenceKeyFilter().include(sessionCookie)); + byte[] output = connector.getBaos().toByteArray(); + assertFalse(new String(output, StandardCharsets.UTF_8).contains( + "session cookie"), "the session cookie is never sent"); + Document doc = docBuilderFactory.newDocumentBuilder().parse( + new ByteArrayInputStream(output)); + NodeList devices = (NodeList) xpath.compile("//Device") + .evaluate(doc, XPathConstants.NODESET); + assertEquals(2, devices.getLength(), "1 device expected for each session"); + } } diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageEvidenceReadTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageEvidenceReadTests.java new file mode 100644 index 000000000..d3d919e73 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageEvidenceReadTests.java @@ -0,0 +1,187 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; +import fiftyone.pipeline.engines.services.DataUploader; +import fiftyone.pipeline.engines.testhelpers.data.MockFlowData; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.util.AbstractMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * Usage sharing is called for every request, and most calls repeat one + * already shared, which the tracker does not share again. Such a call must + * not copy every value of the Evidence, or of the web request, more than + * once. These tests count how often all the values of each are read in a + * call that is not shared. + */ +public class ShareUsageEvidenceReadTests { + + /** + * Enough values that copying them is most of the cost of a call. + */ + private static final int VALUE_COUNT = 100; + + /** + * A map keyed without regard to case and holding no null value, as + * Evidence and the web request are, that counts how often all its + * values are read. + */ + private static final class CountingMap + extends AbstractMap { + + private final Map values = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + + private int reads; + + /** + * @return the reads since this was last called + */ + int takeReads() { + int result = reads; + reads = 0; + return result; + } + + @Override + public Set> entrySet() { + reads++; + return values.entrySet(); + } + + @Override + public Object get(Object key) { + return values.get(key); + } + + @Override + public boolean containsKey(Object key) { + return values.containsKey(key); + } + + @Override + public Object put(String key, Object value) { + return values.put(key, value); + } + + @Override + public int size() { + return values.size(); + } + } + + private static CountingMap headers() { + CountingMap headers = new CountingMap(); + for (int i = 0; i < VALUE_COUNT; i++) { + headers.put("header.x-test-" + i, "value-" + i); + } + return headers; + } + + /** + * An element with the default tracker, sharing every call that is not + * a repeat, and sending nothing until 50 calls are queued. + */ + private static ShareUsageElement createElement() throws Exception { + return new ShareUsageBuilder( + LoggerFactory.getILoggerFactory(), + mock(DataUploader.class)) + .build(); + } + + /** + * With no web request a call that is not shared reads the Evidence once, + * to make the session tracking key, and does not copy it. + */ + @Test + public void NoWebRequest_EvidenceReadOnceWhenNotShared() + throws Exception { + CountingMap evidence = headers(); + ShareUsageElement element = createElement(); + try { + FlowData data = MockFlowData.createFromEvidence(evidence, false); + element.process(data); + assertEquals(1, element.evidenceCollection.size(), + "The first call was not shared."); + + evidence.takeReads(); + element.process(data); + int reads = evidence.takeReads(); + assertEquals(1, element.evidenceCollection.size(), + "The repeat was shared."); + + assertEquals(1, reads, + "A call that is not shared read all the Evidence more than " + + "once, when the session tracking key needs it once."); + } finally { + element.close(); + } + } + + /** + * With a web request a call that is not shared reads the Evidence and + * the web request once each, to make the one copy that the session + * tracking key is made from. + */ + @Test + public void WebRequest_EachCopiedOnceWhenNotShared() throws Exception { + CountingMap request = headers(); + CountingMap evidence = new CountingMap(); + evidence.put("header.user-agent", "evidence-ua"); + evidence.put(ShareUsageEvidence.HTTP_REQUEST_KEY, request); + ShareUsageElement element = createElement(); + try { + FlowData data = MockFlowData.createFromEvidence(evidence, false); + element.process(data); + assertEquals(1, element.evidenceCollection.size(), + "The first call was not shared."); + + evidence.takeReads(); + request.takeReads(); + element.process(data); + int evidenceReads = evidence.takeReads(); + int requestReads = request.takeReads(); + assertEquals(1, element.evidenceCollection.size(), + "The repeat was shared."); + + assertEquals(1, requestReads, + "A call that is not shared copied the web request more " + + "than once."); + assertEquals(1, evidenceReads, + "A call that is not shared copied the Evidence more than " + + "once."); + } finally { + element.close(); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java new file mode 100644 index 000000000..65a920482 --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageOptionsTests.java @@ -0,0 +1,282 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.caching.Cache; +import fiftyone.caching.CacheBuilder; +import fiftyone.caching.LruPutCache; +import fiftyone.pipeline.core.configuration.ElementOptions; +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.configuration.CacheConfiguration; +import fiftyone.pipeline.engines.fiftyone.data.EvidenceKeyFilterShareUsage; +import fiftyone.pipeline.engines.fiftyone.trackers.ShareUsageTracker; +import fiftyone.pipeline.engines.services.DataUploader; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.XPathFactory; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the usage sharing options "Share all Evidence" and "Share all query + * string parameters" from a configuration file, the case of the keys the + * element reads and the number of caches a tracker holds. + * @see Specification + */ +public class ShareUsageOptionsTests { + + /** + * Keeps what the element sends so a test can read it. + */ + private static class CapturingUploader implements DataUploader { + private final ByteArrayOutputStream sent = new ByteArrayOutputStream(); + + @Override + public OutputStream getOutputStream() { + return sent; + } + + @Override + public int getResponseCode() { + return 200; + } + + Document document() throws Exception { + return DocumentBuilderFactory.newInstance() + .newDocumentBuilder() + .parse(new InputSource( + new ByteArrayInputStream(sent.toByteArray()))); + } + } + + /** + * Counts the caches the tracker asks for. + */ + private static class CountingCacheBuilder implements CacheBuilder { + final AtomicInteger built = new AtomicInteger(); + private final LruPutCache.Builder inner = new LruPutCache.Builder(); + + @Override + public Cache build(Cache c, int cacheSize) { + built.incrementAndGet(); + return inner.build(c, cacheSize); + } + } + + /** + * Options for a usage sharing element that sends every request at once, + * as a configuration file holds them. + */ + private static PipelineOptions options(String... nameAndValue) { + ElementOptions element = new ElementOptions(); + element.builderName = "ShareUsage"; + element.buildParameters.put("SharePercentage", "1"); + element.buildParameters.put("MinimumEntriesPerMessage", "1"); + element.buildParameters.put("RepeatEvidenceIntervalMinutes", "0"); + element.buildParameters.put("BlockedHttpHeader", "x-blocked"); + for (int i = 0; i < nameAndValue.length; i += 2) { + element.buildParameters.put(nameAndValue[i], nameAndValue[i + 1]); + } + PipelineOptions options = new PipelineOptions(); + options.elements.add(element); + return options; + } + + private static Map evidence() { + Map evidence = new HashMap<>(); + evidence.put("header.user-agent", "agent"); + evidence.put("header.x-blocked", "blocked"); + evidence.put("header.cookie", "a=b"); + evidence.put("cookie.plain", "cookie"); + evidence.put("session.plain", "session"); + evidence.put("query.plain", "query"); + evidence.put("query.id.email", "a@example.com"); + evidence.put("id.email", "a@example.com"); + evidence.put("id.salt", "salt"); + evidence.put("server.request-path", "/path"); + evidence.put("server.request-query", "?plain=query"); + return evidence; + } + + /** + * Builds the element from the options, gives it the Evidence and returns + * what it sends. + */ + private static Document send( + PipelineOptions options, + Map evidence) throws Exception { + CapturingUploader uploader = new CapturingUploader(); + try (Pipeline pipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .buildFromConfiguration(options)) { + ShareUsageElement element = + pipeline.getElement(ShareUsageElement.class); + assertNotNull(element); + element.dataUploader = uploader; + try (FlowData data = pipeline.createFlowData()) { + data.addEvidence(evidence); + element.process(data); + assertNotNull(element.getSendDataFuture()); + element.getSendDataFuture().get(); + } + } + return uploader.document(); + } + + private static boolean shared( + Document sent, + String category, + String name) throws Exception { + return !XPathFactory.newInstance().newXPath().evaluate( + "//Device/" + category + "[@Name='" + name + "']", + sent).isEmpty(); + } + + @Test + public void defaults_ShareOnlyWhatTheRulesAllow() throws Exception { + Document sent = send(options(), evidence()); + + assertTrue(shared(sent, "header", "user-agent")); + assertFalse(shared(sent, "header", "x-blocked")); + assertFalse(shared(sent, "cookie", "plain")); + assertFalse(shared(sent, "session", "plain")); + assertFalse(shared(sent, "query", "plain")); + } + + @Test + public void shareAllEvidence_FromConfiguration_SharesEverythingButNeverShared() + throws Exception { + Document sent = send(options("ShareAllEvidence", "true"), evidence()); + + assertTrue(shared(sent, "header", "user-agent")); + assertTrue(shared(sent, "header", "x-blocked")); + assertTrue(shared(sent, "cookie", "plain")); + assertTrue(shared(sent, "session", "plain")); + assertTrue(shared(sent, "query", "plain")); + assertFalse(shared(sent, "query", "id.email")); + assertFalse(shared(sent, "id", "email")); + assertFalse(shared(sent, "id", "salt")); + assertFalse(shared(sent, "header", "cookie")); + assertFalse(shared(sent, "server", "request-path")); + assertFalse(shared(sent, "server", "request-query")); + } + + @Test + public void shareAllEvidence_False_FromConfiguration_KeepsTheRules() + throws Exception { + Document sent = send(options("ShareAllEvidence", "false"), evidence()); + + assertFalse(shared(sent, "header", "x-blocked")); + assertFalse(shared(sent, "cookie", "plain")); + assertFalse(shared(sent, "query", "plain")); + } + + @Test + public void shareAllQueryStringParameters_FromConfiguration_SharesEveryParameter() + throws Exception { + Document sent = send( + options("ShareAllQueryStringParameters", "true"), + evidence()); + + assertTrue(shared(sent, "query", "plain")); + assertFalse(shared(sent, "query", "id.email")); + assertFalse(shared(sent, "server", "request-query")); + // The other rules still apply. + assertFalse(shared(sent, "header", "x-blocked")); + assertFalse(shared(sent, "cookie", "plain")); + assertFalse(shared(sent, "session", "plain")); + assertTrue(shared(sent, "header", "user-agent")); + } + + @Test + public void shareAllQueryStringParameters_False_FromConfiguration_KeepsTheRules() + throws Exception { + Document sent = send( + options("ShareAllQueryStringParameters", "false"), + evidence()); + + assertFalse(shared(sent, "query", "plain")); + } + + @Test + public void builder_ShareOptionsDefaultToFalse() { + ShareUsageBuilder builder = new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()); + + assertFalse(builder.isShareAllEvidence()); + assertFalse(builder.isShareAllQueryStringParameters()); + } + + @Test + public void keysAreReadWithoutRegardToCase() throws Exception { + Map evidence = new HashMap<>(); + evidence.put("Server.Client-IP", "1.2.3.4"); + evidence.put("Query.Session-ID", "session-1"); + evidence.put("Query.Sequence", "5"); + + Document sent = send(options(), evidence); + + assertEquals("1.2.3.4", + sent.getElementsByTagName("ClientIP").item(0).getTextContent()); + assertEquals("session-1", + sent.getElementsByTagName("SessionId").item(0).getTextContent()); + assertEquals("5", + sent.getElementsByTagName("Sequence").item(0).getTextContent()); + } + + @Test + public void tracker_BuildsOneCache() { + CountingCacheBuilder builder = new CountingCacheBuilder(); + + try (ShareUsageTracker tracker = new ShareUsageTracker( + new CacheConfiguration(builder, 10), + 1000, + new EvidenceKeyFilterShareUsage( + new ArrayList(), + new ArrayList(), + false, + DEFAULT_SESSION_COOKIE_NAME))) { + assertEquals(1, builder.built.get()); + } catch (java.io.IOException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageSendTimeoutTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageSendTimeoutTests.java new file mode 100644 index 000000000..780cb80bb --- /dev/null +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/flowelements/ShareUsageSendTimeoutTests.java @@ -0,0 +1,133 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.fiftyone.flowelements; + +import fiftyone.pipeline.engines.services.HttpClient; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * A send through an {@link HttpClient} that never answers must end, or it + * would stop all sending. + */ +public class ShareUsageSendTimeoutTests { + + /** + * Opens a connection that records the timeouts set on it. + */ + private static class RecordingHttpClient implements HttpClient { + private HttpURLConnection connection; + + @Override + public HttpURLConnection connect(URL url) { + connection = new HttpURLConnection(url) { + @Override + public int getResponseCode() { + return 200; + } + + @Override + public String getResponseMessage() { + return ""; + } + + @Override + public void disconnect() { } + + @Override + public boolean usingProxy() { + return false; + } + + @Override + public void connect() { } + }; + return connection; + } + + @Override + public String postData( + HttpURLConnection connection, + Map headers, + byte[] data) { + return ""; + } + + @Override + public String getResponseString(HttpURLConnection connection) { + return ""; + } + + @Override + public String getResponseString( + HttpURLConnection connection, + Map headers) { + return ""; + } + } + + @Test + public void ShareUsageElement_LegacySend_ConnectionHasTimeouts() + throws Exception { + RecordingHttpClient client = new RecordingHttpClient(); + ShareUsageElement element = new ShareUsageElement( + LoggerFactory.getLogger(ShareUsageElement.class), + 1, + 1, + 100, + 100, + 100, + 1, + true, + "http://localhost/usage", + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME, + flowData -> true); + element.httpClient = client; + try { + element.legacySendAsXML(new ArrayList<>()); + + assertEquals( + element.httpSendTimeout, + client.connection.getConnectTimeout(), + "The connection has no connect timeout."); + assertEquals( + element.httpSendTimeout, + client.connection.getReadTimeout(), + "The connection has no read timeout."); + assertTrue(element.httpSendTimeout > 0); + } finally { + element.close(); + } + } +} diff --git a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java index 32550b140..eaf8670d8 100644 --- a/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java +++ b/pipeline/pipeline.engines.fiftyone/src/test/java/fiftyone/pipeline/engines/fiftyone/trackers/ShareUsageTrackerTests.java @@ -40,12 +40,29 @@ import static fiftyone.pipeline.core.Constants.*; import static fiftyone.pipeline.engines.Constants.DEFAULT_SESSION_COOKIE_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.any; import static org.mockito.Mockito.when; public class ShareUsageTrackerTests { - private long interval; + /** + * An interval no run of these tests comes near, for the tests that do + * not test expiry, so a slow runner cannot make a repeat count as new. + */ + private static final long NO_EXPIRY_INTERVAL_MILLIS = 60 * 60 * 1000; + + /** + * A short interval, for a test that sleeps past it to test expiry. + */ + private static final long EXPIRY_INTERVAL_MILLIS = 50; + + /** + * Added to the wait for an interval to pass, so that a sleep that returns + * early or a slow clock cannot leave a repeat inside the interval. + */ + private static final long EXPIRY_MARGIN_MILLIS = 250; private EvidenceKeyFilter evidenceKeyFilter; @@ -62,7 +79,6 @@ public void Init() { includedQueryStringParameters = new ArrayList<>(); sessionCookieName = DEFAULT_SESSION_COOKIE_NAME; - interval = 50; evidenceKeyFilter = new EvidenceKeyFilterShareUsage( blockedHttpHeaders, includedQueryStringParameters, @@ -73,7 +89,7 @@ public void Init() { 1000); shareUsageTracker = new ShareUsageTracker( cacheConfig, - interval, + NO_EXPIRY_INTERVAL_MILLIS, evidenceKeyFilter); evidenceData = new HashMap<>(); @@ -102,26 +118,56 @@ public void ShareUsageTracker_RepeatEvidence_BeforeSessionTimeout() { @Test public void ShareUsageTracker_RepeatEvidence_AfterSessionTimeout() throws InterruptedException { + ShareUsageTracker shareUsageTracker = new ShareUsageTracker( + new CacheConfiguration(new LruPutCache.Builder(), 1000), + EXPIRY_INTERVAL_MILLIS, + evidenceKeyFilter); int trackedEvents = 0; for (int i = 0; i < 2; i++) { if (shareUsageTracker.track(data)) { trackedEvents++; } - // Wait some time equal to the interval to elapse. - Thread.sleep(interval); + // Wait for the interval to pass, with a margin. + Thread.sleep(EXPIRY_INTERVAL_MILLIS + EXPIRY_MARGIN_MILLIS); } assertEquals(2, trackedEvents); } + /** + * A repeat is shared at most once in each interval. Sharing a repeat after + * the interval starts a new interval, so a repeat straight after it is not + * shared. + */ @Test - public void ShareUsageTracker_Session_Track() throws InterruptedException { - shareUsageTracker = new ShareUsageTracker( + public void ShareUsageTracker_RepeatEvidence_SharedOncePerInterval() + throws InterruptedException { + long interval = 500; + ShareUsageTracker shareUsageTracker = new ShareUsageTracker( new CacheConfiguration(new LruPutCache.Builder(), 1000), interval, evidenceKeyFilter); + assertTrue(shareUsageTracker.track(data), + "New evidence was not shared."); + Thread.sleep(interval + EXPIRY_MARGIN_MILLIS); + assertTrue(shareUsageTracker.track(data), + "A repeat after the interval was not shared."); + assertFalse(shareUsageTracker.track(data), + "A repeat straight after a repeat was shared was also shared, " + + "so the time it was shared was not stored."); + } + + @Test + public void ShareUsageTracker_Session_Track() { + // The interval does not expire, so only the session cookie keeps the + // two requests apart. + shareUsageTracker = new ShareUsageTracker( + new CacheConfiguration(new LruPutCache.Builder(), 1000), + NO_EXPIRY_INTERVAL_MILLIS, + evidenceKeyFilter); + int trackedEvents = 0; for (int i = 0; i < 2; i++) { final Map evidenceData = new HashMap<>(); @@ -138,7 +184,6 @@ public DataKey answer(InvocationOnMock invocationOnMock) throws Throwable { } } ); - Thread.sleep(interval); if (shareUsageTracker.track(data)) { trackedEvents++; } @@ -156,7 +201,7 @@ public void ShareUsageTracker_Session_DoNotTrack() { sessionCookieName); ShareUsageTracker shareUsageTracker = new ShareUsageTracker( new CacheConfiguration(new LruPutCache.Builder(), 1000), - interval, + NO_EXPIRY_INTERVAL_MILLIS, evidenceKeyFilter); int trackedEvents = 0; @@ -184,6 +229,57 @@ public DataKey answer(InvocationOnMock invocationOnMock) throws Throwable { assertEquals(1, trackedEvents); } + /** + * The tracker's key is of the values usage sharing would share, which + * take in those of the web request. Two requests that differ only in the + * session cookie the web request holds are both tracked. + */ + @Test + public void ShareUsageTracker_WebRequest_SessionCookieTracked() { + assertEquals(2, trackWebRequests( + "cookie." + DEFAULT_SESSION_COOKIE_NAME, "session-0", "session-1")); + } + + /** + * Two requests whose web requests differ only in a value usage sharing + * leaves out, here a cookie other than the session or 51Degrees ones, + * have the same key, so only the first is tracked. + */ + @Test + public void ShareUsageTracker_WebRequest_UnsharedValueIgnored() { + assertEquals(1, trackWebRequests( + "cookie.other", "value-0", "value-1")); + } + + private int trackWebRequests(String key, String... values) { + // An interval no run of these tests comes near, so a slow runner + // cannot make the second request count as new. + long noExpiryIntervalMillis = 60 * 60 * 1000; + ShareUsageTracker tracker = new ShareUsageTracker( + new CacheConfiguration(new LruPutCache.Builder(), 1000), + noExpiryIntervalMillis, + evidenceKeyFilter); + int trackedEvents = 0; + for (String value : values) { + Map request = new HashMap<>(); + request.put(EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR + + "user-agent", "iPhone"); + request.put(key, value); + final Map evidenceData = new HashMap<>(); + evidenceData.put("server.http-request", request); + FlowData data = MockFlowData.createFromEvidence(evidenceData, false); + // A real FlowData makes its key from its Evidence and a filter. + when(data.generateKey(any(EvidenceKeyFilter.class))).then( + invocation -> generateKey( + (EvidenceKeyFilter) invocation.getArgument(0), + evidenceData)); + if (tracker.track(data)) { + trackedEvents++; + } + } + return trackedEvents; + } + public DataKey generateKey( EvidenceKeyFilter filter, Map evidence) { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/caching/DataKeyedCacheBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/caching/DataKeyedCacheBase.java index fd16b2c1b..348f613ac 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/caching/DataKeyedCacheBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/caching/DataKeyedCacheBase.java @@ -70,14 +70,26 @@ public DataKeyedCacheBase(CacheConfiguration configuration) { */ protected abstract EvidenceKeyFilter getFilter(); + /** + * Creates the key the internal cache holds the data of a {@link FlowData} + * under. By default this is the key the {@link FlowData} generates from + * the Evidence that {@link #getFilter()} includes. A class whose key is + * made from other values overrides this method. + * @param flowData the {@link FlowData} to make a key for + * @return the key to use in the internal cache + */ + protected DataKey generateKey(FlowData flowData) { + return flowData.generateKey(getFilter()); + } + @Override public V get(FlowData flowData) { - return internalCache.get(flowData.generateKey(getFilter())); + return internalCache.get(generateKey(flowData)); } @Override public void put(FlowData flowData, V value) { - internalCache.put(flowData.generateKey(getFilter()), value); + internalCache.put(generateKey(flowData), value); } @Override diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java index 61589448b..aeb9a1b29 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/configuration/DataFileConfigurationBuilderBase.java @@ -148,8 +148,9 @@ public TBuilder setAutoUpdate(boolean enabled) { * disk and automatically refresh the engine as soon as the file is updated. * This setting enables/disables that feature. * - * The {@link #setAutoUpdate(boolean)} feature must also be enabled in order - * for the file system watcher to work. + * The file system watcher runs on its own setting, whether + * {@link #setAutoUpdate(boolean)} is on or off, and needs no license key, + * as watching a file downloads nothing. * If the engine is built from a byte[] then this setting does nothing. * @param enabled the cache configuration to use * @return this builder instance diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectEngineDataFileDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectEngineDataFileDefault.java index 100d23207..55ecaa049 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectEngineDataFileDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/data/AspectEngineDataFileDefault.java @@ -41,6 +41,7 @@ public class AspectEngineDataFileDefault implements AspectEngineDataFile { private DataFileConfiguration configuration; private DataUpdateService dataUpdateService = null; private String tempDataDirPath; + private String engineTypeName; @Override public String getIdentifier() { @@ -87,6 +88,23 @@ public void setTempDataFilePath(String path) { this.tempDataFilePath = path; } + /** + * The type name of the engine this data file is for, which the update + * service logs until the engine is set. + * @return the engine type name, or null if it is not known + */ + public String getEngineTypeName() { + return engineTypeName; + } + + /** + * Set the type name of the engine this data file is for. + * @param engineTypeName the engine type name + */ + public void setEngineTypeName(String engineTypeName) { + this.engineTypeName = engineTypeName; + } + @Override public String getTempDataDirPath() { if (tempDataDirPath == null && diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java index 37c9dc7c9..879f55f44 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBase.java @@ -23,6 +23,7 @@ package fiftyone.pipeline.engines.flowelements; import fiftyone.pipeline.annotations.DefaultValue; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.engines.Constants; import fiftyone.pipeline.engines.configuration.DataFileConfiguration; import fiftyone.pipeline.engines.data.AspectEngineDataFile; @@ -181,28 +182,109 @@ protected void preCreateEngine() { // Any files that have the 'update on startup' flag set // will be updated now. // Create the auto-update configuration and register it. + for (DataFileConfiguration dataFileConfig : dataFileConfigs) { + checkUpdateConfiguration(dataFileConfig); + } for (DataFileConfiguration dataFileConfig : dataFileConfigs) { AspectEngineDataFile dataFile = newAspectEngineDataFile(); dataFile.setIdentifier(dataFileConfig.getIdentifier()); dataFile.setConfiguration(dataFileConfig); dataFile.setTempDataDirPath(tempDir); + if (dataFile instanceof AspectEngineDataFileDefault) { + // By convention an engine is named for its builder. + String builderName = getClass().getSimpleName(); + ((AspectEngineDataFileDefault) dataFile).setEngineTypeName( + builderName.endsWith("Builder") ? + builderName.substring( + 0, builderName.length() - "Builder".length()) : + builderName); + } - if (dataFileConfig.getAutomaticUpdatesEnabled()) { - if (dataUpdateService == null) { - throw new RuntimeException( - "Auto update enabled by data update service does not exist. " + - "This can be corrected by passing an DataUpdateService " + - "instance to the engine builder constructor. " + - "If building from configuration, this can be corrected " + - "by adding an DataUpdateService instance to the " + - "the pipeline builder via addService() method."); - } + if (dataFileConfig.getAutomaticUpdatesEnabled() || + dataFileConfig.getUpdateOnStartup() || + isWatched(dataFileConfig)) { dataUpdateService.registerDataFile(dataFile); } dataFiles.add(dataFile); } } + /** + * Whether the data file is to be watched for a new file on disk. The + * file system watcher runs on its own setting and needs neither + * automatic updates nor a license key, only a data update service to run + * it and a data file on disk. + */ + private boolean isWatched(DataFileConfiguration dataFileConfig) { + return dataUpdateService != null && + dataFileConfig.getFileSystemWatcherEnabled() && + dataFileConfig.getDataFilePath() != null && + dataFileConfig.getDataFilePath().isEmpty() == false; + } + + /** + * Refuse a data file configuration that needs a data update service or + * a license key it does not have. Every data file is checked before any + * is registered, so a refused build leaves nothing registered. + */ + private void checkUpdateConfiguration(DataFileConfiguration dataFileConfig) { + boolean autoUpdate = dataFileConfig.getAutomaticUpdatesEnabled(); + boolean updateOnStartup = dataFileConfig.getUpdateOnStartup(); + if (autoUpdate == false && updateOnStartup == false) { + return; + } + String features = (autoUpdate ? "auto update" : "") + + (autoUpdate && updateOnStartup ? " and " : "") + + (updateOnStartup ? "update on startup" : ""); + if (dataUpdateService == null) { + throw new RuntimeException( + "The data update service does not exist but is " + + "needed for " + features + ". " + + "This can be corrected by passing a DataUpdateService " + + "instance to the engine builder constructor. " + + "If building from configuration, this can be corrected " + + "by adding a DataUpdateService instance to the " + + "pipeline builder via addService() method."); + } + if (isLicenseKeyRequired(dataFileConfig) && + hasLicenseKey(dataFileConfig) == false) { + throw new PipelineConfigurationException( + "No license key is set, and the data update URL " + + "serves nothing without one, so the engine cannot " + + "be built with " + features + " on. Set a license " + + "key with setDataUpdateLicenseKey, or turn off " + + (autoUpdate ? "auto update with setAutoUpdate(false)" : "") + + (autoUpdate && updateOnStartup ? " and " : "") + + (updateOnStartup ? + "update on startup with setDataUpdateOnStartup(false)" : "") + + "."); + } + } + + /** + * Whether the data update URL in the configuration serves nothing + * without a license key. Engines whose update source needs a key + * override this, and the engine then refuses to build with a data update + * option on and no key set. + * @param dataFileConfig the data file configuration to check + * @return true if the update source needs a license key + */ + protected boolean isLicenseKeyRequired(DataFileConfiguration dataFileConfig) { + return false; + } + + private static boolean hasLicenseKey(DataFileConfiguration dataFileConfig) { + List keys = dataFileConfig.getDataUpdateLicenseKeys(); + if (keys != null) { + for (String key : keys) { + if (key != null && key.isEmpty() == false) { + return true; + } + } + } + return false; + } + @SuppressWarnings("unchecked") @Override protected void configureEngine(TEngine engine) throws Exception { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java index 4a0cf8667..84df5a4f9 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/flowelements/SingleFileAspectEngineBuilderBase.java @@ -216,8 +216,9 @@ public TBuilder setAutoUpdate(boolean enabled) { * disk and automatically refresh the engine as soon as the file is updated. * This setting enables/disables that feature. *

      - * The {@link #setAutoUpdate(boolean)} feature must also be enabled in order - * for the file system watcher to work. + * The file system watcher runs on its own setting, whether + * {@link #setAutoUpdate(boolean)} is on or off, and needs no license key, + * as watching a file downloads nothing. *

      * If the engine is built from a byte[] then this setting does nothing. *

      diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java index bb262ae78..2ee0231c0 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateService.java @@ -127,7 +127,9 @@ enum AutoUpdateStatus { // The new data file can't be renamed to replace the previous one. AUTO_UPDATE_NEW_FILE_CANT_RENAME, // Refreshing the engine with the new data caused an error to occur. - AUTO_UPDATE_REFRESH_FAILED + AUTO_UPDATE_REFRESH_FAILED, + // An operation could not be completed within the expected time. + AUTO_UPDATE_TIMEOUT } class DataUpdateCompleteArgs { diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java index 7430ea3cc..398bfd9b9 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUpdateServiceDefault.java @@ -36,6 +36,7 @@ import fiftyone.pipeline.engines.services.update.FutureFactory; import fiftyone.pipeline.engines.services.update.FutureFactoryDefault; import fiftyone.pipeline.util.Check; +import fiftyone.pipeline.util.LicenseKeyMask; import jakarta.xml.bind.DatatypeConverter; import org.apache.commons.lang3.ObjectUtils; import org.slf4j.Logger; @@ -44,7 +45,9 @@ import java.io.*; import java.net.HttpURLConnection; import java.net.URI; +import java.net.URLDecoder; import java.nio.file.*; +import java.time.Instant; import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -64,11 +67,50 @@ */ public class DataUpdateServiceDefault implements DataUpdateService { + /** + * The query parameter of a data update URL that holds the license keys, + * as the 51Degrees URL formatter writes it. + */ + private static final String LICENSE_KEYS_PARAMETER = "LicenseKeys"; + + /** + * How long a watched data file is tried before the update gives up with + * {@link AutoUpdateStatus#AUTO_UPDATE_TIMEOUT}, as a copy over it may + * still be in progress. + */ + static final long WATCHED_FILE_OPEN_TIMEOUT_MILLIS = + TimeUnit.MINUTES.toMillis(10); + + /** + * The pause between tries to open a watched data file. + */ + static final long WATCHED_FILE_OPEN_PAUSE_MILLIS = 500; + + /** + * The wait for a watched data file, which tests shorten. + */ + long watchedFileOpenTimeoutMillis = WATCHED_FILE_OPEN_TIMEOUT_MILLIS; + final List configurations = new ArrayList<>(); + /** + * The watch service of each watched data file, held under the + * configuration lock. A watch service runs a thread of its own, so it is + * closed when its data file is unregistered. + */ + private final Map watchServices = + new IdentityHashMap<>(); private final Logger logger; private final boolean closeFutureFactory; private final List onUpdateCompleteList = new ArrayList<>(); private final Object configLock = new Object(); + /** + * Held for every read and write this service makes of a data file's + * future, so a check cannot read or clear its future before + * {@link #registerDataFile} has stored it. Only the future is scheduled, + * cancelled and stored while holding it, so it is never held while + * waiting for the configuration lock or a data file's update lock. + */ + private final Object futureLock = new Object(); /** * HTTP client used to download new data files. */ @@ -183,10 +225,14 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { null; boolean newDataAvailable = false; if (dataFile != null) { - // Only check the file system if the file system watcher - // is not enabled and the engine is using a temporary file. + logger.info("Checking for update"); logger.debug("CheckForUpdate {}", dataFile.getDataFilePath()); - if (dataFile.getConfiguration().getFileSystemWatcherEnabled() == false && + // Compare the data file with the temporary file the engine uses. + // A programmatic update always does, whether or not Automatic + // Update from File is enabled. The timer does only when no file + // system watcher exists to notice a new file. + if ((manualUpdate || + dataFile.getConfiguration().getWatchKey() == null) && dataFile.getDataFilePath() != null && dataFile.getDataFilePath().isEmpty() == false && dataFile.getTempDataFilePath() != null && @@ -207,7 +253,11 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { if (newDataAvailable == false && dataFile.getConfiguration().getDataUpdateUrl() != null && dataFile.getConfiguration().getDataUpdateUrl().isEmpty() == false) { - result = checkForUpdateFromUrl(dataFile); + // A failed update is tried again only when the timer is on. + result = checkForUpdateFromUrl( + dataFile, + manualUpdate == false && + dataFile.getAutomaticUpdatesEnabled()); newDataAvailable = result == AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS || result == AutoUpdateStatus.AUTO_UPDATE_SUCCESS; @@ -218,6 +268,15 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { // yet been refreshed. logger.debug("Calling updatedFileAvailable"); result = updatedFileAvailable(dataFile); + } else if (result == AUTO_UPDATE_SUCCESS) { + // The update was applied in memory. Clear the timer, as + // updatedFileAvailable does, so a new one is registered. + synchronized (futureLock) { + if (dataFile.getFuture() != null) { + dataFile.getFuture().cancel(true); + dataFile.setFuture(null); + } + } } else { // No update available. // If this was a manual call to update then do nothing. @@ -226,22 +285,30 @@ private AutoUpdateStatus checkForUpdate(Object state, boolean manualUpdate) { // This will repeat until the update is acquired. if (manualUpdate == false) { logger.debug("Cancelling and resetting update timer"); - if (dataFile.getFuture() != null) { - dataFile.getFuture().cancel(true); - dataFile.setFuture(futureFactory.schedule( - new Runnable() { - @Override - public void run() { - checkForUpdate(dataFile, false); - } - }, - getInterval(dataFile.getConfiguration()))); + synchronized (futureLock) { + if (dataFile.getFuture() != null) { + dataFile.getFuture().cancel(true); + dataFile.setFuture(futureFactory.schedule( + new Runnable() { + @Override + public void run() { + checkForUpdate(dataFile, false); + } + }, + getIntervalMillis(dataFile.getConfiguration()))); + } } } } - if (manualUpdate == false) { + if (manualUpdate == false || + ((result == AUTO_UPDATE_SUCCESS || + result == AUTO_UPDATE_REFRESH_FAILED) && + dataFile.getIsRegistered())) { // Re-register the engine with the data update service // so it knows when next set of data should be available. + // A manual update of a registered data file does this too, + // whether or not the refresh worked, as a refresh from the + // data file clears the timer, so polling resumes after it. registerDataFile(dataFile); } @@ -257,9 +324,10 @@ public void run() { /** * Called when a data update is available and the file at * engine.DataFilePath contains this new data. - * 1. Refresh the engine. - * 2. Dispose of the existing update timer if there is one. - * 3. Re-register the engine with the update service. + * 1. Dispose of the existing update timer if there is one. + * 2. Refresh the engine. + * Where the data file is registered, the caller then registers it again, + * which sets a new update timer. * * @param dataFile the data file to update * @return {@link AutoUpdateStatus#AUTO_UPDATE_SUCCESS} if the data file was @@ -268,10 +336,12 @@ public void run() { private AutoUpdateStatus updatedFileAvailable(AspectEngineDataFile dataFile) { AspectEngineDataFileDefault aspectDataFile = (AspectEngineDataFileDefault) dataFile; - if (aspectDataFile.getFuture() != null) { - // Dispose of the old timer object - aspectDataFile.getFuture().cancel(true); - aspectDataFile.setFuture(null); + synchronized (futureLock) { + if (aspectDataFile.getFuture() != null) { + // Dispose of the old timer object + aspectDataFile.getFuture().cancel(true); + aspectDataFile.setFuture(null); + } } if (dataFile.getEngine() == null) { @@ -281,15 +351,18 @@ private AutoUpdateStatus updatedFileAvailable(AspectEngineDataFile dataFile) { return AutoUpdateStatus.AUTO_UPDATE_SUCCESS; } + logger.info("Attempting to refresh Engine '{}' with new data", + getEngineType(dataFile)); // Try to update the file multiple times to ensure the file is not locked. int tries = 0; + Exception lastFailure = null; while (tries < 10) { try { dataFile.getEngine().refreshData(dataFile.getIdentifier()); return AUTO_UPDATE_SUCCESS; } catch (Exception ex) { - logger.warn("File Update: Error applying a data update to engine '{}'", - dataFile.getEngine().getClass().getSimpleName(), ex); + lastFailure = ex; + logger.debug("Refresh attempt {} failed", tries + 1, ex); try { Thread.sleep(200); } catch (InterruptedException ignored) { @@ -297,6 +370,7 @@ private AutoUpdateStatus updatedFileAvailable(AspectEngineDataFile dataFile) { } tries++; } + logRefreshFailure(dataFile, lastFailure, tries); return AUTO_UPDATE_REFRESH_FAILED; } @@ -337,17 +411,41 @@ private void dataFileUpdated(Object sender) { // before notifying the engine, otherwise the copy // may still be in progress. boolean fileLockable = false; - while (fileLockable == false) { + boolean interrupted = false; + long deadline = System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos( + watchedFileOpenTimeoutMillis); + while (fileLockable == false && + System.nanoTime() - deadline < 0) { try (BinaryReader reader = fileWrapperFactory.build( aspectDataFile.getDataFilePath()).getReader()) { fileLockable = true; - // Complete the update - status = updatedFileAvailable(dataFile); - } catch (IOException e) { - logger.warn("Exception while reading the data file at '{}'", - aspectDataFile.getDataFilePath(), e); + } catch (IOException ignored) { + try { + Thread.sleep(WATCHED_FILE_OPEN_PAUSE_MILLIS); + } catch (InterruptedException ex) { + // Interrupted, as unregistering the data + // file does, so stop trying. + Thread.currentThread().interrupt(); + interrupted = true; + break; + } + } + } + if (fileLockable) { + // Complete the update + status = updatedFileAvailable(dataFile); + // The refresh cancelled the polling timer, so + // register the file again to set a new one. + if (dataFile.getIsRegistered()) { + registerDataFile(dataFile); } + } else if (interrupted == false) { + // The polling timer is left as it was. + status = AutoUpdateStatus.AUTO_UPDATE_TIMEOUT; } + // An interrupted wait leaves the update unfinished, + // so the status stays AUTO_UPDATE_IN_PROGRESS. } } } @@ -368,22 +466,19 @@ public AutoUpdateStatus updateFromMemory(AspectEngineDataFile dataFile, byte[] d .writeBytes(data); } catch (Exception ex) { logger.error("An error occurred when writing to " + - "'" + dataFile.getDataFilePath() + - "'. The engine will be updated to use the new data " + - "but the file on disk will still contain old data.", - ex); + dataFile.getDataFilePath() + + ". Error detail: " + ex.getMessage(), ex); } } try { // Refresh the engine using the new data. + logger.info("Attempting to refresh Engine '{}' with new data", + getEngineType(dataFile)); dataFile.getEngine().refreshData(dataFile.getIdentifier(), data); result = AUTO_UPDATE_SUCCESS; } catch (Exception ex) { - logger.error("An error occurred when applying a " + - "data update to engine '" + dataFile.getEngine().getClass().getSimpleName() + - "'.", - ex); + logRefreshFailure(dataFile, ex, null); result = AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED; } @@ -404,6 +499,7 @@ public void registerDataFile(final AspectEngineDataFile dataFile) { // We also want to do this synchronously so that execution // will block until the engine is ready. if (dataFile.getConfiguration().getUpdateOnStartup() && isFalse(alreadyRegistered)) { + logger.info("Updating on start-up"); logger.debug("Checking for Update on Startup {}", dataFile.getDataFilePath()); AutoUpdateStatus status = checkForUpdate(dataFile, false); if (ObjectUtils.notEqual(status, AUTO_UPDATE_SUCCESS) && @@ -417,47 +513,56 @@ public void registerDataFile(final AspectEngineDataFile dataFile) { // Only create an automatic update timer if auto updates are // enabled for this engine and there is not already an associated - // timer. - if (aspectDataFile.getAutomaticUpdatesEnabled() && - aspectDataFile.getFuture() == null) { - long delay = getInterval(aspectDataFile.getConfiguration()); - if (aspectDataFile.getUpdateAvailableTime() != null) { - if (aspectDataFile.getUpdateAvailableTime().getTime() - > System.currentTimeMillis()) { - delay = applyIntervalRandomisation( - aspectDataFile.getUpdateAvailableTime().getTime() - - System.currentTimeMillis(), - aspectDataFile.getConfiguration()); + // timer. The lock is held from scheduling to storing, because + // the check can start as soon as it is scheduled and must find + // its own future. + synchronized (futureLock) { + if (aspectDataFile.getAutomaticUpdatesEnabled() && + aspectDataFile.getFuture() == null) { + long delayMillis = + getIntervalMillis(aspectDataFile.getConfiguration()); + if (aspectDataFile.getUpdateAvailableTime() != null) { + if (aspectDataFile.getUpdateAvailableTime().getTime() + > System.currentTimeMillis()) { + delayMillis = applyIntervalRandomisationMillis( + aspectDataFile.getUpdateAvailableTime().getTime() + - System.currentTimeMillis(), + aspectDataFile.getConfiguration()); + } + } + // Create a timer that will go off when the engine expects + // updated data to be available. + logger.debug("Scheduling update check with delay {} millis", + delayMillis); + try { + ScheduledFuture future = futureFactory.schedule( + new Runnable() { + @Override + public void run() { + logger.debug("running checkForUpdate {}", + aspectDataFile.getDataFilePath()); + checkForUpdate(aspectDataFile, false); + logger.debug("Returning from checkForUpdate"); + } + }, + delayMillis); + logger.debug("Update check scheduled for {} millis", + future.getDelay(TimeUnit.MILLISECONDS)); + aspectDataFile.setFuture(future); + } catch (Throwable t) { + logger.error("Exception when scheduling update check", t); } - } - // Create a timer that will go off when the engine expects - // updated data to be available. - logger.debug("Scheduling update check with delay {}", delay); - try { - ScheduledFuture future = futureFactory.schedule( - new Runnable() { - @Override - public void run() { - logger.debug("running checkForUpdate {}", - aspectDataFile.getDataFilePath()); - checkForUpdate(aspectDataFile, false); - logger.debug("Returning from checkForUpdate"); - } - }, - delay); - logger.debug("Update check scheduled for {} millis", - future.getDelay(TimeUnit.MILLISECONDS)); - aspectDataFile.setFuture(future); - } catch (Throwable t) { - logger.error("Exception when scheduling update check", t); } } - // If file system watcher is enabled then set it up. + // If file system watcher is enabled then set it up. The watcher + // runs on its own setting. Watching a file downloads nothing, so + // it needs neither automatic updates nor a license key. if (aspectDataFile.getConfiguration().getFileSystemWatcherEnabled() && aspectDataFile.getConfiguration().getWatchKey() == null && aspectDataFile.getDataFilePath() != null && aspectDataFile.getDataFilePath().isEmpty() == false) { + logger.info("Creating file system watcher"); logger.debug("Initialising watch service"); final Path aspectDataFilePath = Paths.get(aspectDataFile.getDataFilePath()).toAbsolutePath(); @@ -469,9 +574,8 @@ public void run() { throw new IllegalStateException("Cannot find directory to watch"); } logger.debug("Getting a watcher using {}", aspectDataFilePath); - WatchService watcher = aspectDataFilePath - .getFileSystem() - .newWatchService(); + WatchService watcher = + watchService(aspectDataFile, aspectDataFilePath); logger.debug("Setting a watcher for {}", aspectDataFilePath.getParent()); aspectDataFile.getConfiguration().setWatchKey(aspectDataFilePath.getParent().register( watcher, @@ -480,33 +584,49 @@ public void run() { logger.error("File watcher for '{}' could not be initialised. " + e.getMessage(), aspectDataFile.getDataFilePath(), e); } - try { - long pollingInterval = - aspectDataFile.getConfiguration().getPollingIntervalSeconds() * 1000L; - logger.debug("Setting poll future {} millis", pollingInterval); - aspectDataFile.setPollFuture(futureFactory.scheduleRepeating( - new Runnable() { - @Override - public void run() { - logger.debug("Watcher is running"); - for (WatchEvent event : - aspectDataFile.getConfiguration().getWatchKey().pollEvents()) { - if (event.kind() == StandardWatchEventKinds.ENTRY_MODIFY) { - if (((Path) event.context()).endsWith(aspectDataFilePath.getFileName())) { - dataFileUpdated(aspectDataFile); + // The watcher's task polls the watch key, so it is scheduled + // only when there is one, and replaces any task scheduled + // before rather than run beside it. + if (aspectDataFile.getConfiguration().getWatchKey() != null) { + try { + if (aspectDataFile.getPollFuture() != null) { + aspectDataFile.getPollFuture().cancel(false); + } + long pollingInterval = + aspectDataFile.getConfiguration().getPollingIntervalSeconds() * 1000L; + logger.debug("Setting poll future {} millis", pollingInterval); + aspectDataFile.setPollFuture(futureFactory.scheduleRepeating( + new Runnable() { + @Override + public void run() { + logger.debug("Watcher is running"); + // The watch is paused while a downloaded + // file is copied into place. A repeating + // task that fails never runs again, so + // this run does nothing. + WatchKey watchKey = + aspectDataFile.getConfiguration().getWatchKey(); + if (watchKey == null) { + return; + } + for (WatchEvent event : watchKey.pollEvents()) { + if (event.kind() == StandardWatchEventKinds.ENTRY_MODIFY) { + if (((Path) event.context()).endsWith(aspectDataFilePath.getFileName())) { + dataFileUpdated(aspectDataFile); + } } } + // make sure to reset the watch key or we are not called again + watchKey.reset(); } - // make sure to reset the watch key or we are not called again - aspectDataFile.getConfiguration().getWatchKey().reset(); - } - }, - pollingInterval)); - logger.debug("Poll check scheduled for {} mins", - aspectDataFile.getPollFuture().getDelay(TimeUnit.MINUTES)); - } catch (Exception e) { - logger.error("Could not schedule watcher for {}.", - aspectDataFile.getDataFilePath(), e); + }, + pollingInterval)); + logger.debug("Poll check scheduled for {} mins", + aspectDataFile.getPollFuture().getDelay(TimeUnit.MINUTES)); + } catch (Exception e) { + logger.error("Could not schedule watcher for {}.", + aspectDataFile.getDataFilePath(), e); + } } } @@ -531,52 +651,145 @@ public void unregisterDataFile(AspectEngineDataFile dataFile) { } AspectEngineDataFileDefault aspectDataFile = (AspectEngineDataFileDefault) dataFile; + // The watcher's task is stopped even while the watch is + // paused for a download. + if (aspectDataFile.getPollFuture() != null) { + aspectDataFile.getPollFuture().cancel(true); + aspectDataFile.setPollFuture(null); + } if (aspectDataFile.getConfiguration().getWatchKey() != null) { - if (aspectDataFile.getPollFuture() != null) { - aspectDataFile.getPollFuture().cancel(true); - aspectDataFile.setPollFuture(null); - } aspectDataFile.getConfiguration().getWatchKey().cancel(); aspectDataFile.getConfiguration().setWatchKey(null); } - if (aspectDataFile.getFuture() != null) { - aspectDataFile.getFuture().cancel(true); - aspectDataFile.setFuture(null); + synchronized (futureLock) { + if (aspectDataFile.getFuture() != null) { + aspectDataFile.getFuture().cancel(true); + aspectDataFile.setFuture(null); + } } } + closeWatchService(dataFile); + } + if (dataFile != null) { + dataFile.setDataUpdateService(null); } } /** - * Get the random delay in milliseconds to add on to the scheduled time - * for a specific Engine, no greater than the Engine's max randomisation. + * Get the data file's watch service, creating it the first time. The + * data file keeps the one watch service until it is unregistered, so the + * watch resumes on it after a download. * - * @param config to get the delay for - * @return a random delay in milliseconds. + * @param dataFile the data file to watch + * @param path the absolute path of the data file + * @return the data file's watch service + * @throws IOException if a watch service cannot be created */ - private long getInterval(DataFileConfiguration config) { - int seconds = 0; - if (config.getPollingIntervalSeconds() > 0) { - seconds = config.getPollingIntervalSeconds(); + private WatchService watchService( + AspectEngineDataFile dataFile, + Path path) throws IOException { + synchronized (configLock) { + WatchService watcher = watchServices.get(dataFile); + if (watcher == null) { + watcher = path.getFileSystem().newWatchService(); + watchServices.put(dataFile, watcher); + } + return watcher; } - return applyIntervalRandomisation(seconds, config) * 1000; + } + /** + * Close the data file's watch service, if it has one, which ends the + * watch service's thread. + * + * @param dataFile the data file no longer watched + */ + private void closeWatchService(AspectEngineDataFile dataFile) { + WatchService watcher; + synchronized (configLock) { + watcher = watchServices.remove(dataFile); + } + if (watcher != null) { + try { + watcher.close(); + } catch (IOException e) { + logger.warn("Could not close the file watcher for '{}'", + dataFile.getDataFilePath(), e); + } + } } /** - * Add a random amount of time to the specified interval + * Get the polling interval for a data file plus a random number of + * seconds, no greater than the configured maximum randomisation. * - * @param interval the interval to add a random amount of time to - * @param config the {@link DataFileConfiguration} object that specifies the - * maximum number of seconds to add - * @return the new interval + * @param config to get the interval for + * @return the interval in milliseconds. */ - private long applyIntervalRandomisation(long interval, DataFileConfiguration config) { - long seconds = 0; + private long getIntervalMillis(DataFileConfiguration config) { + long intervalMillis = 0; + if (config.getPollingIntervalSeconds() > 0) { + intervalMillis = config.getPollingIntervalSeconds() * 1000L; + } + return applyIntervalRandomisationMillis(intervalMillis, config); + } + + /** + * Add a random number of whole seconds, up to the maximum in the + * configuration, to the specified interval. + * + * @param intervalMillis the interval in milliseconds to add to + * @param config the {@link DataFileConfiguration} object that + * specifies the maximum number of seconds to add + * @return the new interval in milliseconds + */ + private long applyIntervalRandomisationMillis( + long intervalMillis, + DataFileConfiguration config) { + long randomSeconds = 0; if (config.getMaxRandomisationSeconds() > 0) { - seconds = random.nextInt(config.getMaxRandomisationSeconds()); + randomSeconds = random.nextInt(config.getMaxRandomisationSeconds()); } - return interval + seconds; + return intervalMillis + randomSeconds * 1000; + } + + /** + * The license keys to mask in messages that name the data update URL, + * being the data update license keys and any key in the URL's + * LicenseKeys parameter, such as one typed straight into the data + * update URL. + * @param dataFile the data file being updated + * @param url the formatted data update URL + * @return the keys to mask + */ + private static List licenseKeysToMask( + AspectEngineDataFile dataFile, + String url) { + List keys = new ArrayList<>(); + List configured = + dataFile.getConfiguration().getDataUpdateLicenseKeys(); + if (configured != null) { + keys.addAll(configured); + } + int query = url == null ? -1 : url.indexOf('?'); + if (query < 0) { + return keys; + } + for (String parameter : url.substring(query + 1).split("&")) { + int equals = parameter.indexOf('='); + if (equals > 0 && parameter.substring(0, equals) + .equalsIgnoreCase(LICENSE_KEYS_PARAMETER)) { + String value = parameter.substring(equals + 1); + try { + value = URLDecoder.decode(value, "UTF-8"); + } catch (UnsupportedEncodingException | + IllegalArgumentException e) { + // Masked as it is written. + } + keys.addAll(Arrays.asList(value.split("\\|"))); + } + } + return keys; } /** @@ -584,32 +797,39 @@ private long applyIntervalRandomisation(long interval, DataFileConfiguration con * If the data currently used by the engine is the newest available then * nothing will be downloaded. * - * @param dataFile the data file to use - * @param tempFile the temp file to write the data to + * @param dataFile the data file to use + * @param tempFile the temp file to write the data to + * @param automatic true if the update will be tried again if it fails * @return the {@link AutoUpdateStatus} value indicating the result */ private AutoUpdateStatus downloadFile( AspectEngineDataFile dataFile, - FileWrapper tempFile) { + FileWrapper tempFile, + boolean automatic) { String url = dataFile.getFormattedUrl(); - logger.debug("downloadFile from {}", url); + String engineType = getEngineType(dataFile); + // The URL carries the license keys, so every message names it with + // each key masked. + List licenseKeys = licenseKeysToMask(dataFile, url); + String loggedUrl = LicenseKeyMask.maskIn(url, licenseKeys); + logger.debug("downloadFile from {}", loggedUrl); try { + logger.info("Checking for update from '{}' for Engine '{}'", + loggedUrl, engineType); HttpURLConnection connection = httpClient.connect(new URI(url.trim()).toURL()); if (connection == null) { - logger.error("No response from data update service at '{}' for engine '{}'", - dataFile.getFormattedUrl(), Objects.nonNull(dataFile.getEngine()) ? - dataFile.getEngine().getClass().getSimpleName() : "No Engine"); + logger.warn(connectFailure(loggedUrl, engineType, automatic, + "No response from the data update service")); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } try { // Set last-modified header to ensure that a file will only // be downloaded if it is newer than the data we already have. + long ifModifiedSince = 0; if (dataFile.getConfiguration().getVerifyModifiedSince() == true) { - long ifModifiedSince = dataFile - .getDataPublishedDateTime() - .getTime(); + ifModifiedSince = getIfModifiedSince(dataFile); connection.setIfModifiedSince(ifModifiedSince); } @@ -617,37 +837,134 @@ private AutoUpdateStatus downloadFile( switch (connection.getResponseCode()) { case (HttpURLConnection.HTTP_OK): - return downloadOkResponse(connection, dataFile, tempFile); + AutoUpdateStatus downloaded; + try { + downloaded = downloadOkResponse( + connection, dataFile, tempFile, automatic); + } catch (Exception e) { + // An HTTP library's exception often names the + // URL it requested. + logger.warn(downloadFailure(loggedUrl, engineType, + automatic, LicenseKeyMask.maskIn( + e.getMessage(), licenseKeys)), + LicenseKeyMask.maskIn(e, licenseKeys)); + return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; + } + if (downloaded == AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS) { + logger.info("Downloaded new data from '{}' for Engine '{}'", + loggedUrl, engineType); + } + return downloaded; // Note: needed because TooManyRequests is not available // in some versions of the HttpStatusCode enum. case (429): - logger.error("Too many requests to '{}' for {}", - dataFile.getFormattedUrl(), getIdForLogging(dataFile)); + logger.warn(downloadFailure(loggedUrl, engineType, automatic, + "HTTP status code 429 (too many requests)")); return AutoUpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS; case HttpURLConnection.HTTP_NOT_MODIFIED: - logger.debug("No data update available from '{}' for {}", - dataFile.getFormattedUrl(), getIdForLogging(dataFile)); + logger.info("No data newer than {} found at '{}' for Engine '{}'", + Instant.ofEpochMilli(ifModifiedSince), loggedUrl, + engineType); return AutoUpdateStatus.AUTO_UPDATE_NOT_NEEDED; case HttpURLConnection.HTTP_FORBIDDEN: - logger.error("Access denied to data update service at '{}' for {}", - dataFile.getFormattedUrl(), getIdForLogging(dataFile)); + logger.warn(downloadFailure(loggedUrl, engineType, automatic, + "HTTP status code 403 (access denied)")); return AutoUpdateStatus.AUTO_UPDATE_ERR_403_FORBIDDEN; default: - logger.error("HTTP status code '{}' from data update service at '{}' for {}", - connection.getResponseCode(), dataFile.getFormattedUrl(), - getIdForLogging(dataFile)); + logger.warn(downloadFailure(loggedUrl, engineType, automatic, + "HTTP status code " + connection.getResponseCode())); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } } finally { connection.disconnect(); } } catch (Exception e) { - logger.error("Error while processing data file download", e); + // An HTTP library's exception often names the URL it requested. + logger.warn(connectFailure(loggedUrl, engineType, automatic, + LicenseKeyMask.maskIn(e.getMessage(), licenseKeys)), + LicenseKeyMask.maskIn(e, licenseKeys)); return AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; } } + /** + * Get the time to send as If-Modified-Since. This is the publication date + * of the data file, or the last modified time of the data file on disk + * where the data file has no publication date. + */ + private long getIfModifiedSince(AspectEngineDataFile dataFile) { + Date published = dataFile.getDataPublishedDateTime(); + long publishedMillis = published == null ? 0 : published.getTime(); + if (publishedMillis > 0) { + return publishedMillis; + } + String path = dataFile.getDataFilePath(); + if (path != null && path.isEmpty() == false) { + return fileWrapperFactory.getLastModified(path); + } + return publishedMillis; + } + + private static String retryNote(boolean automatic) { + return automatic ? " Update will be attempted again later." : ""; + } + + /** + * The warning for a failed connection. The URL and the detail are + * given with each license key already masked. + */ + private static String connectFailure( + String loggedUrl, + String engineType, + boolean automatic, + String detail) { + return "An error occurred when connecting to " + loggedUrl + + " in order to check for data file updates for " + engineType + + "." + retryNote(automatic) + " Error detail: " + detail; + } + + /** + * The warning for a failed download. The URL and the detail are given + * with each license key already masked. + */ + private static String downloadFailure( + String loggedUrl, + String engineType, + boolean automatic, + String detail) { + return "An error occurred while downloading a data file update for " + + engineType + " from " + loggedUrl + "." + retryNote(automatic) + + " Error detail: " + detail; + } + + /** + * Log the Errors table's message for a refresh that failed, with the + * number of tries for a refresh from the file and without it for one + * from data held in memory. The exception is passed to the logger, so + * its detail is kept beside the message. + */ + private void logRefreshFailure( + AspectEngineDataFile dataFile, + Exception ex, + Integer tries) { + logger.error("An error occurred when applying a data update to " + + "engine '" + getEngineType(dataFile) + "'" + + (tries == null ? "." : " after " + tries + " tries."), ex); + } + + private String getEngineType(AspectEngineDataFile dataFile) { + if (dataFile.getEngine() != null) { + return dataFile.getEngine().getClass().getSimpleName(); + } + // The engine is built after update on start-up has run. + if (dataFile instanceof AspectEngineDataFileDefault && + ((AspectEngineDataFileDefault) dataFile).getEngineTypeName() != null) { + return ((AspectEngineDataFileDefault) dataFile).getEngineTypeName(); + } + return "No Engine"; + } + /** * Persist the body of a 200 OK download to {@code tempFile}. *

      @@ -656,11 +973,16 @@ private AutoUpdateStatus downloadFile( * {@link GZIPInputStream}, so the JDK's concatenated-gzip-member * decoder behaves correctly on top of {@link HttpURLConnection} on * every JDK from 7 upwards. See that method for the gory details. + *

      + * A failure reading the download is thrown, for the caller to report. + * A failure writing {@code tempFile} is logged here as the + * specification's write error. */ private AutoUpdateStatus downloadOkResponse( HttpURLConnection connection, AspectEngineDataFile dataFile, - FileWrapper tempFile) throws IOException { + FileWrapper tempFile, + boolean automatic) throws IOException { logger.debug("HTTP Status is OK"); boolean verifyMd5 = dataFile.getConfiguration().getVerifyMd5(); boolean decompress = dataFile.getConfiguration().getDecompressContent(); @@ -681,9 +1003,25 @@ private AutoUpdateStatus downloadOkResponse( src = new GZIPInputStream(src); } - writeBodyTo(tempFile, src); + ReadFailureRecorder body = new ReadFailureRecorder(src); + try { + writeBodyTo(tempFile, body); + } catch (IOException e) { + if (body.readFailed) { + // The download failed, which the caller reports. + throw e; + } + logger.error("An error occurred when writing to " + + tempFile.getPath() + ". Error detail: " + e.getMessage(), e); + return AutoUpdateStatus.AUTO_UPDATE_ERR_READING_STREAM; + } if (verifyMd5 && !md5Matches(md, connection)) { + logger.warn("An error occurred during the integrity check of new " + + "data file for " + getEngineType(dataFile) + "." + + retryNote(automatic) + + " Error detail: the MD5 hash of the download does not " + + "match the Content-MD5 header"); return AutoUpdateStatus.AUTO_UPDATE_ERR_MD5_VALIDATION_FAILED; } return AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS; @@ -721,6 +1059,38 @@ public int available() throws IOException { return new BufferedInputStream(lying); } + /** + * Records whether a read of the download failed, so that a failure to + * write the temporary file can be told from a failure to download. + */ + private static final class ReadFailureRecorder extends FilterInputStream { + private boolean readFailed; + + ReadFailureRecorder(InputStream in) { + super(in); + } + + @Override + public int read() throws IOException { + try { + return super.read(); + } catch (IOException e) { + readFailed = true; + throw e; + } + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + try { + return super.read(b, off, len); + } catch (IOException e) { + readFailed = true; + throw e; + } + } + } + private void writeBodyTo(FileWrapper tempFile, InputStream body) throws IOException { if (tempFile.getPath() != null) { logger.debug("Copying download stream"); @@ -754,20 +1124,17 @@ public void close() { } } - private String getIdForLogging(AspectEngineDataFile dataFile) { - return dataFile.getEngine() == null ? - "data file '" + dataFile.getIdentifier() + "'" : - "engine '" + dataFile.getEngine().getClass().getSimpleName() + "'"; - } - - private AutoUpdateStatus checkForUpdateFromUrl(AspectEngineDataFile dataFile) { + private AutoUpdateStatus checkForUpdateFromUrl( + AspectEngineDataFile dataFile, + boolean automatic) { if (dataFile.getDataFilePath() == null || dataFile.getDataFilePath().isEmpty()) { // There is no data file path specified so perform the // update entirely in memory. try { FileWrapper uncompressedData = new FileWrapperMemory(); - AutoUpdateStatus memoryDownloadResult = downloadFile(dataFile, uncompressedData); + AutoUpdateStatus memoryDownloadResult = + downloadFile(dataFile, uncompressedData, automatic); if (memoryDownloadResult != AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS) { return memoryDownloadResult; } @@ -775,6 +1142,8 @@ private AutoUpdateStatus checkForUpdateFromUrl(AspectEngineDataFile dataFile) { if (dataFile.getEngine() != null) { // Tell the engine to refresh itself with // the new data. + logger.info("Attempting to refresh Engine '{}' with new data", + getEngineType(dataFile)); dataFile.getEngine().refreshData(dataFile.getIdentifier(), reader.toByteArray()); } else { @@ -783,23 +1152,29 @@ private AutoUpdateStatus checkForUpdateFromUrl(AspectEngineDataFile dataFile) { dataFile.getConfiguration().setData(reader.toByteArray()); } } - return memoryDownloadResult; + // The engine, or the configuration, now holds the new data. + return AUTO_UPDATE_SUCCESS; } catch (Exception ex) { - logger.error("An error occurred when applying a data update to engine '{}'" , - Objects.nonNull(dataFile.getEngine())? - dataFile.getEngine().getClass().getName(): "no engine",ex); + logRefreshFailure(dataFile, ex, null); return AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED; } } - + if (dataFile.getTempDataDirPath() == null || + dataFile.getTempDataDirPath().isEmpty()) { + logger.error("The data file '" + dataFile.getIdentifier() + + "' is checking for updates but does not have a temporary " + + "file path configured."); + return AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED; + } Path uncompressedTempFile = Paths.get( dataFile.getTempDataDirPath(), dataFile.getIdentifier() + "-" + randomUUID() + ".tmp"); FileWrapper uncompressedData = fileWrapperFactory.build(uncompressedTempFile.toString()); try { // Check if there is an update and download it if there is - AutoUpdateStatus result = downloadFile(dataFile, uncompressedData); + AutoUpdateStatus result = + downloadFile(dataFile, uncompressedData, automatic); if (result == AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS) { WatchKey watchKey = dataFile.getConfiguration().getWatchKey(); @@ -825,22 +1200,25 @@ private AutoUpdateStatus checkForUpdateFromUrl(AspectEngineDataFile dataFile) { logger.debug("Reading file {}", d.read()); } } catch (Exception ex) { - logger.error("An error occurred when moving a data file to replace " + - "the existing one at {} for engine '{}'.", - dataFile.getDataFilePath(), - Objects.nonNull(dataFile.getEngine()) ? - dataFile.getEngine().getClass().getSimpleName() : "No engine defined", - ex); + logger.error("An error occurred when copying a data file to " + + "replace the existing one at '{}'.", + dataFile.getDataFilePath(), ex); result = AutoUpdateStatus.AUTO_UPDATE_NEW_FILE_CANT_RENAME; } - // Make sure to enable the file watcher again if needed. + // Make sure to enable the file watcher again if needed. The + // watch resumes on the data file's own watch service, unless + // the data file was unregistered during the download. if (watchKey != null) { final Path filePath = Paths.get(dataFile.getDataFilePath()).toAbsolutePath(); - WatchService watcher = filePath.getFileSystem().newWatchService(); - watchKey = filePath.getParent().register(watcher, - StandardWatchEventKinds.ENTRY_MODIFY); - dataFile.getConfiguration().setWatchKey(watchKey); + synchronized (configLock) { + if (configurations.contains(dataFile)) { + watchKey = filePath.getParent().register( + watchService(dataFile, filePath), + StandardWatchEventKinds.ENTRY_MODIFY); + dataFile.getConfiguration().setWatchKey(watchKey); + } + } } } return result; diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploader.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploader.java index 38df1b000..b790a6415 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploader.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploader.java @@ -44,5 +44,12 @@ public interface DataUploader { */ int getResponseCode() throws IOException; + /** + * @return the content of the answer, which is logged when the code is + * not 200, or an empty string if the uploader cannot give it + */ + default String getResponseContent() { + return ""; + } } diff --git a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java index 777584bf5..9ad54e674 100644 --- a/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java +++ b/pipeline/pipeline.engines/src/main/java/fiftyone/pipeline/engines/services/DataUploaderHttp.java @@ -23,9 +23,11 @@ package fiftyone.pipeline.engines.services; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.zip.GZIPOutputStream; @@ -34,13 +36,19 @@ public class DataUploaderHttp implements DataUploader{ private final String url; private final Map headers; private final int timeout; + + /** + * The most characters of an answer's content that are given. + */ + private static final int LOGGED_CONTENT_LENGTH = 500; HttpURLConnection connection; /** * the URL to which data is to be uploaded * @param url the URL rto which data should be uploaded * @param headers HTTP headers for the connection - * @param timeout connection timeout in milliseconds + * @param timeout milliseconds to wait to connect, and then to wait for + * an answer */ public DataUploaderHttp(String url, Map headers, int timeout) { this.url = url; @@ -55,6 +63,7 @@ public DataUploaderHttp(String url, Map headers, int timeout) { public OutputStream getOutputStream() throws Exception{ connection = (HttpURLConnection) new URI(url.trim()).toURL().openConnection(); connection.setConnectTimeout(timeout); + connection.setReadTimeout(timeout); connection.setRequestMethod("POST"); for (Map.Entry header : headers.entrySet()) { connection.setRequestProperty(header.getKey(), header.getValue()); @@ -67,5 +76,28 @@ public OutputStream getOutputStream() throws Exception{ public int getResponseCode() throws IOException { return connection.getResponseCode(); } + + /** + * Reads the content of a failed answer, as far as the most characters + * that are logged. + */ + @Override + public String getResponseContent() { + try (InputStream error = connection.getErrorStream()) { + if (error == null) { + return ""; + } + byte[] content = new byte[LOGGED_CONTENT_LENGTH]; + int length = 0; + int read; + while (length < content.length && (read = error.read( + content, length, content.length - length)) > 0) { + length += read; + } + return new String(content, 0, length, StandardCharsets.UTF_8); + } catch (IOException e) { + return ""; + } + } } diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBaseTest.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBaseTest.java index 68d84213d..efbd8ade3 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBaseTest.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/flowelements/OnPremiseAspectEngineBuilderBaseTest.java @@ -22,7 +22,15 @@ package fiftyone.pipeline.engines.flowelements; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.engines.Constants; +import fiftyone.pipeline.engines.configuration.DataFileConfiguration; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectEngineDataFile; +import fiftyone.pipeline.engines.services.DataUpdateService; +import fiftyone.pipeline.engines.services.DataUpdateServiceDefault; +import fiftyone.pipeline.engines.services.HttpClient; +import fiftyone.pipeline.engines.services.update.FutureFactory; import org.junit.After; import org.junit.Assume; import org.junit.Before; @@ -35,11 +43,21 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.PosixFilePermission; +import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.concurrent.ScheduledFuture; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; @SuppressWarnings("rawtypes") public class OnPremiseAspectEngineBuilderBaseTest { @@ -131,4 +149,140 @@ public void CreateTemporaryFolder_VerifyPermissions() { assertTrue(tempDir.canWrite()); assertTrue(tempDir.canExecute()); } + + /** + * Every data file's license key is checked before any data file is + * registered, so a build refused for a later data file leaves nothing + * registered with the data update service. + */ + @Test + public void LaterDataFileHasNoLicenseKey_NothingRegistered() { + DataUpdateService updateService = mock(DataUpdateService.class); + OnPremiseAspectEngineBuilderBase builder = + new OnPremiseAspectEngineBuilderBase(null, updateService) { + @Override + public OnPremiseAspectEngineBuilderBase setPerformanceProfile( + Constants.PerformanceProfiles profile) { + return this; + } + + @Override + protected AspectEngine newEngine(List properties) { + return null; + } + + @Override + protected boolean isLicenseKeyRequired( + DataFileConfiguration dataFileConfig) { + return true; + } + }; + builder.addDataFile(autoUpdateConfiguration("First", "SOMELICENSEKEY")); + builder.addDataFile(autoUpdateConfiguration("Second", null)); + + assertThrows(PipelineConfigurationException.class, builder::buildEngine); + + verify(updateService, never()).registerDataFile(any()); + } + + /** + * The file system watcher runs on its own setting and needs no license + * key. An engine whose update source needs a key it does not have is + * built with automatic updates and the update on start-up off, and its + * data file is still registered to be watched. With the watcher off as + * well, nothing is registered. + */ + @Test + public void WatcherAlone_RegistersTheDataFile() throws Exception { + String[] names = { + "watcher on, automatic updates off, no license key", + "watcher off, automatic updates off, no license key"}; + boolean[] watching = {true, false}; + for (int i = 0; i < names.length; i++) { + checkWatcherAlone(names[i], watching[i]); + } + } + + private static void checkWatcherAlone(String name, boolean watching) + throws Exception { + FutureFactory futureFactory = mock(FutureFactory.class); + when(futureFactory.scheduleRepeating(any(Runnable.class), anyLong())) + .then(invocation -> mock(ScheduledFuture.class)); + DataUpdateService updateService = new DataUpdateServiceDefault( + null, mock(HttpClient.class), null, futureFactory); + OnPremiseAspectEngineBuilderBase builder = + keyRequiredBuilder(updateService); + + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Data".getBytes()); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setIdentifier("Watched"); + config.setDataFilePath(dataFile.toString()); + config.setAutomaticUpdatesEnabled(false); + config.setUpdateOnStartup(false); + config.setFileSystemWatcherEnabled(watching); + builder.addDataFile(config); + + try { + builder.buildEngine(); + AspectEngineDataFile file = + (AspectEngineDataFile) builder.dataFiles.get(0); + assertEquals(name + ", the data file is registered", + watching, file.getIsRegistered()); + assertEquals(name + ", a file system watcher was created", + watching, config.getWatchKey() != null); + } + finally { + for (Object file : builder.dataFiles) { + updateService.unregisterDataFile((AspectEngineDataFile) file); + } + if (config.getWatchKey() != null) { + config.getWatchKey().cancel(); + } + directory.toFile().deleteOnExit(); + if (dataFile.toFile().delete() == false) { + dataFile.toFile().deleteOnExit(); + } + directory.toFile().delete(); + } + } + + /** + * A builder whose update source serves nothing without a license key. + */ + private static OnPremiseAspectEngineBuilderBase keyRequiredBuilder( + DataUpdateService updateService) { + return new OnPremiseAspectEngineBuilderBase(null, updateService) { + @Override + public OnPremiseAspectEngineBuilderBase setPerformanceProfile( + Constants.PerformanceProfiles profile) { + return this; + } + + @Override + protected AspectEngine newEngine(List properties) { + return null; + } + + @Override + protected boolean isLicenseKeyRequired( + DataFileConfiguration dataFileConfig) { + return true; + } + }; + } + + private static DataFileConfiguration autoUpdateConfiguration( + String identifier, + String licenseKey) { + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setIdentifier(identifier); + config.setAutomaticUpdatesEnabled(true); + if (licenseKey != null) { + config.setDataUpdateLicenseKeys( + Collections.singletonList(licenseKey)); + } + return config; + } } \ No newline at end of file diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java new file mode 100644 index 000000000..1ea82ece7 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLicenseKeyTests.java @@ -0,0 +1,223 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import fiftyone.common.wrappers.io.FileWrapperFactory; +import fiftyone.pipeline.engines.configuration.DataFileConfiguration; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import fiftyone.pipeline.engines.services.update.FutureFactory; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.Invocation; +import org.slf4j.Logger; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Arrays; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The data update URL carries the license keys in its query. Every log + * entry the data update service writes about a request, at any level and + * with any exception it logs, shows each key only by its last 4 + * characters. + */ +public class DataUpdateServiceLicenseKeyTests { + + private static final String KEY = "TESTKEYNOTREALABCDEFGHIJ1234"; + private static final String MASKED = "************************1234"; + private static final String OTHER_KEY = "OTHERTESTKEYNOTREALXYZ5678"; + private static final String OTHER_MASKED = "**********************5678"; + private static final String UPDATE_URL = + "https://distributor.example/api/v2/download"; + + private Logger logger; + private HttpClient httpClient; + private HttpURLConnection connection; + private DataUpdateServiceDefault service; + private OnPremiseAspectEngine engine; + + @Before + @SuppressWarnings("unchecked") + public void init() throws IOException { + logger = mock(Logger.class); + when(logger.isDebugEnabled()).thenReturn(true); + when(logger.isInfoEnabled()).thenReturn(true); + when(logger.isWarnEnabled()).thenReturn(true); + when(logger.isErrorEnabled()).thenReturn(true); + connection = mock(HttpURLConnection.class); + httpClient = mock(HttpClient.class); + when(httpClient.connect(any(URL.class))).thenReturn(connection); + service = new DataUpdateServiceDefault( + logger, + httpClient, + mock(FileWrapperFactory.class), + mock(FutureFactory.class)); + + // A data file updated in memory, whose URL carries the keys as the + // 51Degrees URL formatter puts them. + final DataFileConfiguration config = new DataFileConfigurationDefault(); + config.setDataUpdateUrl(UPDATE_URL); + config.setDataUpdateLicenseKeys(Arrays.asList(KEY, OTHER_KEY)); + config.setUrlFormatter(dataFile -> config.getDataUpdateUrl() + + "?LicenseKeys=" + KEY + "|" + OTHER_KEY + + "&Download=True&Type=HashV41"); + config.setVerifyModifiedSince(false); + config.setFileSystemWatcherEnabled(false); + config.setAutomaticUpdatesEnabled(false); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setConfiguration(config); + + engine = mock(OnPremiseAspectEngine.class); + when(engine.getDataFileMetaData()).thenReturn(file); + } + + @Test + public void Forbidden_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_FORBIDDEN); + assertLogsMaskedKeys(); + } + + @Test + public void TooManyRequests_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()).thenReturn(429); + assertLogsMaskedKeys(); + } + + @Test + public void OtherStatus_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_INTERNAL_ERROR); + assertLogsMaskedKeys(); + } + + @Test + public void NotModified_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + assertLogsMaskedKeys(); + } + + @Test + public void NoResponse_LogsMaskedKeys() throws IOException { + when(httpClient.connect(any(URL.class))).thenReturn(null); + assertLogsMaskedKeys(); + } + + /** + * The exception an HTTP library raises often names the URL it + * requested, and the exception is logged with its stack trace. + */ + @Test + public void RequestException_LogsMaskedKeys() throws IOException { + when(connection.getResponseCode()).thenThrow(new IOException( + "Server returned HTTP response code: 403 for URL: " + + UPDATE_URL + "?LicenseKeys=" + KEY + "|" + OTHER_KEY)); + assertLogsMaskedKeys(); + } + + /** + * A key typed straight into the data update URL, rather than given as + * a data update license key, is masked too. + */ + @Test + @SuppressWarnings("unchecked") + public void KeyTypedIntoTheUrl_LogsMaskedKey() throws IOException { + DataFileConfiguration config = new DataFileConfigurationDefault(); + config.setDataUpdateUrl(UPDATE_URL + "?LicenseKeys=" + KEY + + "&Download=True&Type=HashV41"); + config.setVerifyModifiedSince(false); + config.setFileSystemWatcherEnabled(false); + config.setAutomaticUpdatesEnabled(false); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setConfiguration(config); + OnPremiseAspectEngine typed = + mock(OnPremiseAspectEngine.class); + when(typed.getDataFileMetaData()).thenReturn(file); + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_FORBIDDEN); + + service.checkForUpdate(typed); + + String logged = logged(); + assertFalse(logged, logged.contains(KEY)); + assertTrue(logged, logged.contains(MASKED)); + } + + private void assertLogsMaskedKeys() { + service.checkForUpdate(engine); + + String logged = logged(); + assertFalse(logged, logged.contains(KEY)); + assertFalse(logged, logged.contains(OTHER_KEY)); + assertTrue(logged, logged.contains(MASKED)); + assertTrue(logged, logged.contains(OTHER_MASKED)); + } + + /** + * Everything passed to the logger, at every level, with the stack trace + * of each throwable as it would be printed. + */ + private String logged() { + StringBuilder text = new StringBuilder(); + for (Invocation invocation : + Mockito.mockingDetails(logger).getInvocations()) { + text.append(invocation.getMethod().getName()).append(':'); + for (Object argument : invocation.getArguments()) { + append(text, argument); + } + text.append('\n'); + } + return text.toString(); + } + + private static void append(StringBuilder text, Object argument) { + if (argument instanceof Object[]) { + for (Object item : (Object[]) argument) { + append(text, item); + } + } else if (argument instanceof Throwable) { + StringWriter trace = new StringWriter(); + ((Throwable) argument).printStackTrace(new PrintWriter(trace)); + text.append(' ').append(trace); + } else { + text.append(' ').append(argument); + } + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java new file mode 100644 index 000000000..dcd972ae1 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceLoggingTests.java @@ -0,0 +1,551 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.ThrowableProxyUtil; +import ch.qos.logback.core.read.ListAppender; +import fiftyone.common.wrappers.io.FileWrapperFactoryDefault; +import fiftyone.pipeline.engines.configuration.DataFileConfiguration; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The messages and levels in the Logging section of the data update + * specification, and the If-Modified-Since time the update URL is sent. + * @see Specification + */ +@SuppressWarnings("unchecked") +public class DataUpdateServiceLoggingTests { + + private static final String UPDATE_URL = "http://test.com"; + // 13 September 2020, 12:26:40 UTC. + private static final long FILE_MODIFIED_MILLIS = 1600000000000L; + // A license key of the length and characters the Distributor issues. + private static final String FULL_KEY = "QF8ZR2KD7NMB4XWT9HJC3LPA"; + private static final String MASKED_KEY = "********************3LPA"; + private static final String KEYED_URL = UPDATE_URL + "?LicenseKeys=" + + FULL_KEY + "&Download=True&Type=HashV41"; + private static final String MASKED_URL = UPDATE_URL + "?LicenseKeys=" + + MASKED_KEY + "&Download=True&Type=HashV41"; + + private ch.qos.logback.classic.Logger logger; + private ListAppender appender; + private HttpClient httpClient; + private HttpURLConnection connection; + private DataUpdateServiceDefault dataUpdate; + private OnPremiseAspectEngine engine; + private String engineType; + private File dataFile; + + @Before + public void init() throws IOException { + logger = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger( + "DataUpdateServiceLoggingTests"); + logger.setLevel(Level.INFO); + appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + + connection = mock(HttpURLConnection.class); + httpClient = mock(HttpClient.class); + when(httpClient.connect(any(URL.class))).thenReturn(connection); + // The futures are real but their first run is a day away. + dataUpdate = new DataUpdateServiceDefault( + logger, httpClient, new FileWrapperFactoryDefault(), null); + + engine = mock(OnPremiseAspectEngine.class); + engineType = engine.getClass().getSimpleName(); + dataFile = File.createTempFile("logging-test", ".tmp"); + assertTrue(dataFile.setLastModified(FILE_MODIFIED_MILLIS)); + } + + @After + public void cleanup() { + dataUpdate.close(); + logger.detachAppender(appender); + dataFile.delete(); + } + + private AspectEngineDataFileDefault newDataFile( + boolean inMemory, + boolean autoUpdate, + boolean updateOnStartup) { + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(autoUpdate); + config.setUpdateOnStartup(updateOnStartup); + config.setFileSystemWatcherEnabled(false); + config.setDataUpdateUrl(UPDATE_URL); + config.setVerifyMd5(false); + config.setDecompressContent(false); + config.setPollingIntervalSeconds(86400); + config.setMaxRandomisationSeconds(0); + if (inMemory == false) { + config.setDataFilePath(dataFile.getAbsolutePath()); + } + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setIdentifier("default"); + file.setEngine(engine); + file.setConfiguration(config); + file.setTempDataDirPath(System.getProperty("java.io.tmpdir")); + when(engine.getDataFileMetaData()).thenReturn(file); + return file; + } + + private List logged(Level level) { + List messages = new ArrayList<>(); + for (ILoggingEvent event : appender.list) { + if (event.getLevel() == level) { + messages.add(event.getFormattedMessage()); + } + } + return messages; + } + + private void assertLogged(Level level, String message) { + List messages = logged(level); + assertTrue( + "Expected '" + message + "' at " + level + " but got " + messages, + messages.contains(message)); + } + + @Test + public void IfModifiedSince_NoPublishedDate_UsesFileLastModified() + throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newDataFile(false, false, false); + file.getConfiguration().setVerifyModifiedSince(true); + + dataUpdate.checkForUpdate(engine); + + verify(connection).setIfModifiedSince(FILE_MODIFIED_MILLIS); + } + + @Test + public void IfModifiedSince_PublishedDate_UsedInsteadOfFileLastModified() + throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newDataFile(false, false, false); + file.getConfiguration().setVerifyModifiedSince(true); + file.setDataPublishedDateTime(new java.util.Date(1500000000000L)); + + dataUpdate.checkForUpdate(engine); + + verify(connection).setIfModifiedSince(1500000000000L); + } + + @Test + public void Check_NotModified_LogsTheSpecifiedMessages() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newDataFile(false, false, false); + file.getConfiguration().setVerifyModifiedSince(true); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.INFO, "Checking for update"); + assertLogged(Level.INFO, "Checking for update from '" + UPDATE_URL + + "' for Engine '" + engineType + "'"); + assertLogged(Level.INFO, "No data newer than 2020-09-13T12:26:40Z " + + "found at '" + UPDATE_URL + "' for Engine '" + engineType + "'"); + } + + @Test + public void Check_Downloaded_LogsDownloadAndRefresh() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + newDataFile(true, false, false); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.INFO, "Downloaded new data from '" + UPDATE_URL + + "' for Engine '" + engineType + "'"); + assertLogged(Level.INFO, "Attempting to refresh Engine '" + + engineType + "' with new data"); + } + + @Test + public void UpdateFromMemory_LogsRefresh() { + AspectEngineDataFileDefault file = newDataFile(true, false, false); + + dataUpdate.updateFromMemory(file, new byte[]{1, 2, 3}); + + assertLogged(Level.INFO, "Attempting to refresh Engine '" + + engineType + "' with new data"); + } + + @Test + public void UpdateFromMemory_RefreshFails_LogsTheTableMessage() + throws Exception { + AspectEngineDataFileDefault file = newDataFile(true, false, false); + doThrow(new RuntimeException("no room")) + .when(engine).refreshData(anyString(), any(byte[].class)); + + dataUpdate.updateFromMemory(file, new byte[]{1, 2, 3}); + + assertLogged(Level.ERROR, "An error occurred when applying a data " + + "update to engine '" + engineType + "'."); + } + + @Test + public void UpdateOnStartup_LogsUpdatingOnStartUp() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newDataFile(false, false, true); + + dataUpdate.registerDataFile(file); + + assertLogged(Level.INFO, "Updating on start-up"); + } + + @Test + public void UpdateOnStartup_EngineNotBuiltYet_LogsTheEngineTypeName() + throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newDataFile(false, false, true); + file.setEngine(null); + file.setEngineTypeName("ExampleEngine"); + + dataUpdate.registerDataFile(file); + + assertLogged(Level.INFO, "Checking for update from '" + UPDATE_URL + + "' for Engine 'ExampleEngine'"); + } + + @Test + public void UpdateOnStartup_AutoUpdateOff_StartsNoTimer() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newDataFile(false, false, true); + + dataUpdate.registerDataFile(file); + + assertEquals(null, file.getFuture()); + } + + @Test + public void FileSystemWatcher_LogsCreatingFileSystemWatcher() { + // The watcher needs no automatic updates. + AspectEngineDataFileDefault file = newDataFile(false, false, false); + file.getConfiguration().setFileSystemWatcherEnabled(true); + + dataUpdate.registerDataFile(file); + + assertLogged(Level.INFO, "Creating file system watcher"); + assertNotNull(file.getConfiguration().getWatchKey()); + } + + @Test + public void ConnectFails_Manual_WarnsWithoutRetryText() throws Exception { + when(httpClient.connect(any(URL.class))) + .thenThrow(new IOException("refused")); + newDataFile(false, false, false); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred when connecting to " + + UPDATE_URL + " in order to check for data file updates for " + + engineType + ". Error detail: refused"); + } + + @Test + public void ConnectFails_Automatic_WarnsThatUpdateWillBeTriedAgain() + throws Exception { + when(httpClient.connect(any(URL.class))) + .thenThrow(new IOException("refused")); + AspectEngineDataFileDefault file = newDataFile(false, true, true); + + dataUpdate.registerDataFile(file); + + assertLogged(Level.WARN, "An error occurred when connecting to " + + UPDATE_URL + " in order to check for data file updates for " + + engineType + ". Update will be attempted again later. " + + "Error detail: refused"); + } + + @Test + public void ConnectFails_AutoUpdateOff_DoesNotPromiseAnotherTry() + throws Exception { + when(httpClient.connect(any(URL.class))) + .thenThrow(new IOException("refused")); + AspectEngineDataFileDefault file = newDataFile(false, false, true); + + dataUpdate.registerDataFile(file); + + assertLogged(Level.WARN, "An error occurred when connecting to " + + UPDATE_URL + " in order to check for data file updates for " + + engineType + ". Error detail: refused"); + } + + private void assertStatusWarned(int status, String detail) + throws Exception { + when(connection.getResponseCode()).thenReturn(status); + newDataFile(true, false, false); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred while downloading a data " + + "file update for " + engineType + " from " + UPDATE_URL + ". " + + "Error detail: " + detail); + assertTrue(logged(Level.ERROR).isEmpty()); + } + + @Test + public void TooManyRequests_WarnsAboutTheDownload() throws Exception { + assertStatusWarned(429, "HTTP status code 429 (too many requests)"); + } + + @Test + public void Forbidden_WarnsAboutTheDownload() throws Exception { + assertStatusWarned(403, "HTTP status code 403 (access denied)"); + } + + @Test + public void OtherStatus_WarnsAboutTheDownload() throws Exception { + assertStatusWarned(500, "HTTP status code 500"); + } + + /** + * The download is written to a temporary file in a folder that does not + * exist, so the write fails while the download itself succeeds. + */ + @Test + public void TemporaryFileCannotBeWritten_LogsTheWriteError() + throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + AspectEngineDataFileDefault file = newDataFile(false, false, false); + String missingDirectory = new File( + System.getProperty("java.io.tmpdir"), + "no-such-directory-" + System.nanoTime()).getAbsolutePath(); + file.setTempDataDirPath(missingDirectory); + // The engine's copy is the data file itself, so the file on disk is + // not newer and the update is downloaded. + file.setTempDataFilePath(dataFile.getAbsolutePath()); + + dataUpdate.checkForUpdate(engine); + + verify(connection).getInputStream(); + boolean found = false; + for (String message : logged(Level.ERROR)) { + found |= message.startsWith("An error occurred when writing to " + + missingDirectory) && message.contains(". Error detail: "); + } + assertTrue("No write error in " + logged(Level.ERROR) + + ", and the warnings are " + logged(Level.WARN), found); + assertTrue(logged(Level.WARN).isEmpty()); + } + + @Test + public void Md5DoesNotMatch_WarnsAboutIntegrityCheck() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + when(connection.getHeaderField("Content-MD5")).thenReturn("0"); + AspectEngineDataFileDefault file = newDataFile(true, false, false); + file.getConfiguration().setVerifyMd5(true); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred during the integrity " + + "check of new data file for " + engineType + ". Error detail: " + + "the MD5 hash of the download does not match the Content-MD5 " + + "header"); + } + + @Test + public void NoTemporaryFilePath_LogsError() { + AspectEngineDataFileDefault file = newDataFile(false, false, false); + file.setTempDataDirPath(null); + when(engine.getTempDataDirPath()).thenReturn(null); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.ERROR, "The data file 'default' is checking for " + + "updates but does not have a temporary file path configured."); + } + + /** + * A data file whose formatted URL carries a license key, as the + * Distributor's URL does, with every level logged. + */ + private AspectEngineDataFileDefault newKeyedDataFile(boolean inMemory) { + logger.setLevel(Level.DEBUG); + AspectEngineDataFileDefault file = newDataFile(inMemory, false, false); + DataFileConfiguration config = file.getConfiguration(); + config.setDataUpdateLicenseKeys( + Collections.singletonList(FULL_KEY)); + config.setUrlFormatter(dataFile -> KEYED_URL); + return file; + } + + /** + * Fails if any log entry, or any throwable logged with one, holds the + * license key in full. + */ + private void assertKeyOnlyMasked() { + assertFalse(appender.list.isEmpty()); + for (ILoggingEvent event : appender.list) { + String text = event.getFormattedMessage(); + if (event.getThrowableProxy() != null) { + text += "\n" + + ThrowableProxyUtil.asString(event.getThrowableProxy()); + } + assertFalse( + "The license key is in full in: " + text, + text.contains(FULL_KEY)); + } + } + + @Test + public void LicenseKey_NotModified_LogsTheMaskedUrl() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_NOT_MODIFIED); + AspectEngineDataFileDefault file = newKeyedDataFile(false); + file.getConfiguration().setVerifyModifiedSince(true); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.INFO, "Checking for update from '" + MASKED_URL + + "' for Engine '" + engineType + "'"); + assertLogged(Level.INFO, "No data newer than 2020-09-13T12:26:40Z " + + "found at '" + MASKED_URL + "' for Engine '" + engineType + "'"); + assertKeyOnlyMasked(); + } + + @Test + public void LicenseKey_Downloaded_LogsTheMaskedUrl() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenReturn(new ByteArrayInputStream(new byte[]{1, 2, 3})); + newKeyedDataFile(true); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.INFO, "Downloaded new data from '" + MASKED_URL + + "' for Engine '" + engineType + "'"); + assertKeyOnlyMasked(); + } + + /** + * An HTTP library's exception often names the URL it requested, so the + * error detail and the logged exception are masked too. + */ + @Test + public void LicenseKey_ConnectFails_MasksTheUrlAndTheException() + throws Exception { + when(httpClient.connect(any(URL.class))) + .thenThrow(new IOException("Connection refused: " + KEYED_URL)); + newKeyedDataFile(false); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred when connecting to " + + MASKED_URL + " in order to check for data file updates for " + + engineType + ". Error detail: Connection refused: " + MASKED_URL); + assertKeyOnlyMasked(); + } + + @Test + public void LicenseKey_DownloadFails_MasksTheUrlAndTheException() + throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()) + .thenThrow(new IOException("Premature EOF from " + KEYED_URL)); + newKeyedDataFile(true); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred while downloading a data " + + "file update for " + engineType + " from " + MASKED_URL + ". " + + "Error detail: Premature EOF from " + MASKED_URL); + assertKeyOnlyMasked(); + } + + @Test + public void LicenseKey_Forbidden_LogsTheMaskedUrl() throws Exception { + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_FORBIDDEN); + newKeyedDataFile(true); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred while downloading a data " + + "file update for " + engineType + " from " + MASKED_URL + ". " + + "Error detail: HTTP status code 403 (access denied)"); + assertKeyOnlyMasked(); + } + + @Test + public void LicenseKey_NoResponse_LogsTheMaskedUrl() throws Exception { + when(httpClient.connect(any(URL.class))).thenReturn(null); + newKeyedDataFile(false); + + dataUpdate.checkForUpdate(engine); + + assertLogged(Level.WARN, "An error occurred when connecting to " + + MASKED_URL + " in order to check for data file updates for " + + engineType + ". Error detail: No response from the data update " + + "service"); + assertKeyOnlyMasked(); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java index 1e24bb485..b10b06a8a 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceTests.java @@ -42,8 +42,10 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatcher; import org.mockito.internal.matchers.LessOrEqual; +import org.mockito.invocation.Invocation; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.slf4j.LoggerFactory; @@ -51,21 +53,28 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; import java.net.HttpURLConnection; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; import java.util.Date; +import java.util.List; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static fiftyone.pipeline.engines.services.DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_HTTPS_ERR; import static fiftyone.pipeline.engines.services.DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_NOT_NEEDED; +import static fiftyone.pipeline.engines.services.DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_SUCCESS; import static org.junit.Assert.*; import static org.junit.Assume.assumeFalse; import static org.mockito.Mockito.*; @@ -228,6 +237,46 @@ public boolean matches(Long aLong) { ); } + @Test + public void DataUpdateService_Register_UpdateAvailableRandomDelayInSeconds() { + // Arrange + final int registrations = 50; + final long availableInMillis = 3600000; + final long maxDelayMillis = 600 * 1000; + final Date availableAt = + new Date(System.currentTimeMillis() + availableInMillis); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(true); + config.setMaxRandomisationSeconds(600); + + // Act + for (int i = 0; i < registrations; i++) { + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setUpdateAvailableTime(availableAt); + file.setEngine(mock(OnPremiseAspectEngine.class)); + file.setConfiguration(config); + dataUpdate.registerDataFile(file); + } + + // Assert + // The delay is the time until the update is available plus a random + // number of seconds, so across many registrations at least one adds + // more than a second, and none adds more than the maximum. + ArgumentCaptor delays = ArgumentCaptor.forClass(Long.class); + verify(futureFactory, times(registrations)).schedule( + any(Runnable.class), + delays.capture()); + long largestExtra = Long.MIN_VALUE; + for (long delay : delays.getAllValues()) { + long extra = delay - availableInMillis; + largestExtra = Math.max(largestExtra, extra); + assertTrue("Delay " + delay + " adds more than the maximum", + extra <= maxDelayMillis); + } + assertTrue("Largest random delay was " + largestExtra + " millis", + largestExtra > 1000); + } + @Test public void DataUpdateService_Register_AutoUpdateConfiguredInterval() { // Arrange @@ -325,6 +374,8 @@ public void DataUpdateService_Register_FileSystemWatcher() throws IOException { dataUpdate.registerDataFile(file); // Assert + // The watcher runs on its own setting, whether or not automatic + // updates are on. assertNotNull(config.getWatchKey()); } finally { // Make sure we tidy up the temp file. @@ -344,7 +395,8 @@ public void DataUpdateService_UpdateFromWatcher() throws IOException, Interrupte // Arrange OnPremiseAspectEngine engine = mock(OnPremiseAspectEngine.class); DataFileConfigurationDefault config = new DataFileConfigurationDefault(); - config.setAutomaticUpdatesEnabled(true); + // The watcher needs no automatic updates. + config.setAutomaticUpdatesEnabled(false); config.setFileSystemWatcherEnabled(true); config.setPollingIntervalSeconds(1); @@ -369,14 +421,16 @@ public Object answer(InvocationOnMock invocationOnMock) throws Throwable { } }); - // Configure a flag to be set when processing - // is complete. + // Configure a flag to be set when the watcher has applied the + // update. final Semaphore completeFlag = new Semaphore(1); completeFlag.acquire(); dataUpdate.onUpdateComplete(new OnUpdateComplete() { @Override public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { - completeFlag.release(); + if (args.getStatus() == AUTO_UPDATE_SUCCESS) { + completeFlag.release(); + } } }); @@ -492,6 +546,73 @@ public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { anyLong()); } + /** + * A check that finds an update and starts before registration has + * stored its future must still leave the next check scheduled and + * stored. + */ + @Test + public void DataUpdateService_UpdateFromFile_FileUpdated_CheckBeforeFutureStored() + throws InterruptedException { + // Arrange + RaceDataFile file = new RaceDataFile(); + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(true); + config.setFileSystemWatcherEnabled(false); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + configureFileUpdate(engine); + configureScheduleRace(file); + Semaphore completeFlag = onUpdateCompleteFlag(); + + // Act + dataUpdate.registerDataFile(file); + boolean completed = completeFlag.tryAcquire( + RACE_WAIT_SECONDS * 3, TimeUnit.SECONDS); + + // Assert + assertTrue("The OnUpdateComplete event was never fired", completed); + verify(engine, times(1)).refreshData((String) any()); + assertNextCheckScheduled(file); + } + + /** + * A check that finds no update and starts before registration has + * stored its future must still leave the next check scheduled and + * stored. A check whose download fails takes the same path, the retry + * after a 429 response among them. + */ + @Test + public void DataUpdateService_UpdateFromFile_FileNotUpdated_CheckBeforeFutureStored() + throws InterruptedException { + // Arrange + RaceDataFile file = new RaceDataFile(); + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(true); + config.setFileSystemWatcherEnabled(false); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + configureFileNoUpdate(engine); + configureScheduleRace(file); + Semaphore completeFlag = onUpdateCompleteFlag(); + + // Act + dataUpdate.registerDataFile(file); + boolean completed = completeFlag.tryAcquire( + RACE_WAIT_SECONDS * 3, TimeUnit.SECONDS); + + // Assert + assertTrue("The OnUpdateComplete event was never fired", completed); + verify(engine, never()).refreshData(anyString()); + assertNextCheckScheduled(file); + } + @Test public void DataUpdateService_UpdateFromUrl_NoUpdate() throws InterruptedException, IOException { // Arrange @@ -771,10 +892,12 @@ public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { // reconfigured to try again later. verify(futureFactory, atLeast(1)) .schedule(any(Runnable.class), anyLong()); - // We expect one error to be logged so make sure it's + // We expect one warning to be logged so make sure it's // ignored in cleanup and verify its presence. - ignoreErrors = 1; - assertEquals(1, logger.errorsLogged.size()); + ignoreWranings = 1; + assertEquals(1, logger.warningsLogged.size()); + assertTrue(logger.warningsLogged.get(0).contains(errorText)); + assertEquals(0, logger.errorsLogged.size()); assertEquals(AUTO_UPDATE_HTTPS_ERR, completeEventArgs[0].getStatus()); } finally { @@ -900,6 +1023,32 @@ public void DataUpdateService_CheckForUpdate_FileUpdated() { anyLong()); } + /** + * A programmatic update compares the data file on disk whether or not + * Automatic Update from File is enabled, here with automatic update off + * and the watcher flag at its default. + */ + @Test + public void DataUpdateService_CheckForUpdate_FileUpdated_AutoUpdateOffWatcherFlagDefault() { + // Arrange + OnPremiseAspectEngine engine = mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + + configureFileUpdate(engine); + + // Act + dataUpdate.checkForUpdate(engine); + + // Assert + verify(engine, times(1)).refreshData((String)any()); + } + @Test public void DataUpdateService_CheckForUpdate_UrlNoUpdate() throws IOException { // Getting no data from the URL will cause an error to be logged @@ -1116,6 +1265,303 @@ public boolean matches(byte[] argument) { verify(futureFactory, times(1)).schedule(any(Runnable.class), anyLong()); } + /** + * A check that finds an update for a data file with no file path + * applies it in memory, so it refreshes the engine once, with the + * downloaded data, and does not refresh it again. + */ + @Test + public void DataUpdateService_CheckForUpdate_InMemory_RefreshesOnce() + throws IOException { + // Arrange + configureNoFileSystem(); + when(httpClientConnection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(httpClientConnection.getResponseMessage()).thenReturn(null); + when(httpClientConnection.getHeaderField("Content-MD5")) + .thenReturn("08527dcbdd437e7fa6c084423d06dba6"); + when(httpClientConnection.getInputStream()).thenReturn( + DataUpdateServiceTests.class.getClassLoader() + .getResourceAsStream("file.gz")); + + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setDataUpdateUrl("http://www.test.com"); + config.setVerifyMd5(true); + config.setDecompressContent(true); + config.setVerifyModifiedSince(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + + // Act + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + // Assert + assertEquals(AUTO_UPDATE_SUCCESS, status); + List refreshes = new ArrayList<>(); + for (Invocation invocation : mockingDetails(engine).getInvocations()) { + if (invocation.getMethod().getName().equals("refreshData")) { + refreshes.add(invocation.getMethod().toGenericString()); + } + } + assertEquals( + "The engine was not refreshed exactly once: " + refreshes, + 1, + refreshes.size()); + verify(engine, times(1)).refreshData(any(), any(byte[].class)); + } + + /** + * A manual check that applies an update from the data file to a data + * file registered for automatic updates sets the next update timer, as + * a check the timer started does. + */ + @Test + public void DataUpdateService_CheckForUpdate_FileUpdated_TimerSetAgain() { + // Arrange + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(autoUpdateEnabled); + config.setFileSystemWatcherEnabled(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + configureFileUpdate(engine); + + dataUpdate.registerDataFile(file); + int timers = autoUpdateEnabled ? 1 : 0; + verify(futureFactory, times(timers)).schedule( + any(Runnable.class), + anyLong()); + + ScheduledFuture pending = file.getFuture(); + + // Act + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + // Assert + assertEquals(AUTO_UPDATE_SUCCESS, status); + verify(engine, times(1)).refreshData((String) any()); + verify(futureFactory, times(2 * timers)).schedule( + any(Runnable.class), + anyLong()); + assertEquals( + "The next update timer was not set after the manual update.", + autoUpdateEnabled, + file.getFuture() != null); + + if (autoUpdateEnabled) { + // The pending timer is cancelled, so only the new one is left. + assertTrue( + "The timer pending before the manual update was not cancelled.", + pending.isCancelled()); + assertNotSame(pending, file.getFuture()); + } + } + + /** + * A manual check whose refresh from the data file fails still leaves + * the next update timer set, as a check the timer started does, so + * polling resumes. + */ + @Test + public void DataUpdateService_CheckForUpdate_FileRefreshFails_TimerSetAgain() { + // updatedFileAvailable logs one error once its tries are used up. + ignoreErrors = 1; + + // Arrange + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + doThrow(new IllegalStateException("The refresh failed.")) + .when(engine).refreshData((String) any()); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(autoUpdateEnabled); + config.setFileSystemWatcherEnabled(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + configureFileUpdate(engine); + + dataUpdate.registerDataFile(file); + int timers = autoUpdateEnabled ? 1 : 0; + ScheduledFuture pending = file.getFuture(); + + // Act + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + // Assert + assertEquals( + DataUpdateService.AutoUpdateStatus.AUTO_UPDATE_REFRESH_FAILED, + status); + String expected = "An error occurred when applying a data update " + + "to engine '" + engine.getClass().getSimpleName() + + "' after 10 tries."; + assertTrue( + "Expected '" + expected + "' but got " + logger.errorsLogged, + logger.errorsLogged.contains(expected)); + verify(futureFactory, times(2 * timers)).schedule( + any(Runnable.class), + anyLong()); + assertEquals( + "The next update timer was not set after the failed refresh.", + autoUpdateEnabled, + file.getFuture() != null); + if (autoUpdateEnabled) { + assertTrue(pending.isCancelled()); + assertNotSame(pending, file.getFuture()); + } + } + + /** + * A data file that has been unregistered is no longer registered, so a + * manual update of it sets no update timer. + */ + @Test + public void DataUpdateService_Unregister_NoLongerRegistered() { + // Arrange + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(autoUpdateEnabled); + config.setFileSystemWatcherEnabled(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + configureFileUpdate(engine); + dataUpdate.registerDataFile(file); + int timers = autoUpdateEnabled ? 1 : 0; + + // Act + dataUpdate.unregisterDataFile(file); + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + // Assert + assertFalse(file.getIsRegistered()); + assertEquals(AUTO_UPDATE_SUCCESS, status); + verify(futureFactory, times(timers)).schedule( + any(Runnable.class), + anyLong()); + assertNull(file.getFuture()); + } + + /** + * A manual check that applies an update in memory to a data file + * registered for automatic updates sets the next update timer, as a + * check the timer started does. + */ + @Test + public void DataUpdateService_CheckForUpdate_InMemory_TimerSetAgain() + throws IOException { + // Arrange + configureNoFileSystem(); + when(httpClientConnection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(httpClientConnection.getResponseMessage()).thenReturn(null); + when(httpClientConnection.getHeaderField("Content-MD5")) + .thenReturn("08527dcbdd437e7fa6c084423d06dba6"); + when(httpClientConnection.getInputStream()).thenReturn( + DataUpdateServiceTests.class.getClassLoader() + .getResourceAsStream("file.gz")); + + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(autoUpdateEnabled); + config.setDataUpdateUrl("http://www.test.com"); + config.setVerifyMd5(true); + config.setDecompressContent(true); + config.setVerifyModifiedSince(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + + dataUpdate.registerDataFile(file); + int timers = autoUpdateEnabled ? 1 : 0; + verify(futureFactory, times(timers)).schedule( + any(Runnable.class), + anyLong()); + + ScheduledFuture pending = file.getFuture(); + + // Act + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + // Assert + assertEquals(AUTO_UPDATE_SUCCESS, status); + verify(futureFactory, times(2 * timers)).schedule( + any(Runnable.class), + anyLong()); + assertEquals( + "The next update timer was not set after the manual update.", + autoUpdateEnabled, + file.getFuture() != null); + + if (autoUpdateEnabled) { + // The pending timer is cancelled, so only the new one is left. + assertTrue( + "The timer pending before the manual update was not cancelled.", + pending.isCancelled()); + assertNotSame(pending, file.getFuture()); + } + } + + /** + * A manual check that finds no update leaves the timer of a data file + * registered for automatic updates as it was. + */ + @Test + public void DataUpdateService_CheckForUpdate_NoUpdate_TimerKept() + throws IOException { + // Arrange + configureNoFileSystem(); + configureHttpNoUpdateAvailable(); + + OnPremiseAspectEngine engine = + mock(OnPremiseAspectEngine.class); + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(autoUpdateEnabled); + config.setDataUpdateUrl("http://www.test.com"); + config.setVerifyModifiedSince(false); + + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + when(engine.getDataFileMetaData()).thenReturn(file); + + dataUpdate.registerDataFile(file); + Object timer = file.getFuture(); + + // Act + DataUpdateService.AutoUpdateStatus status = + dataUpdate.checkForUpdate(engine); + + // Assert + assertEquals(AUTO_UPDATE_NOT_NEEDED, status); + verify(engine, never()).refreshData(anyString()); + verify(futureFactory, times(autoUpdateEnabled ? 1 : 0)).schedule( + any(Runnable.class), + anyLong()); + assertSame(timer, file.getFuture()); + } + /** @@ -1278,8 +1724,8 @@ public void call(Object sender, DataUpdateService.DataUpdateCompleteArgs args) { assertTrue("The 'checkForUpdateComplete' " + "event was never fired", completed); - // Ignore the error that is logged due to the 429 - ignoreErrors = 1; + // Ignore the warning that is logged due to the 429 + ignoreWranings = 1; // Check that the timer has been set to go off again. assertNotNull(file.getFuture()); // Check that the timer has been set to expire in 10 seconds. @@ -1409,6 +1855,154 @@ private void configureTimerImmediateCallbackOnce() { configureTimerImmediateCallback(1, null); } + private static final long RACE_WAIT_SECONDS = 10; + + /** + * Data file that records every write of its future and every read the + * check makes. The check's first read waits until registration has + * stored the future, so a check that reads it too early goes on with a + * stale value. + */ + private static class RaceDataFile extends AspectEngineDataFileDefault { + private final Thread registering = Thread.currentThread(); + private final List events = new CopyOnWriteArrayList<>(); + private final AtomicInteger schedules = new AtomicInteger(); + private final AtomicReference> first = + new AtomicReference<>(); + private final AtomicReference> next = + new AtomicReference<>(); + private final AtomicReference check = new AtomicReference<>(); + private final CountDownLatch checkRead = new CountDownLatch(1); + private final CountDownLatch stored = new CountDownLatch(1); + private volatile boolean registrationReleased; + private volatile boolean storedInTime = true; + + private String who() { + return Thread.currentThread() == registering ? + "registration" : "check"; + } + + private String name(ScheduledFuture future) { + if (future == null) { + return "no future"; + } + if (future == first.get()) { + return "the first future"; + } + return future == next.get() ? "the next future" : "another future"; + } + + @Override + public ScheduledFuture getFuture() { + ScheduledFuture future = super.getFuture(); + if (Thread.currentThread() != registering) { + events.add("check read " + name(future)); + if (checkRead.getCount() > 0) { + checkRead.countDown(); + try { + storedInTime = stored.await( + RACE_WAIT_SECONDS, TimeUnit.SECONDS); + } catch (InterruptedException e) { + storedInTime = false; + Thread.currentThread().interrupt(); + } + } + } + return future; + } + + @Override + public void setFuture(ScheduledFuture future) { + super.setFuture(future); + events.add(future == null ? + who() + " cleared the future" : + who() + " stored " + name(future)); + if (Thread.currentThread() == registering) { + stored.countDown(); + } + } + } + + /** + * Run the first check at once, and hold registration after scheduling + * it until the check has read the future or waits for a lock that + * registration holds. Later checks are never run. + */ + private void configureScheduleRace(final RaceDataFile file) { + final ThreadMXBean threads = ManagementFactory.getThreadMXBean(); + doAnswer(invocation -> { + final Runnable check = invocation.getArgument(0); + if (file.schedules.incrementAndGet() > 1) { + ScheduledFuture next = mock(ScheduledFuture.class); + file.next.set(next); + file.events.add(file.who() + " scheduled the next check"); + return next; + } + file.events.add(file.who() + " scheduled the first check"); + ScheduledFuture first = executorService.schedule(() -> { + file.check.set(Thread.currentThread()); + check.run(); + }, 0, TimeUnit.MILLISECONDS); + file.first.set(first); + file.registrationReleased = waitForCheck(file, threads); + return first; + }).when(futureFactory).schedule(any(Runnable.class), anyLong()); + } + + /** + * Wait until the check has read the future, or is blocked on a lock + * that the registering thread holds and so cannot read it until + * registration has finished. Thread.getId() is deprecated from Java 19 + * but is the id that ThreadMXBean reports on every version. + */ + @SuppressWarnings("deprecation") + private static boolean waitForCheck( + RaceDataFile file, + ThreadMXBean threads) throws InterruptedException { + long deadline = System.nanoTime() + + TimeUnit.SECONDS.toNanos(RACE_WAIT_SECONDS); + while (System.nanoTime() < deadline) { + if (file.checkRead.await(10, TimeUnit.MILLISECONDS)) { + return true; + } + Thread check = file.check.get(); + if (check != null) { + ThreadInfo info = threads.getThreadInfo(check.getId()); + if (info != null && + info.getLockOwnerId() == file.registering.getId()) { + return true; + } + } + } + return false; + } + + /** + * Assert that one check was scheduled at registration and one after it, + * and that the future stored is the next check's. Each message carries + * the order in which the two threads read and wrote the future. + */ + private static void assertNextCheckScheduled(RaceDataFile file) { + String events = " Events: " + file.events; + assertTrue("The check neither read the future nor waited for " + + "registration." + events, + file.registrationReleased); + assertTrue("Registration did not store the future while the " + + "check waited." + events, + file.storedInTime); + assertEquals("Schedules." + events, 2, file.schedules.get()); + assertSame("The future stored is not the next check's." + events, + file.next.get(), + file.getFuture()); + } + + private Semaphore onUpdateCompleteFlag() throws InterruptedException { + final Semaphore completeFlag = new Semaphore(1); + completeFlag.acquire(); + dataUpdate.onUpdateComplete((sender, args) -> completeFlag.release()); + return completeFlag; + } + private void configureFileNoUpdate(OnPremiseAspectEngine engine) { String dataFile = "C:/test/tempFile.dat"; String tempFile = "C:/test/dataFile.dat"; diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceWatchedFileTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceWatchedFileTests.java new file mode 100644 index 000000000..b3e6d13c1 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUpdateServiceWatchedFileTests.java @@ -0,0 +1,767 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import fiftyone.common.wrappers.data.BinaryReader; +import fiftyone.common.wrappers.io.FileWrapper; +import fiftyone.common.wrappers.io.FileWrapperDefault; +import fiftyone.common.wrappers.io.FileWrapperFactory; +import fiftyone.pipeline.engines.configuration.DataFileConfigurationDefault; +import fiftyone.pipeline.engines.data.AspectData; +import fiftyone.pipeline.engines.data.AspectEngineDataFileDefault; +import fiftyone.pipeline.engines.data.AspectPropertyMetaData; +import fiftyone.pipeline.engines.flowelements.OnPremiseAspectEngine; +import fiftyone.pipeline.engines.services.DataUpdateService.AutoUpdateStatus; +import fiftyone.pipeline.engines.services.update.FutureFactory; +import org.junit.Test; +import org.mockito.Answers; +import org.slf4j.Logger; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.WatchKey; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeFalse; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockingDetails; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +/** + * A new data file found by the file watcher is loaded, and then polling for + * updates carries on, as the newFileAvailable steps of the data update + * specification say ("resume polling, if any"). A new file that cannot be + * opened is given up on after a time, as its Errors table says, or at + * once when the wait is interrupted. + */ +public class DataUpdateServiceWatchedFileTests { + + private static final long WAIT_FOR_EVENT_MILLIS = 10000; + + private static final long RETRY_MILLIS = 100; + + /** + * The wait for a file that cannot be opened, shortened from ten minutes. + */ + private static final long OPEN_TIMEOUT_MILLIS = 1000; + + /** + * The most tries that wait allows, being one at the start, one after + * each pause and one more for a slow runner. + */ + private static final long MAX_TRIES = OPEN_TIMEOUT_MILLIS / + DataUpdateServiceDefault.WATCHED_FILE_OPEN_PAUSE_MILLIS + 2; + + /** + * A wait long enough that only an interrupt can end it within + * {@link #INTERRUPTED_WAIT_MILLIS}. + */ + private static final long LONG_OPEN_TIMEOUT_MILLIS = 60000; + + private static final long INTERRUPTED_WAIT_MILLIS = 5000; + + private static final String MAC_MESSAGE = + "File watchers are not well implemented in OS X, " + + "so don't run this unit test as it is unlikely to pass."; + + private static final String POLLER_MESSAGE = + "The watch service on OS X polls from a thread this test cannot " + + "pick out."; + + @Test + public void WatchedFileRefresh_PollingCarriesOn() throws Exception { + assumeFalse(MAC_MESSAGE, isMac()); + + final List timers = new ArrayList<>(); + final AtomicReference watcher = new AtomicReference<>(); + FileWrapperFactory fileWrapperFactory = mock(FileWrapperFactory.class); + when(fileWrapperFactory.build(anyString())).then(invocation -> + new FileWrapperDefault((String) invocation.getArgument(0))); + when(fileWrapperFactory.getLastModified(anyString())).then( + invocation -> lastModified(invocation.getArgument(0))); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + mock(HttpClient.class), + fileWrapperFactory, + futureFactory(timers, watcher)); + + OnPremiseAspectEngine engine = engine(); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = watchedFile(engine, dataFile); + + try { + dataUpdate.registerDataFile(file); + assertEquals("Registering sets one polling timer", + 1, timers.size()); + assertNotNull("Registering sets up the file watcher", + watcher.get()); + + Files.write(dataFile, "New".getBytes()); + long end = System.currentTimeMillis() + WAIT_FOR_EVENT_MILLIS; + while (refreshCount(engine) == 0 && + System.currentTimeMillis() < end) { + watcher.get().run(); + Thread.sleep(RETRY_MILLIS); + } + + assertEquals("The watched file refreshes the engine once", + 1, refreshCount(engine)); + assertEquals("A new polling timer is set after the refresh", + 2, timers.size()); + assertNotNull("The data file holds the new polling timer", + file.getFuture()); + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * The file system watcher runs on its own setting. Watching a file + * downloads nothing, so the watcher does not wait for automatic updates + * to be on, and with its own setting off a replaced file is not loaded. + * Polling runs only with automatic updates on. + */ + @Test + public void WatcherRunsOnItsOwnSetting() throws Exception { + String[] names = { + "watcher on, automatic updates on", + "watcher on, automatic updates off", + "watcher off, automatic updates on"}; + boolean[] automatic = {true, false, true}; + boolean[] watching = {true, true, false}; + for (int i = 0; i < names.length; i++) { + checkWatcherSetting(names[i], automatic[i], watching[i]); + } + } + + private static void checkWatcherSetting( + String name, + boolean automatic, + boolean watching) throws Exception { + final List timers = new ArrayList<>(); + final AtomicReference watcher = new AtomicReference<>(); + FileWrapperFactory fileWrapperFactory = mock(FileWrapperFactory.class); + when(fileWrapperFactory.build(anyString())).then(invocation -> + new FileWrapperDefault((String) invocation.getArgument(0))); + when(fileWrapperFactory.getLastModified(anyString())).then( + invocation -> lastModified(invocation.getArgument(0))); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + mock(HttpClient.class), + fileWrapperFactory, + futureFactory(timers, watcher)); + + OnPremiseAspectEngine engine = engine(); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = + watchedFile(engine, dataFile, automatic, watching); + + try { + dataUpdate.registerDataFile(file); + assertEquals(name + ", a file system watcher was created", + watching, file.getConfiguration().getWatchKey() != null); + assertEquals(name + ", the watcher's task was scheduled", + watching, watcher.get() != null); + assertEquals(name + ", polling timers set", + automatic ? 1 : 0, timers.size()); + + // macOS watchers are too slow to wait for. The test holds the + // polling timers, so only a watcher can load the replaced file. + if (isMac() == false) { + Files.write(dataFile, "New".getBytes()); + Runnable task = watcher.get(); + long end = System.currentTimeMillis() + WAIT_FOR_EVENT_MILLIS; + while (task != null && refreshCount(engine) == 0 && + System.currentTimeMillis() < end) { + task.run(); + Thread.sleep(RETRY_MILLIS); + } + assertEquals(name + ", the replaced file was loaded", + watching ? 1 : 0, refreshCount(engine)); + } + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * Each watch service runs a thread of its own. Unregistering a watched + * data file, as closing its engine does, closes the watch service so its + * thread ends. + */ + @Test + public void Unregistering_ClosesTheWatchService() throws Exception { + assumeFalse(POLLER_MESSAGE, isMac()); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + mock(HttpClient.class), + fileWrapperFactory(), + futureFactory(new ArrayList<>(), new AtomicReference<>())); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = + watchedFile(engine(), dataFile, false, true); + + try { + Set before = watchServiceThreads(); + dataUpdate.registerDataFile(file); + assertEquals("Registering starts a watch service thread", + 1, waitForThreadsStartedSince(before, 1).size()); + + dataUpdate.unregisterDataFile(file); + assertEquals("Unregistering ends the watch service thread", + 0, waitForThreadsStartedSince(before, 0).size()); + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * A downloaded data file is copied into place with the watch paused. + * The watch then resumes on the data file's own watch service, so no + * other watch service is left running, and unregistering ends the one + * there is. + */ + @Test + public void DownloadedFile_WatchResumesOnTheSameWatchService() + throws Exception { + assumeFalse(POLLER_MESSAGE, isMac()); + final List timers = new ArrayList<>(); + HttpURLConnection connection = mock(HttpURLConnection.class); + when(connection.getResponseCode()) + .thenReturn(HttpURLConnection.HTTP_OK); + when(connection.getInputStream()).thenReturn( + new ByteArrayInputStream("New".getBytes())); + HttpClient httpClient = mock(HttpClient.class); + when(httpClient.connect(any(URL.class))).thenReturn(connection); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + httpClient, + fileWrapperFactory(), + timerFactory(timers)); + + OnPremiseAspectEngine engine = engine(); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + Path tempDirectory = Files.createTempDirectory("download"); + AspectEngineDataFileDefault file = + watchedFile(engine, dataFile, true, true); + file.setTempDataDirPath(tempDirectory.toString()); + file.getConfiguration().setDataUpdateUrl("http://www.test.com"); + file.getConfiguration().setDecompressContent(false); + file.getConfiguration().setVerifyMd5(false); + file.getConfiguration().setVerifyModifiedSince(false); + + try { + Set before = watchServiceThreads(); + dataUpdate.registerDataFile(file); + assertEquals("Registering starts a watch service thread", + 1, waitForThreadsStartedSince(before, 1).size()); + + // The polling timer downloads the new file. + timers.get(0).run(); + assertEquals("The downloaded file refreshes the engine", + 1, refreshCount(engine)); + assertNotNull("The watch resumes after the download", + file.getConfiguration().getWatchKey()); + assertEquals("The watch resumes on the same watch service", + 1, waitForThreadsStartedSince(before, 1).size()); + + dataUpdate.unregisterDataFile(file); + assertEquals("Unregistering ends the watch service thread", + 0, waitForThreadsStartedSince(before, 0).size()); + } + finally { + cleanUp(file, directory, dataFile); + tempDirectory.toFile().delete(); + } + } + + /** + * The watcher's task repeats, and a repeating task that fails is never + * run again. So while the watch is paused, as it is when a downloaded + * file is copied into place, the task does nothing and carries on. + * Unregistering the data file while the watch is paused still stops the + * task. + */ + @Test + public void WatchPaused_WatcherTaskCarriesOn() throws Exception { + final AtomicReference watcher = new AtomicReference<>(); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + mock(HttpClient.class), + fileWrapperFactory(), + futureFactory(new ArrayList<>(), watcher)); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = + watchedFile(engine(), dataFile, false, true); + + try { + dataUpdate.registerDataFile(file); + WatchKey watchKey = file.getConfiguration().getWatchKey(); + assertNotNull("A file system watcher was created", watchKey); + ScheduledFuture task = file.getPollFuture(); + + file.getConfiguration().setWatchKey(null); + watcher.get().run(); + file.getConfiguration().setWatchKey(watchKey); + watcher.get().run(); + + file.getConfiguration().setWatchKey(null); + dataUpdate.unregisterDataFile(file); + verify(task).cancel(true); + watchKey.cancel(); + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * The watcher's task polls the watch key, so with no watch key, here + * because the data file's folder does not exist, no task is scheduled. + */ + @Test + public void NoWatchKey_NoWatcherTask() throws Exception { + final AtomicReference watcher = new AtomicReference<>(); + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + mock(HttpClient.class), + fileWrapperFactory(), + futureFactory(new ArrayList<>(), watcher)); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("missing").resolve("data.dat"); + AspectEngineDataFileDefault file = + watchedFile(engine(), dataFile, false, true); + + try { + dataUpdate.registerDataFile(file); + assertNull("No watch key without the data file's folder", + file.getConfiguration().getWatchKey()); + assertNull("No watcher task without a watch key", + watcher.get()); + dataUpdate.unregisterDataFile(file); + } + finally { + directory.toFile().delete(); + } + } + + /** + * Registering a data file again that has no watch key, as it has while + * the watch is paused for a download, creates the watch key again and + * replaces the watcher's task rather than leave the old one running. + */ + @Test + public void RegisteredAgain_WatcherTaskReplaced() throws Exception { + DataUpdateService dataUpdate = new DataUpdateServiceDefault( + null, + mock(HttpClient.class), + fileWrapperFactory(), + futureFactory(new ArrayList<>(), new AtomicReference<>())); + Path directory = Files.createTempDirectory("watched"); + Path dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + AspectEngineDataFileDefault file = + watchedFile(engine(), dataFile, false, true); + + try { + dataUpdate.registerDataFile(file); + ScheduledFuture first = file.getPollFuture(); + assertNotNull("The watcher's task was scheduled", first); + + file.getConfiguration().setWatchKey(null); + dataUpdate.registerDataFile(file); + assertNotNull("The watch key is created again", + file.getConfiguration().getWatchKey()); + verify(first).cancel(false); + dataUpdate.unregisterDataFile(file); + } + finally { + cleanUp(file, directory, dataFile); + } + } + + /** + * A watched file that cannot be opened is tried, with a pause between + * tries, until the wait ends. The update then reports + * AUTO_UPDATE_TIMEOUT, logs nothing, and leaves the engine and the + * polling timer as they were. + */ + @Test + public void WatchedFileCannotBeOpened_GivesUpAfterTheWait() + throws Exception { + assumeFalse(MAC_MESSAGE, isMac()); + try (LockedFile locked = new LockedFile(OPEN_TIMEOUT_MILLIS)) { + locked.start(); + boolean gaveUp = locked.done.await( + WAIT_FOR_EVENT_MILLIS, TimeUnit.MILLISECONDS); + + assertTrue("The watcher tries to open the new file", + locked.tries.get() > 0); + assertTrue("The update gives up on a file it cannot open", + gaveUp); + assertEquals(AutoUpdateStatus.AUTO_UPDATE_TIMEOUT, + locked.status.get()); + assertTrue("The tries are paused, but " + locked.tries.get() + + " were made", locked.tries.get() <= MAX_TRIES); + locked.assertLeftAsItWas(); + } + } + + /** + * An interrupt, as unregistering the data file gives the watcher's task, + * ends the wait at once. The update is left unfinished, so it reports + * AUTO_UPDATE_IN_PROGRESS rather than a timeout, logs nothing, and leaves + * the engine and the polling timer as they were. + */ + @Test + public void WatchedFileWaitInterrupted_LeavesTheUpdateUnfinished() + throws Exception { + assumeFalse(MAC_MESSAGE, isMac()); + try (LockedFile locked = new LockedFile(LONG_OPEN_TIMEOUT_MILLIS)) { + locked.start(); + long end = System.currentTimeMillis() + WAIT_FOR_EVENT_MILLIS; + while (locked.tries.get() == 0 && + System.currentTimeMillis() < end) { + Thread.sleep(RETRY_MILLIS); + } + assertTrue("The watcher tries to open the new file", + locked.tries.get() > 0); + + locked.runner.interrupt(); + boolean ended = locked.done.await( + INTERRUPTED_WAIT_MILLIS, TimeUnit.MILLISECONDS); + + assertTrue("The wait ends when it is interrupted", ended); + assertEquals(AutoUpdateStatus.AUTO_UPDATE_IN_PROGRESS, + locked.status.get()); + locked.assertLeftAsItWas(); + } + } + + /** + * A watched data file that throws on every open, the service watching + * it, and the thread that runs the watcher's task. + */ + private static class LockedFile implements AutoCloseable { + + final List timers = new ArrayList<>(); + final AtomicReference watcher = new AtomicReference<>(); + final AtomicBoolean locked = new AtomicBoolean(true); + final AtomicInteger tries = new AtomicInteger(); + final AtomicInteger logged = new AtomicInteger(); + final AtomicReference status = + new AtomicReference<>(); + final CountDownLatch done = new CountDownLatch(1); + final AtomicBoolean stop = new AtomicBoolean(false); + final OnPremiseAspectEngine engine = engine(); + final DataUpdateServiceDefault service; + final Path directory; + final Path dataFile; + final AspectEngineDataFileDefault file; + final Thread runner; + int loggedAtStart; + + LockedFile(long timeoutMillis) throws Exception { + // The service may call these many times while it waits, so they + // are stub only mocks, which keep no record of their calls. + final BinaryReader reader = + mock(BinaryReader.class, withSettings().stubOnly()); + FileWrapper wrapper = + mock(FileWrapper.class, withSettings().stubOnly()); + when(wrapper.getReader()).then(invocation -> { + tries.incrementAndGet(); + if (locked.get()) { + // Stops a loop with no pause using a whole processor, + // and leaves an interrupt for the service to see. + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + throw new IOException("The file is locked"); + } + return reader; + }); + FileWrapperFactory fileWrapperFactory = + mock(FileWrapperFactory.class, withSettings().stubOnly()); + when(fileWrapperFactory.build(anyString())).thenReturn(wrapper); + when(fileWrapperFactory.getLastModified(anyString())).then( + invocation -> lastModified(invocation.getArgument(0))); + Logger logger = mock(Logger.class, withSettings().stubOnly() + .defaultAnswer(invocation -> { + String name = invocation.getMethod().getName(); + if (name.equals("warn") || name.equals("error")) { + logged.incrementAndGet(); + } + return Answers.RETURNS_DEFAULTS.answer(invocation); + })); + service = new DataUpdateServiceDefault( + logger, + mock(HttpClient.class), + fileWrapperFactory, + futureFactory(timers, watcher)); + service.watchedFileOpenTimeoutMillis = timeoutMillis; + // The first update after the change is the one that waits, + // because only the watcher sets the time it compares with. + service.onUpdateComplete((sender, args) -> { + if (status.compareAndSet(null, args.getStatus())) { + done.countDown(); + } + }); + + directory = Files.createTempDirectory("watched"); + dataFile = directory.resolve("data.dat"); + Files.write(dataFile, "Old".getBytes()); + file = watchedFile(engine, dataFile); + + // The watcher runs on its own thread, so the test can stop + // waiting for an update that does not end. + runner = new Thread(() -> { + while (stop.get() == false && done.getCount() > 0) { + watcher.get().run(); + try { + Thread.sleep(RETRY_MILLIS); + } catch (InterruptedException e) { + return; + } + } + }); + runner.setDaemon(true); + } + + /** + * Registers the file, changes it, and starts the watcher's thread. + */ + void start() throws Exception { + service.registerDataFile(file); + assertNotNull("Registering sets up the file watcher", + watcher.get()); + loggedAtStart = logged.get(); + Files.write(dataFile, "New".getBytes()); + runner.start(); + } + + void assertLeftAsItWas() { + assertEquals("The engine is not refreshed", + 0, refreshCount(engine)); + assertEquals("Nothing is logged", loggedAtStart, logged.get()); + assertEquals("The polling timer is kept", 1, timers.size()); + assertNotNull("The data file holds the polling timer", + file.getFuture()); + } + + @Override + public void close() throws Exception { + // Lets an update that would not give up end, so the thread stops. + stop.set(true); + locked.set(false); + if (runner.isAlive()) { + runner.join(WAIT_FOR_EVENT_MILLIS); + } + cleanUp(file, directory, dataFile); + } + } + + /** + * Records the polling timers scheduled, and keeps the watcher's task for + * the test to run itself. + */ + private static FutureFactory futureFactory( + final List timers, + final AtomicReference watcher) { + FutureFactory futureFactory = mock(FutureFactory.class); + when(futureFactory.schedule(any(Runnable.class), anyLong())) + .then(invocation -> { + timers.add((Long) invocation.getArgument(1)); + return mock(ScheduledFuture.class); + }); + when(futureFactory.scheduleRepeating(any(Runnable.class), anyLong())) + .then(invocation -> { + watcher.set((Runnable) invocation.getArgument(0)); + return mock(ScheduledFuture.class); + }); + return futureFactory; + } + + /** + * Keeps each one-off timer's task for the test to run itself. + */ + private static FutureFactory timerFactory(final List timers) { + FutureFactory futureFactory = mock(FutureFactory.class); + when(futureFactory.schedule(any(Runnable.class), anyLong())) + .then(invocation -> { + timers.add((Runnable) invocation.getArgument(0)); + return mock(ScheduledFuture.class); + }); + when(futureFactory.scheduleRepeating(any(Runnable.class), anyLong())) + .then(invocation -> mock(ScheduledFuture.class)); + return futureFactory; + } + + private static FileWrapperFactory fileWrapperFactory() { + FileWrapperFactory fileWrapperFactory = mock(FileWrapperFactory.class); + when(fileWrapperFactory.build(anyString())).then(invocation -> + new FileWrapperDefault((String) invocation.getArgument(0))); + when(fileWrapperFactory.getLastModified(anyString())).then( + invocation -> lastModified(invocation.getArgument(0))); + return fileWrapperFactory; + } + + /** + * The watch service threads running, found by the watch service's + * poller in each thread's stack. + */ + private static Set watchServiceThreads() { + Set threads = new HashSet<>(); + for (Map.Entry thread : + Thread.getAllStackTraces().entrySet()) { + if (Arrays.stream(thread.getValue()).anyMatch(frame -> + frame.getClassName().endsWith("WatchService$Poller"))) { + threads.add(thread.getKey()); + } + } + return threads; + } + + /** + * Wait for the watch service threads running that were not running + * before to number the count given, as a thread starts and ends a moment + * after its watch service is created or closed. Threads running before + * are left out, so one of those ending makes no difference. + */ + private static Set waitForThreadsStartedSince( + Set before, + int count) throws InterruptedException { + long end = System.currentTimeMillis() + WAIT_FOR_EVENT_MILLIS; + Set started = watchServiceThreads(); + started.removeAll(before); + while (started.size() != count && + System.currentTimeMillis() < end) { + Thread.sleep(RETRY_MILLIS); + started = watchServiceThreads(); + started.removeAll(before); + } + return started; + } + + @SuppressWarnings("unchecked") + private static OnPremiseAspectEngine engine() { + return mock(OnPremiseAspectEngine.class); + } + + private static AspectEngineDataFileDefault watchedFile( + OnPremiseAspectEngine engine, + Path dataFile) { + return watchedFile(engine, dataFile, true, true); + } + + private static AspectEngineDataFileDefault watchedFile( + OnPremiseAspectEngine engine, + Path dataFile, + boolean automatic, + boolean watching) { + DataFileConfigurationDefault config = new DataFileConfigurationDefault(); + config.setAutomaticUpdatesEnabled(automatic); + config.setFileSystemWatcherEnabled(watching); + config.setPollingIntervalSeconds(1); + config.setDataFilePath(dataFile.toString()); + AspectEngineDataFileDefault file = new AspectEngineDataFileDefault(); + file.setEngine(engine); + file.setConfiguration(config); + return file; + } + + private static void cleanUp( + AspectEngineDataFileDefault file, + Path directory, + Path dataFile) { + if (file.getConfiguration().getWatchKey() != null) { + file.getConfiguration().getWatchKey().cancel(); + } + // Windows keeps the file until the reader's mapped buffer is + // collected, so whatever cannot be deleted now is deleted when the + // test run ends. Those are deleted in the reverse order they are + // registered, so the folder is registered first. + directory.toFile().deleteOnExit(); + if (dataFile.toFile().delete() == false) { + dataFile.toFile().deleteOnExit(); + } + directory.toFile().delete(); + } + + private static boolean isMac() { + return System.getProperty("os.name").contains("Mac OS X"); + } + + private static long lastModified(Object path) { + return new File((String) path).lastModified(); + } + + private static long refreshCount(Object engine) { + return mockingDetails(engine).getInvocations().stream() + .filter(i -> i.getMethod().getName().equals("refreshData")) + .count(); + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java new file mode 100644 index 000000000..d9094b438 --- /dev/null +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/DataUploaderHttpTests.java @@ -0,0 +1,132 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.engines.services; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import com.sun.net.httpserver.HttpServer; + +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.util.HashMap; + +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * An upload to a service that accepts the connection and never answers must + * end, or it would stop all usage sharing. + */ +public class DataUploaderHttpTests { + + /** + * How long the uploader waits for an answer. + */ + private static final int TIMEOUT_MILLIS = 500; + + /** + * How long the test service holds a connection open without answering. + */ + private static final int HOLD_MILLIS = 30000; + + /** + * The test fails if the upload has not ended by then. + */ + private static final int LIMIT_MILLIS = 15000; + + /** + * The JDK decides whether a request may carry a restricted header, Origin + * among them, from this property when it first loads its HTTP connection + * class, and the value it reads then stands for the rest of the run. + * HttpClientDefault.connect sets it and relies on being the first to open + * a connection. These tests open one as well, so they set the same value + * first and leave that class working whichever test runs earlier. + */ + @BeforeClass + public static void AllowRestrictedHeaders() { + System.setProperty("sun.net.http.allowRestrictedHeaders", "true"); + } + + @Test(timeout = LIMIT_MILLIS) + public void DataUploaderHttp_NoAnswer_TimesOut() throws Exception { + try (ServerSocket server = new ServerSocket(0)) { + Thread holder = new Thread(() -> { + try (Socket socket = server.accept()) { + Thread.sleep(HOLD_MILLIS); + } catch (Exception ignored) { + // The test has ended and closed the server. + } + }); + holder.setDaemon(true); + holder.start(); + DataUploaderHttp uploader = new DataUploaderHttp( + "http://localhost:" + server.getLocalPort() + "/usage", + new HashMap<>(), + TIMEOUT_MILLIS); + try (OutputStream out = uploader.getOutputStream()) { + out.write(new byte[] {1}); + } + + try { + uploader.getResponseCode(); + fail("The upload was answered."); + } catch (SocketTimeoutException expected) { + // The wait for an answer ended. + } + } + } + + @Test(timeout = LIMIT_MILLIS) + public void DataUploaderHttp_FailedAnswer_GivesTheContent() + throws Exception { + byte[] answer = "the service says no".getBytes(StandardCharsets.UTF_8); + HttpServer server = HttpServer.create(new InetSocketAddress(0), 0); + server.createContext("/usage", exchange -> { + exchange.sendResponseHeaders(500, answer.length); + exchange.getResponseBody().write(answer); + exchange.close(); + }); + server.start(); + try { + DataUploaderHttp uploader = new DataUploaderHttp( + "http://localhost:" + server.getAddress().getPort() + "/usage", + new HashMap<>(), + TIMEOUT_MILLIS * 10); + try (OutputStream out = uploader.getOutputStream()) { + out.write(new byte[] {1}); + } + + assertEquals(500, uploader.getResponseCode()); + assertEquals( + "the service says no", + uploader.getResponseContent()); + } finally { + server.stop(0); + } + } +} diff --git a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/HttpClientTests.java b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/HttpClientTests.java index f09b13400..e7cf616f5 100644 --- a/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/HttpClientTests.java +++ b/pipeline/pipeline.engines/src/test/java/fiftyone/pipeline/engines/services/HttpClientTests.java @@ -35,6 +35,17 @@ import org.junit.Before; import org.junit.Test; +/** + * Calls the live 51Degrees cloud service, and fails rather than skips when + * it cannot be reached, because a test of an HTTP client that passes + * without making a request proves nothing. + *

      + * The resource key below is fixed on purpose and must not be replaced with + * a key from the environment. It is restricted to the 51degrees.com + * origin, which is what gives the two answers these tests need from one + * address, being 401 with no Origin header and 200 with one. A key without + * that restriction would answer 200 to both. + */ public class HttpClientTests { private HttpClient client; diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java index 4118f86f9..c9118d980 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrl.java @@ -30,10 +30,12 @@ * The callback URL the client script posts to for updated JSON. The protocol * and host usually come from the request, and the script holds the URL in a * single-quoted string, so the URL is checked before it is written into the - * script. A URL that is not valid is refused. In the user information, in - * anything after the host and port, and in the endpoint, the characters a - * URL cannot hold or that could end the string or the script are - * percent-encoded. + * script. A URL that is not valid, including one whose protocol is not http + * or https, is refused. In anything after the authority, and in the + * endpoint, the characters a URL cannot hold or that could end the string or + * the script are percent-encoded. The rules are applied to the text given, + * and not to the result of parsing it, because a parser can accept hosts the + * rules refuse and rewrite others, such as 127.1. * @see Specification */ final class CallbackUrl { @@ -71,27 +73,28 @@ final class CallbackUrl { "\\u2028\\u2029\\u202F\\u205F\\u3000]*"); /** - * A scheme as RFC 3986 section 3.1 defines one, being a letter followed - * by letters, digits, '+', '-' and '.'. + * The schemes the script can send its request with. It uses the URL only + * with fetch and XMLHttpRequest, which reach a server with no other + * scheme. */ - private static final Pattern SCHEME = - Pattern.compile("[A-Za-z][A-Za-z0-9+.-]*"); + private static final String[] SCHEMES = {"http", "https"}; /** - * A label of a host name, being ASCII letters, digits, '-' and '_'. + * The characters the host value cannot hold. A '?' or '#' would make the + * endpoint part of the query or fragment, and an '@' means user + * information, which fetch refuses and XMLHttpRequest sends. */ - private static final Pattern LABEL = Pattern.compile("[A-Za-z0-9_-]+"); + private static final String NOT_IN_HOST = "?#@"; /** - * An IPv6 zone identifier, being unreserved characters as RFC 3986 - * section 2.3 defines them. + * A label of a host name, being ASCII letters, digits, '-' and '_'. */ - private static final Pattern ZONE = Pattern.compile("[A-Za-z0-9._~-]+"); + private static final Pattern LABEL = Pattern.compile("[A-Za-z0-9_-]+"); /** - * A port as it follows the host, being ':' and any number of digits. + * A port as it follows the host, being ':' and at least one digit. */ - private static final Pattern PORT = Pattern.compile(":[0-9]*"); + private static final Pattern PORT = Pattern.compile(":[0-9]+"); /** * One group of an IPv6 address. @@ -107,6 +110,13 @@ final class CallbackUrl { "(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])" + "(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}"); + /** + * A label that the URL Standard reads as a number, being decimal digits + * or "0x" followed by hexadecimal digits. + */ + private static final Pattern NUMBER = + Pattern.compile("[0-9]+|0[xX][0-9A-Fa-f]*"); + /** * Whether the URL can be written into the script. */ @@ -114,7 +124,8 @@ final class CallbackUrl { /** * The URL to write into the script when it is valid, and otherwise the - * URL as it was built, for the log. + * URL as it was built, for the log, with *** in place of everything in + * the host value before its last '@'. */ final String url; @@ -128,9 +139,8 @@ private CallbackUrl(boolean valid, String url) { * separates the host and the endpoint, being added when neither has one * and removed when both do. The protocol and the host are written in * lower case and the rest as given, apart from the percent-encoding. - * @param protocol the protocol, such as https - * @param host the host, which can start with user information and end - * with a port, a path, a query or a fragment + * @param protocol the protocol, http or https in any case + * @param host the host, which can end with a port and a path * @param endpoint the endpoint * @return the callback URL, or null when the protocol, host or endpoint * is not set @@ -146,67 +156,104 @@ static CallbackUrl of(String protocol, String host, String endpoint) { } else if (hostSlash && endpointSlash) { endpoint = endpoint.substring(1); } - String built = protocol + "://" + host + endpoint; - if (SCHEME.matcher(protocol).matches() == false) { - return new CallbackUrl(false, built); - } int end = authorityEnd(host); - String authority = checkedAuthority(host.substring(0, end)); + String authority = isScheme(protocol) && hasNoForbiddenCharacter(host) + ? checkedHostPort(host.substring(0, end)) + : null; if (authority == null) { - return new CallbackUrl(false, built); + return new CallbackUrl( + false, + protocol + "://" + withoutUserInformation(host) + endpoint); } return new CallbackUrl( true, - protocol.toLowerCase(Locale.ROOT) + "://" + authority + + asciiLowerCase(protocol) + "://" + authority + encoded(host.substring(end)) + encoded(endpoint)); } /** - * Whether a part of the URL is not set, being null, empty or only - * whitespace. - * @param text the part - * @return true if the part is not set + * Whether the host value holds none of '?', '#' and '@'. */ - static boolean isBlank(String text) { - return text == null || BLANK.matcher(text).matches(); + private static boolean hasNoForbiddenCharacter(String host) { + for (int i = 0; i < host.length(); i++) { + if (NOT_IN_HOST.indexOf(host.charAt(i)) >= 0) { + return false; + } + } + return true; } /** - * Where the authority ends, which is at the first '/', '?' or '#' of the - * host, or at its end. + * The host value with everything before its last '@' replaced by ***. + * The last '@' of the whole value is used, and not that of the + * authority, because a password can hold a '/', '?' or '#'. */ - private static int authorityEnd(String host) { - for (int i = 0; i < host.length(); i++) { - char c = host.charAt(i); - if (c == '/' || c == '?' || c == '#') { - return i; + private static String withoutUserInformation(String host) { + int at = host.lastIndexOf('@'); + return at < 0 ? host : "***" + host.substring(at); + } + + /** + * Whether the protocol is http or https, ignoring the case of its ASCII + * letters only. String.equalsIgnoreCase also matches a letter outside + * ASCII to an ASCII one, such as the long s (U+017F) to 's', so it is + * not used. + * @param protocol the protocol + * @return true if the protocol is http or https + */ + static boolean isScheme(String protocol) { + if (protocol == null) { + return false; + } + String lower = asciiLowerCase(protocol); + for (String scheme : SCHEMES) { + if (scheme.equals(lower)) { + return true; } } - return host.length(); + return false; } /** - * The user information, host and port as they are written into the URL, - * or null when they are not valid. The user information ends at the - * first '@', so a second '@' is part of the host and refused. + * The text with the ASCII letters A to Z in lower case and every other + * character as it is. + * @param text the text + * @return the text with A to Z in lower case */ - private static String checkedAuthority(String authority) { - int at = authority.indexOf('@'); - if (at < 0) { - return checkedHostPort(authority); + private static String asciiLowerCase(String text) { + StringBuilder lower = new StringBuilder(text.length()); + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + lower.append(c >= 'A' && c <= 'Z' ? (char) (c + ('a' - 'A')) : c); } - String hostPort = checkedHostPort(authority.substring(at + 1)); - return hostPort == null - ? null - : encoded(authority.substring(0, at)) + "@" + hostPort; + return lower.toString(); + } + + /** + * Whether a part of the URL is not set, being null, empty or only + * whitespace. + * @param text the part + * @return true if the part is not set + */ + static boolean isBlank(String text) { + return text == null || BLANK.matcher(text).matches(); + } + + /** + * Where the authority ends, which is at the first '/' of the host, or at + * its end. + */ + private static int authorityEnd(String host) { + int end = host.indexOf('/'); + return end < 0 ? host.length() : end; } /** * The host and optional port in lower case, or null when they are not - * valid. The host is an IPv6 address in brackets with an optional zone - * identifier, or a name of labels separated by dots with a dot allowed - * at the end. The port is ':' followed by digits up to 65535, or by - * nothing. + * valid. The host is an IPv6 address in brackets, or a name of labels + * separated by dots with a dot allowed at the end, which must be an IPv4 + * address when its last label is a number. The port is ':' followed by + * digits up to 65535. */ private static String checkedHostPort(String hostPort) { String host; @@ -217,14 +264,6 @@ private static String checkedHostPort(String hostPort) { return null; } String address = hostPort.substring(1, close); - int zone = address.indexOf("%25"); - if (zone >= 0) { - if (ZONE.matcher(address.substring(zone + 3)).matches() == - false) { - return null; - } - address = address.substring(0, zone); - } if (isIpv6(address) == false) { return null; } @@ -238,11 +277,16 @@ private static String checkedHostPort(String hostPort) { String name = host.endsWith(".") ? host.substring(0, host.length() - 1) : host; - for (String label : name.split("\\.", -1)) { + String[] labels = name.split("\\.", -1); + for (String label : labels) { if (LABEL.matcher(label).matches() == false) { return null; } } + if (NUMBER.matcher(labels[labels.length - 1]).matches() && + IPV4.matcher(host).matches() == false) { + return null; + } } if (port.isEmpty() == false && isPort(port) == false) { return null; @@ -252,7 +296,7 @@ private static String checkedHostPort(String hostPort) { /** * Whether the text is ':' followed by digits whose value is no more than - * 65535, or by nothing. + * 65535. */ private static boolean isPort(String text) { if (PORT.matcher(text).matches() == false) { diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java index f8d828dc5..509fc1b0b 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElement.java @@ -32,6 +32,8 @@ import fiftyone.pipeline.core.flowelements.FlowElementBase; import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.fiftyone.data.SequenceData; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; import fiftyone.pipeline.javascriptbuilder.Constants; import fiftyone.pipeline.javascriptbuilder.data.JavaScriptBuilderData; import fiftyone.pipeline.javascriptbuilder.templates.JavaScriptResource; @@ -89,9 +91,11 @@ public class JavaScriptBuilderElement * Words the pattern accepts that cannot be the name of the object. These * are the reserved words of the language, including those reserved only * in strict mode, plus the three global values a top level var cannot - * replace, where the object would silently never be created. The last - * entry is the constructor the script itself defines and calls to create - * the object, so that name would clash with it. + * replace, eval, and the properties of the browser's Window object that a + * top level var cannot replace or that navigate the page away, where the + * object would never be created. The last entry is the constructor the + * script itself defines and calls to create the object, so that name + * would clash with it. */ private static final Set RESERVED_OBJECT_NAMES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( @@ -103,6 +107,15 @@ public class JavaScriptBuilderElement "return", "static", "super", "switch", "this", "throw", "true", "try", "typeof", "var", "void", "while", "with", "yield", "Infinity", "NaN", "undefined", + "eval", "arguments", + "window", "document", "top", "name", "status", "history", + "navigator", "closed", "location", "frameElement", + "isSecureContext", "crossOriginIsolated", "trustedTypes", + "crypto", "indexedDB", "localStorage", "sessionStorage", + "caches", "cookieStore", "speechSynthesis", "styleMedia", + "crashReport", "credentialless", "customElements", + "documentPictureInPicture", "fence", "launchQueue", + "originAgentCluster", "fiftyoneDegreesManager"))); /** @@ -232,8 +245,9 @@ protected void processInternal(FlowData data) throws Exception { String url = getUrl(reqProtocol, reqHost); - String sessionId = getSessionId(data); - int sequence = getSequence(data); + ElementData sequenceData = getSequenceData(data); + String sessionId = getSessionId(sequenceData); + int sequence = getSequence(sequenceData); String serializedParameters = serializeParameters(parameters); // With the gathered resources, build a new JavaScriptResource. @@ -462,52 +476,77 @@ private String getPath() { /** * Gets the session id to write into the script. This is the session id - * evidence, set by the Sequence Element or by the request, when it - * matches SESSION_ID_PATTERN, and an empty string otherwise. - * @param data the flow data + * the Sequence Element wrote, when it matches SESSION_ID_PATTERN, and an + * empty string otherwise, including when the pipeline has no Sequence + * Element. + * @param sequenceData the Sequence Element's data, or null * @return the session id, or an empty string */ - private String getSessionId(FlowData data) { - TryGetResult trySessionId = - data.tryGetEvidence(EVIDENCE_SESSIONID, Object.class); - if (trySessionId.hasValue() && - trySessionId.getValue() instanceof String) { - String sessionId = (String) trySessionId.getValue(); - if (SESSION_ID_PATTERN.matcher(sessionId).matches()) { - return sessionId; - } + private String getSessionId(ElementData sequenceData) { + Object value = getSequenceValue( + sequenceData, SequenceData.SESSION_ID_PROPERTY_NAME); + if (value instanceof String && + SESSION_ID_PATTERN.matcher((String) value).matches()) { + return (String) value; } return ""; } /** - * Gets the sequence to write into the script. This is the sequence - * evidence, as a number or as text, when it is a positive 32 bit - * integer, and 1 otherwise, including when there is no Sequence Element. - * The value is written into the script as code, so text that is not a - * number could break the script, and a number below 1 is not a sequence - * the Sequence Element would ever give. - * @param data the flow data + * Gets the sequence to write into the script. This is the sequence the + * Sequence Element wrote, as a number or as text, when it is a positive + * 32 bit integer, and 1 otherwise, including when the pipeline has no + * Sequence Element. The value is written into the script as code, so + * text that is not a number could break the script, and a number below 1 + * is not a sequence the Sequence Element would ever give. + * @param sequenceData the Sequence Element's data, or null * @return the sequence, which is always at least 1 */ - private int getSequence(FlowData data) { - TryGetResult trySequence = - data.tryGetEvidence(EVIDENCE_SEQUENCE, Object.class); + private int getSequence(ElementData sequenceData) { + Object value = getSequenceValue( + sequenceData, SequenceData.SEQUENCE_PROPERTY_NAME); long sequence = 0; - if (trySequence.hasValue()) { - Object value = trySequence.getValue(); - if (value instanceof Integer) { - sequence = (Integer) value; - } else if (value instanceof String && - SEQUENCE_PATTERN.matcher((String) value).matches()) { - sequence = Long.parseLong((String) value); - } + if (value instanceof Integer) { + sequence = (Integer) value; + } else if (value instanceof String && + SEQUENCE_PATTERN.matcher((String) value).matches()) { + sequence = Long.parseLong((String) value); } return sequence >= 1 && sequence <= Integer.MAX_VALUE ? (int) sequence : 1; } + /** + * The Sequence Element's data, or null when the pipeline has no Sequence + * Element. Step 5 of the specification takes the session id and the + * sequence from that data and not from Evidence, so a pipeline with no + * Sequence Element renders the defaults even when the request carried a + * session id or a sequence of its own. + * @param data the flow data + * @return the Sequence Element's data, or null + * @see Specification + */ + private static ElementData getSequenceData(FlowData data) { + // get(String) throws when the Flow Data holds nothing under the key, + // so the keys it does hold are checked first. + return data.getDataKeys().contains(SequenceElement.DEFAULT_ELEMENT_DATA_KEY) + ? data.get(SequenceElement.DEFAULT_ELEMENT_DATA_KEY) + : null; + } + + /** + * One value from the Sequence Element's data. + * @param sequenceData the Sequence Element's data, or null + * @param name the name of the value within that data + * @return the value, or null + */ + private static Object getSequenceValue( + ElementData sequenceData, + String name) { + return sequenceData == null ? null : sequenceData.get(name); + } + private void buildJavaScript( FlowData data, String jsonObject, diff --git a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java index 28e77260c..47a449373 100644 --- a/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java +++ b/pipeline/pipeline.javascriptbuilder/src/main/java/fiftyone/pipeline/javascriptbuilder/flowelements/JavaScriptBuilderElementBuilder.java @@ -105,20 +105,21 @@ public JavaScriptBuilderElementBuilder setContextRoot(String contextRoot) { *

      * By default, the protocol from the request will be * used. - * @param protocol The protocol to use (http / https) + * @param protocol The protocol to use, http or https in any case of its + * ASCII letters * @return this builder + * @throws PipelineConfigurationException if the protocol is neither http + * nor https */ @DefaultValue("The protocol from the request") public JavaScriptBuilderElementBuilder setProtocol(String protocol) { - if (protocol.equalsIgnoreCase("http") || - protocol.equalsIgnoreCase("https")) { + if (CallbackUrl.isScheme(protocol)) { this.protocol = protocol; } else { throw new PipelineConfigurationException( "Invalid protocol in configuration (" + protocol + - "), must be 'http' or https'"); - + "), must be 'http' or 'https'"); } return this; } diff --git a/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates b/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates index d3f96049b..41e7561f8 160000 --- a/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates +++ b/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates @@ -1 +1 @@ -Subproject commit d3f96049bde389951c79638047b8ea80dfdd5cdf +Subproject commit 41e7561f82dd4a3a498e6b16d81d2c5cdbcdda8a diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java index 5cfde9467..7ec0c95e1 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptBuilderTests.java @@ -30,6 +30,8 @@ import fiftyone.pipeline.engines.data.AspectPropertyValue; import fiftyone.pipeline.engines.data.AspectPropertyValueDefault; import fiftyone.pipeline.engines.exceptions.PropertyMissingException; +import fiftyone.pipeline.engines.fiftyone.data.SequenceData; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; import fiftyone.pipeline.javascriptbuilder.data.JavaScriptBuilderData; import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; @@ -57,6 +59,7 @@ import java.io.IOException; import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -527,6 +530,19 @@ private void configureMocks() { when(flowData.get(anyString())).thenReturn(elementDataMock); + // The Sequence Element's data, which is where the builder reads the + // session id and the sequence from, as a web integration's + // pipeline holds it. + ElementData sequenceData = mock(ElementData.class); + when(sequenceData.get(SequenceData.SESSION_ID_PROPERTY_NAME)) + .thenReturn(sessionId); + when(sequenceData.get(SequenceData.SEQUENCE_PROPERTY_NAME)) + .thenReturn(sequence); + when(flowData.getDataKeys()).thenReturn(Collections.singletonList( + SequenceElement.DEFAULT_ELEMENT_DATA_KEY)); + when(flowData.get(SequenceElement.DEFAULT_ELEMENT_DATA_KEY)) + .thenReturn(sequenceData); + when(flowData.getAs(anyString(), any(Class.class))).thenReturn(new AspectPropertyValueDefault<>("None")); doAnswer((Answer) invocation -> { diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java index 4c49c3fa6..712bc6873 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptObjectNameTests.java @@ -223,6 +223,110 @@ public void ObjectName_InvalidFromConstructor_Refused(String name) { "")); } + /** + * The names the specification refuses that are not reserved words of the + * language, being eval, arguments, and the properties of the browser's + * Window object that a top level var cannot replace or that navigate the + * page away. + */ + static final String[] REFUSED_NAMES = { + "eval", "arguments", "window", "document", "top", "name", "status", + "history", "navigator", "closed", "location", "frameElement", + "isSecureContext", "crossOriginIsolated", "trustedTypes", "crypto", + "indexedDB", "localStorage", "sessionStorage", "caches", + "cookieStore", "speechSynthesis", "styleMedia", "crashReport", + "credentialless", "customElements", "documentPictureInPicture", + "fence", "launchQueue", "originAgentCluster" + }; + + /** + * Window properties that a top level var can replace, so the names stay + * allowed. + */ + static final String[] ALLOWED_WINDOW_NAMES = { + "self", "frames", "parent", "opener", "length" + }; + + static Stream refusedNames() { + return Stream.of(REFUSED_NAMES); + } + + static Stream allowedWindowNames() { + return Stream.of(ALLOWED_WINDOW_NAMES); + } + + @ParameterizedTest + @MethodSource("refusedNames") + public void ObjectName_RefusedNameFromEvidence_DefaultUsed(String name) { + JavaScriptBuilderElement element = + new JavaScriptBuilderElementBuilder(loggerFactory).build(); + evidence.put(EVIDENCE_OBJECT_NAME, name); + + String script = render(element); + + assertWorkingObject(script, Constants.DEFAULT_OBJECT_NAME); + assertFalse(Pattern.compile("var " + Pattern.quote(name) + + " = new fiftyoneDegreesManager\\(\\);") + .matcher(withoutParameters(script)).find(), + "the script should not declare '" + name + "'"); + assertEquals(1, count(script, "new fiftyoneDegreesManager()")); + List warnings = warnings(); + assertEquals(1, warnings.size(), String.join("\n", warnings)); + } + + @ParameterizedTest + @MethodSource("refusedNames") + public void ObjectName_RefusedNameFromBuilder_Refused(String name) { + assertThrows( + PipelineConfigurationException.class, + () -> new JavaScriptBuilderElementBuilder(loggerFactory) + .setObjectName(name) + .build()); + } + + @ParameterizedTest + @MethodSource("refusedNames") + public void ObjectName_RefusedNameFromConstructor_Refused(String name) { + assertThrows( + PipelineConfigurationException.class, + () -> new JavaScriptBuilderElement( + loggerFactory.getLogger("test"), + null, + "", + name, + false, + "", + "")); + } + + @ParameterizedTest + @MethodSource("refusedNames") + public void ObjectName_IsValidObjectName_RefusedName(String name) { + assertFalse(JavaScriptBuilderElement.isValidObjectName(name)); + } + + @ParameterizedTest + @MethodSource("allowedWindowNames") + public void ObjectName_WindowPropertyVarCanReplace_Allowed(String name) { + assertTrue(JavaScriptBuilderElement.isValidObjectName(name)); + JavaScriptBuilderElement element = + new JavaScriptBuilderElementBuilder(loggerFactory) + .setObjectName(name) + .build(); + + String script = render(element); + + assertTrue(script.contains("var " + name + " = new"), + "the script should declare '" + name + "'"); + assertEquals(0, warnings().size()); + } + + @ParameterizedTest + @ValueSource(strings = {"windows", "Window", "cachesX"}) + public void ObjectName_NearMissOfRefusedName_Allowed(String name) { + assertTrue(JavaScriptBuilderElement.isValidObjectName(name)); + } + /** * No name configured means the default name, whether the name was never * set or was set to null. diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java index 2178a5f66..fc8cb2f61 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/JavaScriptUrlTests.java @@ -25,6 +25,7 @@ import fiftyone.common.testhelpers.TestLogger; import fiftyone.common.testhelpers.TestLoggerFactory; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; import fiftyone.pipeline.core.flowelements.Pipeline; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; @@ -32,6 +33,9 @@ import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import org.slf4j.ILoggerFactory; import org.slf4j.Logger; @@ -43,6 +47,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -226,6 +231,94 @@ public void Url_RefusedHost_NoCallbackAndLoggedOnce() throws Exception { errors.get(0)); } + /** + * A refused host is logged with *** in place of everything in the host + * value before its last '@', so a password is never written to the log. + * Each case has its own element, because a refusal is logged once per + * element. + */ + @ParameterizedTest(name = "[{index}]") + @CsvSource(delimiter = '|', value = { + "user:secret@example.com|\"https://***@example.com/json\"", + "a@b@example.com|\"https://***@example.com/json\"", + "u@[::1]:80x|\"https://***@[::1]:80x/json\"", + "user:pa/ss@example.com|\"https://***@example.com/json\"", + "user:pa?ss@example.com|\"https://***@example.com/json\"", + "exa mple.com/a@b|\"https://***@b/json\"", + "example.123|\"https://example.123/json\"", + "https://example.com|\"https://https://example.com/json\""}) + public void Url_RefusedHost_LoggedWithoutUserInformation( + String host, String logged) throws Exception { + renderWithHosts(host); + + List errors = builderErrors(); + assertEquals(1, errors.size(), String.join("\n", errors)); + assertTrue(errors.get(0).contains(logged), errors.get(0)); + for (String secret : new String[] {"secret", "pa/", "pa?", "ss@"}) { + assertFalse(errors.get(0).contains(secret), errors.get(0)); + } + } + + /** + * A protocol from the request that is neither http nor https gives a + * script with no callback, and is logged once like any other URL that + * is not valid. + */ + @Test + public void Url_RefusedProtocol_NoCallbackAndLoggedOnce() throws Exception { + TestLoggerFactory factory = hostLoggerFactory; + JavaScriptBuilderElement jsElement = + new JavaScriptBuilderElementBuilder(factory) + .setEndpoint("/json") + .build(); + try (Pipeline pipeline = new PipelineBuilder(factory) + .addFlowElement(new SequenceElementBuilder(factory).build()) + .addFlowElement(new JsonBuilderElementBuilder(factory).build()) + .addFlowElement(jsElement) + .build()) { + for (int i = 0; i < 2; i++) { + try (FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence( + Constants.EVIDENCE_HOST_KEY, "example.com"); + flowData.addEvidence( + fiftyone.pipeline.core.Constants.EVIDENCE_PROTOCOL, + "ftp"); + flowData.process(); + assertTrue(requestUrls(flowData.getFromElement(jsElement) + .getJavaScript()).isEmpty(), + "A refused protocol reached the script."); + } + } + } + List errors = builderErrors(); + assertEquals(1, errors.size(), String.join("\n", errors)); + assertTrue(errors.get(0).contains("\"ftp://example.com/json\""), + errors.get(0)); + } + + /** + * A protocol configured on the builder is used for every request, so one + * the script cannot send its request with is refused. The long s is + * refused too, which String.equalsIgnoreCase would match to 's'. + */ + @ParameterizedTest + @ValueSource(strings = {"ftp", "ws", "javascript", "https ", "http\u017f", + " ", "\t", "\u00a0"}) + public void Protocol_ConfiguredOtherThanHttpOrHttps_IsRefused( + String protocol) { + JavaScriptBuilderElementBuilder builder = + new JavaScriptBuilderElementBuilder(loggerFactory); + assertThrows(PipelineConfigurationException.class, + () -> builder.setProtocol(protocol)); + } + + @ParameterizedTest + @ValueSource(strings = {"http", "https", "HTTPS"}) + public void Protocol_ConfiguredHttpOrHttps_IsAccepted(String protocol) { + assertNotNull(new JavaScriptBuilderElementBuilder(loggerFactory) + .setProtocol(protocol)); + } + /** * The URL is quoted and escaped in the log, so a line break in it cannot * start a new log line. diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java index b7bbe2e9b..06bc9b6b6 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/SessionIdAndSequenceTests.java @@ -30,6 +30,8 @@ import fiftyone.pipeline.core.flowelements.FlowElementBase; import fiftyone.pipeline.core.flowelements.PipelineBuilder; import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.engines.fiftyone.data.SequenceData; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; @@ -66,6 +68,11 @@ * the builder's own handling can be tested, the tests "without the Sequence * Element" use a stand in for the JSON builder that supplies the payload and * leaves the evidence as it was given. + *

      + * The builder takes both values from the Sequence Element's data, so the + * tests of the rule above put the value straight into that data with a + * stand in for the Sequence Element, which lets a value the real element + * would never write be tried as well. */ public class SessionIdAndSequenceTests { @@ -122,9 +129,65 @@ protected void unmanagedResourcesCleanup() { } } + /** + * Writes the two values given straight into the Sequence Element's data, + * in place of that element, so that a value the real element would never + * write can be tried as well. A null value is not written. + */ + private static class SequenceStandIn extends + FlowElementBase { + + private final Object sessionId; + private final Object sequence; + + SequenceStandIn(Logger logger, Object sessionId, Object sequence) { + super(logger, null); + this.sessionId = sessionId; + this.sequence = sequence; + } + + @Override + protected void processInternal(FlowData data) { + SequenceData sequenceData = new SequenceData(logger, data); + if (sessionId != null) { + sequenceData.put( + SequenceData.SESSION_ID_PROPERTY_NAME, sessionId); + } + if (sequence != null) { + sequenceData.put( + SequenceData.SEQUENCE_PROPERTY_NAME, sequence); + } + data.getOrAdd(getElementDataKey(), f -> sequenceData); + } + + @Override + public String getElementDataKey() { + return SequenceElement.DEFAULT_ELEMENT_DATA_KEY; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist( + Collections.emptyList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } + } + @Test public void SessionId_Valid_Rendered() throws Exception { - String script = render(false, "abc-123", null); + String script = renderWithSequenceData("abc-123", null); assertEquals("abc-123", sessionId(script)); assertEquals("1", sequence(script)); } @@ -132,7 +195,7 @@ public void SessionId_Valid_Rendered() throws Exception { @Test public void SessionId_LongestValid_Rendered() throws Exception { String id = repeat('a', 64); - String script = render(false, id, null); + String script = renderWithSequenceData(id, null); assertEquals(id, sessionId(script)); } @@ -149,20 +212,27 @@ static Stream invalidSessionIds() { @ParameterizedTest @MethodSource("invalidSessionIds") public void SessionId_Invalid_RenderedEmpty(String id) throws Exception { - String script = render(false, id, null); + String script = renderWithSequenceData(id, null); assertEquals("", sessionId(script)); } /** - * The Sequence Element keeps a session id that is already in the - * evidence, so an invalid one reaches the builder that way too. + * The Sequence Element keeps a session id that is a non-empty string in + * the evidence, so an invalid one reaches the builder that way too. */ @ParameterizedTest @MethodSource("invalidSessionIds") public void SessionId_InvalidWithSequenceElement_RenderedEmpty(String id) throws Exception { String script = render(true, id, 1); - assertEquals("", sessionId(script)); + if (id.isEmpty()) { + // The Sequence Element counts an empty session id as absent + // and creates a new one. + assertTrue(sessionId(script).matches("[A-Za-z0-9-]{1,64}"), + sessionId(script)); + } else { + assertEquals("", sessionId(script)); + } assertEquals("2", sequence(script)); } @@ -183,14 +253,31 @@ public void NoSequenceElement_NoEvidence_DefaultsRendered() } /** - * A sequence from a web request is text. With no Sequence Element it - * reaches the builder as it is. + * With no Sequence Element the defaults are rendered even when the + * request carried a session id and a sequence of its own, because step 5 + * of the + * specification + * takes both from the Sequence Element's data and says "Where there is + * no Sequence Element, the sequence MUST be rendered as 1 and the + * session id as an empty string, so the script always parses." + */ + @Test + public void NoSequenceElement_WithEvidence_DefaultsRendered() + throws Exception { + String script = render(false, "abc-123", 7); + assertEquals("", sessionId(script)); + assertEquals("1", sequence(script)); + } + + /** + * A sequence another Sequence Element wrote as text is read as a number + * when it is one, because the Element Data holds values of any type. */ @ParameterizedTest @ValueSource(strings = {"abc", "-1", "0", "99999999999", "", "2147483648"}) public void Sequence_InvalidText_RenderedOne(String value) throws Exception { - String script = render(false, "abc-123", value); + String script = renderWithSequenceData("abc-123", value); assertEquals("1", sequence(script)); } @@ -202,7 +289,7 @@ static Stream invalidSequenceNumbers() { @MethodSource("invalidSequenceNumbers") public void Sequence_InvalidNumber_RenderedOne(Object value) throws Exception { - String script = render(false, "abc-123", value); + String script = renderWithSequenceData("abc-123", value); assertEquals("1", sequence(script)); } @@ -218,7 +305,7 @@ static Stream validSequences() { @MethodSource("validSequences") public void Sequence_Valid_Rendered(Object value, String expected) throws Exception { - String script = render(false, "abc-123", value); + String script = renderWithSequenceData("abc-123", value); assertEquals(expected, sequence(script)); } @@ -238,6 +325,31 @@ public void Sequence_InvalidWithSequenceElement_RenderedOne(int value) assertEquals("1", sequence(script)); } + /** + * Builds a pipeline whose Sequence Element data holds the two values + * given, processes it, checks the script parses and returns it. A null + * value is not written into that data. + */ + private String renderWithSequenceData(Object sessionId, Object sequence) + throws Exception { + JavaScriptBuilderElement jsElement = + new JavaScriptBuilderElementBuilder(loggerFactory).build(); + PipelineBuilder builder = new PipelineBuilder(loggerFactory); + builder.addFlowElement(new SequenceStandIn( + loggerFactory.getLogger("sequence"), sessionId, sequence)); + builder.addFlowElement( + new JsonStandIn(loggerFactory.getLogger("json"))); + builder.addFlowElement(jsElement); + String script; + try (Pipeline pipeline = builder.build(); + FlowData flowData = pipeline.createFlowData()) { + flowData.process(); + script = flowData.getFromElement(jsElement).getJavaScript(); + } + assertParses(script); + return script; + } + /** * Builds a pipeline with the Sequence Element and the JSON builder, or * with the stand in for them, processes it diff --git a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java index fb7f6d91d..12b7e3c9f 100644 --- a/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java +++ b/pipeline/pipeline.javascriptbuilder/src/test/java/fiftyone/pipeline/javascriptbuilder/flowelements/CallbackUrlTests.java @@ -99,8 +99,6 @@ static Stream acceptedHosts() { "https://example.com/51dpipeline/json"), Arguments.of("example.com:8080", "https://example.com:8080/51dpipeline/json"), - Arguments.of("example.com:", - "https://example.com:/51dpipeline/json"), Arguments.of("example.com:65535", "https://example.com:65535/51dpipeline/json"), Arguments.of("a_b.com", @@ -109,26 +107,38 @@ static Stream acceptedHosts() { "https://-ab.com/51dpipeline/json"), Arguments.of("127.0.0.1:80", "https://127.0.0.1:80/51dpipeline/json"), + Arguments.of("example.com", + "https://example.com/51dpipeline/json"), + Arguments.of("1.2.3.4", + "https://1.2.3.4/51dpipeline/json"), + Arguments.of("1.2.3.4:8080", + "https://1.2.3.4:8080/51dpipeline/json"), + Arguments.of("1.2.3.4/path", + "https://1.2.3.4/path/51dpipeline/json"), + Arguments.of("0.0.0.0", + "https://0.0.0.0/51dpipeline/json"), + Arguments.of("101.2.3.4", + "https://101.2.3.4/51dpipeline/json"), + Arguments.of("example.co1", + "https://example.co1/51dpipeline/json"), + Arguments.of("123.example.com", + "https://123.example.com/51dpipeline/json"), Arguments.of("[::1]", "https://[::1]/51dpipeline/json"), Arguments.of("[::1]:8080", "https://[::1]:8080/51dpipeline/json"), - Arguments.of("[fe80::1%25eth0]", - "https://[fe80::1%25eth0]/51dpipeline/json"), - Arguments.of("user:pw@example.com", - "https://user:pw@example.com/51dpipeline/json"), - Arguments.of("u x@example.com", - "https://u%20x@example.com/51dpipeline/json"), - Arguments.of("u\tx@example.com", - "https://u%09x@example.com/51dpipeline/json"), + Arguments.of("[1:2:3:4:5:6:7:8]", + "https://[1:2:3:4:5:6:7:8]/51dpipeline/json"), + Arguments.of("[::1.2.3.4]", + "https://[::1.2.3.4]/51dpipeline/json"), Arguments.of("example.com.", "https://example.com./51dpipeline/json"), Arguments.of("example.com/path", "https://example.com/path/51dpipeline/json"), - Arguments.of("example.com?x", - "https://example.com?x/51dpipeline/json"), Arguments.of("example.com/a b", "https://example.com/a%20b/51dpipeline/json"), + Arguments.of("example.com/a\"b", + "https://example.com/a%22b/51dpipeline/json"), Arguments.of("example.com/a%zzb", "https://example.com/a%25zzb/51dpipeline/json"), Arguments.of("example.com/a%41b", @@ -183,7 +193,63 @@ static Stream refusedHosts() { "a\nb.com", " example.com", "example.com ", - "example.com:8080');alert(1);//"); + "example.com\n", + "example.com:80\n", + "[::1]\n", + "example.com:8080');alert(1);//", + // User information, or an @ anywhere in the host value. + "user@example.com", + "user:pw@example.com", + "@example.com", + ":@example.com", + "user@example.com:8080", + "user@[::1]", + "user@example.com/path", + "u%41@example.com", + "u x@example.com", + "u\tx@example.com", + "u\"x@example.com", + "u'x@example.com", + "u\\x@example.com", + "u\nx@example.com", + "u\u0001x@example.com", + "example.com/a@b", + "user:12/ss@example.com", + "user:pa/ss@example.com", + "user:pa?ss@example.com", + "exa mple.com/a@b", + // A query or fragment. + "example.com?x", + "example.com#x", + "example.com?", + "example.com#", + "example.com:8080?x", + // A port that is present but empty. + "example.com:", + "example.com:/path", + "[::1]:", + "https://example.com", + "http://example.com", + "[::1]#x", + "example.com/path?x", + "example.com/path#x", + "example.com?');alert(1);//", + "example.com#');alert(1);//", + // A name whose last label is a number must be an IPv4 address. + "example.123", + "example.0x1f", + "example.0X", + "1.2.3.999", + "1.2.3", + "127.1", + "0x7f.0.0.1", + "01.2.3.4", + "example.123.", + "1.2.3.4.", + "1.2.3.4.5", + "1", + "300.1.1.1", + "0x7f.1"); } @ParameterizedTest(name = "[{index}]") @@ -192,6 +258,119 @@ public void CallbackUrl_RefusesHost(String host) { assertInvalid(url("https", host), "host " + host); } + static Stream refusedZonesAndTextAfterTheBracket() { + return Stream.of( + "[fe80::1%25]", + "[fe80::1%25eth0]", + "[fe80::1%25eth0]:8080", + "[FE80::1%25ETH0]", + "[fe80::1%251]", + "[::1%25lo]", + "[fe80::1%]", + "[fe80::1%25eth0]/path", + "[fe80::1%25'+alert(1)+']", + "[fe80::1%eth0]", + "[fe80::1%25eth%0]", + "[::1]x", + "[::1]80", + "[::1]:80x", + "[::1]]", + "[1::2::3]", + "[12345::1]"); + } + + /** + * Inside the brackets only an IPv6 address, so a zone identifier is + * refused, and after them only a port. + */ + @ParameterizedTest(name = "[{index}]") + @MethodSource("refusedZonesAndTextAfterTheBracket") + public void CallbackUrl_RefusesZoneOrTextAfterTheBracket(String host) { + assertInvalid(url("https", host), "host " + host); + } + + /** + * A port goes by its value, so leading zeros are allowed, and a port too + * long to read is refused rather than read as small enough. + */ + @Test + public void CallbackUrl_ChecksThePortByItsValue() { + assertValid( + "https://example.com:00080/51dpipeline/json", + url("https", "example.com:00080"), + "host example.com:00080"); + assertValid( + "https://example.com:0065535/51dpipeline/json", + url("https", "example.com:0065535"), + "host example.com:0065535"); + for (String host : new String[] { + "example.com:0065536", + "example.com:123456", + "example.com:4294967296", + "example.com:99999999999999999999"}) { + assertInvalid(url("https", host), "host " + host); + } + } + + /** + * Each character rule 8 lists, after the host and in the endpoint, and a + * '%' without two hexadecimal digits after it, even at the end. + */ + @Test + public void CallbackUrl_EncodesTheCharactersRule8Lists() { + assertValid( + "https://example.com/%22%5E%60%7B%7C%7D/j%22%5E%60%7B%7C%7D", + CallbackUrl.of("https", "example.com/\"^`{|}", "/j\"^`{|}"), + "rule 8 characters"); + assertValid( + "https://example.com/a%25/51dpipeline/json", + url("https", "example.com/a%"), + "host example.com/a%"); + assertValid( + "https://example.com/a%254/51dpipeline/json", + url("https", "example.com/a%4"), + "host example.com/a%4"); + assertValid( + "https://example.com/a%41/51dpipeline/json", + url("https", "example.com/a%41"), + "host example.com/a%41"); + assertValid( + "https://example.com/json%25", + CallbackUrl.of("https", "example.com", "/json%"), + "endpoint /json%"); + } + + static Stream refusedUrlsForTheLog() { + return Stream.of( + Arguments.of("user:secret@example.com", + "https://***@example.com/51dpipeline/json"), + Arguments.of("a@b@example.com", + "https://***@example.com/51dpipeline/json"), + Arguments.of("u@[::1]:80x", + "https://***@[::1]:80x/51dpipeline/json"), + Arguments.of("user:pa/ss@example.com", + "https://***@example.com/51dpipeline/json"), + Arguments.of("user:pa?ss@example.com", + "https://***@example.com/51dpipeline/json"), + Arguments.of("exa mple.com/a@b", + "https://***@b/51dpipeline/json"), + Arguments.of("@example.com", + "https://***@example.com/51dpipeline/json")); + } + + /** + * The URL kept for the log shows *** in place of everything in the host + * value before its last '@'. + */ + @ParameterizedTest(name = "[{index}]") + @MethodSource("refusedUrlsForTheLog") + public void CallbackUrl_RefusedUrlForTheLogHasNoUserInformation( + String host, String expected) { + CallbackUrl url = url("https", host); + assertInvalid(url, "host " + host); + assertEquals(expected, url.url, "host " + host); + } + @Test public void CallbackUrl_InvalidUrlIsTheOneBuilt() { CallbackUrl url = url("https", "exa mple.com"); @@ -200,7 +379,7 @@ public void CallbackUrl_InvalidUrlIsTheOneBuilt() { } @ParameterizedTest - @ValueSource(strings = {"https", "http", "ftp", "h+t-p.s"}) + @ValueSource(strings = {"https", "http", "HtTp"}) public void CallbackUrl_AcceptsScheme(String protocol) { CallbackUrl url = url(protocol, "example.com"); assertNotNull(url); @@ -215,8 +394,15 @@ public void CallbackUrl_WritesTheSchemeInLowerCase() { "protocol HTTPS"); } + /** + * Only http and https, ignoring the case of ASCII letters only, so a + * letter outside ASCII that a case-insensitive comparison can match to an + * ASCII one, such as the long s, is refused. + */ @ParameterizedTest - @ValueSource(strings = {"1http", "h t", "x');alert(1);//", "a://b"}) + @ValueSource(strings = {"ftp", "h+t-p.s", "ws", "javascript", "httpx", + "https ", "1http", "h t", "x');alert(1);//", "a://b", "http\u017f", + "HTTP\u017f"}) public void CallbackUrl_RefusesScheme(String protocol) { assertInvalid(url(protocol, "example.com"), "protocol " + protocol); } @@ -224,10 +410,7 @@ public void CallbackUrl_RefusesScheme(String protocol) { static Stream hostsThatCouldEndTheString() { return Stream.of( "example.com/');alert(1);//", - "example.com?');alert(1);//", - "example.com#');alert(1);//", - "u'x@example.com", - "u\\x@example.com", + "example.com/a\"b", "example.com/a\\b", "example.com/"); } diff --git a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/Constants.java b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/Constants.java index e1c422eeb..f72d41ee8 100644 --- a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/Constants.java +++ b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/Constants.java @@ -25,5 +25,17 @@ public class Constants { public static final Integer MAX_JAVASCRIPT_ITERATIONS = 10; - + + /** + * What the element says when no Sequence Element has produced a sequence + * number. The + * JSON + * Builder specification gives this text and every language gives the + * same, so that the message reads the same whichever one a deployment + * uses. + */ + public static final String NO_SEQUENCE_NUMBER_MESSAGE = + "The JSON Builder needs a sequence number. Place a Sequence Element " + + "before the JSON Builder in the Pipeline."; + } diff --git a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java index 23d4ddf6b..72f037049 100644 --- a/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java +++ b/pipeline/pipeline.jsonbuilder/src/main/java/fiftyone/pipeline/jsonbuilder/flowelements/JsonBuilderElement.java @@ -29,6 +29,7 @@ import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; import fiftyone.pipeline.core.data.FlowData; import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.data.IWeightedValue; import fiftyone.pipeline.core.data.PropertyMatcher; import fiftyone.pipeline.core.data.TryGetResult; import fiftyone.pipeline.core.data.factories.ElementDataFactory; @@ -62,6 +63,8 @@ public class JsonBuilderElement extends FlowElementBase implements JsonBuilder { private static final String JAVASCRIPT_PROPERTIES_NAME = "javascriptProperties"; + private static final String RAW_WEIGHTING_KEY = "rawweighting"; + private static final String VALUE_KEY = "value"; private final EvidenceKeyFilter evidenceKeyFilter; private final List properties; private final List blacklist; @@ -200,9 +203,10 @@ private String buildJson(FlowData data, PipelineConfig config) throws Exception try { sequenceNumber = getSequenceNumber(data); } catch (Exception e) { - throw new PipelineConfigurationException("Make sure there is a " - + "SequenceElement placed before this JsonBuilderElement " - + "in the pipeline", e); + // The caller reads this one, so it carries the message the + // specification gives, and the cause carries the same. + throw new PipelineConfigurationException( + Constants.NO_SEQUENCE_NUMBER_MESSAGE, e); } Map allProperties = getAllProperties(data, config); @@ -222,8 +226,7 @@ private int getSequenceNumber(FlowData data) throws Exception { "query.sequence", Integer.class); if(sequence.hasValue() == false) { - throw new Exception("Sequence number not present in evidence. " + - "this is mandatory."); + throw new Exception(Constants.NO_SEQUENCE_NUMBER_MESSAGE); } return sequence.getValue(); } @@ -319,6 +322,11 @@ private Map getValues( } propertyValue = results; } + else if (propertyValue instanceof List && + ((List) propertyValue).size() > 0 && + ((List) propertyValue).get(0) instanceof IWeightedValue) { + propertyValue = getWeightedValues((List) propertyValue); + } // Add this value to the output values.put(value.getKey().toLowerCase(), propertyValue); @@ -339,6 +347,25 @@ private Map getValues( return values; } + /** + * Weighted values as objects with the lower-case keys rawweighting and + * value, as the weighted values page of the specification shows. + * @param weightedValues the weighted values + * @return the objects to write + */ + private static List getWeightedValues(List weightedValues) { + List results = new ArrayList<>(); + for (Object item : weightedValues) { + IWeightedValue weighted = (IWeightedValue) item; + Map result = new LinkedHashMap<>(); + result.put(RAW_WEIGHTING_KEY, weighted.getRawWeighting()); + result.put(VALUE_KEY, weighted.getValue() == null ? + JSONObject.NULL : weighted.getValue()); + results.add(result); + } + return results; + } + private void addJavaScriptProperties( FlowData data, Map allProperties) { diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderSequenceTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderSequenceTests.java new file mode 100644 index 000000000..0a0329e18 --- /dev/null +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderSequenceTests.java @@ -0,0 +1,268 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.jsonbuilder; + +import fiftyone.common.testhelpers.TestLoggerFactory; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.data.FlowError; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngine; +import fiftyone.pipeline.engines.testhelpers.flowelements.EmptyEngineBuilder; +import fiftyone.pipeline.exceptions.AggregateException; +import fiftyone.pipeline.jsonbuilder.data.JsonBuilderData; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElement; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; + +import org.junit.jupiter.api.Test; +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; + +import java.util.Collection; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * The JsonBuilderElement needs a sequence number, and fails without one. + *

      + * The + * JSON + * Builder specification says the element MUST fail with an error and MUST + * NOT set its JSON value when no Sequence Element has produced a sequence + * number, that a query.sequence value taken from a query string is text and so + * does not count, and that an integer a caller added to query.sequence does + * count in Java because the element reads the integer in Evidence whoever + * added it. + *

      + * The error MUST carry the message the page gives, which is + * {@link Constants#NO_SEQUENCE_NUMBER_MESSAGE}, and where the error is wrapped + * in another the one the caller reads carries it. + */ +public class JsonBuilderSequenceTests { + + private final TestLoggerFactory loggerFactory; + + public JsonBuilderSequenceTests() { + ILoggerFactory internalLogger = mock(ILoggerFactory.class); + when(internalLogger.getLogger(anyString())).thenReturn(mock(Logger.class)); + loggerFactory = new TestLoggerFactory(internalLogger); + } + + /** + * Build a pipeline of an engine and the JSON builder, with a sequence + * element in front when asked for. + * @param withSequenceElement whether to place a sequence element first + * @param suppress whether the pipeline suppresses processing exceptions + * @return the built pipeline, which closes its elements + * @throws Exception if the engine or the pipeline cannot be built + */ + private Pipeline buildPipeline( + boolean withSequenceElement, + boolean suppress) throws Exception { + EmptyEngine engine = new EmptyEngineBuilder(loggerFactory).build(); + JsonBuilderElement jsonBuilder = + new JsonBuilderElementBuilder(loggerFactory).build(); + PipelineBuilder builder = new PipelineBuilder(loggerFactory); + if (withSequenceElement) { + SequenceElement sequenceElement = + new SequenceElementBuilder(loggerFactory).build(); + builder = builder.addFlowElement(sequenceElement); + } + return builder + .addFlowElement(engine) + .addFlowElement(jsonBuilder) + .setAutoCloseElements(true) + .setSuppressProcessException(suppress) + .build(); + } + + /** + * Process one request with the pipeline's exceptions suppressed, and + * assert that the element recorded one error carrying the specification's + * message and set no JSON. + * @param sequence the query.sequence Evidence to add, or null for none + * @throws Exception if the pipeline cannot be built + */ + private void assertFailsWithoutASequenceNumber(Object sequence) + throws Exception { + try (Pipeline pipeline = buildPipeline(false, true); + FlowData flowData = pipeline.createFlowData()) { + if (sequence != null) { + flowData.addEvidence("query.sequence", sequence); + } + flowData.process(); + + Collection errors = flowData.getErrors(); + assertEquals(1, errors.size(), + "Expected the JsonBuilderElement to record one error."); + String message = errors.iterator().next().getThrowable().getMessage(); + assertEquals(Constants.NO_SEQUENCE_NUMBER_MESSAGE, message, + "Expected the error the caller reads to carry the message " + + "the specification gives, word for word."); + + JsonBuilderData result = flowData.get(JsonBuilderData.class); + assertNull(result == null ? null : result.getJson(), + "Expected no JSON to have been set."); + } + } + + /** + * The message is written out here as well as in the element, so that + * changing the element's constant fails this test rather than moving the + * other assertions along with it. + */ + @Test + public void JsonBuilder_TheMessage_IsTheOneTheSpecificationGives() { + assertEquals( + "The JSON Builder needs a sequence number. Place a Sequence " + + "Element before the JSON Builder in the Pipeline.", + Constants.NO_SEQUENCE_NUMBER_MESSAGE); + } + + /** + * With no sequence element and no Evidence there is no sequence number, + * so the element fails and sets no JSON. + */ + @Test + public void JsonBuilder_NoSequenceElement_Fails() throws Exception { + assertFailsWithoutASequenceNumber(null); + } + + /** + * A query string carries text. Text is not a sequence number, so the + * element fails as it does with no Evidence at all. + */ + @Test + public void JsonBuilder_SequenceFromQueryString_Fails() throws Exception { + assertFailsWithoutASequenceNumber("4"); + } + + /** + * Text that is not a number at all fails the same way. + */ + @Test + public void JsonBuilder_SequenceThatIsNotANumber_Fails() throws Exception { + assertFailsWithoutASequenceNumber("abc"); + } + + /** + * The pipeline's own exception handling decides whether the error is + * thrown or recorded. With exceptions not suppressed the failure reaches + * the caller, and still no JSON is set. + */ + @Test + public void JsonBuilder_NoSequenceElement_ThrowsWhenNotSuppressed() + throws Exception { + try (Pipeline pipeline = buildPipeline(false, false); + FlowData flowData = pipeline.createFlowData()) { + AggregateException thrown = assertThrows( + AggregateException.class, + flowData::process); + // The aggregate carries each element's error as a suppressed + // exception. + assertEquals(1, thrown.getSuppressed().length, + "Expected one suppressed exception in the aggregate."); + String message = thrown.getSuppressed()[0].getMessage(); + assertEquals(Constants.NO_SEQUENCE_NUMBER_MESSAGE, message, + "Expected the error the caller reads to carry the message " + + "the specification gives, word for word."); + + JsonBuilderData result = flowData.get(JsonBuilderData.class); + assertNull(result == null ? null : result.getJson(), + "Expected no JSON to have been set."); + } + } + + /** + * The error the element records carries the one getSequenceNumber threw + * as its cause, and the cause says the same thing, because it is the line + * a caller meets at the top of the stack trace. + */ + @Test + public void JsonBuilder_NoSequenceElement_TheCauseSaysWhatIsNeeded() + throws Exception { + try (Pipeline pipeline = buildPipeline(false, true); + FlowData flowData = pipeline.createFlowData()) { + flowData.process(); + + Collection errors = flowData.getErrors(); + assertEquals(1, errors.size(), + "Expected the JsonBuilderElement to record one error."); + Throwable cause = + errors.iterator().next().getThrowable().getCause(); + assertNotNull(cause, "Expected the error to carry a cause."); + assertEquals(Constants.NO_SEQUENCE_NUMBER_MESSAGE, + cause.getMessage(), + "Expected the cause to carry the same message as the error " + + "the caller reads."); + } + } + + /** + * An integer a caller added to query.sequence is a sequence number, even + * with no sequence element, because the element reads the integer in + * Evidence whoever added it. + */ + @Test + public void JsonBuilder_IntegerFromTheCaller_BuildsTheJson() + throws Exception { + try (Pipeline pipeline = buildPipeline(false, false); + FlowData flowData = pipeline.createFlowData()) { + flowData.addEvidence("query.sequence", 3); + flowData.process(); + + assertEquals(0, flowData.getErrors().size(), + "Expected no error with an integer sequence number."); + JsonBuilderData result = flowData.get(JsonBuilderData.class); + assertNotNull(result, "Expected the JsonBuilderData to be present."); + assertNotNull(result.getJson(), "Expected the JSON to be set."); + } + } + + /** + * With a sequence element in front the JSON is built, which is the order + * the web integration assembles. + */ + @Test + public void JsonBuilder_WithSequenceElement_BuildsTheJson() + throws Exception { + try (Pipeline pipeline = buildPipeline(true, false); + FlowData flowData = pipeline.createFlowData()) { + flowData.process(); + + assertEquals(0, flowData.getErrors().size(), + "Expected no error with a sequence element in the pipeline."); + JsonBuilderData result = flowData.get(JsonBuilderData.class); + assertNotNull(result, "Expected the JsonBuilderData to be present."); + assertNotNull(result.getJson(), "Expected the JSON to be set."); + } + } +} diff --git a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java index 965526329..fe9923ff7 100644 --- a/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java +++ b/pipeline/pipeline.jsonbuilder/src/test/java/fiftyone/pipeline/jsonbuilder/JsonBuilderTests.java @@ -482,6 +482,75 @@ public void JsonBuilder_LazyLoading() throws Exception { } } + /** + * The Sequence Element's Element Data is in the JSON like any other + * element's, because the JSON Builder Element does not exclude it. + * @throws Exception + */ + @Test + public void JsonBuilder_SequenceElementData_InJson() throws Exception { + EmptyEngine engine = new EmptyEngineBuilder(loggerFactory) + .setLazyLoading(new LazyLoadingConfiguration(1000)) + .setProcessCost(500) + .build(); + JsonBuilderElement jsonBuilder = new JsonBuilderElementBuilder(loggerFactory) + .build(); + SequenceElement sequenceElement = new SequenceElementBuilder(loggerFactory) + .build(); + try (Pipeline pipeline = new PipelineBuilder(loggerFactory) + .addFlowElement(sequenceElement) + .addFlowElement(engine) + .addFlowElement(jsonBuilder) + .setAutoCloseElements(true) + .build(); + FlowData flowData = pipeline.createFlowData()) { + + flowData.process(); + + JsonBuilderData jsonResult = flowData.get(JsonBuilderData.class); + JSONObject jsonData = new JSONObject(jsonResult.getJson()); + JSONObject sequence = jsonData.getJSONObject("sequence"); + assertEquals(1, sequence.getLong("sequence")); + assertFalse(sequence.getString("session-id").isEmpty()); + } + } + + /** + * A weighted value is written as an object with the lower-case keys + * rawweighting and value, and no others. + * See the Serializing section of the weighted values page of the + * specification. + */ + @Test + public void JsonBuilder_WeightedValues() throws Exception { + List> weighted = new ArrayList<>(); + weighted.add(new WeightedValue<>(32768, "value 1")); + weighted.add(new WeightedValue<>(32767, "value 2")); + Map map = new HashMap<>(); + map.put("weighted", + new AspectPropertyValueDefault>>( + weighted)); + when(elementDataMock.asKeyMap()).thenReturn(map); + + String json = testIteration(1); + + JSONArray values = new JSONObject(json) + .getJSONObject("test") + .getJSONArray("weighted"); + assertEquals(2, values.length(), json); + for (int i = 0; i < values.length(); i++) { + JSONObject value = values.getJSONObject(i); + assertEquals( + new HashSet<>(Arrays.asList("rawweighting", "value")), + value.keySet(), + json); + assertEquals(weighted.get(i).getRawWeighting(), + value.getInt("rawweighting"), json); + assertEquals(weighted.get(i).getValue(), + value.getString("value"), json); + } + } + private String testIteration(int iteration) throws Exception { return testIteration(iteration, null); } diff --git a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/data/MissingTranslationBehavior.java b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/data/MissingTranslationBehavior.java index 7704143bb..4adb068fc 100644 --- a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/data/MissingTranslationBehavior.java +++ b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/data/MissingTranslationBehavior.java @@ -32,16 +32,57 @@ public enum MissingTranslationBehavior { * Use the original value if there is no translation for it. This is the * default behaviour. */ - ORIGINAL, + ORIGINAL("Original"), /** * Use an empty string if there is no translation for the value. */ - EMPTY_STRING, + EMPTY_STRING("EmptyString"), /** * Add a flow error if there is no translation for the value. The error * contains the reason there was no translation. */ - FLOW_ERROR + FLOW_ERROR("FlowError"); + + /** + * The name the specification gives the behaviour, which a configuration + * file uses. + */ + private final String specificationName; + + MissingTranslationBehavior(String specificationName) { + this.specificationName = specificationName; + } + + /** + * Reads a behaviour from text, as a configuration file gives it. The + * names the specification uses, Original, EmptyString and FlowError, and + * the names of these constants are accepted, ignoring case and any white + * space around the text. + * @param value the text to read + * @return the behaviour the text names + * @throws IllegalArgumentException if the text names no behaviour + */ + public static MissingTranslationBehavior parse(String value) { + if (value != null) { + String text = value.trim(); + for (MissingTranslationBehavior behavior : values()) { + if (behavior.specificationName.equalsIgnoreCase(text) || + behavior.name().equalsIgnoreCase(text)) { + return behavior; + } + } + } + StringBuilder accepted = new StringBuilder(); + MissingTranslationBehavior[] behaviors = values(); + for (int i = 0; i < behaviors.length; i++) { + if (i > 0) { + accepted.append(i == behaviors.length - 1 ? " or " : ", "); + } + accepted.append(behaviors[i].specificationName); + } + throw new IllegalArgumentException("'" + value + "' is not a " + + "missing translation behavior. Use " + accepted + "."); + } } diff --git a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java index 2da02839a..3c460191d 100644 --- a/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java +++ b/pipeline/pipeline.translation/src/main/java/fiftyone/pipeline/translation/flowelements/TranslationEngineBuilder.java @@ -22,6 +22,7 @@ package fiftyone.pipeline.translation.flowelements; +import fiftyone.pipeline.annotations.ElementBuilder; import fiftyone.pipeline.translation.data.MissingTranslationBehavior; import fiftyone.pipeline.translation.data.TranslationData; import fiftyone.pipeline.translation.data.TranslationProperty; @@ -41,7 +42,16 @@ /** * Fluent builder for {@link TranslationEngine}. + *

      + * A configuration file names this builder as {@code TranslationEngine} and + * can set {@code SourceElementDataKey}, {@code FixedLanguage} and + * {@code MissingTranslationBehavior}. Sources and translations are added in + * code, so to build the engine from a configuration file, add them to a + * builder and give it to the pipeline builder with + * {@code PipelineBuilder.addElementBuilder}, which then uses it for the + * element the file names. */ +@ElementBuilder public class TranslationEngineBuilder { private final ILoggerFactory loggerFactory; @@ -171,8 +181,21 @@ private static String readUtf8(Path path) throws IOException { /** * Build a translation engine. * @return a new {@link TranslationEngine} + * @throws IllegalArgumentException if no source or no translation has + * been added */ public TranslationEngine build() { + // Checked here as well as in the engine, so that a configuration + // file naming this builder is told where sources and translations + // come from. + if (sources.isEmpty() || translationProperties.isEmpty()) { + throw new IllegalArgumentException( + "A Translation Engine needs at least one source and one " + + "translation, which are added in code with addSource and " + + "addTranslation. To build one from a configuration file, " + + "add them to a TranslationEngineBuilder and pass it to " + + "PipelineBuilder.addElementBuilder."); + } return new TranslationEngine( sourceElementDataKey, translationProperties, diff --git a/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/data/MissingTranslationBehaviorTests.java b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/data/MissingTranslationBehaviorTests.java new file mode 100644 index 000000000..2124abef3 --- /dev/null +++ b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/data/MissingTranslationBehaviorTests.java @@ -0,0 +1,71 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.translation.data; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class MissingTranslationBehaviorTests { + + @ParameterizedTest + @CsvSource({ + "Original, ORIGINAL", + "original, ORIGINAL", + "ORIGINAL, ORIGINAL", + "EmptyString, EMPTY_STRING", + "EMPTYSTRING, EMPTY_STRING", + "empty_string, EMPTY_STRING", + "FlowError, FLOW_ERROR", + "flowerror, FLOW_ERROR", + "FLOW_ERROR, FLOW_ERROR" + }) + public void parse_SpecificationAndConstantNames_IgnoringCase( + String text, MissingTranslationBehavior expected) { + assertEquals(expected, MissingTranslationBehavior.parse(text)); + } + + @Test + public void parse_WhiteSpaceAroundTheText_IsIgnored() { + assertEquals(MissingTranslationBehavior.FLOW_ERROR, + MissingTranslationBehavior.parse(" FlowError ")); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {"Sometimes", "Empty String", "Flow-Error"}) + public void parse_AnythingElse_IsRefusedNamingTheAcceptedValues( + String text) { + IllegalArgumentException thrown = assertThrows( + IllegalArgumentException.class, + () -> MissingTranslationBehavior.parse(text)); + assertEquals("'" + text + "' is not a missing translation " + + "behavior. Use Original, EmptyString or FlowError.", + thrown.getMessage()); + } +} diff --git a/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElement.java b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElement.java new file mode 100644 index 000000000..1ca836f1c --- /dev/null +++ b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElement.java @@ -0,0 +1,84 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.translation.flowelements; + +import fiftyone.pipeline.core.data.ElementData; +import fiftyone.pipeline.core.data.ElementDataBase; +import fiftyone.pipeline.core.data.ElementPropertyMetaData; +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.EvidenceKeyFilterWhitelist; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.FlowElementBase; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; + +/** + * Stands in for the element a Translation Engine reads from, giving one + * country name under the element data key {@code source}. + */ +public class CountrySourceElement + extends FlowElementBase { + + private final String country; + + CountrySourceElement(String country) { + super(LoggerFactory.getLogger(CountrySourceElement.class), + (flowData, element) -> new ElementDataBase( + LoggerFactory.getLogger(CountrySourceElement.class), + flowData) { + }); + this.country = country; + } + + @Override + public String getElementDataKey() { + return "source"; + } + + @Override + public EvidenceKeyFilter getEvidenceKeyFilter() { + return new EvidenceKeyFilterWhitelist(Collections.emptyList()); + } + + @Override + public List getProperties() { + return Collections.emptyList(); + } + + @Override + protected void processInternal(FlowData data) { + ElementData elementData = + data.getOrAdd(getTypedDataKey(), getDataFactory()); + elementData.put("Country", country); + } + + @Override + protected void managedResourcesCleanup() { + } + + @Override + protected void unmanagedResourcesCleanup() { + } +} diff --git a/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElementBuilder.java b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElementBuilder.java new file mode 100644 index 000000000..a4e022d4c --- /dev/null +++ b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/CountrySourceElementBuilder.java @@ -0,0 +1,51 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.translation.flowelements; + +import fiftyone.pipeline.annotations.ElementBuilder; + +/** + * Builds a {@link CountrySourceElement}, so that a configuration file can + * put a source element in front of a Translation Engine. + */ +@ElementBuilder +public class CountrySourceElementBuilder { + + private String country = ""; + + /** + * @param country the country name the element gives + * @return this builder + */ + public CountrySourceElementBuilder setCountry(String country) { + this.country = country; + return this; + } + + /** + * @return a new element + */ + public CountrySourceElement build() { + return new CountrySourceElement(country); + } +} diff --git a/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/TranslationEngineConfigurationTests.java b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/TranslationEngineConfigurationTests.java new file mode 100644 index 000000000..75f8fde48 --- /dev/null +++ b/pipeline/pipeline.translation/src/test/java/fiftyone/pipeline/translation/flowelements/TranslationEngineConfigurationTests.java @@ -0,0 +1,192 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.translation.flowelements; + +import fiftyone.pipeline.core.configuration.PipelineOptions; +import fiftyone.pipeline.core.configuration.PipelineOptionsFactory; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.exceptions.PipelineConfigurationException; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.exceptions.AggregateException; +import fiftyone.pipeline.translation.data.MissingTranslationBehavior; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests that a configuration file builds a Translation Engine from a builder + * that code gave its sources and translations, with the file setting the + * options that take a single value, under the names the specification + * gives them. + */ +public class TranslationEngineConfigurationTests { + + @TempDir + Path folder; + + /** + * A builder prepared in code with the one source and the one translation + * a configuration file cannot give it. + */ + private static TranslationEngineBuilder prepared() { + return new TranslationEngineBuilder(LoggerFactory.getILoggerFactory()) + .addSource("countries.fr_FR.yml", "England: Angleterre") + .addTranslation("Country", "CountryTranslated"); + } + + /** + * Options read from an XML configuration file that puts a country + * source in front of a Translation Engine and sets each of the + * engine's single-value options. + */ + private PipelineOptions options(String country, String behavior) + throws Exception { + String xml = "" + + "" + + "CountrySourceElement" + + "" + country + "" + + "" + + "" + + "" + + "TranslationEngine" + + "" + + "source" + + "fr_FR" + + "" + behavior + + "" + + "" + + "" + + ""; + Path file = folder.resolve("pipeline.xml"); + Files.write(file, xml.getBytes(StandardCharsets.UTF_8)); + return PipelineOptionsFactory.getOptionsFromFile(file.toFile()); + } + + /** Processes one request and gives the translated country. */ + private static Object translate(Pipeline pipeline) throws Exception { + try (FlowData data = pipeline.createFlowData()) { + data.process(); + return data.get("translation").get("CountryTranslated"); + } + } + + @Test + public void preparedBuilder_ConfigurationSetsTheOptionsAndTranslates() + throws Exception { + // No language is in the evidence, so the French translation shows + // the fixed language from the file was used, and the translation + // shows the source element data key from the file was used. + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(prepared()) + .buildFromConfiguration(options("England", "Original"))) { + assertEquals("Angleterre", translate(pipeline)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"Original", "original", "ORIGINAL", " Original "}) + public void originalSpellings_KeepAValueWithNoTranslation(String spelling) + throws Exception { + // The prepared builder says EmptyString, so the original value + // shows the file's setting replaced it. + TranslationEngineBuilder builder = prepared() + .setMissingTranslationBehavior( + MissingTranslationBehavior.EMPTY_STRING); + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(builder) + .buildFromConfiguration(options("Wales", spelling))) { + assertEquals("Wales", translate(pipeline)); + } + } + + @ParameterizedTest + @ValueSource(strings = { + "EmptyString", "emptystring", "EMPTY_STRING", "empty_string"}) + public void emptyStringSpellings_GiveAnEmptyString(String spelling) + throws Exception { + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(prepared()) + .buildFromConfiguration(options("Wales", spelling))) { + assertEquals("", translate(pipeline)); + } + } + + @ParameterizedTest + @ValueSource(strings = { + "FlowError", "flowerror", "FLOW_ERROR", "flow_error"}) + public void flowErrorSpellings_AddAFlowError(String spelling) + throws Exception { + try (Pipeline pipeline = new PipelineBuilder() + .addElementBuilder(prepared()) + .buildFromConfiguration(options("Wales", spelling))) { + AggregateException thrown = assertThrows( + AggregateException.class, () -> translate(pipeline)); + // The errors processing added are the suppressed exceptions. + StringBuilder errors = new StringBuilder(); + for (Throwable error : thrown.getSuppressed()) { + errors.append(error.getMessage()).append(' '); + } + assertTrue(errors.toString().contains( + "There was no translation found for the value 'Wales'"), + errors.toString()); + } + } + + @Test + public void unknownBehavior_IsRefusedNamingTheAcceptedValues() + throws Exception { + PipelineOptions options = options("England", "Sometimes"); + PipelineConfigurationException thrown = assertThrows( + PipelineConfigurationException.class, + () -> new PipelineBuilder() + .addElementBuilder(prepared()) + .buildFromConfiguration(options)); + assertTrue(thrown.getMessage().contains( + "'Sometimes' is not a missing translation behavior. " + + "Use Original, EmptyString or FlowError."), + thrown.getMessage()); + } + + @Test + public void noPreparedBuilder_SaysSourcesAndTranslationsComeFromCode() + throws Exception { + PipelineOptions options = options("England", "Original"); + PipelineConfigurationException thrown = assertThrows( + PipelineConfigurationException.class, + () -> new PipelineBuilder().buildFromConfiguration(options)); + assertTrue(thrown.getMessage().contains( + "which are added in code with addSource and addTranslation"), + thrown.getMessage()); + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java index 309b7669b..61022376d 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/components/FiftyOneInterceptor.java @@ -51,6 +51,7 @@ import static fiftyone.pipeline.web.Constants.CORE_JS_NAME; import static fiftyone.pipeline.web.Constants.CORE_JSON_NAME; +import static fiftyone.pipeline.web.Constants.DEFAULT_JSON_PATH; /** * The 51Degrees middleware component. @@ -93,6 +94,10 @@ public FiftyOneInterceptor( configFileName = Constants.DEFAULT_CONFIG_FILE; } + String jsonPath = config.getJsonPath() == null || + config.getJsonPath().isEmpty() ? + DEFAULT_JSON_PATH : config.getJsonPath(); + File configFile = new File(configFileName); PipelineBuilder builder = new PipelineBuilder() .addService(new DataUpdateServiceDefault( @@ -104,9 +109,13 @@ public FiftyOneInterceptor( // Bind the configuration to a pipeline options instance PipelineOptions options = (PipelineOptions) unmarshaller.unmarshal(configFile); pipeline = StartupHelpers.buildFromConfiguration( - builder, - options, - config.getClientsidePropertiesEnabled()); + builder, + options, + config.getClientsidePropertiesEnabled(), + null, + jsonPath, + config.getSequenceElementEnabled(), + config.getSetHeadersElementEnabled()); } catch (Exception e) { throw new RuntimeException(e); } @@ -114,13 +123,40 @@ public FiftyOneInterceptor( resultService.setPipeline(pipeline); clientsidePropertyService.setPipeline(pipeline); fiftyOneJsService.enable(config.getClientsidePropertiesEnabled()); + fiftyOneJsService.setPaths(config.getJavaScriptPath(), jsonPath); } public static void enableClientsideProperties(ViewControllerRegistry viewControllerRegistry) { viewControllerRegistry.addViewController("/" + CORE_JS_NAME); + viewControllerRegistry.addViewController(DEFAULT_JSON_PATH); + viewControllerRegistry.addViewController("/" + CORE_JSON_NAME); + } + + /** + * Map the JavaScript and JSON paths that are not the defaults, which + * the application sets in its {@link FiftyOneInterceptorConfig}. + * @param viewControllerRegistry the registry to add the paths to + * @param javaScriptPath the JavaScript path, or null or empty for the + * default + * @param jsonPath the JSON path, or null or empty for the default + */ + public static void enableClientsideProperties( + ViewControllerRegistry viewControllerRegistry, + String javaScriptPath, + String jsonPath) { + viewControllerRegistry.addViewController("/" + withoutSlash( + javaScriptPath == null || javaScriptPath.isEmpty() ? + CORE_JS_NAME : javaScriptPath)); + viewControllerRegistry.addViewController("/" + withoutSlash( + jsonPath == null || jsonPath.isEmpty() ? + DEFAULT_JSON_PATH : jsonPath)); viewControllerRegistry.addViewController("/" + CORE_JSON_NAME); } + private static String withoutSlash(String path) { + return path.startsWith("/") ? path.substring(1) : path; + } + @Override public boolean preHandle( HttpServletRequest request, diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfig.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfig.java index c2d6ddfbc..5dcdf3cd2 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfig.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfig.java @@ -40,4 +40,38 @@ public interface FiftyOneInterceptorConfig { * @return true if client-side properties are enabled */ boolean getClientsidePropertiesEnabled(); + + /** + * Get the end of the request path the JavaScript is served for. + * @return the path, or null or empty for the default + */ + default String getJavaScriptPath() { + return null; + } + + /** + * Get the end of the request path the JSON is served for. + * @return the path, or null or empty for the default + */ + default String getJsonPath() { + return null; + } + + /** + * Get whether the Sequence Element is added to the Pipeline when the + * configuration does not include it. + * @return true to add the Sequence Element + */ + default boolean getSequenceElementEnabled() { + return true; + } + + /** + * Get whether the Set Headers Element is added to the Pipeline when the + * configuration does not include it. + * @return true to add the Set Headers Element + */ + default boolean getSetHeadersElementEnabled() { + return true; + } } diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefault.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefault.java index 4b3172ff2..bd22601cf 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefault.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/configuration/FiftyOneInterceptorConfigDefault.java @@ -34,6 +34,14 @@ public class FiftyOneInterceptorConfigDefault implements FiftyOneInterceptorConf private boolean clientsidePropertiesEnabled; + private String javaScriptPath; + + private String jsonPath; + + private boolean sequenceElementEnabled = true; + + private boolean setHeadersElementEnabled = true; + @Autowired public ServletContext servletContext; @@ -45,6 +53,42 @@ public void setClientsidePropertiesEnabled(boolean clientsidePropertiesEnabled) this.clientsidePropertiesEnabled = clientsidePropertiesEnabled; } + public void setJavaScriptPath(String javaScriptPath) { + this.javaScriptPath = javaScriptPath; + } + + public void setJsonPath(String jsonPath) { + this.jsonPath = jsonPath; + } + + public void setSequenceElementEnabled(boolean sequenceElementEnabled) { + this.sequenceElementEnabled = sequenceElementEnabled; + } + + public void setSetHeadersElementEnabled(boolean setHeadersElementEnabled) { + this.setHeadersElementEnabled = setHeadersElementEnabled; + } + + @Override + public String getJavaScriptPath() { + return javaScriptPath; + } + + @Override + public String getJsonPath() { + return jsonPath; + } + + @Override + public boolean getSequenceElementEnabled() { + return sequenceElementEnabled; + } + + @Override + public boolean getSetHeadersElementEnabled() { + return setHeadersElementEnabled; + } + @Override public String getDataFilePath() { return dataFilePath; diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/services/FiftyOneJSService.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/services/FiftyOneJSService.java index 57c8ff3d8..974b1be61 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/services/FiftyOneJSService.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/main/java/fiftyone/pipeline/web/mvc/services/FiftyOneJSService.java @@ -34,6 +34,16 @@ public interface FiftyOneJSService extends FiftyOneJSServiceCore { void enable(boolean enabled); + /** + * Set the paths the JavaScript and the JSON are served from. + * @param javaScriptPath the end of the request path that the JavaScript + * is served for, or null or empty for the default + * @param jsonPath the end of the request path that the JSON is served + * for, or null or empty for the default + */ + default void setPaths(String javaScriptPath, String jsonPath) { + } + @Service class Default extends FiftyOneJSServiceCore.Default implements FiftyOneJSService { diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java index 077a88b46..3f000fa3b 100644 --- a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/java/fiftyone/pipeline/web/mvc/FiftyOneInterceptorTests.java @@ -1,141 +1,283 @@ -/* ********************************************************************* - * This Original Work is copyright of 51 Degrees Mobile Experts Limited. - * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, - * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. - * - * This Original Work is licensed under the European Union Public Licence - * (EUPL) v.1.2 and is subject to its terms as set out below. - * - * If a copy of the EUPL was not distributed with this file, You can obtain - * one at https://opensource.org/licenses/EUPL-1.2. - * - * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be - * amended by the European Commission) shall be deemed incompatible for - * the purposes of the Work and the provisions of the compatibility - * clause in Article 5 of the EUPL shall not apply. - * - * If using the Work as, or as part of, a network application, by - * including the attribution notice(s) required under Article 5 of the EUPL - * in the end user terms of the application under an appropriate heading, - * such notice(s) shall fulfill the requirements of that article. - * ********************************************************************* */ - -package fiftyone.pipeline.web.mvc; - -import static org.junit.Assert.assertFalse; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -import java.io.IOException; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; - -import fiftyone.pipeline.core.data.FlowData; -import fiftyone.pipeline.web.mvc.components.FiftyOneInterceptor; -import fiftyone.pipeline.web.mvc.components.FlowDataProvider; -import fiftyone.pipeline.web.mvc.configuration.FiftyOneInterceptorConfig; -import fiftyone.pipeline.web.mvc.services.ClientsidePropertyService; -import fiftyone.pipeline.web.mvc.services.FiftyOneJSService; -import fiftyone.pipeline.web.mvc.services.PipelineResultService; -import fiftyone.pipeline.web.mvc.services.UACHService; - -public class FiftyOneInterceptorTests { - private HttpServletRequest request; - private HttpServletResponse response; - private Object handler; - - private FiftyOneInterceptor interceptor; - private PipelineResultService resultService; - private FlowData flowData; - - @Before - public void init() throws IOException { - FiftyOneInterceptorConfig config = - mock(FiftyOneInterceptorConfig.class); - when(config.getDataFilePath()) - .thenReturn("src/test/resources/Test.xml"); - - FlowDataProvider flowDataProvider = mock(FlowDataProvider.class); - ClientsidePropertyService clientsidePropertyService = - mock(ClientsidePropertyService.class); - - FiftyOneJSService fiftyOneJsService = mock(FiftyOneJSService.class); - when(fiftyOneJsService.serveJS( - any(HttpServletRequest.class), - any(HttpServletResponse.class))) - .thenReturn(true); - when(fiftyOneJsService.serveJson( - any(HttpServletRequest.class), - any(HttpServletResponse.class))) - .thenReturn(true); - - UACHService uachService = mock(UACHService.class); - - resultService = mock(PipelineResultService.class); - flowData = mock(FlowData.class); - - // Mock request, response and handler object - request = mock(HttpServletRequest.class); - response = mock(HttpServletResponse.class); - handler = mock(Object.class); - - when(flowDataProvider.getFlowData(any(HttpServletRequest.class))) - .thenReturn(flowData); - - // Create the interceptor object for testing - interceptor = new FiftyOneInterceptor( - config, - resultService, - flowDataProvider, - clientsidePropertyService, - fiftyOneJsService, - uachService); - } - - /** - * Check that 'resultService.process()' is called during - * interceptor's preHandle process - * @throws Exception - */ - @Test - public void FiftyOneInterceptor_PreHandle() throws Exception { - interceptor.preHandle(request, response, handler); - verify(resultService, times(1)) - .process(any(HttpServletRequest.class)); - } - - /** - * Check that 'close()' is called on the FlowData instance in the - * interceptor's afterCompletion processed - * @throws Exception - */ - @Test - public void FiftyOneInterceptor_AfterCompletion() throws Exception { - interceptor.afterCompletion( - request, - response, - handler, - new Exception()); - verify(flowData, times(1)) - .close(); - } - - /** - * Check that the FlowData is closed when preHandle serves the JavaScript - * itself, as Spring does not call afterCompletion when preHandle returns - * false. - * @throws Exception - */ - @Test - public void FiftyOneInterceptor_PreHandle_ServesJavaScript_ClosesFlowData() - throws Exception { - assertFalse(interceptor.preHandle(request, response, handler)); - verify(flowData, times(1)) - .close(); - } -} +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web.mvc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.List; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import fiftyone.pipeline.core.data.FlowData; +import org.springframework.web.servlet.config.annotation.ViewControllerRegistry; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElement; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.web.mvc.components.FiftyOneInterceptor; +import fiftyone.pipeline.web.mvc.components.FlowDataProvider; +import fiftyone.pipeline.web.mvc.configuration.FiftyOneInterceptorConfig; +import fiftyone.pipeline.web.mvc.services.ClientsidePropertyService; +import fiftyone.pipeline.web.mvc.services.FiftyOneJSService; +import fiftyone.pipeline.web.mvc.services.PipelineResultService; +import fiftyone.pipeline.web.mvc.services.UACHService; + +public class FiftyOneInterceptorTests { + private HttpServletRequest request; + private HttpServletResponse response; + private Object handler; + + private FiftyOneInterceptor interceptor; + private PipelineResultService resultService; + private FlowData flowData; + + @Before + public void init() throws IOException { + FiftyOneInterceptorConfig config = + mock(FiftyOneInterceptorConfig.class); + when(config.getDataFilePath()) + .thenReturn("src/test/resources/Test.xml"); + + FlowDataProvider flowDataProvider = mock(FlowDataProvider.class); + ClientsidePropertyService clientsidePropertyService = + mock(ClientsidePropertyService.class); + + FiftyOneJSService fiftyOneJsService = mock(FiftyOneJSService.class); + when(fiftyOneJsService.serveJS( + any(HttpServletRequest.class), + any(HttpServletResponse.class))) + .thenReturn(true); + when(fiftyOneJsService.serveJson( + any(HttpServletRequest.class), + any(HttpServletResponse.class))) + .thenReturn(true); + + UACHService uachService = mock(UACHService.class); + + resultService = mock(PipelineResultService.class); + flowData = mock(FlowData.class); + + // Mock request, response and handler object + request = mock(HttpServletRequest.class); + response = mock(HttpServletResponse.class); + handler = mock(Object.class); + + when(flowDataProvider.getFlowData(any(HttpServletRequest.class))) + .thenReturn(flowData); + + // Create the interceptor object for testing + interceptor = new FiftyOneInterceptor( + config, + resultService, + flowDataProvider, + clientsidePropertyService, + fiftyOneJsService, + uachService); + } + + /** + * Check that 'resultService.process()' is called during + * interceptor's preHandle process + * @throws Exception + */ + @Test + public void FiftyOneInterceptor_PreHandle() throws Exception { + interceptor.preHandle(request, response, handler); + verify(resultService, times(1)) + .process(any(HttpServletRequest.class)); + } + + /** + * Check that 'close()' is called on the FlowData instance in the + * interceptor's afterCompletion processed + * @throws Exception + */ + @Test + public void FiftyOneInterceptor_AfterCompletion() throws Exception { + interceptor.afterCompletion( + request, + response, + handler, + new Exception()); + verify(flowData, times(1)) + .close(); + } + + /** + * Check that the FlowData is closed when preHandle serves the JavaScript + * itself, as Spring does not call afterCompletion when preHandle returns + * false. + * @throws Exception + */ + @Test + public void FiftyOneInterceptor_PreHandle_ServesJavaScript_ClosesFlowData() + throws Exception { + assertFalse(interceptor.preHandle(request, response, handler)); + verify(flowData, times(1)) + .close(); + } + + private FiftyOneInterceptor newInterceptor( + FiftyOneInterceptorConfig config, + FiftyOneJSService jsService) throws IOException { + return new FiftyOneInterceptor( + config, + mock(PipelineResultService.class), + mock(FlowDataProvider.class), + mock(ClientsidePropertyService.class), + jsService, + mock(UACHService.class)); + } + + private FiftyOneInterceptorConfig newConfig( + boolean sequence, + boolean setHeaders, + String jsonPath, + String javaScriptPath) { + FiftyOneInterceptorConfig config = mock(FiftyOneInterceptorConfig.class); + when(config.getDataFilePath()) + .thenReturn("src/test/resources/NoSequence.xml"); + when(config.getClientsidePropertiesEnabled()).thenReturn(true); + when(config.getSequenceElementEnabled()).thenReturn(sequence); + when(config.getSetHeadersElementEnabled()).thenReturn(setHeaders); + when(config.getJsonPath()).thenReturn(jsonPath); + when(config.getJavaScriptPath()).thenReturn(javaScriptPath); + return config; + } + + private static boolean hasElement( + FiftyOneInterceptor interceptor, + Class type) { + for (FlowElement element : interceptor.pipeline.getFlowElements()) { + if (type.isInstance(element)) { + return true; + } + } + return false; + } + + /** + * Check that both elements are added when the configuration allows it. + */ + @Test + public void FiftyOneInterceptor_ElementsEnabled_Added() throws Exception { + FiftyOneInterceptor created = newInterceptor( + newConfig(true, true, null, null), mock(FiftyOneJSService.class)); + + assertTrue(hasElement(created, SequenceElement.class)); + assertTrue(hasElement(created, SetHeadersElement.class)); + } + + /** + * Check that the switches stop each element being added. + */ + @Test + public void FiftyOneInterceptor_ElementsDisabled_NotAdded() + throws Exception { + FiftyOneInterceptorConfig config = newConfig(false, false, null, null); + when(config.getClientsidePropertiesEnabled()).thenReturn(false); + when(config.getDataFilePath()) + .thenReturn("src/test/resources/NoJson.xml"); + FiftyOneInterceptor created = newInterceptor( + config, mock(FiftyOneJSService.class)); + + assertFalse(hasElement(created, SequenceElement.class)); + assertFalse(hasElement(created, SetHeadersElement.class)); + } + + /** + * Check that the build is refused when the JSON Builder Element would + * have no Sequence Element before it. + */ + @Test + public void FiftyOneInterceptor_NoSequenceElementWithJson_Refused() { + RuntimeException e = assertThrows(RuntimeException.class, + () -> newInterceptor( + newConfig(false, true, null, null), + mock(FiftyOneJSService.class))); + + assertTrue(e.getCause().getMessage().contains("Sequence Element")); + } + + /** + * Check that the JSON path is the JavaScript Builder Element's callback + * endpoint and, with the JavaScript path, goes to the service that serves + * them. + */ + @Test + public void FiftyOneInterceptor_Paths_UsedByBuilderAndService() + throws Exception { + FiftyOneJSService jsService = mock(FiftyOneJSService.class); + FiftyOneInterceptor created = newInterceptor( + newConfig(true, true, "/51dpipeline/json", "/scripts/51d.js"), + jsService); + + verify(jsService).setPaths("/scripts/51d.js", "/51dpipeline/json"); + for (FlowElement element : created.pipeline.getFlowElements()) { + if (element instanceof JavaScriptBuilderElement) { + Field field = + JavaScriptBuilderElement.class.getDeclaredField("endpoint"); + field.setAccessible(true); + assertEquals("/51dpipeline/json", field.get(element)); + } + } + } + + /** + * Check that no paths in the configuration mean the default paths. + */ + @Test + public void FiftyOneInterceptor_NoPaths_DefaultPaths() throws Exception { + FiftyOneJSService jsService = mock(FiftyOneJSService.class); + newInterceptor(newConfig(true, true, null, null), jsService); + + verify(jsService).setPaths(null, "/51dpipeline/json"); + } + + /** + * Check that the paths the application can call are mapped, the old JSON + * path included. + */ + @Test + public void FiftyOneInterceptor_EnableClientsideProperties_MapsBothJsonPaths() { + ViewControllerRegistry registry = mock(ViewControllerRegistry.class); + + FiftyOneInterceptor.enableClientsideProperties(registry); + + verify(registry).addViewController("/51Degrees.core.js"); + verify(registry).addViewController("/51dpipeline/json"); + verify(registry).addViewController("/51Degrees.core.json"); + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoJson.xml b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoJson.xml new file mode 100644 index 000000000..eece81fbe --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoJson.xml @@ -0,0 +1,7 @@ + + + + JavaScriptBuilderElement + + + diff --git a/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoSequence.xml b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoSequence.xml new file mode 100644 index 000000000..3fd181f26 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web.mvc/src/test/resources/NoSequence.xml @@ -0,0 +1,10 @@ + + + + JsonBuilderElement + + + JavaScriptBuilderElement + + + diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/Constants.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/Constants.java index acd9791db..7298dd10e 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/Constants.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/Constants.java @@ -30,10 +30,17 @@ public class Constants { public static final String CORE_JS_NAME = "51Degrees.core.js"; /** - * The name of the JSON served to the client. + * The name the JSON was served under before {@link #DEFAULT_JSON_PATH} + * became the default. It is still answered. */ public static final String CORE_JSON_NAME = "51Degrees.core.json"; + /** + * The default path the JSON is served from, which the specification + * gives. + */ + public static final String DEFAULT_JSON_PATH = "/51dpipeline/json"; + /** * Key prefix used for 51Degrees data stored in the HTTP context. */ @@ -74,4 +81,36 @@ public class Constants { */ public static final boolean DEFAULT_CLIENTSIDE_ENABLED = true; + /** + * Default value for the options that add the Sequence Element and the + * Set Headers Element. + */ + public static final boolean DEFAULT_ELEMENT_ENABLED = true; + + /** + * Name of the filter init parameter that sets the path the JavaScript is + * served from. + */ + public static final String INIT_PARAM_JAVASCRIPT_PATH = "javascript-path"; + + /** + * Name of the filter init parameter that sets the path the JSON is + * served from. + */ + public static final String INIT_PARAM_JSON_PATH = "json-path"; + + /** + * Name of the filter init parameter that sets whether the Sequence + * Element is added to the Pipeline when it is missing. + */ + public static final String INIT_PARAM_SEQUENCE_ELEMENT_ENABLED = + "sequence-element-enabled"; + + /** + * Name of the filter init parameter that sets whether the Set Headers + * Element is added to the Pipeline when it is missing. + */ + public static final String INIT_PARAM_SET_HEADERS_ELEMENT_ENABLED = + "set-headers-element-enabled"; + } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java index 098da0926..b1090a6ef 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/PipelineFilter.java @@ -40,8 +40,15 @@ import java.io.File; import java.io.IOException; +import static fiftyone.pipeline.web.Constants.DEFAULT_JSON_PATH; +import static fiftyone.pipeline.web.Constants.CORE_JS_NAME; import static fiftyone.pipeline.web.Constants.DEFAULT_CLIENTSIDE_ENABLED; import static fiftyone.pipeline.web.Constants.DEFAULT_CONFIG_FILE; +import static fiftyone.pipeline.web.Constants.DEFAULT_ELEMENT_ENABLED; +import static fiftyone.pipeline.web.Constants.INIT_PARAM_JAVASCRIPT_PATH; +import static fiftyone.pipeline.web.Constants.INIT_PARAM_JSON_PATH; +import static fiftyone.pipeline.web.Constants.INIT_PARAM_SEQUENCE_ELEMENT_ENABLED; +import static fiftyone.pipeline.web.Constants.INIT_PARAM_SET_HEADERS_ELEMENT_ENABLED; /** * Servlet filter used to intercept HTTP requests and process them using the @@ -81,6 +88,19 @@ public void init(FilterConfig filterConfig) throws ServletException { clientsideEnabled = Boolean.parseBoolean(clientsideEnabledString); } + boolean sequenceElementEnabled = getBooleanInitParameter( + filterConfig, + INIT_PARAM_SEQUENCE_ELEMENT_ENABLED, + DEFAULT_ELEMENT_ENABLED); + boolean setHeadersElementEnabled = getBooleanInitParameter( + filterConfig, + INIT_PARAM_SET_HEADERS_ELEMENT_ENABLED, + DEFAULT_ELEMENT_ENABLED); + String javaScriptPath = getPathInitParameter( + filterConfig, INIT_PARAM_JAVASCRIPT_PATH, CORE_JS_NAME); + String jsonPath = getPathInitParameter( + filterConfig, INIT_PARAM_JSON_PATH, DEFAULT_JSON_PATH); + ServletContext context = config.getServletContext(); File configFile = new File(context.getRealPath(configFileName)); @@ -96,7 +116,8 @@ public void init(FilterConfig filterConfig) throws ServletException { PipelineOptions options = (PipelineOptions) unmarshaller.unmarshal(configFile); pipeline = StartupHelpers.buildFromConfiguration(builder, options, - clientsideEnabled, context.getContextPath()); + clientsideEnabled, context.getContextPath(), jsonPath, + sequenceElementEnabled, setHeadersElementEnabled); } catch (Exception e) { throw new ServletException(e); } @@ -110,7 +131,26 @@ public void init(FilterConfig filterConfig) throws ServletException { pipeline); jsService = new FiftyOneJSServiceCore.Default( clientsidePropertyServiceCore, - clientsideEnabled); + clientsideEnabled, + javaScriptPath, + jsonPath); + } + + private static boolean getBooleanInitParameter( + FilterConfig filterConfig, + String name, + boolean defaultValue) { + String value = filterConfig.getInitParameter(name); + return value == null ? defaultValue : Boolean.parseBoolean(value); + } + + private static String getPathInitParameter( + FilterConfig filterConfig, + String name, + String defaultValue) { + String value = filterConfig.getInitParameter(name); + return value == null || value.trim().isEmpty() ? + defaultValue : value.trim(); } @Override diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/StartupHelpers.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/StartupHelpers.java index 77a373429..7e13e8a2a 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/StartupHelpers.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/StartupHelpers.java @@ -100,6 +100,43 @@ public static Pipeline buildFromConfiguration( boolean clientSideEvidenceEnabled, String contextRoot) throws Exception { + return StartupHelpers.buildFromConfiguration(builder, options, + clientSideEvidenceEnabled, contextRoot, Constants.DEFAULT_JSON_PATH, + Constants.DEFAULT_ELEMENT_ENABLED, + Constants.DEFAULT_ELEMENT_ENABLED); + } + + /** + * Configure the extra web elements required and build the Pipeline using + * the {@link PipelineBuilder#buildFromConfiguration(PipelineOptions)} + * method. + * @param builder to build the Pipeline + * @param options to build the Pipeline with + * @param clientSideEvidenceEnabled true if client-side evidence is enabled + * in the configuration. This will add JSON + * and JavaScript elements to the Pipeline + * @param contextRoot The context-root setting from the web.xml. This is + * needed in order to create the correct callback URL + * for the JavascriptBuilder. + * @param jsonPath the path the JSON is served from, which the + * JavascriptBuilder calls back to + * @param sequenceElementEnabled true to add the Sequence Element when the + * configuration does not include one + * @param setHeadersElementEnabled true to add the Set Headers Element + * when the configuration does not + * include one + * @return new {@link Pipeline} instance + * @throws Exception if there was an error building the Pipeline + */ + public static Pipeline buildFromConfiguration( + PipelineBuilder builder, + PipelineOptions options, + boolean clientSideEvidenceEnabled, + String contextRoot, + String jsonPath, + boolean sequenceElementEnabled, + boolean setHeadersElementEnabled) throws Exception { + if (options == null || options.elements == null || options.elements.size() == 0) { @@ -107,8 +144,16 @@ public static Pipeline buildFromConfiguration( "Could not find pipeline configuration information"); } + // A caller that has no path set gives the default, so that the + // JavaScript still calls back to somewhere the JSON is served from. + String path = jsonPath == null || jsonPath.isEmpty() ? + Constants.DEFAULT_JSON_PATH : jsonPath; + String endpoint = "/" + (path.startsWith("/") ? + path.substring(1) : path); + // Add the sequence element. - if (getElementIndex( + if (sequenceElementEnabled && + getElementIndex( options, SequenceElement.class.getSimpleName()) == -1) { // The sequence element is not included so add it. @@ -147,7 +192,7 @@ public static Pipeline buildFromConfiguration( // The js builder element is not included so add it. ElementOptions javaScriptBuilderElement = new ElementOptions(); javaScriptBuilderElement.builderName = JavaScriptBuilderElement.class.getSimpleName(); - javaScriptBuilderElement.buildParameters.put("Endpoint", "/" + Constants.CORE_JSON_NAME); + javaScriptBuilderElement.buildParameters.put("Endpoint", endpoint); if(contextRoot != null) { javaScriptBuilderElement.buildParameters.put("ContextRoot", contextRoot); } @@ -157,7 +202,7 @@ public static Pipeline buildFromConfiguration( // There is already a JavaScript builder config so check if // the endpoint is specified. If not, add it. if (options.elements.get(jsIndex).buildParameters.containsKey("Endpoint") == false) { - options.elements.get(jsIndex).buildParameters.put("Endpoint", "/" + Constants.CORE_JSON_NAME); + options.elements.get(jsIndex).buildParameters.put("Endpoint", endpoint); } // Same for context root. if(options.elements.get(jsIndex).buildParameters.containsKey("ContextRoot") == false && @@ -168,8 +213,26 @@ public static Pipeline buildFromConfiguration( } - // Add the set-headers element. + // The JSON Builder Element fails on every request without a + // sequence number, which only the Sequence Element supplies. if (getElementIndex( + options, + JsonBuilderElement.class.getSimpleName()) != -1 && + getElementIndex( + options, + SequenceElement.class.getSimpleName()) == -1) { + throw new PipelineConfigurationException( + "The Pipeline has a JSON Builder Element but no Sequence " + + "Element, and the JSON Builder Element needs the " + + "sequence number the Sequence Element supplies. Turn " + + "sequence-element-enabled back on, add a Sequence " + + "Element to the configuration, or turn off " + + "client-side properties."); + } + + // Add the set-headers element. + if (setHeadersElementEnabled && + getElementIndex( options, SetHeadersElement.class.getSimpleName()) == -1) { // The set-headers element is not included so add it. diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java index 67439d7cc..d51d2cfe9 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/ClientsidePropertyServiceCore.java @@ -108,6 +108,17 @@ private enum ContentTypes { */ protected Pipeline pipeline; + /** + * True once the Vary headers were worked out while every element + * had loaded its properties, so they can be kept. + */ + private volatile boolean headersComplete = false; + + /** + * Held while the Vary headers are worked out again. + */ + private final Object headersLock = new Object(); + /** * A list of all the HTTP headers that are requested evidence for * elements that populate JavaScript properties. @@ -142,31 +153,52 @@ public Default( */ @SuppressWarnings("rawtypes") protected final void init() { - headersAffectingJavaScript = new ArrayList<>(); + List headers = new ArrayList<>(); + boolean complete = true; // Get evidence filters for all elements. List filters = new ArrayList<>(); for (FlowElement element : pipeline.getFlowElements()) { + complete = complete && element.hasLoadedProperties(); filters.add(element.getEvidenceKeyFilter()); } for (EvidenceKeyFilter filter : filters) { - // If the filter is a white list or derived type then - // get all HTTP header evidence keys from white list - // and add them to the headers that could affect the - // generated JavaScript. - if (filter.getClass().equals(EvidenceKeyFilterWhitelist.class)) { + // If the filter is a white list or derived type, such as + // the filter of a parallel group, then get all HTTP header + // evidence keys from white list and add them to the + // headers that could affect the generated JavaScript. + if (filter instanceof EvidenceKeyFilterWhitelist) { EvidenceKeyFilterWhitelist whitelist = (EvidenceKeyFilterWhitelist) filter; for (String key : whitelist.getWhitelist().keySet()) { if (key.startsWith(EVIDENCE_HTTPHEADER_PREFIX + EVIDENCE_SEPERATOR) && hasControlChar(key) == false) { String header = key.substring(key.indexOf(EVIDENCE_SEPERATOR) + 1); - if(headersAffectingJavaScript.contains(header) == false){ - headersAffectingJavaScript.add(header); + if(headers.contains(header) == false){ + headers.add(header); } } } } } + headersAffectingJavaScript = headers; + headersComplete = complete; + } + + /** + * The headers for the Vary header. Until every element has loaded its + * properties, as after a cloud element could not reach its service + * when it was built, they are worked out again on each call, so they + * gain the cloud's headers once those are known. + */ + private List getHeadersAffectingJavaScript() { + if (headersComplete == false && pipeline != null) { + synchronized (headersLock) { + if (headersComplete == false) { + init(); + } + } + } + return headersAffectingJavaScript; } private boolean hasControlChar(String str) { @@ -269,7 +301,7 @@ private void setHeaders( response.setContentLength(contentLength); response.setStatus(200); response.setHeader("Cache-Control", stringJoin(cacheControl, ",")); - response.setHeader("Vary", stringJoin(headersAffectingJavaScript, ",")); + response.setHeader("Vary", stringJoin(getHeadersAffectingJavaScript(), ",")); response.setHeader("ETag", Integer.toString(hash)); response.setHeader("Access-Control-Allow-Origin", "*"); } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FiftyOneJSServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FiftyOneJSServiceCore.java index f1737d9b0..70d27fa0b 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FiftyOneJSServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/FiftyOneJSServiceCore.java @@ -30,6 +30,7 @@ import static fiftyone.pipeline.web.Constants.CORE_JS_NAME; import static fiftyone.pipeline.web.Constants.CORE_JSON_NAME; +import static fiftyone.pipeline.web.Constants.DEFAULT_JSON_PATH; /** * Service that provides the 51Degrees javascript when requested. @@ -71,9 +72,12 @@ boolean serveJson(HttpServletRequest request, HttpServletResponse response) class Default implements FiftyOneJSServiceCore { protected final ClientsidePropertyServiceCore clientsidePropertyServiceCore; protected boolean enabled; + protected String javaScriptPath; + protected String jsonPath; /** - * Construct a new instance. + * Construct a new instance that serves the JavaScript and the JSON + * from the default paths. * @param clientsidePropertyServiceCore used to get the JavaScript to * add to the returned file * @param enabled true if the service should be enabled @@ -81,8 +85,46 @@ class Default implements FiftyOneJSServiceCore { public Default( ClientsidePropertyServiceCore clientsidePropertyServiceCore, boolean enabled) { + this(clientsidePropertyServiceCore, enabled, null, null); + } + + /** + * Construct a new instance. + * @param clientsidePropertyServiceCore used to get the JavaScript to + * add to the returned file + * @param enabled true if the service should be enabled + * @param javaScriptPath the path inside the application that the + * JavaScript is served for, or null or empty + * for the default + * @param jsonPath the path inside the application that the JSON + * is served for, or null or empty for the default + */ + public Default( + ClientsidePropertyServiceCore clientsidePropertyServiceCore, + boolean enabled, + String javaScriptPath, + String jsonPath) { this.clientsidePropertyServiceCore = clientsidePropertyServiceCore; this.enabled = enabled; + this.javaScriptPath = orDefault(javaScriptPath, CORE_JS_NAME); + this.jsonPath = orDefault(jsonPath, DEFAULT_JSON_PATH); + } + + private static String orDefault(String path, String defaultPath) { + return path == null || path.isEmpty() ? defaultPath : path; + } + + /** + * Set the paths the JavaScript and the JSON are served from. + * @param javaScriptPath the path inside the application that the + * JavaScript is served for, or null or empty + * for the default + * @param jsonPath the path inside the application that the JSON + * is served for, or null or empty for the default + */ + public void setPaths(String javaScriptPath, String jsonPath) { + this.javaScriptPath = orDefault(javaScriptPath, CORE_JS_NAME); + this.jsonPath = orDefault(jsonPath, DEFAULT_JSON_PATH); } @Override @@ -90,14 +132,36 @@ public boolean serveJS( HttpServletRequest request, HttpServletResponse response) throws IOException { boolean result = false; - if (request.getRequestURL().toString().toLowerCase() - .endsWith(CORE_JS_NAME.toLowerCase())) { + if (matches(request, javaScriptPath)) { serveCoreJS(request, response); result = true; } return result; } + /** + * Whether the request is for the path. The path is compared with + * the whole path of the request inside the application, after the + * context path, ignoring case and with or without a leading slash. + */ + private static boolean matches( + HttpServletRequest request, + String path) { + String requested = request.getRequestURI(); + if (requested == null) { + return false; + } + String contextPath = request.getContextPath(); + if (contextPath != null && requested.startsWith(contextPath)) { + requested = requested.substring(contextPath.length()); + } + return withSlash(requested).equalsIgnoreCase(withSlash(path)); + } + + private static String withSlash(String path) { + return path.startsWith("/") ? path : "/" + path; + } + private void serveCoreJS( HttpServletRequest request, HttpServletResponse response) throws IOException { @@ -119,8 +183,8 @@ public boolean serveJson( boolean result = false; - if (request.getRequestURI().toString().toLowerCase() - .endsWith(CORE_JSON_NAME.toLowerCase())) { + if (matches(request, jsonPath) || + matches(request, CORE_JSON_NAME)) { serveCoreJson(request, response); result = true; } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java new file mode 100644 index 000000000..6206d41e5 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidence.java @@ -0,0 +1,101 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web.services; + +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.engines.fiftyone.data.ShareUsageEvidence; + +import java.util.AbstractMap; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * A read-only view of every value the web integration offered as Evidence + * for a request, accepted or not, keyed the way Evidence is keyed, such as + * header.user-agent. It is added to Evidence as + * {@link ShareUsageEvidence#HTTP_REQUEST_KEY} for the usage sharing element, + * which accepts no Evidence and reads the request here instead. + *

      + * It is a map rather than the HttpServletRequest because the usage sharing + * element does not depend on the servlet API, and the values are copied so + * that they stay the same once the request is recycled. + */ +public class WebRequestEvidence extends AbstractMap { + + private final Map values = + new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + + private final Set> entries = + Collections.unmodifiableMap(values).entrySet(); + + WebRequestEvidence() { + } + + /** + * The view in the Evidence of the {@link FlowData}, which is added to it + * the first time. + */ + static WebRequestEvidence of(FlowData flowData) { + Object existing = flowData.getEvidence().asKeyMap() + .get(ShareUsageEvidence.HTTP_REQUEST_KEY); + if (existing instanceof WebRequestEvidence) { + return (WebRequestEvidence) existing; + } + WebRequestEvidence created = new WebRequestEvidence(); + flowData.addEvidence(ShareUsageEvidence.HTTP_REQUEST_KEY, created); + return created; + } + + void record(String key, Object value) { + if (value != null) { + values.put(key, value); + } + } + + @Override + public Set> entrySet() { + return entries; + } + + @Override + public Object get(Object key) { + return key instanceof String ? values.get(key) : null; + } + + @Override + public boolean containsKey(Object key) { + return key instanceof String && values.containsKey(key); + } + + /** + * Says how many values the view holds rather than listing them, so that + * code which prints every Evidence value, such as a debug log or an + * example page, does not print every header and cookie of the request. + */ + @Override + public String toString() { + return "web request (" + values.size() + " values)"; + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java index a12e080f5..03f853e17 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/main/java/fiftyone/pipeline/web/services/WebRequestEvidenceServiceCore.java @@ -37,7 +37,9 @@ * Service used to populate the {@link Evidence} from a * {@link HttpServletRequest} ready for it to be processed by the Pipeline. * The Pipeline's {@link EvidenceKeyFilter} is used to determine what should be - * collected from the request. + * collected from the request. Every value offered, accepted or not, is also + * added in one {@link WebRequestEvidence} for the usage sharing element, + * which accepts no Evidence. * @see Specification */ public interface WebRequestEvidenceServiceCore { @@ -57,10 +59,18 @@ public interface WebRequestEvidenceServiceCore { */ class Default implements WebRequestEvidenceServiceCore { + private static final String EVIDENCE_REQUEST_METHOD = + EVIDENCE_SERVER_PREFIX + EVIDENCE_SEPERATOR + "request-method"; + + private static final String EVIDENCE_REQUEST_PATH = + EVIDENCE_SERVER_PREFIX + EVIDENCE_SEPERATOR + "request-path"; + @Override public void addEvidenceFromRequest( FlowData flowData, HttpServletRequest request) { + // Added on every request, whether or not any element accepts it. + WebRequestEvidence.of(flowData); Enumeration headerNames = request.getHeaderNames(); if (headerNames != null) { @@ -116,11 +126,19 @@ public void addEvidenceFromRequest( checkAndAdd(flowData, EVIDENCE_CLIENTIP_KEY, request.getRemoteAddr()); checkAndAdd(flowData, EVIDENCE_PROTOCOL, request.getScheme()); + + // The method and the path of the request line, as sent. + checkAndAdd(flowData, EVIDENCE_REQUEST_METHOD, request.getMethod()); + checkAndAdd( + flowData, + EVIDENCE_REQUEST_PATH, + request.getRequestURI()); } /** * Check if the {@link FlowData} requires this item of evidence, and add - * it if it does. + * it if it does. It is recorded in the {@link WebRequestEvidence} + * either way. * @param flowData to check and add to * @param key the key to check and add * @param value the value to add if the check passes @@ -129,6 +147,7 @@ protected void checkAndAdd(FlowData flowData, String key, Object value) { if (flowData.getEvidenceKeyFilter().include(key)) { flowData.addEvidence(key, value); } + WebRequestEvidence.of(flowData).record(key, value); } } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsidePropertyServiceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsidePropertyServiceTests.java index f819bd994..31c8f9d46 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsidePropertyServiceTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/ClientsidePropertyServiceTests.java @@ -115,6 +115,28 @@ public void JavaScript_VaryHeader_UA() throws IOException { JavaScript_VaryHeader(elements, "User-Agent"); } + /** + * An element that has not loaded its properties, as a cloud element + * after a failure to reach its service when it was built, gains its + * headers in the Vary header once it has them. + */ + @Test + @SuppressWarnings("rawtypes") + public void JavaScript_VaryHeader_ReadAgainUntilLoaded() throws IOException { + FlowElement element = mock(FlowElement.class); + when(element.getEvidenceKeyFilter()).thenReturn( + new EvidenceKeyFilterWhitelist(new ArrayList()), + new EvidenceKeyFilterWhitelist(Arrays.asList("header.User-Agent"))); + when(element.hasLoadedProperties()).thenReturn(false, true); + List elements = new ArrayList<>(); + elements.add(element); + Configure(elements); + + service.serveJavascript(request, response); + + verify(response).setHeader(eq("Vary"), eq("User-Agent")); + } + @Test public void JavaScript_VaryHeader_UA_AND_SEC() throws IOException { List> elements = new ArrayList>(); diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FiftyOneJSServiceCoreTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FiftyOneJSServiceCoreTests.java new file mode 100644 index 000000000..c0f8ae388 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/FiftyOneJSServiceCoreTests.java @@ -0,0 +1,188 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.web.services.ClientsidePropertyServiceCore; +import fiftyone.pipeline.web.services.FiftyOneJSServiceCore; +import org.junit.Before; +import org.junit.Test; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The paths the JavaScript and the JSON are served from. + * @see Specification + */ +public class FiftyOneJSServiceCoreTests { + + private ClientsidePropertyServiceCore clientside; + private HttpServletRequest request; + private HttpServletResponse response; + + @Before + public void init() { + clientside = mock(ClientsidePropertyServiceCore.class); + request = mock(HttpServletRequest.class); + response = mock(HttpServletResponse.class); + } + + private void requestPath(String path) { + requestPath("", path); + } + + /** + * The request URI holds the context path and then the path inside the + * application. + */ + private void requestPath(String contextPath, String path) { + when(request.getRequestURL()).thenReturn( + new StringBuffer("http://example.com" + contextPath + path)); + when(request.getRequestURI()).thenReturn(contextPath + path); + when(request.getContextPath()).thenReturn(contextPath); + } + + @Test + public void DefaultPaths_JavaScriptAndJsonServed() throws IOException { + FiftyOneJSServiceCore service = + new FiftyOneJSServiceCore.Default(clientside, true); + + requestPath("/51Degrees.core.js"); + assertTrue(service.serveJS(request, response)); + requestPath("/51dpipeline/json"); + assertTrue(service.serveJson(request, response)); + + verify(clientside, times(1)).serveJavascript(request, response); + verify(clientside, times(1)).serveJson(request, response); + } + + /** + * The path the JSON was served from before the specification's path + * became the default is still answered. + */ + @Test + public void DefaultPaths_OldJsonPathStillServed() throws IOException { + FiftyOneJSServiceCore service = + new FiftyOneJSServiceCore.Default(clientside, true); + + requestPath("/51Degrees.core.json"); + assertTrue(service.serveJson(request, response)); + requestPath("/app", "/51Degrees.core.json"); + assertTrue(service.serveJson(request, response)); + + verify(clientside, times(2)).serveJson(request, response); + } + + @Test + public void ConfiguredPaths_ServedInsteadOfTheDefaults() + throws IOException { + FiftyOneJSServiceCore service = new FiftyOneJSServiceCore.Default( + clientside, true, "/scripts/51d.js", "/51dpipeline/json"); + + requestPath("/scripts/51d.js"); + assertTrue(service.serveJS(request, response)); + requestPath("/51dpipeline/json"); + assertTrue(service.serveJson(request, response)); + + requestPath("/51Degrees.core.js"); + assertFalse(service.serveJS(request, response)); + requestPath("/other/json"); + assertFalse(service.serveJson(request, response)); + + verify(clientside, times(1)).serveJavascript(request, response); + verify(clientside, times(1)).serveJson(request, response); + } + + /** + * The configured path is the whole path inside the application, so a + * longer path that ends the same way is not answered. + */ + @Test + public void ConfiguredPath_OnlyAnExactMatchIsAnswered() + throws IOException { + for (String configured : new String[] {"json", "/json"}) { + FiftyOneJSServiceCore service = new FiftyOneJSServiceCore.Default( + clientside, true, configured, configured); + + requestPath("/json"); + assertTrue(configured, service.serveJson(request, response)); + assertTrue(configured, service.serveJS(request, response)); + requestPath("/anything/json"); + assertFalse(configured, service.serveJson(request, response)); + assertFalse(configured, service.serveJS(request, response)); + requestPath("/json/more"); + assertFalse(configured, service.serveJson(request, response)); + requestPath("/notjson"); + assertFalse(configured, service.serveJson(request, response)); + } + } + + @Test + public void ContextPath_IsNotPartOfThePath() throws IOException { + FiftyOneJSServiceCore service = + new FiftyOneJSServiceCore.Default(clientside, true); + + requestPath("/app", "/51dpipeline/json"); + assertTrue(service.serveJson(request, response)); + requestPath("/app", "/other/51dpipeline/json"); + assertFalse(service.serveJson(request, response)); + requestPath("/app", "/51Degrees.core.js"); + assertTrue(service.serveJS(request, response)); + requestPath("/app", "/other/51Degrees.core.js"); + assertFalse(service.serveJS(request, response)); + } + + @Test + public void EmptyPaths_UseTheDefaults() throws IOException { + FiftyOneJSServiceCore service = new FiftyOneJSServiceCore.Default( + clientside, true, "", null); + + requestPath("/51Degrees.core.js"); + assertTrue(service.serveJS(request, response)); + requestPath("/51dpipeline/json"); + assertTrue(service.serveJson(request, response)); + } + + @Test + public void ClientSideDisabled_NothingWritten() throws IOException { + FiftyOneJSServiceCore service = new FiftyOneJSServiceCore.Default( + clientside, false, "/51d.js", "/51dpipeline/json"); + + requestPath("/51d.js"); + assertTrue(service.serveJS(request, response)); + requestPath("/51dpipeline/json"); + assertTrue(service.serveJson(request, response)); + + verify(clientside, never()).serveJavascript(request, response); + verify(clientside, never()).serveJson(request, response); + } +} diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java index e0f7b7508..d0e60d9e4 100644 --- a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/PipelineFilterTests.java @@ -22,32 +22,53 @@ package fiftyone.pipeline.web; +import fiftyone.pipeline.core.configuration.PipelineOptions; import fiftyone.pipeline.core.data.ElementData; import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.FlowElement; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.services.DataUpdateService; +import fiftyone.pipeline.engines.services.DataUpdateServiceDefault; +import fiftyone.pipeline.engines.services.HttpClientDefault; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElement; +import fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElement; +import fiftyone.pipeline.javascriptbuilder.data.JavaScriptBuilderData; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElement; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElement; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.stubbing.Answer; +import org.slf4j.LoggerFactory; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.Unmarshaller; import javax.servlet.*; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; import java.util.HashMap; +import java.util.List; import java.util.Map; import static fiftyone.pipeline.engines.fiftyone.data.SetHeadersData.*; import static fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElement.*; import static fiftyone.pipeline.web.Constants.*; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; +@SuppressWarnings("rawtypes") public class PipelineFilterTests { private FilterConfig config; @@ -183,4 +204,182 @@ public void PipelineFilter_CloseFlowData_ChainFails() throws Exception { } verify(flowData, times(1)).close(); } + + private List elementsOf(PipelineFilter initialised) + throws Exception { + Field field = PipelineFilter.class.getDeclaredField("pipeline"); + field.setAccessible(true); + return ((Pipeline) field.get(initialised)).getFlowElements(); + } + + private static boolean hasElement( + List elements, + Class type) { + for (FlowElement element : elements) { + if (type.isInstance(element)) { + return true; + } + } + return false; + } + + private static String endpointOf(List elements) + throws Exception { + for (FlowElement element : elements) { + if (element instanceof JavaScriptBuilderElement) { + Field field = + JavaScriptBuilderElement.class.getDeclaredField("endpoint"); + field.setAccessible(true); + return (String) field.get(element); + } + } + return null; + } + + private PipelineFilter initFilterWith(String name, String value) + throws Exception { + doReturn(value).when(config).getInitParameter(name); + PipelineFilter initialised = new PipelineFilter(); + initialised.init(config); + return initialised; + } + + /** + * Check that the Sequence Element and the Set Headers Element are added + * when the init parameters do not say otherwise. + */ + @Test + public void PipelineFilter_Defaults_AddsSequenceAndSetHeaders() + throws Exception { + filter = new PipelineFilter(); + filter.init(config); + + List elements = elementsOf(filter); + assertTrue(hasElement(elements, SequenceElement.class)); + assertTrue(hasElement(elements, SetHeadersElement.class)); + assertTrue(hasElement(elements, JsonBuilderElement.class)); + assertTrue(hasElement(elements, JavaScriptBuilderElement.class)); + assertEquals("/51dpipeline/json", endpointOf(elements)); + } + + @Test + public void PipelineFilter_SequenceElementDisabled_NotAdded() + throws Exception { + doReturn("false").when(config) + .getInitParameter("clientside-properties-enabled"); + filter = initFilterWith("sequence-element-enabled", "false"); + + List elements = elementsOf(filter); + assertFalse(hasElement(elements, SequenceElement.class)); + assertTrue(hasElement(elements, SetHeadersElement.class)); + } + + /** + * Check that the build is refused when the JSON Builder Element would + * have no Sequence Element before it, as the JSON Builder Element then + * fails on every request. + */ + @Test + public void PipelineFilter_SequenceElementDisabledWithClientSide_Refused() { + doReturn("false").when(config) + .getInitParameter("sequence-element-enabled"); + filter = new PipelineFilter(); + + ServletException e = assertThrows( + ServletException.class, () -> filter.init(config)); + + assertTrue(e.getCause().getMessage().contains("Sequence Element")); + assertTrue(e.getCause().getMessage() + .contains("sequence-element-enabled")); + } + + @Test + public void PipelineFilter_SetHeadersElementDisabled_NotAdded() + throws Exception { + filter = initFilterWith("set-headers-element-enabled", "false"); + + List elements = elementsOf(filter); + assertTrue(hasElement(elements, SequenceElement.class)); + assertFalse(hasElement(elements, SetHeadersElement.class)); + } + + /** + * Check that the client-side switch still removes only the JSON Builder + * Element and the JavaScript Builder Element. + */ + @Test + public void PipelineFilter_ClientSideDisabled_KeepsSequenceAndSetHeaders() + throws Exception { + filter = initFilterWith("clientside-properties-enabled", "false"); + + List elements = elementsOf(filter); + assertTrue(hasElement(elements, SequenceElement.class)); + assertTrue(hasElement(elements, SetHeadersElement.class)); + assertFalse(hasElement(elements, JsonBuilderElement.class)); + assertFalse(hasElement(elements, JavaScriptBuilderElement.class)); + } + + /** + * Check that the script the JavaScript Builder Element renders calls back + * to the specification's JSON path when the init parameter is not set. + */ + @Test + public void PipelineFilter_Defaults_ScriptCallsTheNewJsonPath() + throws Exception { + filter = new PipelineFilter(); + filter.init(config); + Field field = PipelineFilter.class.getDeclaredField("pipeline"); + field.setAccessible(true); + Pipeline built = (Pipeline) field.get(filter); + + try (FlowData processed = built.createFlowData()) { + processed.addEvidence("header.Host", "example.com"); + processed.addEvidence("header.protocol", "https"); + processed.process(); + + String script = processed + .get(JavaScriptBuilderData.class) + .getJavaScript(); + assertTrue(script.contains("https://example.com/51dpipeline/json")); + } + } + + /** + * Check that the JavaScript Builder Element calls back to the JSON path + * set in the init parameter. + */ + @Test + public void PipelineFilter_JsonPath_IsTheCallbackEndpoint() + throws Exception { + filter = initFilterWith("json-path", "/custom/json"); + + assertEquals("/custom/json", endpointOf(elementsOf(filter))); + } + + /** + * Check that a caller of the public method that passes no JSON path gets + * the default path, rather than a null pointer exception. + */ + @Test + public void StartupHelpers_NoJsonPath_IsTheDefault() throws Exception { + for (String jsonPath : new String[] { null, "" }) { + File configFile = new File(getClass().getClassLoader() + .getResource("config.xml").getFile()); + Unmarshaller unmarshaller = JAXBContext + .newInstance(PipelineOptions.class).createUnmarshaller(); + PipelineOptions options = + (PipelineOptions) unmarshaller.unmarshal(configFile); + PipelineBuilder builder = new PipelineBuilder() + .addService(new DataUpdateServiceDefault( + LoggerFactory.getLogger( + DataUpdateService.class.getSimpleName()), + new HttpClientDefault())); + + try (Pipeline built = StartupHelpers.buildFromConfiguration( + builder, options, true, null, jsonPath, true, true)) { + assertEquals(DEFAULT_JSON_PATH, + endpointOf(built.getFlowElements())); + } + } + } } diff --git a/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java new file mode 100644 index 000000000..b06d5e8b2 --- /dev/null +++ b/pipeline/pipeline.web.packages/pipeline.web/src/test/java/fiftyone/pipeline/web/UsageSharingEvidenceTests.java @@ -0,0 +1,294 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +package fiftyone.pipeline.web; + +import fiftyone.pipeline.core.data.EvidenceKeyFilter; +import fiftyone.pipeline.core.data.FlowData; +import fiftyone.pipeline.core.flowelements.Pipeline; +import fiftyone.pipeline.core.flowelements.PipelineBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SequenceElementBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.SetHeadersElementBuilder; +import fiftyone.pipeline.engines.fiftyone.flowelements.ShareUsageBuilder; +import fiftyone.pipeline.javascriptbuilder.flowelements.JavaScriptBuilderElementBuilder; +import fiftyone.pipeline.jsonbuilder.flowelements.JsonBuilderElementBuilder; +import fiftyone.pipeline.web.services.ClientsidePropertyServiceCore; +import fiftyone.pipeline.web.services.FlowDataProviderCore; +import fiftyone.pipeline.web.services.WebRequestEvidenceServiceCore; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The web integration gives usage sharing the whole web request, and the + * usage sharing element accepts no Evidence, so the client-side endpoints' + * ETag does not change with headers no element uses, If-None-Match among + * them. + */ +public class UsageSharingEvidenceTests { + + private static final String USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"; + + private static final String SESSION_ID = + "1f2e3d4c-5b6a-4987-8a1b-2c3d4e5f6a7b"; + + private Pipeline pipeline; + + private final WebRequestEvidenceServiceCore evidenceService = + new WebRequestEvidenceServiceCore.Default(); + + @Before + public void init() throws Exception { + pipeline = new PipelineBuilder(LoggerFactory.getILoggerFactory()) + .addFlowElement(new SequenceElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new JsonBuilderElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()) + .setSharePercentage(0) + .build()) + .setAutoCloseElements(true) + .build(); + } + + @After + public void cleanUp() throws Exception { + pipeline.close(); + } + + private static HttpServletRequest request(Map headers) { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getMethod()).thenReturn("GET"); + when(request.getRequestURI()).thenReturn("/a%2Fb/c"); + when(request.getHeaderNames()).thenReturn( + Collections.enumeration(headers.keySet())); + for (Map.Entry header : headers.entrySet()) { + when(request.getHeader(header.getKey())) + .thenReturn(header.getValue()); + } + when(request.getCookies()).thenReturn(new Cookie[]{ + new Cookie("51D_Profile", "123456")}); + // A callback from the client-side script carries its session id and + // sequence in the query string. + when(request.getParameterNames()).thenReturn( + Collections.enumeration( + Arrays.asList("q", "session-id", "sequence"))); + when(request.getParameter("q")).thenReturn("search"); + when(request.getParameter("session-id")).thenReturn(SESSION_ID); + when(request.getParameter("sequence")).thenReturn("2"); + when(request.getRemoteAddr()).thenReturn("1.2.3.4"); + when(request.getScheme()).thenReturn("https"); + return request; + } + + private static Map headers() { + Map headers = new LinkedHashMap<>(); + headers.put("User-Agent", USER_AGENT); + headers.put("X-Test", "shared"); + headers.put("Cookie", "51D_Profile=123456"); + return headers; + } + + /** + * The web request is added as server.http-request on every request, + * although no element accepts it, keyed the way Evidence is keyed. A + * header no element accepts is not added as Evidence of its own. + */ + @Test + public void addEvidenceFromRequest_AddsTheWebRequest() throws Exception { + try (FlowData flowData = pipeline.createFlowData()) { + evidenceService.addEvidenceFromRequest( + flowData, request(headers())); + + Map evidence = flowData.getEvidence().asKeyMap(); + assertTrue("The web request was not added.", + evidence.get("server.http-request") instanceof Map); + Map webRequest = (Map) evidence.get("server.http-request"); + assertEquals("shared", webRequest.get("header.X-Test")); + assertEquals(USER_AGENT, webRequest.get("header.User-Agent")); + assertEquals("123456", webRequest.get("cookie.51D_Profile")); + assertEquals("search", webRequest.get("query.q")); + assertEquals("1.2.3.4", webRequest.get("server.client-ip")); + assertFalse("A header no element accepts was added as Evidence.", + evidence.containsKey("header.X-Test")); + } + } + + /** + * The web request holds the method and the path of the request line as + * sent, with no decoding, beside the other parts the specification names. + */ + @Test + public void addEvidenceFromRequest_WebRequestHoldsMethodAndPath() + throws Exception { + try (FlowData flowData = pipeline.createFlowData()) { + evidenceService.addEvidenceFromRequest( + flowData, request(headers())); + + Map webRequest = (Map) flowData.getEvidence() + .asKeyMap().get("server.http-request"); + assertEquals("The method is missing.", + "GET", webRequest.get("server.request-method")); + assertEquals("The path is missing or was decoded.", + "/a%2Fb/c", webRequest.get("server.request-path")); + assertEquals("https", webRequest.get("header.protocol")); + } + } + + /** + * Every element a web pipeline of this repository holds accepts only the + * Evidence that can change its results, so the Pipeline accepts neither + * If-None-Match nor a header no element uses, while it still accepts the + * keys its elements read. + */ + @Test + public void WebPipeline_AcceptsOnlyWhatItsElementsUse() throws Exception { + try (Pipeline webPipeline = new PipelineBuilder( + LoggerFactory.getILoggerFactory()) + .addFlowElement(new SequenceElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new SetHeadersElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new JsonBuilderElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new JavaScriptBuilderElementBuilder( + LoggerFactory.getILoggerFactory()).build()) + .addFlowElement(new ShareUsageBuilder( + LoggerFactory.getILoggerFactory()) + .setSharePercentage(0) + .build()) + .setAutoCloseElements(true) + .build()) { + EvidenceKeyFilter accepted = webPipeline.getEvidenceKeyFilter(); + + assertFalse("If-None-Match is accepted.", + accepted.include("header.if-none-match")); + assertFalse("A header no element uses is accepted.", + accepted.include("header.x-test")); + assertFalse("The web request is accepted.", + accepted.include("server.http-request")); + assertTrue("The JavaScript builder's host is not accepted.", + accepted.include("header.host")); + assertTrue("The sequence element's session id is not accepted.", + accepted.include("query.session-id")); + } + } + + /** + * The sequence element accepts the session id and sequence of a + * callback, so they reach Evidence although usage sharing accepts + * nothing, and the sequence counts on from the callback's. + */ + @Test + public void Callback_SessionAndSequence_CarriedOn() throws Exception { + try (FlowData flowData = pipeline.createFlowData()) { + evidenceService.addEvidenceFromRequest( + flowData, request(headers())); + flowData.process(); + + Map evidence = flowData.getEvidence().asKeyMap(); + assertEquals("The callback's session id was not kept.", + SESSION_ID, evidence.get("query.session-id")); + assertEquals("The sequence did not count on from the callback's.", + 3, evidence.get("query.sequence")); + assertTrue("Processing recorded errors: " + flowData.getErrors(), + flowData.getErrors().isEmpty()); + } + } + + /** + * A browser revalidates with the ETag it was given in If-None-Match. + * That header changes no result, so the ETag of the revalidation is the + * one it sends, and it gets a 304. + */ + @Test + public void Revalidation_SameETag_Gets304() throws Exception { + Map revalidationHeaders = headers(); + HttpServletRequest first = request(headers()); + try (FlowData firstData = pipeline.createFlowData(); + FlowData revalidationData = pipeline.createFlowData()) { + evidenceService.addEvidenceFromRequest(firstData, first); + firstData.process(); + + FlowDataProviderCore provider = mock(FlowDataProviderCore.class); + when(provider.getFlowData(first)).thenReturn(firstData); + ClientsidePropertyServiceCore service = + new ClientsidePropertyServiceCore.Default(provider, pipeline); + + HttpServletResponse firstResponse = response(); + service.serveJson(first, firstResponse); + ArgumentCaptor etag = ArgumentCaptor.forClass(String.class); + verify(firstResponse).setHeader(eq("ETag"), etag.capture()); + + revalidationHeaders.put("If-None-Match", etag.getValue()); + HttpServletRequest revalidation = request(revalidationHeaders); + evidenceService.addEvidenceFromRequest( + revalidationData, revalidation); + revalidationData.process(); + when(provider.getFlowData(revalidation)) + .thenReturn(revalidationData); + + assertEquals("The revalidation's ETag differs from the one it sends.", + etag.getValue(), + Integer.toString(revalidationData.generateKey( + pipeline.getEvidenceKeyFilter()).hashCode())); + + HttpServletResponse revalidationResponse = response(); + service.serveJson(revalidation, revalidationResponse); + verify(revalidationResponse).setStatus(304); + verify(revalidationResponse, never()).setStatus(200); + verify(revalidationResponse, never()).setContentType(anyString()); + } + } + + private static HttpServletResponse response() throws Exception { + HttpServletResponse response = mock(HttpServletResponse.class); + when(response.getWriter()).thenReturn( + new PrintWriter(new StringWriter())); + return response; + } +}