diff --git a/codemagic.yaml b/codemagic.yaml index 0ab453a7f..70b310047 100644 --- a/codemagic.yaml +++ b/codemagic.yaml @@ -86,27 +86,39 @@ workflows: echo "Debug keystore setup at $DEBUG_KEYSTORE_PATH" - name: "Setup config and TSL files" script: | + set -e -o pipefail + + TSL_URL='${{ inputs.defaultCentralConfigurationTslUrl_input }}' + MEMBER_TSLS='${{ inputs.memberTsl_input }}' + CONFIG_DIR=config-lib/src/main/assets/config TSL_FILES_DIR=config-lib/src/main/assets/tslFiles + TSL_BASE_DIR="$CM_BUILD_DIR/tslBaseFiles" + CONFIGURED_TSL="$(basename "$TSL_URL")" # Create central configuration and TSL folders mkdir -p $CONFIG_DIR mkdir -p $TSL_FILES_DIR + mkdir -p $TSL_BASE_DIR + + echo "Clearing TSL files that are bundled in the repository" + rm -f $TSL_FILES_DIR/*.xml + + echo "Downloading latest 'eu-lotl.xml' file" + wget -q "https://ec.europa.eu/tools/lotl/eu-lotl.xml" -O "$TSL_BASE_DIR/eu-lotl.xml" # Get repo.ria values from environmental variables, as CodeMagic doesn't have access to repo.ria domain - if [[ "${{ inputs.defaultCentralConfigurationTslUrl_input }}" == *"ria.test"* ]]; then + if [[ "$TSL_URL" == *"ria.test"* ]]; then echo "Using repo.ria configuration" echo $REPO_RIA_DEFAULT_PROPERTIES | base64 --decode > $CONFIG_DIR/configuration.properties echo $REPO_RIA_CONFIG | base64 --decode > $CONFIG_DIR/default-config.json echo $REPO_RIA_PUB_KEY | base64 --decode > $CONFIG_DIR/default-config.ecpub echo $REPO_RIA_SIGNATURE | base64 --decode > $CONFIG_DIR/default-config.ecc - echo $REPO_RIA_TRUSTED_TEST_MP | base64 --decode > $TSL_FILES_DIR/trusted-test-mp.xml - # EE_T.xml is too large to use in environmental variable. Using compressed variant - set -o pipefail - echo $REPO_RIA_EE_TSL | base64 --decode | gunzip > $TSL_FILES_DIR/EE_T.xml + echo $REPO_RIA_TRUSTED_TEST_MP | base64 --decode > "$TSL_BASE_DIR/trusted-test-mp.xml" + for configFile in "$CONFIG_DIR/configuration.properties" "$CONFIG_DIR/default-config.json" \ "$CONFIG_DIR/default-config.ecpub" "$CONFIG_DIR/default-config.ecc" \ - "$TSL_FILES_DIR/EE_T.xml"; do + "$TSL_BASE_DIR/trusted-test-mp.xml"; do if [ ! -s "$configFile" ]; then echo "ERROR: $configFile is missing or empty" exit 1 @@ -114,27 +126,71 @@ workflows: done else echo "Not using repo.ria configuration" + + if [ ! -f "$TSL_BASE_DIR/$CONFIGURED_TSL" ]; then + echo "Downloading '$CONFIGURED_TSL' file" + wget -q "$TSL_URL" -O "$TSL_BASE_DIR/$CONFIGURED_TSL" + fi fi - - if [ -f "$TSL_FILES_DIR/eu-lotl.xml" ]; then - echo "Getting TSL (local LOTL)" - # Repo.ria TSL files already in assets folder, doesn't need to download separately anymore - if [[ "${{ inputs.defaultCentralConfigurationTslUrl_input }}" == *"ria.test"* ]]; then - python3 scripts/TSLXMLDownloader.py --tslFile="$TSL_FILES_DIR/eu-lotl.xml" --countries="${{ inputs.memberTsl_input }}" --isDevBuild="True" + + echo "Writing requested test member TSLs from environment variables" + MEMBERS_TO_DOWNLOAD="" + for member in $(echo "$MEMBER_TSLS" | tr ';' ' '); do + if [[ ! "$member" =~ ^[A-Za-z0-9_]+$ ]]; then + echo "Invalid TSL member code: '$member'" + exit 1 + fi + + if [[ "$member" != *_T ]]; then + echo "Member '$member' is downloaded from a base file" + MEMBERS_TO_DOWNLOAD="$MEMBERS_TO_DOWNLOAD;$member" + continue + fi + + variable_name="REPO_RIA_${member}_TSL" + variable_value="$(printenv "$variable_name" || true)" + + if [ -z "$variable_value" ]; then + echo "Test member '$member' was requested but $variable_name is not set" + exit 1 + fi + + echo "Writing $member.xml from $variable_name" + member_file="$TSL_FILES_DIR/$member.xml" + printf '%s' "$variable_value" | base64 --decode > "$member_file.raw" + if gzip -t "$member_file.raw" 2> /dev/null; then + gunzip -c "$member_file.raw" > "$member_file" else - python3 scripts/TSLXMLDownloader.py --tslFile="$TSL_FILES_DIR/eu-lotl.xml" --tslTestFile="$TSL_FILES_DIR/$(basename ${{ inputs.defaultCentralConfigurationTslUrl_input }})" --countries="${{ inputs.memberTsl_input }}" --isDevBuild="True" + cp "$member_file.raw" "$member_file" fi + rm -f "$member_file.raw" + done + MEMBERS_TO_DOWNLOAD="${MEMBERS_TO_DOWNLOAD#;}" - echo "Finished downloading TSL" + echo "Getting TSL" + python3 scripts/tsl_xml_downloader.py \ + --tslFile="$TSL_BASE_DIR/eu-lotl.xml" \ + --tslFile="$TSL_BASE_DIR/trusted-test-mp.xml" \ + --members="$MEMBERS_TO_DOWNLOAD" + echo "Finished downloading TSL" + + if [ -n "$(ls -A scripts/TSL 2> /dev/null)" ]; then + echo "Moving TSLs to app's assets" + mv -v scripts/TSL/* $TSL_FILES_DIR + echo "Done moving TSLs" else - echo "Getting TSL (remote LOTL)" - python3 scripts/TSLXMLDownloader.py --tslTestFile="$TSL_FILES_DIR/$(basename 'https://ec.europa.eu/tools/lotl/eu-lotl.xml')" --countries="${{ inputs.memberTsl_input }}" --isDevBuild="True" - echo "Finished downloading TSL" + echo "No downloaded TSLs to move" fi - echo "Moving TSLs to app's assets" - mv -v scripts/TSL/* $TSL_FILES_DIR/ - echo "Done moving TSLs" + echo "Adding $CONFIGURED_TSL to app's assets" + cp -v "$TSL_BASE_DIR/$CONFIGURED_TSL" "$TSL_FILES_DIR/" + + if [[ ! "$TSL_URL" == *"ria.test"* ]]; then + echo "Adding eu-lotl.xml to app's assets" + cp -v "$TSL_BASE_DIR/eu-lotl.xml" "$TSL_FILES_DIR/" + fi + + echo "TSL files bundled in assets: $(ls $TSL_FILES_DIR)" - name: Fetch default configuration script: | # Not running 'fetchAndPackageDefaultConfiguration' for repo.ria as necessary files are already setup @@ -250,6 +306,8 @@ workflows: echo "Debug keystore created at $DEBUG_KEYSTORE_PATH" - name: Fetch default configuration and setup TSLs script: | + set -e -o pipefail + ./gradlew clean --no-daemon fetchAndPackageDefaultConfiguration -Dorg.gradle.jvmargs="-Xmx4g" TSL_FILES_DIR=config-lib/src/main/assets/tslFiles @@ -264,8 +322,10 @@ workflows: echo "Downloading latest 'eu-lotl.xml' file" wget "https://ec.europa.eu/tools/lotl/eu-lotl.xml" -P $TSL_FILES_DIR - echo "Getting TSL (remote LOTL)" - python3 scripts/TSLXMLDownloader.py --tslFile="$TSL_FILES_DIR/$(basename 'https://ec.europa.eu/tools/lotl/eu-lotl.xml')" --tslTestFile="$TSL_FILES_DIR/$(basename 'https://ec.europa.eu/tools/lotl/eu-lotl.xml')" --countries="EE" --isDevBuild="False" + echo "Getting TSL" + python3 scripts/tsl_xml_downloader.py \ + --tslFile="$TSL_FILES_DIR/eu-lotl.xml" \ + --members="EE" echo "Finished downloading TSL" echo "Moving TSLs to app's assets" diff --git a/scripts/TSLXMLDownloader.py b/scripts/TSLXMLDownloader.py deleted file mode 100644 index 0c95f73d3..000000000 --- a/scripts/TSLXMLDownloader.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/python - -import urllib.request -import ssl -import argparse -import os -from xml.dom import minidom - -class TSLDownloader: - def __init__(self): - self.mimetypeFound = False - - def parse_arguments(self): - """Parse command-line arguments.""" - parser = argparse.ArgumentParser(description='TSL XML Downloader') - parser.add_argument('--tslFile', default='eu-lotl.xml', type=str, help='TSL file location') - parser.add_argument('--tslTestFile', type=str, help='Test TSL file location') - parser.add_argument('--countries', required=True, type=str, help='Countries, e.g., "EE;EE_T"') - parser.add_argument('--isDevBuild', required=True, type=str, help='Indicates if build is dev version, e.g., "True"') - return parser.parse_args() - - def load_xml(self, tsl_file_location, tsl_test_file_location, is_test_file): - """Load the appropriate XML file based on country and test file presence.""" - if is_test_file: - if tsl_test_file_location is None or tsl_test_file_location == tsl_file_location: - return None - return minidom.parse(tsl_test_file_location) - - if os.path.isfile(tsl_file_location): - return minidom.parse(tsl_file_location) - return minidom.parse(tsl_test_file_location) - - def download_tsl(self, tsl_url, country_code, is_dev_build, processed_urls): - """Download TSL data from the given URL and save it.""" - # Check if the URL has already been processed - if tsl_url in processed_urls: - print(f"Skipping already processed URL: {tsl_url}") - return - - print(f"{country_code} TSL-URL: {tsl_url}") - - # Get data from TSL-URL - response = urllib.request.urlopen(tsl_url).read() - - # Decode the response from bytes to string - response_str = response.decode('utf-8') - - # Get the directory of the current script - script_directory = os.path.dirname(os.path.abspath(__file__)) - - # Define the TSL subfolder path - tsl_folder_path = os.path.join(script_directory, "TSL") - - # Create the TSL subfolder if it doesn't exist - if not os.path.exists(tsl_folder_path): - os.makedirs(tsl_folder_path) - - # Save data to a file in the TSL subfolder - filename = f"{country_code}.xml" - file_path = os.path.join(tsl_folder_path, filename) - - with open(file_path, 'w', encoding='utf-8') as file: - file.write(response_str) - - # Mark this URL as processed - processed_urls.add(tsl_url) - - def process_country(self, input_country, xmldoc, is_dev_build, processed_urls): - """Process XML data for a specific country and download TSL if applicable.""" - pointers_to_other_tsl = xmldoc.getElementsByTagName('PointersToOtherTSL') - - for pointer in pointers_to_other_tsl: - other_tsl_pointers = pointer.getElementsByTagName('OtherTSLPointer') - - for pointer_data in other_tsl_pointers: - tsl_locations = pointer_data.getElementsByTagName('TSLLocation') - additional_info = pointer_data.getElementsByTagName('AdditionalInformation') - - for info in additional_info: - other_info = info.getElementsByTagName('OtherInformation') - - for other_data in other_info: - scheme_territory = other_data.getElementsByTagName('SchemeTerritory') - mime_type = other_data.getElementsByTagName('ns3:MimeType') or other_data.getElementsByTagName('ns4:MimeType') - test_mime_type = other_data.getElementsByTagName('tslx:MimeType') - - for territory in scheme_territory: - pass - - if "_T" in input_country: - for test_type in test_mime_type: - if territory.firstChild.nodeValue == input_country and test_type.firstChild.nodeValue == 'application/vnd.etsi.tsl+xml': - tsl_url = pointer_data.getElementsByTagName('TSLLocation')[0].firstChild.nodeValue - self.download_tsl(tsl_url, territory.firstChild.nodeValue, is_dev_build, processed_urls) - break - else: - for mime in mime_type: - if territory.firstChild.nodeValue == input_country and mime.firstChild.nodeValue == 'application/vnd.etsi.tsl+xml': - self.mimetypeFound = True - tsl_url = pointer_data.getElementsByTagName('TSLLocation')[0].firstChild.nodeValue - self.download_tsl(tsl_url, territory.firstChild.nodeValue, is_dev_build, processed_urls) - break - - def main(self): - """Main function to execute the TSL downloader.""" - # Parse arguments - args = self.parse_arguments() - - input_countries = args.countries.split(';') if ';' in args.countries else [args.countries] - is_dev_build = args.isDevBuild.lower() == "true" - - # Processed TSL URLs - processed_urls = set() - - for input_country in input_countries: - xmldoc = self.load_xml(args.tslFile, args.tslTestFile, "_T" in input_country) - - if xmldoc is None: - continue - - # Process country-specific data - self.process_country(input_country, xmldoc, is_dev_build, processed_urls) - - # Validate country and mime type - if len(input_country) == 2 and not self.mimetypeFound: - raise Exception(f"Could not find mimetype for '{input_country}'. Verify input country code and check if the MimeType tag name has changed (nsx:MimeType) in TSL file.") - - -if __name__ == "__main__": - downloader = TSLDownloader() - downloader.main() \ No newline at end of file diff --git a/scripts/tsl_xml_downloader.py b/scripts/tsl_xml_downloader.py new file mode 100644 index 000000000..f53f3c9b8 --- /dev/null +++ b/scripts/tsl_xml_downloader.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 + +import argparse +import urllib.request +from pathlib import Path +from xml.dom import minidom + +TSL_MIME_TYPE = 'application/vnd.etsi.tsl+xml' +DOWNLOAD_TIMEOUT_SECONDS = 60 +DOWNLOAD_DIRECTORY = Path(__file__).resolve().parent / 'TSL' + + +def parse_arguments(): + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description='TSL XML Downloader') + parser.add_argument('--tslFile', action='append', default=[], type=Path, metavar='PATH', + help='Base file (LOTL) to look members up in. Repeatable, searched in order') + parser.add_argument('--members', required=True, type=str, + help='Scheme territories to download, e.g. "EE;EE_T;SE_TESTING"') + return parser.parse_args() + + +def child_elements(parent, local_name): + """Direct child elements with this local name, whatever XML namespace prefix the file uses.""" + return [ + node + for node in parent.childNodes + if node.nodeType == node.ELEMENT_NODE and node.localName == local_name + ] + + +def text_of(element): + """Text content of an element, or None when it is empty.""" + return element.firstChild.nodeValue if element.firstChild is not None else None + + +def other_information(pointer, local_name): + """Values of AdditionalInformation/OtherInformation/ for one pointer.""" + return [ + text_of(element) + for additional_information in child_elements(pointer, 'AdditionalInformation') + for information in child_elements(additional_information, 'OtherInformation') + for element in child_elements(information, local_name) + ] + + +def find_tsl_url(member, lotl): + """The TSL location the base file points at for this member, or None.""" + for pointers in lotl.getElementsByTagName('PointersToOtherTSL'): + for pointer in child_elements(pointers, 'OtherTSLPointer'): + if member not in other_information(pointer, 'SchemeTerritory'): + continue + if TSL_MIME_TYPE not in other_information(pointer, 'MimeType'): + continue + + locations = child_elements(pointer, 'TSLLocation') + + if locations: + return text_of(locations[0]) + + return None + + +def existing_base_files(paths): + """The given base files that exist, in the order given, reporting the ones that do not.""" + existing = [] + + for path in paths: + if path.is_file(): + existing.append(path) + else: + print(f"Skipping missing base file '{path}'") + + return existing + + +def resolve_member(member, base_files): + """Look the member up in each base file in turn. Returns the first TSL location found.""" + for base_file in base_files: + tsl_url = find_tsl_url(member, minidom.parse(str(base_file))) + + if tsl_url is not None: + print(f"Found '{member}' in '{base_file}'") + return tsl_url + + print(f"No '{member}' pointer in '{base_file}'") + + return None + + +def download_tsl(tsl_url, destination_path): + """Download a TSL and save it.""" + print(f"Downloading {tsl_url} to {destination_path}") + + with urllib.request.urlopen(tsl_url, timeout=DOWNLOAD_TIMEOUT_SECONDS) as response: + destination_path.write_bytes(response.read()) + + +def main(): + """Main function to execute the TSL downloader.""" + args = parse_arguments() + members = [member for member in args.members.split(';') if member] + base_files = existing_base_files(args.tslFile) + + DOWNLOAD_DIRECTORY.mkdir(parents=True, exist_ok=True) + + for member in members: + tsl_url = resolve_member(member, base_files) + + if tsl_url is None: + searched = ', '.join(str(base_file) for base_file in base_files) or 'none' + raise SystemExit( + f"Could not find a TSL pointer for '{member}'. Base files searched: {searched}. " + f"Either the member code is wrong, or the base file listing it was not given." + ) + + download_tsl(tsl_url, DOWNLOAD_DIRECTORY / f"{member}.xml") + + +if __name__ == "__main__": + main()