diff --git a/utils/data_download/README.md b/utils/data_download/README.md new file mode 100644 index 0000000000..e6d1035915 --- /dev/null +++ b/utils/data_download/README.md @@ -0,0 +1,52 @@ +Using `download_gdex` +===================== + +`download_gdex` is an executable module that constructs API requests for the +[NCAR Geoscience Data Exchange (GDEX)](https://gdex.ucar.edu/) data repository, +where for example, NCEP gridded forecasts and reanalysis are archived. + +This module currently only defines requests for the NCEP Climate Forecast +System Reanalysis (CFSR) and Climate Forecast System Version 2 (CFSv2). +Additional definitions should be added to `gdex_codes.yaml`. + +A GDEX token is required for data downloads. A token can be retrieved at +https://gdex.ucar.edu/accounts/profile/ (account required). + +Instructions +------------ + + 1. Navigate to the directory where data will be downloaded, and symlink + `download_gdex.py` and `gdex_codes.yaml`. + ``` + cd /path/to/download + ln -s /path/to/download_gdex.py ./ + ln -s /path/to/gdex_codes.yaml ./ + ``` + + 2. Initiate the request by executing `download_gdex.py`. The options are: + ``` + python download_gdex.py --help + + usage: download_gdex.py [-h] [-f FREQUENCY] [-b REGION_BOX] parameter_group model start_date end_date # noqa: E501 + + Download CFS data from gdex.ucar.edu + + positional arguments: + parameter_group Name of parameter group (e.g., wind) + model Model name (CFSR or CFSv2) + start_date Start date (yyyy-mm-dd) + end_date End date (yyyy-mm-dd) + + options: + -h, --help show this help message and exit + -f, --frequency FREQUENCY + Frequency (e.g., hourly) + -b, --region_box REGION_BOX + Bounding box [wlon, elon, slat, nlat] + ``` + Options for `parameter_group` can be browsed in `gdex_codes.yaml`. To + download u and v wind velocity from the CFSv2 model from 2011-01-01 to + 2014-01-01, for example, the command would be the following + ``` + python download_gdex.py wind CFSv2 2011-01-01 2014-01-01 + ``` diff --git a/utils/data_download/download_gdex.py b/utils/data_download/download_gdex.py new file mode 100644 index 0000000000..b21d68665d --- /dev/null +++ b/utils/data_download/download_gdex.py @@ -0,0 +1,213 @@ +import argparse +import os +import time + +import requests +import yaml +from dateutil.parser import parse + + +def parse_args(argv=None): + parser = argparse.ArgumentParser( + description='Download CFS data from gdex.ucar.edu') + parser.add_argument( + 'parameter_group', help='Name of parameter group (e.g., wind)') + parser.add_argument('model', help='Model name (CFSR or CFSv2)') + parser.add_argument('start_date', help='Start date (yyyy-mm-dd)') + parser.add_argument('end_date', help='End date (yyyy-mm-dd)') + parser.add_argument( + '-f', '--frequency', default='hourly', help='Frequency (e.g., hourly)') + parser.add_argument( + '-b', '--region_box', default=None, + help='Bounding box [wlon, elon, slat, nlat]') + + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + download_cfs_data( + args.parameter_group, + args.model, + args.start_date, + args.end_date, + frequency=args.frequency, + region_box=args.region_box, + ) + + +def download_cfs_data(parameter_group, model, start_date, end_date, + frequency='hourly', region_box=None): + """ + Download CFS data from gdex.ucar.edu + + Attributes + ---------- + parameter_group : str + Label for the desired parameter group (e.g., 'wind') + + model : str + Label for the desired model ('CFSR' or 'CFSv2') + + start_date : str + Start date + + end_date : str + End date + + frequency : str + Desired frequency (default 'hourly') + + region_box : list + Bounding box [wlon, elon, slat, nlat] (default global) + """ + + if region_box is None: + region_box = ['-180', '180', '-90', '90'] + + fmt = '%Y%m%d%H%M' + date_range = [parse(dt).strftime(fmt) for dt in (start_date, end_date)] + + # Load GDEX codes definitions + with open('gdex_codes.yaml', 'r') as f: + codes = yaml.safe_load(f) + dataset = codes['datasets'][model][frequency] + parameter_codes = codes['parameter_groups'][parameter_group] + + # Build grid codes + grid = parameter_codes['grid'] + if grid == 'full': + if model == 'CFSR': + grid = 'T382' + elif model == 'CFSv2': + grid = 'T574' + grid_codes = codes['grids'][grid] + + # Build product code list (Hours 1-6) + product_type = parameter_codes['product'] + products = [] + for hour in range(1, 7): + if product_type == '1-hour Average': + product = f'{product_type} (initial+{hour - 1} to initial+{hour})' + else: + product = f'{hour}-hour {product_type}' + if product_type != 'Forecast': + product = f'{product} (initial+0 to initial+1)' + if 'pressure' in parameter_group and hour == 6: + products.append('Analysis') + else: + products.append(product) + + # Build request dict + control = { + 'dataset': dataset, + 'date': '/to/'.join(date_range), + 'param': '/'.join(parameter_codes['parameters']), + 'level': parameter_codes['level'], + 'product': '/'.join(products), + 'oformat': 'netCDF', + 'nlat': region_box[3], + 'slat': region_box[2], + 'wlon': region_box[0], + 'elon': region_box[1], + 'gridproj': grid_codes[0], + 'griddef': grid_codes[1], + } + + submit_request(control) + + +def submit_request(control): + """ + Submit download request to GDEX API + + Attributes + ---------- + control : dict + Dictionary of request parameters + """ + + token = get_token() + + print('Submitting request to GDEX...') + print(f' Dataset : {control['dataset']}') + print(f' Date : {control['date']}') + print(f' Params : {control['param']}') + print(f' Level : {control['level']}') + print(f' Product : {control['product']}') + + ret = api_post('submit/', token, control) + if ret.status_code != 200: + print(f'Submit failed (HTTP {ret.status_code}):\n{ret.text}') + raise SystemExit(1) + + result = ret.json() + if result.get('status') != 'ok': + print(f'Submit error:\n{result}') + raise SystemExit(1) + + request_idx = result['data']['request_id'] + print(f'Request accepted. ID: {request_idx}') + + print('Waiting for data preparation (this may take several minutes)...') + while True: + st = api_get(f'status/{request_idx}/', token).json() + state = st['data']['status'] + print(f' Status: {state}', flush=True) + if state == 'Completed': + break + if state in ('Error', 'Cancelled'): + print(f'Request ended with status: {state}') + raise SystemExit(1) + time.sleep(60) + + print('Fetching file list...') + fl = api_get(f'get_req_files/{request_idx}/', token).json() + web_files = [f['web_path'] for f in fl['data']['web_files']] + print(f'{len(web_files)} file(s) to download.') + + for url in web_files: + filename = os.path.basename(url) + print(f'Downloading {filename} ...', end=' ', flush=True) + resp = requests.get(url, stream=True) + with open(filename, 'wb') as fh: + for chunk in resp.iter_content(chunk_size=1 << 20): + fh.write(chunk) + print('done') + + print('All files downloaded.') + + +def get_token(): + """Return cached API token, or prompt the user to paste one. + + Get your token from: https://gdex.ucar.edu/accounts/profile/ + """ + TOKEN_FILE = './gdex_token.txt' + if os.path.isfile(TOKEN_FILE) and os.path.getsize(TOKEN_FILE) > 0: + with open(TOKEN_FILE) as fh: + return fh.read().strip() + print('No GDEX token found.') + print('Please log in at https://gdex.ucar.edu and visit:') + print(' https://gdex.ucar.edu/accounts/profile/') + print('then copy your API token.') + token = input('Paste token here: ').strip() + with open(TOKEN_FILE, 'w') as fh: + fh.write(token) + return token + + +def api_get(endpoint, token): + BASE_URL = 'https://gdex.ucar.edu/api/' + url = BASE_URL + endpoint + '?token=' + token + return requests.get(url) + + +def api_post(endpoint, token, payload): + BASE_URL = 'https://gdex.ucar.edu/api/' + url = BASE_URL + endpoint + '?token=' + token + return requests.post(url, json=payload) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/data_download/gdex_codes.yaml b/utils/data_download/gdex_codes.yaml new file mode 100644 index 0000000000..5eecdf96b8 --- /dev/null +++ b/utils/data_download/gdex_codes.yaml @@ -0,0 +1,81 @@ +parameter_groups: + currents: + parameters: [UOGRD, VOGRD] + level: DBSL:5 + grid: 0.5x0.5_ocn + product: 1-hour Average + humidity: + parameters: [SPF H] + level: HTGL:2 + grid: full + product: Forecast + ice: + parameters: [ICEC] + level: SFC:0 + grid: full + product: Forecast + precipitation: + parameters: [PRATE] + level: SFC:0 + grid: full + product: Average + pressure_surf: + parameters: [PRES] + level: SFC:0 + grid: full + product: Forecast + pressure_msl: + parameters: [PRMSL] + level: MSL:0 + grid: 0.5x0.5_atm + product: Forecast + runoff: + parameters: [WATR] + level: SFC:0 + grid: full + product: Accumulation + ssh: + parameters: [SSHG] + level: SFC:0 + grid: 0.5x0.5_ocn + product: 1-hour Average + temperature: + parameters: [TMP] + level: HTGL:2 + grid: full + product: Forecast + wind: + parameters: [U GRD, V GRD] + level: HTGL:10 + grid: full + product: Forecast + +datasets: + CFSR: + 6-hourly: ds093.0 + hourly: ds093.1 + monthly: ds093.2 + CFSv2: + 6-hourly: ds094.0 + hourly: ds094.1 + monthly: ds094.2 + +grids: + T62: + - latLon + - 192:94:88.542N:0E:88.542S:358.125E:1.875:1.904 + T382: + - gaussLatLon + - 1152:576:89.761N:0E:89.761S:359.688E:0.312:288 + T574: + - gaussLatLon + - 1760:880:89.844N:0E:89.844S:359.795E:0.205:440 + 0.5x0.5_atm: + - latLon + - 720:361:90N:0E:90S:359.5E:0.5:0.5 + 0.5x0.5_ocn: + - latLon + - 720:360:89.75N:0E:89.75S:359.5E:0.5:0.5 + 2.5x2.5: + - latLon + - 144:73:90N:0E:90S:357.5E:2.5:2.5