Add GDEX download tools to new utils/data_download folder

utils/data_download/README.md

@@ -0,0 +1,52 @@
+Using `download_gdex`
+=====================
+
+`download_gdex` is an executable module that constructs API requests for the
+[NCAR Geoscience Data Exchange (GDEX)](https://gdex.ucar.edu/) data repository,
+where for example, NCEP gridded forecasts and reanalysis are archived.
+
+This module currently only defines requests for the NCEP Climate Forecast
+System Reanalysis (CFSR) and Climate Forecast System Version 2 (CFSv2).
+Additional definitions should be added to `gdex_codes.yaml`.
+
+A GDEX token is required for data downloads. A token can be retrieved at
+https://gdex.ucar.edu/accounts/profile/ (account required).
+
+Instructions
+------------
+
+   1. Navigate to the directory where data will be downloaded, and symlink
+      `download_gdex.py` and `gdex_codes.yaml`.
+      ```
+      cd /path/to/download
+      ln -s /path/to/download_gdex.py ./
+      ln -s /path/to/gdex_codes.yaml ./
+      ```
+
+   2. Initiate the request by executing `download_gdex.py`. The options are:
+      ```
+      python download_gdex.py --help
+
+      usage: download_gdex.py [-h] [-f FREQUENCY] [-b REGION_BOX] parameter_group model start_date end_date  # noqa: E501
+
+      Download CFS data from gdex.ucar.edu
+
+      positional arguments:
+      parameter_group       Name of parameter group (e.g., wind)
+      model                 Model name (CFSR or CFSv2)
+      start_date            Start date (yyyy-mm-dd)
+      end_date              End date (yyyy-mm-dd)
+
+      options:
+      -h, --help            show this help message and exit
+      -f, --frequency FREQUENCY
+                            Frequency (e.g., hourly)
+      -b, --region_box REGION_BOX
+                            Bounding box [wlon, elon, slat, nlat]
+      ```
+      Options for `parameter_group` can be browsed in `gdex_codes.yaml`. To
+      download u and v wind velocity from the CFSv2 model from 2011-01-01 to
+      2014-01-01, for example, the command would be the following
+      ```
+      python download_gdex.py wind CFSv2 2011-01-01 2014-01-01
+      ```

utils/data_download/download_gdex.py

@@ -0,0 +1,213 @@
+import argparse
+import os
+import time
+
+import requests
+import yaml
+from dateutil.parser import parse
+
+
+def parse_args(argv=None):
+    parser = argparse.ArgumentParser(
+        description='Download CFS data from gdex.ucar.edu')
+    parser.add_argument(
+        'parameter_group', help='Name of parameter group (e.g., wind)')
+    parser.add_argument('model', help='Model name (CFSR or CFSv2)')
+    parser.add_argument('start_date', help='Start date (yyyy-mm-dd)')
+    parser.add_argument('end_date', help='End date (yyyy-mm-dd)')
+    parser.add_argument(
+        '-f', '--frequency', default='hourly', help='Frequency (e.g., hourly)')
+    parser.add_argument(
+        '-b', '--region_box', default=None,
+        help='Bounding box [wlon, elon, slat, nlat]')
+
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    args = parse_args(argv)
+    download_cfs_data(
+        args.parameter_group,
+        args.model,
+        args.start_date,
+        args.end_date,
+        frequency=args.frequency,
+        region_box=args.region_box,
+    )
+
+
+def download_cfs_data(parameter_group, model, start_date, end_date,
+                      frequency='hourly', region_box=None):
+    """
+    Download CFS data from gdex.ucar.edu
+
+    Attributes
+    ----------
+    parameter_group : str
+        Label for the desired parameter group (e.g., 'wind')
+
+    model : str
+        Label for the desired model ('CFSR' or 'CFSv2')
+
+    start_date : str
+        Start date
+
+    end_date : str
+        End date
+
+    frequency : str
+        Desired frequency (default 'hourly')
+
+    region_box : list
+        Bounding box [wlon, elon, slat, nlat] (default global)
+    """
+
+    if region_box is None:
+        region_box = ['-180', '180', '-90', '90']
+
+    fmt = '%Y%m%d%H%M'
+    date_range = [parse(dt).strftime(fmt) for dt in (start_date, end_date)]
+
+    # Load GDEX codes definitions
+    with open('gdex_codes.yaml', 'r') as f:
+        codes = yaml.safe_load(f)
+    dataset = codes['datasets'][model][frequency]
+    parameter_codes = codes['parameter_groups'][parameter_group]
+
+    # Build grid codes
+    grid = parameter_codes['grid']
+    if grid == 'full':
+        if model == 'CFSR':
+            grid = 'T382'
+        elif model == 'CFSv2':
+            grid = 'T574'
+    grid_codes = codes['grids'][grid]
+
+    # Build product code list (Hours 1-6)
+    product_type = parameter_codes['product']
+    products = []
+    for hour in range(1, 7):
+        if product_type == '1-hour Average':
+            product = f'{product_type} (initial+{hour - 1} to initial+{hour})'
+        else:
+            product = f'{hour}-hour {product_type}'
+            if product_type != 'Forecast':
+                product = f'{product} (initial+0 to initial+1)'
+        if 'pressure' in parameter_group and hour == 6:
+            products.append('Analysis')
+        else:
+            products.append(product)
+
+    # Build request dict
+    control = {
+        'dataset': dataset,
+        'date': '/to/'.join(date_range),
+        'param': '/'.join(parameter_codes['parameters']),
+        'level': parameter_codes['level'],
+        'product': '/'.join(products),
+        'oformat': 'netCDF',
+        'nlat': region_box[3],
+        'slat': region_box[2],
+        'wlon': region_box[0],
+        'elon': region_box[1],
+        'gridproj': grid_codes[0],
+        'griddef': grid_codes[1],
+    }
+
+    submit_request(control)
+
+
+def submit_request(control):
+    """
+    Submit download request to GDEX API
+
+    Attributes
+    ----------
+    control : dict
+        Dictionary of request parameters
+    """
+
+    token = get_token()
+
+    print('Submitting request to GDEX...')
+    print(f'  Dataset  : {control['dataset']}')
+    print(f'  Date     : {control['date']}')
+    print(f'  Params   : {control['param']}')
+    print(f'  Level    : {control['level']}')
+    print(f'  Product  : {control['product']}')
+
+    ret = api_post('submit/', token, control)
+    if ret.status_code != 200:
+        print(f'Submit failed (HTTP {ret.status_code}):\n{ret.text}')
+        raise SystemExit(1)
+
+    result = ret.json()
+    if result.get('status') != 'ok':
+        print(f'Submit error:\n{result}')
+        raise SystemExit(1)
+
+    request_idx = result['data']['request_id']
+    print(f'Request accepted. ID: {request_idx}')
+
+    print('Waiting for data preparation (this may take several minutes)...')
+    while True:
+        st = api_get(f'status/{request_idx}/', token).json()
+        state = st['data']['status']
+        print(f'  Status: {state}', flush=True)
+        if state == 'Completed':
+            break
+        if state in ('Error', 'Cancelled'):
+            print(f'Request ended with status: {state}')
+            raise SystemExit(1)
+        time.sleep(60)
+
+    print('Fetching file list...')
+    fl = api_get(f'get_req_files/{request_idx}/', token).json()
+    web_files = [f['web_path'] for f in fl['data']['web_files']]
+    print(f'{len(web_files)} file(s) to download.')
+
+    for url in web_files:
+        filename = os.path.basename(url)
+        print(f'Downloading {filename} ...', end=' ', flush=True)
+        resp = requests.get(url, stream=True)
+        with open(filename, 'wb') as fh:
+            for chunk in resp.iter_content(chunk_size=1 << 20):
+                fh.write(chunk)
+        print('done')
+
+    print('All files downloaded.')
+
+
+def get_token():
+    """Return cached API token, or prompt the user to paste one.
+
+    Get your token from: https://gdex.ucar.edu/accounts/profile/
+    """
+    TOKEN_FILE = './gdex_token.txt'
+    if os.path.isfile(TOKEN_FILE) and os.path.getsize(TOKEN_FILE) > 0:
+        with open(TOKEN_FILE) as fh:
+            return fh.read().strip()
+    print('No GDEX token found.')
+    print('Please log in at https://gdex.ucar.edu and visit:')
+    print('  https://gdex.ucar.edu/accounts/profile/')
+    print('then copy your API token.')
+    token = input('Paste token here: ').strip()
+    with open(TOKEN_FILE, 'w') as fh:
+        fh.write(token)
+    return token
+
+
+def api_get(endpoint, token):
+    BASE_URL = 'https://gdex.ucar.edu/api/'
+    url = BASE_URL + endpoint + '?token=' + token
+    return requests.get(url)
+
+
+def api_post(endpoint, token, payload):
+    BASE_URL = 'https://gdex.ucar.edu/api/'
+    url = BASE_URL + endpoint + '?token=' + token
+    return requests.post(url, json=payload)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

utils/data_download/gdex_codes.yaml

@@ -0,0 +1,81 @@
+parameter_groups:
+  currents:
+    parameters: [UOGRD, VOGRD]
+    level: DBSL:5
+    grid: 0.5x0.5_ocn
+    product: 1-hour Average
+  humidity:
+    parameters: [SPF H]
+    level: HTGL:2
+    grid: full
+    product: Forecast
+  ice:
+    parameters: [ICEC]
+    level: SFC:0
+    grid: full
+    product: Forecast
+  precipitation:
+    parameters: [PRATE]
+    level: SFC:0
+    grid: full
+    product: Average
+  pressure_surf:
+    parameters: [PRES]
+    level: SFC:0
+    grid: full
+    product: Forecast
+  pressure_msl:
+    parameters: [PRMSL]
+    level: MSL:0
+    grid: 0.5x0.5_atm
+    product: Forecast
+  runoff:
+    parameters: [WATR]
+    level: SFC:0
+    grid: full
+    product: Accumulation
+  ssh:
+    parameters: [SSHG]
+    level: SFC:0
+    grid: 0.5x0.5_ocn
+    product: 1-hour Average
+  temperature:
+    parameters: [TMP]
+    level: HTGL:2
+    grid: full
+    product: Forecast
+  wind:
+    parameters: [U GRD, V GRD]
+    level: HTGL:10
+    grid: full
+    product: Forecast
+
+datasets:
+  CFSR:
+    6-hourly: ds093.0
+    hourly: ds093.1
+    monthly: ds093.2
+  CFSv2:
+    6-hourly: ds094.0
+    hourly: ds094.1
+    monthly: ds094.2
+
+grids:
+  T62:
+    - latLon
+    - 192:94:88.542N:0E:88.542S:358.125E:1.875:1.904
+  T382:
+    - gaussLatLon
+    - 1152:576:89.761N:0E:89.761S:359.688E:0.312:288
+  T574:
+    - gaussLatLon
+    - 1760:880:89.844N:0E:89.844S:359.795E:0.205:440
+  0.5x0.5_atm:
+    - latLon
+    - 720:361:90N:0E:90S:359.5E:0.5:0.5
+  0.5x0.5_ocn:
+    - latLon
+    - 720:360:89.75N:0E:89.75S:359.5E:0.5:0.5
+  2.5x2.5:
+    - latLon
+    - 144:73:90N:0E:90S:357.5E:2.5:2.5