diff --git a/.gitmodules b/.gitmodules index fb97e80..4eedccb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -37,3 +37,9 @@ [submodule "src/eCLM_atmforcing"] path = src/eCLM_atmforcing url = https://github.com/HPSCTerrSys/eCLM_static-file-generator.git +[submodule "src/PDAF_input-file-generator"] + path = src/PDAF_input-file-generator + url = https://github.com/HPSCTerrSys/PDAF_input-file-generator.git +[submodule "src/eCLM_namelist-generator"] + path = src/eCLM_namelist-generator + url = https://github.com/HPSCTerrSys/eCLM_namelist-generator.git diff --git a/ctl/pre_ctl/pre_config.sh b/ctl/pre_ctl/pre_config.sh index 7cb7a97..c9c1eaa 100755 --- a/ctl/pre_ctl/pre_config.sh +++ b/ctl/pre_ctl/pre_config.sh @@ -56,6 +56,42 @@ echo "List of forcing files: "${listfrcfile[@]} fi +#################### +# PDAF +#################### +if [[ "${modelid}" == *pdaf* ]]; then + +parse_config_file ${conf_file} "pre_config_pdaf" + +# --------------------------------------------------------------------------- +# Perturb atmospheric forcing +# --------------------------------------------------------------------------- +num_ensemble=${num_ensemble:-50} + +if [[ "${modelid}" == *clm* ]]; then + +lsmforcpertsrc_dir=${lsmforcpertsrc_dir:-${ctl_dir}/../src/eCLM_atmforcing/mkperturb} +eclmfrc_dir=${eclmfrc_dir:-${frc_dir}/eclm/forcing/} +pdaffrc_dir=${pdaffrc_dir:-${eclmfrc_dir}} + +check_var_def pre_config_pdaf_env $(find ${lsmforcpertsrc_dir}/../ -type f -name "*sh") "Using environment file " + +pyvenv_pre_config_pdaf=${pyvenv_pre_config_pdaf:-${ctl_dir}/virtualenvs/pyenv_pre_config_pdaf} + +# load environment +source ${pre_config_pdaf_env} + +# create virtual env if it does not exist yet +if [ ! -d "${pyvenv_pre_config_pdaf}" ]; then + echo "Virtual env not found at ${pyvenv_pre_config_pdaf}, installing " + python -m venv ${pyvenv_pre_config_pdaf} + source ${pyvenv_pre_config_pdaf}/bin/activate + pip install ${ctl_dir}/../src/eCLM_atmforcing/ + deactivate +fi + +fi # clm +fi # pdaf if ${debugmode}; then diff --git a/ctl/pre_ctl/pre_run.sh b/ctl/pre_ctl/pre_run.sh index 8a6e907..7c0686a 100755 --- a/ctl/pre_ctl/pre_run.sh +++ b/ctl/pre_ctl/pre_run.sh @@ -41,7 +41,44 @@ for yyyymm in "${listfrcfile[@]}"; do done unset yyyymm year month -fi +fi #eclm + +if [[ "${modelid}" == *pdaf* ]]; then +if [[ "${modelid}" == *clm* ]]; then + +cd "${lsmforcpertsrc_dir}" + +# load environment +source ${pre_config_pdaf_env} +source ${pyvenv_pre_config_pdaf}/bin/activate + +# list contain dates needed for forcing +for yyyymm in "${listfrcfile[@]}"; do + year="${yyyymm%%-*}" + month="${yyyymm#*-}" + + ( + exec srun --exclusive -n 1 python perturb_forcings.py \ + --years "${year}" \ + --months "${month}" \ + --num-ensemble ${num_ensemble} \ + --fdir ${eclmfrc_dir} \ + --outdir ${pdaffrc_dir} + ) & + + # cap concurrent jobs at npnode (cores/tasks per node) + while (( $(jobs -rp | wc -l) >= npnode )); do + wait -n + done + +done +wait +unset yyyymm year month +deactivate + + +fi #eclm +fi #pdaf } # pre_run diff --git a/ctl/sim_ctl/sim_config.sh b/ctl/sim_ctl/sim_config.sh index 9f8d330..745a7f7 100755 --- a/ctl/sim_ctl/sim_config.sh +++ b/ctl/sim_ctl/sim_config.sh @@ -28,33 +28,10 @@ if [[ "${scheduler}" == "pbs" || "${scheduler}" == "local" ]]; then # Open MPI app-context file, ranks assigned per-line count (used with 'mpiexec --app') mpmd_mapping_file=mpi_multiprog_mapping.conf mpmd_run_cmd="mpiexec --app" - if [[ "${modelid}" == *icon* ]]; then - echo "-np ${ico_proc} ./icon" >> ${sim_dir}/${mpmd_mapping_file} - fi - if [[ "${modelid}" == *eclm* ]]; then - echo "-np ${clm_proc} ./eclm" >> ${sim_dir}/${mpmd_mapping_file} - fi - if [[ "${modelid}" == *parflow* ]]; then - echo "-np ${pfl_proc} ./parflow __pfl_expid__" >> ${sim_dir}/${mpmd_mapping_file} - fi else # Slurm multi-prog rank-range mapping file (used with 'srun --multi-prog') mpmd_mapping_file=slm_multiprog_mapping.conf mpmd_run_cmd="srun --multi-prog" - if [[ "${modelid}" == *icon* ]]; then - echo "0-__icon_pe__ ./icon" >> ${sim_dir}/${mpmd_mapping_file} - fi - if [[ "${modelid}" == *eclm* ]]; then - echo "__clm_ps__-__clm_pe__ ./eclm" >> ${sim_dir}/${mpmd_mapping_file} - fi - if [[ "${modelid}" == *parflow* ]]; then - echo "__pfl_ps__-__pfl_pe__ ./parflow __pfl_expid__" >> ${sim_dir}/${mpmd_mapping_file} - fi - sed -i "s/__icon_pe__/$(($ico_proc-1))/" ${sim_dir}/${mpmd_mapping_file} - sed -i "s/__clm_ps__/$(($ico_proc))/" ${sim_dir}/${mpmd_mapping_file} - sed -i "s/__clm_pe__/$(($ico_proc+$clm_proc-1))/" ${sim_dir}/${mpmd_mapping_file} - sed -i "s/__pfl_ps__/$(($ico_proc+$clm_proc))/" ${sim_dir}/${mpmd_mapping_file} - sed -i "s/__pfl_pe__/$(($ico_proc+$clm_proc+$pfl_proc-1))/" ${sim_dir}/${mpmd_mapping_file} fi # change to run directory @@ -277,7 +254,6 @@ if [[ "${modelid}" == *parflow* ]]; then sed -i "s#__inifile__#$(basename "$fini_pfl")#" coup_oas.tcl sed -i "s/__pfltsfilerst__/${pfltsfilerst}/" coup_oas.tcl sed -i "s/__pfloutmfilt__/${pfloutmfilt}/" coup_oas.tcl - sed -i "s/__pfl_expid__/$EXP_ID/" ${mpmd_mapping_file} # --- execute ParFlow distributeing tcl-scripts export PARFLOW_DIR=${tsmp2_install_dir} @@ -319,6 +295,134 @@ if [[ "${run_oasis}" == true ]]; then fi # if modelid == oasis +if [[ "${modelid}" == *pdaf* ]]; then + +parse_config_file ${conf_file} "sim_config_pdaf" + +# --------------------------------------------------------------------------- +# PDAF configuration +# --------------------------------------------------------------------------- +num_ensemble=${num_ensemble:-50} +pdafinputfilegen_dir=${pdafinputfilegen_dir:-${ctl_dir}/../src/PDAF_input-file-generator/mkenkfpfpar} +pdafnamelistgen_dir=${pdafnamelistgen_dir:-${ctl_dir}/../src/eCLM_namelist-generator} + +check_var_def sim_config_pdaf_env $(find ${pdafinputfilegen_dir}/../ -type f -name "*sh") "Using environment file " + +pyvenv_sim_config_pdaf=${pyvenv_sim_config_pdaf:-${ctl_dir}/virtualenvs/pyenv_sim_config_pdaf} + +# load environment +source ${sim_config_pdaf_env} + +# create virtual env if it does not exist yet +if [ ! -d "${pyvenv_sim_config_pdaf}" ]; then + echo "Virtual env not found at ${pyvenv_sim_config_pdaf}, installing " + python -m venv ${pyvenv_sim_config_pdaf} + source ${pyvenv_sim_config_pdaf}/bin/activate + pip install ${ctl_dir}/../src/PDAF_input-file-generator/ + pip install ${pdafnamelistgen_dir} + deactivate +fi + +# activate virtual env +source ${pyvenv_sim_config_pdaf}/bin/activate + +if [[ "${modelid}" == *clm* ]]; then + +# eCLM_namelist-generator defaults (see create_ensemble_namelists.py argparse defaults) +eclmfrc_dir=${eclmfrc_dir:-${frc_dir}/eclm/forcing/} +pdaffrc_dir=${pdaffrc_dir:-${eclmfrc_dir}} +pdaf_clm_backend=${pdaf_clm_backend:-re} +pdaf_clm_forcings_dir=${pdaf_clm_forcings_dir:-${pdaffrc_dir}} +pdaf_clm_suffix_fsurdat=${pdaf_clm_suffix_fsurdat:-false} +pdaf_clm_suffix_finidat=${pdaf_clm_suffix_finidat:-false} +pdaf_clm_suffix_paramfile=${pdaf_clm_suffix_paramfile:-false} + +python ${pdafnamelistgen_dir}/create_ensemble_namelists.py \ + --num_ensemble ${num_ensemble} \ + --backend ${pdaf_clm_backend} \ + --forcings-dir ${pdaf_clm_forcings_dir} \ + $( [[ "${pdaf_clm_suffix_fsurdat}" == "true" ]] && echo --suffix-fsurdat ) \ + $( [[ "${pdaf_clm_suffix_finidat}" == "true" ]] && echo --suffix-finidat ) \ + $( [[ "${pdaf_clm_suffix_paramfile}" == "true" ]] && echo --suffix-paramfile ) + +fi # clm + +python ${pdafinputfilegen_dir}/adapt_enkfpf_par.py \ + --pf-nprocs ${pfl_proc} \ + --clm-nprocs ${clm_proc} \ + --da-nreal ${num_ensemble} + +deactivate + +# tsmp-pdaf command line options (see PDAF/interface/framework/init_pdaf.F90 for defaults) +pdaf_screen=${pdaf_screen:-2} +pdaf_filtertype=${pdaf_filtertype:-2} +pdaf_subtype=${pdaf_subtype:-0} +pdaf_delt_obs=${pdaf_delt_obs:-2} +pdaf_rms_obs=${pdaf_rms_obs:-0.5} +pdaf_use_omi=${pdaf_use_omi:-false} +pdaf_forget=${pdaf_forget:-1.0} +pdaf_type_forget=${pdaf_type_forget:-0} +pdaf_locweight=${pdaf_locweight:-0} +pdaf_cradius=${pdaf_cradius:-0.0} +pdaf_sradius=${pdaf_sradius:-${pdaf_cradius}} +pdaf_obs_file=${pdaf_obs_file:-output.dat} + +pdaf_cmd_options="-n_modeltasks ${num_ensemble} -screen ${pdaf_screen} \ +-filtertype ${pdaf_filtertype} -subtype ${pdaf_subtype} \ +-delt_obs ${pdaf_delt_obs} -rms_obs ${pdaf_rms_obs} \ +-use_omi .${pdaf_use_omi}. -forget ${pdaf_forget} -type_forget ${pdaf_type_forget} \ +-locweight ${pdaf_locweight} -cradius ${pdaf_cradius} -sradius ${pdaf_sradius} \ +-obs_filename ${pdaf_obs_file}" + +fi # modelid == pdaf + +#### +## MPMD mapping file +#### + +# MPMD program-rank mapping file and run command for the coupled run (format depend on scheduler) +if [[ "${scheduler}" == "pbs" || "${scheduler}" == "local" ]]; then + if [[ "${modelid}" != *pdaf* ]]; then + # Open MPI app-context file, ranks assigned per-line count (used with 'mpiexec --app') + if [[ "${modelid}" == *icon* ]]; then + echo "-np ${ico_proc} ./icon" >> ${sim_dir}/${mpmd_mapping_file} + fi + if [[ "${modelid}" == *eclm* ]]; then + echo "-np ${clm_proc} ./eclm" >> ${sim_dir}/${mpmd_mapping_file} + fi + if [[ "${modelid}" == *parflow* ]]; then + echo "-np ${pfl_proc} ./parflow ${EXP_ID}" >> ${sim_dir}/${mpmd_mapping_file} + fi + else + echo "-np ${tot_proc} ./tsmp-pdaf ${pdaf_cmd_options}" >> ${sim_dir}/${mpmd_mapping_file} + fi +else + # Slurm multi-prog rank-range mapping file (used with 'srun --multi-prog') + if [[ "${modelid}" != *pdaf* ]]; then + if [[ "${modelid}" == *icon* ]]; then + echo "0-__icon_pe__ ./icon" >> ${sim_dir}/${mpmd_mapping_file} + fi + if [[ "${modelid}" == *eclm* ]]; then + echo "__clm_ps__-__clm_pe__ ./eclm" >> ${sim_dir}/${mpmd_mapping_file} + fi + if [[ "${modelid}" == *parflow* ]]; then + echo "__pfl_ps__-__pfl_pe__ ./parflow ${EXP_ID}" >> ${sim_dir}/${mpmd_mapping_file} + fi + sed -i "s/__icon_pe__/$(($ico_proc-1))/" ${sim_dir}/${mpmd_mapping_file} + sed -i "s/__clm_ps__/$(($ico_proc))/" ${sim_dir}/${mpmd_mapping_file} + sed -i "s/__clm_pe__/$(($ico_proc+$clm_proc-1))/" ${sim_dir}/${mpmd_mapping_file} + sed -i "s/__pfl_ps__/$(($ico_proc+$clm_proc))/" ${sim_dir}/${mpmd_mapping_file} + sed -i "s/__pfl_pe__/$(($ico_proc+$clm_proc+$pfl_proc-1))/" ${sim_dir}/${mpmd_mapping_file} + else + echo "0-$((tot_proc-1)) ./tsmp-pdaf ${pdaf_cmd_options}" >> ${sim_dir}/${mpmd_mapping_file} + fi +fi + +##### +## DEBUG MODE +##### + if ${debugmode}; then # create job submission script (tsmp2.job) diff --git a/src/PDAF_input-file-generator b/src/PDAF_input-file-generator new file mode 160000 index 0000000..4ab43bb --- /dev/null +++ b/src/PDAF_input-file-generator @@ -0,0 +1 @@ +Subproject commit 4ab43bb0af41ee16678523e233dd479a8de83c85 diff --git a/src/eCLM_namelist-generator b/src/eCLM_namelist-generator new file mode 160000 index 0000000..36c77e9 --- /dev/null +++ b/src/eCLM_namelist-generator @@ -0,0 +1 @@ +Subproject commit 36c77e9f4072b8229c534be402f29b1bf0e2a9ba