diff --git a/.gitignore b/.gitignore index 714c7ba4..4c088872 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,9 @@ src/exozippy/models/NextGen/runtime/ src/exozippy/models/Claret/ATLAS_L_vt2/ src/exozippy/models/Claret/build/ src/exozippy/models/Claret/sources/ +# Third-party extinction maps fetched on first use by exozippy-mkdust +# (utilities/mkdust.py): Zelakiewicz+2025 (Zenodo) and Nataf+2013 Table 3. +src/exozippy/models/dustmaps/ src/exozippy/components/evolutionarymodel/testing.ipynb src/exozippy/components/evolutionarymodel/massradius_mann.py src/exozippy/components/evolutionarymodel/massradius_mist.py diff --git a/CLAUDE.md b/CLAUDE.md index 1ab223d5..71d867d4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -95,6 +95,7 @@ Each line names when to read the doc. Read it first; do not reconstruct its reas - Before touching the microlensing likelihood, the seeding (the built-in peak finder; the removed MMEXOFAST keys), or the lens/source body maps, read `src/exozippy/components/mulensing/mulensing.md`. - Before comparing a microlensing parameter to a published one, converting to or from another modelling code, or "fixing" a microlensing sign, read `src/exozippy/components/mulensing/conventions.md` -- the origins, the parallax signs, `alpha`, `q > 1`, and the mappings onto the other common literature conventions. The Conventions section of paper III carries the same numbered claims. The chi2 tables behind the `alpha` rules (C18, C21, C22) are in `docs/alpha_conventions.md`. - Before giving the blended light of a microlensing target a parameter of its own, touching the unresolved-background component (its per-band flux, its light-curve tie, its photocenter and proper-motion prior), adding a prior on the background's fluxes or colors, or letting another component read it, read `src/exozippy/components/background/background.md`. +- Before changing the dust-map prior tool (`exozippy-mkdust`: which maps it reads, the default map, the Nataf+2013 conversion to A_V, the widths), read `docs/dust_maps.md`. - Before touching the red-clump reference (`components/redclump/`), the SED flux-source seam (`Component.sed_flux_sources`) it is read through, the interim source-extinction tie, or how a published clump extinction reaches a fit, read `src/exozippy/components/redclump/redclump.md`. - Before changing a stellar prior (IMF, the FFP mass function, distance), an empirical stellar or planetary relation, the planet mass coordinate, or the planet radius prior (uniform in R_P/R_*), read `src/exozippy/components/star/star.md`. - Before touching the galactic model's coordinates -- the galactocentric frame, `line_of_sight_basis`, `galactic_xyz`, `polar_velocity`, the bar angle, or any sign in the velocity transform -- read `src/exozippy/components/galacticmodel/galacticmodel.md`. Positions and velocities come from ONE astropy transform, mirrored in x into genulens's orientation; building either by hand is how review 1.7.1 happened. diff --git a/docs/dust_maps.md b/docs/dust_maps.md new file mode 100644 index 00000000..26a59ae8 --- /dev/null +++ b/docs/dust_maps.md @@ -0,0 +1,113 @@ +# Dust-map priors: `exozippy-mkdust` + +`src/exozippy/utilities/mkdust.py`. Given a sky position, it reads every extinction map it can +reach and writes `.dust.params.yaml`: candidate priors on `sed.dust_anchor`, ONE live and +every other commented out (the way `exozippy-mkticsed` comments out non-default catalog rows). +Read this before changing a map, a conversion ratio, a width, or which map is the default. + +``` +exozippy-mkdust 17:53:48.01 -29:15:47.48 --name KB242087 --outpath . +``` + +RA/Dec in decimal degrees, or sexagesimal (RA in hours) with colons. A negative sexagesimal Dec +needs no quoting or `--`. + +## What it feeds, and why no new key + +The SED's sight-line dust profile (`components/sed/sed.md`, "The sight-line dust profile") +samples one total, `sed.dust_anchor`: the extinction at the profile's anchor, in A_V unless the +system config names `sed: {dust_anchor_band: }`. A red-clump map states the extinction to +the clump, at the bar, which is where the anchor sits when the fit has a red clump or a galactic +model (the bar's mean distance); with neither, the anchor is the whole column, a few percent more +at |b| ~ 3-4 deg. So every map value is an ordinary `{mu, sigma, citation}` prior on +`sed.dust_anchor`. The prior needs the profile on: two or more SED objects at different +distances, or `sed: {distance_constrains_av: true}` for one. + +A line written in a band other than V (marked `[band X]` in the file) also needs +`sed: {dust_anchor_band: X}` in the system config; without it the number is read as A_V. The +file says so above the lines. The live line is always A_V, so it can never be misread that way. + +With calibrated red-clump rows in the fit (`components/redclump/redclump.md`), use none of the +bulge maps: they are built from the same clump stars, so the clump would be counted twice. + +## The default + +Following Nataf's 2026 recommendation for bulge sight lines, the live line is the FIRST of these +that gives a usable value: + +1. Surot et al. (2020), VVV E(J-Ks) (VizieR J/A+A/644/A140; the nearest node in a 1' cone). +2. Zelakiewicz et al. (2025), UKIRT A_Ks (Zenodo record 14919461, fetched md5-checked on first + use; the 2' x 2' pixel holding the target). A pixel the authors flag as a lower limit is + written, marked, and never the default. +3. Nataf et al. (2013), OGLE-III A_I and E(V-I) (Table 3 from the OGLE archive, fetched size- and + md5-checked; the nearest reliable point within 600", as the OGLE calculator does). +4. Schlafly & Finkbeiner (2011)'s recalibrated SFD E(B-V) (IRSA), only at |b| >= 5 deg: it is the + whole column, and not valid toward the inner bulge. + +Surot's map covers Zelakiewicz's, so Zelakiewicz is the default only when Surot is unreachable; +the file then says which preferred map it passed over and why (a failed query is listed as +FAILED with the error, and logged as a WARNING). + +### Why the default is A_V converted by hand, not Ks + +`sed.md` lets a map's value enter in its own band (`dust_anchor_band`), converted by the fit's +law. That is the right channel for a Ks map ONLY once the law's near-infrared slope is right. The +fit's law today is CCM89 + O'Donnell, tabulated at R_V = 3.1 or with a fitted R_V, and its +near-infrared slope is fixed: at R_V = 3.1, A_V/A_Ks ~ 8.4, while the bulge measures 13-15 +(Nataf et al. 2016's median 13.44; 15.3 from the Nataf et al. 2013 means below). A Ks anchor would +put ~40% too little dust in V and I, the bands most microlensing data are in. So the live line +converts the map to A_V with Nataf et al. (2013)'s bulge ratios, and the Ks-native lines are +written commented. They become the better choice when the law's near-infrared slope is fitted. + +## The conversions and widths + +- **Nataf et al. (2013) bulge means.** A_I = 1.217 E(V-I) (their eq. 29 at the mean R_JKVI) and + R_JKVI = E(J-Ks)/E(V-I) = 0.3433 (their Sec. 8.1). Hence A_V = A_I + E(V-I) = 2.217 E(V-I), and + + A_V / E(J-Ks) = (1.217 + 1) / 0.3433 = 6.458. + +- **The ratio's sight-line scatter, 7%.** Over the reliable Table 3 sight lines with E(V-I) > 0.5, + (A_I + E(V-I)) / (R_JKVI E(V-I)) has mean 6.465 (agreeing with 6.458) and a fractional standard + deviation of 0.070 (0.070 again for |b| < 3). That is the width a mean ratio carries on one + sight line; it is added in quadrature to the map's own error. +- **Surot et al. (2020):** A_V = 6.458 E(J-Ks); sigma from the node's E(J-Ks) error and the 7%. + The catalog's error is the error of the node MEAN (the star-to-star dispersion is sqrt(20) times + larger); the anchor is a pencil-beam value whose per-object scatter the profile models + separately (`star.av_frac`, `redclump.av_frac`), so the mean's error is the right one. The + Ks line is A_Ks = 0.422167 E(J-Ks), Surot's own measured VVV bulge vector (their eq. 1), in + `Paranal/VISTA.Ks`. +- **Zelakiewicz et al. (2025):** A_Ks to E(J-Ks) through the same Surot vector, then the Nataf + ratio: A_V = 6.458 / 0.422167 A_Ks = 15.30 A_Ks. The width adds a 10% term for the A_Ks to + E(J-Ks) step, the spread of published bulge A_Ks/E(J-Ks) (0.422 Surot et al. 2020, 0.428 + Alonso-Garcia et al. 2017, 0.423 Nagatomo et al. 2019, 0.49 Wang & Chen 2019), to the map error + (the mean of the asymmetric errors) and the 7%. The Ks line is written in `2MASS/2MASS.Ks`: + UKIRT WFCAM K has no bolometric-correction column. +- **Nataf et al. (2013) Table 3:** A_V = A_I + E(V-I) directly, no NIR step. The width is the + listed differential reddening S_E(V-I) propagated, (A_I/E(V-I) + 1) S_E(V-I); a point that lists + S_E(V-I) = 0 gets Nataf's 9% field-scale granularity (their eq. 20), and the note says so. The + A_I line is in `Generic/Cousins.I`. +- **SFD:** A_V = 3.1 E(B-V) on the Schlafly & Finkbeiner (2011) recalibration, sigma from the 5' + standard deviation IRSA reports. + +On KMT-2024-BLG-2087 ((l, b) = (0.64, -1.70)) the two independent routes agree: Zelakiewicz's +A_Ks = 0.216 gives A_V = 3.30 +/- 0.86 and Nataf's OGLE-III point gives A_V = 3.36 +/- 0.35. + +## Maps it does not read + +Listed in every output with the reason, and never approximated: + +- Zucker et al. (2025), 3D: a multi-GB cube read through the `dustmaps` package, not a web + query; it saturates at A_V ~ 12 and does not reach the bulge itself. +- Green et al. (2019), Bayestar19: the `dustmaps` package (multi-GB); dec > -30 only. +- Gonzalez et al. (2012), BEAM: an interactive web calculator only; Surot et al. (2020) + supersedes it over the same VVV area. +- VVV-ExMaps (the BaSTI web service of Surot et al. 2020): needs a user ID; the same map is read + from VizieR. +- Marshall et al. (2006): a 15', Besancon-model 3D map; not read. + +## Tests + +`tests/test_mkdust.py`: each reader on a fixture in the map's own format (no network), the +default order and a failed query, the SFD latitude rule, the CLI's negative Dec, every written +line parsing as YAML with references.bib citations, and the live line resolving as the +`sed.dust_anchor` prior of a dust-profile system. diff --git a/pyproject.toml b/pyproject.toml index fa677a4a..643a3547 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,6 +234,7 @@ exozippy-gui = "exozippy.gui.app:main" # main()s, kept for the historical `python scripts/X.py` spelling). exozippy-getdata = "exozippy.utilities.getdata:main" exozippy-mkticsed = "exozippy.utilities.mkticsed:main" +exozippy-mkdust = "exozippy.utilities.mkdust:main" exozippy-exofast2exozippy = "exozippy.utilities.exofast2exozippy:main" # Example data that is NOT redistributed with EXOZIPPy (licence # compatibility -- see components/pharmacokinetics/fetch_theoph.py). diff --git a/scripts/zenodo_publish.py b/scripts/zenodo_publish.py index 924935eb..74a5d88f 100644 --- a/scripts/zenodo_publish.py +++ b/scripts/zenodo_publish.py @@ -835,7 +835,18 @@ def load_registry(path: Path): def _q(s: str) -> str: - """A double-quoted Python string literal, as ruff format writes it.""" + """A Python string literal, as ruff format writes it. + + Double-quoted, unless the text holds a double quote and no single one: + then ruff prefers single quotes to escaping (a Zenodo title that quotes + its paper's title, e.g. 'Data for "..."'). + """ + if '"' in s and "'" not in s: + return ( + "'" + + json.dumps(s, ensure_ascii=True)[1:-1].replace('\\"', '"') + + "'" + ) return json.dumps(s, ensure_ascii=True) diff --git a/src/exozippy/components/redclump/redclump.md b/src/exozippy/components/redclump/redclump.md index 0de25fb8..6239d907 100644 --- a/src/exozippy/components/redclump/redclump.md +++ b/src/exozippy/components/redclump/redclump.md @@ -125,7 +125,10 @@ through the passband on the clump's own spectrum at the fit's law (the tables, o A published value is a params-file prior on that element, with its citation: `redclump.RedClump.extinction_0: {mu: 1.80, sigma: 0.05, citation: ...}` (the owner spelling, `element_owners`). Better still, when the paper gives the calibrated I_RC and (V-I)_RC, write -them as SED rows naming the clump: that is the measurement, not a derived number. +them as SED rows naming the clump: that is the measurement, not a derived number. A 2D MAP value +(Surot et al. 2020, Zelakiewicz et al. 2025, Nataf et al. 2013) goes on `sed.dust_anchor` instead, +written by `exozippy-mkdust` (`docs/dust_maps.md`) -- and not beside calibrated clump rows, +since the maps are built from the same clump stars. This REPLACES the monochromatic inversions (review 2.9.16(b)): `extinction. av_from_band_extinction` / `av_from_colour_excess` and mkticsed's `band_extinction` (which now diff --git a/src/exozippy/components/sed/defaults.yaml b/src/exozippy/components/sed/defaults.yaml index 9dc3df0d..ff91f831 100644 --- a/src/exozippy/components/sed/defaults.yaml +++ b/src/exozippy/components/sed/defaults.yaml @@ -65,8 +65,10 @@ sed: # The anchored total: the extinction in sed.dust_anchor_band (default A_V # itself) at the anchor -- the red clump's distance when a clump is in the # fit, else the whole column. It carries the AMOUNT; the bins only the - # shape. A 2D map's value is a prior HERE (mu/sigma + citation), in the - # map's own band (dust_anchor_band), with no law conversion by hand. + # shape. A 2D map's value is a prior HERE (mu/sigma + citation): + # `exozippy-mkdust` writes one per map (docs/dust_maps.md), the default as + # A_V through Nataf+2013's bulge ratios, since the law's fixed NIR slope + # makes a Ks anchor (dust_anchor_band) put too little dust in the optical. # DEFAULT_WEAK: a user's A_V start on a star seeds it (a hint), and this # default yields to any other start. Lower bound strictly positive (the # anchor-band inversion divides by the band extinction). diff --git a/src/exozippy/components/sed/sed.md b/src/exozippy/components/sed/sed.md index 64ca73b0..1ff112e6 100644 --- a/src/exozippy/components/sed/sed.md +++ b/src/exozippy/components/sed/sed.md @@ -233,7 +233,7 @@ The tie has a second table source since 2026-10-20: tables EXOZIPPy generates it **The anchor: one sampled total, the bins only shape** (ruling 12). `A(d) = A_anchor C(d) / C(d_anchor)`. `sed.dust_anchor` is sampled directly (DEFAULT_WEAK, so a user's start on any star seeds it): the well-measured total is one axis, not a curved ridge through K log-increments. `d_anchor` is the red clump's SAMPLED distance when a clump is in the fit (an `SEDFluxSource` with `extinction_anchor=True`; at most one); else, with a galactic model, the Galactic bar's MEAN distance on the sight line (`GalacticModel.bar_distance_moments`, the distance a clump block's prior is centered on), a FIXED number, not sampled; else infinity (`C(inf) = sum 10**z_k`: the whole column), with a DEBUG line. The bar, not the column, because a red-clump extinction map (Surot et al. 2020, Zelakiewicz et al. 2025) states the extinction TO THE CLUMP, at the bar, and a source behind the bar sees 3-5% more at |b| ~ 3-4 deg (JDE 2026-11-10). The clump's own `redclump.av = A(d_clump) (1 + redclump.av_frac)`, `av_frac` ~ N(0, 0.09) (Nataf et al. 2013's field-scale granularity: the centroid measures the pencil beam to 9%), which SUPERSEDES PR 2's interim `av_ratio` tie (kept only with the profile off; `clump_constrains_source_av: true` under the profile raises). -**The anchor's band** (`sed: {dust_anchor_band: }`). Absent: the anchor IS A_V (no conversion; every shipped example). With a filter (a NIR map's 2MASS Ks), `sed.dust_anchor` is the extinction in THAT band, integrated through the passband on a reference red-clump spectrum (Teff 4700 K, logg 2.5, [Fe/H] 0 -- the giants the maps are built from) at the fit's law, so a map value is a prior natively with no hand conversion and, under `fit_rv_dust`, a tight A_Ks no longer makes A_V and R_V a curved ridge. `sed.av_anchor` (derived, reported) is A_V at the anchor: the band-integrated A_band(A_V) inverted on the SED's OWN grid (`bc_node`, table or spectral path) by three fixed-point steps from the A_V = 1 ratio, converging to < 1e-5 (the ratio A_band/A_V varies by a few percent over A_V 1-8). The filter must have a BC column (it is requested through `sed_flux_filters`). Why A_V is the default and not A_I: it keeps every shipped example bit-identical, it is the grid's own axis, and every per-star `av` the BC path consumes is A_V. +**The anchor's band** (`sed: {dust_anchor_band: }`). Absent: the anchor IS A_V (no conversion; every shipped example). With a filter (a NIR map's 2MASS Ks), `sed.dust_anchor` is the extinction in THAT band, integrated through the passband on a reference red-clump spectrum (Teff 4700 K, logg 2.5, [Fe/H] 0 -- the giants the maps are built from) at the fit's law, so a map value is a prior natively with no hand conversion and, under `fit_rv_dust`, a tight A_Ks no longer makes A_V and R_V a curved ridge. `sed.av_anchor` (derived, reported) is A_V at the anchor: the band-integrated A_band(A_V) inverted on the SED's OWN grid (`bc_node`, table or spectral path) by three fixed-point steps from the A_V = 1 ratio, converging to < 1e-5 (the ratio A_band/A_V varies by a few percent over A_V 1-8). The filter must have a BC column (it is requested through `sed_flux_filters`). Why A_V is the default and not A_I: it keeps every shipped example bit-identical, it is the grid's own axis, and every per-star `av` the BC path consumes is A_V. **A map value** comes from `exozippy-mkdust` (`docs/dust_maps.md`), which writes every map it can read as a commented `sed.dust_anchor` line and leaves one live. The live line is A_V converted with Nataf et al. (2013)'s bulge ratios, not a native Ks line: the law's near-infrared slope is fixed (A_V/A_Ks ~ 8.4 at R_V = 3.1 against the bulge's 13-15), so a Ks anchor would put ~40% too little dust in V and I until that slope is fitted. **Each object's own coordinate.** `star.av_frac` ~ N(0, 0.03) per read star (Sale 2012's per-star term; JDE's 0.02-0.05), bounds [-0.9, 10] as validity limits (A_V > 0). THE one spelling of the width is its sigma, `star..av_frac: {sigma: 0.05}`. Under the profile `av_frac` joins the `STAR_KNOBS` (pinned and inactive on unread stars, as av), counts in `PHOTOMETRIC_DOF` in place of av, and the errscale count adds the anchor once (and under the split the near R_V and the boundary); the z_k are prior-only shape and are not counted. An orbit group links `av_frac` instead of `av` (`Star.orbit_linked_params`: a derived member would raise). A derived av has no lower grid barrier (A(d) > 0 by construction; a barrier at 0 would tax a nearby lens), and keeps the grid's upper one, as `loggsed` does. diff --git a/src/exozippy/components/sed/sed.py b/src/exozippy/components/sed/sed.py index 3fa18575..7472f987 100644 --- a/src/exozippy/components/sed/sed.py +++ b/src/exozippy/components/sed/sed.py @@ -583,7 +583,7 @@ def prefix(self): @classmethod def get_utilities(cls): - from ...utilities import mkticsed + from ...utilities import mkdust, mkticsed from ...utilities.registry import ( UtilitySpec, argparse_subprocess_runner, @@ -602,6 +602,19 @@ def get_utilities(cls): build_parser=mkticsed.build_parser, run=argparse_subprocess_runner("exozippy.utilities.mkticsed"), ), + UtilitySpec( + name="mkdust", + label="Dust-map prior for the sight line", + description=( + "Query the extinction maps at a position and write the " + "sed.dust_anchor prior (every map, all but one commented " + "out)." + ), + component_keys=["sed"], + available=True, + build_parser=mkdust.build_parser, + run=argparse_subprocess_runner("exozippy.utilities.mkdust"), + ), ] @classmethod diff --git a/src/exozippy/latex/references.bib b/src/exozippy/latex/references.bib index b8ae8323..7b83fbc2 100644 --- a/src/exozippy/latex/references.bib +++ b/src/exozippy/latex/references.bib @@ -451,3 +451,5 @@ @article{Bensby:2013 @article{Green:2014, title={A Method for Mapping the Three-dimensional Distribution of Dust in the {Milky Way}}, volume={783}, DOI={10.1088/0004-637X/783/2/114}, number={2}, journal={\apj}, author={Green, Gregory M. and Schlafly, Edward F. and Finkbeiner, Douglas P. and others}, year={2014}, pages={114} } @article{Sale:2012, title={{3D} extinction mapping using hierarchical {Bayesian} models}, volume={427}, DOI={10.1111/j.1365-2966.2012.21662.x}, number={3}, journal={\mnras}, author={Sale, Stuart E.}, year={2012}, pages={2119} } @article{Schlafly:2016, title={The Optical-infrared Extinction Curve and its Variation in the {Milky Way}}, volume={821}, DOI={10.3847/0004-637X/821/2/78}, number={2}, journal={\apj}, author={Schlafly, E. F. and Meisner, A. M. and Stutz, A. M. and others}, year={2016}, pages={78} } +@article{Schlafly:2011, title={Measuring Reddening with {Sloan Digital Sky Survey} Stellar Spectra and Recalibrating {SFD}}, volume={737}, DOI={10.1088/0004-637X/737/2/103}, number={2}, journal={\apj}, author={Schlafly, Edward F. and Finkbeiner, Douglas P.}, year={2011}, pages={103} } +@article{Zelakiewicz:2025, title={A Near-Infrared Extinction and Reddening Map Towards the {Galactic} Bulge Using {UKIRT}}, volume={170}, journal={\aj}, author={Zelakiewicz, Aiden S. and Johnson, Samson A. and Gaudi, B. Scott and Bryden, Geoffrey and Nataf, David M. and Shvartzvald, Yossi}, year={2025}, pages={328}, eprint={2509.25440}, archivePrefix={arXiv}, note={Map data: Zenodo, doi:10.5281/zenodo.14919461} } diff --git a/src/exozippy/utilities/mkdust.py b/src/exozippy/utilities/mkdust.py new file mode 100644 index 00000000..b029bc8d --- /dev/null +++ b/src/exozippy/utilities/mkdust.py @@ -0,0 +1,649 @@ +#!/usr/bin/env python3 +""" +mkdust.py + +Query the extinction maps at a sky position and write the dust-profile +prior a fit consumes, as a commented params-file fragment the user edits. +The dust analogue of mkticsed: every map that answers is written, and all +but ONE are commented out, so switching maps is uncommenting a line. + +What it feeds +------------- +The SED's sight-line dust profile (components/sed/sed.md, "The sight-line +dust profile") samples ONE total, ``sed.dust_anchor``: the extinction at the +profile's anchor, in A_V unless the system config's ``sed:`` block names a +``dust_anchor_band``. A 2D red-clump map states the extinction TO THE +CLUMP, at the bar, which is where the anchor sits when the fit has a red +clump or a galactic model. So a map value is a ``{mu, sigma, citation}`` +prior on ``sed.dust_anchor`` -- no new config channel. + +The default (the live line) +--------------------------- +Following Nataf (2026 Sagan workshop): Surot et al. (2020)'s VVV E(J-Ks) +map, else Zelakiewicz et al. (2025)'s UKIRT A_Ks map, converted to A_V with +the Nataf et al. (2013) bulge-mean ratios (A_I = 1.217 E(V-I), +E(J-Ks)/E(V-I) = 0.3433). Why converted by hand rather than written in Ks +with ``dust_anchor_band``: the fit's own law (CCM89 + O'Donnell, tabulated +at R_V = 3.1 or fitted) has A_V/A_Ks ~ 8.4 at R_V = 3.1, where the bulge +measures 13-15, so a Ks anchor would put ~40% too little dust in the optical +bands most microlensing data are in. The Ks-native lines are written too +(commented); they become the better choice once the NIR slope of the law is +fitted. The numbers behind each conversion and width are in +docs/dust_maps.md. + +Maps that cannot be queried programmatically are listed in the output as +skipped, with the reason. None is ever approximated. + +Usage: + exozippy-mkdust [--name NAME] [--outpath DIR] + + ra/dec in decimal degrees, or sexagesimal (ra in hours) with colons: + exozippy-mkdust 17:53:48.01 -29:15:47.48 --name KB242087 +""" + +from __future__ import annotations + +import argparse +import logging +import math +import re +import sys +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path + +import numpy as np +from astropy import units as u +from astropy.coordinates import Angle, SkyCoord + +from ..components.sed.bc_grid import DEFAULT_MODEL_ROOT +from . import zenodo, zenodo_assets + +logger = logging.getLogger(__name__) + +# --- the conversions (docs/dust_maps.md) -------------------------------------- + +# Nataf et al. (2013) bulge means: A_I/E(V-I) (eq. 29 at the mean R_JKVI) and +# R_JKVI = E(J-Ks)/E(V-I) (Sec. 8.1). +NATAF13_AI_EVI = 1.217 +NATAF13_RJKVI = 0.3433 +# A_V / E(J-Ks) = (A_I + E(V-I)) / E(J-Ks) = (1.217 + 1) / 0.3433 = 6.458. +AV_PER_EJK = (NATAF13_AI_EVI + 1.0) / NATAF13_RJKVI +# Sight-line scatter of that ratio: the 1-sigma fractional spread of +# (A_I + E(V-I)) / (R_JKVI E(V-I)) over the reliable Nataf+2013 Table 3 +# sight lines with E(V-I) > 0.5 (mean 6.465, sd/mean 0.070). +AV_PER_EJK_FRAC_SIGMA = 0.07 +# Surot et al. (2020) eq. 1, their measured VVV bulge reddening vector: +# A_Ks = 0.422167 E(J-Ks). Used (1) to write Surot's map in Ks and (2) to +# take Zelakiewicz's A_Ks to E(J-Ks) before the Nataf+2013 ratio. +SUROT20_AKS_PER_EJK = 0.422167 +# Fractional uncertainty of that A_Ks -> E(J-Ks) step for a map measured in +# Ks: published bulge A_Ks/E(J-Ks) run 0.42-0.49 (Surot+2020, Alonso-Garcia +# +2017 0.428, Nagatomo+2019 0.423, Wang & Chen 2019 0.49). +AKS_TO_EJK_FRAC_SIGMA = 0.10 +# Schlafly & Finkbeiner (2011) recalibrated E(B-V), to A_V at R_V = 3.1. +SF11_RV = 3.1 +# Nataf et al. (2013) eq. 20: field-scale extinction granularity, ~9% of the +# total. Used as the width of a Nataf+2013 Table 3 sight line that lists no +# differential reddening (S_E(V-I) = 0). +NATAF13_GRANULARITY = 0.09 + +# The anchor's band for each map's NATIVE-band line (sed.dust_anchor_band). +VVV_KS = "Paranal/VISTA.Ks" +# UKIRT WFCAM K has no BC column; 2MASS Ks is the nearest band that does. +UKIRT_KS = "2MASS/2MASS.Ks" +COUSINS_I = "Generic/Cousins.I" + +# --- where the downloaded map files live -------------------------------------- + +DUSTMAP_DIR = DEFAULT_MODEL_ROOT / "dustmaps" +ZELAKIEWICZ_RECORD = "zelakiewicz2025_ukirt_extinction" +ZELAKIEWICZ_FILE = "ukirt_map.csv" +# Nataf et al. (2013) Table 3, from the OGLE archive (not on Zenodo). Size +# and md5 pin the 2013-05-31 version (the ~19,000-point grid). +NATAF13_TABLE3 = { + "Table3.dat": { + "url": "https://www.astrouw.edu.pl/ogle/ogle3/ext/blg/Table3.dat", + "size": 3227029, + "md5": "9fbbcb21fc019f947fc0335949f32c5f", + } +} + +SUROT20_VIZIER = "J/A+A/644/A140" +# The Surot nodes reach < 10" resolution near the plane; a 1' cone always +# holds one inside the map. +SUROT20_RADIUS_ARCMIN = 1.0 +# Zelakiewicz pixels are 2' x 2': a target inside the map is within the +# half-diagonal (1.41') of one pixel center. +ZELAKIEWICZ_MATCH_ARCMIN = 1.5 +# The OGLE calculator's own match radius for Table 3. +NATAF13_MATCH_ARCSEC = 600.0 + + +class MapFormatError(RuntimeError): + """A map service answered, but not in the format this tool reads.""" + + +class MapFetchError(RuntimeError): + """A map file could not be downloaded (zenodo.fetch_assets gave up).""" + + +def _fetch(assets: dict) -> None: + """``zenodo.fetch_assets`` into DUSTMAP_DIR, its give-up as MapFetchError.""" + try: + zenodo.fetch_assets(assets, DUSTMAP_DIR) + except RuntimeError as e: + raise MapFetchError(str(e)) from e + + +@dataclass +class Prior: + """One candidate ``sed.dust_anchor`` prior.""" + + mu: float + sigma: float + citation: tuple[str, ...] + band: str | None = None # None: A_V (no dust_anchor_band) + note: str = "" + + +@dataclass +class MapReading: + """What one map said at the target (or why it said nothing).""" + + key: str + title: str + status: str # "ok", "outside", "failed", "skipped" + detail: str = "" + av: Prior | None = None + native: list[Prior] = field(default_factory=list) + + +# --- coordinates -------------------------------------------------------------- + + +def parse_coord(ra: str, dec: str) -> SkyCoord: + """ICRS position from decimal degrees, or sexagesimal with ra in hours.""" + if ":" in ra or ":" in dec: + return SkyCoord(ra, dec, unit=(u.hourangle, u.deg), frame="icrs") + return SkyCoord(float(ra), float(dec), unit=u.deg, frame="icrs") + + +def _lb(coord: SkyCoord) -> tuple[float, float]: + """Galactic (l, b) in degrees, l wrapped to (-180, 180].""" + g = coord.galactic + return float(g.l.wrap_at(180 * u.deg).deg), float(g.b.deg) + + +def _sep_arcmin(l1, b1, l2, b2): + """Angular separation in arcmin (haversine), vectorized over (l2, b2).""" + l1, b1, l2, b2 = map(np.radians, (l1, b1, l2, b2)) + h = ( + np.sin((b2 - b1) / 2) ** 2 + + np.cos(b1) * np.cos(b2) * np.sin((l2 - l1) / 2) ** 2 + ) + return np.degrees(2 * np.arcsin(np.sqrt(h))) * 60.0 + + +# --- the maps ----------------------------------------------------------------- + + +def _av_from_ejk(ejk, e_ejk, citation, note): + """A_V from E(J-Ks) through the Nataf+2013 bulge-mean ratios.""" + av = AV_PER_EJK * ejk + sigma = av * math.hypot(e_ejk / ejk, AV_PER_EJK_FRAC_SIGMA) + return Prior(av, sigma, citation, note=note) + + +def _vizier_cone(coord, catalog, radius_arcmin): + """Rows of `catalog` within `radius_arcmin`, nearest first (``_r``).""" + from astroquery.vizier import Vizier + + tables = Vizier(columns=["**", "_r"], row_limit=5000).query_region( + coord, radius=radius_arcmin * u.arcmin, catalog=catalog + ) + if len(tables) == 0: + return None + return tables[0] + + +def read_surot2020(coord: SkyCoord) -> MapReading: + """Surot et al. (2020) VVV E(J-Ks), nearest node (VizieR cone).""" + m = MapReading("surot2020", "Surot et al. (2020), VVV E(J-Ks)", "ok") + table = _vizier_cone(coord, SUROT20_VIZIER, SUROT20_RADIUS_ARCMIN) + if table is None or len(table) == 0: + m.status = "outside" + m.detail = ( + f"no map node within {SUROT20_RADIUS_ARCMIN}' (outside the VVV " + f"bulge area, |l| < 10, -10 < b < 5)" + ) + return m + for col in ("E(J-Ks)", "e_E(J-Ks)", "_r"): + if col not in table.colnames: + raise MapFormatError( + f"VizieR {SUROT20_VIZIER} has no column '{col}' (columns: " + f"{', '.join(table.colnames)}); mkdust must be updated to " + f"the catalog's current format." + ) + row = table[int(np.argmin(np.asarray(table["_r"], dtype=float)))] + ejk = float(row["E(J-Ks)"]) + e_ejk = float(row["e_E(J-Ks)"]) + r_arcsec = float(row["_r"]) * 60.0 # VizieR _r is in arcmin + m.detail = ( + f"E(J-Ks) = {ejk:.3f} +/- {e_ejk:.3f} (error of the node mean) at " + f'the nearest node, {r_arcsec:.1f}" away' + ) + m.av = _av_from_ejk( + ejk, + e_ejk, + ("Surot:2020", "Nataf:2013"), + "A_V = 6.458 E(J-Ks) (Nataf+2013 means)", + ) + aks = SUROT20_AKS_PER_EJK * ejk + m.native.append( + Prior( + aks, + SUROT20_AKS_PER_EJK * e_ejk, + ("Surot:2020",), + band=VVV_KS, + note="A_Ks = 0.422 E(J-Ks), Surot+2020 eq. 1", + ) + ) + return m + + +def zelakiewicz_path() -> Path: + """The Zelakiewicz+2025 map, fetched (md5-checked) on first use.""" + fetch_all() + return DUSTMAP_DIR / ZELAKIEWICZ_FILE + + +def fetch_all() -> list[Path]: + """Fetch the Zenodo-hosted map into DUSTMAP_DIR (the record's owner hook). + + Zenodo's ``fetch_every_record`` skips it (an on-demand record): only this + tool reads it. + """ + _fetch(zenodo_assets.assets(ZELAKIEWICZ_RECORD)) + return [DUSTMAP_DIR / ZELAKIEWICZ_FILE] + + +def read_zelakiewicz2025(coord: SkyCoord, path: Path) -> MapReading: + """Zelakiewicz et al. (2025) UKIRT A_Ks, the 2' x 2' pixel holding it.""" + m = MapReading( + "zelakiewicz2025", "Zelakiewicz et al. (2025), UKIRT A_Ks", "ok" + ) + data = np.genfromtxt(path, delimiter=",", names=True) + expected = ("l", "b", "aks", "aks_err_up", "aks_err_low", "Flag") + if data.dtype.names is None or not set(expected) <= set(data.dtype.names): + raise MapFormatError( + f"{path} lacks the columns {expected} (has {data.dtype.names})." + ) + l0, b0 = _lb(coord) + sep = _sep_arcmin(l0, b0, data["l"], data["b"]) + i = int(np.argmin(sep)) + if sep[i] > ZELAKIEWICZ_MATCH_ARCMIN: + m.status = "outside" + m.detail = ( + "outside the map (-2.15 < l < 2.71, -2.69 < b < 2.03, and the " + "UKIRT 2017 fields within it)" + ) + return m + row = data[i] + aks = float(row["aks"]) + e_aks = 0.5 * (float(row["aks_err_up"]) + float(row["aks_err_low"])) + flagged = int(row["Flag"]) == 1 + m.detail = ( + f"A_Ks = {aks:.3f} +{row['aks_err_up']:.3f}/-{row['aks_err_low']:.3f} " + f'in the pixel centered {sep[i] * 60:.0f}" away' + ) + if flagged: + m.detail += ( + "; FLAGGED by the authors as a LOWER LIMIT (too few clump stars " + "detected in H): do not use it as a prior" + ) + ejk = aks / SUROT20_AKS_PER_EJK + av = AV_PER_EJK * ejk + sigma = av * math.sqrt( + (e_aks / aks) ** 2 + + AKS_TO_EJK_FRAC_SIGMA**2 + + AV_PER_EJK_FRAC_SIGMA**2 + ) + m.av = Prior( + av, + sigma, + ("Zelakiewicz:2025", "Surot:2020", "Nataf:2013"), + note="A_V = 15.30 A_Ks (Surot+2020 eq. 1, then Nataf+2013 means)", + ) + m.native.append( + Prior( + aks, + e_aks, + ("Zelakiewicz:2025",), + band=UKIRT_KS, + note="UKIRT K written as 2MASS Ks (no UKIRT K BC column)", + ) + ) + if flagged: + m.status = "lower_limit" + return m + + +def nataf2013_path() -> Path: + """Nataf+2013 Table 3, fetched (size- and md5-checked) on first use.""" + _fetch(NATAF13_TABLE3) + return DUSTMAP_DIR / "Table3.dat" + + +def read_nataf2013(coord: SkyCoord, path: Path) -> MapReading: + """Nataf et al. (2013) OGLE-III A_I, E(V-I): nearest reliable point.""" + m = MapReading( + "nataf2013", "Nataf et al. (2013), OGLE-III A_I and E(V-I)", "ok" + ) + data = np.loadtxt(path, comments="#") + if data.ndim != 2 or data.shape[1] != 11: + raise MapFormatError( + f"{path} is not the 11-column Nataf+2013 Table 3 " + f"(shape {data.shape})." + ) + data = data[data[:, 10] == 0] # QualityFlag 0: a reliable sight line + grid = SkyCoord(data[:, 2] * 15.0, data[:, 3], unit=u.deg) + sep = coord.separation(grid).arcsec + i = int(np.argmin(sep)) + if sep[i] > NATAF13_MATCH_ARCSEC: + m.status = "outside" + m.detail = ( + f"no reliable OGLE-III sight line within " + f'{NATAF13_MATCH_ARCSEC:.0f}"' + ) + return m + _, _, _, _, ai, evi, s_evi, _, _, rjkvi, _ = data[i] + av = ai + evi + m.detail = ( + f"A_I = {ai:.3f}, E(V-I) = {evi:.3f}, differential reddening " + f"S_E(V-I) = {s_evi:.3f}, R_JKVI = {rjkvi:.3f}, at the grid point " + f'{sep[i]:.0f}" away' + ) + if s_evi > 0: + sigma_av = (ai / evi + 1.0) * s_evi + sigma_ai = (ai / evi) * s_evi + note = "sigma from S_E(V-I)" + else: + sigma_av = NATAF13_GRANULARITY * av + sigma_ai = NATAF13_GRANULARITY * ai + note = "no S_E(V-I) listed: sigma = 9% (Nataf+2013 eq. 20)" + m.av = Prior( + av, sigma_av, ("Nataf:2013",), note=f"A_V = A_I + E(V-I); {note}" + ) + m.native.append( + Prior(ai, sigma_ai, ("Nataf:2013",), band=COUSINS_I, note=note) + ) + return m + + +def read_sfd(coord: SkyCoord) -> MapReading: + """Schlafly & Finkbeiner (2011) recalibrated SFD E(B-V), via IRSA.""" + from astroquery.ipac.irsa.irsa_dust import IrsaDust + + m = MapReading( + "sfd", "Schlafly & Finkbeiner (2011) / SFD (1998), E(B-V)", "ok" + ) + t = IrsaDust.get_query_table(coord, section="ebv") + for col in ("ext SandF mean", "ext SandF std"): + if col not in t.colnames: + raise MapFormatError( + f"IRSA dust table has no '{col}' (columns: " + f"{', '.join(t.colnames)})." + ) + ebv = float(t["ext SandF mean"][0]) + e_ebv = float(t["ext SandF std"][0]) + m.detail = ( + f"E(B-V)_SF11 = {ebv:.3f} +/- {e_ebv:.3f} (5' mean/std). The WHOLE " + f"column, not the extinction to the bar" + ) + m.av = Prior( + SF11_RV * ebv, + SF11_RV * e_ebv, + ("Schlafly:2011", "Schlegel:1998"), + note="A_V = 3.1 E(B-V)", + ) + return m + + +# Maps that exist but this tool cannot read, and why. Written to the output. +SKIPPED = ( + ( + "Zucker et al. (2025), 3D (DECaPS)", + "a multi-GB 3D cube read through the `dustmaps` package, not a web " + "query; it also saturates at A_V ~ 12 and does not reach the bulge " + "itself", + ), + ( + "Green et al. (2019), Bayestar19 3D", + "read through the `dustmaps` package (multi-GB download); dec > -30 " + "only", + ), + ( + "Gonzalez et al. (2012), BEAM E(J-Ks)", + "an interactive web calculator only (no catalog service); Surot+2020 " + "supersedes it at higher resolution over the same VVV area", + ), + ( + "VVV-ExMaps (BaSTI web service of Surot+2020)", + "needs a user ID from the BaSTI team; the same map is read from " + "VizieR above", + ), + ( + "Marshall et al. (2006), 3D Besancon-model map", + "not read by this tool (15' resolution, model-dependent)", + ), +) + +# The order the default (live) line is chosen in: the first map that gives a +# usable value. Nataf's 2026 recommendation (Surot, Zelakiewicz), then the +# OGLE-III optical map; SFD only off the bulge, where it is valid. +DEFAULT_ORDER = ("surot2020", "zelakiewicz2025", "nataf2013", "sfd") +SFD_MIN_ABS_B = 5.0 + +_NETWORK_ERRORS = (OSError, MapFormatError, MapFetchError) + + +def query_maps(coord: SkyCoord) -> list[MapReading]: + """Read every map at `coord`; a map that fails says so, and why.""" + readers = ( + ("surot2020", "Surot et al. (2020)", lambda: read_surot2020(coord)), + ( + "zelakiewicz2025", + "Zelakiewicz et al. (2025)", + lambda: read_zelakiewicz2025(coord, zelakiewicz_path()), + ), + ( + "nataf2013", + "Nataf et al. (2013)", + lambda: read_nataf2013(coord, nataf2013_path()), + ), + ("sfd", "Schlafly & Finkbeiner (2011)", lambda: read_sfd(coord)), + ) + out = [] + for key, title, read in readers: + try: + out.append(read()) + except _NETWORK_ERRORS as e: + logger.warning( + f"[mkdust] {title}: the query failed ({type(e).__name__}: " + f"{e}); it is listed as failed in the output.\n" + f"Rerun when the service is back to include it." + ) + out.append( + MapReading(key, title, "failed", f"{type(e).__name__}: {e}") + ) + return out + + +def choose_default(readings: list[MapReading], b_deg: float): + """The reading whose A_V line is live, or None.""" + by_key = {m.key: m for m in readings} + for key in DEFAULT_ORDER: + m = by_key.get(key) + if m is None or m.status != "ok" or m.av is None: + continue + if key == "sfd" and abs(b_deg) < SFD_MIN_ABS_B: + continue + return m + return None + + +# --- the output --------------------------------------------------------------- + + +def _fmt(p: Prior) -> str: + cites = ", ".join(p.citation) + return ( + f"sed.dust_anchor: {{mu: {p.mu:.3f}, sigma: {p.sigma:.3f}, " + f"citation: [{cites}]}}" + ) + + +def render(coord: SkyCoord, readings: list[MapReading], name: str) -> str: + """The params-file fragment, as text.""" + l0, b0 = _lb(coord) + default = choose_default(readings, b0) + ra_s = coord.ra.to_string(u.hourangle, sep=":", precision=2, pad=True) + dec_s = coord.dec.to_string( + u.deg, sep=":", precision=2, alwayssign=True, pad=True + ) + lines = [ + f"# Dust-map priors for {name}: RA, Dec = {ra_s} {dec_s} " + f"({coord.ra.deg:.5f}, {coord.dec.deg:.5f}); " + f"l, b = {l0:.4f}, {b0:.4f}", + f"# Written by exozippy-mkdust on {date.today().isoformat()}.", + "#", + "# Paste into the params file. Each line is a prior on", + "# sed.dust_anchor, the total of the SED's sight-line dust profile", + "# (components/sed/sed.md, 'The sight-line dust profile'), which needs", + "# the profile on: two or more SED objects at different distances, or", + "# `sed: {distance_constrains_av: true}` for one.", + "#", + "# A red-clump map states the extinction TO THE CLUMP (at the bar).", + "# The anchor sits there when the fit has a red clump or a galactic", + "# model; with neither it is the whole column, a few percent more.", + "# With calibrated red-clump rows in the fit, use NONE of the bulge", + "# maps: they are built from the same clump stars (counted twice).", + "#", + "# Uncomment exactly ONE sed.dust_anchor line. A line marked", + "# [band X] is in that band, not A_V, and ALSO needs", + "# sed: {dust_anchor_band: X}", + "# in the system config, or it is read as A_V.", + "", + ] + if default is None: + lines += [ + "# No map gave a usable default here; every line below is", + "# commented out. State an A_V prior by hand if you have one.", + "", + ] + else: + passed = [ + m.title + for m in readings + if DEFAULT_ORDER.index(m.key) < DEFAULT_ORDER.index(default.key) + ] + lines.append( + f"# DEFAULT: {default.title}, as A_V. {default.av.note}." + ) + if passed: + lines.append( + f"# (Preferred but unusable here, see below: " + f"{'; '.join(passed)}.)" + ) + lines += [_fmt(default.av), ""] + for m in readings: + lines.append(f"# --- {m.title}: {m.status.upper()}") + if m.detail: + lines.append(f"# {m.detail}") + if m.av is not None and m is not default: + lines.append(f"# A_V: {m.av.note}") + lines.append(f"# {_fmt(m.av)}") + for p in m.native: + lines.append(f"# [band {p.band}]: {p.note}") + lines.append(f"# {_fmt(p)}") + lines.append("") + lines.append("# --- Not queried (cannot be read programmatically here):") + for title, why in SKIPPED: + lines.append(f"# {title}: {why}.") + return "\n".join(lines) + "\n" + + +def mkdust( + ra: str, + dec: str, + name: str = "dust", + outpath: str = ".", + readings: list[MapReading] | None = None, +) -> Path: + """Query every map at (ra, dec) and write ``.dust.params.yaml``. + + ``readings`` replaces the queries (tests, or a caller that has them). + """ + coord = parse_coord(ra, dec) + if readings is None: + readings = query_maps(coord) + text = render(coord, readings, name) + out = Path(outpath) / f"{name}.dust.params.yaml" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(text, encoding="utf-8") + logger.info(f"[mkdust] Wrote {out}") + return out + + +def build_parser(): + """Return the argparse parser for the mkdust utility.""" + p = argparse.ArgumentParser( + prog="exozippy-mkdust", + description=( + "Query the extinction maps at a position and write the " + "sed.dust_anchor prior (all maps, all but one commented out)" + ), + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p.add_argument( + "ra", help="RA: decimal degrees, or hh:mm:ss.ss (hours) with colons" + ) + p.add_argument( + "dec", help="Dec: decimal degrees, or +dd:mm:ss.s with colons" + ) + p.add_argument( + "--name", + default="dust", + help="Base name of the output file .dust.params.yaml", + ) + p.add_argument("--outpath", default=".", help="Output directory") + return p + + +def _protect_negative_sexagesimal(argv): + """Let a negative sexagesimal Dec through argparse. + + argparse takes "-29:15:47.48" for an option (it accepts only plain + negative NUMBERS as positionals), so such a token is rewritten to the + same angle in decimal degrees before parsing. + """ + out = [] + for tok in argv: + if re.fullmatch(r"-\d+:\d+(:\d+(\.\d*)?)?", tok): + tok = f"{Angle(tok, unit=u.deg).deg:.8f}" + out.append(tok) + return out + + +def main(argv=None): + """CLI entry point. Parses argv (or sys.argv) and runs mkdust.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + argv = sys.argv[1:] if argv is None else list(argv) + args = build_parser().parse_args(_protect_negative_sexagesimal(argv)) + out = mkdust(args.ra, args.dec, name=args.name, outpath=args.outpath) + print(out.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + main() diff --git a/src/exozippy/utilities/zenodo.py b/src/exozippy/utilities/zenodo.py index f27b922d..8d244a03 100644 --- a/src/exozippy/utilities/zenodo.py +++ b/src/exozippy/utilities/zenodo.py @@ -94,6 +94,7 @@ import tempfile import time import urllib.error +import urllib.parse import urllib.request from pathlib import Path from typing import Callable, Iterator, Mapping @@ -157,6 +158,12 @@ _adoption_attempted: set[tuple[str, str]] = set() +def _host(meta: Mapping[str, object]) -> str: + """The server an asset comes from, for messages (Zenodo, or e.g. the + OGLE archive for a third-party file exozippy-mkdust reads).""" + return urllib.parse.urlparse(str(meta["url"])).netloc + + def _md5(path: Path, chunk: int = 1 << 20) -> str: """Streaming md5 of a file (the spectra grid is ~250 MB).""" h = hashlib.md5() @@ -570,7 +577,7 @@ def _download_verified( if digest != meta["md5"]: raise RuntimeError( f"{filename} has md5 {digest}, expected " - f"{meta['md5']}. The file on Zenodo may have been " + f"{meta['md5']}. The file on {_host(meta)} may have been " f"replaced, or the download was corrupted." ) return @@ -596,7 +603,7 @@ def _download_verified( time.sleep(delay) raise RuntimeError( - f"Could not download {filename} from Zenodo after " + f"Could not download {filename} from {_host(meta)} after " f"{_DOWNLOAD_ATTEMPTS} attempts: {last_error}" ) from last_error @@ -633,7 +640,7 @@ def _fetch_via_cache( if on_fetch is not None: on_fetch(filename) - logger.info(f"Downloading {filename} from Zenodo...") + logger.info(f"Downloading {filename} from {_host(meta)}...") fd, tmp_name = tempfile.mkstemp( dir=cache, prefix=Path(filename).name + ".", suffix=".part" @@ -741,7 +748,7 @@ def fetch_assets( if on_fetch is not None: on_fetch(filename) - logger.info(f"Downloading {filename} from Zenodo...") + logger.info(f"Downloading {filename} from {_host(meta)}...") fd, tmp_name = tempfile.mkstemp( dir=dest_dir, prefix=dest.name + ".", suffix=".part" diff --git a/src/exozippy/utilities/zenodo_assets.py b/src/exozippy/utilities/zenodo_assets.py index 9158f807..b7e710ea 100644 --- a/src/exozippy/utilities/zenodo_assets.py +++ b/src/exozippy/utilities/zenodo_assets.py @@ -744,6 +744,35 @@ def _pins(table: Mapping[str, tuple[int, str]]) -> Mapping[str, ZenodoFile]: } ), ), + # Zelakiewicz et al. (2025)'s UKIRT A_Ks / E(H-Ks) red-clump map of + # the bulge (a third-party record, CC-BY-4.0). Consumer: + # utilities/mkdust.py, the dust-map prior tool. ON DEMAND: only that + # tool reads it. Cited through the paper (Zelakiewicz:2025). + "zelakiewicz2025_ukirt_extinction": ZenodoRecord( + record_id=14919461, + concept_record_id=14919460, + title=( + 'Data for "A Near-Infrared Extinction and Reddening Map ' + 'Towards the Galactic Bulge Using UKIRT"' + ), + creators=( + "Zelakiewicz, Aiden", + "Johnson, Samson", + "Gaudi, B. Scott", + "Bryden, Geoffrey", + "Nataf, David", + "Shvartzvald, Yossi", + ), + citation_key=None, + files=_pins( + { + "ukirt_map.csv": ( + 741243, + "910fea2e7f03de661cb8cc9acc22cd30", + ), + } + ), + ), } ) @@ -756,7 +785,11 @@ def _pins(table: Mapping[str, tuple[int, str]]) -> Mapping[str, ZenodoFile]: # has an owner with a ``fetch_all()`` (the explicit pre-fetch for a machine # that will run offline); only the automatic pre-fetch skips it. ON_DEMAND: frozenset[str] = frozenset( - {"nextgen_spectra_r20000", "nextgen_runtime_spectra"} + { + "nextgen_spectra_r20000", + "nextgen_runtime_spectra", + "zelakiewicz2025_ukirt_extinction", + } ) # The module that OWNS each record: the one whose loader reads its files, and @@ -775,6 +808,7 @@ def _pins(table: Mapping[str, tuple[int, str]]) -> Mapping[str, ZenodoFile]: "phoenix_ld_tables": "exozippy.models.PHOENIX.ld_tables", "nextgen_spectra_r20000": "exozippy.models.NextGen.bc_synthesis", "nextgen_runtime_spectra": "exozippy.models.NextGen.runtime_spectra", + "zelakiewicz2025_ukirt_extinction": "exozippy.utilities.mkdust", } ) diff --git a/tests/test_citation_field.py b/tests/test_citation_field.py index ed4f8234..c0237f69 100644 --- a/tests/test_citation_field.py +++ b/tests/test_citation_field.py @@ -25,6 +25,7 @@ / "utilities" / "mkticsed.py" ) +MKDUST = MKTICSED.with_name("mkdust.py") # --- normalization at the params-file boundary -------------------------------- @@ -298,15 +299,16 @@ def test_blanking_a_cited_priors_last_field_in_the_gui_removes_the_entry( # --- mkticsed only writes keys the bibliography has --------------------------- -def test_every_key_mkticsed_can_write_exists_in_references_bib(): +@pytest.mark.parametrize("source", [MKTICSED, MKDUST], ids=lambda p: p.name) +def test_every_key_mkticsed_can_write_exists_in_references_bib(source): """ - Given every Author:Year key literal in mkticsed.py, + Given every Author:Year key literal in mkticsed.py (and mkdust.py), When checked against the shipped references.bib, Then each one exists -- so no generated note can print "?". """ # Arrange keys = set( - re.findall(r"[\"']([A-Z][A-Za-z]+:\d{4})[\"']", MKTICSED.read_text()) + re.findall(r"[\"']([A-Z][A-Za-z]+:\d{4})[\"']", source.read_text()) ) # Act diff --git a/tests/test_mkdust.py b/tests/test_mkdust.py new file mode 100644 index 00000000..23a02ec3 --- /dev/null +++ b/tests/test_mkdust.py @@ -0,0 +1,378 @@ +"""exozippy-mkdust: the dust-map prior tool (utilities/mkdust.py). + +No test touches the network: the map files are tiny fixtures written here in +the maps' own formats, and the VizieR/IRSA queries are monkeypatched. +""" + +import re + +import numpy as np +import pytest +import yaml +from astropy import units as u +from astropy.coordinates import SkyCoord +from astropy.table import Table + +from exozippy.outputs.texutils import known_bib_keys +from exozippy.utilities import mkdust as md + +# KMT-2024-BLG-2087: (l, b) = (0.6374, -1.7011). +RA, DEC = "17:53:48.01", "-29:15:47.48" + + +def _coord(): + return md.parse_coord(RA, DEC) + + +def _lb(): + return md._lb(_coord()) + + +def _zel_csv(tmp_path, flag=0, dl_arcmin=0.5): + """A three-pixel Zelakiewicz+2025 map: one near the target, two far.""" + l0, b0 = _lb() + rows = [ + (l0 + dl_arcmin / 60.0, b0, 0.216, 0.050, 0.048, 0.20, 0.04, flag), + (l0 + 0.5, b0, 0.40, 0.05, 0.05, 0.3, 0.04, 0), + (l0, b0 + 0.5, 0.10, 0.05, 0.05, 0.1, 0.04, 0), + ] + path = tmp_path / "ukirt_map.csv" + with open(path, "w") as f: + f.write("l,b,aks,aks_err_up,aks_err_low,ehks,ehks_err,Flag\n") + for r in rows: + f.write(",".join(str(x) for x in r) + "\n") + return path + + +def _nataf_table3(tmp_path, s_evi=0.157): + """A Nataf+2013 Table 3 with an UNRELIABLE point nearest the target.""" + c = _coord() + ra_h = c.ra.deg / 15.0 + l0, b0 = _lb() + rows = [ + # QualityFlag 1, 5" away: must be ignored. + ( + l0, + b0, + ra_h + 5.0 / 3600 / 15, + c.dec.deg, + 9.9, + 9.9, + 0.1, + 14.5, + 0.3, + 0.34, + 1, + ), + # QualityFlag 0, 62" away. + ( + l0, + b0, + ra_h, + c.dec.deg + 62.0 / 3600, + 1.847, + 1.516, + s_evi, + 14.5, + 0.3, + 0.344, + 0, + ), + ] + path = tmp_path / "Table3.dat" + with open(path, "w") as f: + f.write( + "### lgalac bgalac RA DEC A_{I} E(V-I) S_{E(V-I)} mu S_{mu} " + "RJKVI Quality\n" + ) + for r in rows: + f.write(" ".join(repr(float(x)) for x in r) + "\n") + return path + + +def _surot_table(rows): + return Table( + rows=rows, + names=("E(J-Ks)", "e_E(J-Ks)", "_r"), + dtype=(float, float, float), + ) + + +# --- the conversions ---------------------------------------------------------- + + +def test_the_nataf_ratios_give_the_documented_av_per_ejk(): + """ + Given Nataf+2013's bulge means A_I/E(V-I) = 1.217, R_JKVI = 0.3433, + When A_V/E(J-Ks) and A_V/A_Ks are formed, + Then they are the 6.458 and 15.30 the output notes and the doc quote. + """ + assert md.AV_PER_EJK == pytest.approx(6.458, abs=5e-4) + assert md.AV_PER_EJK / md.SUROT20_AKS_PER_EJK == pytest.approx( + 15.30, abs=5e-3 + ) + + +# --- each reader -------------------------------------------------------------- + + +def test_surot_takes_the_nearest_node_and_converts_through_nataf(monkeypatch): + """ + Given a VizieR cone with two Surot nodes, + When read_surot2020 reads it, + Then the NEAREST node is used; A_V = AV_PER_EJK E(J-Ks) with the map + error and the 7% ratio scatter in quadrature; the Ks line is Surot's + own vector in VVV Ks. + """ + table = _surot_table([(2.0, 0.5, 0.9), (0.5, 0.01, 0.1)]) + monkeypatch.setattr(md, "_vizier_cone", lambda *a: table) + + m = md.read_surot2020(_coord()) + + assert m.status == "ok" + assert m.av.mu == pytest.approx(md.AV_PER_EJK * 0.5) + assert m.av.sigma == pytest.approx( + m.av.mu * np.hypot(0.01 / 0.5, md.AV_PER_EJK_FRAC_SIGMA) + ) + assert m.av.band is None + (ks,) = m.native + assert ks.band == md.VVV_KS + assert ks.mu == pytest.approx(md.SUROT20_AKS_PER_EJK * 0.5) + + +def test_surot_outside_and_a_changed_format(monkeypatch): + """ + Given no node in the cone, then a table missing the E(J-Ks) column, + When read_surot2020 reads each, + Then the first is "outside" and the second raises MapFormatError naming + the column (never a guessed column). + """ + monkeypatch.setattr(md, "_vizier_cone", lambda *a: None) + assert md.read_surot2020(_coord()).status == "outside" + + bad = Table(rows=[(1.0, 0.1)], names=("EJK", "_r")) + monkeypatch.setattr(md, "_vizier_cone", lambda *a: bad) + with pytest.raises(md.MapFormatError, match=r"E\(J-Ks\)"): + md.read_surot2020(_coord()) + + +def test_zelakiewicz_reads_the_pixel_holding_the_target(tmp_path): + """ + Given a three-pixel map with one pixel 30" from the target, + When read_zelakiewicz2025 reads it, + Then that pixel's A_Ks is used, A_V goes through Surot's vector and the + Nataf ratio, and its width carries the extra A_Ks -> E(J-Ks) term. + """ + m = md.read_zelakiewicz2025(_coord(), _zel_csv(tmp_path)) + + assert m.status == "ok" + (ks,) = m.native + assert ks.mu == pytest.approx(0.216) and ks.sigma == pytest.approx(0.049) + assert m.av.mu == pytest.approx( + 0.216 / md.SUROT20_AKS_PER_EJK * md.AV_PER_EJK + ) + frac = np.sqrt( + (0.049 / 0.216) ** 2 + + md.AKS_TO_EJK_FRAC_SIGMA**2 + + md.AV_PER_EJK_FRAC_SIGMA**2 + ) + assert m.av.sigma == pytest.approx(m.av.mu * frac) + + +def test_zelakiewicz_outside_and_flagged(tmp_path): + """ + Given the nearest pixel 3' away, then a flagged (lower-limit) pixel, + When read, + Then the first is "outside"; the second is "lower_limit", says so, and + is never chosen as the default. + """ + far = md.read_zelakiewicz2025(_coord(), _zel_csv(tmp_path, dl_arcmin=3)) + assert far.status == "outside" and far.av is None + + flagged = md.read_zelakiewicz2025(_coord(), _zel_csv(tmp_path, flag=1)) + assert flagged.status == "lower_limit" + assert "LOWER LIMIT" in flagged.detail + assert md.choose_default([flagged], _lb()[1]) is None + + +def test_nataf2013_skips_unreliable_points_and_sums_ai_and_evi(tmp_path): + """ + Given Table 3 with an unreliable point nearest and a reliable one 62" + away, + When read_nataf2013 reads it, + Then the reliable point is used: A_V = A_I + E(V-I), its width from the + differential reddening, and the A_I line in Cousins I. + """ + m = md.read_nataf2013(_coord(), _nataf_table3(tmp_path)) + + assert m.status == "ok" + assert m.av.mu == pytest.approx(1.847 + 1.516) + assert m.av.sigma == pytest.approx((1.847 / 1.516 + 1.0) * 0.157) + (ai,) = m.native + assert ai.band == md.COUSINS_I and ai.mu == pytest.approx(1.847) + assert '62"' in m.detail + + +def test_nataf2013_with_no_differential_reddening_uses_the_granularity( + tmp_path, +): + """A sight line listing S_E(V-I) = 0 gets Nataf's 9%, and says so.""" + m = md.read_nataf2013(_coord(), _nataf_table3(tmp_path, s_evi=0.0)) + assert m.av.sigma == pytest.approx(0.09 * (1.847 + 1.516)) + assert "9%" in m.av.note + + +# --- choosing and writing ----------------------------------------------------- + + +def _readings(tmp_path, monkeypatch, surot_ok=True): + if surot_ok: + table = _surot_table([(0.5, 0.01, 0.1)]) + monkeypatch.setattr(md, "_vizier_cone", lambda *a: table) + else: + + def _down(*a): + raise OSError("404 Client Error") + + monkeypatch.setattr(md, "_vizier_cone", _down) + zel = _zel_csv(tmp_path) + nat = _nataf_table3(tmp_path) + monkeypatch.setattr(md, "zelakiewicz_path", lambda: zel) + monkeypatch.setattr(md, "nataf2013_path", lambda: nat) + + def _sfd(coord): + m = md.MapReading("sfd", "SFD", "ok") + m.av = md.Prior(3.0, 0.1, ("Schlafly:2011", "Schlegel:1998")) + return m + + monkeypatch.setattr(md, "read_sfd", _sfd) + return md.query_maps(_coord()) + + +def _live_and_commented(text): + live = yaml.safe_load(text) + commented = [ + yaml.safe_load(line[2:]) + for line in text.splitlines() + if line.startswith("# sed.dust_anchor:") + ] + return live, commented + + +def test_the_file_has_one_live_prior_and_every_map_commented( + tmp_path, monkeypatch +): + """ + Given every map answering, + When the file is written, + Then it parses as YAML with exactly ONE live key, sed.dust_anchor, from + Surot (the default) as A_V; every other candidate is a commented line + that parses once uncommented; every citation is a references.bib key; + and the maps it cannot read are listed as not queried. + """ + readings = _readings(tmp_path, monkeypatch) + out = md.mkdust(RA, DEC, name="t", outpath=tmp_path, readings=readings) + text = out.read_text() + + live, commented = _live_and_commented(text) + assert list(live) == ["sed.dust_anchor"] + assert live["sed.dust_anchor"]["mu"] == pytest.approx( + md.AV_PER_EJK * 0.5, abs=1e-3 + ) + assert live["sed.dust_anchor"]["citation"] == ["Surot:2020", "Nataf:2013"] + # Surot Ks; Zelakiewicz A_V + Ks; Nataf A_V + A_I; SFD A_V. + assert len(commented) == 6 + bib = known_bib_keys() + for entry in [ + live["sed.dust_anchor"], + *[c["sed.dust_anchor"] for c in commented], + ]: + assert set(entry) == {"mu", "sigma", "citation"} + assert entry["sigma"] > 0 + assert set(entry["citation"]) <= bib + assert "[band Paranal/VISTA.Ks]" in text + assert "dust_anchor_band" in text + for title, _ in md.SKIPPED: + assert title in text + + +def test_a_failed_query_is_reported_and_the_next_map_is_the_default( + tmp_path, monkeypatch, caplog +): + """ + Given VizieR down, + When the maps are queried and written, + Then Surot is listed FAILED with the error, a WARNING says so, and the + live line is Zelakiewicz's A_V with the header naming the preferred + map it passed over. + """ + readings = _readings(tmp_path, monkeypatch, surot_ok=False) + text = md.render(_coord(), readings, "t") + + assert readings[0].status == "failed" + assert "404" in readings[0].detail + assert any( + r.levelname == "WARNING" and "Surot" in r.message + for r in caplog.records + ) + live, _ = _live_and_commented(text) + assert "Zelakiewicz:2025" in live["sed.dust_anchor"]["citation"] + assert re.search(r"Preferred but unusable here.*Surot", text) + + +def test_sfd_is_the_default_only_off_the_bulge(): + """ + Given only an SFD reading, + When the default is chosen at |b| = 1.7 and at |b| = 20, + Then there is no default inside |b| < 5 (SFD is not valid there) and SFD + is the default outside. + """ + m = md.MapReading("sfd", "SFD", "ok") + m.av = md.Prior(3.0, 0.1, ("Schlafly:2011",)) + assert md.choose_default([m], -1.7) is None + assert md.choose_default([m], -20.0) is m + + +def test_the_cli_takes_a_negative_sexagesimal_dec(tmp_path, monkeypatch): + """ + Given `exozippy-mkdust 17:53:48.01 -29:15:47.48`, + When main parses it, + Then argparse does not take the Dec for an option, and the written + position is the target's. + """ + seen = {} + + def _query(coord): + seen["coord"] = coord + return [] + + monkeypatch.setattr(md, "query_maps", _query) + md.main([RA, DEC, "--name", "cli", "--outpath", str(tmp_path)]) + + target = SkyCoord(RA, DEC, unit=(u.hourangle, u.deg)) + assert seen["coord"].separation(target).arcsec < 1e-3 + text = (tmp_path / "cli.dust.params.yaml").read_text() + assert "No map gave a usable default" in text + assert yaml.safe_load(text) is None + + +def test_the_live_line_is_a_prior_the_dust_profile_reads( + tmp_path, monkeypatch +): + """ + Given the tool's live line pasted into a lens + source + clump params + file (the dust profile on), + When the system is prepared, + Then sed.dust_anchor resolves to that mu and sigma -- the line feeds the + existing channel, no new key. + """ + from test_dust_profile import _lens_params, _system + + readings = _readings(tmp_path, monkeypatch) + live, _ = _live_and_commented(md.render(_coord(), readings, "t")) + system = _system(tmp_path, params=_lens_params(**live)) + + assert system.sed._dust_plan.on + resolved = system.config_manager.resolve("sed", "dust_anchor") + assert resolved["mu"] == pytest.approx(live["sed.dust_anchor"]["mu"]) + assert resolved["sigma"] == pytest.approx(live["sed.dust_anchor"]["sigma"])