From 396699a4739787f0d55158b350ddc3e366143c4a Mon Sep 17 00:00:00 2001 From: Jeremy Howard Date: Wed, 11 Nov 2026 08:35:22 +0000 Subject: [PATCH 1/2] Reduce mapped array execution overhead Mapped kernels avoid a redundant error scope. Typed reads share copying and conversion for contiguous and repeated inputs. Contiguous-block recognition and source-permutation reads share one affine proof, and cycle tiling sizes padding for the active run. See Functions, formulas and kernels and Search and gathering in DEV. --- Cargo.toml | 2 +- core/src/agreement.rs | 71 ++++++++++++++++------------------------ kernels/src/operation.rs | 10 +++--- 3 files changed, 35 insertions(+), 48 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3553e05..a9848a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ rustymath = { path = ".", version = ">=0.1.60, <1", default-features = false } rustymath-buffer = { path = "buffer", version = ">=0.1.39, <1" } pyo3 = ">=0.29" pyo3-build-config = ">=0.29" -rustymath-core = { path = "core", version = ">=0.1.60, <1", default-features = false } +rustymath-core = { path = "core", version = ">=0.1.61, <1", default-features = false } rustymath-kernels = { path = "kernels", version = ">=0.1.59, <1" } rustymath-timing = { path = "timing", version = ">=0.1.56, <1" } half = { version = ">=2.7.1, <3", features = ["num-traits"] } diff --git a/core/src/agreement.rs b/core/src/agreement.rs index 6aaf7d7..d13719c 100644 --- a/core/src/agreement.rs +++ b/core/src/agreement.rs @@ -246,18 +246,10 @@ impl ReadPlan { let (found, map) = self.positional(shape)?; if found != source { return None; } if items == 0 { return Some(Mapping::Linear(1)); } - let (base, forward) = map.affine(shape)?; - let mut axes: Vec<_> = (0..shape.len()).filter(|&axis| shape[axis] > 1).collect(); - axes.sort_unstable_by_key(|&axis| forward[axis].unsigned_abs()); - let (mut covered, mut first) = (1usize, base as i128); - for &axis in &axes { - if forward[axis].unsigned_abs() != covered { return None; } - covered = covered.checked_mul(shape[axis])?; - if forward[axis] < 0 { first += forward[axis] as i128 * (shape[axis] - 1) as i128; } - } - if covered != items || first != 0 { return None; } + let (range, axes, forward) = map.contiguous_axes(shape)?; + if range != (0..items) { return None; } let strides = crate::layout::strides(shape); - let steps = axes.into_iter().rev().map(|axis| { + let steps = axes.into_iter().map(|axis| { let stride = isize::try_from(strides[axis]).ok()?; let reverse = forward[axis] < 0; Some(Steps::Stride { start: if reverse { (shape[axis] - 1) * strides[axis] } else { 0 }, len: shape[axis], @@ -447,7 +439,8 @@ impl<'s> Read<'s> { if range.is_empty() { return Ok((Arg::Many(&[]), 0)); } if let Some((view, stride)) = self.run_view(&range) { if let Some(items) = borrowed_as::(view, checked)? { return Ok((items, 0)); } - cast_run(view, stride, range.len(), checked, scratch)?; + scratch.clear(); + append_run(view, stride, range.len(), checked, scratch)?; return Ok((Arg::Many(scratch), range.len())); } scratch.clear(); @@ -508,15 +501,7 @@ impl<'s> Read<'s> { let last = first.checked_add_signed(stride * (len as isize - 1)).ok_or_else(outside)?; let low = first.min(last); let view = self.sources[source].borrow(low..first.max(last) + 1).ok_or_else(outside)?; - if stride == 0 { - let item = kept(T::cast(view, 0), checked)?; - output.resize(output.len() + len, item); - return Ok(()); - } - if stride == 1 { - if let Some(items) = T::of(view) { output.extend_from_slice(items); return Ok(()); } - return T::extend_cast(view, len, checked, output); - } + if stride == 0 || stride == 1 { return append_run(view, stride as usize, len, checked, output); } let walked = Walked::from(Mapping::Steps { base: 0, steps: vec![Steps::Stride { start: first - low, len, step: stride }] }); if let Some(items) = T::of(view) { copy_walked(&walked, items, 0, len, T::FILL, output); return Ok(()); } with_dtype!(view.dtype(), S => { @@ -573,15 +558,8 @@ impl<'s> Read<'s> { if output.dtype() != dtype { *output = crate::buffer::Buffer::with_capacity(dtype, range.len()); } with_dtype!(dtype, T => { let items = T::vec_mut(output).expect("output has the selected dtype"); - if let Some((view, stride)) = run { - if view.dtype() != dtype { return cast_run(view, stride, range.len(), checked, items); } - let source = T::of(view).expect("a typed read selected its dtype"); - items.clear(); - if stride == 0 { items.resize(range.len(), source[0]); } else { items.extend_from_slice(source); } - return Ok(()); - } items.clear(); - self.read_onto(range, checked, items) + match run { Some((view, stride)) => append_run(view, stride, range.len(), checked, items), None => self.read_onto(range, checked, items) } }) } } @@ -625,10 +603,13 @@ fn borrowed_as<'a, T: crate::buffer::Cast>(view: crate::buffer::View<'a>, checke Ok(None) } -/// The `len` items of a borrowed run of another dtype, as `T` in `output`: one typed loop over the run ([`Cast::extend_cast`](crate::buffer::Cast::extend_cast)), in place of a routed read for each item. A stride of 0 repeats the run's one item. -fn cast_run(view: crate::buffer::View<'_>, stride: usize, len: usize, checked: bool, output: &mut Vec) -> Result<(), Error> { - output.clear(); - if stride == 0 { output.resize(len, kept(T::cast(view, 0), checked)?); return Ok(()); } +/// Appends a borrowed run as `T`, copying the same dtype or converting it in one typed loop. A stride of 0 repeats its one item. +fn append_run(view: crate::buffer::View<'_>, stride: usize, len: usize, checked: bool, output: &mut Vec) -> Result<(), Error> { + if let Some(items) = T::of(view) { + if stride == 0 { output.resize(output.len() + len, items[0]); } else { output.extend_from_slice(&items[..len]); } + return Ok(()); + } + if stride == 0 { output.resize(output.len() + len, kept(T::cast(view, 0), checked)?); return Ok(()); } T::extend_cast(view, len, checked, output) } @@ -669,8 +650,8 @@ impl Period<'_> { /// The period and exact backing length written by `tiled_cycle` for this run size. Long plain cycles return `None`; long walking cycles are stored once. pub fn tiling(&self, run: usize) -> Option<(usize, usize)> { match *self { - Self::Cycles(p) | Self::Walks { len: p, .. } if p > 0 && p < LONG_CYCLE => - Some((p, if p < run { p * (run.div_ceil(p) + 1) } else { p })), Self::Walks { len, .. } => Some((len, len)), + Self::Cycles(p) | Self::Walks { len: p, .. } if p > 0 && p < run => Some((p, p * (run.div_ceil(p) + 1))), + Self::Walks { len, .. } => Some((len, len)), Self::Cycles(_) | Self::Repeats(_) => None } } @@ -876,19 +857,25 @@ impl Mapping { } /// The contiguous block of storage that the mapping reads over `shape`, each item once, where the block's axes come in another order or run backwards: as a transpose, a reversal or windows that don't overlap read their source. `None` where the mapping skips, repeats or fills an item. pub fn block(&self, shape: &[usize]) -> Option { + let (range, walked, strides) = self.contiguous_axes(shape)?; + let mut axes = vec![None; shape.len()]; + for (b, &a) in walked.iter().enumerate() { axes[a] = Some((b, strides[a] < 0)); } + Some(Block { range, dims: walked.iter().map(|&a| shape[a]).collect(), axes }) + } + + /// The storage range, view axes in storage order and signed strides of a contiguous affine mapping. + fn contiguous_axes(&self, shape: &[usize]) -> Option<(Range, Vec, Vec)> { if shape.contains(&0) { return None; } let (first, strides) = self.affine(shape)?; let mut walked: Vec = (0..shape.len()).filter(|&a| shape[a] > 1).collect(); - walked.sort_by_key(|&a| std::cmp::Reverse(strides[a].unsigned_abs())); - let mut size = 1; + walked.sort_unstable_by_key(|&a| std::cmp::Reverse(strides[a].unsigned_abs())); + let mut size = 1usize; for &a in walked.iter().rev() { if strides[a].unsigned_abs() != size { return None; } - size *= shape[a]; + size = size.checked_mul(shape[a])?; } let start = walked.iter().filter(|&&a| strides[a] < 0).try_fold(first, |s, &a| s.checked_sub(strides[a].unsigned_abs() * (shape[a] - 1)))?; - let mut axes = vec![None; shape.len()]; - for (b, &a) in walked.iter().enumerate() { axes[a] = Some((b, strides[a] < 0)); } - Some(Block { range: start..start + size, dims: walked.iter().map(|&a| shape[a]).collect(), axes }) + Some((start..start.checked_add(size)?, walked, strides)) } /// The offset for position `i`. Panics where the mapping gives a fill. #[inline] @@ -1216,8 +1203,6 @@ impl Frames { pub fn items(&self, k: usize, i: usize, per: usize) -> std::ops::Range { let c = self.cell(k, i); c * per..(c + 1) * per } } -/// The shortest cycle that reads in place a period at a time. A shorter cycle is written out once (`Period::tiling`, `tiled_cycle`), so that runs a chunk long read it in place at any phase. On comparisons of two cycles of 65,536 characters, writing the cycle out cost 61-65% less than the earlier reads at 16 items and below, 43% less at 43, 5% less at 128 and 2% less at 256, then 11%, 24% and 32% more than reading a period at a time at 512, 1,000 and 2,048. -const LONG_CYCLE: usize = 256; /// Bytes in a cache line on the Apple processors that rustymath is tuned on. An x86-64 line holds half as many. pub(crate) const LINE: usize = 128; /// The most items in a block of whole rows from [`Mapping::line_block`]. diff --git a/kernels/src/operation.rs b/kernels/src/operation.rs index 11e8193..42b8804 100644 --- a/kernels/src/operation.rs +++ b/kernels/src/operation.rs @@ -470,9 +470,11 @@ impl Kernel { if inputs.len() != self.arity() { target.fill(range.len()); return Err(Error::new(Kind::Length, "kernel operand count changed")); } if range.is_empty() { return Ok(()); } let mut output = Output { target, result_dtype: self.result, checked: self.variant.fails_on_overflow(), scratch }; - crate::error::checked(|| match self.policy { - Some(Policy::Kinds(ByKinds::Truth(value))) => map::constant(value, range.len(), &mut output), - Some(Policy::Kinds(side)) => map::copied(inputs.selected(usize::from(side == ByKinds::Right)), range, self.variant.checked, &mut output), + match self.policy { + Some(Policy::Kinds(side)) => crate::error::checked(|| match side { + ByKinds::Truth(value) => map::constant(value, range.len(), &mut output), + side => map::copied(inputs.selected(usize::from(side == ByKinds::Right)), range, self.variant.checked, &mut output), + }), _ => { let mut entered = false; let result = self.projected(inputs.0, Some(range.clone()), Seed::Identity, settings, |reads, _| { @@ -482,7 +484,7 @@ impl Kernel { if !entered { output.target.fill(range.len()); } result } - }) + } } pub fn fold_pieces( &self, dims: [usize; 3], seed: Seed<'_>, direction: Direction, settings: Settings, pieces: &Pieces<'_>, output: &mut Buffer) -> Result<(), Error> { From 9f6ee95464c9203b03eaa21a2ab0934e356d2f95 Mon Sep 17 00:00:00 2001 From: Jeremy Howard Date: Thu, 12 Nov 2026 04:38:12 +1000 Subject: [PATCH 2/2] Reduce mapped array execution overhead --- buffer/src/selection.rs | 9 ++-- core/src/agreement.rs | 73 +++++++++++++--------------- core/src/arc.rs | 3 +- core/src/buffer.rs | 10 ++-- core/src/buffer/output.rs | 17 ++----- core/src/complex/bulk.rs | 4 +- core/src/convert.rs | 66 ++++++++++++++++--------- core/src/distribution.rs | 16 +++--- core/src/error.rs | 4 +- core/src/events.rs | 9 ++-- core/src/facts.rs | 16 +++--- core/src/gather.rs | 11 +++-- core/src/integer.rs | 10 ++-- core/src/ln.rs | 18 ++++--- core/src/parallel.rs | 5 +- core/src/slice.rs | 4 -- core/src/special.rs | 12 +++-- core/src/structure.rs | 10 ++-- core/src/tolerant.rs | 28 +++++++---- kernels/src/operation.rs | 31 +++++------- kernels/src/operation/map.rs | 31 ++++++------ kernels/src/operation/map/complex.rs | 11 +++-- kernels/src/operation/map/io.rs | 10 ++-- py/src/explain.rs | 3 +- src/create.rs | 4 +- src/lib.rs | 35 +++++-------- tests/numeric.rs | 15 +++--- tests/slices.rs | 11 ++--- tests/support/mod.rs | 3 +- 29 files changed, 240 insertions(+), 239 deletions(-) diff --git a/buffer/src/selection.rs b/buffer/src/selection.rs index fda3246..e920508 100644 --- a/buffer/src/selection.rs +++ b/buffer/src/selection.rs @@ -35,9 +35,7 @@ pub fn key_names(obj: &Bound<'_, PyAny>) -> Option> { /// One Python selector, independent of the selectors on the other axes. Keys read by `structure::key_selector`, so a missing key reads the fill, as in BPL. pub fn selector(layout: &Layout, axis: usize, obj: &Bound<'_, PyAny>, array: impl FnOnce() -> PyResult<(Selector, bool)>) -> PyResult<(Selector, bool)> { let size = *layout.shape().get(axis).ok_or_else(|| PyIndexError::new_err("too many axis selectors"))?; - if let Ok(slice) = obj.cast::() { - return slice_spec(slice)?.selector(layout, axis).map(|s| (s, false)).map_err(error); - } + if let Ok(slice) = obj.cast::() { return slice_spec(slice)?.selector(layout, axis).map(|s| (s, false)).map_err(error); } if let Some(names) = key_names(obj) { let one = obj.is_instance_of::(); let within = (!one).then(|| Layout::from([names.len()])); @@ -75,8 +73,9 @@ pub fn parts<'py>(index: &Bound<'py, PyAny>) -> PyResult() && ["start", "stop", "step"].iter().all(|n| p.getattr(*n).is_ok_and(|v| v.is_none())) { IndexPart::All } - else if let Ok(slice) = p.cast::() { IndexPart::Slice(Arc::new(slice_spec(slice)?)) } + } else if p.is_instance_of::() && ["start", "stop", "step"].iter().all(|n| p.getattr(*n).is_ok_and(|v| v.is_none())) { + IndexPart::All + } else if let Ok(slice) = p.cast::() { IndexPart::Slice(Arc::new(slice_spec(slice)?)) } else { IndexPart::Axis(p) }) }).collect() } diff --git a/core/src/agreement.rs b/core/src/agreement.rs index d13719c..e751384 100644 --- a/core/src/agreement.rs +++ b/core/src/agreement.rs @@ -61,9 +61,7 @@ pub struct ReadPart { pub end: usize, pub plan: ReadPlan } /// Data-free positional routes shared by readers, compilation and read-back. Leaves stay inline; maps and joins share their recursive payloads. #[derive(Clone, Debug, PartialEq, Eq)] -pub enum ReadRoute { - Input(usize), Index { base: i64, step: i64 }, Map(Arc), Join(Arc<[ReadPart]>), Default, Invalid -} +pub enum ReadRoute { Input(usize), Index { base: i64, step: i64 }, Map(Arc), Join(Arc<[ReadPart]>), Default, Invalid } #[derive(Debug, PartialEq, Eq)] pub struct ReadMap { pub input: ReadPlan, pub map: Mapping, pub missing: ReadPlan } @@ -78,8 +76,8 @@ impl ReadPlan { fn fallback(missing: Missing, items: usize) -> Self { match missing { Missing::Input(plan) => plan.remapped(Mapping::Linear(1), items, Missing::Invalid), - Missing::Scalar(source) => Self::input(source, 1).remapped(Mapping::Single, items, Missing::Invalid), - Missing::Default => Self::fill(items), Missing::Invalid => Self { route: ReadRoute::Invalid, items } + Missing::Scalar(source) => Self::input(source, 1).remapped(Mapping::Single, items, Missing::Invalid), Missing::Default => Self::fill(items), + Missing::Invalid => Self { route: ReadRoute::Invalid, items } } } pub fn copy(source: usize, source_items: usize, map: Mapping, items: usize, missing: Missing) -> Self { @@ -279,7 +277,7 @@ impl ReadPlan { if n > after { return None; } part.run(&(at..at.checked_add(n)?))? } - ReadRoute::Index { .. } | ReadRoute::Default | ReadRoute::Invalid => return None + ReadRoute::Index { .. } | ReadRoute::Default | ReadRoute::Invalid => return None, }; Some((source, first, if n == 1 { 1 } else { stride })) } @@ -405,9 +403,8 @@ impl<'s> Read<'s> { } /// A span at its actual storage dtype and the item count copied. Copied spans require sources of one dtype. - pub fn argument<'a>( - self, range: Range, checked: bool, scratch: &'a mut crate::buffer::Buffer - ) -> Result<(crate::buffer::View<'a>, usize), Error> where 's: 'a { + pub fn argument<'a>(self, range: Range, checked: bool, scratch: &'a mut crate::buffer::Buffer) -> Result<(crate::buffer::View<'a>, usize), Error> + where 's: 'a { self.range(&range)?; if let Some((view, _)) = self.run_view(&range) { return Ok((view, 0)); } let mut dtype = None; @@ -433,8 +430,7 @@ impl<'s> Read<'s> { } /// The same read and the number of items it materialized in scratch; borrowed runs and scalar conversions count zero. pub fn part_counted<'a, T: crate::buffer::Cast>( - &'a self, range: Range, checked: bool, scratch: &'a mut Vec - ) -> Result<(Arg<'a, T>, usize), Error> { + &'a self, range: Range, checked: bool, scratch: &'a mut Vec) -> Result<(Arg<'a, T>, usize), Error> { self.range(&range)?; if range.is_empty() { return Ok((Arg::Many(&[]), 0)); } if let Some((view, stride)) = self.run_view(&range) { @@ -552,9 +548,8 @@ impl<'s> Read<'s> { self.write_into(range, dtype, checked, output, run) } fn write_into( - &self, range: Range, dtype: crate::buffer::DType, checked: bool, - output: &mut crate::buffer::Buffer, run: Option<(crate::buffer::View<'s>, usize)> - ) -> Result<(), Error> { + &self, range: Range, dtype: crate::buffer::DType, checked: bool, output: &mut crate::buffer::Buffer, + run: Option<(crate::buffer::View<'s>, usize)>) -> Result<(), Error> { if output.dtype() != dtype { *output = crate::buffer::Buffer::with_capacity(dtype, range.len()); } with_dtype!(dtype, T => { let items = T::vec_mut(output).expect("output has the selected dtype"); @@ -606,7 +601,8 @@ fn borrowed_as<'a, T: crate::buffer::Cast>(view: crate::buffer::View<'a>, checke /// Appends a borrowed run as `T`, copying the same dtype or converting it in one typed loop. A stride of 0 repeats its one item. fn append_run(view: crate::buffer::View<'_>, stride: usize, len: usize, checked: bool, output: &mut Vec) -> Result<(), Error> { if let Some(items) = T::of(view) { - if stride == 0 { output.resize(output.len() + len, items[0]); } else { output.extend_from_slice(&items[..len]); } + if stride == 0 { output.resize(output.len() + len, items[0]); } + else { output.extend_from_slice(&items[..len]); } return Ok(()); } if stride == 0 { output.resize(output.len() + len, kept(T::cast(view, 0), checked)?); return Ok(()); } @@ -620,10 +616,14 @@ enum Filled { Fill, Value(i64), Source(usize), Never } impl Filled { fn of(missing: &ReadPlan) -> Option { match missing.route() { - ReadRoute::Default => Some(Self::Fill), ReadRoute::Invalid => Some(Self::Never), &ReadRoute::Index { base, step: 0 } => Some(Self::Value(base)), + ReadRoute::Default => Some(Self::Fill), + ReadRoute::Invalid => Some(Self::Never), + &ReadRoute::Index { base, step: 0 } => Some(Self::Value(base)), &ReadRoute::Input(source) if missing.len() == 1 => Some(Self::Source(source)), - ReadRoute::Map(read) if matches!(&read.map, Mapping::Single) => match *read.input.route() { ReadRoute::Input(source) => Some(Self::Source(source)), _ => None }, - _ => None + ReadRoute::Map(read) if matches!(&read.map, Mapping::Single) => match *read.input.route() { + ReadRoute::Input(source) => Some(Self::Source(source)), _ => None + }, + _ => None, } } /// The item as an `A`, where it is exactly one: `None` where an index value doesn't convert, or the source holds another dtype or no first item. @@ -651,8 +651,7 @@ impl Period<'_> { pub fn tiling(&self, run: usize) -> Option<(usize, usize)> { match *self { Self::Cycles(p) | Self::Walks { len: p, .. } if p > 0 && p < run => Some((p, p * (run.div_ceil(p) + 1))), - Self::Walks { len, .. } => Some((len, len)), - Self::Cycles(_) | Self::Repeats(_) => None + Self::Walks { len, .. } => Some((len, len)), Self::Cycles(_) | Self::Repeats(_) => None } } } @@ -728,8 +727,7 @@ impl<'a> Walked<'a> { /// The length of the cycle whose items the mapping's positions read again in order (`Period::Cycles`), where it has one. pub fn cycle(&self) -> Option { *self.cycle.get_or_init(|| match self.mapping.period() { Some(Period::Cycles(p)) => Some(p), _ => None }) } - /// A walking copy, or one cycle written out to cover runs of up to `cover` items at any phase. - /// `None` only when the mapping has neither a tiling period nor explicit walking entries. + /// A walking copy, or one cycle written out to cover runs of up to `cover` items at any phase. `None` only when the mapping has neither a tiling period nor explicit walking entries. pub fn gathered(&self, input: crate::buffer::View<'_>, cover: usize) -> Option { with_dtype!(input.dtype(), A => { let data = A::of(input).expect("the view has its dtype"); @@ -771,7 +769,10 @@ impl Mapping { }); } if let Self::Linear(period) | Self::Tiled(period) = *self { - if period < count(shape) && period != 1 && !shape.iter().rev().scan(1, |p, &n| { *p *= n; Some(*p) }).any(|p| p == period) { return None; } + if period < count(shape) && period != 1 && !shape.iter().rev().scan(1, |p, &n| { + *p *= n; + Some(*p) + }).any(|p| p == period) { return None; } } let (base, mut stride) = (self.get(0)?, count(shape)); for (k, &len) in shape.iter().enumerate() { @@ -885,12 +886,9 @@ impl Mapping { pub fn run(&self, start: usize, n: usize) -> Option<(usize, isize)> { if n == 0 { return None; } match *self { - Self::Single => Some((0, 0)), Self::Linear(0) | Self::Tiled(0) => None, - Self::Linear(1) => Some((start, 1)), - Self::Linear(r) if n <= r - start % r => Some((start / r, 0)), - Self::Tiled(p) if n <= p - start % p => Some((start % p, 1)), - Self::Steps { base, ref steps } => Steps::run(base, steps, start, n), - _ => None + Self::Single => Some((0, 0)), Self::Linear(0) | Self::Tiled(0) => None, Self::Linear(1) => Some((start, 1)), + Self::Linear(r) if n <= r - start % r => Some((start / r, 0)), Self::Tiled(p) if n <= p - start % p => Some((start % p, 1)), + Self::Steps { base, ref steps } => Steps::run(base, steps, start, n), _ => None } } /// The offset for position `i`, or `None` where it reads a fill. @@ -1212,29 +1210,28 @@ const LINE_BLOCK: usize = 1 << 16; pub fn expanded_items<'a, A: Element>(mapping: &Walked, data: &'a [A], start: usize, n: usize, buf: &'a mut Vec) -> &'a [A] { match in_place(mapping, data, start, n) { Some(Arg::Many(items)) => items, - Some(Arg::One(x)) => { buf.clear(); buf.resize(n, x); buf }, - None => copied(mapping, data, start, n, buf) + Some(Arg::One(x)) => { + buf.clear(); + buf.resize(n, x); + buf + } + None => copied(mapping, data, start, n, buf), } } /// Items `start..start + n` of an argument that `mapping` lays out, where they need no copy: a single item, a contiguous run, items within one repeat, or items of a cycle: within one period of a mapping that cycles (`Period::Cycles`), and as far as `data` holds a `Tiled` cycle, once or written out past its end (`tiled_cycle`). A `Steps` mapping reads a forward run in place through `run`. fn in_place<'a, A: Copy>(mapping: &Walked, data: &'a [A], start: usize, n: usize) -> Option> { match mapping.run(start, n) { - Some((o, 0)) => return data.get(o).copied().map(Arg::One), - Some((o, 1)) => return data.get(o..o.checked_add(n)?).map(Arg::Many), - _ => {} + Some((o, 0)) => return data.get(o).copied().map(Arg::One), Some((o, 1)) => return data.get(o..o.checked_add(n)?).map(Arg::Many), _ => {} } // A `Tiled` source may already hold several periods (`tiled_cycle`); a cyclic Steps view only borrows within one. let period = match **mapping { - Mapping::Tiled(p) if p > 0 => p, - Mapping::Steps { .. } => mapping.cycle().filter(|&p| p > 0 && n <= p - start % p)?, - _ => return None + Mapping::Tiled(p) if p > 0 => p, Mapping::Steps { .. } => mapping.cycle().filter(|&p| p > 0 && n <= p - start % p)?, _ => return None }; let from = start % period; data.get(from..from.checked_add(n)?).map(Arg::Many) } - /// The cycle that `mapping`'s positions read from `data`, written into `buf` as `Period::tiling` says for runs of up to `run` positions, with the `Tiled` mapping that then reads it in place: a cycle read in order copies its first period, and one that walks entries (`Period::Walks`) gathers its period through them. A cycle shorter than its cover is written out past its end, so that a run reads it in place at any phase. `None` where `tiling` gives none, leaving `buf` alone. pub fn tiled_cycle(data: &[A], mapping: &Mapping, run: usize, buf: &mut Vec) -> Option { let period = mapping.period()?; diff --git a/core/src/arc.rs b/core/src/arc.rs index 7e80cae..8e31085 100644 --- a/core/src/arc.rs +++ b/core/src/arc.rs @@ -70,7 +70,8 @@ pub(crate) fn atan(x: R) -> R { (hi + ((u * z).mla::(p, u) + lo)).copysign(x) } -pub(crate) fn atan_slice(_: &Atan, _: &crate::recipe::NativeOps<{ crate::FUSED }>, x: [crate::Arg<'_, R>; 1], n: usize, out: &mut [std::mem::MaybeUninit]) -> bool { +pub(crate) fn atan_slice( + _: &Atan, _: &crate::recipe::NativeOps<{ crate::FUSED }>, x: [crate::Arg<'_, R>; 1], n: usize, out: &mut [std::mem::MaybeUninit]) -> bool { each!(x, n, out, |[x]| crate::slice::Total(atan::<_, { crate::FUSED }>(x))) } diff --git a/core/src/buffer.rs b/core/src/buffer.rs index 4cfeb62..d2993a8 100644 --- a/core/src/buffer.rs +++ b/core/src/buffer.rs @@ -787,7 +787,6 @@ pub(crate) fn kept_as(view: View<'_>, i: usize) -> Result { T::cast(view, i).map_err(|_| lost()) } - impl Buffer { /// Flattened positions whose item differs from its type's fill (zero, false or a space). pub fn nonzero(view: View<'_>) -> Vec { @@ -838,11 +837,13 @@ impl Buffer { } /// The items of `view` at `dtype`. Matching dtypes copy. Otherwise each item converts as [`Source`] reads it, and conversions to a narrower dtype follow these rules. A float converts to an integer by truncating toward zero. A value that doesn't fit its target gives an Overflow error, as do NaN, infinities and anything but exactly 0 or 1 for Booleans. A complex number converts to a real dtype only when its imaginary part is zero, and otherwise gives a Domain error. Characters convert only to characters. pub fn cast(view: View<'_>, dtype: DType) -> Result { - crate::convert::Conversion { input: view.dtype(), output: dtype, kind: crate::convert::ConversionKind::Cast }.buffer(view, true, crate::tolerant::Tolerance::EXACT) + crate::convert::Conversion { input: view.dtype(), output: dtype, kind: crate::convert::ConversionKind::Cast }.buffer( + view, true, crate::tolerant::Tolerance::EXACT) } /// The items of `view` at `dtype`, where every item keeps its value. Items convert as [`Buffer::cast`] converts them, and a float that becomes a Boolean or an integer must be whole. A float rounds to a float of another width. A Domain error where an item doesn't keep its value. pub fn kept(view: View<'_>, dtype: DType) -> Result { - crate::convert::Conversion { input: view.dtype(), output: dtype, kind: crate::convert::ConversionKind::Keep }.buffer(view, true, crate::tolerant::Tolerance::EXACT) + crate::convert::Conversion { input: view.dtype(), output: dtype, kind: crate::convert::ConversionKind::Keep }.buffer( + view, true, crate::tolerant::Tolerance::EXACT) } } @@ -853,7 +854,8 @@ impl Buffer { crate::convert::Conversion { input: view.dtype(), output: dtype, kind: crate::convert::ConversionKind::Floor }.buffer(view, true, tolerance) } pub fn cast_results(view: View<'_>, dtype: DType, checked: bool) -> Result { - crate::convert::Conversion { input: view.dtype(), output: dtype, kind: crate::convert::ConversionKind::Truncate }.buffer(view, checked, crate::tolerant::Tolerance::EXACT) + crate::convert::Conversion { input: view.dtype(), output: dtype, kind: crate::convert::ConversionKind::Truncate }.buffer( + view, checked, crate::tolerant::Tolerance::EXACT) } } diff --git a/core/src/buffer/output.rs b/core/src/buffer/output.rs index de201d3..e2fb76c 100644 --- a/core/src/buffer/output.rs +++ b/core/src/buffer/output.rs @@ -59,16 +59,12 @@ impl Scratch { } } -pub struct Output<'a, 'b> { - pub target: MapOutput<'a, 'b>, pub result_dtype: DType, pub checked: bool, pub scratch: Option<&'a mut Scratch> -} +pub struct Output<'a, 'b> { pub target: MapOutput<'a, 'b>, pub result_dtype: DType, pub checked: bool, pub scratch: Option<&'a mut Scratch> } impl Output<'_, '_> { /// The block body is erased before output conversion. pub fn write( - &mut self, n: usize, - apply: &(dyn Fn(std::ops::Range, &mut Slots<'_, O>, Option<&mut Scratch>) -> bool + Sync) - ) -> Result<(), Error> { + &mut self, n: usize, apply: &(dyn Fn(std::ops::Range, &mut Slots<'_, O>, Option<&mut Scratch>) -> bool + Sync)) -> Result<(), Error> { let checked = self.checked; if let MapOutput::Buffer(buffer) = &mut self.target { if let Some(values) = O::vec_mut(buffer) { extended(values, n, self.scratch.as_deref_mut(), apply); } else { @@ -88,8 +84,7 @@ impl Output<'_, '_> { fn extended( values: &mut Vec, n: usize, scratch: Option<&mut Scratch>, - apply: &(dyn Fn(std::ops::Range, &mut Slots<'_, O>, Option<&mut Scratch>) -> bool + Sync) -) { + apply: &(dyn Fn(std::ops::Range, &mut Slots<'_, O>, Option<&mut Scratch>) -> bool + Sync)) { match scratch { Some(scratch) => { Slots::append(values, n, |slots| apply(0..n, slots, Some(scratch))); } None => { crate::parallel::extended(values, n, crate::CHUNK, &|part, slots| apply(part, slots, None)); } @@ -98,8 +93,7 @@ fn extended( fn converted( target: &mut MapOutput<'_, '_>, range: std::ops::Range, checked: bool, mut work: Option<&mut Scratch>, - apply: &(dyn Fn(std::ops::Range, &mut Slots<'_, O>, Option<&mut Scratch>) -> bool + Sync) -) -> Result<(), Error> { + apply: &(dyn Fn(std::ops::Range, &mut Slots<'_, O>, Option<&mut Scratch>) -> bool + Sync)) -> Result<(), Error> { let mut values = work.as_deref_mut().map_or_else(Vec::new, |work| std::mem::take(work.result::())); let conversion = crate::convert::Conversion { input: O::DTYPE, output: target.dtype(), kind: crate::convert::ConversionKind::Truncate }; let result = (|| { @@ -107,8 +101,7 @@ fn converted( let n = crate::CHUNK.min(range.end - at); values.clear(); Slots::append(&mut values, n, |slots| apply(at..at + n, slots, work.as_deref_mut())); - if let Err(error) = conversion.append(O::view(&values), checked, crate::tolerant::Tolerance::EXACT, - work.as_deref_mut(), target.reborrow()) { + if let Err(error) = conversion.append(O::view(&values), checked, crate::tolerant::Tolerance::EXACT, work.as_deref_mut(), target.reborrow()) { target.fill(range.end - at - n); return Err(error); } diff --git a/core/src/complex/bulk.rs b/core/src/complex/bulk.rs index 3e785f0..897f1fd 100644 --- a/core/src/complex/bulk.rs +++ b/core/src/complex/bulk.rs @@ -39,8 +39,7 @@ fn sin_cos(y: &[R], sin: &mut Vec, cos: &mut Vec) { } /// Real-input cis through the shared real sin/cos slices, with exact quarter turns. -pub(crate) fn real_cis( - k: &crate::Cis, ops: &NativeOps<{ FUSED }>, [arg]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit>]) { +pub(crate) fn real_cis(k: &crate::Cis, ops: &NativeOps<{ FUSED }>, [arg]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit>]) { let ys = match arg { Arg::One(y) => { let value = k.complex(ops, [c(y, R::zero())]).expect("cis has a complex scalar body"); @@ -62,7 +61,6 @@ pub(crate) fn real_cis( } } - fn split, R: Native + Cast, const N: usize>( k: &K, ops: &NativeOps<{ FUSED }>, args: [Arg<'_, Complex>; N], n: usize, out: &mut [MaybeUninit>], fast: impl FnOnce([(&[R], &[R]); N], &mut Vec, &mut Vec), rare: impl Fn([Complex; N], Complex) -> bool) { diff --git a/core/src/convert.rs b/core/src/convert.rs index a63ccd9..0dbcd93 100644 --- a/core/src/convert.rs +++ b/core/src/convert.rs @@ -91,37 +91,49 @@ impl Conversion { .is_some_and(|(_, valid)| valid != Some(true))) } /// Converts contiguous storage through the mapped conversion executor. - pub(crate) fn buffer(self, view: crate::buffer::View<'_>, checked: bool, tolerance: crate::tolerant::Tolerance) -> Result { + pub(crate) fn buffer( + self, view: crate::buffer::View<'_>, checked: bool, tolerance: crate::tolerant::Tolerance) -> Result { let mut result = crate::buffer::Buffer::with_capacity(self.output, view.len()); self.append(view, checked, tolerance, None, crate::buffer::MapOutput::Buffer(&mut result))?; Ok(result) } pub(crate) fn append<'a, 'b>( - self, view: crate::buffer::View<'_>, checked: bool, tolerance: crate::tolerant::Tolerance, - scratch: Option<&'a mut crate::buffer::Scratch>, output: crate::buffer::MapOutput<'a, 'b> - ) -> Result<(), crate::error::Error> { + self, view: crate::buffer::View<'_>, checked: bool, tolerance: crate::tolerant::Tolerance, scratch: Option<&'a mut crate::buffer::Scratch>, + output: crate::buffer::MapOutput<'a, 'b>) -> Result<(), crate::error::Error> { let sources = [crate::agreement::Binding::full(view)]; let plan = crate::agreement::ReadPlan::input(0, view.len()); self.map_in(crate::agreement::Read { sources: &sources, plan: &plan }, 0..view.len(), checked, tolerance, scratch, output) } pub fn map_in<'a, 'b>( self, input: crate::agreement::Read<'_>, range: std::ops::Range, checked: bool, tolerance: crate::tolerant::Tolerance, - scratch: Option<&'a mut crate::buffer::Scratch>, target: crate::buffer::MapOutput<'a, 'b> - ) -> Result<(), crate::error::Error> { + scratch: Option<&'a mut crate::buffer::Scratch>, target: crate::buffer::MapOutput<'a, 'b>) -> Result<(), crate::error::Error> { let mut output = crate::buffer::Output { target, result_dtype: self.output, checked, scratch }; macro_rules! write { - ($s:ty, $t:ty, $convert:expr) => { written::<$s, $t>(input, range, checked, &mut output, $convert) }; + ($s:ty, $t:ty, $convert:expr) => { + written::<$s, $t>(input, range, checked, &mut output, $convert) + }; (@cast $t:ty) => {{ - if let crate::buffer::MapOutput::Buffer(buffer) = &mut output.target && let Some(values) = <$t>::vec_mut(buffer) { + if let crate::buffer::MapOutput::Buffer(buffer) = &mut output.target + && let Some(values) = <$t>::vec_mut(buffer) + { let (at, n) = (values.len(), range.len()); if let Err(error) = input.append(range, checked, values) { - crate::error::record(error); values.truncate(at); values.resize(at + n, <$t>::FILL); + crate::error::record(error); + values.truncate(at); + values.resize(at + n, <$t>::FILL); } Ok(()) - } else { write!($t, $t, Ok) } + } else { + write!($t, $t, Ok) + } + }}; + (@keep $t:ty) => { + kept_mapped::<$t>(self.input, input, range, checked, &mut output) + }; + (@unsupported) => {{ + output.target.fill(range.len()); + Err(Error::new(Kind::Domain, "unsupported resolved conversion")) }}; - (@keep $t:ty) => { kept_mapped::<$t>(self.input, input, range, checked, &mut output) }; - (@unsupported) => {{ output.target.fill(range.len()); Err(Error::new(Kind::Domain, "unsupported resolved conversion")) }}; } crate::error::checked(|| conversion!(self, checked, tolerance, write)) } @@ -135,15 +147,14 @@ fn converted_item(value: Result( input: crate::agreement::Read<'_>, range: std::ops::Range, checked: bool, output: &mut crate::buffer::Output<'_, '_>, - convert: impl Fn(S) -> Result + Sync -) -> Result<(), crate::error::Error> { + convert: impl Fn(S) -> Result + Sync) -> Result<(), crate::error::Error> { output.write::(range.len(), &|part, slots, work| { let part = part.start + range.start..part.end + range.start; let in_place = input.in_place::(&part); let len = if in_place { part.len().max(1) } else { crate::CHUNK }; let mut local = Vec::new(); let (mut work, mut copied) = (work, 0); - let scratch = match work.as_deref_mut() { Some(work) if !in_place => { let [scratch] = work.inputs::(); scratch }, _ => &mut local }; + let scratch = match work.as_deref_mut() { Some(work) if !in_place => { let [scratch] = work.inputs::(); scratch } _ => &mut local }; let ok = part.clone().step_by(len).fold(true, |ok, start| { let range = start..(start + len).min(part.end); let n = range.len(); @@ -317,15 +328,22 @@ fn kept_mapped( let mut local = crate::buffer::Buffer::with_capacity(dtype, 0); let scratch = match work { Some(work) => work.buffer(dtype), None => &mut local }; match input.part_view(part.start + range.start..part.end + range.start, dtype, checked, scratch) { - Ok(values) => unsafe { slots.fill(part.len(), |out| { - let mut good = true; - for (i, slot) in out.iter_mut().enumerate() { - slot.write(match crate::buffer::kept_as::(values, i) { - Ok(value) => value, Err(error) => { crate::error::record(error); good = false; T::FILL } - }); - } - good - }) }, + Ok(values) => unsafe { + slots.fill(part.len(), |out| { + let mut good = true; + for (i, slot) in out.iter_mut().enumerate() { + slot.write(match crate::buffer::kept_as::(values, i) { + Ok(value) => value, + Err(error) => { + crate::error::record(error); + good = false; + T::FILL + } + }); + } + good + }) + }, Err(error) => { crate::error::record(error); unsafe { slots.fill(part.len(), |out| { for slot in out { slot.write(T::FILL); } false }) } diff --git a/core/src/distribution.rs b/core/src/distribution.rs index 4c5c5b9..b5d3ef6 100644 --- a/core/src/distribution.rs +++ b/core/src/distribution.rs @@ -66,18 +66,13 @@ trait Formulas { #[inline(always)] fn quantile_map(&self) -> impl Fn(F) -> F + Copy { move |p| narrowed(p, |p| self.quantile(p)) } #[inline(always)] - fn pdf_many(&self, x: Arg<'_, F>, n: usize, out: &mut [MaybeUninit]) -> bool { - let pdf = self.pdf_map(); - each!([x], n, out, |[x]| pdf(x)) - } + fn pdf_many(&self, x: Arg<'_, F>, n: usize, out: &mut [MaybeUninit]) -> bool { let pdf = self.pdf_map(); each!([x], n, out, |[x]| pdf(x)) } #[inline(always)] - fn cdf_many(&self, x: Arg<'_, F>, n: usize, out: &mut [MaybeUninit]) -> bool { - let cdf = self.cdf_map(); - each!([x], n, out, |[x]| cdf(x)) - } + fn cdf_many(&self, x: Arg<'_, F>, n: usize, out: &mut [MaybeUninit]) -> bool { let cdf = self.cdf_map(); each!([x], n, out, |[x]| cdf(x)) } #[inline(always)] fn quantile_many(&self, p: Arg<'_, F>, n: usize, out: &mut [MaybeUninit]) -> bool { - let q = self.quantile_map(); each!([p], n, out, |[p]| q(p)) + let q = self.quantile_map(); + each!([p], n, out, |[p]| q(p)) } } @@ -648,7 +643,8 @@ impl Formulas for LogNormal { fn cdf_many(&self, x: Arg<'_, F>, n: usize, out: &mut [MaybeUninit]) -> bool { let (line, half) = (self.normal().erfc_line(), F::narrow(0.5)); composed( - x, n, out, #[inline(always)] move |x| line(ln(x)), through(Erfc, crate::special::erfc_slice), #[inline(always)] move |x, e| if x <= F::zero() { F::zero() } else { half * e }) + x, n, out, #[inline(always)] move |x| line(ln(x)), through(Erfc, crate::special::erfc_slice), #[inline(always)] + move |x, e| if x <= F::zero() { F::zero() } else { half * e }) } #[inline(always)] fn quantile_many(&self, p: Arg<'_, F>, n: usize, out: &mut [MaybeUninit]) -> bool { diff --git a/core/src/error.rs b/core/src/error.rs index a5f49d1..31e7677 100644 --- a/core/src/error.rs +++ b/core/src/error.rs @@ -87,9 +87,7 @@ pub fn record(error: Error) { }); } -/// Checked eager entry for Rust and Python callers. -/// Returns direct failures and, when no slot is active, the first recorded failure. With an -/// active slot, recorded failures remain for its enclosing operation to collect. +/// Checked eager entry for Rust and Python callers. Returns direct failures and, when no slot is active, the first recorded failure. With an active slot, recorded failures remain for its enclosing operation to collect. pub fn checked(run: impl FnOnce() -> Result) -> Result { let (mut run, mut result) = (Some(run), None); let recorded = recorded(&mut || result = run.take().map(|run| run())); diff --git a/core/src/events.rs b/core/src/events.rs index 480adc5..f4fe25e 100644 --- a/core/src/events.rs +++ b/core/src/events.rs @@ -71,8 +71,8 @@ impl Event { /// The producer's designated text label, if this event has one. pub fn slow_name(&self) -> Option<&str> { match self { - Self::Route(Record { kind: EventKind::Stopped, reason, .. }) => Some(*reason), - Self::Row { fields, label: Some(i), .. } => Some(fields[*i].text()), _ => None, + Self::Route(Record { kind: EventKind::Stopped, reason, .. }) => Some(*reason), Self::Row { fields, label: Some(i), .. } => Some(fields[*i].text()), + _ => None } } pub fn fields(&self) -> Vec> { @@ -94,7 +94,9 @@ impl Report { pub fn record(&mut self, event: Event) { if let Event::Route(r) = &event { if let Some(existing) = self.records.iter_mut().find_map(|event| match event { - Event::Route(existing) if existing.kind == r.kind && existing.route == r.route && existing.reason == r.reason && existing.site == r.site => Some(existing), + Event::Route(existing) if existing.kind == r.kind && existing.route == r.route && existing.reason == r.reason && existing.site == r.site => { + Some(existing) + } _ => None, }) { existing.calls += r.calls; @@ -108,7 +110,6 @@ impl Report { pub fn routes(&self) -> impl Iterator { self.records.iter().filter_map(Event::route) } } - pub trait Sink: Send + Sync { fn sites(&self) -> bool { false } fn record(&self, event: Event); diff --git a/core/src/facts.rs b/core/src/facts.rs index 1595bd2..aa8eb62 100644 --- a/core/src/facts.rs +++ b/core/src/facts.rs @@ -186,11 +186,12 @@ impl Op { } /// A formula's native slice bodies. Real and integer variants run the body at their dtype (f32 for f16). Complex variants keep the real-input shortcut and final validity check at the map boundary; an absent body uses the scalar formula. pub struct NativeSlices { - pub single: Option>, pub double: Option>, pub byte: Option>, - pub short: Option>, pub int: Option>, pub long: Option>, - pub complex_single: Option>, pub complex_double: Option>, - pub real_complex_single: Option>, pub real_complex_double: Option>, - pub complex_test_single: Option>, pub complex_test_double: Option> + pub single: Option>, pub double: Option>, pub byte: Option>, pub short: Option>, + pub int: Option>, pub long: Option>, pub complex_single: Option>, + pub complex_double: Option>, pub real_complex_single: Option>, + pub real_complex_double: Option>, + pub complex_test_single: Option>, + pub complex_test_double: Option> } /// A slice form writes all `n` results at `U` from arguments at `T`. The caller reports errors from its verdict `R`: `bool` for real/integer forms, `()` for complex forms. pub type Slice = fn(&K, &NativeOps<{ FUSED }>, [crate::Arg<'_, T>; N], usize, &mut [std::mem::MaybeUninit]) -> R; @@ -224,9 +225,8 @@ impl NativeSlices { } impl NativeSlices { fn residue() -> Self { - Self { single: Some(crate::tolerant::residue_slice), double: Some(crate::tolerant::residue_slice), - byte: Some(crate::integer::residue_byte), short: Some(crate::integer::residue_short), - int: Some(crate::integer::residue_int), long: Some(crate::integer::residue_long), ..Self::NONE } + Self { single: Some(crate::tolerant::residue_slice), double: Some(crate::tolerant::residue_slice), byte: Some(crate::integer::residue_byte), + short: Some(crate::integer::residue_short), int: Some(crate::integer::residue_int), long: Some(crate::integer::residue_long), ..Self::NONE } } } diff --git a/core/src/gather.rs b/core/src/gather.rs index 108e25c..1210f07 100644 --- a/core/src/gather.rs +++ b/core/src/gather.rs @@ -1008,10 +1008,13 @@ impl Steps { pub fn furthest(base: usize, steps: &[impl std::borrow::Borrow]) -> Option { steps.iter().try_fold(base, |at, s| { let furthest = match *s.borrow() { - ref s if s.len() == 0 => return None, Self::Stride { start, len, step } => start.wrapping_add_signed(step.max(0) * (len - 1) as isize), - Self::Padded { size, stride, .. } => size.saturating_sub(1) * stride, Self::Rotated { len, stride, .. } => (len - 1) * stride, - Self::Table(ref t) => t.span().map_or_else( - || (0..t.len()).filter_map(|j| t.get(j)).max().unwrap_or(0), |(scale, size)| size.saturating_sub(1) * scale) + ref s if s.len() == 0 => return None, + Self::Stride { start, len, step } => start.wrapping_add_signed(step.max(0) * (len - 1) as isize), + Self::Padded { size, stride, .. } => size.saturating_sub(1) * stride, + Self::Rotated { len, stride, .. } => (len - 1) * stride, + Self::Table(ref t) => { + t.span().map_or_else(|| (0..t.len()).filter_map(|j| t.get(j)).max().unwrap_or(0), |(scale, size)| size.saturating_sub(1) * scale) + } }; Some(at + furthest) }) diff --git a/core/src/integer.rs b/core/src/integer.rs index 30cbacf..cf4dedb 100644 --- a/core/src/integer.rs +++ b/core/src/integer.rs @@ -211,16 +211,14 @@ fn decoded(bases: &[i64], digits: &[i64], step: impl Fn(i64, i64, i64) -> Option digits.chunks(bases.len()).map(|d| (1..d.len()).try_fold(d[0], |v, k| step(v, bases[k], d[k]))).collect() } -pub(crate) fn residue_long(_: &Residue, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [x, y]: [Arg<'_, i64>; 2], n: usize, out: &mut [std::mem::MaybeUninit]) -> bool { +pub(crate) fn residue_long( + _: &Residue, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [x, y]: [Arg<'_, i64>; 2], n: usize, out: &mut [std::mem::MaybeUninit]) -> bool { if let Arg::One(d) = x && let Some(m) = mask(d) { each!([y], n, out, |[y]| y & m) } else if let Arg::One(d) = x && let Some(r) = FloatDivisor::new(d) && fit(y, n) { each!([y], n, out, |[y]| r.residue(y)) - } else if let Arg::One(d) = x && let Some(d) = Divisor::new(d) { - each!([y], n, out, |[y]| d.residue(y)) - } else { - each!([x, y], n, out, |[x, y]| residue(x, y)) - } + } else if let Arg::One(d) = x && let Some(d) = Divisor::new(d) { each!([y], n, out, |[y]| d.residue(y)) } + else { each!([x, y], n, out, |[x, y]| residue(x, y)) } } macro_rules! residues { diff --git a/core/src/ln.rs b/core/src/ln.rs index 479bd12..e72339a 100644 --- a/core/src/ln.rs +++ b/core/src/ln.rs @@ -176,15 +176,21 @@ pub struct Log10; pub struct Log1p; pub(crate) fn ln_slice(_: &Ln, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [x]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { - each_usual(x, n, out, (usual, #[inline(always)] |x| log_of::<_, { crate::FUSED }>(x.split(), natural())), none(), ln::<_, { crate::FUSED }>) + each_usual(x, n, out, (usual, #[inline(always)] |x| log_of::<_, { crate::FUSED }>(x.split(), natural()),), none(), ln::<_, { crate::FUSED }>) } -pub(crate) fn log2_slice(_: &Log2, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [x]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { - each_usual(x, n, out, (usual, #[inline(always)] |x| log_of::<_, { crate::FUSED }>(x.split(), base2(&crate::recipe::Floats))), none(), log2::<_, { crate::FUSED }>) +pub(crate) fn log2_slice( + _: &Log2, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [x]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { + each_usual( + x, n, out, (usual, #[inline(always)] |x| log_of::<_, { crate::FUSED }>(x.split(), base2(&crate::recipe::Floats)),), none(), log2::<_, { crate::FUSED }>) } -pub(crate) fn log10_slice(_: &Log10, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [x]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { - each_usual(x, n, out, (usual, #[inline(always)] |x| log_of::<_, { crate::FUSED }>(x.split(), base10(&crate::recipe::Floats))), none(), log10::<_, { crate::FUSED }>) +pub(crate) fn log10_slice( + _: &Log10, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [x]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { + each_usual( + x, n, out, (usual, #[inline(always)] |x| log_of::<_, { crate::FUSED }>(x.split(), base10(&crate::recipe::Floats)),), none(), + log10::<_, { crate::FUSED }>) } -pub(crate) fn log1p_slice(_: &Log1p, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [x]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { +pub(crate) fn log1p_slice( + _: &Log1p, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [x]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { each_usual(x, n, out, (usual1p, log1p_usual::<_, { crate::FUSED }>), none(), log1p::<_, { crate::FUSED }>) } /// ln|x|, the real part of ln x, as a function object. diff --git a/core/src/parallel.rs b/core/src/parallel.rs index 461b3a2..9277a61 100644 --- a/core/src/parallel.rs +++ b/core/src/parallel.rs @@ -134,9 +134,7 @@ fn each_cell(count: usize, write: impl Fn(usize, &mut S) -> bool + S for first in (timed..count).step_by(round) { check()?; let cells = round.min(count - first); - if threads == 1 { - ok &= (first..first + cells).fold(true, |ok, i| write(i, &mut scratch) & ok); - } else { + if threads == 1 { ok &= (first..first + cells).fold(true, |ok, i| write(i, &mut scratch) & ok); } else { ok &= pieces(cells, cells.div_ceil(threads * PIECES).max(least).max(1), usize::MAX, &|range| { let mut scratch = S::default(); range.map(|c| first + c).fold(true, |ok, i| write(i, &mut scratch) & ok) @@ -258,7 +256,6 @@ impl Slots<'_, B> { } } - #[cfg(not(target_arch = "wasm32"))] mod pool { use std::sync::{Arc, Mutex}; diff --git a/core/src/slice.rs b/core/src/slice.rs index 385f5b5..5610409 100644 --- a/core/src/slice.rs +++ b/core/src/slice.rs @@ -104,7 +104,6 @@ impl Outcome for Total { fn split(self) -> (O, bool) { (self.0, true) } } - /// The items of `x` that result `i` takes. #[inline(always)] fn items(x: &[Arg<'_, E>; N], i: usize) -> [E; N] { std::array::from_fn(|k| x[k].nth(i)) } @@ -163,7 +162,6 @@ pub(crate) fn each_usual>( }) } - /// [`each`] with the item body marked `#[inline(always)]`, so native bulk loops inline their scalar body. macro_rules! each { ($x:expr, $n:expr, $out:expr, |$args:pat_param| $body:expr $(,)?) => { @@ -173,7 +171,6 @@ macro_rules! each { }; } - /// Runs `$body` for each item `$i` of a result of `$n` items, with `$a` and `$b` the items of `$x` and `$y` that pair with it. Each pairing of `One` and `Many` has a loop of its own. The loops index slices of the result's length with no iterator adapters, and vectorize at any build settings. macro_rules! each_pair { ($x:expr, $y:expr, $n:expr, |$i:ident, $a:ident, $b:ident| $body:expr) => {{ @@ -311,7 +308,6 @@ pub(crate) fn two_pass( ok } - /// `f`, a slice form writing `n` outputs of type `O`, run into a buffer, with `each` turning each output into the `P` that `out` takes. It gives `f`'s result. /// /// # Safety diff --git a/core/src/special.rs b/core/src/special.rs index 8acfa08..696ba37 100644 --- a/core/src/special.rs +++ b/core/src/special.rs @@ -630,10 +630,10 @@ fn erfcinv64(q: f64) -> f64 { /// [`erfc`](crate::erfc) as a formula object, with a native bulk routine that groups inputs by region. #[derive(Clone, Copy, Debug)] pub struct Erfc; -pub(crate) fn erfc_slice(_: &Erfc, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [x]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { +pub(crate) fn erfc_slice( + _: &Erfc, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [x]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { match x { - Arg::Many(x) => erfc_blocks::<_, { crate::FUSED }>(&x[..n], out), - x => each!([x], n, out, |[x]| crate::slice::Total(erfc::<_, { crate::FUSED }>(x))), + Arg::Many(x) => erfc_blocks::<_, { crate::FUSED }>(&x[..n], out), x => each!([x], n, out, |[x]| crate::slice::Total(erfc::<_, { crate::FUSED }>(x))) } } @@ -666,14 +666,16 @@ fn erfc_blocks(x: &[R], out: &mut [MaybeUninit] /// [`erfinv`](crate::erfinv) as a formula object, with a native bulk routine that groups f64 inputs by region. #[derive(Clone, Copy, Debug)] pub struct Erfinv; -pub(crate) fn erfinv_slice(_: &Erfinv, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [p]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { +pub(crate) fn erfinv_slice( + _: &Erfinv, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [p]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { inverses(p, n, out, #[inline(always)] |p| erfinv::<_, { crate::FUSED }>(p), #[inline(always)] |p| (p.abs(), 1.0 - p.abs(), p)) } /// [`erfcinv`](crate::erfcinv) as a formula object, with a native bulk routine that groups f64 inputs by region. #[derive(Clone, Copy, Debug)] pub struct Erfcinv; -pub(crate) fn erfcinv_slice(_: &Erfcinv, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [q]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { +pub(crate) fn erfcinv_slice( + _: &Erfcinv, _: &crate::recipe::NativeOps<{ crate::FUSED }>, [q]: [Arg<'_, R>; 1], n: usize, out: &mut [MaybeUninit]) -> bool { inverses(q, n, out, #[inline(always)] |q| erfcinv::<_, { crate::FUSED }>(q), #[inline(always)] |q| ((q - 1.0).abs(), q.min(2.0 - q), 1.0 - q)) } diff --git a/core/src/structure.rs b/core/src/structure.rs index 7957164..fc3b432 100644 --- a/core/src/structure.rs +++ b/core/src/structure.rs @@ -47,10 +47,9 @@ impl SliceSpec { let bound = |value: Option<&SliceBound>, default: i128, stop: bool| -> Result { let at = match value { None => return Ok(default), - Some(SliceBound::Position(i)) => { let i = *i as i128; if i < 0 { i + n } else { i } }, + Some(SliceBound::Position(i)) => { let i = *i as i128; if i < 0 { i + n } else { i } } Some(SliceBound::Key(name)) => { - let i = layout.keys(axis).and_then(|keys| keys.position(name)) - .ok_or_else(|| Error::new(Kind::Index, format!("missing key: {name}")))?; + let i = layout.keys(axis).and_then(|keys| keys.position(name)).ok_or_else(|| Error::new(Kind::Index, format!("missing key: {name}")))?; i as i128 + if stop { direction } else { 0 } } }; @@ -110,7 +109,10 @@ pub fn index_claims<'a>(rank: usize, parts: &[IndexPart<(Option<&'a str>, usize) written.push(IndexPart::All); claims.push((None, 1)); } - IndexPart::Slice(slice) => { written.push(IndexPart::Slice(slice.clone())); claims.push((None, 1)); } + IndexPart::Slice(slice) => { + written.push(IndexPart::Slice(slice.clone())); + claims.push((None, 1)); + } IndexPart::NewAxis => written.push(IndexPart::NewAxis), IndexPart::Rest => { for _ in used..rank { diff --git a/core/src/tolerant.rs b/core/src/tolerant.rs index 429d26c..8d82d01 100644 --- a/core/src/tolerant.rs +++ b/core/src/tolerant.rs @@ -293,7 +293,6 @@ macro_rules! with_tolerance { }}; } - /// Where an item must lie against the range of floats equal to one number (`Tolerant::range`) for a comparison with that number to hold. `NotAbove` and `NotBelow` include the range itself. #[derive(Clone, Copy)] pub enum Band { Inside, Outside, Below, NotAbove, Above, NotBelow } @@ -360,7 +359,6 @@ pub fn against( Some(true) } - /// Wasm has no instruction that narrows a 64-bit lane to a byte. LLVM writes a float comparison's results two items at a time, in about ten instructions for each item. Where it can prove that lanes hold masks, it also replaces wasm's narrowing instructions with slower code. Here each block of 16 items compares with vector instructions, then shuffles the first byte of each mask into place: 7 shuffles for `f64`'s 8 vectors, and 3 for `f32`'s 4. #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] mod narrowed { @@ -425,18 +423,24 @@ mod narrowed { #[derive(Clone, Copy, Debug, PartialEq)] pub struct Equal(pub Tolerance); impl Banding for Equal { const BANDS: (Band, Band) = (Band::Inside, Band::Inside); } -pub(crate) fn complex_equal_slice(k: &Equal, o: &recipe::NativeOps<{ crate::FUSED }>, args: [Arg<'_, Complex>; 2], n: usize, out: &mut [std::mem::MaybeUninit]) { +pub(crate) fn complex_equal_slice( + k: &Equal, o: &recipe::NativeOps<{ crate::FUSED }>, args: [Arg<'_, Complex>; 2], n: usize, out: &mut [std::mem::MaybeUninit]) { let tolerance = k.0.overridden(o.tolerance); - with_tolerance!(tolerance, |t| { complex_equals(t, args, n, out, false); }); + with_tolerance!(tolerance, |t| { + complex_equals(t, args, n, out, false); + }); } /// The negation of [`Equal`] at the tolerance it holds, as a function object, for real and complex items. #[derive(Clone, Copy, Debug, PartialEq)] pub struct NotEqual(pub Tolerance); impl Banding for NotEqual { const BANDS: (Band, Band) = (Band::Outside, Band::Outside); } -pub(crate) fn complex_not_equal_slice(k: &NotEqual, o: &recipe::NativeOps<{ crate::FUSED }>, args: [Arg<'_, Complex>; 2], n: usize, out: &mut [std::mem::MaybeUninit]) { +pub(crate) fn complex_not_equal_slice( + k: &NotEqual, o: &recipe::NativeOps<{ crate::FUSED }>, args: [Arg<'_, Complex>; 2], n: usize, out: &mut [std::mem::MaybeUninit]) { let tolerance = k.0.overridden(o.tolerance); - with_tolerance!(tolerance, |t| { complex_equals(t, args, n, out, true); }); + with_tolerance!(tolerance, |t| { + complex_equals(t, args, n, out, true); + }); } /// `Equal`'s slice form on complex items, or with `negate` `NotEqual`'s. Each `CHUNK` of items splits into slices of real and imaginary parts, as in `slice::split`, so `squared_equal` runs over plain slices with no branch and vectorizes. `scaled_equal` then takes the pairs it leaves undecided, one at a time. No result fails. @@ -529,7 +533,6 @@ impl Stepped for GreaterEqual { fn steps + ?Sized>(self, o: &O, x: O::V, y: O::V) -> O::B { LessEqual(self.0).steps(o, y, x) } } - /// [`Tolerant::floor`], or [`TolerantComplex::complex_floor`] for complex items, at the tolerance it holds, as a function object. #[derive(Clone, Copy, Debug, PartialEq)] pub struct Floor(pub Tolerance); @@ -541,11 +544,16 @@ pub struct Ceil(pub Tolerance); /// [`Tolerant::residue`], or [`TolerantComplex::complex_residue`] for complex items, at the tolerance it holds, as a function object: `y` modulo `x`, for arguments `x` and `y` in that order. At no tolerance, where `y/x` passes the float's precision ([`far`]), the rounded quotient has lost its low digits, and [`exact_residue`] gives the residue. The slice form recomputes those items in a second pass. Its `i64` form is [`integer::residue`](crate::integer::residue), which needs no tolerance. #[derive(Clone, Copy, Debug, PartialEq)] pub struct Residue(pub Tolerance); -pub(crate) fn residue_slice(k: &Residue, o: &recipe::NativeOps<{ crate::FUSED }>, [x, y]: [Arg<'_, R>; 2], n: usize, out: &mut [std::mem::MaybeUninit]) -> bool { +pub(crate) fn residue_slice( + k: &Residue, o: &recipe::NativeOps<{ crate::FUSED }>, [x, y]: [Arg<'_, R>; 2], n: usize, out: &mut [std::mem::MaybeUninit]) -> bool { let tolerance = k.0.overridden(o.tolerance); with_tolerance!(tolerance, |t| { - if t.value() != 0.0 { return each!([x, y], n, out, |[x, y]| Total(t.residue(x, y))); } - if !each!([x, y], n, out, |[x, y]| (t.residue(x, y), near(x, y))) { replace_far(x, y, n, out); } + if t.value() != 0.0 { + return each!([x, y], n, out, |[x, y]| Total(t.residue(x, y))); + } + if !each!([x, y], n, out, |[x, y]| (t.residue(x, y), near(x, y))) { + replace_far(x, y, n, out); + } true }) } diff --git a/kernels/src/operation.rs b/kernels/src/operation.rs index 42b8804..b722cbf 100644 --- a/kernels/src/operation.rs +++ b/kernels/src/operation.rs @@ -68,19 +68,14 @@ impl std::fmt::Debug for Kernel { pub enum Map { Kernel(Kernel, Settings), Convert(crate::convert::Conversion, Settings, bool) } impl Map { pub fn arity(&self) -> usize { match self { Self::Kernel(kernel, _) => kernel.arity(), Self::Convert(..) => 1 } } - /// Appends every result, including initialized fallback items on a numerical error. - /// Errors are execution failures, not a pre-write refusal to handle the call. - /// Both variants follow [`crate::error::checked`]: an enclosing checked scope owns recorded - /// numerical failures. Under an active error slot, `Ok` alone does not establish success. - /// Discard private outputs when that scope fails; neither `Err` nor a recorded failure is a route refusal. + /// Appends every result, including initialized fallback items on a numerical error. Errors are execution failures, not a pre-write refusal to handle the call. Both variants follow [`crate::error::checked`]: an enclosing checked scope owns recorded numerical failures. Under an active error slot, `Ok` alone does not establish success. Discard private outputs when that scope fails; neither `Err` nor a recorded failure is a route refusal. pub fn run<'a, 'b>( - &self, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, - scratch: &'a mut crate::buffer::Scratch, mut output: MapOutput<'a, 'b> + &self, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, scratch: &'a mut crate::buffer::Scratch, mut output: MapOutput<'a, 'b> ) -> Result<(), Error> { if inputs.len() != self.arity() { output.fill(range.len()); return Err(Error::new(Kind::Length, "map arity does not match its operands")); } match self { Self::Kernel(kernel, settings) => kernel.map_in(inputs, range, *settings, scratch, output), - Self::Convert(conversion, settings, checked) => conversion.map_in(inputs[0], range, *checked, settings.tolerance, Some(scratch), output) + Self::Convert(conversion, settings, checked) => conversion.map_in(inputs[0], range, *checked, settings.tolerance, Some(scratch), output), } } } @@ -382,9 +377,11 @@ impl Kernel { let bounds = Range::of_dtype(self.result); // f64 endpoints cannot prove i64 wrapping bounds. Narrower integers are exact here until their result exceeds their type. if self.variant.dtype.is_integral() && !self.variant.fails_on_overflow() && self.result.is_integral() - && (self.result == DType::Integer(Width::I64) || !range.within(bounds.lo, bounds.hi)) { bounds } + && (self.result == DType::Integer(Width::I64) || !range.within(bounds.lo, bounds.hi)) + { bounds } // Checked arithmetic can rerun wider; only a Boolean result is bounded independently of that attempt. - else if self.result == DType::Boolean { range.meet(bounds) } else { range } + else if self.result == DType::Boolean { range.meet(bounds) } + else { range } } pub fn identity(&self) -> Option { self.definition.identity() } /// The argument that a dyadic call returns unchanged where `constants` gives the arguments that are one number, 0 for the left and 1 for the right, from the formula's identities: `x+0` and `0+x` keep `x`, `x-0` keeps `x`, and `0-x` keeps nothing. @@ -456,16 +453,12 @@ impl Kernel { /// Appends a mapped range using the caller's execution-local conversion buffers. pub fn map_in<'a, 'b>( - &self, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, settings: Settings, - scratch: &'a mut crate::buffer::Scratch, output: MapOutput<'a, 'b> - ) -> Result<(), Error> { - self.mapping(Inputs(inputs), range, settings, output, Some(scratch)) - } + &self, inputs: &[crate::agreement::Read<'_>], range: std::ops::Range, settings: Settings, scratch: &'a mut crate::buffer::Scratch, + output: MapOutput<'a, 'b>) -> Result<(), Error> { self.mapping(Inputs(inputs), range, settings, output, Some(scratch)) } fn mapping<'a, 'b>( &self, inputs: Inputs<'_, '_>, range: std::ops::Range, settings: Settings, mut target: MapOutput<'a, 'b>, - scratch: Option<&'a mut crate::buffer::Scratch> - ) -> Result<(), Error> { + scratch: Option<&'a mut crate::buffer::Scratch>) -> Result<(), Error> { use crate::characters::{ByKinds, Policy}; if inputs.len() != self.arity() { target.fill(range.len()); return Err(Error::new(Kind::Length, "kernel operand count changed")); } if range.is_empty() { return Ok(()); } @@ -680,7 +673,9 @@ fn with_projected( if !read.sources.iter().any(|s| matches!(s.view.dtype(), DType::Float(_))) { return Ok(None); } let range = range.clone().unwrap_or(0..read.plan.len()); let mut floats = Buffer::with_capacity(DType::Float(FloatWidth::F64), 0); - let View::Floats(Floats::F64(values)) = read.part_view(range.clone(), floats.dtype(), true, &mut floats)? else { unreachable!("a read into 64-bit floats gives them") }; + let View::Floats(Floats::F64(values)) = read.part_view(range.clone(), floats.dtype(), true, &mut floats)? else { + unreachable!("a read into 64-bit floats gives them") + }; Ok(Some((range.start, Buffer::Integers(IntBuf::I64(whole_numbers(t, values)?))))) }).collect::, Error>>()?; let bindings = converted.iter().zip(inputs) diff --git a/kernels/src/operation/map.rs b/kernels/src/operation/map.rs index 35fa3ec..ff5584f 100644 --- a/kernels/src/operation/map.rs +++ b/kernels/src/operation/map.rs @@ -50,12 +50,10 @@ pub(super) fn map_formula, const N: usize>( DType::Complex(crate::buffer::ComplexWidth::F64) if K::REAL_COMPLEX && inputs.real() => { map_real_complex::(object, variant, inputs, range, &ops, slices.and_then(|s| s.real_complex_double), output) } - DType::Complex(crate::buffer::ComplexWidth::F32) => { - map_complex::(object, variant, inputs, range, &ops, slices.as_ref().and_then(|s| s.complex_single), slices.and_then(|s| s.complex_test_single), judge, output) - } - DType::Complex(crate::buffer::ComplexWidth::F64) => { - map_complex::(object, variant, inputs, range, &ops, slices.as_ref().and_then(|s| s.complex_double), slices.and_then(|s| s.complex_test_double), judge, output) - } + DType::Complex(crate::buffer::ComplexWidth::F32) => map_complex::( + object, variant, inputs, range, &ops, slices.as_ref().and_then(|s| s.complex_single), slices.and_then(|s| s.complex_test_single), judge, output), + DType::Complex(crate::buffer::ComplexWidth::F64) => map_complex::( + object, variant, inputs, range, &ops, slices.as_ref().and_then(|s| s.complex_double), slices.and_then(|s| s.complex_test_double), judge, output), DType::Character => Err(Error::new(Kind::Domain, "characters are not numerical kernel operands")), } } @@ -108,14 +106,18 @@ fn map_blocks( let recorded = |result: Result<(), Kind>| result.map_err(|kind| crate::error::record(Error::new(kind, "numerical kernel argument is outside its domain"))).is_ok(); match inputs.borrowed::(range.clone(), checked) { - Ok(Some(args)) => return output.write(range.len(), &|part, slots, _| unsafe { - slots.fill(part.len(), |out| recorded(apply(args.map(|arg| arg.part(part.start, part.len())), part.len(), out))) - }), - Ok(None) => {}, Err(error) => { output.target.fill(range.len()); return Err(error); } + Ok(Some(args)) => { + return output.write(range.len(), &|part, slots, _| unsafe { + slots.fill(part.len(), |out| recorded(apply(args.map(|arg| arg.part(part.start, part.len())), part.len(), out))) + }); + } + Ok(None) => {} + Err(error) => { output.target.fill(range.len()); return Err(error); } } if let Some((len, rows)) = by_rows::(inputs.0) { - return output - .write(range.len(), &|part, slots, _| unsafe { slots.fill(part.len(), |out| recorded(fill_rows(&rows, len, part.start + range.start, out, apply))) }); + return output.write(range.len(), &|part, slots, _| unsafe { + slots.fill(part.len(), |out| recorded(fill_rows(&rows, len, part.start + range.start, out, apply))) + }); } output.write(range.len(), &|part, slots, work| { let part = part.start + range.start..part.end + range.start; @@ -150,10 +152,9 @@ fn by_rows<'a, F: crate::buffer::Cast, const N: usize>(inputs: &[crate::agreemen let len = inputs.iter().find_map(|read| match read.plan.route() { ReadRoute::Map(mapped) => match &mapped.map { Mapping::Linear(k) | Mapping::Tiled(k) => Some(*k), - Mapping::Steps { steps, .. } if Steps::count(steps) == read.plan.len() => steps.last().map(Steps::len), - _ => Some(read.plan.len()), + Mapping::Steps { steps, .. } if Steps::count(steps) == read.plan.len() => steps.last().map(Steps::len), _ => Some(read.plan.len()) }, - _ => None + _ => None, })?; let items = inputs.first()?.plan.len(); if len < ROWS || items <= len || items % len != 0 { return None; } diff --git a/kernels/src/operation/map/complex.rs b/kernels/src/operation/map/complex.rs index 8786472..c2febd5 100644 --- a/kernels/src/operation/map/complex.rs +++ b/kernels/src/operation/map/complex.rs @@ -3,14 +3,15 @@ use super::*; pub(super) fn map_complex( object: &K, variant: crate::domain::Variant, inputs: Inputs<'_, '_>, range: std::ops::Range, ops: &NativeOps<{ FUSED }>, slice: Option, N, ()>>, test: Option, N, (), bool>>, - judge: crate::domain::Variant, output: &mut Output<'_, '_>) -> Result<(), Error> -where K: crate::facts::Formula, F: crate::Native + crate::buffer::Cast + OutputItem, num_complex::Complex: crate::buffer::Cast + OutputItem { + judge: crate::domain::Variant, output: &mut Output<'_, '_>) -> Result<(), Error> where K: crate::facts::Formula, + F: crate::Native + crate::buffer::Cast + OutputItem, num_complex::Complex: crate::buffer::Cast + OutputItem { let value = |x| object.complex(ops, x).unwrap_or(num_complex::Complex::new(F::zero(), F::zero())); let invalid = |x, r| (!complex_valid::(ops, x, r)).then_some(Kind::Domain); if const { K::BOOLEAN } { if let Some(test) = test { return map_blocks::, bool, N>(inputs, range, variant.checked, output, &|args, n, out| { - test(object, ops, args, n, out); Ok(()) + test(object, ops, args, n, out); + Ok(()) }); } map_values::, bool, N>( @@ -30,8 +31,8 @@ where K: crate::facts::Formula, F: crate::Native + crate::buffer::Cast + Outp /// Complex results from real arguments through a native slice, or `Formula::real_complex` with `Formula::complex` for the items it declines. pub(super) fn map_real_complex( object: &K, variant: crate::domain::Variant, inputs: Inputs<'_, '_>, range: std::ops::Range, ops: &NativeOps<{ FUSED }>, - slice: Option>>, output: &mut Output<'_, '_> -) -> Result<(), Error> where K: crate::facts::Formula, F: crate::Native + crate::buffer::Cast, num_complex::Complex: OutputItem { + slice: Option>>, output: &mut Output<'_, '_>) -> Result<(), Error> where + K: crate::facts::Formula, F: crate::Native + crate::buffer::Cast, num_complex::Complex: OutputItem { map_blocks::, N>(inputs, range, variant.checked, output, &|args, n, out| { if let Some(slice) = slice { slice(object, ops, args, n, out); return Ok(()); } let nan = num_complex::Complex::new(F::nan(), F::nan()); diff --git a/kernels/src/operation/map/io.rs b/kernels/src/operation/map/io.rs index 1025880..5eee817 100644 --- a/kernels/src/operation/map/io.rs +++ b/kernels/src/operation/map/io.rs @@ -18,13 +18,13 @@ impl<'a, 'b> Inputs<'a, 'b> { Ok(Some(args)) } pub(crate) fn part<'s, F: Cast, const N: usize>( - self, range: std::ops::Range, checked: bool, scratch: &'s mut [&mut Vec; N] - ) -> Result<([Arg<'s, F>; N], usize), Error> where 'a: 's, 'b: 's { + self, range: std::ops::Range, checked: bool, scratch: &'s mut [&mut Vec; N]) -> Result<([Arg<'s, F>; N], usize), Error> where 'a: 's, 'b: 's { let (mut args, mut copied) = ([Arg::Many(&[]); N], 0); for (k, scratch) in scratch.iter_mut().enumerate() { - if let Some(j) = self.0[..k].iter().position(|read| - std::ptr::eq(read.plan, self.0[k].plan) && std::ptr::eq(read.sources, self.0[k].sources) - ) { args[k] = args[j]; continue; } + if let Some(j) = self.0[..k].iter().position(|read| std::ptr::eq(read.plan, self.0[k].plan) && std::ptr::eq(read.sources, self.0[k].sources)) { + args[k] = args[j]; + continue; + } let (items, n) = self.0[k].part_counted(range.clone(), checked, scratch)?; args[k] = items; copied += n; diff --git a/py/src/explain.rs b/py/src/explain.rs index 5f5ce19..13bb8bd 100644 --- a/py/src/explain.rs +++ b/py/src/explain.rs @@ -32,7 +32,8 @@ impl Explanation { record.set_item("kind", event.name())?; for ((name, _), field) in event.table().1.iter().zip(event.fields()) { match field { - Field::Text(t) => record.set_item(name, t.as_ref())?, Field::Count(n) => record.set_item(name, n)?, Field::Truth(t) => record.set_item(name, t)? + Field::Text(t) => record.set_item(name, t.as_ref())?, Field::Count(n) => record.set_item(name, n)?, + Field::Truth(t) => record.set_item(name, t)? } } Ok(record.unbind()) diff --git a/src/create.rs b/src/create.rs index 93611ec..8d2e954 100644 --- a/src/create.rs +++ b/src/create.rs @@ -85,6 +85,8 @@ pub fn linspace(start: f64, stop: f64, n: usize, endpoint: bool) -> Result Result, Error> { let exponents = linspace(start, stop, n, endpoint)?; let mut out = Vec::with_capacity(n); - rustymath_core::slice::pairs(crate::Arg::One(base), crate::Arg::Many(&exponents), n, &mut out, |x, y| (crate::facts::formula_at(&crate::Pow, [x, y]), true)); + rustymath_core::slice::pairs(crate::Arg::One(base), crate::Arg::Many(&exponents), n, &mut out, |x, y| { + (crate::facts::formula_at(&crate::Pow, [x, y]), true) + }); Ok(out) } diff --git a/src/lib.rs b/src/lib.rs index e2dfc4e..d9a62fb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,30 +2,19 @@ #![doc = include_str!("lib.md")] #[macro_use] extern crate rustymath_core; -// The public surface is math, arrays and their execution facts, not every helper -// made public inside core so the kernels crate can call it. +// The public surface is math, arrays and their execution facts, not every helper made public inside core so the kernels crate can call it. pub use rustymath_core::{ - agreement, buffer, characters, convert, derivative, distribution, domain, error, events, exact, facts, gather, hashing, integer, interrupt, - layout, memory, number_theory, optimize, parallel, polynomial, random, recipe, reduce, search, signal, stats, structure, tolerant, - fused, unfused, half, num_complex, -}; -pub use rustymath_core::{ - FUSED, Float, Native, Arg, - Abs, Acos, Acosh, Add, Angle, Asin, Asinh, Atan, Atan2, Atanh, Betainc, BetaincAt, Betaincinv, Binom, BinomDk, BinomDn, - BitAnd, BitOr, Bitwise, Cbrt, Choose, Cis, Clip, Conj, Cos, Cosh, Digamma, Divide, Erf, Erfc, Erfcinv, Erfinv, Exp, Expm1, - Factorial, Gamma, Gammainc, Gammaincc, Gammaincinv, Hypot, Imag, Isfinite, Isinf, Isnan, LambertW, LeftShift, Lgamma, Ln, Log1p, Log2, - Log10, Logb, MathishDivide, MathishLogb, Multiply, Nand, Negate, Nor, Not, PiTimes, Polygamma, Pow, Real, RealAcos, RealAcosh, RealAsin, - RealAtanh, RealLn, RealPow, RealSqrt, RealSqrt1mx2, RealSqrtm1mx2, RealSqrtx2m1, Reciprocal, RightShift, Rint, Root, Round, Scale, - Sign, Sin, Sinh, Sqrt, Sqrt1mx2, Sqrt1px2, Sqrtm1mx2, Sqrtx2m1, Subtract, Tan, Tanh, Trunc, Exponent, -}; -pub use rustymath_core::{ - acos, acosh, asin, asinh, atan, atan2, atanh, beta, betainc, betaincinv, binom, binom_dk, binom_dn, bitwise, cbrt, cos, cosh, digamma, - divide, erf, erfc, erfcinv, erfinv, exp, expm1, factorial, gamma, gammainc, gammaincc, gammaincinv, hypot, lambertw, lgamma, ln, log1p, - log2, log10, logb, polygamma, pow, root, sin, sinh, sqrt1mx2, sqrt1px2, sqrtx2m1, tan, tanh, -}; -pub use rustymath_core::{ - element_types, function_doc, native_reduction, with_complex_width, with_complexes, with_dtype, with_float_width, with_floats, with_inexact, - with_int_pair, with_ints, with_reduce, with_tolerance, with_width, + Abs, Acos, Acosh, Add, Angle, Arg, Asin, Asinh, Atan, Atan2, Atanh, Betainc, BetaincAt, Betaincinv, Binom, BinomDk, BinomDn, BitAnd, BitOr, Bitwise, Cbrt, + Choose, Cis, Clip, Conj, Cos, Cosh, Digamma, Divide, Erf, Erfc, Erfcinv, Erfinv, Exp, Expm1, Exponent, FUSED, Factorial, Float, Gamma, Gammainc, Gammaincc, + Gammaincinv, Hypot, Imag, Isfinite, Isinf, Isnan, LambertW, LeftShift, Lgamma, Ln, Log1p, Log2, Log10, Logb, MathishDivide, MathishLogb, Multiply, Nand, + Native, Negate, Nor, Not, PiTimes, Polygamma, Pow, Real, RealAcos, RealAcosh, RealAsin, RealAtanh, RealLn, RealPow, RealSqrt, RealSqrt1mx2, RealSqrtm1mx2, + RealSqrtx2m1, Reciprocal, RightShift, Rint, Root, Round, Scale, Sign, Sin, Sinh, Sqrt, Sqrt1mx2, Sqrt1px2, Sqrtm1mx2, Sqrtx2m1, Subtract, Tan, Tanh, Trunc, + acos, acosh, agreement, asin, asinh, atan, atan2, atanh, beta, betainc, betaincinv, binom, binom_dk, binom_dn, bitwise, buffer, cbrt, characters, convert, + cos, cosh, derivative, digamma, distribution, divide, domain, element_types, erf, erfc, erfcinv, erfinv, error, events, exact, exp, expm1, factorial, + facts, function_doc, fused, gamma, gammainc, gammaincc, gammaincinv, gather, half, hashing, hypot, integer, interrupt, lambertw, layout, lgamma, ln, log1p, + log2, log10, logb, memory, native_reduction, num_complex, number_theory, optimize, parallel, polygamma, polynomial, pow, random, recipe, reduce, root, + search, signal, sin, sinh, sqrt1mx2, sqrt1px2, sqrtx2m1, stats, structure, tan, tanh, tolerant, unfused, with_complex_width, with_complexes, with_dtype, + with_float_width, with_floats, with_inexact, with_int_pair, with_ints, with_reduce, with_tolerance, with_width, }; pub use rustymath_kernels::operation; pub mod array; diff --git a/tests/numeric.rs b/tests/numeric.rs index e1469a4..5f3cb1f 100644 --- a/tests/numeric.rs +++ b/tests/numeric.rs @@ -406,7 +406,8 @@ mod integer { let t = rustymath::tolerant::Tolerance::DEFAULT; // Every pair of `u8`s agrees with the `i64` functions where the result fits, and fails where it doesn't. let v = Variant::checked(u8::DTYPE); - let (gcd_kernel, lcm_kernel, pow_kernel, residue_kernel) = (Kernel::new(Gcd(t), v), Kernel::new(Lcm(t), v), Kernel::new(Pow, v), Kernel::new(Residue(t), v)); + let (gcd_kernel, lcm_kernel, pow_kernel, residue_kernel) = + (Kernel::new(Gcd(t), v), Kernel::new(Lcm(t), v), Kernel::new(Pow, v), Kernel::new(Residue(t), v)); for x in 0..=u8::MAX { for y in 0..=u8::MAX { let (a, b) = (i64::from(x), i64::from(y)); @@ -589,9 +590,9 @@ mod operation { fn convert(data: rustymath::buffer::View<'_>, dtype: DType, values: &mut Buffer) -> Result<(), Error> { let sources = [Binding::full(data)]; let plan = ReadPlan::input(0, data.len()); - let map = rustymath::operation::Map::Convert(rustymath::convert::Conversion { - input: data.dtype(), output: dtype, kind: rustymath::convert::ConversionKind::Cast - }, Settings { tolerance: Tolerance::EXACT }, true); + let map = rustymath::operation::Map::Convert( + rustymath::convert::Conversion { input: data.dtype(), output: dtype, kind: rustymath::convert::ConversionKind::Cast }, + Settings { tolerance: Tolerance::EXACT }, true); map.run(&[Read { sources: &sources, plan: &plan }], 0..data.len(), &mut Scratch::default(), MapOutput::Buffer(values)) } let float = f64::DTYPE; @@ -833,9 +834,9 @@ mod operation { if direction == Direction::Right { ordered.reverse(); } let (mut carry, mut expected) = (initial, Vec::new()); for x in ordered { - let value = carry.map_or(x, |c| { - u8::from(if direction == Direction::Left { tolerance.less(c, x) } else { tolerance.less(x, c) }) as f64 - }); + let value = + carry.map_or(x, |c| u8::from(if direction == Direction::Left { tolerance.less(c, x) } + else { tolerance.less(x, c) }) as f64); expected.push(value); carry = Some(value); } diff --git a/tests/slices.rs b/tests/slices.rs index 7156d06..8256695 100644 --- a/tests/slices.rs +++ b/tests/slices.rs @@ -869,17 +869,15 @@ mod tolerant { let f64s = DType::Float(FloatWidth::F64); let quotient = Kernel::resolve(FloorDivide(Tolerance::EXACT), Variant::checked(f64s), [f64s; 2]).unwrap(); assert_eq!( - (quotient.real(&[x, y], rustymath::operation::Settings { tolerance: Tolerance::EXACT }).unwrap(), Exactly.residue(y, x)), - (2.0, x - 2.0 * y) + (quotient.real(&[x, y], rustymath::operation::Settings { tolerance: Tolerance::EXACT }).unwrap(), Exactly.residue(y, x)), (2.0, x - 2.0 * y) ); if rustymath::FUSED { assert_ne!(Exactly.residue(0.1, 1.0), 0.0); } assert_eq!(Exactly.residue(1e300, 1e-300), 1e-300); assert!((T.gcd(0.6, 0.4) - 0.2).abs() < 1e-15 && Exactly.gcd(0.6, 0.4) < 1e-15); let tiny = f64::from_bits(1); - for (x, y, g) in [ - (0.75, 0.5, 0.25), (1.0, 0.75, 0.25), (3.0 * tiny, 9.0 * tiny, 3.0 * tiny), - (f64::MAX, tiny, tiny), (f64::MAX, f64::MAX, f64::MAX), - ] { assert_eq!(Exactly.gcd(x, y), g); } + for (x, y, g) in [(0.75, 0.5, 0.25), (1.0, 0.75, 0.25), (3.0 * tiny, 9.0 * tiny, 3.0 * tiny), (f64::MAX, tiny, tiny), (f64::MAX, f64::MAX, f64::MAX)] { + assert_eq!(Exactly.gcd(x, y), g); + } assert_eq!(Exactly.gcd(0.6, 0.4), f64::EPSILON / 2.0); assert_eq!((T.gcd(12.0, 18.0), T.gcd(-12.0, 18.0), T.gcd(0.0, 2.5), T.gcd(0.0, 0.0), T.gcd(12.0f32, 18.0)), (6.0, 6.0, 2.5, 0.0, 6.0)); assert!(T.gcd(f64::NAN, 1.0).is_nan() && T.gcd(f64::NAN, f64::NAN).is_nan() && T.gcd(f64::INFINITY, 1.0).is_nan()); @@ -1002,7 +1000,6 @@ mod tolerant { assert_eq!(lcm.complex(&[c(2.0, 0.0), c(0.0, 3.0)], Settings { tolerance: T }).unwrap(), c(0.0, 6.0)); } - /// Tolerant search finds the first match that a brute-force search finds, at every width and at small and large tolerances, among floats crowded near a few values, and among zeros, infinities and NaN, whether it scans for a few items or indexes many. #[test] fn slice_search() { diff --git a/tests/support/mod.rs b/tests/support/mod.rs index cdae223..33022a4 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -38,8 +38,7 @@ pub fn map( pub fn map_with( kernel: &rustymath::operation::Kernel, args: [rustymath::buffer::View<'_>; N], n: usize, out: &mut rustymath::buffer::Buffer, - settings: rustymath::operation::Settings -) -> Result<(), rustymath::error::Error> { + settings: rustymath::operation::Settings) -> Result<(), rustymath::error::Error> { use rustymath::agreement::{Binding, Mapping, Missing, Read, ReadPlan}; let sources = args.map(Binding::full); let plans: [_; N] = std::array::from_fn(|i| {