Reduce memory usage of sample-level result accumulators

PRASCore.jl/src/Results/DemandResponseAvailability.jl

@@ -30,6 +30,9 @@ struct DRAvailabilityAccumulator <: ResultAccumulator{DemandResponseAvailability
 
 end
 
+sampledata(acc::DRAvailabilityAccumulator) = acc.available
+usesamplepartitions(::DemandResponseAvailability) = true
+
 function accumulator(
     sys::SystemModel{N}, nsamples::Int, ::DemandResponseAvailability
 ) where {N}

PRASCore.jl/src/Results/DemandResponseEnergySamples.jl

@@ -32,6 +32,9 @@ struct DemandResponseEnergySamplesAccumulator <: ResultAccumulator{DemandRespons
 
 end
 
+sampledata(acc::DemandResponseEnergySamplesAccumulator) = acc.energy
+usesamplepartitions(::DemandResponseEnergySamples) = true
+
 function accumulator(
     sys::SystemModel{N}, nsamples::Int, ::DemandResponseEnergySamples
 ) where {N}

PRASCore.jl/src/Results/FlowSamples.jl

@@ -39,6 +39,9 @@ struct FlowSamplesAccumulator <: ResultAccumulator{FlowSamples}
 
 end
 
+sampledata(acc::FlowSamplesAccumulator) = acc.flow
+usesamplepartitions(::FlowSamples) = true
+
 function accumulator(
     sys::SystemModel{N}, nsamples::Int, ::FlowSamples
 ) where {N}

PRASCore.jl/src/Results/GeneratorAvailability.jl

@@ -42,6 +42,9 @@ struct GenAvailabilityAccumulator <:
 
 end
 
+sampledata(acc::GenAvailabilityAccumulator) = acc.available
+usesamplepartitions(::GeneratorAvailability) = true
+
 function merge!(
     x::GenAvailabilityAccumulator, y::GenAvailabilityAccumulator
 )

PRASCore.jl/src/Results/GeneratorStorageAvailability.jl

@@ -31,6 +31,9 @@ struct GenStorAvailabilityAccumulator <: ResultAccumulator{GeneratorStorageAvail
 
 end
 
+sampledata(acc::GenStorAvailabilityAccumulator) = acc.available
+usesamplepartitions(::GeneratorStorageAvailability) = true
+
 function accumulator(
     sys::SystemModel{N}, nsamples::Int, ::GeneratorStorageAvailability
 ) where {N}

PRASCore.jl/src/Results/GeneratorStorageEnergySamples.jl

@@ -33,6 +33,9 @@ struct GenStorageEnergySamplesAccumulator <: ResultAccumulator{GeneratorStorageE
 
 end
 
+sampledata(acc::GenStorageEnergySamplesAccumulator) = acc.energy
+usesamplepartitions(::GeneratorStorageEnergySamples) = true
+
 function accumulator(
     sys::SystemModel{N}, nsamples::Int, ::GeneratorStorageEnergySamples
 ) where {N}

PRASCore.jl/src/Results/LineAvailability.jl

@@ -47,6 +47,9 @@ struct LineAvailabilityAccumulator <: ResultAccumulator{LineAvailability}
 
 end
 
+sampledata(acc::LineAvailabilityAccumulator) = acc.available
+usesamplepartitions(::LineAvailability) = true
+
 accumulatortype(::LineAvailability) = LineAvailabilityAccumulator
 
 function accumulator(

PRASCore.jl/src/Results/Results.jl

@@ -35,6 +35,8 @@ abstract type ResultSpec end
 
 abstract type ResultAccumulator{R<:ResultSpec} end
 
+usesamplepartitions(::ResultSpec) = false
+
 abstract type Result{
     N, # Number of timesteps simulated
     L, # Length of each simulation timestep
@@ -194,29 +196,68 @@ include("GeneratorStorageEnergySamples.jl")
 include("DemandResponseEnergySamples.jl")
 
 function resultchannel(
-    results::T, threads::Int
+    results::T, nworkers::Int
 ) where T <: Tuple{Vararg{ResultSpec}}
 
     types = accumulatortype.(results)
-    return Channel{Tuple{types...}}(threads)
+    return Channel{Tuple{Tuple{types...},UnitRange{Int}}}(nworkers)
 
 end
 
-merge!(xs::T, ys::T) where T <: Tuple{Vararg{ResultAccumulator}} =
-    foreach(merge!, xs, ys)
+function copy_sample_partition!(
+    x::A,
+    y::A,
+    sampleids::UnitRange{Int},
+) where {A<:ResultAccumulator}
+
+    xarr = sampledata(x)
+    yarr = sampledata(y)
+
+    xarr[:, :, sampleids] .= yarr
+    return
+end
 
 function finalize(
-    results::Channel{<:Tuple{Vararg{ResultAccumulator}}},
+    results::Channel{Tuple{A,UnitRange{Int}}},
     system::SystemModel{N,L,T,P,E},
-    threads::Int
-) where {N,L,T,P,E}
+    nworkers::Int,
+    nsamples::Int,
+    resultspecs::Tuple{Vararg{ResultSpec}},
+) where {A<:Tuple{Vararg{ResultAccumulator}},N,L,T,P,E}
+
+    first_recorders, first_sampleids = take!(results)
 
-    total_result = take!(results)
+    if nworkers == 1 && first_sampleids == 1:nsamples
+        close(results)
+        return finalize.(first_recorders, system)
+    end
 
-    for _ in 2:threads
-        thread_result = take!(results)
-        merge!(total_result, thread_result)
+    total_result = map(resultspecs, first_recorders) do spec, recorder
+        usesamplepartitions(spec) ? accumulator(system, nsamples, spec) : recorder
     end
+
+    for i in eachindex(total_result)
+        if usesamplepartitions(resultspecs[i])
+            copy_sample_partition!(
+                total_result[i], first_recorders[i], first_sampleids
+            )
+        end
+    end
+
+    for _ in 2:nworkers
+        thread_recorders, sampleids = take!(results)
+
+        for i in eachindex(total_result)
+            if usesamplepartitions(resultspecs[i])
+                copy_sample_partition!(
+                    total_result[i], thread_recorders[i], sampleids
+                )
+            else
+                merge!(total_result[i], thread_recorders[i])
+            end
+        end
+    end
+
     close(results)
 
     return finalize.(total_result, system)

PRASCore.jl/src/Results/Shortfall.jl

@@ -108,7 +108,7 @@ function accumulator(
 
 end
 
-function merge!(
+function merge_shortfall_statistics!(
     x::ShortfallAccumulator, y::ShortfallAccumulator
 )
 
@@ -122,13 +122,44 @@ function merge!(
     foreach(merge!, x.unservedload_period, y.unservedload_period)
     foreach(merge!, x.unservedload_regionperiod, y.unservedload_regionperiod)
 
+    return
+
+end
+
+
+function merge!(
+    x::ShortfallAccumulator, y::ShortfallAccumulator
+)
+
+    merge_shortfall_statistics!(x, y)
+
     x.unservedload_sample .+= y.unservedload_sample
     x.unservedload_region_sample .+= y.unservedload_region_sample
 
     return
 
 end
 
+
+function copy_sample_partition!(
+    x::ShortfallAccumulator,
+    y::ShortfallAccumulator,
+    sampleids::UnitRange{Int},
+)
+
+    merge_shortfall_statistics!(x, y)
+
+    x.unservedload_sample[sampleids] .= y.unservedload_sample
+    x.unservedload_region_sample[:, sampleids] .=
+        y.unservedload_region_sample
+
+    return
+
+end
+
+usesamplepartitions(::Shortfall) = true
+usesamplepartitions(::DemandResponseShortfall) = true
+
 accumulatortype(::S) where {
         S<:Union{Shortfall,DemandResponseShortfall}
     } = ShortfallAccumulator{S}

PRASCore.jl/src/Results/ShortfallSamples.jl

@@ -57,6 +57,10 @@ struct ShortfallSamplesAccumulator{S} <: ResultAccumulator{ShortfallSamples}
 
 end
 
+sampledata(acc::ShortfallSamplesAccumulator) = acc.shortfall
+usesamplepartitions(::ShortfallSamples) = true
+usesamplepartitions(::DemandResponseShortfallSamples) = true
+
 function accumulator(
     sys::SystemModel{N}, nsamples::Int, ::S
 ) where {N,S<:Union{ShortfallSamples,DemandResponseShortfallSamples}}

PRASCore.jl/src/Results/StorageAvailability.jl

@@ -30,6 +30,9 @@ struct StorAvailabilityAccumulator <: ResultAccumulator{StorageAvailability}
 
 end
 
+sampledata(acc::StorAvailabilityAccumulator) = acc.available
+usesamplepartitions(::StorageAvailability) = true
+
 function accumulator(
     sys::SystemModel{N}, nsamples::Int, ::StorageAvailability
 ) where {N}

PRASCore.jl/src/Results/StorageEnergySamples.jl

@@ -32,6 +32,9 @@ struct StorageEnergySamplesAccumulator <: ResultAccumulator{StorageEnergySamples
 
 end
 
+sampledata(acc::StorageEnergySamplesAccumulator) = acc.energy
+usesamplepartitions(::StorageEnergySamples) = true
+
 function accumulator(
     sys::SystemModel{N}, nsamples::Int, ::StorageEnergySamples
 ) where {N}

PRASCore.jl/src/Results/SurplusSamples.jl

@@ -32,6 +32,9 @@ struct SurplusSamplesAccumulator <: ResultAccumulator{SurplusSamples}
 
 end
 
+sampledata(acc::SurplusSamplesAccumulator) = acc.surplus
+usesamplepartitions(::SurplusSamples) = true
+
 function accumulator(
     sys::SystemModel{N}, nsamples::Int, ::SurplusSamples
 ) where {N}

PRASCore.jl/src/Results/UtilizationSamples.jl

@@ -49,6 +49,9 @@ struct UtilizationSamplesAccumulator <: ResultAccumulator{UtilizationSamples}
 
 end
 
+sampledata(acc::UtilizationSamplesAccumulator) = acc.utilization
+usesamplepartitions(::UtilizationSamples) = true
+
 function accumulator(
     sys::SystemModel{N}, nsamples::Int, ::UtilizationSamples
 ) where {N}