[rANS] Enforce Consistent Histograms

Author: MichaelLettrich

DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h

@@ -339,7 +339,10 @@ class EncodedBlocks
   using dictionaryType = std::variant<rans::RenormedSparseHistogram<source_T>, rans::RenormedDenseHistogram<source_T>>;
 #endif
 
-  void setHeader(const H& h) { mHeader = h; }
+  void setHeader(const H& h)
+  {
+    mHeader = h;
+  }
   const H& getHeader() const { return mHeader; }
   H& getHeader() { return mHeader; }
   std::shared_ptr<H> cloneHeader() const { return std::shared_ptr<H>(new H(mHeader)); } // for dictionary creation
@@ -371,6 +374,22 @@ class EncodedBlocks
     assert(static_cast<int64_t>(std::numeric_limits<source_T>::min()) <= static_cast<int64_t>(metadata.max));
     assert(static_cast<int64_t>(std::numeric_limits<source_T>::max()) >= static_cast<int64_t>(metadata.min));
 
+    // check consistency of metadata and type
+    [&]() {
+      const int64_t sourceMin = std::numeric_limits<source_T>::min();
+      const int64_t sourceMax = std::numeric_limits<source_T>::max();
+
+      auto view = rans::trim(rans::HistogramView{block.getDict(), block.getDict() + block.getNDict(), metadata.min});
+      const int64_t dictMin = view.getMin();
+      const int64_t dictMax = view.getMax();
+      assert(dictMin >= metadata.min);
+      assert(dictMax <= metadata.max);
+
+      if ((dictMin < sourceMin) || (dictMax > sourceMax)) {
+        throw std::runtime_error(fmt::format("value range of dictionary and target datatype are incompatible: target type [{},{}] vs dictionary [{},{}]", sourceMin, sourceMax, dictMin, dictMax));
+      }
+    }();
+
     if (ansVersion == ANSVersionCompat) {
       rans::DenseHistogram<source_T> histogram{block.getDict(), block.getDict() + block.getNDict(), metadata.min};
       return rans::compat::renorm(std::move(histogram), metadata.probabilityBits);

Utilities/rANS/include/rANS/internal/containers/DenseHistogram.h

@@ -586,7 +586,16 @@ auto DenseHistogram<source_T, std::enable_if_t<sizeof(source_T) <= 2>>::addFrequ
   addedHistogramView = trim(addedHistogramView);
 
   if constexpr (std::is_unsigned_v<source_T>) {
-    LOG_IF(warning, addedHistogramView.getMin() < 0) << fmt::format("trying to add frequencies for a signed symbol to a DenseHistogram of an unsiged type.");
+    LOG_IF(warning, addedHistogramView.getMin() < 0) << fmt::format("trying to add frequencies of a signed symbol type to a DenseHistogram of an unsiged type.");
+  }
+  if constexpr (std::is_signed_v<source_T>) {
+    const std::ptrdiff_t sourceTypeMax = std::numeric_limits<source_T>::max();
+    const bool isMinOutOfRange = addedHistogramView.getMin() > sourceTypeMax;
+    const bool isMaxOutOfRange = addedHistogramView.getMax() > sourceTypeMax;
+
+    if (isMaxOutOfRange || isMinOutOfRange) {
+      LOGP(warning, "trying to add frequencies of an unsigned symbol type to a DenseHistogram of a signed type");
+    }
   }
 
   const auto thisHistogramView = makeHistogramView(this->mContainer);

Utilities/rANS/include/rANS/internal/transform/renorm.h

@@ -143,7 +143,7 @@ decltype(auto) renorm(histogram_T histogram, Metrics<typename histogram_T::sourc
     return nSymbols;
   }();
 
-  if (nSorted < correctableIndices.size()) {
+  if ((nSorted < correctableIndices.size()) && (renormingPolicy != RenormingPolicy::ForceIncompressible)) {
     std::partial_sort(correctableIndices.begin(), correctableIndices.begin() + nSorted, correctableIndices.end(), [](const auto& a, const auto& b) { return a.second < b.second; });
   } else {
     std::stable_sort(correctableIndices.begin(), correctableIndices.end(), [](const auto& a, const auto& b) { return a.second < b.second; });