From 7334810f5525e8135e80998b23f6d4194814e910 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 22 Jan 2026 10:16:16 -0800 Subject: [PATCH 001/162] Update to NAM Core 0.4.0 --- NeuralAudio/CMakeLists.txt | 11 ++++++++++- NeuralAudio/NAMModel.h | 40 ++++---------------------------------- deps/NeuralAmpModelerCore | 2 +- 3 files changed, 15 insertions(+), 38 deletions(-) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index b406ca3..866fb34 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -61,7 +61,16 @@ set(SOURCES if(BUILD_NAMCORE) set(NAM_SOURCES ../deps/NeuralAmpModelerCore/NAM/activations.h ../deps/NeuralAmpModelerCore/NAM/activations.cpp + ../deps/NeuralAmpModelerCore/NAM/conv1d.cpp + ../deps/NeuralAmpModelerCore/NAM/conv1d.h + ../deps/NeuralAmpModelerCore/NAM/film.h + ../deps/NeuralAmpModelerCore/NAM/gating_activations.h + ../deps/NeuralAmpModelerCore/NAM/get_dsp.h + ../deps/NeuralAmpModelerCore/NAM/get_dsp.cpp ../deps/NeuralAmpModelerCore/NAM/lstm.h + ../deps/NeuralAmpModelerCore/NAM/registry.h + ../deps/NeuralAmpModelerCore/NAM/ring_buffer.h + ../deps/NeuralAmpModelerCore/NAM/ring_buffer.cpp ../deps/NeuralAmpModelerCore/NAM/lstm.cpp ../deps/NeuralAmpModelerCore/NAM/dsp.h ../deps/NeuralAmpModelerCore/NAM/dsp.cpp @@ -76,7 +85,7 @@ set(RTNEURAL_WN_SOURCES ../deps/RTNeural-NAM/wavenet/wavenet_layer.hpp ../deps/RTNeural-NAM/wavenet/arena.hpp) endif() -add_library(NeuralAudio STATIC ${SOURCES} ${NAM_SOURCES} ${RTNEURAL_WN_SOURCES}) +add_library(NeuralAudio OBJECT ${SOURCES} ${NAM_SOURCES} ${RTNEURAL_WN_SOURCES}) target_include_directories(NeuralAudio PUBLIC ..) target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/NeuralAmpModelerCore) diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index 814ee33..a9c5ff7 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -2,9 +2,9 @@ #include "NeuralModel.h" #include +#include #include -#include -#include +#include namespace NeuralAudio { @@ -34,46 +34,14 @@ namespace NeuralAudio ReadNAMConfig(modelJson); - std::string arch = modelJson.at("architecture"); - - nlohmann::json config = modelJson.at("config"); - - std::vector weights = modelJson.at("weights"); - - if (arch == "WaveNet") - { - std::vector layer_array_params; - - for (size_t i = 0; i < config.at("layers").size(); i++) - { - nlohmann::json layerConfig = config.at("layers").at(i); - - layer_array_params.push_back( - nam::wavenet::LayerArrayParams(layerConfig.at("input_size"), layerConfig.at("condition_size"), layerConfig.at("head_size"), - layerConfig.at("channels"), layerConfig.at("kernel_size"), layerConfig.at("dilations"), - layerConfig.at("activation"), layerConfig.at("gated"), layerConfig.at("head_bias"))); - } - - const bool with_head = !config.at("head").is_null(); - const float head_scale = config.at("head_scale"); - - namModel = std::make_unique(layer_array_params, head_scale, with_head, weights, sampleRate); - } - else if (arch == "LSTM") - { - const int num_layers = config.at("num_layers"); - const int input_size = config.at("input_size"); - const int hidden_size = config.at("hidden_size"); - - namModel = std::make_unique(num_layers, input_size, hidden_size, weights, sampleRate); - } + namModel = nam::get_dsp(modelJson); return true; } void Process(float* input, float* output, size_t numSamples) { - namModel->process(input, output, (int)numSamples); + namModel->process(&input, &output, (int)numSamples); } void Prewarm() diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index e181f61..925da40 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit e181f61efb8d05d34add45b5eecb3893ff21177c +Subproject commit 925da40bec6fb9a45594ca7fe7e0e330601d26e3 From 0645e40de7106a17858ac89f22509e572f32398d Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 22 Jan 2026 11:31:24 -0800 Subject: [PATCH 002/162] Version check for A2. Add example A2 model for testing --- NeuralAudio/NeuralModel.cpp | 24 +- Utils/Models/wavenet_a2_max.nam | 2802 +++++++++++++++++++++++++++++++ 2 files changed, 2819 insertions(+), 7 deletions(-) create mode 100644 Utils/Models/wavenet_a2_max.nam diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index d02d05b..6c6df7e 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -122,6 +122,17 @@ namespace NeuralAudio return CreateFromStream(jsonStream, modelPath.extension()); } + bool NAMIsA2(std::string version) + { + int major = 0, minor = 0, patch = 0; + char dot; + std::stringstream ss(version); + + ss >> major >> dot >> minor >> dot >> patch; + + return (major > 0) || (minor > 5) || ((minor == 5) && (patch > 4)); + } + NeuralModel* NeuralModel::CreateFromStream(std::basic_istream& jsonStream, std::filesystem::path extension) { EnsureModelDefsAreLoaded(); @@ -133,10 +144,10 @@ namespace NeuralAudio if (extension == ".nam") { - std::string arch = modelJson.at("architecture"); - #ifdef BUILD_NAMCORE - if (wavenetLoadMode == EModelLoadMode::NAMCore) + std::string version = modelJson.at("version"); + + if ((wavenetLoadMode == EModelLoadMode::NAMCore) || NAMIsA2(version)) { NAMModel* model = new NAMModel; @@ -146,12 +157,13 @@ namespace NeuralAudio } #endif + std::string arch = modelJson.at("architecture"); + nlohmann::json config = modelJson.at("config"); + if (newModel == nullptr) { if (arch == "WaveNet") { - nlohmann::json config = modelJson.at("config"); - if (config.at("layers").size() == 2) { nlohmann::json firstLayerConfig = config.at("layers").at(0); @@ -215,8 +227,6 @@ namespace NeuralAudio } else if (arch == "LSTM") { - nlohmann::json config = modelJson.at("config"); - #ifdef BUILD_STATIC_RTNEURAL if (lstmLoadMode == EModelLoadMode::RTNeural) { diff --git a/Utils/Models/wavenet_a2_max.nam b/Utils/Models/wavenet_a2_max.nam new file mode 100644 index 0000000..cea7420 --- /dev/null +++ b/Utils/Models/wavenet_a2_max.nam @@ -0,0 +1,2802 @@ +{ + "notes": [ + "This model is meant as a 'test case' to contain all of the new features that are being considered for A2.", + "It doesn't have slimmability." + ], + "version": "0.5.4", + "metadata": { + "date": { + "year": 2026, + "month": 1, + "day": 19, + "hour": 14, + "minute": 51, + "second": 21 + }, + "loudness": -20.0, + "gain": 0.2, + "name": "Generated WaveNet Model", + "modeled_by": "create_wavenet.py" + }, + "architecture": "WaveNet", + "config": { + "condition_dsp": { + "version": "0.5.4", + "metadata": { + "date": { + "year": 2026, + "month": 1, + "day": 19, + "hour": 14, + "minute": 51, + "second": 18 + }, + "loudness": -20.0, + "gain": 0.2, + "name": "Generated WaveNet Model", + "modeled_by": "create_wavenet.py" + }, + "architecture": "WaveNet", + "config": { + "layers": [ + { + "input_size": 1, + "condition_size": 1, + "head_size": 4, + "channels": 3, + "bottleneck": 6, + "kernel_size": 2, + "dilations": [ + 1, + 2 + ], + "activation": { + "type": "SiLU" + }, + "gating_mode": "gated", + "head_bias": false, + "groups_input": 3, + "groups_input_mixin": 1, + "groups_1x1": 3, + "head_1x1": { + "active": true, + "out_channels": 6, + "groups": 3 + }, + "secondary_activation": "Hardswish", + "conv_pre_film": { + "active": true, + "shift": true + }, + "conv_post_film": { + "active": true, + "shift": true + }, + "input_mixin_pre_film": { + "active": true, + "shift": true + }, + "input_mixin_post_film": { + "active": true, + "shift": true + }, + "activation_pre_film": { + "active": true, + "shift": true + }, + "activation_post_film": { + "active": true, + "shift": true + }, + "1x1_post_film": { + "active": true, + "shift": true + }, + "head1x1_post_film": { + "active": true, + "shift": true + } + }, + { + "input_size": 3, + "condition_size": 1, + "head_size": 8, + "channels": 4, + "bottleneck": 2, + "kernel_size": 3, + "dilations": [ + 1, + 3, + 5 + ], + "activation": { + "type": "PReLU", + "negative_slopes": [ + 0.04, + 0.05 + ] + }, + "gating_mode": "blended", + "head_bias": false, + "groups_input": 1, + "groups_input_mixin": 1, + "groups_1x1": 1, + "head_1x1": { + "active": true, + "out_channels": 4, + "groups": 2 + }, + "secondary_activation": { + "type": "LeakyHardtanh", + "min_val": 0.0, + "max_val": 0.9, + "min_slope": 0.0, + "max_slope": 0.02 + }, + "conv_pre_film": { + "active": true, + "shift": false + }, + "conv_post_film": { + "active": true, + "shift": false + }, + "input_mixin_pre_film": { + "active": true, + "shift": false + }, + "input_mixin_post_film": { + "active": true, + "shift": false + }, + "activation_pre_film": { + "active": true, + "shift": false + }, + "activation_post_film": { + "active": true, + "shift": false + }, + "1x1_post_film": { + "active": true, + "shift": false + }, + "head1x1_post_film": { + "active": true, + "shift": false + } + } + ], + "head": null, + "head_scale": 0.02 + }, + "weights": [ + 0.2788535969157675, + -0.9499784895546661, + -0.4499413632617615, + -0.5535785237023545, + 0.4729424283280248, + 0.3533989748458226, + 0.7843591354096908, + -0.8261223347411677, + -0.15615636062945915, + -0.9404055611238593, + -0.5627240503927933, + 0.010710576206724776, + -0.9469280606322728, + -0.602324698626703, + 0.2997688755590464, + 0.08988296120643335, + -0.5591187559186066, + 0.17853136775181744, + 0.6188609133556533, + -0.987002480643878, + 0.6116385036656158, + 0.3962787899764537, + -0.31949896696401625, + -0.6890410003764369, + 0.9144261444135624, + -0.32681090977474647, + -0.8145083132397042, + -0.806567246333072, + 0.6949887326949196, + 0.20745206273378214, + 0.6142565465487604, + 0.45946357338763577, + 0.07245618290940148, + 0.9462315279587412, + -0.24293124558329304, + 0.104081262546454, + 0.6588093285059897, + 0.2370395047284921, + 0.7234138006215545, + 0.15470429051352408, + 0.40914367242984695, + -0.9083512326886756, + -0.5442034486969063, + -0.42122407279578566, + -0.840416046152745, + -0.5344182272779396, + -0.7979971411805418, + -0.44405279377981577, + 0.27136888852880037, + -0.2703356420598315, + -0.2596380657662347, + -0.5809859384570246, + -0.4660443559017733, + 0.873309175424988, + 0.2960707704931871, + 0.21826201133397638, + -0.657722703603806, + 0.45825359590069836, + -0.6731950124761432, + -0.24108911648470444, + 0.9790467012731905, + 0.2799995197081857, + 0.11389948754929247, + 0.3692285019797492, + 0.6857038403796192, + 0.5519998230924896, + -0.5419038560717913, + -0.9357995121919245, + -0.36909390388183616, + -0.46451824804859454, + -0.5780343128273471, + 0.8858194286701089, + 0.7527352529453377, + -0.37064423840304417, + 0.3108773305897601, + -0.20873619787867148, + 0.829095179481087, + -0.0822962948252024, + -0.4702396670038951, + -0.5067449846120331, + 0.12273626832630158, + -0.47451678295412947, + 0.16917198044708104, + 0.795645767204954, + -0.20119898971920547, + -0.5613584816854333, + 0.9950752129902205, + 0.01905258735292903, + -0.8181811756524122, + -0.9057672491505309, + -0.7807017392986817, + 0.2548920834061801, + 0.5841587287259282, + -0.15568006640063192, + -0.8729445876960857, + -0.23676142698692648, + 0.9922427604801936, + 0.058228690198274036, + 0.9421567552272363, + 0.7215594044689961, + -0.9770379561143607, + 0.4414436387203893, + 0.36342073805314956, + 0.07394066081759032, + -0.46634962009491443, + 0.2819235971596161, + -0.7768956528082471, + -0.13046949866179003, + -0.09255258734158711, + 0.9076318550421603, + 0.7517058807563881, + -0.4732218984978185, + 0.0011722261005966406, + -0.6426962389397373, + 0.825255678689641, + 0.7410371396735338, + -0.4031104171027342, + 0.2778989897320103, + 0.21794042287634463, + -0.6943214629007304, + 0.5250216001503025, + 0.07875806023925147, + 0.5572529572611165, + 0.06070734439035497, + -0.998856207744113, + -0.3516878859906538, + -0.9610465152283354, + 0.8581972325292342, + 0.7574437556463685, + 0.6633310587223589, + -0.38497174919467714, + -0.8841496670116249, + 0.7560191984080811, + 0.8938988905959881, + -0.8286930958642424, + -0.02801907336677245, + -0.8615749630632328, + 0.5212043305144631, + 0.5316688586139755, + -0.7432170710004744, + -0.04943524380253739, + 0.0996071869898878, + -0.4698867421198818, + 0.7448660821705149, + -0.15372411959822618, + -0.5764035891158359, + 0.07859217755891668, + 0.45986213817995236, + -0.5976978732206082, + -0.3765674173982101, + 0.9902987133217893, + 0.299756115278907, + -0.12379983217099189, + 0.035151682071181245, + -0.7579916082634686, + -0.5506053259368853, + -0.32382887570508934, + 0.17661743691446663, + -0.539770534806846, + -0.559565231096881, + -0.8580138279819349, + 0.2622059145401978, + -0.5421164323776912, + 0.8108400260122559, + 0.719270800507493, + -0.8582853002226931, + -0.5239907312620096, + 0.33795555659256116, + -0.5715263852591228, + -0.73537630254995, + 0.871028481161342, + 0.14208618665056894, + -0.05465794737641172, + 0.5692388485815068, + 0.6149939955332868, + -0.6191801712762446, + -0.8061383715423533, + -0.13789763518724496, + -0.15284275396015845, + -0.06595066392665005, + 0.4581516989197012, + 0.34672909458660306, + 0.9683304227319323, + -0.8031642576960822, + -0.19475743579546245, + -0.3213947892100737, + 0.7233450727055821, + -0.5026873321594287, + -0.619582183118377, + -0.1027729043337362, + -0.15623672033119163, + -0.44290971066611906, + -0.500387104235799, + 0.8465311985520256, + -0.11373850989308609, + 0.7226982095236612, + 0.10065062489969612, + -0.8988233409502375, + 0.9985649368254532, + 0.6720551701599038, + 0.9379925145695025, + 0.8527339660162552, + 0.6973914688286109, + -0.667377778792172, + -0.02871774909856306, + -0.5725054016016367, + -0.19791941490109477, + -0.8827292000556421, + -0.2420537620461678, + 0.9706176875594519, + -0.4695938836556961, + 0.5681412038971387, + -0.08998326532171341, + -0.15398502801967417, + 0.9146352817193464, + 0.9908453789854277, + 0.11153664681123643, + 0.436816550592652, + -0.6904063494518717, + -0.4065843490108716, + 0.9374187299383177, + 0.15836058163251243, + 0.08439040274854848, + 0.4959511207581282, + -0.8856694541850338, + 0.16835518891794243, + 0.005700765839027122, + 0.7054397840965707, + -0.6851345441210335, + 0.9215578065489007, + -0.8397770695188262, + -0.6283500780385536, + 0.19007021290005532, + 0.3504251072081803, + -0.5295922099981376, + -0.7602267721057516, + 0.780574628258875, + -0.5075693044227503, + 0.1890383070668824, + 0.23876302066420618, + -0.16155016932825506, + 0.16734457858244944, + 0.04556543106391775, + 0.8694125154728545, + -0.5914816011529271, + 0.4323836015788296, + -0.522628094768308, + -0.208428306417491, + 0.34338044591994255, + -0.40000584040247555, + -0.36764560745629193, + 0.5037289848288042, + -0.8549137710136854, + -0.08342895476282775, + 0.9969088817088847, + 0.9921928957101889, + -0.853478557800734, + -0.5736913754659192, + -0.4695991704991973, + 0.8665187559874181, + 0.7617283473728791, + 0.7585404849690855, + -0.2609458225222321, + -0.6845063352855361, + 0.667489909279614, + 0.4070798501747419, + 0.22335553145190024, + 0.9744661272630086, + 0.3079526354214652, + -0.9843537856956841, + 0.6342082702309233, + -0.4012424956000442, + 0.32677742993215464, + 0.8778600078542078, + -0.7314177712132646, + -0.7691426591617956, + -0.7859280445811647, + 0.10644728176963181, + -0.45530357537036736, + 0.20965965406044784, + 0.43522437427759586, + -0.5928053753450941, + 0.26847591777015944, + -0.4720321967391812, + -0.02293629570124689, + 0.8106729821586465, + 0.6922074265897109, + -0.8154030645745332, + -0.15284845487254728, + -0.44663955205549666, + -0.9929086218244354, + 0.5422384460392542, + 0.27422675460275925, + -0.4760894751313036, + 0.48246181669586163, + 0.10336084225278253, + -0.14462616203864131, + -0.9806606007833201, + -0.8495122798524659, + 0.7662127866002859, + 0.8078571431197863, + 0.09118057841104465, + 0.6691900397720334, + 0.16501913297958803, + -0.7038124288650347, + -0.7451089614357225, + -0.38348330013973264, + 0.79796297748518, + 0.5922446097760834, + 0.7214051640018055, + 0.7978492730529492, + -0.5798469233204919, + -0.5009405215541511, + -0.7944127566564287, + 0.5602324837428854, + 0.7682694029020178, + -0.1872452203357664, + 0.2413230203014256, + -0.6908933233355907, + 0.8597620313873489, + 0.7292113924399279, + 0.9524120658619257, + 0.6215434398807937, + 0.7628324093266488, + -0.9504272762036226, + 0.4731289435101642, + -0.33562906410714266, + 0.8616317720966511, + 0.6044702778742779, + 0.7281280567505588, + 0.621498633148778, + -0.46638858081105594, + 0.5747490182709423, + -0.7838087471940858, + 0.7443335658121795, + 0.7171865026755633, + -0.5551325649086711, + 0.6331732111938579, + -0.07939353064211585, + -0.38961826532279886, + 0.5906909983057236, + -0.5448090251844593, + -0.952671130597097, + -0.6137404233445827, + -0.34347609760458697, + 0.7287058840605727, + 0.9337782080967223, + -0.4417500145562572, + 0.2829634772152554, + -0.20064323127987826, + 0.9622993743965202, + 0.07243146495744379, + 0.8784742806494314, + -0.7693164962971448, + 0.9408012220444559, + -0.6428643676550727, + 0.9250686315231109, + -0.46906727495406275, + -0.7831949055705778, + -0.1308724828707113, + 0.4570901213054086, + -0.37264537161001754, + 0.21241770661228654, + 0.022846119338956195, + -0.22960913331054567, + 0.15317608699319907, + -0.4905549877228361, + 0.4175705676683412, + -0.9966174435627411, + 0.8511503309981654, + 0.0769039941855838, + 0.438859998289691, + 0.48390015567895306, + 0.34125700886599897, + -0.27155705643747163, + -0.8600523777473796, + 0.32847536982254466, + -0.3395999279148072, + -0.37216870988328066, + 0.696030559012671, + 0.43950852602790036, + -0.39935546357747165, + -0.3814306755826935, + -0.1832141827615663, + -0.19519922588455074, + -0.40868959494810597, + -0.7454244018816936, + -0.1591073324541834, + 0.880727341460366, + 0.35463589054546585, + 0.8056110914651653, + 0.23102983190276105, + -0.39810025086886935, + 0.09587442627139642, + -0.999188120605425, + -0.4261725662621456, + -0.1402237000203308, + 0.15996956239136395, + 0.30941124740614323, + -0.07002361950597158, + -0.11568040139038516, + -0.5725971980217994, + -0.053627628181347475, + 0.8023616516565084, + 0.5920495202535605, + -0.6606172076038905, + -0.8304089265497565, + 0.030904019830432894, + 0.26588171153159146, + -0.3296234891803982, + 0.6368469290733285, + 0.5022762750814644, + 0.3455913411143341, + -0.5507186680054235, + -0.6017401345468467, + -0.9511492245463473, + -0.5103149118432997, + -0.04972731156238974, + 0.6994753892494638, + -0.8543435416308618, + -0.1711179780045613, + 0.2595307614754274, + -0.6111295265205814, + 0.39270850098100984, + -0.011245661979126131, + -0.5120311208431223, + 0.31211602222356816, + -0.9889103637239365, + 0.5019289532369458, + 0.5400923771480501, + -0.7868254068729221, + -0.14970761211453176, + -0.6482266365869367, + 0.9159320845590795, + 0.03591550088748163, + -0.8995632297187182, + -0.5016034406800567, + 0.6966726947033195, + -0.08707634905965489, + 0.602833203444529, + 0.3351554651727062, + 0.975784906132896, + 0.1909046369388394, + 0.9000792168863119, + 0.782851851620874, + 0.2253046455235257, + 0.4385479225519342, + 0.00955632964880393, + 0.6611383394428301, + 0.09574390122165677, + 0.7944162064665243, + 0.4873108843191698, + -0.05065112635389335, + -0.4816169030699613, + -0.5055205249806809, + 0.27532287355231255, + 0.5316273685943309, + 0.0425996256559642, + 0.2534968739635626, + -0.4508051061649234, + -0.8450332922705284, + -0.428543698273695, + -0.4565697858356308, + -0.3605808631624754, + 0.08030444503691281, + -0.7232518769676886, + -0.5374770405436264, + 0.38789962459810456, + 0.41283828338910444, + -0.8715422985722439, + -0.18480126066682656, + 0.08522228100783069, + -0.1684515317936881, + -0.5863312209711795, + -0.15971296445314875, + 0.8096769566803539, + 0.1681588284084503, + 0.39104597299593613, + 0.7134640646078685, + 0.5311891522361389, + -0.23923794214457672, + -0.9882078328320139, + -0.2964823946563506, + 0.5069502501187717, + 0.7068959011382092, + 0.9068606769402126, + -0.1619574347403563, + 0.49503133795610155, + 0.09226461946767817, + 0.20650517788248268, + -0.5589226113523622, + -0.5611567307571277, + -0.1283280479067268, + -0.9419503601065695, + -0.3277409126032351, + 0.3582837700566994, + -0.19136666172472583, + -0.6699105375929824, + -0.0652197015353797, + -0.7447444054376786, + 0.24451392194812938, + -0.9460670961897246, + -0.21195948732059056, + 0.12878396604954823, + -0.9457959073193751, + 0.2854992960186715, + -0.7286010255388715, + -0.07660311189697655, + -0.8994307330227449, + -0.24179227162372086, + -0.5766794315770369, + -0.3463083902373829, + 0.5224594157880542, + -0.24174756887172744, + 0.5040196471095697, + 0.6638485703105452, + -0.4954569364352388, + -0.8361875344767149, + -0.9612334258999786, + 0.0788380958450674, + 0.9998156570184185, + -0.30007931255963216, + 0.30028818649975, + 0.5624660992217898, + 0.30350931048777885, + 0.5084664081190524, + 0.8992234654319777, + -0.6012786352749342, + -0.9592399653593353, + -0.6952353084304193, + -0.7475580502515875, + 0.33891768923982135, + 0.12793916386003823, + -0.5640709181869927, + 0.3989299424923016, + 0.5337961967124816, + -0.6644217132643955, + 0.21449498778186338, + 0.49585130391057164, + -0.7709342572422047, + 0.6386023486221701, + 0.9294415460681753, + -0.7838025006847869, + -0.948643149005068, + -0.3760855112106096, + 0.3546945737008176, + 0.9163456764117919, + -0.20669111696674536, + 0.4300294100989368, + -0.8480070443138801, + 0.3812288318659607, + 0.2544847912020889, + -0.7961973891080469, + 0.544961769902448, + 0.7005864781775926, + 0.20082322963368826, + -0.7578898698653698, + 0.9676887030293426, + 0.5652706927220392, + -0.30559246938310203, + -0.1432439735305111, + -0.25885824756388764, + 0.011921579354155831, + -0.31753765027743364, + 0.6991512539991547, + 0.6446618361801162, + -0.7889222587120028, + 0.9215751344291572, + 0.27117021220289184, + 0.6574146220049164, + 0.41461728741215387, + -0.1290257099846459, + 0.46759060802678376, + 0.9309474624761553, + -0.45983520722519833, + 0.6163984376135119, + 0.07634581289651154, + -0.033004992233607755, + -0.12885101399221144, + 0.46205242861024454, + -0.46320892390154933, + 0.7034263200386637, + 0.6614620377812068, + -0.8266742038886512, + 0.7632623680055439, + -0.512273121618088, + -0.07058306679353654, + 0.22066340846104127, + -0.24202139174347193, + -0.9426000044598208, + 0.7019056726249182, + -0.6363202856846164, + -0.575760299640522, + 0.595664713656193, + -0.3193223136714327, + 0.7606399595164508, + 0.4023675006644032, + -0.4474628484876302, + -0.9796977711226458, + 0.8961251555540624, + -0.8287740760839575, + 0.4401493282083886, + -0.022844306302425288, + 0.5163293069649328, + 0.38121867889304606, + 0.29180579948190455, + -0.018357329842827674, + 0.5858657362646349, + -0.8138932988906566, + -0.5568071990549397, + 0.3835743105904037, + -0.3875879397398232, + 0.16311117066473435, + -0.05347902248095049, + 0.0618438622915356, + -0.1489923745895705, + 0.49187087342721925, + -0.3384174056081397, + 0.4057098843715379, + -0.45816714604760733, + -0.497192647598117, + -0.7586882304953884, + -0.6148314130969652, + -0.7608905174898943, + 0.07172793076749961, + 0.5243792189682059, + -0.6297003151246761, + -0.5672307202427886, + -0.031602825574582605, + 0.44917000218610315, + 0.9532140457661136, + 0.0492737382172157, + -0.43400259345008996, + -0.7989477838184185, + -0.6117648438178531, + -0.5450336730348906, + -0.6411169126291933, + -0.9717032654878723, + 0.06827017853532658, + -0.4513773464334654, + 0.9485898622054758, + 0.1067179335972166, + 0.3948347858203889, + -0.7474410140831822, + 0.736922394525888, + -0.018242610952442284, + 0.7454394699970692, + 0.1481284392526645, + -0.061206110138979586, + -0.1190624037279795, + -0.6312726592198366, + -0.8972465656328443, + 0.8821271935362065, + -0.04454162546862461, + 0.6442312905988608, + -0.19858511549612046, + -0.8518356594887204, + 0.2588914139039966, + -0.8927818514149337, + -0.7016048310526897, + 0.12567919416912843, + -0.39232897631554464, + 0.9878362454204404, + -0.7630968755770768, + 0.5288868906523885, + 0.21263530248592555, + 0.5814816596868388, + -0.5486257258699307, + 0.04514507026043568, + -0.09897107022249152, + -0.11455799196134331, + 0.7203333316523601, + 0.9800625209335958, + -0.3892395113612792, + 0.24205464214429706, + 0.21926182449026066, + 0.48017861096909, + 0.895180400650756, + -0.5844241883528973, + -0.577949609389214, + 0.32085627438407327, + -0.6858858132382457, + -0.652372903347131, + -0.8498702621976642, + -0.9946485547942285, + -0.09899259076459521, + 0.18762239023912652, + -0.4174814219379448, + -0.5370475308879483, + 0.41391165975806654, + 0.4059751161874343, + -0.09193734718591351, + 0.3747698401424855, + 0.8478220889650974, + 0.5756560532934627, + 0.25011601432844377, + 0.32236608570673564, + 0.8673369168910265, + -0.1497220677915767, + 0.08912475742115733, + 0.2952694468049615, + 0.8168228904252122, + 0.6532623193001192, + -0.8571803262883717, + -0.6681544215585506, + -0.3847763747715349, + 0.4979154441392466, + 0.13841409863818477, + -0.4227788233883165, + -0.7512926836505835, + 0.37735598242418744, + 0.3994673699514346, + 0.8853524814880815, + 0.0009443542358511525, + -0.012409561309831396, + -0.839116296201398, + -0.9202784316327739, + -0.13594267167845486, + -0.35535683328340006, + -0.4992641951759922, + -0.8173462267380296, + 0.9238222043856734, + 0.6719172278123626, + 0.1503982184398609, + 0.9015725556127048, + 0.9991448337549027, + 0.3445631686064785, + -0.4609779480658922, + -0.9195366537107676, + 0.5125376608251142, + -0.05899835349673288, + 0.3030189788673727, + 0.832145575853535, + -0.6370217055537482, + 0.17065925055570874, + 0.26956943890817375, + -0.016548395617973277, + -0.8175151874123456, + -0.3040778874106991, + -0.33338321266981596, + 0.3402670190423709, + 0.7154661888007052, + -0.3403926728420692, + 0.387347347966865, + -0.4235644092770885, + 0.8903870791264212, + 0.6271320694759452, + 0.10019321794356584, + -0.09034818279654111, + -0.37096568565966415, + -0.35345242744801064, + 0.9403694536171543, + -0.19164988599850874, + 0.029192504658210217, + 0.9762384295635083, + 0.31532077296625793, + 0.08518718871439002, + -0.173504858401357, + -0.6248349172109458, + -0.2764412816966495, + 0.5128863081111474, + 0.25081748429001793, + 0.5199810721220339, + -0.5928835232994523, + 0.09843927817066334, + 0.8553455216888954, + -0.1237678098552435, + 0.39650005824434453, + -0.7571478332747603, + 0.9462936316433888, + 0.21774333416392988, + -0.5214050753446993, + -0.6832436723895352, + 0.10167801400533993, + 0.10450281810746609, + -0.8135815974369549, + 0.9845142787904393, + 0.8258597569683359, + -0.0771042116424332, + -0.7650677018208849, + 0.6642863475378025, + -0.003248990585609457, + 0.4332066519835822, + 0.01774403013902348, + -0.4531502065734516, + 0.6694478911533888, + 0.9604892652067509, + -0.5125381878497233, + 0.10253015376093932, + -0.23282797350570017, + 0.8437362998631237, + 0.016481783185787968, + 0.7586525102929522, + 0.7280538688571763, + -0.44750519435591807, + 0.5800123640062269, + -0.17011515289730972, + 0.8684967873650402, + 0.015475351619312905, + 0.6410989463711567, + -0.4343220334343467, + -0.4028883004564656, + 0.17387544482832218, + 0.9978046664586775, + -0.020719306688598005, + -0.7028091632344218, + 0.07716115540774737, + -0.30975211661398494, + 0.10383483414162198, + 0.08686012591734604, + -0.08931076626699785, + -0.35644529954192516, + -0.6226952525857892, + 0.3949968552411427, + 0.14359528396980226, + -0.5328751078842138, + 0.5510889501985177, + -0.9127054018053962, + 0.4894103031303918, + 0.4104557620500051, + 0.6228178051297537, + -0.22784249501803533, + 0.32737765896906934, + 0.6414951034191101, + 0.9616362773195701, + -0.009342700767152934, + -0.9259607773088476, + 0.00458230026593176, + 0.1803608586191341, + 0.7394006267244309, + 0.7483807481086167, + -0.11938758045864573, + 0.05190217361512284, + -0.08614385105151556, + 0.44488765514128525, + -0.1800427605072712, + 0.3095626528554676, + -0.6912775624569023, + -0.061018798030536114, + 0.9384072611484344, + -0.32287753189713664, + 0.3854091971737641, + 0.2996733051585452, + 0.7035305847013829, + 0.7046826731317861, + 0.7186843683365332, + -0.2399812049546899, + -0.36667769213320067, + 0.437434850445966, + 0.5188036186687273, + 0.7447660347970726, + -0.9282018003246268, + -0.8631585056368971, + 0.2623220337092811, + 0.8418581975605934, + 0.9948518459124307, + 0.4935327334758539, + -0.13205705615374863, + -0.8031137472336041, + 0.26749565756158256, + 0.7451584652141814, + -0.11264289667658844, + 0.3880023255864691, + 0.806848124100972, + -0.9080180626455745, + 0.5922869303244236, + -0.4132644480695824, + -0.25031782044937034, + -0.7088604086012031, + 0.06233263629752006, + 0.13185612383187362, + 0.5850389477619065, + -0.6600327018638052, + -0.8420632986800107, + 0.7416791972897097, + 0.2394207370748307, + -0.5183404162873739, + 0.8256580320471096, + -0.7137645597429807, + -0.07770017329000489, + -0.49204532117291544, + -0.4893465836816919, + -0.9812051370902579, + 0.6092661539502926, + 0.802418847197766, + 0.3552217713981589, + -0.6840487558564261, + -0.11654043279237114, + -0.3088687511138366, + 0.17514341025284286, + 0.2778774047200243, + -0.15138212307820464, + -0.4998035511845975, + 0.6906078502851973, + -0.6015660017822153, + -0.23061350206530906, + -0.0335838778815678, + -0.5255885961448508, + 0.14384538470147779, + 0.14962386035729014, + 0.9853840872537951, + -0.4095384922334613, + 0.9558889691537253, + 0.31645963185736137, + -0.45103923964354453, + 0.13185803391139794, + 0.37159898546802417, + 0.48933768233065034, + -0.9019114984480001, + 0.21281298615294908, + -0.006545426952259348, + 0.8083105817874507, + -0.42761169708161106, + 0.5977202390151974, + 0.2141299963285319, + -0.29535808832936694, + 0.27323575601178374, + 0.24178232626060825, + 0.35552891724671887, + 0.44185675334141794, + 0.3183630806335074, + 0.6766742339250331, + 0.25649620737379664, + 0.8068074081459358, + 0.2926812177810456, + -0.38213423209472785, + -0.11835361967437685, + 0.1591476107368055, + 0.4647195358784766, + -0.8197332485075621, + -0.40977909674168944, + 0.49496172987674303, + -0.6487198591113867, + -0.7356804045064329, + 0.07881551796884212, + 0.9429791624226798, + 0.06170474740557008, + 0.8269739489633938, + 0.6609452391347947, + -0.4860598308734754, + 0.6493796250848145, + -0.03630434025251761, + 0.612976987587533, + 0.4931187014340901, + -0.32256949239622745, + -0.7696605851004812, + 0.9257865857377552, + -0.7184859699882347, + 0.9330004189255041, + 0.7202811937976437, + 0.4484334241510364, + 0.9598844855638069, + 0.9345394946000647, + 0.6091752880411239, + -0.26844990118886725, + 0.5813639371778749, + -0.9721626897981976, + 0.07314461653811821, + -0.09042794453225067, + 0.3456567637475072, + 0.3446815947020263, + 0.1691201833043321, + 0.6448346024535485, + 0.8805837835591084, + -0.7833077956015386, + -0.5323561955768337, + -0.9499507007070354, + 0.7684696904297044, + 0.1228147644997577, + 0.8305118174863189, + -0.55726559985202, + -0.8735659176796091, + 0.6477107027808953, + 0.8187752768557841, + -0.39561965094149953, + -0.18340828840917456, + -0.7204459749855578, + 0.8925230657638925, + -0.39127083128798956, + -0.014750762043588272, + -0.8056160027563568, + 0.7745186170570046, + -0.728671902587327, + -0.09271248622221484, + 0.34097243770034247, + 0.4862802430463431, + 0.8919481715588642, + -0.16174649317055434, + 0.48453802953063163, + -0.69095419518033, + -0.17023094512638615, + -0.801956730578943, + -0.02130592442077095, + -0.18376822860459896, + 0.903043050762119, + -0.9345674262899062, + -0.2589400825311292, + -0.1132338278785967, + 0.9011103397028539, + 0.7109003866119092, + -0.8012907507877394, + 0.3713605309625707, + 0.08893172296428986, + 0.9556850589040935, + -0.28265231757536413, + -0.20372071451125384, + -0.6203828756778409, + -0.7556805618254725, + 0.6960663769273621, + -0.09056526285896571, + 0.325537476123956, + 0.2834089344664352, + 0.19429191903909016, + -0.9572850905272587, + 0.5735891809092335, + -0.5128622056719527, + -0.7481522293839142, + 0.12915595181592665, + -0.8627796943512882, + 0.5303147517771689, + -0.585685259306683, + -0.5680972961626505, + 0.7393908535390894, + -0.34288089313553916, + -0.7048916401171126, + 0.8010620712635164, + -0.9943288970399673, + 0.7168122527603527, + -0.710624039358984, + -0.7400157371131264, + -0.49869160654375233, + -0.6510057581972131, + 0.3221152851946336, + -0.9484397004276033, + -0.9702793455385823, + 0.5799693284695078, + -0.5241367878190739, + -0.3524570760759551, + -0.6515075971876456, + -0.895201964277647, + 0.483436113908299, + 0.05217105319573423, + 0.4913305500679914, + -0.04750806915349948, + 0.5560340786284055, + 0.026475915218383328, + -0.781891979992309, + 0.007677379571642717, + 0.8908312859402125, + -0.9132699262016566, + 0.5664539919607592, + 0.7339618155196765 + ], + "sample_rate": 48000 + }, + "layers": [ + { + "input_size": 1, + "condition_size": 8, + "head_size": 1, + "channels": 4, + "bottleneck": 4, + "kernel_size": 4, + "dilations": [ + 1, + 2 + ], + "activation": { + "type": "PReLU", + "negative_slope": 0.015 + }, + "gating_mode": "none", + "head_bias": true, + "groups_input": 1, + "groups_input_mixin": 4, + "groups_1x1": 2, + "head_1x1": { + "active": true, + "out_channels": 4, + "groups": 2 + }, + "secondary_activation": "", + "conv_pre_film": { + "active": true, + "shift": true + }, + "conv_post_film": { + "active": true, + "shift": true + }, + "input_mixin_pre_film": { + "active": true, + "shift": true + }, + "input_mixin_post_film": { + "active": true, + "shift": true + }, + "activation_pre_film": { + "active": true, + "shift": true + }, + "activation_post_film": { + "active": true, + "shift": true + }, + "1x1_post_film": { + "active": true, + "shift": true + }, + "head1x1_post_film": { + "active": true, + "shift": true + } + } + ], + "head": null, + "head_scale": 0.02 + }, + "weights": [ + 0.2788535969157675, + -0.9499784895546661, + -0.4499413632617615, + -0.5535785237023545, + 0.4729424283280248, + 0.3533989748458226, + 0.7843591354096908, + -0.8261223347411677, + -0.15615636062945915, + -0.9404055611238593, + -0.5627240503927933, + 0.010710576206724776, + -0.9469280606322728, + -0.602324698626703, + 0.2997688755590464, + 0.08988296120643335, + -0.5591187559186066, + 0.17853136775181744, + 0.6188609133556533, + -0.987002480643878, + 0.6116385036656158, + 0.3962787899764537, + -0.31949896696401625, + -0.6890410003764369, + 0.9144261444135624, + -0.32681090977474647, + -0.8145083132397042, + -0.806567246333072, + 0.6949887326949196, + 0.20745206273378214, + 0.6142565465487604, + 0.45946357338763577, + 0.07245618290940148, + 0.9462315279587412, + -0.24293124558329304, + 0.104081262546454, + 0.6588093285059897, + 0.2370395047284921, + 0.7234138006215545, + 0.15470429051352408, + 0.40914367242984695, + -0.9083512326886756, + -0.5442034486969063, + -0.42122407279578566, + -0.840416046152745, + -0.5344182272779396, + -0.7979971411805418, + -0.44405279377981577, + 0.27136888852880037, + -0.2703356420598315, + -0.2596380657662347, + -0.5809859384570246, + -0.4660443559017733, + 0.873309175424988, + 0.2960707704931871, + 0.21826201133397638, + -0.657722703603806, + 0.45825359590069836, + -0.6731950124761432, + -0.24108911648470444, + 0.9790467012731905, + 0.2799995197081857, + 0.11389948754929247, + 0.3692285019797492, + 0.6857038403796192, + 0.5519998230924896, + -0.5419038560717913, + -0.9357995121919245, + -0.36909390388183616, + -0.46451824804859454, + -0.5780343128273471, + 0.8858194286701089, + 0.7527352529453377, + -0.37064423840304417, + 0.3108773305897601, + -0.20873619787867148, + 0.829095179481087, + -0.0822962948252024, + -0.4702396670038951, + -0.5067449846120331, + 0.12273626832630158, + -0.47451678295412947, + 0.16917198044708104, + 0.795645767204954, + -0.20119898971920547, + -0.5613584816854333, + 0.9950752129902205, + 0.01905258735292903, + -0.8181811756524122, + -0.9057672491505309, + -0.7807017392986817, + 0.2548920834061801, + 0.5841587287259282, + -0.15568006640063192, + -0.8729445876960857, + -0.23676142698692648, + 0.9922427604801936, + 0.058228690198274036, + 0.9421567552272363, + 0.7215594044689961, + -0.9770379561143607, + 0.4414436387203893, + 0.36342073805314956, + 0.07394066081759032, + -0.46634962009491443, + 0.2819235971596161, + -0.7768956528082471, + -0.13046949866179003, + -0.09255258734158711, + 0.9076318550421603, + 0.7517058807563881, + -0.4732218984978185, + 0.0011722261005966406, + -0.6426962389397373, + 0.825255678689641, + 0.7410371396735338, + -0.4031104171027342, + 0.2778989897320103, + 0.21794042287634463, + -0.6943214629007304, + 0.5250216001503025, + 0.07875806023925147, + 0.5572529572611165, + 0.06070734439035497, + -0.998856207744113, + -0.3516878859906538, + -0.9610465152283354, + 0.8581972325292342, + 0.7574437556463685, + 0.6633310587223589, + -0.38497174919467714, + -0.8841496670116249, + 0.7560191984080811, + 0.8938988905959881, + -0.8286930958642424, + -0.02801907336677245, + -0.8615749630632328, + 0.5212043305144631, + 0.5316688586139755, + -0.7432170710004744, + -0.04943524380253739, + 0.0996071869898878, + -0.4698867421198818, + 0.7448660821705149, + -0.15372411959822618, + -0.5764035891158359, + 0.07859217755891668, + 0.45986213817995236, + -0.5976978732206082, + -0.3765674173982101, + 0.9902987133217893, + 0.299756115278907, + -0.12379983217099189, + 0.035151682071181245, + -0.7579916082634686, + -0.5506053259368853, + -0.32382887570508934, + 0.17661743691446663, + -0.539770534806846, + -0.559565231096881, + -0.8580138279819349, + 0.2622059145401978, + -0.5421164323776912, + 0.8108400260122559, + 0.719270800507493, + -0.8582853002226931, + -0.5239907312620096, + 0.33795555659256116, + -0.5715263852591228, + -0.73537630254995, + 0.871028481161342, + 0.14208618665056894, + -0.05465794737641172, + 0.5692388485815068, + 0.6149939955332868, + -0.6191801712762446, + -0.8061383715423533, + -0.13789763518724496, + -0.15284275396015845, + -0.06595066392665005, + 0.4581516989197012, + 0.34672909458660306, + 0.9683304227319323, + -0.8031642576960822, + -0.19475743579546245, + -0.3213947892100737, + 0.7233450727055821, + -0.5026873321594287, + -0.619582183118377, + -0.1027729043337362, + -0.15623672033119163, + -0.44290971066611906, + -0.500387104235799, + 0.8465311985520256, + -0.11373850989308609, + 0.7226982095236612, + 0.10065062489969612, + -0.8988233409502375, + 0.9985649368254532, + 0.6720551701599038, + 0.9379925145695025, + 0.8527339660162552, + 0.6973914688286109, + -0.667377778792172, + -0.02871774909856306, + -0.5725054016016367, + -0.19791941490109477, + -0.8827292000556421, + -0.2420537620461678, + 0.9706176875594519, + -0.4695938836556961, + 0.5681412038971387, + -0.08998326532171341, + -0.15398502801967417, + 0.9146352817193464, + 0.9908453789854277, + 0.11153664681123643, + 0.436816550592652, + -0.6904063494518717, + -0.4065843490108716, + 0.9374187299383177, + 0.15836058163251243, + 0.08439040274854848, + 0.4959511207581282, + -0.8856694541850338, + 0.16835518891794243, + 0.005700765839027122, + 0.7054397840965707, + -0.6851345441210335, + 0.9215578065489007, + -0.8397770695188262, + -0.6283500780385536, + 0.19007021290005532, + 0.3504251072081803, + -0.5295922099981376, + -0.7602267721057516, + 0.780574628258875, + -0.5075693044227503, + 0.1890383070668824, + 0.23876302066420618, + -0.16155016932825506, + 0.16734457858244944, + 0.04556543106391775, + 0.8694125154728545, + -0.5914816011529271, + 0.4323836015788296, + -0.522628094768308, + -0.208428306417491, + 0.34338044591994255, + -0.40000584040247555, + -0.36764560745629193, + 0.5037289848288042, + -0.8549137710136854, + -0.08342895476282775, + 0.9969088817088847, + 0.9921928957101889, + -0.853478557800734, + -0.5736913754659192, + -0.4695991704991973, + 0.8665187559874181, + 0.7617283473728791, + 0.7585404849690855, + -0.2609458225222321, + -0.6845063352855361, + 0.667489909279614, + 0.4070798501747419, + 0.22335553145190024, + 0.9744661272630086, + 0.3079526354214652, + -0.9843537856956841, + 0.6342082702309233, + -0.4012424956000442, + 0.32677742993215464, + 0.8778600078542078, + -0.7314177712132646, + -0.7691426591617956, + -0.7859280445811647, + 0.10644728176963181, + -0.45530357537036736, + 0.20965965406044784, + 0.43522437427759586, + -0.5928053753450941, + 0.26847591777015944, + -0.4720321967391812, + -0.02293629570124689, + 0.8106729821586465, + 0.6922074265897109, + -0.8154030645745332, + -0.15284845487254728, + -0.44663955205549666, + -0.9929086218244354, + 0.5422384460392542, + 0.27422675460275925, + -0.4760894751313036, + 0.48246181669586163, + 0.10336084225278253, + -0.14462616203864131, + -0.9806606007833201, + -0.8495122798524659, + 0.7662127866002859, + 0.8078571431197863, + 0.09118057841104465, + 0.6691900397720334, + 0.16501913297958803, + -0.7038124288650347, + -0.7451089614357225, + -0.38348330013973264, + 0.79796297748518, + 0.5922446097760834, + 0.7214051640018055, + 0.7978492730529492, + -0.5798469233204919, + -0.5009405215541511, + -0.7944127566564287, + 0.5602324837428854, + 0.7682694029020178, + -0.1872452203357664, + 0.2413230203014256, + -0.6908933233355907, + 0.8597620313873489, + 0.7292113924399279, + 0.9524120658619257, + 0.6215434398807937, + 0.7628324093266488, + -0.9504272762036226, + 0.4731289435101642, + -0.33562906410714266, + 0.8616317720966511, + 0.6044702778742779, + 0.7281280567505588, + 0.621498633148778, + -0.46638858081105594, + 0.5747490182709423, + -0.7838087471940858, + 0.7443335658121795, + 0.7171865026755633, + -0.5551325649086711, + 0.6331732111938579, + -0.07939353064211585, + -0.38961826532279886, + 0.5906909983057236, + -0.5448090251844593, + -0.952671130597097, + -0.6137404233445827, + -0.34347609760458697, + 0.7287058840605727, + 0.9337782080967223, + -0.4417500145562572, + 0.2829634772152554, + -0.20064323127987826, + 0.9622993743965202, + 0.07243146495744379, + 0.8784742806494314, + -0.7693164962971448, + 0.9408012220444559, + -0.6428643676550727, + 0.9250686315231109, + -0.46906727495406275, + -0.7831949055705778, + -0.1308724828707113, + 0.4570901213054086, + -0.37264537161001754, + 0.21241770661228654, + 0.022846119338956195, + -0.22960913331054567, + 0.15317608699319907, + -0.4905549877228361, + 0.4175705676683412, + -0.9966174435627411, + 0.8511503309981654, + 0.0769039941855838, + 0.438859998289691, + 0.48390015567895306, + 0.34125700886599897, + -0.27155705643747163, + -0.8600523777473796, + 0.32847536982254466, + -0.3395999279148072, + -0.37216870988328066, + 0.696030559012671, + 0.43950852602790036, + -0.39935546357747165, + -0.3814306755826935, + -0.1832141827615663, + -0.19519922588455074, + -0.40868959494810597, + -0.7454244018816936, + -0.1591073324541834, + 0.880727341460366, + 0.35463589054546585, + 0.8056110914651653, + 0.23102983190276105, + -0.39810025086886935, + 0.09587442627139642, + -0.999188120605425, + -0.4261725662621456, + -0.1402237000203308, + 0.15996956239136395, + 0.30941124740614323, + -0.07002361950597158, + -0.11568040139038516, + -0.5725971980217994, + -0.053627628181347475, + 0.8023616516565084, + 0.5920495202535605, + -0.6606172076038905, + -0.8304089265497565, + 0.030904019830432894, + 0.26588171153159146, + -0.3296234891803982, + 0.6368469290733285, + 0.5022762750814644, + 0.3455913411143341, + -0.5507186680054235, + -0.6017401345468467, + -0.9511492245463473, + -0.5103149118432997, + -0.04972731156238974, + 0.6994753892494638, + -0.8543435416308618, + -0.1711179780045613, + 0.2595307614754274, + -0.6111295265205814, + 0.39270850098100984, + -0.011245661979126131, + -0.5120311208431223, + 0.31211602222356816, + -0.9889103637239365, + 0.5019289532369458, + 0.5400923771480501, + -0.7868254068729221, + -0.14970761211453176, + -0.6482266365869367, + 0.9159320845590795, + 0.03591550088748163, + -0.8995632297187182, + -0.5016034406800567, + 0.6966726947033195, + -0.08707634905965489, + 0.602833203444529, + 0.3351554651727062, + 0.975784906132896, + 0.1909046369388394, + 0.9000792168863119, + 0.782851851620874, + 0.2253046455235257, + 0.4385479225519342, + 0.00955632964880393, + 0.6611383394428301, + 0.09574390122165677, + 0.7944162064665243, + 0.4873108843191698, + -0.05065112635389335, + -0.4816169030699613, + -0.5055205249806809, + 0.27532287355231255, + 0.5316273685943309, + 0.0425996256559642, + 0.2534968739635626, + -0.4508051061649234, + -0.8450332922705284, + -0.428543698273695, + -0.4565697858356308, + -0.3605808631624754, + 0.08030444503691281, + -0.7232518769676886, + -0.5374770405436264, + 0.38789962459810456, + 0.41283828338910444, + -0.8715422985722439, + -0.18480126066682656, + 0.08522228100783069, + -0.1684515317936881, + -0.5863312209711795, + -0.15971296445314875, + 0.8096769566803539, + 0.1681588284084503, + 0.39104597299593613, + 0.7134640646078685, + 0.5311891522361389, + -0.23923794214457672, + -0.9882078328320139, + -0.2964823946563506, + 0.5069502501187717, + 0.7068959011382092, + 0.9068606769402126, + -0.1619574347403563, + 0.49503133795610155, + 0.09226461946767817, + 0.20650517788248268, + -0.5589226113523622, + -0.5611567307571277, + -0.1283280479067268, + -0.9419503601065695, + -0.3277409126032351, + 0.3582837700566994, + -0.19136666172472583, + -0.6699105375929824, + -0.0652197015353797, + -0.7447444054376786, + 0.24451392194812938, + -0.9460670961897246, + -0.21195948732059056, + 0.12878396604954823, + -0.9457959073193751, + 0.2854992960186715, + -0.7286010255388715, + -0.07660311189697655, + -0.8994307330227449, + -0.24179227162372086, + -0.5766794315770369, + -0.3463083902373829, + 0.5224594157880542, + -0.24174756887172744, + 0.5040196471095697, + 0.6638485703105452, + -0.4954569364352388, + -0.8361875344767149, + -0.9612334258999786, + 0.0788380958450674, + 0.9998156570184185, + -0.30007931255963216, + 0.30028818649975, + 0.5624660992217898, + 0.30350931048777885, + 0.5084664081190524, + 0.8992234654319777, + -0.6012786352749342, + -0.9592399653593353, + -0.6952353084304193, + -0.7475580502515875, + 0.33891768923982135, + 0.12793916386003823, + -0.5640709181869927, + 0.3989299424923016, + 0.5337961967124816, + -0.6644217132643955, + 0.21449498778186338, + 0.49585130391057164, + -0.7709342572422047, + 0.6386023486221701, + 0.9294415460681753, + -0.7838025006847869, + -0.948643149005068, + -0.3760855112106096, + 0.3546945737008176, + 0.9163456764117919, + -0.20669111696674536, + 0.4300294100989368, + -0.8480070443138801, + 0.3812288318659607, + 0.2544847912020889, + -0.7961973891080469, + 0.544961769902448, + 0.7005864781775926, + 0.20082322963368826, + -0.7578898698653698, + 0.9676887030293426, + 0.5652706927220392, + -0.30559246938310203, + -0.1432439735305111, + -0.25885824756388764, + 0.011921579354155831, + -0.31753765027743364, + 0.6991512539991547, + 0.6446618361801162, + -0.7889222587120028, + 0.9215751344291572, + 0.27117021220289184, + 0.6574146220049164, + 0.41461728741215387, + -0.1290257099846459, + 0.46759060802678376, + 0.9309474624761553, + -0.45983520722519833, + 0.6163984376135119, + 0.07634581289651154, + -0.033004992233607755, + -0.12885101399221144, + 0.46205242861024454, + -0.46320892390154933, + 0.7034263200386637, + 0.6614620377812068, + -0.8266742038886512, + 0.7632623680055439, + -0.512273121618088, + -0.07058306679353654, + 0.22066340846104127, + -0.24202139174347193, + -0.9426000044598208, + 0.7019056726249182, + -0.6363202856846164, + -0.575760299640522, + 0.595664713656193, + -0.3193223136714327, + 0.7606399595164508, + 0.4023675006644032, + -0.4474628484876302, + -0.9796977711226458, + 0.8961251555540624, + -0.8287740760839575, + 0.4401493282083886, + -0.022844306302425288, + 0.5163293069649328, + 0.38121867889304606, + 0.29180579948190455, + -0.018357329842827674, + 0.5858657362646349, + -0.8138932988906566, + -0.5568071990549397, + 0.3835743105904037, + -0.3875879397398232, + 0.16311117066473435, + -0.05347902248095049, + 0.0618438622915356, + -0.1489923745895705, + 0.49187087342721925, + -0.3384174056081397, + 0.4057098843715379, + -0.45816714604760733, + -0.497192647598117, + -0.7586882304953884, + -0.6148314130969652, + -0.7608905174898943, + 0.07172793076749961, + 0.5243792189682059, + -0.6297003151246761, + -0.5672307202427886, + -0.031602825574582605, + 0.44917000218610315, + 0.9532140457661136, + 0.0492737382172157, + -0.43400259345008996, + -0.7989477838184185, + -0.6117648438178531, + -0.5450336730348906, + -0.6411169126291933, + -0.9717032654878723, + 0.06827017853532658, + -0.4513773464334654, + 0.9485898622054758, + 0.1067179335972166, + 0.3948347858203889, + -0.7474410140831822, + 0.736922394525888, + -0.018242610952442284, + 0.7454394699970692, + 0.1481284392526645, + -0.061206110138979586, + -0.1190624037279795, + -0.6312726592198366, + -0.8972465656328443, + 0.8821271935362065, + -0.04454162546862461, + 0.6442312905988608, + -0.19858511549612046, + -0.8518356594887204, + 0.2588914139039966, + -0.8927818514149337, + -0.7016048310526897, + 0.12567919416912843, + -0.39232897631554464, + 0.9878362454204404, + -0.7630968755770768, + 0.5288868906523885, + 0.21263530248592555, + 0.5814816596868388, + -0.5486257258699307, + 0.04514507026043568, + -0.09897107022249152, + -0.11455799196134331, + 0.7203333316523601, + 0.9800625209335958, + -0.3892395113612792, + 0.24205464214429706, + 0.21926182449026066, + 0.48017861096909, + 0.895180400650756, + -0.5844241883528973, + -0.577949609389214, + 0.32085627438407327, + -0.6858858132382457, + -0.652372903347131, + -0.8498702621976642, + -0.9946485547942285, + -0.09899259076459521, + 0.18762239023912652, + -0.4174814219379448, + -0.5370475308879483, + 0.41391165975806654, + 0.4059751161874343, + -0.09193734718591351, + 0.3747698401424855, + 0.8478220889650974, + 0.5756560532934627, + 0.25011601432844377, + 0.32236608570673564, + 0.8673369168910265, + -0.1497220677915767, + 0.08912475742115733, + 0.2952694468049615, + 0.8168228904252122, + 0.6532623193001192, + -0.8571803262883717, + -0.6681544215585506, + -0.3847763747715349, + 0.4979154441392466, + 0.13841409863818477, + -0.4227788233883165, + -0.7512926836505835, + 0.37735598242418744, + 0.3994673699514346, + 0.8853524814880815, + 0.0009443542358511525, + -0.012409561309831396, + -0.839116296201398, + -0.9202784316327739, + -0.13594267167845486, + -0.35535683328340006, + -0.4992641951759922, + -0.8173462267380296, + 0.9238222043856734, + 0.6719172278123626, + 0.1503982184398609, + 0.9015725556127048, + 0.9991448337549027, + 0.3445631686064785, + -0.4609779480658922, + -0.9195366537107676, + 0.5125376608251142, + -0.05899835349673288, + 0.3030189788673727, + 0.832145575853535, + -0.6370217055537482, + 0.17065925055570874, + 0.26956943890817375, + -0.016548395617973277, + -0.8175151874123456, + -0.3040778874106991, + -0.33338321266981596, + 0.3402670190423709, + 0.7154661888007052, + -0.3403926728420692, + 0.387347347966865, + -0.4235644092770885, + 0.8903870791264212, + 0.6271320694759452, + 0.10019321794356584, + -0.09034818279654111, + -0.37096568565966415, + -0.35345242744801064, + 0.9403694536171543, + -0.19164988599850874, + 0.029192504658210217, + 0.9762384295635083, + 0.31532077296625793, + 0.08518718871439002, + -0.173504858401357, + -0.6248349172109458, + -0.2764412816966495, + 0.5128863081111474, + 0.25081748429001793, + 0.5199810721220339, + -0.5928835232994523, + 0.09843927817066334, + 0.8553455216888954, + -0.1237678098552435, + 0.39650005824434453, + -0.7571478332747603, + 0.9462936316433888, + 0.21774333416392988, + -0.5214050753446993, + -0.6832436723895352, + 0.10167801400533993, + 0.10450281810746609, + -0.8135815974369549, + 0.9845142787904393, + 0.8258597569683359, + -0.0771042116424332, + -0.7650677018208849, + 0.6642863475378025, + -0.003248990585609457, + 0.4332066519835822, + 0.01774403013902348, + -0.4531502065734516, + 0.6694478911533888, + 0.9604892652067509, + -0.5125381878497233, + 0.10253015376093932, + -0.23282797350570017, + 0.8437362998631237, + 0.016481783185787968, + 0.7586525102929522, + 0.7280538688571763, + -0.44750519435591807, + 0.5800123640062269, + -0.17011515289730972, + 0.8684967873650402, + 0.015475351619312905, + 0.6410989463711567, + -0.4343220334343467, + -0.4028883004564656, + 0.17387544482832218, + 0.9978046664586775, + -0.020719306688598005, + -0.7028091632344218, + 0.07716115540774737, + -0.30975211661398494, + 0.10383483414162198, + 0.08686012591734604, + -0.08931076626699785, + -0.35644529954192516, + -0.6226952525857892, + 0.3949968552411427, + 0.14359528396980226, + -0.5328751078842138, + 0.5510889501985177, + -0.9127054018053962, + 0.4894103031303918, + 0.4104557620500051, + 0.6228178051297537, + -0.22784249501803533, + 0.32737765896906934, + 0.6414951034191101, + 0.9616362773195701, + -0.009342700767152934, + -0.9259607773088476, + 0.00458230026593176, + 0.1803608586191341, + 0.7394006267244309, + 0.7483807481086167, + -0.11938758045864573, + 0.05190217361512284, + -0.08614385105151556, + 0.44488765514128525, + -0.1800427605072712, + 0.3095626528554676, + -0.6912775624569023, + -0.061018798030536114, + 0.9384072611484344, + -0.32287753189713664, + 0.3854091971737641, + 0.2996733051585452, + 0.7035305847013829, + 0.7046826731317861, + 0.7186843683365332, + -0.2399812049546899, + -0.36667769213320067, + 0.437434850445966, + 0.5188036186687273, + 0.7447660347970726, + -0.9282018003246268, + -0.8631585056368971, + 0.2623220337092811, + 0.8418581975605934, + 0.9948518459124307, + 0.4935327334758539, + -0.13205705615374863, + -0.8031137472336041, + 0.26749565756158256, + 0.7451584652141814, + -0.11264289667658844, + 0.3880023255864691, + 0.806848124100972, + -0.9080180626455745, + 0.5922869303244236, + -0.4132644480695824, + -0.25031782044937034, + -0.7088604086012031, + 0.06233263629752006, + 0.13185612383187362, + 0.5850389477619065, + -0.6600327018638052, + -0.8420632986800107, + 0.7416791972897097, + 0.2394207370748307, + -0.5183404162873739, + 0.8256580320471096, + -0.7137645597429807, + -0.07770017329000489, + -0.49204532117291544, + -0.4893465836816919, + -0.9812051370902579, + 0.6092661539502926, + 0.802418847197766, + 0.3552217713981589, + -0.6840487558564261, + -0.11654043279237114, + -0.3088687511138366, + 0.17514341025284286, + 0.2778774047200243, + -0.15138212307820464, + -0.4998035511845975, + 0.6906078502851973, + -0.6015660017822153, + -0.23061350206530906, + -0.0335838778815678, + -0.5255885961448508, + 0.14384538470147779, + 0.14962386035729014, + 0.9853840872537951, + -0.4095384922334613, + 0.9558889691537253, + 0.31645963185736137, + -0.45103923964354453, + 0.13185803391139794, + 0.37159898546802417, + 0.48933768233065034, + -0.9019114984480001, + 0.21281298615294908, + -0.006545426952259348, + 0.8083105817874507, + -0.42761169708161106, + 0.5977202390151974, + 0.2141299963285319, + -0.29535808832936694, + 0.27323575601178374, + 0.24178232626060825, + 0.35552891724671887, + 0.44185675334141794, + 0.3183630806335074, + 0.6766742339250331, + 0.25649620737379664, + 0.8068074081459358, + 0.2926812177810456, + -0.38213423209472785, + -0.11835361967437685, + 0.1591476107368055, + 0.4647195358784766, + -0.8197332485075621, + -0.40977909674168944, + 0.49496172987674303, + -0.6487198591113867, + -0.7356804045064329, + 0.07881551796884212, + 0.9429791624226798, + 0.06170474740557008, + 0.8269739489633938, + 0.6609452391347947, + -0.4860598308734754, + 0.6493796250848145, + -0.03630434025251761, + 0.612976987587533, + 0.4931187014340901, + -0.32256949239622745, + -0.7696605851004812, + 0.9257865857377552, + -0.7184859699882347, + 0.9330004189255041, + 0.7202811937976437, + 0.4484334241510364, + 0.9598844855638069, + 0.9345394946000647, + 0.6091752880411239, + -0.26844990118886725, + 0.5813639371778749, + -0.9721626897981976, + 0.07314461653811821, + -0.09042794453225067, + 0.3456567637475072, + 0.3446815947020263, + 0.1691201833043321, + 0.6448346024535485, + 0.8805837835591084, + -0.7833077956015386, + -0.5323561955768337, + -0.9499507007070354, + 0.7684696904297044, + 0.1228147644997577, + 0.8305118174863189, + -0.55726559985202, + -0.8735659176796091, + 0.6477107027808953, + 0.8187752768557841, + -0.39561965094149953, + -0.18340828840917456, + -0.7204459749855578, + 0.8925230657638925, + -0.39127083128798956, + -0.014750762043588272, + -0.8056160027563568, + 0.7745186170570046, + -0.728671902587327, + -0.09271248622221484, + 0.34097243770034247, + 0.4862802430463431, + 0.8919481715588642, + -0.16174649317055434, + 0.48453802953063163, + -0.69095419518033, + -0.17023094512638615, + -0.801956730578943, + -0.02130592442077095, + -0.18376822860459896, + 0.903043050762119, + -0.9345674262899062, + -0.2589400825311292, + -0.1132338278785967, + 0.9011103397028539, + 0.7109003866119092, + -0.8012907507877394, + 0.3713605309625707, + 0.08893172296428986, + 0.9556850589040935, + -0.28265231757536413, + -0.20372071451125384, + -0.6203828756778409, + -0.7556805618254725, + 0.6960663769273621, + -0.09056526285896571, + 0.325537476123956, + 0.2834089344664352, + 0.19429191903909016, + -0.9572850905272587, + 0.5735891809092335, + -0.5128622056719527, + -0.7481522293839142, + 0.12915595181592665, + -0.8627796943512882, + 0.5303147517771689, + -0.585685259306683, + -0.5680972961626505, + 0.7393908535390894, + -0.34288089313553916, + -0.7048916401171126, + 0.8010620712635164, + -0.9943288970399673, + 0.7168122527603527, + -0.710624039358984, + -0.7400157371131264, + -0.49869160654375233, + -0.6510057581972131, + 0.3221152851946336, + -0.9484397004276033, + -0.9702793455385823, + 0.5799693284695078, + -0.5241367878190739, + -0.3524570760759551, + -0.6515075971876456, + -0.895201964277647, + 0.483436113908299, + 0.05217105319573423, + 0.4913305500679914, + -0.04750806915349948, + 0.5560340786284055, + 0.026475915218383328, + -0.781891979992309, + 0.007677379571642717, + 0.8908312859402125, + -0.9132699262016566, + 0.5664539919607592, + 0.7339618155196765, + 0.042902429426168176, + -0.08391495580463593, + 0.9280523662440574, + -0.8783491850109884, + -0.04203617800327342, + -0.19676549097487905, + 0.3721949921244654, + -0.01946229171154945, + 0.8194016582304586, + -0.853018568467099, + -0.8384190451784161, + 0.21659484726687994, + -0.8686355333597722, + -0.4499680008840936, + 0.26615344860203094, + 0.09671286809672108, + -0.3496291133626641, + 0.9892555117218471, + 0.06111367486264907, + -0.09256916484905675, + 0.21085358307062574, + -0.801643076641916, + 0.40355883709213236, + 0.7055854745911501, + 0.3018333297626108, + 0.5379254602094772, + 0.44167983331519833, + -0.5699538673450062, + -0.09689016806943695, + -0.5430128512708312, + -0.3221367623296496, + -0.09300219418510536, + -0.16802069947700948, + -0.8098283214487383, + -0.146471987478084, + 0.3302157261206178, + -0.2513979531289263, + -0.6947221504625525, + 0.8459700714687688, + -0.8657333837267045, + 0.663543776949709, + -0.8135397965926452, + -0.8068711348684288, + 0.4775919969774314, + 0.6235385705547847, + 0.11274147123070022, + 0.1729301654789539, + 0.12317282798414486, + -0.3407080371675899, + -0.7555374292908814, + -0.2928038407324647, + 0.3306810400058309, + 0.5005685005029565, + 0.7361842977381299, + 0.44212135749229886, + 0.9367972506229489, + 0.20082018244935407, + -0.29670762861370203, + 0.15583703677970973, + -0.5745223886559878, + 0.3134726059763042, + -0.5515102617848688, + -0.7835632361454667, + 0.6907468372026901, + -0.2648778987692997, + 0.5252112638736994, + 0.14820000866292538, + 0.6144427423046888, + 0.6903103226567162, + 0.9490932042514164, + 0.6368537190813393, + 0.22714656107092956, + 0.2853983276596628, + -0.9474923370928332, + 0.8581685819898728, + 0.658921579919326, + -0.4651045496917787, + -0.6391678560782912, + 0.40539754573120934, + -0.3820306223401797, + -0.32035068644548326, + -0.9877884211926826, + 0.7397254130728763, + 0.13264218952275253, + -0.19843131200096886, + -0.7162506916974627, + 0.26634402531102763, + -0.9386858032381882, + 0.49222352401141345, + -0.5697342399329781, + -0.1603350124709937, + -0.31820803646133444, + -0.2598938150459922, + 0.4431919354853464, + 0.553671239933482, + 0.1351871132287945, + -0.8300859200456414, + -0.8947823471489564, + -0.6851802057856937, + 0.23567636385206114, + 0.34793742122622606, + -0.4557943129075621, + 0.32387738561652246, + -0.028676590218007503, + -0.11591162660444176, + -0.4536663112704611, + 0.5098862873367414, + -0.7723649837795965, + -0.1401727332042193, + -0.4335070598391013, + 0.35697250952026605, + -0.02673449327149835, + 0.33426511747269916, + -0.909165274791145, + -0.2094732078247421, + 0.19864991388890085, + -0.9846258282002343, + -0.39716127596312645, + -0.5775320415593754, + -0.725530389485346, + -0.4889609919570814, + -0.34375528812431777, + -0.9845401868613761, + 0.4940282468595356, + -0.6486103962483154, + -0.23958510856952753, + 0.40734252676532723, + 0.000524693112426311, + 0.6667084048397565, + 0.6124003731335277, + -0.8558490068156843, + 0.7235287240451771, + -0.9153954768744172, + -0.9625169268287086, + 0.8423248690048248, + 0.7242200272241328, + 0.15151832147366218, + 0.14679936177168607, + 0.41899792313786843, + -0.16461208031303887, + -0.7696532546724035, + -0.9582868819505024, + -0.3504636411089723, + 0.6026443086209043, + 0.23625052660848045, + 0.6640518261434143, + 0.8395395034827693, + -0.8237402374042306, + 0.6889687196293939, + -0.5133670503545327, + 0.17774257660582382, + 0.04792508600126322, + -0.20846660628133407, + -0.37945087632827734, + -0.3209734377030722, + -0.3338627550137361, + -0.6637345839086268, + 0.020966569084296616, + -0.7719467203228949, + 0.019904124645943932, + 0.8118454631601009, + -0.3012494690552394, + 0.4547582113478086, + 0.6378972030497039, + 0.6300740115000283, + -0.5274623021065514, + -0.7071115634446112, + -0.6054563943520335, + 0.20479797054633164, + 0.5204305910937841, + 0.3110180210374782, + -0.6457077421055417, + 0.5456961784951204, + -0.011765949965222733, + 0.5088916504939716, + 0.5197542992154969, + -0.10218948600987399, + 0.8483085167713187, + 0.12898356680559853, + 0.27059663812118684, + 0.2490435588837896, + 0.7284937496628323, + 0.2544348137995436, + -0.6980851972138462, + -0.8634274830084878, + -0.11558387232743028, + -0.3943591297219742, + -0.45065266502768164, + -0.8876557595738439, + 0.014673770579556322, + -0.37918429878737814, + -0.09617227259863559, + -0.8862198983320952, + 0.6633932633262429, + -0.84653799765309, + 0.7285000678504783, + 0.7105867429121806, + 0.23001677683114718, + 0.014135634667910235, + -0.07457668214545543, + 0.10863274267660783, + 0.583635594530717, + 0.7917535311136052, + -0.10053259269818748, + 0.6196318353959422, + 0.30367490929721974, + -0.3569464742391433, + -0.04874194411386101, + -0.6982778466087229, + -0.8762525997977986, + -0.7929962454592168, + 0.7982536679115471, + -0.31312444832346475, + 0.4286310983349253, + 0.009098003001914767, + -0.6548821771272701, + -0.5045125528034315, + -0.12448345138153405, + -0.1211564164747514, + 0.04549607053109117, + -0.6825075840369172, + -0.25429603579734583, + -0.43421284277116423, + -0.18246120550535938, + -0.3232657063171609, + 0.19577172476597893, + 0.5784538631273295, + 0.2946107139386138, + -0.8681762914405651, + -0.8109881049698113, + 0.356758689680162, + -0.43170605224377256, + 0.44746731000492357, + 0.31312817282095584, + 0.8126853943273544, + 0.7465593241210777, + -0.3332759278788868, + 0.1654790291728392, + -0.7171432388313643, + -0.30035842492831155, + 0.9353930153854892, + 0.3969599256237619, + -0.21608403132927934, + 0.19008245630314957, + 0.8760043991315216, + -0.38083622516807814, + -0.24664138788739964, + 0.5833239157270871, + 0.6263695676296019, + 0.340232799989445, + 0.657917945788929, + 0.47754934425896556, + 0.3708288805551525, + 0.05278667946834803, + 0.29204964146697576, + -0.15318726735579258, + -0.2763438073064308, + -0.27480466203591125, + -0.6394741542463107, + -0.5716146775821993, + 0.8953365350686937, + -0.02745815825454545, + -0.5469139069322255, + -0.7248692936458414, + -0.8456698313982474, + 0.6888567773717666, + -0.7977184726642117, + 0.541749440726202, + 0.670239653269112, + 0.7673643309851232, + -0.9245050152848775, + -0.3264712560714369, + 0.5326152088944831, + -0.737901917140984, + -0.24656025835495066, + -0.675505575823099, + 0.662690113378251, + 0.5421956274619768, + 0.6180874392786022, + -0.6689216685118826, + -0.12465318972330652, + -0.17828277700693307, + 0.3527258443770658, + -0.5249395971061499, + -0.1116025803894487, + -0.4301441348732835, + 0.4970730361908726, + -0.10214407393331926, + 0.06802229939652227, + -0.3810642068744208, + 0.6172477421815727, + -0.061968789935494595, + 0.6702267854514148, + -0.26431808354993436, + 0.8942603404882459, + 0.9688795870631546, + -0.07664004311802164, + -0.43645653459244915, + -0.23625513161857903, + 0.05491957692296534, + 0.9325363064118946, + 0.6337824791624802, + 0.6025184483030952, + -0.7232029307931376, + -0.49999357682198586, + 0.2823580724088943, + 0.748233890104474, + 0.10908149084886243, + -0.7948205365031813, + 0.691784553466877, + 0.7023320961695576, + -0.42987397188134313, + 0.5262336605833817, + -0.45441740081728677, + 0.8106124179564824, + -0.7053026880167914, + -0.12505487961003836, + 0.8928265260235213, + -0.5559239861860388, + -0.09774401955858214, + -0.30082984372270216, + -0.9466596183941229, + -0.8934862256578511, + 0.004014229386470891, + -0.5284438522731347, + 0.9890507024765824, + -0.25017465316446486, + -0.9436249089436999, + 0.8616518094999062, + 0.6783525752232114, + 0.29992136858836327, + 0.582761274964352, + -0.7248008245482567, + -0.42624120537346366, + 0.6595231663056451, + 0.39214397715196725, + -0.7224146163636276, + 0.41107235057816083, + -0.10279705203550682, + -0.9894976033887641, + -0.8415484574585574, + -0.4881521431259106, + 0.669926198564762, + 0.09760849088767087, + 0.4544695706498636, + 0.05554301177344856, + -0.7776262793515232, + -0.42379684392153893, + -0.39769761082756006, + -0.9045011067622974, + -0.1603489124931139, + 0.5877982172788763, + -0.08577276672710266, + -0.77828420941997, + 0.810293771323997, + 0.19347808563789393, + -0.9671292955882111, + 0.03075146041881216, + -0.5161237311580134, + -0.7128463195074799, + -0.14152221379333252, + 0.22961916555190132, + -0.5188715223869129, + -0.16686480958292038, + 0.3287426034840182, + -0.8287720900254765, + 0.9493089819044433, + -0.8646413541823079, + 0.052118890644341054, + 0.014655393159573205, + 0.9766629711929349, + 0.10830390483639096, + -0.2190925348798718, + -0.05972984368302803, + 0.2713415829372601, + 0.9620788451031206, + -0.49269947786157453, + -0.9675155377821048, + 0.5770400325590306, + -0.3103950136732174, + 0.4658820429012833, + 0.25651392495131287, + 0.5430027482197182, + 0.4703739696246225, + -0.33496278325603024, + -0.9113286234095836, + 0.09202749041538305, + 0.6270177311121763, + -0.6498217458965831, + 0.558285186956563, + -0.07075420050592451, + 0.3907785039921279, + 0.2634716955166756, + 0.6229953636952128, + -0.8737989259355508, + 0.5523807994068435, + -0.08464084510529357, + -0.4131148576499375, + -0.9123874486817538, + -0.6010603325696233, + -0.916188116139236, + 0.8667419599007946, + 0.030767178508997572, + 0.9782454045922468, + 0.08606139530837176, + -0.4933724695947652, + 0.5065818376377298, + -0.6177931385321782, + -0.2860516479292732, + 0.56168313395685, + 0.7315965541561151, + -0.33615062723731093, + -0.7510499835112332, + -0.26396165137065397, + 0.7789730340245629, + 0.48661541103924244, + 0.7892749899101066, + -0.22671046347861568, + 0.9474471686306181, + -0.007593546925954708, + -0.004953215012745371, + 0.8486209332539272, + 0.03855170706988398, + 0.6022961748035476, + 0.45416264868527145, + -0.8421459878890643, + 0.2049065976604545, + 0.6446825590797733, + 0.09094879468927397, + -0.35757714357081194, + -0.8398621778500095, + 0.3218384429162733, + -0.38700828781849506, + 0.2052432554611996, + -0.14776785423390915, + 0.37952961689097253, + -0.29690603245601577, + -0.915289674299741, + 0.7400743501127842, + -0.2948813793830354, + 0.9963011955460981, + -0.4508892798502848, + 0.9600545583885582, + 0.8958087572061726, + -0.8499176700236815, + 0.2750250757665966, + -0.2733777386980354, + 0.6021919511243399, + 0.35882121562937974, + 0.9055787925592156, + -0.7144410632749005, + 0.21514580664171068, + 0.562623939486933, + -0.930402068402916 + ], + "sample_rate": 48000 +} \ No newline at end of file From 92db0d5adf070767edd917dccf48444afaa81c20 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 27 Jan 2026 08:17:06 -0800 Subject: [PATCH 003/162] Update NAM Core. ModelTest now handles failure to load internal models. --- Utils/ModelTest/ModelTest.cpp | 44 +- Utils/Models/wavenet_a2_max.nam | 819 ++++---------------------------- deps/NeuralAmpModelerCore | 2 +- 3 files changed, 119 insertions(+), 746 deletions(-) diff --git a/Utils/ModelTest/ModelTest.cpp b/Utils/ModelTest/ModelTest.cpp index 13224a0..a4039f9 100644 --- a/Utils/ModelTest/ModelTest.cpp +++ b/Utils/ModelTest/ModelTest.cpp @@ -11,6 +11,13 @@ NeuralModel* LoadModel(std::filesystem::path modelPath, EModelLoadMode loadMode) NeuralModel::SetWaveNetLoadMode(loadMode); NeuralModel::SetLSTMLoadMode(loadMode); + if (!std::filesystem::exists(modelPath)) + { + std::cout << "Model file does not exist: " << modelPath << std::endl; + + return nullptr; + } + try { auto model = NeuralAudio::NeuralModel::CreateFromFile(modelPath); @@ -133,17 +140,20 @@ void RunNAMTests(std::filesystem::path modelPath, int blockSize) double rms; - if (namCoreModel != nullptr) - { - } - std::tuple internal; std::tuple rtNeural; std::tuple namCore; - internal = BenchModel(internalModel, blockSize, numBlocks); + if (internalModel != nullptr) + { + internal = BenchModel(internalModel, blockSize, numBlocks); - std::cout << "Internal: " << std::get<0>(internal) << " (" << std::get<1>(internal) << ")" << std::endl; + std::cout << "Internal: " << std::get<0>(internal) << " (" << std::get<1>(internal) << ")" << std::endl; + } + else + { + std::cout << "Model can't be loaded as internal model" << std::endl; + } if (namCoreModel != nullptr) { @@ -151,11 +161,15 @@ void RunNAMTests(std::filesystem::path modelPath, int blockSize) namCore = BenchModel(namCoreModel, blockSize, numBlocks); - rms = ComputeError(namCoreModel, internalModel, blockSize, numBlocks); - std::cout << "NAM Core: " << std::get<0>(namCore) << " (" << std::get<1>(namCore) << ")" << std::endl; - std::cout << "NAM vs Internal RMS err: " << rms << std::endl; - std::cout << "Internal is: " << (std::get<0>(namCore) / std::get<0>(internal)) << "x NAM" << std::endl; + + if (internalModel != nullptr) + { + rms = ComputeError(namCoreModel, internalModel, blockSize, numBlocks); + + std::cout << "NAM vs Internal RMS err: " << rms << std::endl; + std::cout << "Internal is: " << (std::get<0>(namCore) / std::get<0>(internal)) << "x NAM" << std::endl; + } } if (rtNeuralModel != nullptr) @@ -165,12 +179,16 @@ void RunNAMTests(std::filesystem::path modelPath, int blockSize) rtNeural = BenchModel(rtNeuralModel, blockSize, numBlocks); std::cout << "RTNeural: " << std::get<0>(rtNeural) << " (" << std::get<1>(rtNeural) << ")" << std::endl; - rms = ComputeError(namCoreModel, rtNeuralModel, blockSize, numBlocks); - std::cout << "NAM vs RTNeural RMS err: " << rms << std::endl; if (namCoreModel != nullptr) { - std::cout << "RTNeural is: " << (std::get<0>(namCore) / std::get<0>(rtNeural)) << "x NAM" << std::endl; + rms = ComputeError(namCoreModel, rtNeuralModel, blockSize, numBlocks); + std::cout << "NAM vs RTNeural RMS err: " << rms << std::endl; + + if (namCoreModel != nullptr) + { + std::cout << "RTNeural is: " << (std::get<0>(namCore) / std::get<0>(rtNeural)) << "x NAM" << std::endl; + } } } diff --git a/Utils/Models/wavenet_a2_max.nam b/Utils/Models/wavenet_a2_max.nam index cea7420..e820526 100644 --- a/Utils/Models/wavenet_a2_max.nam +++ b/Utils/Models/wavenet_a2_max.nam @@ -3,7 +3,7 @@ "This model is meant as a 'test case' to contain all of the new features that are being considered for A2.", "It doesn't have slimmability." ], - "version": "0.5.4", + "version": "0.6.0", "metadata": { "date": { "year": 2026, @@ -21,7 +21,7 @@ "architecture": "WaveNet", "config": { "condition_dsp": { - "version": "0.5.4", + "version": "0.6.0", "metadata": { "date": { "year": 2026, @@ -66,35 +66,43 @@ "secondary_activation": "Hardswish", "conv_pre_film": { "active": true, - "shift": true + "shift": true, + "groups": 1 }, "conv_post_film": { "active": true, - "shift": true + "shift": true, + "groups": 1 }, "input_mixin_pre_film": { "active": true, - "shift": true + "shift": true, + "groups": 1 }, "input_mixin_post_film": { "active": true, - "shift": true + "shift": true, + "groups": 1 }, "activation_pre_film": { "active": true, - "shift": true + "shift": true, + "groups": 1 }, "activation_post_film": { "active": true, - "shift": true + "shift": true, + "groups": 1 }, "1x1_post_film": { "active": true, - "shift": true + "shift": true, + "groups": 1 }, "head1x1_post_film": { "active": true, - "shift": true + "shift": true, + "groups": 1 } }, { @@ -109,14 +117,30 @@ 3, 5 ], - "activation": { - "type": "PReLU", - "negative_slopes": [ - 0.04, - 0.05 - ] - }, - "gating_mode": "blended", + "activation": [ + { + "type": "PReLU", + "negative_slopes": [ + 0.04, + 0.05 + ] + }, + { + "type": "PReLU", + "negative_slopes": [ + 0.03, + 0.01 + ] + }, + { + "type": "Softsign" + } + ], + "gating_mode": [ + "blended", + "gated", + "gated" + ], "head_bias": false, "groups_input": 1, "groups_input_mixin": 1, @@ -126,44 +150,56 @@ "out_channels": 4, "groups": 2 }, - "secondary_activation": { - "type": "LeakyHardtanh", - "min_val": 0.0, - "max_val": 0.9, - "min_slope": 0.0, - "max_slope": 0.02 - }, + "secondary_activation": [ + { + "type": "LeakyHardtanh", + "min_val": 0.0, + "max_val": 0.9, + "min_slope": 0.0, + "max_slope": 0.02 + }, + "ReLU", + "Sigmoid" + ], "conv_pre_film": { "active": true, - "shift": false + "shift": false, + "groups": 1 }, "conv_post_film": { "active": true, - "shift": false + "shift": false, + "groups": 1 }, "input_mixin_pre_film": { "active": true, - "shift": false + "shift": false, + "groups": 1 }, "input_mixin_post_film": { "active": true, - "shift": false + "shift": false, + "groups": 1 }, "activation_pre_film": { "active": true, - "shift": false + "shift": false, + "groups": 1 }, "activation_post_film": { "active": true, - "shift": false + "shift": false, + "groups": 1 }, "1x1_post_film": { "active": true, - "shift": false + "shift": false, + "groups": 1 }, "head1x1_post_film": { "active": true, - "shift": false + "shift": false, + "groups": 1 } } ], @@ -1239,8 +1275,7 @@ 2 ], "activation": { - "type": "PReLU", - "negative_slope": 0.015 + "type": "Softsign" }, "gating_mode": "none", "head_bias": true, @@ -1255,35 +1290,43 @@ "secondary_activation": "", "conv_pre_film": { "active": true, - "shift": true + "shift": true, + "groups": 2 }, "conv_post_film": { "active": true, - "shift": true + "shift": true, + "groups": 4 }, "input_mixin_pre_film": { "active": true, - "shift": true + "shift": true, + "groups": 4 }, "input_mixin_post_film": { "active": true, - "shift": true + "shift": true, + "groups": 2 }, "activation_pre_film": { "active": true, - "shift": true + "shift": true, + "groups": 1 }, "activation_post_film": { "active": true, - "shift": true + "shift": true, + "groups": 2 }, "1x1_post_film": { "active": true, - "shift": true + "shift": true, + "groups": 8 }, "head1x1_post_film": { "active": true, - "shift": true + "shift": true, + "groups": 4 } } ], @@ -2108,695 +2151,7 @@ 0.14359528396980226, -0.5328751078842138, 0.5510889501985177, - -0.9127054018053962, - 0.4894103031303918, - 0.4104557620500051, - 0.6228178051297537, - -0.22784249501803533, - 0.32737765896906934, - 0.6414951034191101, - 0.9616362773195701, - -0.009342700767152934, - -0.9259607773088476, - 0.00458230026593176, - 0.1803608586191341, - 0.7394006267244309, - 0.7483807481086167, - -0.11938758045864573, - 0.05190217361512284, - -0.08614385105151556, - 0.44488765514128525, - -0.1800427605072712, - 0.3095626528554676, - -0.6912775624569023, - -0.061018798030536114, - 0.9384072611484344, - -0.32287753189713664, - 0.3854091971737641, - 0.2996733051585452, - 0.7035305847013829, - 0.7046826731317861, - 0.7186843683365332, - -0.2399812049546899, - -0.36667769213320067, - 0.437434850445966, - 0.5188036186687273, - 0.7447660347970726, - -0.9282018003246268, - -0.8631585056368971, - 0.2623220337092811, - 0.8418581975605934, - 0.9948518459124307, - 0.4935327334758539, - -0.13205705615374863, - -0.8031137472336041, - 0.26749565756158256, - 0.7451584652141814, - -0.11264289667658844, - 0.3880023255864691, - 0.806848124100972, - -0.9080180626455745, - 0.5922869303244236, - -0.4132644480695824, - -0.25031782044937034, - -0.7088604086012031, - 0.06233263629752006, - 0.13185612383187362, - 0.5850389477619065, - -0.6600327018638052, - -0.8420632986800107, - 0.7416791972897097, - 0.2394207370748307, - -0.5183404162873739, - 0.8256580320471096, - -0.7137645597429807, - -0.07770017329000489, - -0.49204532117291544, - -0.4893465836816919, - -0.9812051370902579, - 0.6092661539502926, - 0.802418847197766, - 0.3552217713981589, - -0.6840487558564261, - -0.11654043279237114, - -0.3088687511138366, - 0.17514341025284286, - 0.2778774047200243, - -0.15138212307820464, - -0.4998035511845975, - 0.6906078502851973, - -0.6015660017822153, - -0.23061350206530906, - -0.0335838778815678, - -0.5255885961448508, - 0.14384538470147779, - 0.14962386035729014, - 0.9853840872537951, - -0.4095384922334613, - 0.9558889691537253, - 0.31645963185736137, - -0.45103923964354453, - 0.13185803391139794, - 0.37159898546802417, - 0.48933768233065034, - -0.9019114984480001, - 0.21281298615294908, - -0.006545426952259348, - 0.8083105817874507, - -0.42761169708161106, - 0.5977202390151974, - 0.2141299963285319, - -0.29535808832936694, - 0.27323575601178374, - 0.24178232626060825, - 0.35552891724671887, - 0.44185675334141794, - 0.3183630806335074, - 0.6766742339250331, - 0.25649620737379664, - 0.8068074081459358, - 0.2926812177810456, - -0.38213423209472785, - -0.11835361967437685, - 0.1591476107368055, - 0.4647195358784766, - -0.8197332485075621, - -0.40977909674168944, - 0.49496172987674303, - -0.6487198591113867, - -0.7356804045064329, - 0.07881551796884212, - 0.9429791624226798, - 0.06170474740557008, - 0.8269739489633938, - 0.6609452391347947, - -0.4860598308734754, - 0.6493796250848145, - -0.03630434025251761, - 0.612976987587533, - 0.4931187014340901, - -0.32256949239622745, - -0.7696605851004812, - 0.9257865857377552, - -0.7184859699882347, - 0.9330004189255041, - 0.7202811937976437, - 0.4484334241510364, - 0.9598844855638069, - 0.9345394946000647, - 0.6091752880411239, - -0.26844990118886725, - 0.5813639371778749, - -0.9721626897981976, - 0.07314461653811821, - -0.09042794453225067, - 0.3456567637475072, - 0.3446815947020263, - 0.1691201833043321, - 0.6448346024535485, - 0.8805837835591084, - -0.7833077956015386, - -0.5323561955768337, - -0.9499507007070354, - 0.7684696904297044, - 0.1228147644997577, - 0.8305118174863189, - -0.55726559985202, - -0.8735659176796091, - 0.6477107027808953, - 0.8187752768557841, - -0.39561965094149953, - -0.18340828840917456, - -0.7204459749855578, - 0.8925230657638925, - -0.39127083128798956, - -0.014750762043588272, - -0.8056160027563568, - 0.7745186170570046, - -0.728671902587327, - -0.09271248622221484, - 0.34097243770034247, - 0.4862802430463431, - 0.8919481715588642, - -0.16174649317055434, - 0.48453802953063163, - -0.69095419518033, - -0.17023094512638615, - -0.801956730578943, - -0.02130592442077095, - -0.18376822860459896, - 0.903043050762119, - -0.9345674262899062, - -0.2589400825311292, - -0.1132338278785967, - 0.9011103397028539, - 0.7109003866119092, - -0.8012907507877394, - 0.3713605309625707, - 0.08893172296428986, - 0.9556850589040935, - -0.28265231757536413, - -0.20372071451125384, - -0.6203828756778409, - -0.7556805618254725, - 0.6960663769273621, - -0.09056526285896571, - 0.325537476123956, - 0.2834089344664352, - 0.19429191903909016, - -0.9572850905272587, - 0.5735891809092335, - -0.5128622056719527, - -0.7481522293839142, - 0.12915595181592665, - -0.8627796943512882, - 0.5303147517771689, - -0.585685259306683, - -0.5680972961626505, - 0.7393908535390894, - -0.34288089313553916, - -0.7048916401171126, - 0.8010620712635164, - -0.9943288970399673, - 0.7168122527603527, - -0.710624039358984, - -0.7400157371131264, - -0.49869160654375233, - -0.6510057581972131, - 0.3221152851946336, - -0.9484397004276033, - -0.9702793455385823, - 0.5799693284695078, - -0.5241367878190739, - -0.3524570760759551, - -0.6515075971876456, - -0.895201964277647, - 0.483436113908299, - 0.05217105319573423, - 0.4913305500679914, - -0.04750806915349948, - 0.5560340786284055, - 0.026475915218383328, - -0.781891979992309, - 0.007677379571642717, - 0.8908312859402125, - -0.9132699262016566, - 0.5664539919607592, - 0.7339618155196765, - 0.042902429426168176, - -0.08391495580463593, - 0.9280523662440574, - -0.8783491850109884, - -0.04203617800327342, - -0.19676549097487905, - 0.3721949921244654, - -0.01946229171154945, - 0.8194016582304586, - -0.853018568467099, - -0.8384190451784161, - 0.21659484726687994, - -0.8686355333597722, - -0.4499680008840936, - 0.26615344860203094, - 0.09671286809672108, - -0.3496291133626641, - 0.9892555117218471, - 0.06111367486264907, - -0.09256916484905675, - 0.21085358307062574, - -0.801643076641916, - 0.40355883709213236, - 0.7055854745911501, - 0.3018333297626108, - 0.5379254602094772, - 0.44167983331519833, - -0.5699538673450062, - -0.09689016806943695, - -0.5430128512708312, - -0.3221367623296496, - -0.09300219418510536, - -0.16802069947700948, - -0.8098283214487383, - -0.146471987478084, - 0.3302157261206178, - -0.2513979531289263, - -0.6947221504625525, - 0.8459700714687688, - -0.8657333837267045, - 0.663543776949709, - -0.8135397965926452, - -0.8068711348684288, - 0.4775919969774314, - 0.6235385705547847, - 0.11274147123070022, - 0.1729301654789539, - 0.12317282798414486, - -0.3407080371675899, - -0.7555374292908814, - -0.2928038407324647, - 0.3306810400058309, - 0.5005685005029565, - 0.7361842977381299, - 0.44212135749229886, - 0.9367972506229489, - 0.20082018244935407, - -0.29670762861370203, - 0.15583703677970973, - -0.5745223886559878, - 0.3134726059763042, - -0.5515102617848688, - -0.7835632361454667, - 0.6907468372026901, - -0.2648778987692997, - 0.5252112638736994, - 0.14820000866292538, - 0.6144427423046888, - 0.6903103226567162, - 0.9490932042514164, - 0.6368537190813393, - 0.22714656107092956, - 0.2853983276596628, - -0.9474923370928332, - 0.8581685819898728, - 0.658921579919326, - -0.4651045496917787, - -0.6391678560782912, - 0.40539754573120934, - -0.3820306223401797, - -0.32035068644548326, - -0.9877884211926826, - 0.7397254130728763, - 0.13264218952275253, - -0.19843131200096886, - -0.7162506916974627, - 0.26634402531102763, - -0.9386858032381882, - 0.49222352401141345, - -0.5697342399329781, - -0.1603350124709937, - -0.31820803646133444, - -0.2598938150459922, - 0.4431919354853464, - 0.553671239933482, - 0.1351871132287945, - -0.8300859200456414, - -0.8947823471489564, - -0.6851802057856937, - 0.23567636385206114, - 0.34793742122622606, - -0.4557943129075621, - 0.32387738561652246, - -0.028676590218007503, - -0.11591162660444176, - -0.4536663112704611, - 0.5098862873367414, - -0.7723649837795965, - -0.1401727332042193, - -0.4335070598391013, - 0.35697250952026605, - -0.02673449327149835, - 0.33426511747269916, - -0.909165274791145, - -0.2094732078247421, - 0.19864991388890085, - -0.9846258282002343, - -0.39716127596312645, - -0.5775320415593754, - -0.725530389485346, - -0.4889609919570814, - -0.34375528812431777, - -0.9845401868613761, - 0.4940282468595356, - -0.6486103962483154, - -0.23958510856952753, - 0.40734252676532723, - 0.000524693112426311, - 0.6667084048397565, - 0.6124003731335277, - -0.8558490068156843, - 0.7235287240451771, - -0.9153954768744172, - -0.9625169268287086, - 0.8423248690048248, - 0.7242200272241328, - 0.15151832147366218, - 0.14679936177168607, - 0.41899792313786843, - -0.16461208031303887, - -0.7696532546724035, - -0.9582868819505024, - -0.3504636411089723, - 0.6026443086209043, - 0.23625052660848045, - 0.6640518261434143, - 0.8395395034827693, - -0.8237402374042306, - 0.6889687196293939, - -0.5133670503545327, - 0.17774257660582382, - 0.04792508600126322, - -0.20846660628133407, - -0.37945087632827734, - -0.3209734377030722, - -0.3338627550137361, - -0.6637345839086268, - 0.020966569084296616, - -0.7719467203228949, - 0.019904124645943932, - 0.8118454631601009, - -0.3012494690552394, - 0.4547582113478086, - 0.6378972030497039, - 0.6300740115000283, - -0.5274623021065514, - -0.7071115634446112, - -0.6054563943520335, - 0.20479797054633164, - 0.5204305910937841, - 0.3110180210374782, - -0.6457077421055417, - 0.5456961784951204, - -0.011765949965222733, - 0.5088916504939716, - 0.5197542992154969, - -0.10218948600987399, - 0.8483085167713187, - 0.12898356680559853, - 0.27059663812118684, - 0.2490435588837896, - 0.7284937496628323, - 0.2544348137995436, - -0.6980851972138462, - -0.8634274830084878, - -0.11558387232743028, - -0.3943591297219742, - -0.45065266502768164, - -0.8876557595738439, - 0.014673770579556322, - -0.37918429878737814, - -0.09617227259863559, - -0.8862198983320952, - 0.6633932633262429, - -0.84653799765309, - 0.7285000678504783, - 0.7105867429121806, - 0.23001677683114718, - 0.014135634667910235, - -0.07457668214545543, - 0.10863274267660783, - 0.583635594530717, - 0.7917535311136052, - -0.10053259269818748, - 0.6196318353959422, - 0.30367490929721974, - -0.3569464742391433, - -0.04874194411386101, - -0.6982778466087229, - -0.8762525997977986, - -0.7929962454592168, - 0.7982536679115471, - -0.31312444832346475, - 0.4286310983349253, - 0.009098003001914767, - -0.6548821771272701, - -0.5045125528034315, - -0.12448345138153405, - -0.1211564164747514, - 0.04549607053109117, - -0.6825075840369172, - -0.25429603579734583, - -0.43421284277116423, - -0.18246120550535938, - -0.3232657063171609, - 0.19577172476597893, - 0.5784538631273295, - 0.2946107139386138, - -0.8681762914405651, - -0.8109881049698113, - 0.356758689680162, - -0.43170605224377256, - 0.44746731000492357, - 0.31312817282095584, - 0.8126853943273544, - 0.7465593241210777, - -0.3332759278788868, - 0.1654790291728392, - -0.7171432388313643, - -0.30035842492831155, - 0.9353930153854892, - 0.3969599256237619, - -0.21608403132927934, - 0.19008245630314957, - 0.8760043991315216, - -0.38083622516807814, - -0.24664138788739964, - 0.5833239157270871, - 0.6263695676296019, - 0.340232799989445, - 0.657917945788929, - 0.47754934425896556, - 0.3708288805551525, - 0.05278667946834803, - 0.29204964146697576, - -0.15318726735579258, - -0.2763438073064308, - -0.27480466203591125, - -0.6394741542463107, - -0.5716146775821993, - 0.8953365350686937, - -0.02745815825454545, - -0.5469139069322255, - -0.7248692936458414, - -0.8456698313982474, - 0.6888567773717666, - -0.7977184726642117, - 0.541749440726202, - 0.670239653269112, - 0.7673643309851232, - -0.9245050152848775, - -0.3264712560714369, - 0.5326152088944831, - -0.737901917140984, - -0.24656025835495066, - -0.675505575823099, - 0.662690113378251, - 0.5421956274619768, - 0.6180874392786022, - -0.6689216685118826, - -0.12465318972330652, - -0.17828277700693307, - 0.3527258443770658, - -0.5249395971061499, - -0.1116025803894487, - -0.4301441348732835, - 0.4970730361908726, - -0.10214407393331926, - 0.06802229939652227, - -0.3810642068744208, - 0.6172477421815727, - -0.061968789935494595, - 0.6702267854514148, - -0.26431808354993436, - 0.8942603404882459, - 0.9688795870631546, - -0.07664004311802164, - -0.43645653459244915, - -0.23625513161857903, - 0.05491957692296534, - 0.9325363064118946, - 0.6337824791624802, - 0.6025184483030952, - -0.7232029307931376, - -0.49999357682198586, - 0.2823580724088943, - 0.748233890104474, - 0.10908149084886243, - -0.7948205365031813, - 0.691784553466877, - 0.7023320961695576, - -0.42987397188134313, - 0.5262336605833817, - -0.45441740081728677, - 0.8106124179564824, - -0.7053026880167914, - -0.12505487961003836, - 0.8928265260235213, - -0.5559239861860388, - -0.09774401955858214, - -0.30082984372270216, - -0.9466596183941229, - -0.8934862256578511, - 0.004014229386470891, - -0.5284438522731347, - 0.9890507024765824, - -0.25017465316446486, - -0.9436249089436999, - 0.8616518094999062, - 0.6783525752232114, - 0.29992136858836327, - 0.582761274964352, - -0.7248008245482567, - -0.42624120537346366, - 0.6595231663056451, - 0.39214397715196725, - -0.7224146163636276, - 0.41107235057816083, - -0.10279705203550682, - -0.9894976033887641, - -0.8415484574585574, - -0.4881521431259106, - 0.669926198564762, - 0.09760849088767087, - 0.4544695706498636, - 0.05554301177344856, - -0.7776262793515232, - -0.42379684392153893, - -0.39769761082756006, - -0.9045011067622974, - -0.1603489124931139, - 0.5877982172788763, - -0.08577276672710266, - -0.77828420941997, - 0.810293771323997, - 0.19347808563789393, - -0.9671292955882111, - 0.03075146041881216, - -0.5161237311580134, - -0.7128463195074799, - -0.14152221379333252, - 0.22961916555190132, - -0.5188715223869129, - -0.16686480958292038, - 0.3287426034840182, - -0.8287720900254765, - 0.9493089819044433, - -0.8646413541823079, - 0.052118890644341054, - 0.014655393159573205, - 0.9766629711929349, - 0.10830390483639096, - -0.2190925348798718, - -0.05972984368302803, - 0.2713415829372601, - 0.9620788451031206, - -0.49269947786157453, - -0.9675155377821048, - 0.5770400325590306, - -0.3103950136732174, - 0.4658820429012833, - 0.25651392495131287, - 0.5430027482197182, - 0.4703739696246225, - -0.33496278325603024, - -0.9113286234095836, - 0.09202749041538305, - 0.6270177311121763, - -0.6498217458965831, - 0.558285186956563, - -0.07075420050592451, - 0.3907785039921279, - 0.2634716955166756, - 0.6229953636952128, - -0.8737989259355508, - 0.5523807994068435, - -0.08464084510529357, - -0.4131148576499375, - -0.9123874486817538, - -0.6010603325696233, - -0.916188116139236, - 0.8667419599007946, - 0.030767178508997572, - 0.9782454045922468, - 0.08606139530837176, - -0.4933724695947652, - 0.5065818376377298, - -0.6177931385321782, - -0.2860516479292732, - 0.56168313395685, - 0.7315965541561151, - -0.33615062723731093, - -0.7510499835112332, - -0.26396165137065397, - 0.7789730340245629, - 0.48661541103924244, - 0.7892749899101066, - -0.22671046347861568, - 0.9474471686306181, - -0.007593546925954708, - -0.004953215012745371, - 0.8486209332539272, - 0.03855170706988398, - 0.6022961748035476, - 0.45416264868527145, - -0.8421459878890643, - 0.2049065976604545, - 0.6446825590797733, - 0.09094879468927397, - -0.35757714357081194, - -0.8398621778500095, - 0.3218384429162733, - -0.38700828781849506, - 0.2052432554611996, - -0.14776785423390915, - 0.37952961689097253, - -0.29690603245601577, - -0.915289674299741, - 0.7400743501127842, - -0.2948813793830354, - 0.9963011955460981, - -0.4508892798502848, - 0.9600545583885582, - 0.8958087572061726, - -0.8499176700236815, - 0.2750250757665966, - -0.2733777386980354, - 0.6021919511243399, - 0.35882121562937974, - 0.9055787925592156, - -0.7144410632749005, - 0.21514580664171068, - 0.562623939486933, - -0.930402068402916 + -0.9127054018053962 ], "sample_rate": 48000 } \ No newline at end of file diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index 925da40..7050f78 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit 925da40bec6fb9a45594ca7fe7e0e330601d26e3 +Subproject commit 7050f780022884f09bff5bc739c983e7ae13a36a From 3ffa5ee740d3c8fcfbcd95c4e4b7b767a459705b Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 27 Feb 2026 08:51:11 -0800 Subject: [PATCH 004/162] Update to latest NAM Core --- deps/NeuralAmpModelerCore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index 7050f78..20a04fc 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit 7050f780022884f09bff5bc739c983e7ae13a36a +Subproject commit 20a04fcf466dc4233730412b120e5bbad72402c3 From 7e4efeb0f4e35573591ccbe92a03d4042f8ae396 Mon Sep 17 00:00:00 2001 From: mikeoliphant Date: Thu, 12 Mar 2026 10:34:08 +0700 Subject: [PATCH 005/162] Add option to enable NAM Core inline GEMM. Update NAM Core. --- NeuralAudio/CMakeLists.txt | 10 +++++++++- deps/NeuralAmpModelerCore | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 866fb34..d08f6b0 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -19,6 +19,14 @@ else() message(STATUS "NOT Building NAM Core implementation") endif() +option(NAM_USE_INLINE_GEMM "Use inline matrix multiplications in NAM Core" OFF) +if(NAM_USE_INLINE_GEMM) + message(STATUS "Using NAM Core inline matrix multiplications") + add_definitions(-DNAM_USE_INLINE_GEMM) +else() + message(STATUS "NOT Using NAM Core inline matrix multiplications") +endif() + set(LSTM_MATH "FastMath" CACHE STRING "LSTM math functions") add_definitions(-DLSTM_MATH=${LSTM_MATH}) message(STATUS "LSTM math is: ${LSTM_MATH}") @@ -103,4 +111,4 @@ target_link_libraries(NeuralAudio LINK_PUBLIC RTNeural math_approx) source_group(NeuralAudio ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES}) source_group(NAM ${CMAKE_CURRENT_SOURCE_DIR} FILES ${NAM_SOURCES}) -source_group(RTNeural-NAM ${CMAKE_CURRENT_SOURCE_DIR} FILES ${RTNEURAL_WN_SOURCES}) \ No newline at end of file +source_group(RTNeural-NAM ${CMAKE_CURRENT_SOURCE_DIR} FILES ${RTNEURAL_WN_SOURCES}) diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index 20a04fc..32d43dd 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit 20a04fcf466dc4233730412b120e5bbad72402c3 +Subproject commit 32d43dda35dc7faf195ed4fbf64ed19dcdd19776 From c7a3d62b82b7145e96f7bfeea989272650dc0b4d Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 12 Mar 2026 12:50:27 +0700 Subject: [PATCH 006/162] Added option to enable inline matrix multiplication in NAM Core. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 3ff664e..7558683 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,8 @@ Note - you'll have to change the Visual Studio version if you are using a differ ```-DBUILD_NAMCORE=ON```: Support loading models using the NAM Core implemenations. +```-DNAM_USE_INLINE_GEMM=ON```: Enable use of inline matrix multiplication in NAM Core. + ```-DBUILD_STATIC_RTNEURAL=ON```: Build static RTNeural model architectures (slower compile, larger size - only use if you plan on forcing RTNeural model loading). ```-DWAVENET_FRAMES=XXX```: Sample buffer size for the internal WaveNet implementation. Defaults to **64**. If you know you are going to be using a fixed sample buffer smaller or larger than this, use that instead. Note that the model will still be able to process any buffer size - it is just optimized for this size. From f046c433c33fe85c12319ed8ec9f9eda98fca4b8 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 12 Mar 2026 14:35:54 +0700 Subject: [PATCH 007/162] Switch build NAM Core option to default ON --- NeuralAudio/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index d08f6b0..2d2538e 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -11,7 +11,7 @@ else() message(STATUS "NOT Building static RTNeural models") endif() -option(BUILD_NAMCORE "Build NAM Core" OFF) +option(BUILD_NAMCORE "Build NAM Core" ON) if(BUILD_NAMCORE) message(STATUS "Building NAM Core implementation") add_definitions(-DBUILD_NAMCORE) From b75c0d3923cab836928446b83031ba062d2cbb0d Mon Sep 17 00:00:00 2001 From: mikeoliphant Date: Fri, 13 Mar 2026 09:11:46 +0700 Subject: [PATCH 008/162] Update NAM Core --- deps/NeuralAmpModelerCore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index 32d43dd..a90dd14 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit 32d43dda35dc7faf195ed4fbf64ed19dcdd19776 +Subproject commit a90dd14364146d6bff85fde75d0890b16b263f9d From 17842f2d1da8829a8d9271c7e219870d39208c00 Mon Sep 17 00:00:00 2001 From: Florian Paul Schmidt Date: Mon, 13 Apr 2026 09:20:11 +0200 Subject: [PATCH 009/162] only use leftCols(numFrames) --- NeuralAudio/WaveNet.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index d8d2054..d5caf23 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -223,9 +223,9 @@ namespace NeuralAudio // data[pos] = WAVENET_MATH::Tanh(data[pos]); //} - const_cast&>(headInput).noalias() += block.topRows(Channels); + const_cast&>(headInput).leftCols(numFrames).noalias() += block; - oneByOne.Process(block.topRows(Channels), const_cast&>(output).middleCols(outputStart, numFrames)); + oneByOne.Process(block, const_cast&>(output).middleCols(outputStart, numFrames)); const_cast&>(output).middleCols(outputStart, numFrames).noalias() += layerBuffer.middleCols(bufferStart, numFrames); } @@ -439,4 +439,4 @@ namespace NeuralAudio Eigen::Matrix headArray; float headScale; }; -} \ No newline at end of file +} From 31ffaa538983c266528f30c663964ff9743e2fec Mon Sep 17 00:00:00 2001 From: Florian Paul Schmidt Date: Mon, 13 Apr 2026 09:25:44 +0200 Subject: [PATCH 010/162] Ust std::runtime_error instead of assert to signal wrong number of weights --- NeuralAudio/WaveNet.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index d8d2054..12ecc5d 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -371,7 +371,12 @@ namespace NeuralAudio headScale = *(it++); - assert(std::distance(weights.begin(), it) == (long)weights.size()); + if (std::distance(weights.begin(), it) != (long)weights.size()) + { + std::stringstream str; + str << "Wrong number of weights. Remaining: " << std::distance(weights.begin(), it); + throw std::runtime_error(str.str()); + } } size_t GetMaxFrames() @@ -439,4 +444,4 @@ namespace NeuralAudio Eigen::Matrix headArray; float headScale; }; -} \ No newline at end of file +} From 0cd5c1a9937f060d560e7a0cbcdd79d10e4bd7fb Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 13 Apr 2026 08:32:44 -0700 Subject: [PATCH 011/162] Don't do leftCols() ahead of time - rely on layer Process() to do the correct thing --- NeuralAudio/WaveNet.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 481e890..cf27ed1 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -394,11 +394,11 @@ namespace NeuralAudio { if constexpr (layerIndex == 0) { - std::get(layerArrays).Prewarm(condition, condition, headArray.leftCols(1)); + std::get(layerArrays).Prewarm(condition, condition, headArray); } else { - std::get(layerArrays).Prewarm(std::get(layerArrays).arrayOutputs.leftCols(1), condition, std::get(layerArrays).headOutputs.leftCols(1)); + std::get(layerArrays).Prewarm(std::get(layerArrays).arrayOutputs, condition, std::get(layerArrays).headOutputs); } }); } @@ -415,11 +415,11 @@ namespace NeuralAudio { if constexpr (layerIndex == 0) { - std::get(layerArrays).Process(condition, condition, headArray.leftCols(numFrames), numFrames); + std::get(layerArrays).Process(condition, condition, headArray, numFrames); } else { - std::get(layerArrays).Process(std::get(layerArrays).arrayOutputs.leftCols(numFrames), condition, std::get(layerArrays).headOutputs.leftCols(numFrames), numFrames); + std::get(layerArrays).Process(std::get(layerArrays).arrayOutputs, condition, std::get(layerArrays).headOutputs, numFrames); } }); From bb367177aad495de03298e5469389e3fd94f28e1 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 13 Apr 2026 10:48:39 -0700 Subject: [PATCH 012/162] Formatting --- NeuralAudio/WaveNet.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index cf27ed1..a8ee597 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -371,12 +371,12 @@ namespace NeuralAudio headScale = *(it++); - if (std::distance(weights.begin(), it) != (long)weights.size()) - { - std::stringstream str; - str << "Wrong number of weights. Remaining: " << std::distance(weights.begin(), it); - throw std::runtime_error(str.str()); - } + if (std::distance(weights.begin(), it) != (long)weights.size()) + { + std::stringstream str; + str << "Wrong number of weights. Remaining: " << std::distance(weights.begin(), it); + throw std::runtime_error(str.str()); + } } size_t GetMaxFrames() From 89fb36a15cc652e3f79921c3a6a6157a432dfd77 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 14 Apr 2026 06:53:18 -0700 Subject: [PATCH 013/162] Put lefCols() back in Dense layer Process calls --- NeuralAudio/WaveNet.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index a8ee597..7d20f60 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -299,7 +299,7 @@ namespace NeuralAudio template void Prewarm(const Eigen::MatrixBase& layerInputs, const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInputs) { - rechannel.Process(layerInputs, std::get<0>(layers).layerBuffer.middleCols(std::get<0>(layers).bufferStart, 1)); + rechannel.Process(layerInputs.leftCols(1), std::get<0>(layers).layerBuffer.middleCols(std::get<0>(layers).bufferStart, 1)); ForEachIndex([&](auto layerIndex) { @@ -315,13 +315,13 @@ namespace NeuralAudio } }); - headRechannel.Process(headInputs, headOutputs.leftCols(1)); + headRechannel.Process(headInputs.leftCols(1), headOutputs.leftCols(1)); } template void Process(const Eigen::MatrixBase& layerInputs, const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInputs, const size_t numFrames) { - rechannel.Process(layerInputs, std::get<0>(layers).layerBuffer.middleCols(std::get<0>(layers).bufferStart, numFrames)); + rechannel.Process(layerInputs.leftCols(numFrames), std::get<0>(layers).layerBuffer.middleCols(std::get<0>(layers).bufferStart, numFrames)); ForEachIndex([&](auto layerIndex) { @@ -337,7 +337,7 @@ namespace NeuralAudio std::get(layers).AdvanceFrames(numFrames); }); - headRechannel.Process(headInputs, headOutputs.leftCols(numFrames)); + headRechannel.Process(headInputs.leftCols(numFrames), headOutputs.leftCols(numFrames)); } }; From 14fb7bfb3b47f696e520142c676dae9aabf28387 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 14 Apr 2026 13:00:05 -0700 Subject: [PATCH 014/162] Removed redundant returns. Check for correct number of weights up front. --- NeuralAudio/InternalModel.h | 2 -- NeuralAudio/RTNeuralModel.h | 2 -- NeuralAudio/WaveNet.h | 58 ++++++++++++++++++++++++++++++++----- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index ee41d2a..8658c80 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -20,8 +20,6 @@ namespace NeuralAudio ReadKerasConfig(modelJson); return CreateModelFromKerasJson(modelJson); - - return true; } virtual bool CreateModelFromKerasJson(const nlohmann::json& modelJson) diff --git a/NeuralAudio/RTNeuralModel.h b/NeuralAudio/RTNeuralModel.h index b207fff..63aba73 100644 --- a/NeuralAudio/RTNeuralModel.h +++ b/NeuralAudio/RTNeuralModel.h @@ -45,8 +45,6 @@ namespace NeuralAudio ReadKerasConfig(modelJson); return CreateModelFromKerasJson(modelJson); - - return true; } virtual bool CreateModelFromKerasJson(const nlohmann::json& modelJson) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 7d20f60..bf3f47b 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -26,6 +26,11 @@ namespace NeuralAudio class Conv1DT { public: + size_t GetNumWeights() + { + return OutChannels * InChannels * KernelSize + (DoBias ? OutChannels : 0); + } + void SetWeights(std::vector::iterator& inWeights) { weights.resize(KernelSize); @@ -70,6 +75,11 @@ namespace NeuralAudio class DenseLayerT { public: + size_t GetNumWeights() + { + return OutSize * InSize + (DoBias ? OutSize : 0); + } + void SetWeights(std::vector::iterator& inWeights) { for (size_t i = 0; i < OutSize; i++) @@ -168,6 +178,11 @@ namespace NeuralAudio #endif } + size_t GetNumWeights() + { + return conv1D.GetNumWeights() + inputMixin.GetNumWeights() + oneByOne.GetNumWeights(); + } + void SetWeights(std::vector::iterator& weights) { conv1D.SetWeights(weights); @@ -284,6 +299,20 @@ namespace NeuralAudio return allocNum; } + size_t GetNumWeights() + { + size_t numWeights = rechannel.GetNumWeights(); + + ForEachIndex([&](auto layerIndex) + { + numWeights += std::get(layers).GetNumWeights(); + }); + + numWeights += headRechannel.GetNumWeights(); + + return numWeights; + } + void SetWeights(std::vector::iterator& weights) { rechannel.SetWeights(weights); @@ -357,26 +386,41 @@ namespace NeuralAudio allocNum = std::get(layerArrays).AllocBuffers(allocNum); }); - } - void SetWeights(std::vector weights) + size_t GetNumWeights() { - std::vector::iterator it = weights.begin(); + size_t numWeights = 0; ForEachIndex([&](auto layerIndex) { - std::get(layerArrays).SetWeights(it); + numWeights += std::get(layerArrays).GetNumWeights(); }); - headScale = *(it++); + numWeights++; // headScale; + + return numWeights; + } + + void SetWeights(std::vector weights) + { + size_t numWeights = GetNumWeights(); - if (std::distance(weights.begin(), it) != (long)weights.size()) + if (numWeights != weights.size()) { std::stringstream str; - str << "Wrong number of weights. Remaining: " << std::distance(weights.begin(), it); + str << "Wrong number of weights. Expected " << numWeights << " but got " << weights.size(); throw std::runtime_error(str.str()); } + + std::vector::iterator it = weights.begin(); + + ForEachIndex([&](auto layerIndex) + { + std::get(layerArrays).SetWeights(it); + }); + + headScale = *(it++); } size_t GetMaxFrames() From 81ac2f2cc8064b81f55ed064e3824fddeba30024 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 16 Apr 2026 15:25:06 -0700 Subject: [PATCH 015/162] Fix cmake --- NeuralAudio/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 222ca69..a2936b9 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -25,6 +25,8 @@ if(NAM_USE_INLINE_GEMM) add_definitions(-DNAM_USE_INLINE_GEMM) else() message(STATUS "NOT Using NAM Core inline matrix multiplications") +endif() + option(BUILD_INTERNAL_STATIC_WAVENET "Build Internal static WaveNet models" ON) if(BUILD_INTERNAL_STATIC_WAVENET) message(STATUS "Building Internal static WaveNet models") From 61f3dee730420991b9fe2a6fe8f8e74c330665bd Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 16 Apr 2026 15:33:51 -0700 Subject: [PATCH 016/162] Update to latest NAM Core --- NeuralAudio/CMakeLists.txt | 8 ++++++-- deps/NeuralAmpModelerCore | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index a2936b9..7cdc137 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -95,11 +95,15 @@ if(BUILD_NAMCORE) ../deps/NeuralAmpModelerCore/NAM/registry.h ../deps/NeuralAmpModelerCore/NAM/ring_buffer.h ../deps/NeuralAmpModelerCore/NAM/ring_buffer.cpp + ../deps/NeuralAmpModelerCore/NAM/slimmable.h ../deps/NeuralAmpModelerCore/NAM/lstm.cpp ../deps/NeuralAmpModelerCore/NAM/dsp.h ../deps/NeuralAmpModelerCore/NAM/dsp.cpp - ../deps/NeuralAmpModelerCore/NAM/wavenet.cpp - ../deps/NeuralAmpModelerCore/NAM/wavenet.h) + ../deps/NeuralAmpModelerCore/NAM/wavenet/slimmable.cpp + ../deps/NeuralAmpModelerCore/NAM/wavenet/detail.h + ../deps/NeuralAmpModelerCore/NAM/wavenet/params.h + ../deps/NeuralAmpModelerCore/NAM/wavenet/model.h + ../deps/NeuralAmpModelerCore/NAM/wavenet/model.cpp) endif() if(BUILD_STATIC_RTNEURAL) diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index a90dd14..8b5a7ce 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit a90dd14364146d6bff85fde75d0890b16b263f9d +Subproject commit 8b5a7cef1c8a4b6faed26a5767e1fc553f0f55f5 From e0a20bfe05b58e65b770e7fb5fbd87c69507b4b6 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 16 Apr 2026 15:47:01 -0700 Subject: [PATCH 017/162] NAM A2 updates --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7eeef98..b60cc46 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ NeuralAudio is a C++ library designed to make it easy to use neural network mach NeuralAudio currently supports the following model types: -- [Neural Amp Modeler](https://github.com/sdatkinson/neural-amp-modeler) (NAM) WaveNet and LSTM models +- [Neural Amp Modeler](https://github.com/sdatkinson/neural-amp-modeler) (NAM) WaveNet and LSTM models, A1 and A2 support - [RTNeural](https://github.com/jatinchowdhury18/RTNeural) keras models (LSTM, GRU) # Underlying Libraries and Performance @@ -17,7 +17,7 @@ For completeness, and to facilitate accuracy and performance benchmarking, it ca The internal NeuralAudio implmentation currently outperforms the other implementations on all tested platforms (Windows x64, Linux x64/Arm64). It also uses significantly less memory than the NAM Core WaveNet implementation (which, for example, uses about 10x as much memory for a "standard" WaveNet model). -For WaveNet, the internal implmeentation supports optimized static models of the offical NAM network architectures: "Standard", "Lite", "Feather", "Nano". +For WaveNet, the internal implmeentation supports optimized static models of the offical NAM A1 network architectures: "Standard", "Lite", "Feather", "Nano". For LSTM, the internal implementation supports optimized static models architectures for 1x8, 1x12, 1x16, 1x24, 2x8, 2x12, and 2x16 models. @@ -25,6 +25,8 @@ All NAM files with WaveNet and LSTM architectures not supported internally will All keras models not supported internally will fall back to the RTNeural implmentation. +NAM A2 models currently use the NAM Core implementation. + # API overview To load a model: @@ -86,7 +88,7 @@ NeuralAudio::EModelLoadMode::RTNeural You can check which implementation was actually used to load the model with ```model->GetLoadMode()```. -**NOTE:** Because of compile time and executable size considerations, only the internal and dynamic RTNeural implementations are built by default. If you want to use RTNeural, it is recommended that you add ```-DBUILD_STATIC_RTNEURAL=ON``` to your cmake commandline. This will create static model implmentations for the same sets of WaveNet and LSTM models as the internal implmentation, and results in increased performance. If you want to use NAM Core, add ```DBUILD_NAMCORE=ON``` to your cmake commandline. +**NOTE:** Because of compile time and executable size considerations, only the internal, NAM Core and dynamic RTNeural implementations are built by default. If you want to use RTNeural, it is recommended that you add ```-DBUILD_STATIC_RTNEURAL=ON``` to your cmake commandline. This will create static model implmentations for the same sets of WaveNet and LSTM models as the internal implmentation, and results in increased performance. ## Getting the model receptive field size From 8d79bf065657859e25c9a6212b7fc2ab72200c4d Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 16 Apr 2026 15:50:23 -0700 Subject: [PATCH 018/162] Update LSTM notes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b60cc46..f81093a 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ NeuralAudio::EModelLoadMode::RTNeural You can check which implementation was actually used to load the model with ```model->GetLoadMode()```. -**NOTE:** Because of compile time and executable size considerations, only the internal, NAM Core and dynamic RTNeural implementations are built by default. If you want to use RTNeural, it is recommended that you add ```-DBUILD_STATIC_RTNEURAL=ON``` to your cmake commandline. This will create static model implmentations for the same sets of WaveNet and LSTM models as the internal implmentation, and results in increased performance. +**NOTE:** Because of compile time and executable size considerations, only the internal, NAM Core and dynamic RTNeural implementations are built by default. If you want to use RTNeural, it is recommended that you add ```-DBUILD_STATIC_RTNEURAL=ON``` to your cmake commandline. This will create static model implmentations for the same sets of WaveNet and LSTM models as the internal implmentation, and results in increased performance. Interal static LSTM model support is also off by default - to turn it on use ```-DBUILD_INTERNAL_STATIC_LSTM=ON```. ## Getting the model receptive field size From 001eb9a275d7569a1cbfa1899e13087d4674dac7 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 25 Apr 2026 14:17:37 -0700 Subject: [PATCH 019/162] Add neural_tilde to list of projects --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f81093a..6c60faa 100644 --- a/README.md +++ b/README.md @@ -161,3 +161,4 @@ The following applications and devices are using the NeuralAudio library for mod - [stompbox](https://github.com/mikeoliphant/stompbox): Guitar amplification and effects pedalboard simulation. - [NeuralRack](https://github.com/brummer10/NeuralRack): Neural Model and Impulse Response File loader for Linux/Windows. - [Darkglass Anagram](https://www.darkglass.com/creation/anagram): Bass guitar effects unit. +- [neural_tilde](https://github.com/apresta/neural_tilde): Max/MSP external for running neural amplifier captures From 4510ba2034be1f8a40a07a2d77c53f850d31a638 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 27 Apr 2026 09:40:05 -0700 Subject: [PATCH 020/162] Fix issues with floating point metadata with no decimal places not being parsed correctly --- NeuralAudio/NeuralModel.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index b3ad0d7..e186640 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -339,17 +339,17 @@ namespace NeuralAudio { nlohmann::json metaData = modelJson.at("metadata"); - if (metaData.contains("loudness") && metaData.at("loudness").is_number_float()) + if (metaData.contains("loudness") && metaData.at("loudness").is_number()) { modelLoudnessDB = (float)metaData.at("loudness"); } - if (metaData.contains("input_level_dbu") && metaData.at("input_level_dbu").is_number_float()) + if (metaData.contains("input_level_dbu") && metaData.at("input_level_dbu").is_number()) { modelInputLevelDBu = metaData.at("input_level_dbu"); } - if (metaData.contains("output_level_dbu") && metaData.at("output_level_dbu").is_number_float()) + if (metaData.contains("output_level_dbu") && metaData.at("output_level_dbu").is_number()) { modelOutputLevelDBu = metaData.at("output_level_dbu"); } @@ -363,12 +363,12 @@ namespace NeuralAudio sampleRate = modelJson.at("samplerate"); } - if (modelJson.contains("in_gain") && modelJson.at("in_gain").is_number_float()) + if (modelJson.contains("in_gain") && modelJson.at("in_gain").is_number()) { modelInputLevelDBu = modelJson.at("in_gain"); } - if (modelJson.contains("out_gain") && modelJson.at("out_gain").is_number_float()) + if (modelJson.contains("out_gain") && modelJson.at("out_gain").is_number()) { modelLoudnessDB = -18 - (float)modelJson.at("out_gain"); } From 701ab02d16b286094b0bab25212d6b1391604a02 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 27 Apr 2026 11:49:02 -0700 Subject: [PATCH 021/162] Add license section to README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6c60faa..95d039d 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,10 @@ NeuralAudio currently supports the following model types: - [Neural Amp Modeler](https://github.com/sdatkinson/neural-amp-modeler) (NAM) WaveNet and LSTM models, A1 and A2 support - [RTNeural](https://github.com/jatinchowdhury18/RTNeural) keras models (LSTM, GRU) +# License + +This repository is licensed under the [MIT license](https://github.com/mikeoliphant/NeuralAudio/blob/main/LICENSE). It is a liberal license, but please make sure that you comply with the terms - as well as the terms of [this project's dependencies](https://github.com/mikeoliphant/NeuralAudio/blob/main/CREDITS.md). I would also appreciate it if you would let me know if you are using this library. + # Underlying Libraries and Performance By default, NeuralAudio uses its own implementation of WaveNet and LSTM network models. This implementation has been designed to produce **exactly** the same output as the [NAM Core library](https://github.com/sdatkinson/NeuralAmpModelerCore), but with increased performance and reduced memory usage. From f0f3b8accf1e832d710309a6fe9d156d699541a2 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 30 Apr 2026 09:22:18 -0700 Subject: [PATCH 022/162] Use compile-time loop over convolution kernel. Lazy product seems to be faster for small matrices and when we're adding bias. --- NeuralAudio/WaveNet.h | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index bf3f47b..a014f3f 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -50,17 +50,27 @@ namespace NeuralAudio template inline void Process(const Eigen::MatrixBase& input, Eigen::MatrixBase const & output, const size_t iStart, const size_t nCols) const { - for (size_t k = 0; k < KernelSize; k++) + ForEachIndex([&](auto k) { auto offset = Dilation * ((int)k + 1 - KernelSize); auto inBlock = input.middleCols(iStart + offset, nCols); - if (k == 0) - const_cast&>(output).noalias() = weights[k] * inBlock; + if constexpr ((InChannels <= 4) || DoBias) + { + if constexpr (k == 0) + const_cast&>(output).noalias() = weights[k].lazyProduct(inBlock); + else + const_cast&>(output).noalias() += weights[k].lazyProduct(inBlock); + } else - const_cast&>(output).noalias() += weights[k] * inBlock; - } + { + if constexpr (k == 0) + const_cast&>(output).noalias() = weights[k] * inBlock; + else + const_cast&>(output).noalias() += weights[k] * inBlock; + } + }); if constexpr (DoBias) const_cast&>(output).colwise() += bias; From 1ab7a6a8bf1cf03c26652b2a3dcc04060713acc2 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 5 May 2026 08:57:53 -0700 Subject: [PATCH 023/162] Update NAM Core. Add support for A2 fast path wavenet. --- NeuralAudio/CMakeLists.txt | 18 ++++++++++++++---- deps/NeuralAmpModelerCore | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 7cdc137..09a0062 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -24,7 +24,15 @@ if(NAM_USE_INLINE_GEMM) message(STATUS "Using NAM Core inline matrix multiplications") add_definitions(-DNAM_USE_INLINE_GEMM) else() - message(STATUS "NOT Using NAM Core inline matrix multiplications") + message(STATUS "NOT using NAM Core inline matrix multiplications") +endif() + +option(NAM_ENABLE_A2_FAST "Use NAM A2 fast-path WaveNet" ON) +if(NAM_ENABLE_A2_FAST) + message(STATUS "Using NAM A2 fast-path WaveNet") + add_definitions(-DNAM_ENABLE_A2_FAST) +else() + message(STATUS "NOT using NAM A2 fast-path WaveNet") endif() option(BUILD_INTERNAL_STATIC_WAVENET "Build Internal static WaveNet models" ON) @@ -32,7 +40,7 @@ if(BUILD_INTERNAL_STATIC_WAVENET) message(STATUS "Building Internal static WaveNet models") add_definitions(-DBUILD_INTERNAL_STATIC_WAVENET) else() - message(STATUS "NOT Building Internal static WaveNet models") + message(STATUS "NOT building Internal static WaveNet models") endif() option(BUILD_INTERNAL_STATIC_LSTM "Build Internal static LSTM models" ON) @@ -40,7 +48,7 @@ if(BUILD_INTERNAL_STATIC_LSTM) message(STATUS "Building Internal static LSTM models") add_definitions(-DBUILD_INTERNAL_STATIC_LSTM) else() - message(STATUS "NOT Building Internal static LSTM models") + message(STATUS "NOT building Internal static LSTM models") endif() set(LSTM_MATH "FastMath" CACHE STRING "LSTM math functions") @@ -103,7 +111,9 @@ if(BUILD_NAMCORE) ../deps/NeuralAmpModelerCore/NAM/wavenet/detail.h ../deps/NeuralAmpModelerCore/NAM/wavenet/params.h ../deps/NeuralAmpModelerCore/NAM/wavenet/model.h - ../deps/NeuralAmpModelerCore/NAM/wavenet/model.cpp) + ../deps/NeuralAmpModelerCore/NAM/wavenet/model.cpp + ../deps/NeuralAmpModelerCore/NAM/wavenet/a2_fast.h + ../deps/NeuralAmpModelerCore/NAM/wavenet/a2_fast.cpp) endif() if(BUILD_STATIC_RTNEURAL) diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index 8b5a7ce..ab72c07 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit 8b5a7cef1c8a4b6faed26a5767e1fc553f0f55f5 +Subproject commit ab72c078cd141be60d963d357bd501d56cc9606b From 3133c3ad44205b6af4f35246120346c562c03650 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 5 May 2026 09:01:06 -0700 Subject: [PATCH 024/162] Add NAM_ENABLE_A2_FAST cmake option --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 95d039d..2140259 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,9 @@ Note - you'll have to change the Visual Studio version if you are using a differ ```-DBUILD_NAMCORE=ON|OFF```: Support loading models using the NAM Core implemenations. -```-DNAM_USE_INLINE_GEMM=ON```: Enable use of inline matrix multiplication in NAM Core. +```-DNAM_USE_INLINE_GEMM=ON|OFF```: Enable use of inline matrix multiplication in NAM Core. + +```-DNAM_ENABLE_A2_FAST=ON|OFF```: Enable use of A2 fast path wavenet in NAM Core. ```-DBUILD_STATIC_RTNEURAL=ON|OFF```: Build static RTNeural model architectures (slower compile, larger size - only use if you plan on forcing RTNeural model loading). From 37b515554171a7424baf2c7129f644bf7c74860b Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 5 May 2026 10:31:18 -0700 Subject: [PATCH 025/162] Added model quality scaling --- NeuralAudio/NAMModel.h | 42 +++++++++++++++++++++++++++++++++++++++ NeuralAudio/NeuralModel.h | 20 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index a9c5ff7..178ed0b 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace NeuralAudio { @@ -14,6 +15,8 @@ namespace NeuralAudio NAMModel() { nam::activations::Activation::enable_fast_tanh(); + + slimmableSize = defaultQualityScaleFactor; } ~NAMModel() @@ -36,9 +39,46 @@ namespace NeuralAudio namModel = nam::get_dsp(modelJson); + auto* slim = dynamic_cast(namModel.get()); + + if (slim != nullptr) + { + isSlimmable = true; + + slim->SetSlimmableSize(slimmableSize); + } + return true; } + bool HasQualityScaling() + { + return isSlimmable; + } + + float GetQualityScaleFactor() + { + return slimmableSize; + } + + void SetQualityScaleFactor(float scaleFactor) + { + if (HasQualityScaling()) + { + if (slimmableSize != scaleFactor) + { + slimmableSize = scaleFactor; + + if (namModel != nullptr) + { + auto* slim = dynamic_cast(namModel.get()); + + slim->SetSlimmableSize(slimmableSize); + } + } + } + } + void Process(float* input, float* output, size_t numSamples) { namModel->process(&input, &output, (int)numSamples); @@ -51,5 +91,7 @@ namespace NeuralAudio private: std::unique_ptr namModel = nullptr; + float slimmableSize = 1.0f; + bool isSlimmable = false; }; } \ No newline at end of file diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index 0e1483b..c60abc5 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -56,11 +56,30 @@ namespace NeuralAudio defaultMaxAudioBufferSize = maxSize; } + static void SetDefaultQualityScaleFactor(float scaleFactor) + { + defaultQualityScaleFactor = scaleFactor; + } + virtual EModelLoadMode GetLoadMode() { return EModelLoadMode::Internal; } + virtual bool HasQualityScaling() + { + return false; + } + + virtual float GetQualityScaleFactor() + { + return 1.0f; + } + + virtual void SetQualityScaleFactor(float scaleFactor) + { + } + virtual bool IsStatic() { return false; @@ -115,6 +134,7 @@ namespace NeuralAudio inline static EModelLoadMode lstmLoadMode = EModelLoadMode::Internal; inline static EModelLoadMode wavenetLoadMode = EModelLoadMode::Internal; inline static int defaultMaxAudioBufferSize = 128; + inline static float defaultQualityScaleFactor = 1.0f; void Prewarm(size_t numSamples, size_t blockSize) { From 5b570afa35a38253a655e64d1728def07c9a58c0 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 5 May 2026 10:40:02 -0700 Subject: [PATCH 026/162] Document model quality scaling factor usage --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.md b/README.md index 2140259..a341924 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,36 @@ You can check which implementation was actually used to load the model with ```m **NOTE:** Because of compile time and executable size considerations, only the internal, NAM Core and dynamic RTNeural implementations are built by default. If you want to use RTNeural, it is recommended that you add ```-DBUILD_STATIC_RTNEURAL=ON``` to your cmake commandline. This will create static model implmentations for the same sets of WaveNet and LSTM models as the internal implmentation, and results in increased performance. Interal static LSTM model support is also off by default - to turn it on use ```-DBUILD_INTERNAL_STATIC_LSTM=ON```. +## Setting model quality scaling factor + +Some models (notably, slimmable NAM A2 models) support quality scaling - trading off quality for performance. + +Quality scaling is a floating point range from 0.0 (highest performance) to 1.0 (highest quality). + +To set the default quality scaling factor, do: + +``` +NeuralAudio::NeuralModel::SetDefaultQualityScaleFactor(scaleFactor); +``` + +To check if a model supports quality scaling, do: + +``` +if (model->HasQualityScaling()) ... +``` + +To set the quality scaling factor for a model, do: + +``` +model->SetQualityScaleFactor(scaleFactor); +``` + +To get the quality scaling factor for a model, do: + +``` +float scaleFactor = model->GetQualityScaleFactor(); +``` + ## Getting the model receptive field size WaveNet models have a fixed receptive field size (ie: size of the input that the output depends on). From 8cd830f6773c7b6aae625a1a9893e7dfcd481f21 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 8 May 2026 07:33:02 -0700 Subject: [PATCH 027/162] Add NAM Core container source files --- NeuralAudio/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 09a0062..30f3434 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -107,6 +107,8 @@ if(BUILD_NAMCORE) ../deps/NeuralAmpModelerCore/NAM/lstm.cpp ../deps/NeuralAmpModelerCore/NAM/dsp.h ../deps/NeuralAmpModelerCore/NAM/dsp.cpp + ../deps/NeuralAmpModelerCore/NAM/container.cpp + ../deps/NeuralAmpModelerCore/NAM/container.h ../deps/NeuralAmpModelerCore/NAM/wavenet/slimmable.cpp ../deps/NeuralAmpModelerCore/NAM/wavenet/detail.h ../deps/NeuralAmpModelerCore/NAM/wavenet/params.h From 50df68dd1ab7302124fbe0ef573d0102d1a7da10 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 8 May 2026 07:41:56 -0700 Subject: [PATCH 028/162] Add quality scaling option to ModelTest --- Utils/ModelTest/ModelTest.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Utils/ModelTest/ModelTest.cpp b/Utils/ModelTest/ModelTest.cpp index dcc6e5e..9896582 100644 --- a/Utils/ModelTest/ModelTest.cpp +++ b/Utils/ModelTest/ModelTest.cpp @@ -270,7 +270,13 @@ int main(int argc, char* argv[]) .required() .help("Specify the number of iterations") .scan<'i', int>(); - + + program.add_argument("-q", "--quality_scale") + .default_value(1.0f) + .nargs(1) + .required() + .help("Quality scaling (0.0 is fasteset, 1.0 is highest quality") + .scan<'g', float>(); try { program.parse_args(argc, argv); @@ -289,7 +295,13 @@ int main(int argc, char* argv[]) blockSize = program.get("--block_size"); - std::cout << "Block size: " << blockSize << std::endl; + float qualityScale = 1.0f; + + qualityScale = program.get("--quality_scale"); + + NeuralModel::SetDefaultQualityScaleFactor(qualityScale); + + std::cout << "Block size: " << blockSize << " Quality Scale: " << qualityScale << std::endl; if (!modelPath.empty()) { From 05d9fbdc958cc0873b28e7dbb29376f23b39047d Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 13 May 2026 10:39:21 -0700 Subject: [PATCH 029/162] show max block time multiplied by total blocks in ModelTest benchmark --- Utils/ModelTest/ModelTest.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Utils/ModelTest/ModelTest.cpp b/Utils/ModelTest/ModelTest.cpp index 9896582..25f33d7 100644 --- a/Utils/ModelTest/ModelTest.cpp +++ b/Utils/ModelTest/ModelTest.cpp @@ -82,6 +82,8 @@ static std::tuple BenchModel(NeuralModel* model, int blockSize, double tot = std::chrono::duration_cast> (end - start).count(); + maxBlock *= numBlocks; + return std::tie(tot, maxBlock); } From b0d1c8c068ac99e766adfd90463b82729d3325e7 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 18 May 2026 08:20:25 -0700 Subject: [PATCH 030/162] Set NAM Core max buffer size --- NeuralAudio/InternalModel.h | 8 ++++---- NeuralAudio/NAMModel.h | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index 8658c80..f31d21d 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -96,7 +96,7 @@ namespace NeuralAudio return true; } - void SetMaxAudioBufferSize(int maxSize) + void SetMaxAudioBufferSize(const int maxSize) { (void)maxSize; } @@ -214,7 +214,7 @@ namespace NeuralAudio return true; } - void SetMaxAudioBufferSize(int maxSize) + void SetMaxAudioBufferSize(const int maxSize) { model->SetMaxFrames(maxSize); } @@ -353,7 +353,7 @@ namespace NeuralAudio return true; } - void SetMaxAudioBufferSize(int maxSize) + void SetMaxAudioBufferSize(const int maxSize) { (void)maxSize; } @@ -516,7 +516,7 @@ namespace NeuralAudio return true; } - void SetMaxAudioBufferSize(int maxSize) + void SetMaxAudioBufferSize(const int maxSize) { (void)maxSize; } diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index 178ed0b..723f706 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -48,6 +48,8 @@ namespace NeuralAudio slim->SetSlimmableSize(slimmableSize); } + SetMaxAudioBufferSize(defaultMaxAudioBufferSize); + return true; } @@ -79,6 +81,11 @@ namespace NeuralAudio } } + void SetMaxAudioBufferSize(const int maxSize) + { + namModel->Reset(namModel->GetExpectedSampleRate(), maxSize); + } + void Process(float* input, float* output, size_t numSamples) { namModel->process(&input, &output, (int)numSamples); From 9be7ba7ad4cb22258a57265e048ce86beab1607f Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 18 May 2026 08:50:58 -0700 Subject: [PATCH 031/162] Add DEFAULT_QUALITY_SCALE cmake option --- NeuralAudio/CMakeLists.txt | 12 +++++++++--- NeuralAudio/NeuralModel.h | 6 +++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 30f3434..0aa0d74 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -43,6 +43,10 @@ else() message(STATUS "NOT building Internal static WaveNet models") endif() +set(WAVENET_MATH "FastMath" CACHE STRING "WaveNet math functions") +add_definitions(-DWAVENET_MATH=${WAVENET_MATH}) +message(STATUS "WaveNet math is: ${WAVENET_MATH}") + option(BUILD_INTERNAL_STATIC_LSTM "Build Internal static LSTM models" ON) if(BUILD_INTERNAL_STATIC_LSTM) message(STATUS "Building Internal static LSTM models") @@ -55,9 +59,11 @@ set(LSTM_MATH "FastMath" CACHE STRING "LSTM math functions") add_definitions(-DLSTM_MATH=${LSTM_MATH}) message(STATUS "LSTM math is: ${LSTM_MATH}") -set(WAVENET_MATH "FastMath" CACHE STRING "WaveNet math functions") -add_definitions(-DWAVENET_MATH=${WAVENET_MATH}) -message(STATUS "WaveNet math is: ${WAVENET_MATH}") +set(DEFAULT_QUALITY_SCALE "1.0" CACHE STRING "Default model quality scale factor") + +add_definitions(-DDEFAULT_QUALITY_SCALE=${DEFAULT_QUALITY_SCALE}) + +message(STATUS "Default model quality scale factor is: ${DEFAULT_QUALITY_SCALE}") set(WAVENET_FRAMES "64" CACHE STRING "WaveNet frame size") diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index c60abc5..3763f0b 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -4,6 +4,10 @@ #include #include "json.hpp" +#ifndef DEFAULT_QUALITY_SCALE +#define DEFAULT_QUALITY_SCALE 1.0 +#endif + namespace NeuralAudio { enum EModelLoadMode @@ -134,7 +138,7 @@ namespace NeuralAudio inline static EModelLoadMode lstmLoadMode = EModelLoadMode::Internal; inline static EModelLoadMode wavenetLoadMode = EModelLoadMode::Internal; inline static int defaultMaxAudioBufferSize = 128; - inline static float defaultQualityScaleFactor = 1.0f; + inline static float defaultQualityScaleFactor = DEFAULT_QUALITY_SCALE; void Prewarm(size_t numSamples, size_t blockSize) { From 2bbce1b9d51b1f1870d9a868ba78e67ece2f5fee Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 18 May 2026 08:53:50 -0700 Subject: [PATCH 032/162] Update README with DEFAULT_QUALITY_SCALE option --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a341924..bd35bce 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,8 @@ Note - you'll have to change the Visual Studio version if you are using a differ ```-DBUILD_INTERNAL_STATIC_LSTM=ON|OFF```: Build internal static LSTM model architectures (faster internal LSTM, but slower compile, larger size). +```-DDEFAULT_QUALITY_SCALE="X.X"```: Default model quality scale factor (0.0 to 1.0). Be sure to use quotes around value. Defaults to "1.0". + ```-DWAVENET_FRAMES=XXX```: Sample buffer size for the internal WaveNet implementation. Defaults to **64**. If you know you are going to be using a fixed sample buffer smaller or larger than this, use that instead. Note that the model will still be able to process any buffer size - it is just optimized for this size. ```-DBUFFER_PADDING=XXX```: Amount of padding to convolution layer buffers. This allows ring buffer resets to be staggered accross layers to improve performance. It also uses a significant amount of memory. It is set to **24** by default. It can be set all the way down to 0 to reduce memory usage. From 48a78f85f2a70e719a8654b2059de55203f95a48 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 18 May 2026 09:55:11 -0700 Subject: [PATCH 033/162] Added metadata interface --- NeuralAudio/NeuralModel.cpp | 7 +++++++ NeuralAudio/NeuralModel.h | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index e186640..87c85f0 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -330,6 +330,13 @@ namespace NeuralAudio void NeuralModel::ReadNAMConfig(const nlohmann::json& modelJson) { + if (modelJson.contains("metadata")) + { + nlohmann::json metadataJson = modelJson.at("metadata"); + + AddMetadata(metadataJson); + } + if (modelJson.contains("sample_rate") && modelJson.at("sample_rate").is_number()) { sampleRate = modelJson.at("sample_rate"); diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index 3763f0b..bab4f8b 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -82,6 +82,7 @@ namespace NeuralAudio virtual void SetQualityScaleFactor(float scaleFactor) { + (void)scaleFactor; } virtual bool IsStatic() @@ -89,7 +90,7 @@ namespace NeuralAudio return false; } - virtual void SetMaxAudioBufferSize(int maxSize) + virtual void SetMaxAudioBufferSize(const int maxSize) { (void)maxSize; } @@ -114,6 +115,21 @@ namespace NeuralAudio return -1; // No fixed receptive field size (ie: for LSTM) } + virtual std::string GetMetadata(std::string fieldName) + { + auto it = std::find_if(metadata.begin(), metadata.end(), [&](const auto& pair) + { + return pair.first == fieldName; + }); + + if (it != metadata.end()) + { + return it->second; + } + + return ""; + } + virtual void Process(float* input, float* output, size_t numSamples) { (void)input; @@ -133,6 +149,7 @@ namespace NeuralAudio float modelOutputLevelDBu = 12; float modelLoudnessDB = -18; float sampleRate = 48000; + std::vector> metadata; inline static float audioInputLevelDBu = 12; inline static EModelLoadMode lstmLoadMode = EModelLoadMode::Internal; @@ -140,6 +157,22 @@ namespace NeuralAudio inline static int defaultMaxAudioBufferSize = 128; inline static float defaultQualityScaleFactor = DEFAULT_QUALITY_SCALE; + void AddMetadata(std::string fieldName, std::string fieldValue) + { + metadata.push_back({fieldName, fieldValue}); + } + + void AddMetadata(const nlohmann::json& metadataJson) + { + for (auto& [key, value] : metadataJson.items()) + { + if (!value.is_null()) + { + AddMetadata(key, value.dump()); + } + } + } + void Prewarm(size_t numSamples, size_t blockSize) { std::vector input; From 9255a053be5126e20504b695eefcfb954db05504 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 18 May 2026 10:15:19 -0700 Subject: [PATCH 034/162] Update README.md --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index bd35bce..fea9d73 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,22 @@ To get the quality scaling factor for a model, do: float scaleFactor = model->GetQualityScaleFactor(); ``` +## Getting model metadata + +To retrieve arbitrary metadata fields from models that contain them, do: + +``` +std::string fieldName = "this_is_a_field_name"; + +std::string metadataValue = model->GetMetadata(fieldName); + +if (!metadataValue.empty()) +{ + // do something +} +``` +Results are always returned a strings. Field names are case sensitive. + ## Getting the model receptive field size WaveNet models have a fixed receptive field size (ie: size of the input that the output depends on). From 82a09b19daae0e53614347026861413352467309 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 18 May 2026 10:17:32 -0700 Subject: [PATCH 035/162] Pass metadata field by ref --- NeuralAudio/NeuralModel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index bab4f8b..5d59ab7 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -115,7 +115,7 @@ namespace NeuralAudio return -1; // No fixed receptive field size (ie: for LSTM) } - virtual std::string GetMetadata(std::string fieldName) + virtual std::string GetMetadata(const std::string& fieldName) { auto it = std::find_if(metadata.begin(), metadata.end(), [&](const auto& pair) { From 580fd1f0c68089e87a84a0e4aa66a930f3d90f9b Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 18 May 2026 12:03:35 -0700 Subject: [PATCH 036/162] Added GetModelVersion --- NeuralAudio/NeuralModel.cpp | 2 ++ NeuralAudio/NeuralModel.h | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 87c85f0..ad754ef 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -330,6 +330,8 @@ namespace NeuralAudio void NeuralModel::ReadNAMConfig(const nlohmann::json& modelJson) { + modelVersion = modelJson.at("version"); + if (modelJson.contains("metadata")) { nlohmann::json metadataJson = modelJson.at("metadata"); diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index 5d59ab7..1490954 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -115,6 +115,11 @@ namespace NeuralAudio return -1; // No fixed receptive field size (ie: for LSTM) } + virtual std::string GetModelVersion() + { + return modelVersion; + } + virtual std::string GetMetadata(const std::string& fieldName) { auto it = std::find_if(metadata.begin(), metadata.end(), [&](const auto& pair) @@ -149,6 +154,7 @@ namespace NeuralAudio float modelOutputLevelDBu = 12; float modelLoudnessDB = -18; float sampleRate = 48000; + std::string modelVersion = ""; std::vector> metadata; inline static float audioInputLevelDBu = 12; From d43da8efbf1fdd5cef9b7eb21da7e977fae6e44e Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 18 May 2026 12:05:07 -0700 Subject: [PATCH 037/162] Document GetModelVersion --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index fea9d73..32bf92c 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,14 @@ if (!metadataValue.empty()) ``` Results are always returned a strings. Field names are case sensitive. +To get the model version string, do: + +``` +std::string version = model->GetModelVersion() +``` + +The string will be empty if no version information exists. + ## Getting the model receptive field size WaveNet models have a fixed receptive field size (ie: size of the input that the output depends on). From 62a93276437665db8855987e89989cfb5694dbce Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 19 May 2026 09:56:06 -0700 Subject: [PATCH 038/162] Hide private details from public interface --- NeuralAudio/CMakeLists.txt | 1 + NeuralAudio/InternalModel.h | 7 +-- NeuralAudio/NAMModel.h | 3 +- NeuralAudio/NeuralModel.cpp | 55 -------------------- NeuralAudio/NeuralModel.h | 35 ------------- NeuralAudio/NeuralModelImpl.h | 97 +++++++++++++++++++++++++++++++++++ NeuralAudio/RTNeuralLoader.h | 1 + NeuralAudio/RTNeuralModel.h | 3 +- 8 files changed, 107 insertions(+), 95 deletions(-) create mode 100644 NeuralAudio/NeuralModelImpl.h diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 0aa0d74..cfce01e 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -84,6 +84,7 @@ endif() set(SOURCES NeuralModel.h NeuralModel.cpp + NeuralModelImpl.h NAMModel.h RTNeuralModel.h RTNeuralLoader.cpp diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index f31d21d..85ae803 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -1,6 +1,7 @@ #pragma once #include "NeuralModel.h" +#include "NeuralModelImpl.h" #include "WaveNet.h" #include "WaveNetDynamic.h" #include "LSTM.h" @@ -12,7 +13,7 @@ namespace NeuralAudio using ILiteDilations1 = NeuralAudio::Dilations<1, 2, 4, 8, 16, 32, 64>; using ILiteDilations2 = NeuralAudio::Dilations<128, 256, 512, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512>; - class InternalModel : public NeuralModel + class InternalModel : public NeuralModelImpl { public: bool LoadFromKerasJson(const nlohmann::json& modelJson) @@ -365,7 +366,7 @@ namespace NeuralAudio void Prewarm() { - NeuralModel::Prewarm(2048, 64); + NeuralModelImpl::Prewarm(2048, 64); } private: @@ -528,7 +529,7 @@ namespace NeuralAudio void Prewarm() { - NeuralModel::Prewarm(2048, 64); + NeuralModelImpl::Prewarm(2048, 64); } private: diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index 723f706..d16e84e 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -1,6 +1,7 @@ #pragma once #include "NeuralModel.h" +#include "NeuralModelImpl.h" #include #include #include @@ -9,7 +10,7 @@ namespace NeuralAudio { - class NAMModel : public NeuralModel + class NAMModel : public NeuralModelImpl { public: NAMModel() diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index ad754ef..122c647 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -327,59 +327,4 @@ namespace NeuralAudio return newModel; } - - void NeuralModel::ReadNAMConfig(const nlohmann::json& modelJson) - { - modelVersion = modelJson.at("version"); - - if (modelJson.contains("metadata")) - { - nlohmann::json metadataJson = modelJson.at("metadata"); - - AddMetadata(metadataJson); - } - - if (modelJson.contains("sample_rate") && modelJson.at("sample_rate").is_number()) - { - sampleRate = modelJson.at("sample_rate"); - } - - if (modelJson.contains("metadata")) - { - nlohmann::json metaData = modelJson.at("metadata"); - - if (metaData.contains("loudness") && metaData.at("loudness").is_number()) - { - modelLoudnessDB = (float)metaData.at("loudness"); - } - - if (metaData.contains("input_level_dbu") && metaData.at("input_level_dbu").is_number()) - { - modelInputLevelDBu = metaData.at("input_level_dbu"); - } - - if (metaData.contains("output_level_dbu") && metaData.at("output_level_dbu").is_number()) - { - modelOutputLevelDBu = metaData.at("output_level_dbu"); - } - } - } - - void NeuralModel::ReadKerasConfig(const nlohmann::json& modelJson) - { - if (modelJson.contains("samplerate") && modelJson.at("samplerate").is_number()) - { - sampleRate = modelJson.at("samplerate"); - } - - if (modelJson.contains("in_gain") && modelJson.at("in_gain").is_number()) - { - modelInputLevelDBu = modelJson.at("in_gain"); - } - - if (modelJson.contains("out_gain") && modelJson.at("out_gain").is_number()) - { - modelLoudnessDB = -18 - (float)modelJson.at("out_gain"); - } - } } \ No newline at end of file diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index 1490954..e245f27 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -2,7 +2,6 @@ #include #include -#include "json.hpp" #ifndef DEFAULT_QUALITY_SCALE #define DEFAULT_QUALITY_SCALE 1.0 @@ -147,9 +146,6 @@ namespace NeuralAudio } protected: - void ReadNAMConfig(const nlohmann::json& modelJson); - void ReadKerasConfig(const nlohmann::json& modelJson); - float modelInputLevelDBu = 12; float modelOutputLevelDBu = 12; float modelLoudnessDB = -18; @@ -162,36 +158,5 @@ namespace NeuralAudio inline static EModelLoadMode wavenetLoadMode = EModelLoadMode::Internal; inline static int defaultMaxAudioBufferSize = 128; inline static float defaultQualityScaleFactor = DEFAULT_QUALITY_SCALE; - - void AddMetadata(std::string fieldName, std::string fieldValue) - { - metadata.push_back({fieldName, fieldValue}); - } - - void AddMetadata(const nlohmann::json& metadataJson) - { - for (auto& [key, value] : metadataJson.items()) - { - if (!value.is_null()) - { - AddMetadata(key, value.dump()); - } - } - } - - void Prewarm(size_t numSamples, size_t blockSize) - { - std::vector input; - input.resize(blockSize); - std::fill(input.begin(), input.end(), 0.0f); - - std::vector output; - output.resize(blockSize); - - for (size_t block = 0; block < (numSamples / blockSize); block++) - { - Process(input.data(), output.data(), blockSize); - } - } }; } \ No newline at end of file diff --git a/NeuralAudio/NeuralModelImpl.h b/NeuralAudio/NeuralModelImpl.h new file mode 100644 index 0000000..36456aa --- /dev/null +++ b/NeuralAudio/NeuralModelImpl.h @@ -0,0 +1,97 @@ +#pragma once + +#include "json.hpp" +#include "NeuralModel.h" + +namespace NeuralAudio +{ + class NeuralModelImpl : public NeuralModel + { + protected: + void ReadNAMConfig(const nlohmann::json& modelJson) + { + modelVersion = modelJson.at("version"); + + if (modelJson.contains("metadata")) + { + nlohmann::json metadataJson = modelJson.at("metadata"); + + AddMetadata(metadataJson); + } + + if (modelJson.contains("sample_rate") && modelJson.at("sample_rate").is_number()) + { + sampleRate = modelJson.at("sample_rate"); + } + + if (modelJson.contains("metadata")) + { + nlohmann::json metaData = modelJson.at("metadata"); + + if (metaData.contains("loudness") && metaData.at("loudness").is_number()) + { + modelLoudnessDB = (float)metaData.at("loudness"); + } + + if (metaData.contains("input_level_dbu") && metaData.at("input_level_dbu").is_number()) + { + modelInputLevelDBu = metaData.at("input_level_dbu"); + } + + if (metaData.contains("output_level_dbu") && metaData.at("output_level_dbu").is_number()) + { + modelOutputLevelDBu = metaData.at("output_level_dbu"); + } + } + } + + void ReadKerasConfig(const nlohmann::json& modelJson) + { + if (modelJson.contains("samplerate") && modelJson.at("samplerate").is_number()) + { + sampleRate = modelJson.at("samplerate"); + } + + if (modelJson.contains("in_gain") && modelJson.at("in_gain").is_number()) + { + modelInputLevelDBu = modelJson.at("in_gain"); + } + + if (modelJson.contains("out_gain") && modelJson.at("out_gain").is_number()) + { + modelLoudnessDB = -18 - (float)modelJson.at("out_gain"); + } + } + + void AddMetadata(std::string fieldName, std::string fieldValue) + { + metadata.push_back({ fieldName, fieldValue }); + } + + void AddMetadata(const nlohmann::json& metadataJson) + { + for (auto& [key, value] : metadataJson.items()) + { + if (!value.is_null()) + { + AddMetadata(key, value.dump()); + } + } + } + + void Prewarm(size_t numSamples, size_t blockSize) + { + std::vector input; + input.resize(blockSize); + std::fill(input.begin(), input.end(), 0.0f); + + std::vector output; + output.resize(blockSize); + + for (size_t block = 0; block < (numSamples / blockSize); block++) + { + Process(input.data(), output.data(), blockSize); + } + } + }; +} \ No newline at end of file diff --git a/NeuralAudio/RTNeuralLoader.h b/NeuralAudio/RTNeuralLoader.h index 7e6cb23..c96e463 100644 --- a/NeuralAudio/RTNeuralLoader.h +++ b/NeuralAudio/RTNeuralLoader.h @@ -1,6 +1,7 @@ #pragma once #include "NeuralModel.h" +#include "NeuralModelImpl.h" namespace NeuralAudio { diff --git a/NeuralAudio/RTNeuralModel.h b/NeuralAudio/RTNeuralModel.h index 63aba73..5a80346 100644 --- a/NeuralAudio/RTNeuralModel.h +++ b/NeuralAudio/RTNeuralModel.h @@ -1,6 +1,7 @@ #pragma once #include "NeuralModel.h" +#include "NeuralModelImpl.h" #include #ifdef BUILD_STATIC_RTNEURAL #include "wavenet_model.hpp" @@ -32,7 +33,7 @@ namespace NeuralAudio } }; - class RTNeuralModel : public NeuralModel + class RTNeuralModel : public NeuralModelImpl { public: EModelLoadMode GetLoadMode() From 66d02d9bcea08fe8ce0b7f1227cc90af74d76563 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 19 May 2026 10:03:25 -0700 Subject: [PATCH 039/162] Mark method overrides explicitly --- NeuralAudio/InternalModel.h | 44 ++++++++++++++++++------------------- NeuralAudio/NAMModel.h | 14 ++++++------ NeuralAudio/RTNeuralModel.h | 8 +++---- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index 85ae803..699c307 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -73,7 +73,7 @@ namespace NeuralAudio } } - bool IsStatic() + bool IsStatic() override { return true; } @@ -97,17 +97,17 @@ namespace NeuralAudio return true; } - void SetMaxAudioBufferSize(const int maxSize) + void SetMaxAudioBufferSize(const int maxSize) override { (void)maxSize; } - int GetReceptiveFieldSize() + int GetReceptiveFieldSize() override { return model->ReceptiveFieldSize; } - void Process(float* input, float* output, size_t numSamples) + void Process(float* input, float* output, size_t numSamples) override { size_t offset = 0; @@ -122,7 +122,7 @@ namespace NeuralAudio } } - void Prewarm() + void Prewarm() override { model->Prewarm(); } @@ -155,17 +155,17 @@ namespace NeuralAudio class InternalWaveNetDefinitionT : public InternalWaveNetDefinitionBase { public: - InternalModel* CreateModel() + InternalModel* CreateModel() override { return new InternalWaveNetModelT; } - virtual size_t GetNumChannels() + virtual size_t GetNumChannels() override { return numChannels; } - virtual size_t GetHeadSize() + virtual size_t GetHeadSize() override { return headSize; } @@ -187,7 +187,7 @@ namespace NeuralAudio } } - EModelLoadMode GetLoadMode() + EModelLoadMode GetLoadMode() override { return EModelLoadMode::Internal; } @@ -215,12 +215,12 @@ namespace NeuralAudio return true; } - void SetMaxAudioBufferSize(const int maxSize) + void SetMaxAudioBufferSize(const int maxSize) override { model->SetMaxFrames(maxSize); } - void Process(float* input, float* output, size_t numSamples) + void Process(float* input, float* output, size_t numSamples) override { size_t offset = 0; @@ -235,7 +235,7 @@ namespace NeuralAudio } } - void Prewarm() + void Prewarm() override { model->Prewarm(); } @@ -263,7 +263,7 @@ namespace NeuralAudio } } - bool IsStatic() + bool IsStatic() override { return true; } @@ -354,17 +354,17 @@ namespace NeuralAudio return true; } - void SetMaxAudioBufferSize(const int maxSize) + void SetMaxAudioBufferSize(const int maxSize) override { (void)maxSize; } - void Process(float* input, float* output, size_t numSamples) + void Process(float* input, float* output, size_t numSamples) override { model->Process(input, output, numSamples); } - void Prewarm() + void Prewarm() override { NeuralModelImpl::Prewarm(2048, 64); } @@ -397,17 +397,17 @@ namespace NeuralAudio class InternalLSTMDefinitionT : public InternalLSTMDefinitionBase { public: - InternalModel* CreateModel() + InternalModel* CreateModel() override { return new InternalLSTMModelT; } - virtual size_t GetNumLayers() + virtual size_t GetNumLayers() override { return NumLayers; } - virtual size_t GetHiddenSize() + virtual size_t GetHiddenSize() override { return HiddenSize; } @@ -517,17 +517,17 @@ namespace NeuralAudio return true; } - void SetMaxAudioBufferSize(const int maxSize) + void SetMaxAudioBufferSize(const int maxSize) override { (void)maxSize; } - void Process(float* input, float* output, size_t numSamples) + void Process(float* input, float* output, size_t numSamples) override { model->Process(input, output, numSamples); } - void Prewarm() + void Prewarm() override { NeuralModelImpl::Prewarm(2048, 64); } diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index d16e84e..f85d0df 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -26,7 +26,7 @@ namespace NeuralAudio namModel.reset(); } - EModelLoadMode GetLoadMode() + EModelLoadMode GetLoadMode() override { return EModelLoadMode::NAMCore; } @@ -54,17 +54,17 @@ namespace NeuralAudio return true; } - bool HasQualityScaling() + bool HasQualityScaling() override { return isSlimmable; } - float GetQualityScaleFactor() + float GetQualityScaleFactor() override { return slimmableSize; } - void SetQualityScaleFactor(float scaleFactor) + void SetQualityScaleFactor(float scaleFactor) override { if (HasQualityScaling()) { @@ -82,17 +82,17 @@ namespace NeuralAudio } } - void SetMaxAudioBufferSize(const int maxSize) + void SetMaxAudioBufferSize(const int maxSize) override { namModel->Reset(namModel->GetExpectedSampleRate(), maxSize); } - void Process(float* input, float* output, size_t numSamples) + void Process(float* input, float* output, size_t numSamples) override { namModel->process(&input, &output, (int)numSamples); } - void Prewarm() + void Prewarm() override { namModel->prewarm(); } diff --git a/NeuralAudio/RTNeuralModel.h b/NeuralAudio/RTNeuralModel.h index 5a80346..d105d7d 100644 --- a/NeuralAudio/RTNeuralModel.h +++ b/NeuralAudio/RTNeuralModel.h @@ -36,7 +36,7 @@ namespace NeuralAudio class RTNeuralModel : public NeuralModelImpl { public: - EModelLoadMode GetLoadMode() + EModelLoadMode GetLoadMode() override { return EModelLoadMode::RTNeural; } @@ -95,7 +95,7 @@ namespace NeuralAudio } } - bool IsStatic() + bool IsStatic() override { return true; } @@ -214,13 +214,13 @@ namespace NeuralAudio return true; } - void Process(float* input, float* output, size_t numSamples) + void Process(float* input, float* output, size_t numSamples) override { for (size_t i = 0; i < numSamples; i++) output[i] = model->forward(input + i); } - void Prewarm() + void Prewarm() override { float sample = 0; From 71fbfbebca24813160a84a3cb52113ce27472ee6 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 19 May 2026 10:06:53 -0700 Subject: [PATCH 040/162] Fix Linux build (add includes) --- NeuralAudio/NeuralModel.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index e245f27..df249f2 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -2,6 +2,8 @@ #include #include +#include +#include #ifndef DEFAULT_QUALITY_SCALE #define DEFAULT_QUALITY_SCALE 1.0 From 62934358ed7040f1a8095ea49071df80cab2dd0b Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 19 May 2026 10:08:10 -0700 Subject: [PATCH 041/162] More linux includes --- NeuralAudio/NeuralModel.h | 1 + 1 file changed, 1 insertion(+) diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index df249f2..ee0c4a3 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -2,6 +2,7 @@ #include #include +#include #include #include From fb249972ca7c5821e93f38745831dd86825ab0de Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 19 May 2026 10:11:10 -0700 Subject: [PATCH 042/162] Include math for ModelTest --- Utils/ModelTest/ModelTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Utils/ModelTest/ModelTest.cpp b/Utils/ModelTest/ModelTest.cpp index 25f33d7..537c471 100644 --- a/Utils/ModelTest/ModelTest.cpp +++ b/Utils/ModelTest/ModelTest.cpp @@ -1,3 +1,4 @@ +#include #include #include #include "argparse.hpp" From 0127fede0e2dd863d19f5f106a2e87f9bfec0314 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 19 May 2026 10:38:10 -0700 Subject: [PATCH 043/162] Don't build static LSTM models by default (saves memory and compile time) --- NeuralAudio/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index cfce01e..0046129 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -47,7 +47,7 @@ set(WAVENET_MATH "FastMath" CACHE STRING "WaveNet math functions") add_definitions(-DWAVENET_MATH=${WAVENET_MATH}) message(STATUS "WaveNet math is: ${WAVENET_MATH}") -option(BUILD_INTERNAL_STATIC_LSTM "Build Internal static LSTM models" ON) +option(BUILD_INTERNAL_STATIC_LSTM "Build Internal static LSTM models" OFF) if(BUILD_INTERNAL_STATIC_LSTM) message(STATUS "Building Internal static LSTM models") add_definitions(-DBUILD_INTERNAL_STATIC_LSTM) From 91702987c7ce78c18fb59c8ffd5da9ccefaef267 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 20 May 2026 10:46:37 -0700 Subject: [PATCH 044/162] Avoid at least one of the extra A2 NAM Core prewarm calls --- NeuralAudio/NAMModel.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index f85d0df..12d338e 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -40,6 +40,8 @@ namespace NeuralAudio namModel = nam::get_dsp(modelJson); + suppressPrewarm = true; // Hack to work around NAM Core get_dsp() above forcing prewarm + auto* slim = dynamic_cast(namModel.get()); if (slim != nullptr) @@ -94,12 +96,20 @@ namespace NeuralAudio void Prewarm() override { - namModel->prewarm(); + if (suppressPrewarm) + { + suppressPrewarm = false; + } + else + { + namModel->prewarm(); + } } private: std::unique_ptr namModel = nullptr; float slimmableSize = 1.0f; bool isSlimmable = false; + bool suppressPrewarm = false; }; } \ No newline at end of file From 2c6526f07bc327d4e32188b217ed4547621d8de5 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 20 May 2026 10:53:28 -0700 Subject: [PATCH 045/162] Revert "Use compile-time loop over convolution kernel. Lazy product seems to be faster for small matrices and when we're adding bias." This reverts commit f0f3b8accf1e832d710309a6fe9d156d699541a2. --- NeuralAudio/WaveNet.h | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index a014f3f..bf3f47b 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -50,27 +50,17 @@ namespace NeuralAudio template inline void Process(const Eigen::MatrixBase& input, Eigen::MatrixBase const & output, const size_t iStart, const size_t nCols) const { - ForEachIndex([&](auto k) + for (size_t k = 0; k < KernelSize; k++) { auto offset = Dilation * ((int)k + 1 - KernelSize); auto inBlock = input.middleCols(iStart + offset, nCols); - if constexpr ((InChannels <= 4) || DoBias) - { - if constexpr (k == 0) - const_cast&>(output).noalias() = weights[k].lazyProduct(inBlock); - else - const_cast&>(output).noalias() += weights[k].lazyProduct(inBlock); - } + if (k == 0) + const_cast&>(output).noalias() = weights[k] * inBlock; else - { - if constexpr (k == 0) - const_cast&>(output).noalias() = weights[k] * inBlock; - else - const_cast&>(output).noalias() += weights[k] * inBlock; - } - }); + const_cast&>(output).noalias() += weights[k] * inBlock; + } if constexpr (DoBias) const_cast&>(output).colwise() += bias; From bd37b79dae957387e2a358e5b659ba0e6ec1b34c Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 20 May 2026 12:43:14 -0700 Subject: [PATCH 046/162] Update licensing note --- Utils/Models/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Utils/Models/README.md b/Utils/Models/README.md index c00565d..9246273 100644 --- a/Utils/Models/README.md +++ b/Utils/Models/README.md @@ -1,3 +1,5 @@ -Licensing note: +# Licensing Note -The "tw40_blues_deluxe_deerinkstudios.json" model was created by Deer Ink Studios and is under the CC BY-NC-ND 4.0 license. +These models are provided under the [CC BY-NC-ND 4.0 license](https://creativecommons.org/licenses/by-nc-nd/4.0/deed.en). + +The "tw40_blues_deluxe_deerinkstudios.json" model was created by Deer Ink Studios. From e13d1f1b454b58ea196fdad016d5f032cbb46429 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 20 May 2026 12:47:15 -0700 Subject: [PATCH 047/162] Delete Utils/Models/wavenet_a2_max.nam --- Utils/Models/wavenet_a2_max.nam | 2157 ------------------------------- 1 file changed, 2157 deletions(-) delete mode 100644 Utils/Models/wavenet_a2_max.nam diff --git a/Utils/Models/wavenet_a2_max.nam b/Utils/Models/wavenet_a2_max.nam deleted file mode 100644 index e820526..0000000 --- a/Utils/Models/wavenet_a2_max.nam +++ /dev/null @@ -1,2157 +0,0 @@ -{ - "notes": [ - "This model is meant as a 'test case' to contain all of the new features that are being considered for A2.", - "It doesn't have slimmability." - ], - "version": "0.6.0", - "metadata": { - "date": { - "year": 2026, - "month": 1, - "day": 19, - "hour": 14, - "minute": 51, - "second": 21 - }, - "loudness": -20.0, - "gain": 0.2, - "name": "Generated WaveNet Model", - "modeled_by": "create_wavenet.py" - }, - "architecture": "WaveNet", - "config": { - "condition_dsp": { - "version": "0.6.0", - "metadata": { - "date": { - "year": 2026, - "month": 1, - "day": 19, - "hour": 14, - "minute": 51, - "second": 18 - }, - "loudness": -20.0, - "gain": 0.2, - "name": "Generated WaveNet Model", - "modeled_by": "create_wavenet.py" - }, - "architecture": "WaveNet", - "config": { - "layers": [ - { - "input_size": 1, - "condition_size": 1, - "head_size": 4, - "channels": 3, - "bottleneck": 6, - "kernel_size": 2, - "dilations": [ - 1, - 2 - ], - "activation": { - "type": "SiLU" - }, - "gating_mode": "gated", - "head_bias": false, - "groups_input": 3, - "groups_input_mixin": 1, - "groups_1x1": 3, - "head_1x1": { - "active": true, - "out_channels": 6, - "groups": 3 - }, - "secondary_activation": "Hardswish", - "conv_pre_film": { - "active": true, - "shift": true, - "groups": 1 - }, - "conv_post_film": { - "active": true, - "shift": true, - "groups": 1 - }, - "input_mixin_pre_film": { - "active": true, - "shift": true, - "groups": 1 - }, - "input_mixin_post_film": { - "active": true, - "shift": true, - "groups": 1 - }, - "activation_pre_film": { - "active": true, - "shift": true, - "groups": 1 - }, - "activation_post_film": { - "active": true, - "shift": true, - "groups": 1 - }, - "1x1_post_film": { - "active": true, - "shift": true, - "groups": 1 - }, - "head1x1_post_film": { - "active": true, - "shift": true, - "groups": 1 - } - }, - { - "input_size": 3, - "condition_size": 1, - "head_size": 8, - "channels": 4, - "bottleneck": 2, - "kernel_size": 3, - "dilations": [ - 1, - 3, - 5 - ], - "activation": [ - { - "type": "PReLU", - "negative_slopes": [ - 0.04, - 0.05 - ] - }, - { - "type": "PReLU", - "negative_slopes": [ - 0.03, - 0.01 - ] - }, - { - "type": "Softsign" - } - ], - "gating_mode": [ - "blended", - "gated", - "gated" - ], - "head_bias": false, - "groups_input": 1, - "groups_input_mixin": 1, - "groups_1x1": 1, - "head_1x1": { - "active": true, - "out_channels": 4, - "groups": 2 - }, - "secondary_activation": [ - { - "type": "LeakyHardtanh", - "min_val": 0.0, - "max_val": 0.9, - "min_slope": 0.0, - "max_slope": 0.02 - }, - "ReLU", - "Sigmoid" - ], - "conv_pre_film": { - "active": true, - "shift": false, - "groups": 1 - }, - "conv_post_film": { - "active": true, - "shift": false, - "groups": 1 - }, - "input_mixin_pre_film": { - "active": true, - "shift": false, - "groups": 1 - }, - "input_mixin_post_film": { - "active": true, - "shift": false, - "groups": 1 - }, - "activation_pre_film": { - "active": true, - "shift": false, - "groups": 1 - }, - "activation_post_film": { - "active": true, - "shift": false, - "groups": 1 - }, - "1x1_post_film": { - "active": true, - "shift": false, - "groups": 1 - }, - "head1x1_post_film": { - "active": true, - "shift": false, - "groups": 1 - } - } - ], - "head": null, - "head_scale": 0.02 - }, - "weights": [ - 0.2788535969157675, - -0.9499784895546661, - -0.4499413632617615, - -0.5535785237023545, - 0.4729424283280248, - 0.3533989748458226, - 0.7843591354096908, - -0.8261223347411677, - -0.15615636062945915, - -0.9404055611238593, - -0.5627240503927933, - 0.010710576206724776, - -0.9469280606322728, - -0.602324698626703, - 0.2997688755590464, - 0.08988296120643335, - -0.5591187559186066, - 0.17853136775181744, - 0.6188609133556533, - -0.987002480643878, - 0.6116385036656158, - 0.3962787899764537, - -0.31949896696401625, - -0.6890410003764369, - 0.9144261444135624, - -0.32681090977474647, - -0.8145083132397042, - -0.806567246333072, - 0.6949887326949196, - 0.20745206273378214, - 0.6142565465487604, - 0.45946357338763577, - 0.07245618290940148, - 0.9462315279587412, - -0.24293124558329304, - 0.104081262546454, - 0.6588093285059897, - 0.2370395047284921, - 0.7234138006215545, - 0.15470429051352408, - 0.40914367242984695, - -0.9083512326886756, - -0.5442034486969063, - -0.42122407279578566, - -0.840416046152745, - -0.5344182272779396, - -0.7979971411805418, - -0.44405279377981577, - 0.27136888852880037, - -0.2703356420598315, - -0.2596380657662347, - -0.5809859384570246, - -0.4660443559017733, - 0.873309175424988, - 0.2960707704931871, - 0.21826201133397638, - -0.657722703603806, - 0.45825359590069836, - -0.6731950124761432, - -0.24108911648470444, - 0.9790467012731905, - 0.2799995197081857, - 0.11389948754929247, - 0.3692285019797492, - 0.6857038403796192, - 0.5519998230924896, - -0.5419038560717913, - -0.9357995121919245, - -0.36909390388183616, - -0.46451824804859454, - -0.5780343128273471, - 0.8858194286701089, - 0.7527352529453377, - -0.37064423840304417, - 0.3108773305897601, - -0.20873619787867148, - 0.829095179481087, - -0.0822962948252024, - -0.4702396670038951, - -0.5067449846120331, - 0.12273626832630158, - -0.47451678295412947, - 0.16917198044708104, - 0.795645767204954, - -0.20119898971920547, - -0.5613584816854333, - 0.9950752129902205, - 0.01905258735292903, - -0.8181811756524122, - -0.9057672491505309, - -0.7807017392986817, - 0.2548920834061801, - 0.5841587287259282, - -0.15568006640063192, - -0.8729445876960857, - -0.23676142698692648, - 0.9922427604801936, - 0.058228690198274036, - 0.9421567552272363, - 0.7215594044689961, - -0.9770379561143607, - 0.4414436387203893, - 0.36342073805314956, - 0.07394066081759032, - -0.46634962009491443, - 0.2819235971596161, - -0.7768956528082471, - -0.13046949866179003, - -0.09255258734158711, - 0.9076318550421603, - 0.7517058807563881, - -0.4732218984978185, - 0.0011722261005966406, - -0.6426962389397373, - 0.825255678689641, - 0.7410371396735338, - -0.4031104171027342, - 0.2778989897320103, - 0.21794042287634463, - -0.6943214629007304, - 0.5250216001503025, - 0.07875806023925147, - 0.5572529572611165, - 0.06070734439035497, - -0.998856207744113, - -0.3516878859906538, - -0.9610465152283354, - 0.8581972325292342, - 0.7574437556463685, - 0.6633310587223589, - -0.38497174919467714, - -0.8841496670116249, - 0.7560191984080811, - 0.8938988905959881, - -0.8286930958642424, - -0.02801907336677245, - -0.8615749630632328, - 0.5212043305144631, - 0.5316688586139755, - -0.7432170710004744, - -0.04943524380253739, - 0.0996071869898878, - -0.4698867421198818, - 0.7448660821705149, - -0.15372411959822618, - -0.5764035891158359, - 0.07859217755891668, - 0.45986213817995236, - -0.5976978732206082, - -0.3765674173982101, - 0.9902987133217893, - 0.299756115278907, - -0.12379983217099189, - 0.035151682071181245, - -0.7579916082634686, - -0.5506053259368853, - -0.32382887570508934, - 0.17661743691446663, - -0.539770534806846, - -0.559565231096881, - -0.8580138279819349, - 0.2622059145401978, - -0.5421164323776912, - 0.8108400260122559, - 0.719270800507493, - -0.8582853002226931, - -0.5239907312620096, - 0.33795555659256116, - -0.5715263852591228, - -0.73537630254995, - 0.871028481161342, - 0.14208618665056894, - -0.05465794737641172, - 0.5692388485815068, - 0.6149939955332868, - -0.6191801712762446, - -0.8061383715423533, - -0.13789763518724496, - -0.15284275396015845, - -0.06595066392665005, - 0.4581516989197012, - 0.34672909458660306, - 0.9683304227319323, - -0.8031642576960822, - -0.19475743579546245, - -0.3213947892100737, - 0.7233450727055821, - -0.5026873321594287, - -0.619582183118377, - -0.1027729043337362, - -0.15623672033119163, - -0.44290971066611906, - -0.500387104235799, - 0.8465311985520256, - -0.11373850989308609, - 0.7226982095236612, - 0.10065062489969612, - -0.8988233409502375, - 0.9985649368254532, - 0.6720551701599038, - 0.9379925145695025, - 0.8527339660162552, - 0.6973914688286109, - -0.667377778792172, - -0.02871774909856306, - -0.5725054016016367, - -0.19791941490109477, - -0.8827292000556421, - -0.2420537620461678, - 0.9706176875594519, - -0.4695938836556961, - 0.5681412038971387, - -0.08998326532171341, - -0.15398502801967417, - 0.9146352817193464, - 0.9908453789854277, - 0.11153664681123643, - 0.436816550592652, - -0.6904063494518717, - -0.4065843490108716, - 0.9374187299383177, - 0.15836058163251243, - 0.08439040274854848, - 0.4959511207581282, - -0.8856694541850338, - 0.16835518891794243, - 0.005700765839027122, - 0.7054397840965707, - -0.6851345441210335, - 0.9215578065489007, - -0.8397770695188262, - -0.6283500780385536, - 0.19007021290005532, - 0.3504251072081803, - -0.5295922099981376, - -0.7602267721057516, - 0.780574628258875, - -0.5075693044227503, - 0.1890383070668824, - 0.23876302066420618, - -0.16155016932825506, - 0.16734457858244944, - 0.04556543106391775, - 0.8694125154728545, - -0.5914816011529271, - 0.4323836015788296, - -0.522628094768308, - -0.208428306417491, - 0.34338044591994255, - -0.40000584040247555, - -0.36764560745629193, - 0.5037289848288042, - -0.8549137710136854, - -0.08342895476282775, - 0.9969088817088847, - 0.9921928957101889, - -0.853478557800734, - -0.5736913754659192, - -0.4695991704991973, - 0.8665187559874181, - 0.7617283473728791, - 0.7585404849690855, - -0.2609458225222321, - -0.6845063352855361, - 0.667489909279614, - 0.4070798501747419, - 0.22335553145190024, - 0.9744661272630086, - 0.3079526354214652, - -0.9843537856956841, - 0.6342082702309233, - -0.4012424956000442, - 0.32677742993215464, - 0.8778600078542078, - -0.7314177712132646, - -0.7691426591617956, - -0.7859280445811647, - 0.10644728176963181, - -0.45530357537036736, - 0.20965965406044784, - 0.43522437427759586, - -0.5928053753450941, - 0.26847591777015944, - -0.4720321967391812, - -0.02293629570124689, - 0.8106729821586465, - 0.6922074265897109, - -0.8154030645745332, - -0.15284845487254728, - -0.44663955205549666, - -0.9929086218244354, - 0.5422384460392542, - 0.27422675460275925, - -0.4760894751313036, - 0.48246181669586163, - 0.10336084225278253, - -0.14462616203864131, - -0.9806606007833201, - -0.8495122798524659, - 0.7662127866002859, - 0.8078571431197863, - 0.09118057841104465, - 0.6691900397720334, - 0.16501913297958803, - -0.7038124288650347, - -0.7451089614357225, - -0.38348330013973264, - 0.79796297748518, - 0.5922446097760834, - 0.7214051640018055, - 0.7978492730529492, - -0.5798469233204919, - -0.5009405215541511, - -0.7944127566564287, - 0.5602324837428854, - 0.7682694029020178, - -0.1872452203357664, - 0.2413230203014256, - -0.6908933233355907, - 0.8597620313873489, - 0.7292113924399279, - 0.9524120658619257, - 0.6215434398807937, - 0.7628324093266488, - -0.9504272762036226, - 0.4731289435101642, - -0.33562906410714266, - 0.8616317720966511, - 0.6044702778742779, - 0.7281280567505588, - 0.621498633148778, - -0.46638858081105594, - 0.5747490182709423, - -0.7838087471940858, - 0.7443335658121795, - 0.7171865026755633, - -0.5551325649086711, - 0.6331732111938579, - -0.07939353064211585, - -0.38961826532279886, - 0.5906909983057236, - -0.5448090251844593, - -0.952671130597097, - -0.6137404233445827, - -0.34347609760458697, - 0.7287058840605727, - 0.9337782080967223, - -0.4417500145562572, - 0.2829634772152554, - -0.20064323127987826, - 0.9622993743965202, - 0.07243146495744379, - 0.8784742806494314, - -0.7693164962971448, - 0.9408012220444559, - -0.6428643676550727, - 0.9250686315231109, - -0.46906727495406275, - -0.7831949055705778, - -0.1308724828707113, - 0.4570901213054086, - -0.37264537161001754, - 0.21241770661228654, - 0.022846119338956195, - -0.22960913331054567, - 0.15317608699319907, - -0.4905549877228361, - 0.4175705676683412, - -0.9966174435627411, - 0.8511503309981654, - 0.0769039941855838, - 0.438859998289691, - 0.48390015567895306, - 0.34125700886599897, - -0.27155705643747163, - -0.8600523777473796, - 0.32847536982254466, - -0.3395999279148072, - -0.37216870988328066, - 0.696030559012671, - 0.43950852602790036, - -0.39935546357747165, - -0.3814306755826935, - -0.1832141827615663, - -0.19519922588455074, - -0.40868959494810597, - -0.7454244018816936, - -0.1591073324541834, - 0.880727341460366, - 0.35463589054546585, - 0.8056110914651653, - 0.23102983190276105, - -0.39810025086886935, - 0.09587442627139642, - -0.999188120605425, - -0.4261725662621456, - -0.1402237000203308, - 0.15996956239136395, - 0.30941124740614323, - -0.07002361950597158, - -0.11568040139038516, - -0.5725971980217994, - -0.053627628181347475, - 0.8023616516565084, - 0.5920495202535605, - -0.6606172076038905, - -0.8304089265497565, - 0.030904019830432894, - 0.26588171153159146, - -0.3296234891803982, - 0.6368469290733285, - 0.5022762750814644, - 0.3455913411143341, - -0.5507186680054235, - -0.6017401345468467, - -0.9511492245463473, - -0.5103149118432997, - -0.04972731156238974, - 0.6994753892494638, - -0.8543435416308618, - -0.1711179780045613, - 0.2595307614754274, - -0.6111295265205814, - 0.39270850098100984, - -0.011245661979126131, - -0.5120311208431223, - 0.31211602222356816, - -0.9889103637239365, - 0.5019289532369458, - 0.5400923771480501, - -0.7868254068729221, - -0.14970761211453176, - -0.6482266365869367, - 0.9159320845590795, - 0.03591550088748163, - -0.8995632297187182, - -0.5016034406800567, - 0.6966726947033195, - -0.08707634905965489, - 0.602833203444529, - 0.3351554651727062, - 0.975784906132896, - 0.1909046369388394, - 0.9000792168863119, - 0.782851851620874, - 0.2253046455235257, - 0.4385479225519342, - 0.00955632964880393, - 0.6611383394428301, - 0.09574390122165677, - 0.7944162064665243, - 0.4873108843191698, - -0.05065112635389335, - -0.4816169030699613, - -0.5055205249806809, - 0.27532287355231255, - 0.5316273685943309, - 0.0425996256559642, - 0.2534968739635626, - -0.4508051061649234, - -0.8450332922705284, - -0.428543698273695, - -0.4565697858356308, - -0.3605808631624754, - 0.08030444503691281, - -0.7232518769676886, - -0.5374770405436264, - 0.38789962459810456, - 0.41283828338910444, - -0.8715422985722439, - -0.18480126066682656, - 0.08522228100783069, - -0.1684515317936881, - -0.5863312209711795, - -0.15971296445314875, - 0.8096769566803539, - 0.1681588284084503, - 0.39104597299593613, - 0.7134640646078685, - 0.5311891522361389, - -0.23923794214457672, - -0.9882078328320139, - -0.2964823946563506, - 0.5069502501187717, - 0.7068959011382092, - 0.9068606769402126, - -0.1619574347403563, - 0.49503133795610155, - 0.09226461946767817, - 0.20650517788248268, - -0.5589226113523622, - -0.5611567307571277, - -0.1283280479067268, - -0.9419503601065695, - -0.3277409126032351, - 0.3582837700566994, - -0.19136666172472583, - -0.6699105375929824, - -0.0652197015353797, - -0.7447444054376786, - 0.24451392194812938, - -0.9460670961897246, - -0.21195948732059056, - 0.12878396604954823, - -0.9457959073193751, - 0.2854992960186715, - -0.7286010255388715, - -0.07660311189697655, - -0.8994307330227449, - -0.24179227162372086, - -0.5766794315770369, - -0.3463083902373829, - 0.5224594157880542, - -0.24174756887172744, - 0.5040196471095697, - 0.6638485703105452, - -0.4954569364352388, - -0.8361875344767149, - -0.9612334258999786, - 0.0788380958450674, - 0.9998156570184185, - -0.30007931255963216, - 0.30028818649975, - 0.5624660992217898, - 0.30350931048777885, - 0.5084664081190524, - 0.8992234654319777, - -0.6012786352749342, - -0.9592399653593353, - -0.6952353084304193, - -0.7475580502515875, - 0.33891768923982135, - 0.12793916386003823, - -0.5640709181869927, - 0.3989299424923016, - 0.5337961967124816, - -0.6644217132643955, - 0.21449498778186338, - 0.49585130391057164, - -0.7709342572422047, - 0.6386023486221701, - 0.9294415460681753, - -0.7838025006847869, - -0.948643149005068, - -0.3760855112106096, - 0.3546945737008176, - 0.9163456764117919, - -0.20669111696674536, - 0.4300294100989368, - -0.8480070443138801, - 0.3812288318659607, - 0.2544847912020889, - -0.7961973891080469, - 0.544961769902448, - 0.7005864781775926, - 0.20082322963368826, - -0.7578898698653698, - 0.9676887030293426, - 0.5652706927220392, - -0.30559246938310203, - -0.1432439735305111, - -0.25885824756388764, - 0.011921579354155831, - -0.31753765027743364, - 0.6991512539991547, - 0.6446618361801162, - -0.7889222587120028, - 0.9215751344291572, - 0.27117021220289184, - 0.6574146220049164, - 0.41461728741215387, - -0.1290257099846459, - 0.46759060802678376, - 0.9309474624761553, - -0.45983520722519833, - 0.6163984376135119, - 0.07634581289651154, - -0.033004992233607755, - -0.12885101399221144, - 0.46205242861024454, - -0.46320892390154933, - 0.7034263200386637, - 0.6614620377812068, - -0.8266742038886512, - 0.7632623680055439, - -0.512273121618088, - -0.07058306679353654, - 0.22066340846104127, - -0.24202139174347193, - -0.9426000044598208, - 0.7019056726249182, - -0.6363202856846164, - -0.575760299640522, - 0.595664713656193, - -0.3193223136714327, - 0.7606399595164508, - 0.4023675006644032, - -0.4474628484876302, - -0.9796977711226458, - 0.8961251555540624, - -0.8287740760839575, - 0.4401493282083886, - -0.022844306302425288, - 0.5163293069649328, - 0.38121867889304606, - 0.29180579948190455, - -0.018357329842827674, - 0.5858657362646349, - -0.8138932988906566, - -0.5568071990549397, - 0.3835743105904037, - -0.3875879397398232, - 0.16311117066473435, - -0.05347902248095049, - 0.0618438622915356, - -0.1489923745895705, - 0.49187087342721925, - -0.3384174056081397, - 0.4057098843715379, - -0.45816714604760733, - -0.497192647598117, - -0.7586882304953884, - -0.6148314130969652, - -0.7608905174898943, - 0.07172793076749961, - 0.5243792189682059, - -0.6297003151246761, - -0.5672307202427886, - -0.031602825574582605, - 0.44917000218610315, - 0.9532140457661136, - 0.0492737382172157, - -0.43400259345008996, - -0.7989477838184185, - -0.6117648438178531, - -0.5450336730348906, - -0.6411169126291933, - -0.9717032654878723, - 0.06827017853532658, - -0.4513773464334654, - 0.9485898622054758, - 0.1067179335972166, - 0.3948347858203889, - -0.7474410140831822, - 0.736922394525888, - -0.018242610952442284, - 0.7454394699970692, - 0.1481284392526645, - -0.061206110138979586, - -0.1190624037279795, - -0.6312726592198366, - -0.8972465656328443, - 0.8821271935362065, - -0.04454162546862461, - 0.6442312905988608, - -0.19858511549612046, - -0.8518356594887204, - 0.2588914139039966, - -0.8927818514149337, - -0.7016048310526897, - 0.12567919416912843, - -0.39232897631554464, - 0.9878362454204404, - -0.7630968755770768, - 0.5288868906523885, - 0.21263530248592555, - 0.5814816596868388, - -0.5486257258699307, - 0.04514507026043568, - -0.09897107022249152, - -0.11455799196134331, - 0.7203333316523601, - 0.9800625209335958, - -0.3892395113612792, - 0.24205464214429706, - 0.21926182449026066, - 0.48017861096909, - 0.895180400650756, - -0.5844241883528973, - -0.577949609389214, - 0.32085627438407327, - -0.6858858132382457, - -0.652372903347131, - -0.8498702621976642, - -0.9946485547942285, - -0.09899259076459521, - 0.18762239023912652, - -0.4174814219379448, - -0.5370475308879483, - 0.41391165975806654, - 0.4059751161874343, - -0.09193734718591351, - 0.3747698401424855, - 0.8478220889650974, - 0.5756560532934627, - 0.25011601432844377, - 0.32236608570673564, - 0.8673369168910265, - -0.1497220677915767, - 0.08912475742115733, - 0.2952694468049615, - 0.8168228904252122, - 0.6532623193001192, - -0.8571803262883717, - -0.6681544215585506, - -0.3847763747715349, - 0.4979154441392466, - 0.13841409863818477, - -0.4227788233883165, - -0.7512926836505835, - 0.37735598242418744, - 0.3994673699514346, - 0.8853524814880815, - 0.0009443542358511525, - -0.012409561309831396, - -0.839116296201398, - -0.9202784316327739, - -0.13594267167845486, - -0.35535683328340006, - -0.4992641951759922, - -0.8173462267380296, - 0.9238222043856734, - 0.6719172278123626, - 0.1503982184398609, - 0.9015725556127048, - 0.9991448337549027, - 0.3445631686064785, - -0.4609779480658922, - -0.9195366537107676, - 0.5125376608251142, - -0.05899835349673288, - 0.3030189788673727, - 0.832145575853535, - -0.6370217055537482, - 0.17065925055570874, - 0.26956943890817375, - -0.016548395617973277, - -0.8175151874123456, - -0.3040778874106991, - -0.33338321266981596, - 0.3402670190423709, - 0.7154661888007052, - -0.3403926728420692, - 0.387347347966865, - -0.4235644092770885, - 0.8903870791264212, - 0.6271320694759452, - 0.10019321794356584, - -0.09034818279654111, - -0.37096568565966415, - -0.35345242744801064, - 0.9403694536171543, - -0.19164988599850874, - 0.029192504658210217, - 0.9762384295635083, - 0.31532077296625793, - 0.08518718871439002, - -0.173504858401357, - -0.6248349172109458, - -0.2764412816966495, - 0.5128863081111474, - 0.25081748429001793, - 0.5199810721220339, - -0.5928835232994523, - 0.09843927817066334, - 0.8553455216888954, - -0.1237678098552435, - 0.39650005824434453, - -0.7571478332747603, - 0.9462936316433888, - 0.21774333416392988, - -0.5214050753446993, - -0.6832436723895352, - 0.10167801400533993, - 0.10450281810746609, - -0.8135815974369549, - 0.9845142787904393, - 0.8258597569683359, - -0.0771042116424332, - -0.7650677018208849, - 0.6642863475378025, - -0.003248990585609457, - 0.4332066519835822, - 0.01774403013902348, - -0.4531502065734516, - 0.6694478911533888, - 0.9604892652067509, - -0.5125381878497233, - 0.10253015376093932, - -0.23282797350570017, - 0.8437362998631237, - 0.016481783185787968, - 0.7586525102929522, - 0.7280538688571763, - -0.44750519435591807, - 0.5800123640062269, - -0.17011515289730972, - 0.8684967873650402, - 0.015475351619312905, - 0.6410989463711567, - -0.4343220334343467, - -0.4028883004564656, - 0.17387544482832218, - 0.9978046664586775, - -0.020719306688598005, - -0.7028091632344218, - 0.07716115540774737, - -0.30975211661398494, - 0.10383483414162198, - 0.08686012591734604, - -0.08931076626699785, - -0.35644529954192516, - -0.6226952525857892, - 0.3949968552411427, - 0.14359528396980226, - -0.5328751078842138, - 0.5510889501985177, - -0.9127054018053962, - 0.4894103031303918, - 0.4104557620500051, - 0.6228178051297537, - -0.22784249501803533, - 0.32737765896906934, - 0.6414951034191101, - 0.9616362773195701, - -0.009342700767152934, - -0.9259607773088476, - 0.00458230026593176, - 0.1803608586191341, - 0.7394006267244309, - 0.7483807481086167, - -0.11938758045864573, - 0.05190217361512284, - -0.08614385105151556, - 0.44488765514128525, - -0.1800427605072712, - 0.3095626528554676, - -0.6912775624569023, - -0.061018798030536114, - 0.9384072611484344, - -0.32287753189713664, - 0.3854091971737641, - 0.2996733051585452, - 0.7035305847013829, - 0.7046826731317861, - 0.7186843683365332, - -0.2399812049546899, - -0.36667769213320067, - 0.437434850445966, - 0.5188036186687273, - 0.7447660347970726, - -0.9282018003246268, - -0.8631585056368971, - 0.2623220337092811, - 0.8418581975605934, - 0.9948518459124307, - 0.4935327334758539, - -0.13205705615374863, - -0.8031137472336041, - 0.26749565756158256, - 0.7451584652141814, - -0.11264289667658844, - 0.3880023255864691, - 0.806848124100972, - -0.9080180626455745, - 0.5922869303244236, - -0.4132644480695824, - -0.25031782044937034, - -0.7088604086012031, - 0.06233263629752006, - 0.13185612383187362, - 0.5850389477619065, - -0.6600327018638052, - -0.8420632986800107, - 0.7416791972897097, - 0.2394207370748307, - -0.5183404162873739, - 0.8256580320471096, - -0.7137645597429807, - -0.07770017329000489, - -0.49204532117291544, - -0.4893465836816919, - -0.9812051370902579, - 0.6092661539502926, - 0.802418847197766, - 0.3552217713981589, - -0.6840487558564261, - -0.11654043279237114, - -0.3088687511138366, - 0.17514341025284286, - 0.2778774047200243, - -0.15138212307820464, - -0.4998035511845975, - 0.6906078502851973, - -0.6015660017822153, - -0.23061350206530906, - -0.0335838778815678, - -0.5255885961448508, - 0.14384538470147779, - 0.14962386035729014, - 0.9853840872537951, - -0.4095384922334613, - 0.9558889691537253, - 0.31645963185736137, - -0.45103923964354453, - 0.13185803391139794, - 0.37159898546802417, - 0.48933768233065034, - -0.9019114984480001, - 0.21281298615294908, - -0.006545426952259348, - 0.8083105817874507, - -0.42761169708161106, - 0.5977202390151974, - 0.2141299963285319, - -0.29535808832936694, - 0.27323575601178374, - 0.24178232626060825, - 0.35552891724671887, - 0.44185675334141794, - 0.3183630806335074, - 0.6766742339250331, - 0.25649620737379664, - 0.8068074081459358, - 0.2926812177810456, - -0.38213423209472785, - -0.11835361967437685, - 0.1591476107368055, - 0.4647195358784766, - -0.8197332485075621, - -0.40977909674168944, - 0.49496172987674303, - -0.6487198591113867, - -0.7356804045064329, - 0.07881551796884212, - 0.9429791624226798, - 0.06170474740557008, - 0.8269739489633938, - 0.6609452391347947, - -0.4860598308734754, - 0.6493796250848145, - -0.03630434025251761, - 0.612976987587533, - 0.4931187014340901, - -0.32256949239622745, - -0.7696605851004812, - 0.9257865857377552, - -0.7184859699882347, - 0.9330004189255041, - 0.7202811937976437, - 0.4484334241510364, - 0.9598844855638069, - 0.9345394946000647, - 0.6091752880411239, - -0.26844990118886725, - 0.5813639371778749, - -0.9721626897981976, - 0.07314461653811821, - -0.09042794453225067, - 0.3456567637475072, - 0.3446815947020263, - 0.1691201833043321, - 0.6448346024535485, - 0.8805837835591084, - -0.7833077956015386, - -0.5323561955768337, - -0.9499507007070354, - 0.7684696904297044, - 0.1228147644997577, - 0.8305118174863189, - -0.55726559985202, - -0.8735659176796091, - 0.6477107027808953, - 0.8187752768557841, - -0.39561965094149953, - -0.18340828840917456, - -0.7204459749855578, - 0.8925230657638925, - -0.39127083128798956, - -0.014750762043588272, - -0.8056160027563568, - 0.7745186170570046, - -0.728671902587327, - -0.09271248622221484, - 0.34097243770034247, - 0.4862802430463431, - 0.8919481715588642, - -0.16174649317055434, - 0.48453802953063163, - -0.69095419518033, - -0.17023094512638615, - -0.801956730578943, - -0.02130592442077095, - -0.18376822860459896, - 0.903043050762119, - -0.9345674262899062, - -0.2589400825311292, - -0.1132338278785967, - 0.9011103397028539, - 0.7109003866119092, - -0.8012907507877394, - 0.3713605309625707, - 0.08893172296428986, - 0.9556850589040935, - -0.28265231757536413, - -0.20372071451125384, - -0.6203828756778409, - -0.7556805618254725, - 0.6960663769273621, - -0.09056526285896571, - 0.325537476123956, - 0.2834089344664352, - 0.19429191903909016, - -0.9572850905272587, - 0.5735891809092335, - -0.5128622056719527, - -0.7481522293839142, - 0.12915595181592665, - -0.8627796943512882, - 0.5303147517771689, - -0.585685259306683, - -0.5680972961626505, - 0.7393908535390894, - -0.34288089313553916, - -0.7048916401171126, - 0.8010620712635164, - -0.9943288970399673, - 0.7168122527603527, - -0.710624039358984, - -0.7400157371131264, - -0.49869160654375233, - -0.6510057581972131, - 0.3221152851946336, - -0.9484397004276033, - -0.9702793455385823, - 0.5799693284695078, - -0.5241367878190739, - -0.3524570760759551, - -0.6515075971876456, - -0.895201964277647, - 0.483436113908299, - 0.05217105319573423, - 0.4913305500679914, - -0.04750806915349948, - 0.5560340786284055, - 0.026475915218383328, - -0.781891979992309, - 0.007677379571642717, - 0.8908312859402125, - -0.9132699262016566, - 0.5664539919607592, - 0.7339618155196765 - ], - "sample_rate": 48000 - }, - "layers": [ - { - "input_size": 1, - "condition_size": 8, - "head_size": 1, - "channels": 4, - "bottleneck": 4, - "kernel_size": 4, - "dilations": [ - 1, - 2 - ], - "activation": { - "type": "Softsign" - }, - "gating_mode": "none", - "head_bias": true, - "groups_input": 1, - "groups_input_mixin": 4, - "groups_1x1": 2, - "head_1x1": { - "active": true, - "out_channels": 4, - "groups": 2 - }, - "secondary_activation": "", - "conv_pre_film": { - "active": true, - "shift": true, - "groups": 2 - }, - "conv_post_film": { - "active": true, - "shift": true, - "groups": 4 - }, - "input_mixin_pre_film": { - "active": true, - "shift": true, - "groups": 4 - }, - "input_mixin_post_film": { - "active": true, - "shift": true, - "groups": 2 - }, - "activation_pre_film": { - "active": true, - "shift": true, - "groups": 1 - }, - "activation_post_film": { - "active": true, - "shift": true, - "groups": 2 - }, - "1x1_post_film": { - "active": true, - "shift": true, - "groups": 8 - }, - "head1x1_post_film": { - "active": true, - "shift": true, - "groups": 4 - } - } - ], - "head": null, - "head_scale": 0.02 - }, - "weights": [ - 0.2788535969157675, - -0.9499784895546661, - -0.4499413632617615, - -0.5535785237023545, - 0.4729424283280248, - 0.3533989748458226, - 0.7843591354096908, - -0.8261223347411677, - -0.15615636062945915, - -0.9404055611238593, - -0.5627240503927933, - 0.010710576206724776, - -0.9469280606322728, - -0.602324698626703, - 0.2997688755590464, - 0.08988296120643335, - -0.5591187559186066, - 0.17853136775181744, - 0.6188609133556533, - -0.987002480643878, - 0.6116385036656158, - 0.3962787899764537, - -0.31949896696401625, - -0.6890410003764369, - 0.9144261444135624, - -0.32681090977474647, - -0.8145083132397042, - -0.806567246333072, - 0.6949887326949196, - 0.20745206273378214, - 0.6142565465487604, - 0.45946357338763577, - 0.07245618290940148, - 0.9462315279587412, - -0.24293124558329304, - 0.104081262546454, - 0.6588093285059897, - 0.2370395047284921, - 0.7234138006215545, - 0.15470429051352408, - 0.40914367242984695, - -0.9083512326886756, - -0.5442034486969063, - -0.42122407279578566, - -0.840416046152745, - -0.5344182272779396, - -0.7979971411805418, - -0.44405279377981577, - 0.27136888852880037, - -0.2703356420598315, - -0.2596380657662347, - -0.5809859384570246, - -0.4660443559017733, - 0.873309175424988, - 0.2960707704931871, - 0.21826201133397638, - -0.657722703603806, - 0.45825359590069836, - -0.6731950124761432, - -0.24108911648470444, - 0.9790467012731905, - 0.2799995197081857, - 0.11389948754929247, - 0.3692285019797492, - 0.6857038403796192, - 0.5519998230924896, - -0.5419038560717913, - -0.9357995121919245, - -0.36909390388183616, - -0.46451824804859454, - -0.5780343128273471, - 0.8858194286701089, - 0.7527352529453377, - -0.37064423840304417, - 0.3108773305897601, - -0.20873619787867148, - 0.829095179481087, - -0.0822962948252024, - -0.4702396670038951, - -0.5067449846120331, - 0.12273626832630158, - -0.47451678295412947, - 0.16917198044708104, - 0.795645767204954, - -0.20119898971920547, - -0.5613584816854333, - 0.9950752129902205, - 0.01905258735292903, - -0.8181811756524122, - -0.9057672491505309, - -0.7807017392986817, - 0.2548920834061801, - 0.5841587287259282, - -0.15568006640063192, - -0.8729445876960857, - -0.23676142698692648, - 0.9922427604801936, - 0.058228690198274036, - 0.9421567552272363, - 0.7215594044689961, - -0.9770379561143607, - 0.4414436387203893, - 0.36342073805314956, - 0.07394066081759032, - -0.46634962009491443, - 0.2819235971596161, - -0.7768956528082471, - -0.13046949866179003, - -0.09255258734158711, - 0.9076318550421603, - 0.7517058807563881, - -0.4732218984978185, - 0.0011722261005966406, - -0.6426962389397373, - 0.825255678689641, - 0.7410371396735338, - -0.4031104171027342, - 0.2778989897320103, - 0.21794042287634463, - -0.6943214629007304, - 0.5250216001503025, - 0.07875806023925147, - 0.5572529572611165, - 0.06070734439035497, - -0.998856207744113, - -0.3516878859906538, - -0.9610465152283354, - 0.8581972325292342, - 0.7574437556463685, - 0.6633310587223589, - -0.38497174919467714, - -0.8841496670116249, - 0.7560191984080811, - 0.8938988905959881, - -0.8286930958642424, - -0.02801907336677245, - -0.8615749630632328, - 0.5212043305144631, - 0.5316688586139755, - -0.7432170710004744, - -0.04943524380253739, - 0.0996071869898878, - -0.4698867421198818, - 0.7448660821705149, - -0.15372411959822618, - -0.5764035891158359, - 0.07859217755891668, - 0.45986213817995236, - -0.5976978732206082, - -0.3765674173982101, - 0.9902987133217893, - 0.299756115278907, - -0.12379983217099189, - 0.035151682071181245, - -0.7579916082634686, - -0.5506053259368853, - -0.32382887570508934, - 0.17661743691446663, - -0.539770534806846, - -0.559565231096881, - -0.8580138279819349, - 0.2622059145401978, - -0.5421164323776912, - 0.8108400260122559, - 0.719270800507493, - -0.8582853002226931, - -0.5239907312620096, - 0.33795555659256116, - -0.5715263852591228, - -0.73537630254995, - 0.871028481161342, - 0.14208618665056894, - -0.05465794737641172, - 0.5692388485815068, - 0.6149939955332868, - -0.6191801712762446, - -0.8061383715423533, - -0.13789763518724496, - -0.15284275396015845, - -0.06595066392665005, - 0.4581516989197012, - 0.34672909458660306, - 0.9683304227319323, - -0.8031642576960822, - -0.19475743579546245, - -0.3213947892100737, - 0.7233450727055821, - -0.5026873321594287, - -0.619582183118377, - -0.1027729043337362, - -0.15623672033119163, - -0.44290971066611906, - -0.500387104235799, - 0.8465311985520256, - -0.11373850989308609, - 0.7226982095236612, - 0.10065062489969612, - -0.8988233409502375, - 0.9985649368254532, - 0.6720551701599038, - 0.9379925145695025, - 0.8527339660162552, - 0.6973914688286109, - -0.667377778792172, - -0.02871774909856306, - -0.5725054016016367, - -0.19791941490109477, - -0.8827292000556421, - -0.2420537620461678, - 0.9706176875594519, - -0.4695938836556961, - 0.5681412038971387, - -0.08998326532171341, - -0.15398502801967417, - 0.9146352817193464, - 0.9908453789854277, - 0.11153664681123643, - 0.436816550592652, - -0.6904063494518717, - -0.4065843490108716, - 0.9374187299383177, - 0.15836058163251243, - 0.08439040274854848, - 0.4959511207581282, - -0.8856694541850338, - 0.16835518891794243, - 0.005700765839027122, - 0.7054397840965707, - -0.6851345441210335, - 0.9215578065489007, - -0.8397770695188262, - -0.6283500780385536, - 0.19007021290005532, - 0.3504251072081803, - -0.5295922099981376, - -0.7602267721057516, - 0.780574628258875, - -0.5075693044227503, - 0.1890383070668824, - 0.23876302066420618, - -0.16155016932825506, - 0.16734457858244944, - 0.04556543106391775, - 0.8694125154728545, - -0.5914816011529271, - 0.4323836015788296, - -0.522628094768308, - -0.208428306417491, - 0.34338044591994255, - -0.40000584040247555, - -0.36764560745629193, - 0.5037289848288042, - -0.8549137710136854, - -0.08342895476282775, - 0.9969088817088847, - 0.9921928957101889, - -0.853478557800734, - -0.5736913754659192, - -0.4695991704991973, - 0.8665187559874181, - 0.7617283473728791, - 0.7585404849690855, - -0.2609458225222321, - -0.6845063352855361, - 0.667489909279614, - 0.4070798501747419, - 0.22335553145190024, - 0.9744661272630086, - 0.3079526354214652, - -0.9843537856956841, - 0.6342082702309233, - -0.4012424956000442, - 0.32677742993215464, - 0.8778600078542078, - -0.7314177712132646, - -0.7691426591617956, - -0.7859280445811647, - 0.10644728176963181, - -0.45530357537036736, - 0.20965965406044784, - 0.43522437427759586, - -0.5928053753450941, - 0.26847591777015944, - -0.4720321967391812, - -0.02293629570124689, - 0.8106729821586465, - 0.6922074265897109, - -0.8154030645745332, - -0.15284845487254728, - -0.44663955205549666, - -0.9929086218244354, - 0.5422384460392542, - 0.27422675460275925, - -0.4760894751313036, - 0.48246181669586163, - 0.10336084225278253, - -0.14462616203864131, - -0.9806606007833201, - -0.8495122798524659, - 0.7662127866002859, - 0.8078571431197863, - 0.09118057841104465, - 0.6691900397720334, - 0.16501913297958803, - -0.7038124288650347, - -0.7451089614357225, - -0.38348330013973264, - 0.79796297748518, - 0.5922446097760834, - 0.7214051640018055, - 0.7978492730529492, - -0.5798469233204919, - -0.5009405215541511, - -0.7944127566564287, - 0.5602324837428854, - 0.7682694029020178, - -0.1872452203357664, - 0.2413230203014256, - -0.6908933233355907, - 0.8597620313873489, - 0.7292113924399279, - 0.9524120658619257, - 0.6215434398807937, - 0.7628324093266488, - -0.9504272762036226, - 0.4731289435101642, - -0.33562906410714266, - 0.8616317720966511, - 0.6044702778742779, - 0.7281280567505588, - 0.621498633148778, - -0.46638858081105594, - 0.5747490182709423, - -0.7838087471940858, - 0.7443335658121795, - 0.7171865026755633, - -0.5551325649086711, - 0.6331732111938579, - -0.07939353064211585, - -0.38961826532279886, - 0.5906909983057236, - -0.5448090251844593, - -0.952671130597097, - -0.6137404233445827, - -0.34347609760458697, - 0.7287058840605727, - 0.9337782080967223, - -0.4417500145562572, - 0.2829634772152554, - -0.20064323127987826, - 0.9622993743965202, - 0.07243146495744379, - 0.8784742806494314, - -0.7693164962971448, - 0.9408012220444559, - -0.6428643676550727, - 0.9250686315231109, - -0.46906727495406275, - -0.7831949055705778, - -0.1308724828707113, - 0.4570901213054086, - -0.37264537161001754, - 0.21241770661228654, - 0.022846119338956195, - -0.22960913331054567, - 0.15317608699319907, - -0.4905549877228361, - 0.4175705676683412, - -0.9966174435627411, - 0.8511503309981654, - 0.0769039941855838, - 0.438859998289691, - 0.48390015567895306, - 0.34125700886599897, - -0.27155705643747163, - -0.8600523777473796, - 0.32847536982254466, - -0.3395999279148072, - -0.37216870988328066, - 0.696030559012671, - 0.43950852602790036, - -0.39935546357747165, - -0.3814306755826935, - -0.1832141827615663, - -0.19519922588455074, - -0.40868959494810597, - -0.7454244018816936, - -0.1591073324541834, - 0.880727341460366, - 0.35463589054546585, - 0.8056110914651653, - 0.23102983190276105, - -0.39810025086886935, - 0.09587442627139642, - -0.999188120605425, - -0.4261725662621456, - -0.1402237000203308, - 0.15996956239136395, - 0.30941124740614323, - -0.07002361950597158, - -0.11568040139038516, - -0.5725971980217994, - -0.053627628181347475, - 0.8023616516565084, - 0.5920495202535605, - -0.6606172076038905, - -0.8304089265497565, - 0.030904019830432894, - 0.26588171153159146, - -0.3296234891803982, - 0.6368469290733285, - 0.5022762750814644, - 0.3455913411143341, - -0.5507186680054235, - -0.6017401345468467, - -0.9511492245463473, - -0.5103149118432997, - -0.04972731156238974, - 0.6994753892494638, - -0.8543435416308618, - -0.1711179780045613, - 0.2595307614754274, - -0.6111295265205814, - 0.39270850098100984, - -0.011245661979126131, - -0.5120311208431223, - 0.31211602222356816, - -0.9889103637239365, - 0.5019289532369458, - 0.5400923771480501, - -0.7868254068729221, - -0.14970761211453176, - -0.6482266365869367, - 0.9159320845590795, - 0.03591550088748163, - -0.8995632297187182, - -0.5016034406800567, - 0.6966726947033195, - -0.08707634905965489, - 0.602833203444529, - 0.3351554651727062, - 0.975784906132896, - 0.1909046369388394, - 0.9000792168863119, - 0.782851851620874, - 0.2253046455235257, - 0.4385479225519342, - 0.00955632964880393, - 0.6611383394428301, - 0.09574390122165677, - 0.7944162064665243, - 0.4873108843191698, - -0.05065112635389335, - -0.4816169030699613, - -0.5055205249806809, - 0.27532287355231255, - 0.5316273685943309, - 0.0425996256559642, - 0.2534968739635626, - -0.4508051061649234, - -0.8450332922705284, - -0.428543698273695, - -0.4565697858356308, - -0.3605808631624754, - 0.08030444503691281, - -0.7232518769676886, - -0.5374770405436264, - 0.38789962459810456, - 0.41283828338910444, - -0.8715422985722439, - -0.18480126066682656, - 0.08522228100783069, - -0.1684515317936881, - -0.5863312209711795, - -0.15971296445314875, - 0.8096769566803539, - 0.1681588284084503, - 0.39104597299593613, - 0.7134640646078685, - 0.5311891522361389, - -0.23923794214457672, - -0.9882078328320139, - -0.2964823946563506, - 0.5069502501187717, - 0.7068959011382092, - 0.9068606769402126, - -0.1619574347403563, - 0.49503133795610155, - 0.09226461946767817, - 0.20650517788248268, - -0.5589226113523622, - -0.5611567307571277, - -0.1283280479067268, - -0.9419503601065695, - -0.3277409126032351, - 0.3582837700566994, - -0.19136666172472583, - -0.6699105375929824, - -0.0652197015353797, - -0.7447444054376786, - 0.24451392194812938, - -0.9460670961897246, - -0.21195948732059056, - 0.12878396604954823, - -0.9457959073193751, - 0.2854992960186715, - -0.7286010255388715, - -0.07660311189697655, - -0.8994307330227449, - -0.24179227162372086, - -0.5766794315770369, - -0.3463083902373829, - 0.5224594157880542, - -0.24174756887172744, - 0.5040196471095697, - 0.6638485703105452, - -0.4954569364352388, - -0.8361875344767149, - -0.9612334258999786, - 0.0788380958450674, - 0.9998156570184185, - -0.30007931255963216, - 0.30028818649975, - 0.5624660992217898, - 0.30350931048777885, - 0.5084664081190524, - 0.8992234654319777, - -0.6012786352749342, - -0.9592399653593353, - -0.6952353084304193, - -0.7475580502515875, - 0.33891768923982135, - 0.12793916386003823, - -0.5640709181869927, - 0.3989299424923016, - 0.5337961967124816, - -0.6644217132643955, - 0.21449498778186338, - 0.49585130391057164, - -0.7709342572422047, - 0.6386023486221701, - 0.9294415460681753, - -0.7838025006847869, - -0.948643149005068, - -0.3760855112106096, - 0.3546945737008176, - 0.9163456764117919, - -0.20669111696674536, - 0.4300294100989368, - -0.8480070443138801, - 0.3812288318659607, - 0.2544847912020889, - -0.7961973891080469, - 0.544961769902448, - 0.7005864781775926, - 0.20082322963368826, - -0.7578898698653698, - 0.9676887030293426, - 0.5652706927220392, - -0.30559246938310203, - -0.1432439735305111, - -0.25885824756388764, - 0.011921579354155831, - -0.31753765027743364, - 0.6991512539991547, - 0.6446618361801162, - -0.7889222587120028, - 0.9215751344291572, - 0.27117021220289184, - 0.6574146220049164, - 0.41461728741215387, - -0.1290257099846459, - 0.46759060802678376, - 0.9309474624761553, - -0.45983520722519833, - 0.6163984376135119, - 0.07634581289651154, - -0.033004992233607755, - -0.12885101399221144, - 0.46205242861024454, - -0.46320892390154933, - 0.7034263200386637, - 0.6614620377812068, - -0.8266742038886512, - 0.7632623680055439, - -0.512273121618088, - -0.07058306679353654, - 0.22066340846104127, - -0.24202139174347193, - -0.9426000044598208, - 0.7019056726249182, - -0.6363202856846164, - -0.575760299640522, - 0.595664713656193, - -0.3193223136714327, - 0.7606399595164508, - 0.4023675006644032, - -0.4474628484876302, - -0.9796977711226458, - 0.8961251555540624, - -0.8287740760839575, - 0.4401493282083886, - -0.022844306302425288, - 0.5163293069649328, - 0.38121867889304606, - 0.29180579948190455, - -0.018357329842827674, - 0.5858657362646349, - -0.8138932988906566, - -0.5568071990549397, - 0.3835743105904037, - -0.3875879397398232, - 0.16311117066473435, - -0.05347902248095049, - 0.0618438622915356, - -0.1489923745895705, - 0.49187087342721925, - -0.3384174056081397, - 0.4057098843715379, - -0.45816714604760733, - -0.497192647598117, - -0.7586882304953884, - -0.6148314130969652, - -0.7608905174898943, - 0.07172793076749961, - 0.5243792189682059, - -0.6297003151246761, - -0.5672307202427886, - -0.031602825574582605, - 0.44917000218610315, - 0.9532140457661136, - 0.0492737382172157, - -0.43400259345008996, - -0.7989477838184185, - -0.6117648438178531, - -0.5450336730348906, - -0.6411169126291933, - -0.9717032654878723, - 0.06827017853532658, - -0.4513773464334654, - 0.9485898622054758, - 0.1067179335972166, - 0.3948347858203889, - -0.7474410140831822, - 0.736922394525888, - -0.018242610952442284, - 0.7454394699970692, - 0.1481284392526645, - -0.061206110138979586, - -0.1190624037279795, - -0.6312726592198366, - -0.8972465656328443, - 0.8821271935362065, - -0.04454162546862461, - 0.6442312905988608, - -0.19858511549612046, - -0.8518356594887204, - 0.2588914139039966, - -0.8927818514149337, - -0.7016048310526897, - 0.12567919416912843, - -0.39232897631554464, - 0.9878362454204404, - -0.7630968755770768, - 0.5288868906523885, - 0.21263530248592555, - 0.5814816596868388, - -0.5486257258699307, - 0.04514507026043568, - -0.09897107022249152, - -0.11455799196134331, - 0.7203333316523601, - 0.9800625209335958, - -0.3892395113612792, - 0.24205464214429706, - 0.21926182449026066, - 0.48017861096909, - 0.895180400650756, - -0.5844241883528973, - -0.577949609389214, - 0.32085627438407327, - -0.6858858132382457, - -0.652372903347131, - -0.8498702621976642, - -0.9946485547942285, - -0.09899259076459521, - 0.18762239023912652, - -0.4174814219379448, - -0.5370475308879483, - 0.41391165975806654, - 0.4059751161874343, - -0.09193734718591351, - 0.3747698401424855, - 0.8478220889650974, - 0.5756560532934627, - 0.25011601432844377, - 0.32236608570673564, - 0.8673369168910265, - -0.1497220677915767, - 0.08912475742115733, - 0.2952694468049615, - 0.8168228904252122, - 0.6532623193001192, - -0.8571803262883717, - -0.6681544215585506, - -0.3847763747715349, - 0.4979154441392466, - 0.13841409863818477, - -0.4227788233883165, - -0.7512926836505835, - 0.37735598242418744, - 0.3994673699514346, - 0.8853524814880815, - 0.0009443542358511525, - -0.012409561309831396, - -0.839116296201398, - -0.9202784316327739, - -0.13594267167845486, - -0.35535683328340006, - -0.4992641951759922, - -0.8173462267380296, - 0.9238222043856734, - 0.6719172278123626, - 0.1503982184398609, - 0.9015725556127048, - 0.9991448337549027, - 0.3445631686064785, - -0.4609779480658922, - -0.9195366537107676, - 0.5125376608251142, - -0.05899835349673288, - 0.3030189788673727, - 0.832145575853535, - -0.6370217055537482, - 0.17065925055570874, - 0.26956943890817375, - -0.016548395617973277, - -0.8175151874123456, - -0.3040778874106991, - -0.33338321266981596, - 0.3402670190423709, - 0.7154661888007052, - -0.3403926728420692, - 0.387347347966865, - -0.4235644092770885, - 0.8903870791264212, - 0.6271320694759452, - 0.10019321794356584, - -0.09034818279654111, - -0.37096568565966415, - -0.35345242744801064, - 0.9403694536171543, - -0.19164988599850874, - 0.029192504658210217, - 0.9762384295635083, - 0.31532077296625793, - 0.08518718871439002, - -0.173504858401357, - -0.6248349172109458, - -0.2764412816966495, - 0.5128863081111474, - 0.25081748429001793, - 0.5199810721220339, - -0.5928835232994523, - 0.09843927817066334, - 0.8553455216888954, - -0.1237678098552435, - 0.39650005824434453, - -0.7571478332747603, - 0.9462936316433888, - 0.21774333416392988, - -0.5214050753446993, - -0.6832436723895352, - 0.10167801400533993, - 0.10450281810746609, - -0.8135815974369549, - 0.9845142787904393, - 0.8258597569683359, - -0.0771042116424332, - -0.7650677018208849, - 0.6642863475378025, - -0.003248990585609457, - 0.4332066519835822, - 0.01774403013902348, - -0.4531502065734516, - 0.6694478911533888, - 0.9604892652067509, - -0.5125381878497233, - 0.10253015376093932, - -0.23282797350570017, - 0.8437362998631237, - 0.016481783185787968, - 0.7586525102929522, - 0.7280538688571763, - -0.44750519435591807, - 0.5800123640062269, - -0.17011515289730972, - 0.8684967873650402, - 0.015475351619312905, - 0.6410989463711567, - -0.4343220334343467, - -0.4028883004564656, - 0.17387544482832218, - 0.9978046664586775, - -0.020719306688598005, - -0.7028091632344218, - 0.07716115540774737, - -0.30975211661398494, - 0.10383483414162198, - 0.08686012591734604, - -0.08931076626699785, - -0.35644529954192516, - -0.6226952525857892, - 0.3949968552411427, - 0.14359528396980226, - -0.5328751078842138, - 0.5510889501985177, - -0.9127054018053962 - ], - "sample_rate": 48000 -} \ No newline at end of file From 6bcedc36e8005ff03254f4321c3a5213af3958e1 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 21 May 2026 06:49:48 -0700 Subject: [PATCH 048/162] Avoid extra copy in FastMath tanh --- NeuralAudio/Activation.h | 22 +++++++++------------- NeuralAudio/WaveNet.h | 12 +----------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/NeuralAudio/Activation.h b/NeuralAudio/Activation.h index cd77a0c..a04af9b 100644 --- a/NeuralAudio/Activation.h +++ b/NeuralAudio/Activation.h @@ -17,17 +17,15 @@ namespace NeuralAudio struct StdMath { template - static auto Tanh(Matrix& x) + static void Tanh(Matrix *x) { - float* data = x.data(); - size_t size = x.rows() * x.cols(); + float* data = x->data(); + size_t size = x->rows() * x->cols(); for (size_t pos = 0; pos < size; pos++) { data[pos] = Tanh(data[pos]); } - - return x; } static inline float Tanh(const float x) @@ -44,17 +42,15 @@ namespace NeuralAudio struct FastMath { template - static auto Tanh(Matrix& x) - { - float* data = x.data(); - size_t size = x.rows() * x.cols(); + static void Tanh(Matrix *x) + { + float* data = x->data(); + size_t size = x->rows() * x->cols(); for (size_t pos = 0; pos < size; pos++) { data[pos] = Tanh(data[pos]); } - - return x; } static inline float Tanh(const float x) @@ -83,9 +79,9 @@ namespace NeuralAudio struct EigenMath { template - static auto Tanh(const Matrix& x) + static void Tanh(Matrix* x) { - return x.array().tanh(); + *x = x->array().tanh(); } static inline float Tanh(const float x) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index bf3f47b..81da190 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -226,17 +226,7 @@ namespace NeuralAudio inputMixin.ProcessAcc(condition, block); - block = WAVENET_MATH::Tanh(block); - - //block = block.array().tanh(); - - //float* data = block.data(); - //size_t size = block.rows() * block.cols(); - - //for (size_t pos = 0; pos < size; pos++) - //{ - // data[pos] = WAVENET_MATH::Tanh(data[pos]); - //} + WAVENET_MATH::Tanh(&block); const_cast&>(headInput).leftCols(numFrames).noalias() += block; From 3a4cfa7a287e3038083b9cb65e40cb7c232f9a71 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 22 May 2026 16:05:23 -0700 Subject: [PATCH 049/162] First shot at creating a release with ModelTest included --- .github/workflows/release.yml | 150 ++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..77baa30 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,150 @@ +name: Release + +on: + workflow_dispatch: + +env: + BUILD_TYPE: Release + +jobs: + create_release: + name: Create release + runs-on: ubuntu-latest + outputs: + upload_url: ${{steps.create_release.outputs.upload_url}} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Create release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + with: + draft: true + tag_name: ${{github.ref}} + release_name: Release ${{github.ref}} + + build-linux-x64: + name: Build Linux x64 + needs: create_release + strategy: + matrix: + native_arch: [x86-64, x86-64-v3] + runs-on: ubuntu-latest + env: + ARCHIVE_NAME: model_test_linux_${{ matrix.native_arch }}.tgz + steps: + - uses: actions/checkout@v3.3.0 + with: + submodules: recursive + + - name: Build + working-directory: ${{github.workspace}}/build + env: + CXX: clang++ + CXXFLAGS: "-Wall -O3 -march=${{ matrix.native_arch }}" + run: | + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON + cmake --build . --config $BUILD_TYPE -j4 + + - name: Add Archive + working-directory: ${{github.workspace}}/build + run: | + mkdir ModelTest + cp Utils/ModelTest/ModelTest ModelTest + cp -r Utils/Models ModelTest + tar -czf ${{ env.ARCHIVE_NAME }} ModelTest + + - name: Upload Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: ./build/${{ env.ARCHIVE_NAME }} + asset_name: ${{ env.ARCHIVE_NAME }} + asset_content_type: application/tgz + + build-rpi: + name: Build Raspberry Pi + needs: create_release + strategy: + matrix: + native_arch: [rpi4, rpi5] + runs-on: ubuntu-latest + container: + image: ghcr.io/tttapa/docker-arm-cross-toolchain:aarch64-rpi3-linux-gnu-gcc12 + env: + ARCHIVE_NAME: model_test_${{ matrix.native_arch }}.tgz + steps: + - uses: actions/checkout@v3.3.0 + with: + submodules: recursive + + - name: Build + working-directory: ${{github.workspace}}/build + run: | + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON + cmake --build . --config $BUILD_TYPE -j4 + + - name: Add Archive + working-directory: ${{github.workspace}}/build + run: | + mkdir ModelTest + cp Utils/ModelTest/ModelTest ModelTest + cp -r Utils/Models ModelTest + tar -czf ${{ env.ARCHIVE_NAME }} ModelTest + + - name: Upload Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: ./build/${{ env.ARCHIVE_NAME }} + asset_name: ${{ env.ARCHIVE_NAME }} + asset_content_type: application/tgz + + build-windows: + name: Build Windows + needs: create_release + strategy: + matrix: + native_arch: [x86-64, x86-64-v3] + runs-on: windows-latest + env: + ARCHIVE_NAME: model_test_win_${{ matrix.native_arch }}.zip + steps: + - uses: actions/checkout@v3.3.0 + with: + submodules: recursive + + - name: Build + working-directory: ${{github.workspace}}/build + env: + CXXFLAGS: "/O2 -march=${{ matrix.native_arch }}" + run: | + cmake.exe -G "Visual Studio 17 2022" -A x64 -T ClangCL -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON .. + cmake --build . --config=release -j4 + + - name: Add Archive + working-directory: ${{github.workspace}}/build + run: | + mkdir ModelTest + cp Utils/ModelTest/ModelTest.exe ModelTest + cp -r Utils/Models ModelTest + Compress-Archive -Path ModelTest -Destination ${{ env.ARCHIVE_NAME }} + + - name: Upload Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: ./${{ env.ARCHIVE_NAME }} + asset_name: ${{ env.ARCHIVE_NAME }} + asset_content_type: application/zip From 6ec34e37a45da31480363b622fc1f49e7e224d90 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 22 May 2026 16:16:41 -0700 Subject: [PATCH 050/162] Build utils in release. Fix rpi cmake build tools path --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77baa30..21c5616 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,7 +48,7 @@ jobs: CXX: clang++ CXXFLAGS: "-Wall -O3 -march=${{ matrix.native_arch }}" run: | - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON cmake --build . --config $BUILD_TYPE -j4 - name: Add Archive @@ -88,7 +88,7 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build run: | - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DCMAKE_TOOLCHAIN_FILE=/home/develop/opt/x-tools/aarch64-rpi3-linux-gnu/aarch64-${{ matrix.native_arch }}-linux-gnu.toolchain.cmake cmake --build . --config $BUILD_TYPE -j4 - name: Add Archive @@ -128,7 +128,7 @@ jobs: env: CXXFLAGS: "/O2 -march=${{ matrix.native_arch }}" run: | - cmake.exe -G "Visual Studio 17 2022" -A x64 -T ClangCL -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON .. + cmake.exe -G "Visual Studio 17 2022" -A x64 -T ClangCL -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON .. cmake --build . --config=release -j4 - name: Add Archive From ed80e54ffa3f6dda8f58e54423acb901966d6886 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 22 May 2026 16:21:34 -0700 Subject: [PATCH 051/162] Fix Windows ModelTest exe path Update the path for ModelTest.exe in the release workflow. --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 21c5616..9618b90 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -135,7 +135,7 @@ jobs: working-directory: ${{github.workspace}}/build run: | mkdir ModelTest - cp Utils/ModelTest/ModelTest.exe ModelTest + cp Utils/ModelTest/Release/ModelTest.exe ModelTest cp -r Utils/Models ModelTest Compress-Archive -Path ModelTest -Destination ${{ env.ARCHIVE_NAME }} From 51826803f276e7e9014232e05a90a97c7dba2d57 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 22 May 2026 16:27:59 -0700 Subject: [PATCH 052/162] Fix Windows asset path --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9618b90..c49b966 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -145,6 +145,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ./${{ env.ARCHIVE_NAME }} + asset_path: ./build/${{ env.ARCHIVE_NAME }} asset_name: ${{ env.ARCHIVE_NAME }} asset_content_type: application/zip From 6fef2e97412a7d2ffa3a9e2d974039c5e8a08b49 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 22 May 2026 16:44:59 -0700 Subject: [PATCH 053/162] More method override annotations --- NeuralAudio/InternalModel.h | 12 ++++++------ NeuralAudio/NeuralModelImpl.h | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index 699c307..39ec829 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -78,7 +78,7 @@ namespace NeuralAudio return true; } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -192,7 +192,7 @@ namespace NeuralAudio return EModelLoadMode::Internal; } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { nlohmann::json config = modelJson.at("config"); @@ -268,7 +268,7 @@ namespace NeuralAudio return true; } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -307,7 +307,7 @@ namespace NeuralAudio return vec; } - bool CreateModelFromKerasJson(const nlohmann::json& modelJson) + bool CreateModelFromKerasJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -430,7 +430,7 @@ namespace NeuralAudio } } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -469,7 +469,7 @@ namespace NeuralAudio return vec; } - bool CreateModelFromKerasJson(const nlohmann::json& modelJson) + bool CreateModelFromKerasJson(const nlohmann::json& modelJson) override { if (model != nullptr) { diff --git a/NeuralAudio/NeuralModelImpl.h b/NeuralAudio/NeuralModelImpl.h index 36456aa..06d0939 100644 --- a/NeuralAudio/NeuralModelImpl.h +++ b/NeuralAudio/NeuralModelImpl.h @@ -7,6 +7,9 @@ namespace NeuralAudio { class NeuralModelImpl : public NeuralModel { + public: + using NeuralModel::Prewarm; + protected: void ReadNAMConfig(const nlohmann::json& modelJson) { From b5233cc7e485df93b04e0128cfd75a3eb1a2166e Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 23 May 2026 07:43:20 -0700 Subject: [PATCH 054/162] ModelTest release archive rename --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c49b966..8cd180d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,7 +36,7 @@ jobs: native_arch: [x86-64, x86-64-v3] runs-on: ubuntu-latest env: - ARCHIVE_NAME: model_test_linux_${{ matrix.native_arch }}.tgz + ARCHIVE_NAME: ModelTest_linux_${{ matrix.native_arch }}.tgz steps: - uses: actions/checkout@v3.3.0 with: @@ -79,7 +79,7 @@ jobs: container: image: ghcr.io/tttapa/docker-arm-cross-toolchain:aarch64-rpi3-linux-gnu-gcc12 env: - ARCHIVE_NAME: model_test_${{ matrix.native_arch }}.tgz + ARCHIVE_NAME: ModelTest_${{ matrix.native_arch }}.tgz steps: - uses: actions/checkout@v3.3.0 with: @@ -117,7 +117,7 @@ jobs: native_arch: [x86-64, x86-64-v3] runs-on: windows-latest env: - ARCHIVE_NAME: model_test_win_${{ matrix.native_arch }}.zip + ARCHIVE_NAME: ModelTest_win_${{ matrix.native_arch }}.zip steps: - uses: actions/checkout@v3.3.0 with: From 254753c2dc823a84fa1a3e491843012e0c47e11c Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 23 May 2026 08:32:09 -0700 Subject: [PATCH 055/162] Look for ModelTest Model folder in current directory or up the path --- Utils/ModelTest/ModelTest.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Utils/ModelTest/ModelTest.cpp b/Utils/ModelTest/ModelTest.cpp index 537c471..572edba 100644 --- a/Utils/ModelTest/ModelTest.cpp +++ b/Utils/ModelTest/ModelTest.cpp @@ -230,17 +230,21 @@ int RunDefaultTests(int blockSize) { std::filesystem::path modelPath = std::filesystem::current_path(); - while (modelPath.filename() != "Utils") + do { + if (std::filesystem::exists((modelPath / "Models"))) + break; + modelPath = modelPath.parent_path(); + } + while (modelPath != modelPath.root_path()); - if (modelPath == modelPath.root_path()) - { - std::cout << "Unable to find Models: " << std::filesystem::current_path() << std::endl; - std::cout << "ModelTest must be run from within the Utils subdirectory" << std::endl; + if (modelPath == modelPath.root_path()) + { + std::cout << "Unable to find Models: " << std::filesystem::current_path() << std::endl; + std::cout << "ModelTest must be run from within the Utils subdirectory" << std::endl; - return -1; - } + return -1; } modelPath = modelPath / "Models"; @@ -294,6 +298,7 @@ int main(int argc, char* argv[]) int blockSize = 64; + //std::filesystem::path modelPath = R"(C:\Users\oliph\Downloads\Fender Deluxe Reverb (A2)\Deluxe Reverb.nam)"; std::filesystem::path modelPath = program.get("model_file"); blockSize = program.get("--block_size"); From 03280ff1932463ca8da470e8ab1798ccd4eb5ec4 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 23 May 2026 08:33:05 -0700 Subject: [PATCH 056/162] Better error message when ModelTest can't find Models folder --- Utils/ModelTest/ModelTest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Utils/ModelTest/ModelTest.cpp b/Utils/ModelTest/ModelTest.cpp index 572edba..27b4994 100644 --- a/Utils/ModelTest/ModelTest.cpp +++ b/Utils/ModelTest/ModelTest.cpp @@ -242,7 +242,8 @@ int RunDefaultTests(int blockSize) if (modelPath == modelPath.root_path()) { std::cout << "Unable to find Models: " << std::filesystem::current_path() << std::endl; - std::cout << "ModelTest must be run from within the Utils subdirectory" << std::endl; + std::cout << "ModelTest looks for a \"Models\" folder in current folder or up the path." << std::endl; + std::cout << "You can also specify a specific model to test by passing the path on the commandline." << std::endl; return -1; } From 7bc4c24256f88b1a249eb13be706ee9856773ced Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 27 May 2026 07:41:32 -0700 Subject: [PATCH 057/162] Update NeuralAmpModelerCore --- deps/NeuralAmpModelerCore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index ab72c07..5656639 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit ab72c078cd141be60d963d357bd501d56cc9606b +Subproject commit 5656639bb6dd65340369eeeb50e4d70c32a42922 From 3ba502fc75d38b3dcc4fd4e470f936fc16fa79a9 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 27 May 2026 09:35:32 -0700 Subject: [PATCH 058/162] Add NeuralModelLoader class in place of static model creation methods --- NeuralAudio/InternalModel.h | 8 +- NeuralAudio/NAMModel.h | 6 +- NeuralAudio/NeuralModel.cpp | 18 ++- NeuralAudio/NeuralModel.h | 114 +++++++++++-------- NeuralAudio/NeuralModelImpl.h | 7 ++ NeuralAudio/RTNeuralLoader.cpp | 22 ++-- NeuralAudioCAPI/NeuralAudioCApi.cpp | 40 +++++-- NeuralAudioCAPI/NeuralAudioCApi.h | 14 ++- NeuralAudioCSharp/NeuralAudio/NativeApi.cs | 16 ++- NeuralAudioCSharp/NeuralAudio/NeuralModel.cs | 63 ++++++---- NeuralAudioCSharp/NeuralAudioTest/Program.cs | 6 +- Utils/ModelTest/ModelTest.cpp | 42 +++---- 12 files changed, 225 insertions(+), 131 deletions(-) diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index 39ec829..80f4eb7 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -92,7 +92,7 @@ namespace NeuralAudio model->SetWeights(modelJson.at("weights")); - SetMaxAudioBufferSize(defaultMaxAudioBufferSize); + SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); return true; } @@ -210,7 +210,7 @@ namespace NeuralAudio model->SetWeights(modelJson.at("weights")); - SetMaxAudioBufferSize(defaultMaxAudioBufferSize); + SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); return true; } @@ -282,7 +282,7 @@ namespace NeuralAudio model->SetNAMWeights(modelJson.at("weights")); - SetMaxAudioBufferSize(defaultMaxAudioBufferSize); + SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); return true; } @@ -444,7 +444,7 @@ namespace NeuralAudio model->SetNAMWeights(modelJson.at("weights")); - SetMaxAudioBufferSize(defaultMaxAudioBufferSize); + SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); return true; } diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index 12d338e..8492590 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -16,8 +16,6 @@ namespace NeuralAudio NAMModel() { nam::activations::Activation::enable_fast_tanh(); - - slimmableSize = defaultQualityScaleFactor; } ~NAMModel() @@ -36,6 +34,8 @@ namespace NeuralAudio if (namModel) namModel.reset(); + slimmableSize = loader->GetDefaultQualityScaleFactor(); + ReadNAMConfig(modelJson); namModel = nam::get_dsp(modelJson); @@ -51,7 +51,7 @@ namespace NeuralAudio slim->SetSlimmableSize(slimmableSize); } - SetMaxAudioBufferSize(defaultMaxAudioBufferSize); + SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); return true; } diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 122c647..bb0a3c9 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -85,7 +85,7 @@ namespace NeuralAudio return true; } - bool NeuralModel::SupportsWaveNetLoadMode(EModelLoadMode mode) + bool NeuralModelLoader::SupportsWaveNetLoadMode(EModelLoadMode mode) { if (mode == EModelLoadMode::NAMCore) #ifdef BUILD_NAMCORE @@ -104,7 +104,7 @@ namespace NeuralAudio return true; } - bool NeuralModel::SupportsLSTMLoadMode(EModelLoadMode mode) + bool NeuralModelLoader::SupportsLSTMLoadMode(EModelLoadMode mode) { if (mode == EModelLoadMode::NAMCore) #ifdef BUILD_NAMCORE @@ -116,7 +116,7 @@ namespace NeuralAudio return true; } - NeuralModel* NeuralModel::CreateFromFile(std::filesystem::path modelPath) + NeuralModel* NeuralModelLoader::CreateFromFile(std::filesystem::path modelPath) { if (!std::filesystem::exists(modelPath)) return nullptr; @@ -137,7 +137,7 @@ namespace NeuralAudio return (major > 0) || (minor > 5) || ((minor == 5) && (patch > 4)); } - NeuralModel* NeuralModel::CreateFromStream(std::basic_istream& jsonStream, std::filesystem::path extension) + NeuralModel* NeuralModelLoader::CreateFromStream(std::basic_istream& jsonStream, std::filesystem::path extension) { EnsureModelDefsAreLoaded(); @@ -155,6 +155,7 @@ namespace NeuralAudio { NAMModel* model = new NAMModel; + model->SetModelLoader(this); model->LoadFromJson(modelJson); newModel = model; @@ -197,7 +198,7 @@ namespace NeuralAudio if (wavenetLoadMode == EModelLoadMode::RTNeural) { #ifdef BUILD_STATIC_RTNEURAL - newModel = RTNeuralLoadNAMWaveNet(modelJson); + newModel = RTNeuralLoadNAMWaveNet(modelJson, this); #endif } @@ -209,6 +210,7 @@ namespace NeuralAudio { auto model = modelDef->CreateModel(); + model->SetModelLoader(this); model->LoadFromNAMJson(modelJson); newModel = model; @@ -223,6 +225,8 @@ namespace NeuralAudio // Use a dynamic model if we had no static definition InternalWaveNetModelDyn* model = new InternalWaveNetModelDyn; + model->SetModelLoader(this); + if (model->LoadFromNAMJson(modelJson)) { newModel = model; @@ -234,7 +238,7 @@ namespace NeuralAudio #ifdef BUILD_STATIC_RTNEURAL if (lstmLoadMode == EModelLoadMode::RTNeural) { - newModel = RTNeuralLoadNAMLSTM(modelJson); + newModel = RTNeuralLoadNAMLSTM(modelJson, this); } #endif @@ -255,6 +259,8 @@ namespace NeuralAudio { InternalLSTMModelDyn* model = new InternalLSTMModelDyn; + model->SetModelLoader(this); + if (model->LoadFromNAMJson(modelJson)) { newModel = model; diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index ee0c4a3..ef3559c 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -22,51 +22,10 @@ namespace NeuralAudio class NeuralModel { public: - static NeuralModel* CreateFromFile(std::filesystem::path modelPath); - static NeuralModel* CreateFromStream(std::basic_istream& stream, std::filesystem::path extension); - virtual ~NeuralModel() { } - static bool SetLSTMLoadMode(EModelLoadMode val) - { - if (!SupportsLSTMLoadMode(val)) - return false; - - lstmLoadMode = val; - - return true; - } - - static bool SetWaveNetLoadMode(EModelLoadMode val) - { - if (!SupportsWaveNetLoadMode(val)) - return false; - - wavenetLoadMode = val; - - return true; - } - - static bool SupportsWaveNetLoadMode(EModelLoadMode mode); - static bool SupportsLSTMLoadMode(EModelLoadMode mode); - - static void SetAudioInputLevelDBu(float audioDBu) - { - audioInputLevelDBu = audioDBu; - } - - static void SetDefaultMaxAudioBufferSize(int maxSize) - { - defaultMaxAudioBufferSize = maxSize; - } - - static void SetDefaultQualityScaleFactor(float scaleFactor) - { - defaultQualityScaleFactor = scaleFactor; - } - virtual EModelLoadMode GetLoadMode() { return EModelLoadMode::Internal; @@ -149,17 +108,80 @@ namespace NeuralAudio } protected: + float audioInputLevelDBu = 12; float modelInputLevelDBu = 12; float modelOutputLevelDBu = 12; float modelLoudnessDB = -18; float sampleRate = 48000; std::string modelVersion = ""; std::vector> metadata; + }; + + class NeuralModelLoader + { + public: + NeuralModel* CreateFromFile(std::filesystem::path modelPath); + NeuralModel* CreateFromStream(std::basic_istream& stream, std::filesystem::path extension); + + bool SetLSTMLoadMode(EModelLoadMode val) + { + if (!SupportsLSTMLoadMode(val)) + return false; + + lstmLoadMode = val; - inline static float audioInputLevelDBu = 12; - inline static EModelLoadMode lstmLoadMode = EModelLoadMode::Internal; - inline static EModelLoadMode wavenetLoadMode = EModelLoadMode::Internal; - inline static int defaultMaxAudioBufferSize = 128; - inline static float defaultQualityScaleFactor = DEFAULT_QUALITY_SCALE; + return true; + } + + bool SetWaveNetLoadMode(EModelLoadMode val) + { + if (!SupportsWaveNetLoadMode(val)) + return false; + + wavenetLoadMode = val; + + return true; + } + + bool SupportsWaveNetLoadMode(EModelLoadMode mode); + bool SupportsLSTMLoadMode(EModelLoadMode mode); + + void SetAudioInputLevelDBu(float audioDBu) + { + audioInputLevelDBu = audioDBu; + } + + float GetAudioInputLevelDBu() + { + return audioInputLevelDBu; + } + + void SetDefaultMaxAudioBufferSize(int maxSize) + { + defaultMaxAudioBufferSize = maxSize; + } + + int GetDefaultMaxAudioBufferSize() + { + return defaultMaxAudioBufferSize; + } + + void SetDefaultQualityScaleFactor(float scaleFactor) + { + defaultQualityScaleFactor = scaleFactor; + } + + float GetDefaultQualityScaleFactor() + { + return defaultQualityScaleFactor; + } + + protected: + EModelLoadMode lstmLoadMode = EModelLoadMode::Internal; + EModelLoadMode wavenetLoadMode = EModelLoadMode::Internal; + float audioInputLevelDBu = 12; + int defaultMaxAudioBufferSize = 128; + float defaultQualityScaleFactor = DEFAULT_QUALITY_SCALE; }; + } \ No newline at end of file diff --git a/NeuralAudio/NeuralModelImpl.h b/NeuralAudio/NeuralModelImpl.h index 06d0939..8eef75b 100644 --- a/NeuralAudio/NeuralModelImpl.h +++ b/NeuralAudio/NeuralModelImpl.h @@ -10,6 +10,11 @@ namespace NeuralAudio public: using NeuralModel::Prewarm; + void SetModelLoader(NeuralModelLoader* loader) + { + this->loader = loader; + } + protected: void ReadNAMConfig(const nlohmann::json& modelJson) { @@ -96,5 +101,7 @@ namespace NeuralAudio Process(input.data(), output.data(), blockSize); } } + + NeuralModelLoader *loader; }; } \ No newline at end of file diff --git a/NeuralAudio/RTNeuralLoader.cpp b/NeuralAudio/RTNeuralLoader.cpp index 3afde01..ef03c4e 100644 --- a/NeuralAudio/RTNeuralLoader.cpp +++ b/NeuralAudio/RTNeuralLoader.cpp @@ -45,7 +45,7 @@ namespace NeuralAudio return nullptr; } - NeuralModel* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson) + NeuralModel* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson, NeuralModelLoader *loader) { nlohmann::json config = modelJson.at("config"); @@ -58,6 +58,7 @@ namespace NeuralAudio { auto model = modelDef->CreateModel(); + model->SetModelLoader(loader); model->LoadFromNAMJson(modelJson); return model; @@ -66,7 +67,7 @@ namespace NeuralAudio return nullptr; } - NeuralModel* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson) + NeuralModel* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader *loader) { nlohmann::json config = modelJson.at("config"); @@ -75,17 +76,20 @@ namespace NeuralAudio if (modelDef != nullptr) { RTNeuralModel* model = modelDef->CreateModel(); + + model->SetModelLoader(loader); model->LoadFromNAMJson(modelJson); - if (model != nullptr) - return model; + return model; + } - // If we didn't have a static model that matched, use RTNeural's dynamic model - RTNeuralModelDyn* dynModel = new RTNeuralModelDyn; - dynModel->LoadFromNAMJson(modelJson); + // If we didn't have a static model that matched, use RTNeural's dynamic model + RTNeuralModelDyn* dynModel = new RTNeuralModelDyn; - return dynModel; - } + dynmodel->SetModelLoader(loader); + dynModel->LoadFromNAMJson(modelJson); + + return dynModel; return nullptr; } diff --git a/NeuralAudioCAPI/NeuralAudioCApi.cpp b/NeuralAudioCAPI/NeuralAudioCApi.cpp index 04d8cdd..0aac758 100644 --- a/NeuralAudioCAPI/NeuralAudioCApi.cpp +++ b/NeuralAudioCAPI/NeuralAudioCApi.cpp @@ -6,11 +6,31 @@ struct NeuralModel NeuralAudio::NeuralModel* model; }; -NeuralModel* CreateModelFromFile(const wchar_t* modelPath) +struct NeuralModelLoader +{ + NeuralAudio::NeuralModelLoader* loader; +}; + +NeuralModelLoader* CreateLoader() +{ + NeuralModelLoader* loader = new NeuralModelLoader(); + + loader->loader = new NeuralAudio::NeuralModelLoader(); + + return loader; +} + +void DeleteLoader(NeuralModelLoader* loader) +{ + delete loader->loader; + delete loader; +} + +NeuralModel* CreateModelFromFile(NeuralModelLoader* loader, const wchar_t* modelPath) { NeuralModel* model = new NeuralModel(); - model->model = NeuralAudio::NeuralModel::CreateFromFile(modelPath); + model->model = loader->loader->CreateFromFile(modelPath); return model; } @@ -21,24 +41,24 @@ void DeleteModel(NeuralModel* model) delete model; } -void SetLSTMLoadMode(int loadMode) +void SetLSTMLoadMode(NeuralModelLoader* loader, int loadMode) { - NeuralAudio::NeuralModel::SetLSTMLoadMode((NeuralAudio::EModelLoadMode)loadMode); + loader->loader->SetLSTMLoadMode((NeuralAudio::EModelLoadMode)loadMode); } -void SetWaveNetLoadMode(int loadMode) +void SetWaveNetLoadMode(NeuralModelLoader* loader, int loadMode) { - NeuralAudio::NeuralModel::SetWaveNetLoadMode((NeuralAudio::EModelLoadMode)loadMode); + loader->loader->SetWaveNetLoadMode((NeuralAudio::EModelLoadMode)loadMode); } -void SetAudioInputLevelDBu(float audioDBu) +void SetAudioInputLevelDBu(NeuralModelLoader* loader, float audioDBu) { - NeuralAudio::NeuralModel::SetAudioInputLevelDBu(audioDBu); + loader->loader->SetAudioInputLevelDBu(audioDBu); } -void SetDefaultMaxAudioBufferSize(int maxSize) +void SetDefaultMaxAudioBufferSize(NeuralModelLoader* loader, int maxSize) { - NeuralAudio::NeuralModel::SetDefaultMaxAudioBufferSize(maxSize); + loader->loader->SetDefaultMaxAudioBufferSize(maxSize); } int GetLoadMode(NeuralModel* model) diff --git a/NeuralAudioCAPI/NeuralAudioCApi.h b/NeuralAudioCAPI/NeuralAudioCApi.h index 28d8886..18a34b6 100644 --- a/NeuralAudioCAPI/NeuralAudioCApi.h +++ b/NeuralAudioCAPI/NeuralAudioCApi.h @@ -13,19 +13,23 @@ extern "C" { #endif struct NeuralModel; +struct NeuralModelLoader; +NA_EXTERN NeuralModelLoader* CreateLoader(); -NA_EXTERN NeuralModel* CreateModelFromFile(const wchar_t* modelPath); +NA_EXTERN void DeleteLoader(NeuralModelLoader* loader); + +NA_EXTERN NeuralModel* CreateModelFromFile(NeuralModelLoader *loader, const wchar_t* modelPath); NA_EXTERN void DeleteModel(NeuralModel* model); -NA_EXTERN void SetLSTMLoadMode(int loadMode); +NA_EXTERN void SetLSTMLoadMode(NeuralModelLoader* loader, int loadMode); -NA_EXTERN void SetWaveNetLoadMode(int loadMode); +NA_EXTERN void SetWaveNetLoadMode(NeuralModelLoader* loader, int loadMode); -NA_EXTERN void SetAudioInputLevelDBu(float audioDBu); +NA_EXTERN void SetAudioInputLevelDBu(NeuralModelLoader* loader, float audioDBu); -NA_EXTERN void SetDefaultMaxAudioBufferSize(int maxSize); +NA_EXTERN void SetDefaultMaxAudioBufferSize(NeuralModelLoader* loader, int maxSize); NA_EXTERN int GetLoadMode(NeuralModel* model); diff --git a/NeuralAudioCSharp/NeuralAudio/NativeApi.cs b/NeuralAudioCSharp/NeuralAudio/NativeApi.cs index a562e97..54f9393 100644 --- a/NeuralAudioCSharp/NeuralAudio/NativeApi.cs +++ b/NeuralAudioCSharp/NeuralAudio/NativeApi.cs @@ -9,22 +9,28 @@ static class NativeApi public const string NEURAL_AUDIO_LIB_NAME = "NeuralAudioCAPI"; [DllImport(NEURAL_AUDIO_LIB_NAME)] - public static extern IntPtr CreateModelFromFile([MarshalAs(UnmanagedType.LPWStr)]string modelPath); + public static extern IntPtr CreateLoader(); + + [DllImport(NEURAL_AUDIO_LIB_NAME)] + public static extern void DeleteLoader(IntPtr loader); + + [DllImport(NEURAL_AUDIO_LIB_NAME)] + public static extern IntPtr CreateModelFromFile(IntPtr loader, [MarshalAs(UnmanagedType.LPWStr)]string modelPath); [DllImport(NEURAL_AUDIO_LIB_NAME)] public static extern void DeleteModel(IntPtr model); [DllImport(NEURAL_AUDIO_LIB_NAME)] - public static extern void SetLSTMLoadMode(int loadMode); + public static extern void SetLSTMLoadMode(IntPtr loader, int loadMode); [DllImport(NEURAL_AUDIO_LIB_NAME)] - public static extern void SetWaveNetLoadMode(int loadMode); + public static extern void SetWaveNetLoadMode(IntPtr loader, int loadMode); [DllImport(NEURAL_AUDIO_LIB_NAME)] - public static extern void SetAudioInputLevelDBu(float audioDBu); + public static extern void SetAudioInputLevelDBu(IntPtr loader, float audioDBu); [DllImport(NEURAL_AUDIO_LIB_NAME)] - public static extern void SetDefaultMaxAudioBufferSize(int maxSize); + public static extern void SetDefaultMaxAudioBufferSize(IntPtr loader, int maxSize); [DllImport(NEURAL_AUDIO_LIB_NAME)] public static extern int GetLoadMode(IntPtr model); diff --git a/NeuralAudioCSharp/NeuralAudio/NeuralModel.cs b/NeuralAudioCSharp/NeuralAudio/NeuralModel.cs index 482ae99..215e60a 100644 --- a/NeuralAudioCSharp/NeuralAudio/NeuralModel.cs +++ b/NeuralAudioCSharp/NeuralAudio/NeuralModel.cs @@ -3,31 +3,58 @@ namespace NeuralAudio { - public class NeuralModel + public enum EModelLoadMode + { + Internal, + RTNeural, + NAMCore + }; + + public class NeuralModelLoader { - public enum EModelLoadMode + IntPtr nativeLoader; + + public NeuralModelLoader() + { + nativeLoader = NativeApi.CreateLoader(); + } + + ~NeuralModelLoader() { - Internal, - RTNeural, - NAMCore - }; + NativeApi.DeleteLoader(nativeLoader); + } - IntPtr nativeModel; + public void SetLSTMModelLoadMode(EModelLoadMode mode) + { + NativeApi.SetLSTMLoadMode(nativeLoader, (int)mode); + } - public static void SetLSTMModelLoadMode(EModelLoadMode mode) + public void SetWaveNetModelLoadMode(EModelLoadMode mode) { - NativeApi.SetLSTMLoadMode((int)mode); + NativeApi.SetWaveNetLoadMode(nativeLoader, (int)mode); } - public static void SetWaveNetModelLoadMode(EModelLoadMode mode) + public void SetDefaultMaxAudioBufferSize(int bufferSize) { - NativeApi.SetWaveNetLoadMode((int)mode); + NativeApi.SetDefaultMaxAudioBufferSize(nativeLoader, bufferSize); } - public static void SetDefaultMaxAudioBufferSize(int bufferSize) + public NeuralModel CreateModelFromFile(string modelPath) { - NativeApi.SetDefaultMaxAudioBufferSize(bufferSize); + NeuralModel model = new NeuralModel(); + + model.nativeModel = NativeApi.CreateModelFromFile(nativeLoader, modelPath); + + if (model.nativeModel == IntPtr.Zero) + return null; + + return model; } + } + + public class NeuralModel + { + internal IntPtr nativeModel; public bool IsStatic { get { return NativeApi.IsStatic(nativeModel); } } public EModelLoadMode LoadMode { get { return (EModelLoadMode)NativeApi.GetLoadMode(nativeModel); } } @@ -35,15 +62,9 @@ public static void SetDefaultMaxAudioBufferSize(int bufferSize) public float RecommendedInputDBAdjustment { get { return NativeApi.GetRecommendedInputDBAdjustment(nativeModel); } } public float RecommendedOutputDBAdjustment { get { return NativeApi.GetRecommendedOutputDBAdjustment(nativeModel); } } - public static NeuralModel FromFile(string modelPath) + ~NeuralModel() { - NeuralModel model = new NeuralModel(); - - IntPtr nativeModel = NativeApi.CreateModelFromFile(modelPath); - - model.nativeModel = nativeModel; - - return model; + NativeApi.DeleteModel(nativeModel); } public void SetMaxAudioBufferSize(int bufferSize) diff --git a/NeuralAudioCSharp/NeuralAudioTest/Program.cs b/NeuralAudioCSharp/NeuralAudioTest/Program.cs index 77f1b94..8058db6 100644 --- a/NeuralAudioCSharp/NeuralAudioTest/Program.cs +++ b/NeuralAudioCSharp/NeuralAudioTest/Program.cs @@ -6,9 +6,11 @@ internal class Program { static void Main(string[] args) { - NeuralModel.SetWaveNetModelLoadMode(NeuralModel.EModelLoadMode.Internal); + NeuralModelLoader loader = new(); - NeuralModel model = NeuralModel.FromFile("BossWN-standard.nam"); + loader.SetWaveNetModelLoadMode(EModelLoadMode.Internal); + + NeuralModel model = loader.CreateModelFromFile("BossWN-standard.nam"); var input = new float[1024]; var output = new float[1024]; diff --git a/Utils/ModelTest/ModelTest.cpp b/Utils/ModelTest/ModelTest.cpp index 27b4994..3ae0d2f 100644 --- a/Utils/ModelTest/ModelTest.cpp +++ b/Utils/ModelTest/ModelTest.cpp @@ -8,10 +8,10 @@ using namespace NeuralAudio; static std::string LoadModes[] = { "Internal", "RTNeural", "NAMCore" }; -NeuralModel* LoadModel(std::filesystem::path modelPath, EModelLoadMode loadMode) +NeuralModel* LoadModel(std::filesystem::path modelPath, NeuralModelLoader& loader, EModelLoadMode loadMode) { - NeuralModel::SetWaveNetLoadMode(loadMode); - NeuralModel::SetLSTMLoadMode(loadMode); + loader.SetWaveNetLoadMode(loadMode); + loader.SetLSTMLoadMode(loadMode); if (!std::filesystem::exists(modelPath)) { @@ -22,7 +22,7 @@ NeuralModel* LoadModel(std::filesystem::path modelPath, EModelLoadMode loadMode) try { - auto model = NeuralAudio::NeuralModel::CreateFromFile(modelPath); + auto model = loader.CreateFromFile(modelPath); if (model == nullptr) { @@ -127,7 +127,7 @@ static double ComputeError(NeuralModel* model1, NeuralModel* model2, int blockSi return sqrt(totErr / (double)(blockSize * numBlocks)); } -void RunNAMTests(std::filesystem::path modelPath, int blockSize) +void RunNAMTests(std::filesystem::path modelPath, NeuralModelLoader& loader, int blockSize) { std::cout << "Model: " << modelPath << std::endl; std::cout << std::endl; @@ -136,11 +136,11 @@ void RunNAMTests(std::filesystem::path modelPath, int blockSize) int numBlocks = dataSize / blockSize; - NeuralModel::SetDefaultMaxAudioBufferSize(blockSize); + loader.SetDefaultMaxAudioBufferSize(blockSize); - NeuralModel* rtNeuralModel = LoadModel(modelPath, EModelLoadMode::RTNeural); - NeuralModel* namCoreModel = LoadModel(modelPath, EModelLoadMode::NAMCore); - NeuralModel* internalModel = LoadModel(modelPath, EModelLoadMode::Internal); + NeuralModel* rtNeuralModel = LoadModel(modelPath, loader, EModelLoadMode::RTNeural); + NeuralModel* namCoreModel = LoadModel(modelPath, loader, EModelLoadMode::NAMCore); + NeuralModel* internalModel = LoadModel(modelPath, loader, EModelLoadMode::Internal); double rms; @@ -199,7 +199,7 @@ void RunNAMTests(std::filesystem::path modelPath, int blockSize) std::cout << std::endl; } -void RunKerasTests(std::filesystem::path modelPath, int blockSize) +void RunKerasTests(std::filesystem::path modelPath, NeuralModelLoader& loader, int blockSize) { std::cout << "Model: " << modelPath << std::endl; @@ -207,10 +207,10 @@ void RunKerasTests(std::filesystem::path modelPath, int blockSize) int numBlocks = dataSize / blockSize; - NeuralAudio::NeuralModel::SetDefaultMaxAudioBufferSize(blockSize); + loader.SetDefaultMaxAudioBufferSize(blockSize); - auto internalModel = LoadModel(modelPath, EModelLoadMode::Internal); - auto rtNeuralModel = LoadModel(modelPath, EModelLoadMode::RTNeural); + auto internalModel = LoadModel(modelPath, loader, EModelLoadMode::Internal); + auto rtNeuralModel = LoadModel(modelPath, loader, EModelLoadMode::RTNeural); double rms = ComputeError(rtNeuralModel, internalModel, blockSize, numBlocks); std::cout << "Internal vs RTNeural RMS err: " << rms << std::endl; @@ -226,7 +226,7 @@ void RunKerasTests(std::filesystem::path modelPath, int blockSize) std::cout << std::endl; } -int RunDefaultTests(int blockSize) +int RunDefaultTests(NeuralModelLoader& loader, int blockSize) { std::filesystem::path modelPath = std::filesystem::current_path(); @@ -253,12 +253,12 @@ int RunDefaultTests(int blockSize) std::cout << "Loading models from: " << modelPath << std::endl << std::endl; std::cout << "WaveNet (Standard) Test" << std::endl; - RunNAMTests(modelPath / "BossWN-standard.nam", blockSize); + RunNAMTests(modelPath / "BossWN-standard.nam", loader, blockSize); std::cout << std::endl; std::cout << "LSTM (1x16) Test" << std::endl; - RunNAMTests(modelPath / "BossLSTM-1x16.nam", blockSize); + RunNAMTests(modelPath / "BossLSTM-1x16.nam", loader, blockSize); return 0; } @@ -308,7 +308,9 @@ int main(int argc, char* argv[]) qualityScale = program.get("--quality_scale"); - NeuralModel::SetDefaultQualityScaleFactor(qualityScale); + NeuralModelLoader loader; + + loader.SetDefaultQualityScaleFactor(qualityScale); std::cout << "Block size: " << blockSize << " Quality Scale: " << qualityScale << std::endl; @@ -316,16 +318,16 @@ int main(int argc, char* argv[]) { if (modelPath.extension() == ".nam") { - RunNAMTests(modelPath, blockSize); + RunNAMTests(modelPath, loader, blockSize); } else { - RunKerasTests(modelPath, blockSize); + RunKerasTests(modelPath, loader, blockSize); } } else { - if (RunDefaultTests(blockSize) < 0) + if (RunDefaultTests(loader, blockSize) < 0) return -1; } From fb6f89691ace2fa211860b0147f44ad89401c005 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 27 May 2026 10:06:04 -0700 Subject: [PATCH 059/162] Fix static RTNeural model load path --- NeuralAudio/NeuralModel.cpp | 2 +- NeuralAudio/RTNeuralLoader.cpp | 5 +++-- NeuralAudio/RTNeuralLoader.h | 6 +++--- NeuralAudio/RTNeuralModel.h | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index bb0a3c9..a3fd2e3 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -282,7 +282,7 @@ namespace NeuralAudio #ifdef BUILD_STATIC_RTNEURAL if (lstmLoadMode == EModelLoadMode::RTNeural) { - newModel = RTNeuralLoadKeras(modelJson); + newModel = RTNeuralLoadKeras(modelJson, this); } #endif diff --git a/NeuralAudio/RTNeuralLoader.cpp b/NeuralAudio/RTNeuralLoader.cpp index ef03c4e..2d3fda8 100644 --- a/NeuralAudio/RTNeuralLoader.cpp +++ b/NeuralAudio/RTNeuralLoader.cpp @@ -86,7 +86,7 @@ namespace NeuralAudio // If we didn't have a static model that matched, use RTNeural's dynamic model RTNeuralModelDyn* dynModel = new RTNeuralModelDyn; - dynmodel->SetModelLoader(loader); + dynModel->SetModelLoader(loader); dynModel->LoadFromNAMJson(modelJson); return dynModel; @@ -94,7 +94,7 @@ namespace NeuralAudio return nullptr; } - NeuralModel* RTNeuralLoadKeras(const nlohmann::json& modelJson) + NeuralModel* RTNeuralLoadKeras(const nlohmann::json& modelJson, NeuralModelLoader* loader) { const auto layers = modelJson.at("layers"); const size_t numLayers = layers.size() - 1; @@ -107,6 +107,7 @@ namespace NeuralAudio { RTNeuralModel* model = modelDef->CreateModel(); + model->SetModelLoader(loader); model->LoadFromKerasJson(modelJson); return model; diff --git a/NeuralAudio/RTNeuralLoader.h b/NeuralAudio/RTNeuralLoader.h index c96e463..b8b1886 100644 --- a/NeuralAudio/RTNeuralLoader.h +++ b/NeuralAudio/RTNeuralLoader.h @@ -6,7 +6,7 @@ namespace NeuralAudio { extern void EnsureRTNeuralModelDefsAreLoaded(); - extern NeuralModel* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson); - extern NeuralModel* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson); - extern NeuralModel* RTNeuralLoadKeras(const nlohmann::json& modelJson); + extern NeuralModel* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson, NeuralModelLoader* loader); + extern NeuralModel* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader* loader); + extern NeuralModel* RTNeuralLoadKeras(const nlohmann::json& modelJson, NeuralModelLoader* loader); } \ No newline at end of file diff --git a/NeuralAudio/RTNeuralModel.h b/NeuralAudio/RTNeuralModel.h index d105d7d..2a1a129 100644 --- a/NeuralAudio/RTNeuralModel.h +++ b/NeuralAudio/RTNeuralModel.h @@ -282,7 +282,7 @@ namespace NeuralAudio model->load_weights(modelJson); - SetMaxAudioBufferSize(defaultMaxAudioBufferSize); + SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); return true; } From 31bb74db6850e97a8a347d8b92fbd27498ddeb0d Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 27 May 2026 10:18:03 -0700 Subject: [PATCH 060/162] Add model loader info --- README.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 32bf92c..7572eb4 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,12 @@ NAM A2 models currently use the NAM Core implementation. # API overview -To load a model: +Models are loaded with a model loader: + ``` -NeuralModel* model = NeuralAudio::NeuralModel::CreateFromFile(""); +NeuralModelLoader loader; + +NeuralModel* model = loader.CreateFromFile(""); ``` To process a model: @@ -48,10 +51,10 @@ model->Process(pointerToFloatInputData, pointerToFloatOutputData, int numSamples Some models need to allocate memory based on the size of the audio buffers being used. You need to make sure that processing does not exceed the specified maximum buffer size. -The default maximum size is 128 samples. To change it, do: +The default maximum size is 128 samples. To change it, change the default size on the model loader: ``` -NeuralAudio::NeuralModel::SetDefaultMaxAudioBufferSize(maxSize); +loader.SetDefaultMaxAudioBufferSize(maxSize); ``` if you want to change the maximum buffer size of an already created model, do: @@ -66,20 +69,20 @@ model->SetMaxAudioBufferSize(int maxSize); Use ```model->GetRecommendedInputDBAdjustment()``` and ```model->GetRecommendedOutputDBAdjustment()``` to obtain the ideal input and output volume level adjustments in dB. -To set a known audio input level (ie: from an audio interface), use ```model->SetAudioInputLevelDBu(float audioDBu)```. This is set at 12DBu by default. +To set a known audio input level (ie: from an audio interface), use ```loader.SetAudioInputLevelDBu(float audioDBu)```. This is set at 12DBu by default. ## Model load behavior By default, models are loaded using the internal NeuralAudio implementation. If you would like to force the use of the NAM Core or RTNeural implementations, you can use: ``` -NeuralAudio::NeuralModel::SetWaveNetLoadMode(loadMode) +loader.SetWaveNetLoadMode(loadMode) ``` and ``` -NeuralAudio::NeuralModel::SetLSTMLoadMode(loadMode) +loader.SetLSTMLoadMode(loadMode) ``` where "loadMode" is one of: @@ -100,10 +103,10 @@ Some models (notably, slimmable NAM A2 models) support quality scaling - trading Quality scaling is a floating point range from 0.0 (highest performance) to 1.0 (highest quality). -To set the default quality scaling factor, do: +To set the default quality scaling factor, set it on the loader: ``` -NeuralAudio::NeuralModel::SetDefaultQualityScaleFactor(scaleFactor); +loader.SetDefaultQualityScaleFactor(scaleFactor); ``` To check if a model supports quality scaling, do: @@ -112,12 +115,14 @@ To check if a model supports quality scaling, do: if (model->HasQualityScaling()) ... ``` -To set the quality scaling factor for a model, do: +To set the quality scaling factor for a loaded model, do: ``` model->SetQualityScaleFactor(scaleFactor); ``` +***Note: This operation is not real-time safe if the quality scale factor results in a change to the model*** + To get the quality scaling factor for a model, do: ``` From d9d685cabde7672c42dac8a6d7f7224a18d6e04b Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 27 May 2026 11:29:42 -0700 Subject: [PATCH 061/162] More override annotation fixes --- NeuralAudio/NeuralModelImpl.h | 4 ++-- NeuralAudio/RTNeuralModel.h | 42 +++++++++++++++++------------------ 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/NeuralAudio/NeuralModelImpl.h b/NeuralAudio/NeuralModelImpl.h index 8eef75b..d8856e6 100644 --- a/NeuralAudio/NeuralModelImpl.h +++ b/NeuralAudio/NeuralModelImpl.h @@ -10,9 +10,9 @@ namespace NeuralAudio public: using NeuralModel::Prewarm; - void SetModelLoader(NeuralModelLoader* loader) + void SetModelLoader(NeuralModelLoader* modelLoader) { - this->loader = loader; + this->loader = modelLoader; } protected: diff --git a/NeuralAudio/RTNeuralModel.h b/NeuralAudio/RTNeuralModel.h index 2a1a129..123fbb9 100644 --- a/NeuralAudio/RTNeuralModel.h +++ b/NeuralAudio/RTNeuralModel.h @@ -41,7 +41,7 @@ namespace NeuralAudio return EModelLoadMode::RTNeural; } - bool LoadFromKerasJson(const nlohmann::json& modelJson) + virtual bool LoadFromKerasJson(const nlohmann::json& modelJson) { ReadKerasConfig(modelJson); @@ -100,7 +100,7 @@ namespace NeuralAudio return true; } - bool CreateModelFromKerasJson(const nlohmann::json& modelJson) + bool CreateModelFromKerasJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -116,7 +116,7 @@ namespace NeuralAudio return true; } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -263,12 +263,12 @@ namespace NeuralAudio } } - bool IsStatic() + bool IsStatic() override { return true; } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -287,17 +287,17 @@ namespace NeuralAudio return true; } - void SetMaxAudioBufferSize(int maxSize) + void SetMaxAudioBufferSize(int maxSize) override { model->prepare(maxSize); } - void Process(float* input, float* output, size_t numSamples) + void Process(float* input, float* output, size_t numSamples) override { model->forward(input, output, (int)numSamples); } - void Prewarm() + void Prewarm() override { float sample = 0; @@ -321,7 +321,7 @@ namespace NeuralAudio class RTNeuralLSTMDefinitionBase : public RTNeuralModelDefinitionBase { public: - virtual RTNeuralModel* CreateModel() + RTNeuralModel* CreateModel() override { return nullptr; } @@ -341,17 +341,17 @@ namespace NeuralAudio class RTNeuralLSTMDefinitionT : public RTNeuralLSTMDefinitionBase { public: - RTNeuralModel* CreateModel() + RTNeuralModel* CreateModel() override { return new RTNeuralLSTMModelT; } - size_t GetNumLayers() + size_t GetNumLayers() override { return numLayers; } - size_t GetHiddenSize() + size_t GetHiddenSize() override { return hiddenSize; } @@ -360,7 +360,7 @@ namespace NeuralAudio class RTNeuralWaveNetDefinitionBase : public RTNeuralModelDefinitionBase { public: - virtual RTNeuralModel* CreateModel() + virtual RTNeuralModel* CreateModel() override { return nullptr; } @@ -380,17 +380,17 @@ namespace NeuralAudio class RTNeuralWaveNetDefinitionT : public RTNeuralWaveNetDefinitionBase { public: - RTNeuralModel* CreateModel() + RTNeuralModel* CreateModel() override { return new RTNeuralWaveNetModelT; } - virtual size_t GetNumChannels() + size_t GetNumChannels() override { return numChannels; } - virtual size_t GetHeadSize() + size_t GetHeadSize() override { return headSize; } @@ -411,12 +411,12 @@ namespace NeuralAudio model.reset(); } - EModelLoadMode GetLoadMode() + EModelLoadMode GetLoadMode() override { return EModelLoadMode::RTNeural; } - bool CreateModelFromKerasJson(const nlohmann::json& modelJson) + bool CreateModelFromKerasJson(const nlohmann::json& modelJson) override { model = RTNeural::json_parser::parseJson(modelJson, false); model->reset(); @@ -424,7 +424,7 @@ namespace NeuralAudio return true; } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { model = std::make_unique>(1); @@ -535,13 +535,13 @@ namespace NeuralAudio return true; } - void Process(float* input, float* output, size_t numSamples) + void Process(float* input, float* output, size_t numSamples) override { for (size_t i = 0; i < numSamples; i++) output[i] = model->forward(input + i); } - void Prewarm() + void Prewarm() override { float sample = 0; From ad97ea34c1077119fb868b844bb4888a23c007ea Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 27 May 2026 11:45:57 -0700 Subject: [PATCH 062/162] Work around NAM Core prewarm issues --- NeuralAudio/NAMModel.h | 14 ++++---------- deps/NeuralAmpModelerCore | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index 8492590..9fbfdfb 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -40,7 +40,7 @@ namespace NeuralAudio namModel = nam::get_dsp(modelJson); - suppressPrewarm = true; // Hack to work around NAM Core get_dsp() above forcing prewarm + namModel->SuppressPrewarm(true); auto* slim = dynamic_cast(namModel.get()); @@ -53,6 +53,8 @@ namespace NeuralAudio SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); + namModel->SuppressPrewarm(false); + return true; } @@ -96,20 +98,12 @@ namespace NeuralAudio void Prewarm() override { - if (suppressPrewarm) - { - suppressPrewarm = false; - } - else - { - namModel->prewarm(); - } + namModel->prewarm(); } private: std::unique_ptr namModel = nullptr; float slimmableSize = 1.0f; bool isSlimmable = false; - bool suppressPrewarm = false; }; } \ No newline at end of file diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index 5656639..7e34234 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit 5656639bb6dd65340369eeeb50e4d70c32a42922 +Subproject commit 7e3423448e75219c09cce555b5bebccf10950aa2 From cf17ecac0a60139c4a38dab0f15ae85e9dd11bc5 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 27 May 2026 12:11:36 -0700 Subject: [PATCH 063/162] Remove debug code from ModelTest --- Utils/ModelTest/ModelTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/Utils/ModelTest/ModelTest.cpp b/Utils/ModelTest/ModelTest.cpp index 3ae0d2f..8681d57 100644 --- a/Utils/ModelTest/ModelTest.cpp +++ b/Utils/ModelTest/ModelTest.cpp @@ -299,7 +299,6 @@ int main(int argc, char* argv[]) int blockSize = 64; - //std::filesystem::path modelPath = R"(C:\Users\oliph\Downloads\Fender Deluxe Reverb (A2)\Deluxe Reverb.nam)"; std::filesystem::path modelPath = program.get("model_file"); blockSize = program.get("--block_size"); From 7b858227987679773ad97161b68d99e046eaa5b6 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 27 May 2026 12:42:53 -0700 Subject: [PATCH 064/162] Force build of static models in build test --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9f45ba0..4954104 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build run: | - cmake -G "Visual Studio 17 2022" -A x64 -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_STATIC_RTNEURAL=ON -T ClangCL .. + cmake -G "Visual Studio 17 2022" -A x64 -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_STATIC_RTNEURAL=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -T ClangCL .. cmake --build . --config=release -j4 - name: Run ModelTest @@ -46,7 +46,7 @@ jobs: env: CXX: g++-13 run: | - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_STATIC_RTNEURAL=ON + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_STATIC_RTNEURAL=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON cmake --build . --config $BUILD_TYPE -j4 - name: Run ModelTest From 87bed1dd5349604cbc758f9a403541b94e2e190a Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 27 May 2026 12:43:34 -0700 Subject: [PATCH 065/162] Fix static LSTM loading --- NeuralAudio/NeuralModel.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index a3fd2e3..5a2d324 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -249,6 +249,8 @@ namespace NeuralAudio if (modelDef != nullptr) { auto model = modelDef->CreateModel(); + + model->SetModelLoader(this); model->LoadFromNAMJson(modelJson); newModel = model; From 3ad580637050554c6fe18b7ae3265e948901b721 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 2 Jun 2026 07:31:41 -0700 Subject: [PATCH 066/162] Fix NAM Core A2 fast path issue with Tone3000 models --- deps/NeuralAmpModelerCore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index 7e34234..d5e93a7 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit 7e3423448e75219c09cce555b5bebccf10950aa2 +Subproject commit d5e93a7e11f75affe85ccf8b82c0ff2a1105133a From 78ef473b02b8dad0cec34804cc03c287f5ded3c0 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 2 Jun 2026 08:48:45 -0700 Subject: [PATCH 067/162] Add a2 test model --- Utils/Models/BossWN-a2.nam | 1 + 1 file changed, 1 insertion(+) create mode 100644 Utils/Models/BossWN-a2.nam diff --git a/Utils/Models/BossWN-a2.nam b/Utils/Models/BossWN-a2.nam new file mode 100644 index 0000000..1376177 --- /dev/null +++ b/Utils/Models/BossWN-a2.nam @@ -0,0 +1 @@ +{"version":"0.7.0","architecture":"SlimmableContainer","config":{"submodels":[{"max_value":0.5,"model":{"version":"0.7.0","metadata":{"date":{"year":2026,"month":5,"day":17,"hour":22,"minute":50,"second":14},"loudness":-12.17414665222168,"gain":0.6298792269436364},"architecture":"WaveNet","config":{"layers":[{"input_size":1,"condition_size":1,"head":{"out_channels":1,"kernel_size":16,"bias":true},"channels":3,"kernel_sizes":[6,6,6,6,6,6,6,6,6,6,6,6,6,6,15,15,6,6,6,6,6,6,6],"dilations":[1,3,7,17,41,101,239,1,3,7,17,41,101,239,1,13,1,3,7,17,41,101,239],"activation":[{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01}],"bottleneck":3,"head1x1":{"active":false,"out_channels":1,"groups":1},"layer1x1":{"active":true,"groups":1},"groups_input":1,"groups_input_mixin":1,"conv_pre_film":{"active":false,"shift":true,"groups":1},"conv_post_film":{"active":false,"shift":true,"groups":1},"input_mixin_pre_film":{"active":false,"shift":true,"groups":1},"input_mixin_post_film":{"active":false,"shift":true,"groups":1},"activation_pre_film":{"active":false,"shift":true,"groups":1},"activation_post_film":{"active":false,"shift":true,"groups":1},"layer1x1_post_film":{"active":false,"shift":true,"groups":1},"head1x1_post_film":{"active":false,"shift":true,"groups":1},"gating_mode":["none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none"],"secondary_activation":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"slimmable":null}],"head":null,"head_scale":0.013119162069029721},"weights":[-2.2014312744140625,-2.3644347190856934,4.10330057144165,0.013875482603907585,0.15356875956058502,0.3278008699417114,0.13442133367061615,-0.4063030779361725,-1.0289421081542969,0.20730212330818176,0.11322535574436188,0.3868657350540161,0.10031043738126755,-0.31691259145736694,-1.0278282165527344,-0.12845806777477264,-0.3099781572818756,-0.272651731967926,-0.056800853461027145,0.4446815252304077,1.354359745979309,0.15971176326274872,0.08661764115095139,-0.10098310559988022,0.18293239176273346,0.05380669981241226,0.23948296904563904,0.09054432064294815,-0.08613887429237366,-0.1300751268863678,0.15587326884269714,0.11223623901605606,0.19360223412513733,-0.16403675079345703,0.16310861706733704,-0.007865854538977146,0.08529536426067352,-0.34021446108818054,-0.3202733099460602,-0.16050001978874207,-0.15973332524299622,-0.1709902435541153,-0.2039112001657486,0.23347772657871246,0.7998841404914856,-0.15269169211387634,-0.08793331682682037,-0.39089450240135193,-0.1976887583732605,0.25692084431648254,0.9227266311645508,0.254395455121994,0.1028246060013771,0.2839694321155548,0.2400667518377304,-0.27344000339508057,-1.0547384023666382,1.6182467937469482,1.2084239721298218,1.7490651607513428,1.002180814743042,-0.7275207042694092,-1.1934421062469482,-0.25818192958831787,0.43737220764160156,0.17171142995357513,-0.3276022970676422,0.5688443779945374,0.4970926344394684,0.39238572120666504,0.3128047585487366,-0.44379669427871704,0.04468515142798424,0.41651737689971924,0.29364076256752014,-0.04451122134923935,-0.028112323954701424,-0.02097748965024948,-0.012702057138085365,0.06268917769193649,-0.16147184371948242,-0.29312142729759216,-0.09629882127046585,-0.02723022736608982,0.06206013262271881,0.16032196581363678,0.05538935586810112,0.23848466575145721,-0.27474716305732727,-0.33876675367355347,-0.033302415162324905,-0.10256210714578629,-0.1081211045384407,0.11435101181268692,0.2447659820318222,0.2783104181289673,0.2373391091823578,-0.159987673163414,-0.5479333996772766,0.2566385269165039,0.18790094554424286,0.2716050148010254,0.1309361457824707,0.18553760647773743,-0.14580664038658142,-0.014284756034612656,-0.09840688854455948,-0.16540786623954773,0.11237860471010208,0.14221222698688507,0.9043796062469482,-0.0736541599035263,-0.20429277420043945,0.09105397015810013,-0.04670644924044609,0.09283225983381271,0.4834168255329132,0.2033672034740448,-0.05711207911372185,-0.038311850279569626,-0.164163738489151,-0.08022404462099075,0.13044373691082,0.2241334766149521,0.4343363344669342,0.3230689465999603,0.26705202460289,0.26982206106185913,-0.20042891800403595,-0.05943610146641731,0.23236514627933502,0.2435539811849594,0.08832674473524094,1.2771165370941162,-0.19556638598442078,0.14746785163879395,0.06528852134943008,-0.07936146855354309,-0.4645145833492279,-0.6456254720687866,0.2770215570926666,0.003341606119647622,0.31739142537117004,-0.09175405651330948,-0.47012990713119507,-0.13660883903503418,0.20460069179534912,-0.10530567914247513,0.09297942370176315,0.5133674144744873,0.45620226860046387,0.4208831787109375,-0.32265979051589966,-0.03083098866045475,-0.00627914909273386,-0.05858776718378067,-0.11458605527877808,-0.10568328201770782,-0.38585251569747925,-0.056530069559812546,0.05505377799272537,0.030275659635663033,-0.38001692295074463,-0.5056266188621521,0.22018752992153168,-0.07126032561063766,-0.20173831284046173,-0.3261547386646271,-0.359453409910202,-0.015088825486600399,0.42483004927635193,-0.018563169986009598,-0.043394844979047775,0.0406346395611763,0.08621232956647873,0.053887128829956055,0.32103431224823,-0.034591853618621826,-0.04755638167262077,0.15458261966705322,0.3090440034866333,0.37724411487579346,-0.4374990463256836,0.2141016125679016,0.13493311405181885,0.26272568106651306,0.33496421575546265,0.15733586251735687,-0.5267455577850342,0.0708104819059372,0.04849817976355553,0.03587888926267624,-0.07463464140892029,0.001113251899369061,-0.030115514993667603,0.101874940097332,-0.028080172836780548,-0.11411042511463165,-0.0713108479976654,-0.10225676000118256,0.5586898326873779,-0.36880797147750854,0.23473967611789703,0.22187674045562744,-0.08366920053958893,-0.39052024483680725,0.5241316556930542,0.3194989562034607,-0.29806801676750183,0.2259378880262375,-0.8656169176101685,0.6054820418357849,-1.0572688579559326,0.10955116152763367,0.05774696171283722,0.19563937187194824,0.34555792808532715,0.3106386959552765,0.3284628391265869,0.1351257860660553,0.0653991848230362,0.056746706366539,0.115949347615242,-0.5801233649253845,-0.11422739923000336,0.07654698938131332,0.060470253229141235,0.014784484170377254,0.03117985650897026,0.05313900113105774,-0.3193320333957672,0.197667196393013,0.15637464821338654,0.019759058952331543,0.1039513498544693,0.4362679123878479,-0.008409534581005573,-0.11793550103902817,-0.14007437229156494,-0.1831897646188736,-0.01337550487369299,0.24091477692127228,0.1191105768084526,-0.04455364868044853,-0.08310576528310776,-0.08645974844694138,-0.06507641822099686,-0.05547230318188667,-0.7532620429992676,-0.12714485824108124,-0.21965573728084564,-0.2249358892440796,-0.1996520459651947,-0.3197749853134155,-0.08810564875602722,0.1319267600774765,0.20888161659240723,0.15505792200565338,-0.06444970518350601,-0.6391368508338928,0.16648919880390167,0.02057655341923237,0.043972697108983994,0.07323552668094635,0.05857202410697937,-0.05138305202126503,0.5299592614173889,0.0357242152094841,0.06755470484495163,0.1893053948879242,0.170505091547966,0.1796083301305771,-0.14555144309997559,0.17707335948944092,-0.38982537388801575,0.22848962247371674,-0.08042185008525848,-0.07579585909843445,-0.3839818835258484,-0.21525897085666656,-0.049253564327955246,0.23048636317253113,0.4585550129413605,0.6998566389083862,0.5793481469154358,-0.11139228194952011,-0.23135340213775635,-0.2864333987236023,-0.04127928987145424,-0.24587275087833405,0.17506276071071625,-0.024926166981458664,-0.09200415760278702,-0.09672544151544571,0.0023581082932651043,0.0559343658387661,0.20842275023460388,0.03211093321442604,0.04095561057329178,0.09505100548267365,0.08572191745042801,0.011949539184570312,-0.5728508830070496,0.0651869848370552,0.07643463462591171,0.18087291717529297,0.20300918817520142,-0.021814070641994476,0.33623814582824707,0.023220300674438477,0.09634418785572052,0.08783882111310959,0.03317289054393768,-0.013059227727353573,-0.14349544048309326,0.003293004585430026,-0.0003971911792177707,0.01377302035689354,0.004187173675745726,-0.00927561055868864,-0.10491470247507095,0.011596589349210262,0.004819414112716913,0.0596567839384079,0.025614546611905098,0.00022714523947797716,-0.2966059744358063,0.08415508270263672,-0.0512981191277504,-0.027162181213498116,-0.030133729800581932,-0.06166153401136398,0.14925351738929749,0.01606331579387188,-0.01696416363120079,-0.014036207459867,-0.04851203411817551,0.011458122171461582,-0.051346227526664734,0.07395359128713608,0.004695982672274113,0.06185265630483627,-0.04176633805036545,0.027011973783373833,0.27004680037498474,0.2184954732656479,0.08525806665420532,-0.04840027168393135,0.06630273163318634,-0.03354614973068237,0.12632602453231812,-0.020363574847579002,0.16040000319480896,-0.24998949468135834,-0.6133416891098022,0.3964405357837677,-0.08699718117713928,-0.05510358512401581,0.030894216150045395,-0.13260945677757263,-0.38290613889694214,-0.04843368008732796,0.541000485420227,0.019082050770521164,0.007336693350225687,-0.0015728918369859457,-0.05935760587453842,-0.14012505114078522,0.007425954565405846,-0.007963333278894424,-0.007962853647768497,-0.0028553702868521214,-0.016793249174952507,-0.014149646274745464,-0.6677324175834656,-0.029075751081109047,-0.027568113058805466,-0.020008312538266182,-0.08829547464847565,-0.12561483681201935,-0.2089890092611313,-0.0086365332826972,-0.02048121578991413,-0.04040398448705673,-0.07442653924226761,-0.11849841475486755,0.5839603543281555,-0.005353859625756741,-0.012812442146241665,-0.004449067171663046,-0.023512255400419235,-0.012884330004453659,-0.486892431974411,-0.029179057106375694,-0.03881366550922394,-0.018658852204680443,-0.11640860885381699,-0.05758708715438843,-0.02758430503308773,0.010116863064467907,0.03526017814874649,0.0957784578204155,0.18534670770168304,0.34974345564842224,-0.2780730426311493,0.009035254828631878,0.005115012172609568,0.00764759024605155,0.04431099444627762,0.010754058137536049,0.010473896749317646,0.027558807283639908,0.020677799358963966,0.045722510665655136,0.18427978456020355,0.20525576174259186,0.15279117226600647,-0.27704840898513794,-0.09763302654027939,0.1571602076292038,-0.06785169988870621,-0.12322850525379181,-0.04365439713001251,0.025742774829268456,-0.087237149477005,0.16346365213394165,1.0164475440979004,-0.3515949845314026,0.3112505078315735,-0.3225294351577759,-0.018905742093920708,-0.39615869522094727,-0.16421464085578918,-0.3041362762451172,0.3561646640300751,-0.02584652788937092,-0.03723614662885666,-0.0378250852227211,-0.043828085064888,-0.033122528344392776,-0.05960679054260254,-0.0123366117477417,-0.016761887818574905,-0.017051242291927338,-0.018122490495443344,-0.013254676945507526,-0.0065485830418765545,-0.06753022223711014,-0.08714424818754196,-0.07974662631750107,-0.10276509821414948,-0.08740398287773132,0.3400799334049225,-0.04502898454666138,-0.05914813652634621,-0.027337580919265747,-0.029985355213284492,-0.041764311492443085,-0.3453563153743744,-0.019694466143846512,-0.02872847020626068,-0.014559841714799404,-0.01689673773944378,-0.018111858516931534,0.1508011817932129,-0.10746829211711884,-0.14647206664085388,-0.07456900924444199,-0.09562823921442032,-0.06073016673326492,0.03304087743163109,0.0030416434165090322,0.01756305620074272,-0.005867429543286562,-0.0070260995998978615,0.09984858334064484,0.027783868834376335,0.0008591634687036276,0.005053444765508175,-0.0029418659396469593,-0.006573853082954884,-0.02672344446182251,0.4023105800151825,0.0064987437799572945,0.02932850830256939,-0.0160983819514513,-0.03192165121436119,-0.07437776029109955,0.24474897980690002,-0.2958236038684845,-0.3867059350013733,0.0019733794033527374,0.27910539507865906,0.07492859661579132,0.08461777120828629,-0.06402987986803055,-0.13830603659152985,0.09749722480773926,-0.22630615532398224,0.3731735348701477,0.22992894053459167,-0.34267500042915344,-0.5737828016281128,-0.4918070137500763,-0.45648831129074097,0.05826294422149658,0.420463889837265,0.16099809110164642,0.06086291745305061,-0.015186207368969917,0.03053053840994835,0.09299395978450775,0.17543672025203705,-0.1887088567018509,-0.0016415286809206009,0.08831943571567535,0.17218364775180817,0.21438223123550415,-0.25929197669029236,-0.32457491755485535,0.05628906562924385,0.023955170065164566,-0.09208360314369202,0.04444421827793121,0.03765854611992836,-0.022934360429644585,0.2666116952896118,-0.11071984469890594,-0.14637991786003113,-0.2116205096244812,-0.15522781014442444,0.032325442880392075,0.30661794543266296,0.034208860248327255,-0.05703316628932953,0.035322047770023346,-0.10946434736251831,-0.06372347474098206,0.1628868728876114,0.2534482777118683,0.055746499449014664,-0.07275748252868652,0.2312823385000229,-0.102630615234375,-0.15631455183029175,-0.24276049435138702,0.02730661630630493,0.06320495158433914,0.21693485975265503,0.0053083705715835094,0.05945627763867378,0.0044203768484294415,0.14401677250862122,0.6592717170715332,0.6577001810073853,0.06053007394075394,0.033865492790937424,0.0725623071193695,0.24501881003379822,-0.06038414686918259,0.16380807757377625,-0.09601780027151108,-0.060286134481430054,0.2112821340560913,-0.005282821599394083,0.2468477487564087,-0.6443699598312378,-0.4285217225551605,0.19263726472854614,0.6201222538948059,-0.17815150320529938,-0.4198221266269684,0.7243912220001221,0.5521109700202942,-0.28732725977897644,-0.042256489396095276,-0.15002775192260742,-0.37756964564323425,0.6629250645637512,0.1854286789894104,0.11356539279222488,0.17620515823364258,0.21373027563095093,-0.22099517285823822,-0.15375979244709015,-0.05918728932738304,0.011076151393353939,-0.04833417013287544,-0.04595540091395378,-0.2917470335960388,-0.13568267226219177,0.22090356051921844,0.1213298961520195,0.18485966324806213,0.21171437203884125,0.3657015562057495,0.05172756686806679,-0.17758505046367645,-0.18795205652713776,-0.1028355211019516,-0.06293430179357529,-0.006775418296456337,-0.23636171221733093,0.08007046580314636,0.05864149332046509,0.07869783788919449,0.12085473537445068,0.24944719672203064,0.16303913295269012,0.21768684685230255,0.08480014652013779,-0.022742707282304764,0.19164447486400604,0.15293581783771515,0.09564486891031265,0.19310149550437927,0.06410381197929382,0.09536483138799667,0.045794595032930374,-0.02825321815907955,-0.08688146620988846,-0.06949115544557571,-0.028912706300616264,-0.10171185433864594,0.02747916989028454,-0.3905165493488312,0.014015856198966503,0.04064818099141121,0.09153930842876434,0.1189781203866005,0.2024744749069214,0.25191307067871094,0.13821421563625336,0.2617477774620056,0.1649087518453598,0.1616164743900299,-0.13608133792877197,-0.25968968868255615,-0.03105182945728302,0.31666168570518494,-0.005689946934580803,0.15644478797912598,-0.5720689296722412,0.11084175854921341,-0.3403335213661194,-0.40763193368911743,0.26767855882644653,-0.441596657037735,0.2859145402908325,-0.5444028377532959,0.29211774468421936,-0.07738512754440308,-0.1804807186126709,-0.02522926777601242,-0.09450779855251312,-0.13646376132965088,0.29535430669784546,0.040519386529922485,0.04803149029612541,0.02461175061762333,0.016729354858398438,0.07254209369421005,-0.28055301308631897,-0.1413072645664215,-0.19903384149074554,-0.14780324697494507,-0.11357695609331131,-0.21471437811851501,-0.3246343433856964,-0.0510096475481987,-0.08354977518320084,0.12729278206825256,-0.10149084031581879,-0.05479840934276581,0.16306541860103607,-0.015471145510673523,-0.007160392589867115,-0.15260739624500275,-0.11011581867933273,-0.2112150341272354,0.1642652153968811,-0.041242215782403946,0.07066415250301361,0.22906073927879333,-0.012592502869665623,0.03516409546136856,0.00900303479284048,-0.06584323942661285,-0.03831200301647186,-0.21301425993442535,0.04159072786569595,-0.008085627108812332,0.12950187921524048,0.01966777630150318,0.007433672901242971,0.049937669187784195,-0.015088389627635479,0.009842945262789726,-0.3282533288002014,-0.044814977794885635,-0.02022007293999195,-0.16136394441127777,-0.0007228617905639112,0.025680389255285263,-0.4317805767059326,-0.2836703658103943,0.21167778968811035,-0.17584557831287384,-0.03210986778140068,0.007393332198262215,0.06711545586585999,-0.29140788316726685,0.09494075179100037,0.060016632080078125,0.6892462968826294,-0.12644565105438232,0.5437824726104736,0.3083539605140686,-0.09905592352151871,-0.07869111746549606,0.1842239499092102,0.11328814923763275,0.473525732755661,0.0931219682097435,0.04723215475678444,-0.00942918099462986,0.12776996195316315,0.10211355239152908,-0.13509821891784668,-0.026894384995102882,-0.003409402910619974,0.021159030497074127,-0.025657540187239647,0.03933345153927803,-0.10009203106164932,0.019600436091423035,0.009639068506658077,-0.0657072588801384,0.1254866123199463,0.0225848201662302,0.10728231817483902,-0.01374603621661663,-0.07630118727684021,0.11476660519838333,-0.0681256502866745,0.12623153626918793,-0.3740171790122986,-0.006826187949627638,0.02012096345424652,-0.039208441972732544,0.034855764359235764,-0.021081412211060524,-0.08178651332855225,0.04184757545590401,-0.02104531228542328,0.12483389675617218,-0.037485286593437195,0.12625505030155182,0.4177972674369812,0.005522859748452902,0.05570603534579277,0.12235534191131592,0.04680335894227028,0.29607969522476196,-0.28425854444503784,-0.0033091583754867315,-0.02145201526582241,-0.042982712388038635,-0.0012671989388763905,-0.07852332293987274,-0.3635716736316681,0.011116636916995049,0.06510531157255173,0.11740104854106903,-0.0036171970423310995,0.3098653554916382,0.17680472135543823,-0.05556192621588707,0.11135676503181458,0.09721453487873077,0.05879608169198036,-0.03277534246444702,-0.07436835020780563,-0.1520647257566452,0.0529743954539299,-0.09556834399700165,-0.3573068380355835,-0.4127682149410248,-0.5504496097564697,-0.058370739221572876,0.06758173555135727,-0.41957026720046997,0.5789896249771118,-0.5567089915275574,0.02437531016767025,0.01897905021905899,0.008736285381019115,0.13045452535152435,0.05155551806092262,0.03634728863835335,-0.07670553028583527,-0.014921480789780617,-0.0001275207760045305,-0.04321090504527092,-0.013399453833699226,-0.030683742836117744,-0.3170943856239319,0.02865612879395485,-0.02591852843761444,0.050633110105991364,0.03243114426732063,0.049704816192388535,-0.13640345633029938,-0.017414020374417305,-0.09700603038072586,-0.13105517625808716,-0.1012539342045784,-0.008842669427394867,-0.08316817134618759,0.0076532321982085705,0.026950640603899956,0.04738757014274597,0.027548838406801224,0.07576663792133331,0.05371657758951187,0.004232553765177727,-0.029722118750214577,-0.017079457640647888,-0.034212835133075714,0.0678187906742096,0.1804538369178772,-0.010413448326289654,-0.03706008940935135,-0.042372141033411026,0.005118170287460089,0.015261722728610039,-0.0745769813656807,-0.05799354612827301,0.01239084079861641,0.013459793291985989,0.0033835917711257935,-0.0006350994808599353,0.05803684517741203,0.004481020383536816,0.03986607864499092,0.06641774624586105,0.01457990426570177,0.04627985134720802,-0.014160661026835442,0.20562240481376648,0.029852863401174545,0.004694575443863869,-0.09294068068265915,-0.05241657420992851,0.006039710249751806,-0.17241093516349792,0.0891229659318924,0.1347063183784485,-0.23249280452728271,-0.10104064643383026,0.006114705931395292,-0.054887741804122925,0.1170739158987999,-0.021641410887241364,0.4166780412197113,-0.7408673167228699,-0.34534701704978943,-0.004390317015349865,-0.0133216492831707,-0.013962293975055218,-0.04001116007566452,-0.0017552917124703526,0.008528734557330608,0.03392954170703888,0.0360761359333992,0.06000314652919769,0.0953037291765213,0.039254967123270035,-0.015198169276118279,-0.13355867564678192,-0.12131726741790771,-0.20218780636787415,-0.16920284926891327,-0.24456675350666046,-0.04598214477300644,-0.06402838975191116,-0.05914926901459694,-0.11267099529504776,-0.09625673294067383,-0.018514737486839294,-0.09744048863649368,0.02030935324728489,0.02018473669886589,0.037463828921318054,0.03457596153020859,0.006644686684012413,0.825168788433075,-0.03622704744338989,-0.035193800926208496,-0.05690038949251175,-0.07324764132499695,-0.03475940600037575,0.07583461701869965,-0.07657330483198166,-0.08096276968717575,-0.1342262625694275,-0.16198377311229706,-0.19149421155452728,-0.43036723136901855,0.02622113563120365,0.02749551460146904,0.04527297616004944,0.05606325343251228,0.06924230605363846,-0.3003419041633606,-0.034038741141557693,-0.035073067992925644,-0.06426286697387695,-0.06528802216053009,-0.08003722876310349,-0.6930965781211853,-0.032366540282964706,-0.42031794786453247,-0.5722066760063171,0.0567447692155838,0.38014471530914307,-0.029751446098089218,-0.1923818439245224,-0.456892192363739,0.11181174218654633,0.1238681897521019,-1.2726902961730957,0.7340671420097351,0.16623497009277344,0.0877658799290657,0.6717463135719299,0.09809945523738861,0.11721499264240265,0.24386271834373474,-0.0468749925494194,-0.092890165746212,-0.06576604396104813,-0.11520735174417496,-0.02461431920528412,-0.08256372809410095,-0.004163934849202633,-0.0200889240950346,0.00592941464856267,-0.021959252655506134,0.032569244503974915,-0.035753559321165085,0.0015350664034485817,0.06677763164043427,-0.0041939979419112206,0.06927374005317688,-0.05020373314619064,0.05175931379199028,-0.024826476350426674,-0.018296018242836,-0.011916188523173332,-0.018089836463332176,-0.015844322741031647,-0.016764501109719276,0.001426394795998931,0.006738968193531036,0.0193238016217947,0.03287462517619133,0.005511725787073374,0.0040188804268836975,-0.009849431924521923,-0.008338280022144318,0.004751404281705618,0.011433483101427555,0.0008858044166117907,-0.0023382750805467367,-0.0011220235610380769,-0.0036734105087816715,-0.0037256337236613035,-0.0031578040216118097,-0.004243722651153803,-0.0014916519867256284,0.012692187912762165,0.0043168445117771626,0.00796261802315712,0.007874337024986744,0.009626087732613087,0.009238019585609436,-0.0077214716002345085,-0.0023708934895694256,-0.0053356848657131195,-0.007169315591454506,-0.008372701704502106,-0.01434654463082552,-0.0124212009832263,-0.00593726197257638,-0.0014124347362667322,0.014019911177456379,-0.0015306307468563318,-0.0009815986268222332,0.018811382353305817,0.004848717246204615,-0.0023553117644041777,0.13777117431163788,0.010609139688313007,-0.0008198893629014492,0.04232117533683777,0.01120645273476839,-0.003369665704667568,0.057212989777326584,0.2081596553325653,-0.06718017905950546,0.17269113659858704,0.009713338688015938,-0.03404852747917175,0.08491694927215576,0.11330597847700119,0.041843436658382416,0.045436955988407135,0.1359654664993286,-0.01210862398147583,0.10351971536874771,-0.09000325947999954,-0.13927295804023743,-0.15863150358200073,0.22291485965251923,0.03032924234867096,0.01198500581085682,-0.056901611387729645,0.022123921662569046,-0.010731500573456287,-0.051212161779403687,0.012613601051270962,0.02941378764808178,0.02553156204521656,0.1221400648355484,-0.07341241836547852,-0.5046965479850769,-0.6048814654350281,-0.47892361879348755,-0.15346777439117432,-0.03878682106733322,-0.0584254264831543,0.015974368900060654,0.15863056480884552,0.05177295580506325,-0.01242576353251934,-0.05994808301329613,0.10977894812822342,0.03320879116654396,0.039395589381456375,0.10629669576883316,0.058564599603414536,0.10299590975046158,0.03876424953341484,-0.12510058283805847,-0.05931894853711128,0.18011242151260376,0.0008341691573150456,0.0009805859299376607,0.16798320412635803,0.06688108295202255,0.029469454661011696,0.2033063918352127,0.24847964942455292,0.06179556995630264,-0.002073288196697831,-0.1087300106883049,0.10886983573436737,0.12217050790786743,-0.004577497951686382,-0.05989358201622963,-0.14923034608364105,-0.03575071692466736,0.05595117062330246,-0.08653093129396439,-0.0018496819538995624,0.011525840498507023,-0.02607526071369648,-0.09662096947431564,-0.004074508789926767,-0.15243199467658997,-0.015339260920882225,0.38580283522605896,0.5369555354118347,-0.015834636986255646,-0.28406691551208496,0.1948232203722,0.03356671333312988,0.04813729599118233,0.05075075104832649,0.06731343269348145,0.08344151824712753,0.015106960199773312,0.04820121079683304,0.12462487816810608,0.1424451470375061,-0.00261193560436368,0.15644453465938568,0.3097437620162964,0.21650174260139465,0.09976957738399506,-0.004249217454344034,-0.03585461899638176,-0.13884197175502777,0.03310080245137215,0.025318201631307602,0.01099180243909359,-0.0360720194876194,-0.06428101658821106,0.03864077851176262,0.07165437191724777,-0.017714744433760643,-0.1632019430398941,-0.06779570877552032,-0.02947402000427246,-0.0844537764787674,0.08557608723640442,-0.007783283945173025,-0.053134895861148834,-0.11975040286779404,-0.12594766914844513,-0.1929493397474289,-0.0352880097925663,0.04859646409749985,0.04121916741132736,0.09053142368793488,-0.13254739344120026,-0.13205386698246002,-0.1608976274728775,-0.12032034993171692,0.1704205572605133,-0.0199842881411314,0.06420713663101196,0.04240137338638306,0.026440057903528214,0.08491645008325577,0.06401057541370392,0.09507822245359421,0.004296616185456514,0.07048927247524261,0.13774053752422333,0.12633119523525238,0.05486954748630524,0.08617448061704636,0.028352098539471626,0.05629890784621239,0.2628430426120758,0.27364468574523926,0.12121216952800751,-0.02701500989496708,-0.10187099128961563,-0.0019906789530068636,0.08286310732364655,-0.5924199819564819,-0.06184062361717224,0.2982446849346161,-0.03407451510429382,-0.1747429519891739,-0.5763257145881653,0.637191653251648,-0.17938430607318878,-0.04755958169698715,0.038731761276721954,-0.6143099665641785,0.059068404138088226,0.11320030689239502,0.16534854471683502,0.11437245458364487,0.10673084110021591,0.07022818177938461,0.003832305548712611,0.05411868914961815,0.022144947201013565,0.05946105346083641,-0.011302797123789787,0.08274678140878677,0.1359892636537552,-0.17296907305717468,0.3382267951965332,0.01912323758006096,-0.029805863276124,-0.019274523481726646,-0.0012285838602110744,-0.05631855130195618,0.029644066467881203,-0.06144419312477112,0.0773959681391716,0.04722733423113823,0.013274071738123894,0.056786246597766876,-0.0013954784953966737,0.23763790726661682,0.008414988406002522,-0.41124045848846436,0.03231858089566231,0.10895248502492905,0.06642306596040726,0.06283331662416458,0.09885290265083313,0.028784483671188354,0.03015569970011711,0.07237100601196289,0.02827944979071617,0.08045416325330734,0.06920401751995087,0.07699686288833618,0.22619569301605225,0.47617295384407043,-0.535832405090332,0.01129608228802681,-0.04972101375460625,-0.08959131687879562,-0.05961409956216812,-0.1049358919262886,-0.16627180576324463,-0.08263284713029861,-0.16013680398464203,-0.13960857689380646,-0.14504225552082062,-0.07391935586929321,-0.0669969916343689,-0.06543442606925964,0.2150854617357254,-0.2799462080001831,-0.07504361867904663,0.002481478499248624,-0.019103670492768288,-0.028899382799863815,0.013751109130680561,-0.02258693426847458,0.08810557425022125,-0.04907526448369026,-0.036649320274591446,-0.016108041629195213,-0.026935124769806862,-0.012373811565339565,-0.22196374833583832,-0.08736121654510498,0.246938094496727,0.0024825208820402622,-0.0693335086107254,-0.03585716709494591,-0.030721642076969147,-0.07896463572978973,-0.04599202796816826,-0.054013997316360474,-0.10336010158061981,-0.05640396103262901,-0.09523042291402817,-0.09239568561315536,-0.052691757678985596,-0.18009594082832336,-0.4294753968715668,0.5425381064414978,-0.0372757725417614,-0.047823864966630936,-0.0104625029489398,-0.004671900998800993,-0.03973732516169548,-0.0021263654343783855,-0.023855945095419884,-0.0789547860622406,-0.10577692836523056,-0.08959463983774185,-0.03843184560537338,-0.06384005397558212,0.1419914811849594,-0.038728974759578705,0.1009673923254013,0.019970932975411415,0.04192150756716728,0.023478630930185318,0.013865449465811253,0.036482445895671844,0.021509593352675438,0.01584126614034176,0.055049218237400055,0.05191134288907051,0.05466006323695183,0.04351736977696419,0.02834985964000225,-0.022774936631321907,-0.013348772190511227,0.8476651310920715,-0.016304010525345802,-0.013945184648036957,-0.012984920293092728,-0.0027163932099938393,-0.012388846836984158,-0.005535949487239122,-0.0092982342466712,-0.02955726906657219,-0.04773883521556854,-0.02991931512951851,-0.01802026852965355,-0.02324429713189602,0.05831553786993027,-0.03732339292764664,0.24632878601551056,-0.05545299872756004,-0.017101304605603218,-0.1755329966545105,-0.011509559117257595,-0.021610479801893234,-0.07784798741340637,-0.07508524507284164,0.10352034121751785,0.7073677182197571,0.07185958325862885,-0.053868044167757034,0.27208173274993896,0.24398355185985565,-0.2285347580909729,0.2912147045135498,0.5949616432189941,-0.08625003695487976,-0.45112931728363037,0.5575506091117859,-0.3293210566043854,-0.7460311651229858,-0.293647825717926,0.6017194390296936,0.15850253403186798,0.47176310420036316,-0.6422329545021057,-0.5851084589958191,0.1897549033164978,0.04001835361123085,0.607490062713623,0.31152766942977905,-0.31967270374298096,0.014013879932463169,0.10590489953756332,-0.03616946563124657,-0.12072359025478363,0.07594557851552963,0.03325088322162628,-0.20154978334903717,-0.170691579580307,0.3313436210155487,-0.15262366831302643,0.18364350497722626,0.09518761932849884,-0.3837544620037079,0.38900402188301086,0.4690322279930115,-0.21908485889434814,0.06534942984580994,-0.09507252275943756,-0.10922722518444061,-0.05699625611305237,-0.031187966465950012,0.08619870245456696,0.1474815160036087,0.09033043682575226,-0.04655039682984352,-0.18047496676445007,-0.12387923151254654,0.26619669795036316,-0.16795149445533752,-0.12015614658594131,-0.37688353657722473,-0.28292834758758545,0.07180991023778915,0.9948890805244446,0.08901812136173248,0.16902439296245575,0.09275513887405396,0.0014691996620967984,0.005647479090839624,-0.3428989052772522,-0.31666532158851624,-0.4016331136226654,0.1796700656414032,-0.1153717190027237,-0.02604624256491661,0.21946120262145996,0.6540656089782715,0.1338268518447876,-0.1489035189151764,1.1311229467391968,0.9888778328895569,-0.9279384613037109,-0.6369128823280334,-0.2257556915283203,0.916847288608551,0.7586669921875,0.29432398080825806,-0.361706405878067,-0.006152325309813023,0.09592199325561523,0.19373579323291779,0.08641296625137329,0.10175519436597824,0.2515322268009186,0.014469844289124012,-0.17101824283599854,0.04422421008348465,-0.12461630254983902,-0.30275166034698486,0.5333916544914246,0.06643041968345642,0.03210492804646492,0.14926141500473022,0.07832732796669006,-0.03323187306523323,-0.5999218225479126,0.2416476309299469,0.06577883660793304,0.04328596591949463,0.11503281444311142,0.36899903416633606,0.35206255316734314,-0.14235715568065643,0.006640881299972534,-0.057446278631687164,0.015142158605158329,0.40110906958580017,-0.2773725092411041,-0.016524402424693108,0.047951750457286835,0.07167202234268188,-0.16193455457687378,0.03690257668495178,0.25288742780685425,0.04912714287638664,-0.10478393733501434,-0.08806739747524261,-0.026553120464086533,-0.19559796154499054,0.25458988547325134,-0.012580211274325848,-0.053777772933244705,-0.011997280642390251,0.0037699805106967688,-0.07730097323656082,0.9006826877593994,0.06179668381810188,-0.010403776541352272,-0.0886271744966507,0.03487853705883026,0.2578091621398926,-0.5855004191398621,0.3450313210487366,0.20694376528263092,-0.0310494527220726,0.018108515068888664,-0.07476796209812164,0.01882769726216793,0.008881991729140282,-0.33994725346565247,0.20755980908870697,0.36053040623664856,-0.4929382801055908,0.6407672166824341,-0.2640534043312073,0.0756574496626854,-0.09167305380105972,0.04820699989795685,0.12184742838144302,0.3310514986515045,0.01969449408352375,0.04393260180950165,-0.1141587421298027,0.04316375032067299,0.03724924847483635,0.4118242859840393,0.03441966697573662,0.06868327409029007,0.09885704517364502,-0.07200797647237778,-0.17254434525966644,0.46763256192207336,0.022165071219205856,0.1919018030166626,0.3366394639015198,0.3589342534542084,0.16031454503536224,-0.6433174014091492,0.04733126610517502,-0.09951262921094894,0.059655409306287766,0.004702924285084009,0.0761677622795105,-0.6042079329490662,-0.01885995641350746,-0.018688198179006577,-0.10474279522895813,0.06551960110664368,0.06703010946512222,-0.45956951379776,-0.005746491253376007,-0.17116868495941162,-0.3802606761455536,-0.2647726237773895,-0.14840957522392273,0.5841073393821716,0.03802467882633209,0.11474578827619553,0.04503820836544037,0.24843822419643402,0.016823846846818924,0.11341434717178345,-0.0008356450707651675,-0.06703321635723114,0.07848859578371048,0.024161599576473236,-0.0035771820694208145,0.22426073253154755,0.04011731222271919,-0.04321274161338806,0.14549490809440613,-0.025712233036756516,0.02366986870765686,-0.01782803423702717,0.05392251908779144,-0.08337150514125824,0.11046622693538666,-0.013644182123243809,-0.030412456020712852,0.0240046214312315,-0.07773593813180923,0.1039014607667923,-0.0958295688033104,0.01869843900203705,0.019236905500292778,0.21336813271045685,0.34422242641448975,-0.319243460893631,-0.39324823021888733,0.45094138383865356,0.02845090813934803,0.18155111372470856,-0.08651205152273178,-0.09162988513708115,-0.13530051708221436,-0.21564792096614838,-0.2328002154827118,-0.17725519835948944,0.03038480132818222,0.029090847820043564,0.06109246984124184,0.08712486177682877,0.11939854919910431,-0.13102349638938904,-0.026446377858519554,-0.02958752028644085,-0.05399693176150322,-0.01953728124499321,0.04659644141793251,-0.11446240544319153,-0.04875820502638817,0.025174353271722794,0.020836347714066505,-0.029433388262987137,0.050300247967243195,-0.8252735137939453,0.01601352170109749,-0.019899846985936165,-0.005916091613471508,-0.0032847232650965452,-0.02384144626557827,-0.47836223244667053,0.0007863653590902686,0.0023984701838344336,0.006836785469204187,0.014324359595775604,-0.09210367500782013,0.21945828199386597,-0.13566380739212036,-0.062271930277347565,-0.11334778368473053,-0.05663912370800972,-0.017131386324763298,0.06530536711215973,0.039832279086112976,0.03215375542640686,0.03897356241941452,0.03206166252493858,0.0035584254655987024,1.4705729484558105,-0.011424228549003601,-0.025117743760347366,0.017151866108179092,0.017307188361883163,-0.21890093386173248,0.0515814833343029,-0.17531372606754303,0.01111412514001131,-0.03920687735080719,0.018594130873680115,-0.07948680967092514,-0.02049228921532631,-0.0673556849360466,0.2809760570526123,0.10224907845258713,0.013941392302513123,0.13300032913684845,0.738284707069397,0.1895570307970047,-0.11416144669055939,0.20689840614795685,-0.23171168565750122,0.44691362977027893,-0.3195688724517822,-0.029310191050171852,-0.08314323425292969,-0.09653011709451675,-0.02628924697637558,-0.08270561695098877,0.10026688873767853,-0.1106480285525322,-0.006533506326377392,0.0026308593805879354,-0.02729727141559124,-0.01298778411000967,-0.23126523196697235,0.10406032204627991,0.24133187532424927,0.2280656099319458,0.20596632361412048,0.20327994227409363,-0.4009084105491638,-0.09376328438520432,0.06942594796419144,0.10149380564689636,0.13736122846603394,0.21569569408893585,0.154552161693573,0.09739657491445541,-0.0030859860125929117,-0.014125283807516098,-0.03227918967604637,-0.044040746986866,0.29667213559150696,-0.11798983812332153,-0.2397173047065735,-0.2310023307800293,-0.18406134843826294,-0.1826174557209015,0.44684505462646484,0.08694919943809509,-0.0046926033683121204,0.08368433266878128,0.08411066979169846,0.016939710825681686,0.3009517788887024,0.0032006201799958944,-0.0034124997910112143,-0.02417127601802349,-0.0254774559289217,0.01519435178488493,0.3979244530200958,-0.05322978273034096,-0.04194755479693413,-0.04148422181606293,-0.023416774347424507,-0.011169090867042542,-0.2717444896697998,-0.1014086976647377,0.08036136627197266,0.04182131960988045,0.029329003766179085,0.05583757534623146,-0.010043137706816196,-0.1464369297027588,0.11816923320293427,-0.012017850764095783,0.2589063048362732,-0.36908045411109924,-0.35938766598701477,0.8864460587501526,-0.8998825550079346,0.5013027191162109,-0.2257995903491974,0.22479185461997986,-0.0675414428114891,0.002782267052680254,-0.02724994532763958,-0.028992721810936928,-0.009121596813201904,-0.06178080290555954,0.021934743970632553,0.004436344839632511,0.026032842695713043,0.024056412279605865,0.02026424929499626,0.039798177778720856,-0.6955615878105164,0.07282725721597672,0.07608980685472488,0.12042932212352753,0.1753741204738617,0.18118061125278473,-0.5514914393424988,0.002555098617449403,0.03264492005109787,0.022012468427419662,0.07239676266908646,0.02546601928770542,-0.03030533716082573,-0.005074485670775175,-0.029462313279509544,-0.020047131925821304,-0.04671751707792282,-0.037415869534015656,0.7706627249717712,-0.07706645876169205,-0.08271367847919464,-0.13075940310955048,-0.18076038360595703,-0.2071143090724945,0.6055879592895508,0.1262311339378357,0.15871037542819977,0.07941827923059464,0.16202005743980408,0.08094903081655502,-0.04096280783414841,-0.06543227285146713,-0.09776101261377335,-0.05924657732248306,-0.09913641214370728,-0.04335082322359085,0.5658462643623352,-0.13599401712417603,-0.14157456159591675,-0.15046806633472443,-0.07188783586025238,-0.019120121374726295,0.03155511990189552,0.008467300795018673,-0.10507164150476456,-0.042841143906116486,0.0011448387522250414,-0.027831709012389183,0.009626454673707485,-0.014806988649070263,0.16584385931491852,0.28264516592025757,0.4486536681652069,-0.21796318888664246,-0.31900110840797424,0.6371962428092957,-0.6182395219802856,0.03133954480290413,0.09415530413389206,-0.045720357447862625,-0.042131002992391586,0.013573694042861462,0.02541964314877987,-0.010312024503946304,-0.03710467740893364,0.0678456723690033,0.2693142890930176,0.0008730444824323058,-0.01821768842637539,0.01374217588454485,0.04943842440843582,-0.03424232453107834,0.34124118089675903,0.021895170211791992,0.017238054424524307,0.010336000472307205,0.024893512949347496,0.04815058037638664,-0.3108771741390228,0.10412284731864929,0.0861377939581871,0.043321214616298676,0.05031534284353256,0.15001454949378967,0.14362213015556335,-0.1446680724620819,-0.11162202805280685,-0.08736463636159897,-0.09995895624160767,-0.21683676540851593,0.2091200202703476,-0.041534245014190674,-0.03841639682650566,-0.0582425557076931,-0.059896696358919144,-0.0833626240491867,0.33971959352493286,0.11230512708425522,0.0858721062541008,0.04409798979759216,0.0488688200712204,0.044027794152498245,0.31547704339027405,-0.2052435427904129,0.0705895647406578,0.07439230382442474,0.04826352745294571,0.04345687851309776,0.6741552352905273,-0.08405507355928421,0.03277426213026047,0.07917097955942154,0.03888147696852684,0.056686654686927795,-0.1785184144973755,0.07047679275274277,0.17008596658706665,0.09589804708957672,-0.0038658473640680313,0.007242206484079361,-0.00807781983166933,-7.194824032632412e-39,-1.1412282593443063e-38,1.2728917407150304e-38,7.008008526562485e-39,4.360937510572869e-39,-1.5801862644626732e-38,6.941910679298748e-39,-1.0416056074902157e-38,-1.9987167010874985e-38,-1.692427048468623e-38,-2.0657591636460372e-39,7.971978555753098e-39,-0.0007643940043635666,-7.517994527006522e-05,0.0011423630639910698,0.007460514549165964,0.013150885701179504,0.02047545649111271,0.030100321397185326,0.04066313058137894,0.05117891728878021,0.06268710643053055,0.06856943666934967,0.0640471950173378,0.050829995423555374,0.03251292183995247,0.014471824280917645,0.0029876301996409893,0.0006870522629469633,0.00227527623064816,-0.0012763356789946556,-0.004584061913192272,-0.012437601573765278,-0.019491909071803093,-0.02720937691628933,-0.03773266077041626,-0.0506129115819931,-0.06274384260177612,-0.07136803865432739,-0.07072915881872177,-0.05693242698907852,-0.036609478294849396,-0.017761869356036186,-0.00528827914968133,-0.0014733538264408708,0.0016680218977853656,-0.0017726797377690673,0.0025163134559988976,0.00026513051125220954,0.0019456174923107028,0.0020801671780645847,0.0008694561547599733,0.0009016917319968343,0.0036132519599050283,0.0037702464032918215,0.006309461314231157,0.01314935926347971,0.01287764497101307,0.0050207204185426235,-0.0003493284457363188,-0.01819089613854885,0.013119162060320377],"sample_rate":48000.0}},{"max_value":1.0,"model":{"version":"0.7.0","metadata":{"date":{"year":2026,"month":5,"day":17,"hour":9,"minute":1,"second":28},"loudness":-12.049386978149414,"gain":0.612789648007463},"architecture":"WaveNet","config":{"layers":[{"input_size":1,"condition_size":1,"head":{"out_channels":1,"kernel_size":16,"bias":true},"channels":8,"kernel_sizes":[6,6,6,6,6,6,6,6,6,6,6,6,6,6,15,15,6,6,6,6,6,6,6],"dilations":[1,3,7,17,41,101,239,1,3,7,17,41,101,239,1,13,1,3,7,17,41,101,239],"activation":[{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01}],"bottleneck":8,"head1x1":{"active":false,"out_channels":1,"groups":1},"layer1x1":{"active":true,"groups":1},"groups_input":1,"groups_input_mixin":1,"conv_pre_film":{"active":false,"shift":true,"groups":1},"conv_post_film":{"active":false,"shift":true,"groups":1},"input_mixin_pre_film":{"active":false,"shift":true,"groups":1},"input_mixin_post_film":{"active":false,"shift":true,"groups":1},"activation_pre_film":{"active":false,"shift":true,"groups":1},"activation_post_film":{"active":false,"shift":true,"groups":1},"layer1x1_post_film":{"active":false,"shift":true,"groups":1},"head1x1_post_film":{"active":false,"shift":true,"groups":1},"gating_mode":["none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none"],"secondary_activation":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"slimmable":null}],"head":null,"head_scale":0.013119162069029721},"weights":[-2.708199977874756,1.518722653388977,2.0678908824920654,-1.4746893644332886,1.8213872909545898,1.583485722541809,0.43026724457740784,-2.332751750946045,0.010750990360975266,0.011915381997823715,0.011935512535274029,0.016323460265994072,0.01673297770321369,0.017248285934329033,-0.008671868592500687,-0.009857114404439926,-0.009906550869345665,-0.013710119761526585,-0.013825307600200176,-0.013919103890657425,-0.009980551898479462,-0.011135874316096306,-0.01124157290905714,-0.015299483202397823,-0.015604638494551182,-0.015871070325374603,0.008626227267086506,0.009722459129989147,0.009800530038774014,0.013623515143990517,0.013611781410872936,0.013682014308869839,-0.009396251291036606,-0.010530810803174973,-0.010661841370165348,-0.014683360233902931,-0.01494394987821579,-0.01524931937456131,-0.008933411911129951,-0.009977768175303936,-0.010045178234577179,-0.01389186829328537,-0.014010310173034668,-0.014149187132716179,-0.003665241878479719,-0.004495521076023579,-0.00490069342777133,-0.007277719676494598,-0.006902802269905806,-0.006552433595061302,0.010334426537156105,0.011462384834885597,0.01152486726641655,0.015768369659781456,0.016090799123048782,0.016407733783125877,-0.22126078605651855,-0.16176877915859222,-0.09408030658960342,-0.24135459959506989,-0.3601699769496918,-0.5376079678535461,0.10701704025268555,0.07671711593866348,0.07630858570337296,0.1591598093509674,0.24871820211410522,0.32435551285743713,0.14989745616912842,0.14074751734733582,0.1529988795518875,0.15210354328155518,0.350324809551239,0.4424012005329132,-0.12005482614040375,-0.10728505998849869,-0.08462131023406982,-0.17147760093212128,-0.2478979080915451,-0.30178287625312805,0.13311949372291565,0.0843011885881424,0.08390852808952332,0.1688326895236969,0.35139402747154236,0.4230392873287201,0.13811549544334412,0.09212638437747955,0.05540633574128151,0.16856394708156586,0.27142223715782166,0.3446059226989746,0.041794080287218094,0.022458622232079506,0.025374339893460274,0.06077304854989052,0.09303129464387894,0.11520745605230331,-0.14149031043052673,-0.10911538451910019,-0.09488236904144287,-0.1638813465833664,-0.36184218525886536,-0.4446091651916504,0.10268570482730865,0.05700470879673958,0.06608925014734268,0.08909949660301208,0.43510472774505615,0.6243542432785034,-0.06537137925624847,-0.023880504071712494,-0.035500142723321915,-0.07227897644042969,-0.2820356786251068,-0.4140956997871399,-0.07145122438669205,-0.01178249903023243,-0.03827093914151192,-0.09420035779476166,-0.380927175283432,-0.5251759886741638,0.06906753778457642,0.022641761228442192,0.032873645424842834,0.07498612254858017,0.2867022156715393,0.41280636191368103,-0.08002938330173492,-0.03263408690690994,-0.034250929951667786,-0.07832571119070053,-0.34592801332473755,-0.4952373802661896,-0.06393209099769592,-0.029876215383410454,-0.03724568709731102,-0.0830356553196907,-0.3046635687351227,-0.4529934525489807,-0.018433447927236557,-0.005087459925562143,-0.013559828512370586,-0.027506191283464432,-0.09682443737983704,-0.14162777364253998,0.10551492869853973,0.05306422337889671,0.046254027634859085,0.08746477961540222,0.3899362087249756,0.605506181716919,0.18329885601997375,0.1554454267024994,0.1004151776432991,0.15097621083259583,0.2953614890575409,0.25351423025131226,-0.12572002410888672,-0.08539800345897675,-0.058826662600040436,-0.09593193233013153,-0.14477548003196716,-0.14708206057548523,-0.19528046250343323,-0.13524703681468964,-0.15226945281028748,-0.17342600226402283,-0.24479831755161285,-0.15947869420051575,0.12808138132095337,0.10757962614297867,0.06675813347101212,0.1200144812464714,0.14151734113693237,0.17858661711215973,-0.1383955329656601,-0.1436891108751297,-0.13082921504974365,-0.13333868980407715,-0.154122456908226,-0.14799895882606506,-0.16486220061779022,-0.07851517200469971,-0.09308408200740814,-0.13599009811878204,-0.1624670922756195,-0.17618121206760406,-0.04448150098323822,-0.02641434594988823,-0.02333538420498371,-0.03613685443997383,-0.05507902055978775,-0.06228704750537872,0.20076912641525269,0.16247287392616272,0.1400408148765564,0.12912917137145996,0.19108611345291138,0.15839475393295288,-0.005738380365073681,-0.005796929821372032,-0.0065437196753919125,-0.011671068146824837,-0.018894534558057785,-0.03450039401650429,0.002968445885926485,0.0024787806905806065,0.003163605695590377,0.007610717322677374,0.014055093750357628,0.025926558300852776,0.004077491350471973,0.0035650143399834633,0.004303657915443182,0.009375077672302723,0.016614850610494614,0.030676115304231644,-0.0029278157744556665,-0.0024051149375736713,-0.003095983760431409,-0.0074216024950146675,-0.013718685135245323,-0.025535786524415016,0.003937237896025181,0.0035400015767663717,0.004202478565275669,0.008799494244158268,0.01551251020282507,0.02878757193684578,0.0032072158064693213,0.0027412709314376116,0.003467801259830594,0.007937275804579258,0.014387133531272411,0.026693154126405716,0.0004102429957129061,-0.00017850598669610918,0.0002553259255364537,0.0019968091510236263,0.005252325441688299,0.009958280250430107,-0.0052640000358223915,-0.004946265835314989,-0.00552966445684433,-0.01036661583930254,-0.017266152426600456,-0.0319945402443409,-0.3002333641052246,-0.3285265862941742,-0.30392152070999146,0.018921317532658577,0.7487377524375916,1.3850520849227905,0.20561358332633972,0.21890433132648468,0.1611560583114624,-0.005037939641624689,-0.44345253705978394,-0.8231340646743774,0.20645084977149963,0.33162033557891846,0.21083973348140717,0.03017219714820385,-0.6196483969688416,-1.1663920879364014,-0.1683957725763321,-0.24229121208190918,-0.16351088881492615,0.030219541862607002,0.47303763031959534,0.8594751358032227,0.20772896707057953,0.26761293411254883,0.16133293509483337,-0.0037443446926772594,-0.5594484806060791,-1.0485031604766846,0.21391627192497253,0.2463735193014145,0.16063621640205383,0.010698996484279633,-0.49031826853752136,-0.8967832922935486,0.058822423219680786,0.07960306107997894,0.04207496717572212,-0.010659916326403618,-0.164829820394516,-0.2901567220687866,-0.25468313694000244,-0.3543860912322998,-0.2399986833333969,0.03583882376551628,0.7005712985992432,1.2310742139816284,0.15079627931118011,0.19553515315055847,0.20545202493667603,-0.004535813815891743,-0.42152073979377747,-0.8715760707855225,-0.12315821647644043,-0.15299226343631744,-0.10295090079307556,0.0007206678856164217,0.28835031390190125,0.5656916499137878,-0.13561491668224335,-0.23435251414775848,-0.1383228898048401,-0.0016416548751294613,0.43607795238494873,0.7610684633255005,0.14337950944900513,0.17180562019348145,0.11756827682256699,-0.013632307760417461,-0.32243847846984863,-0.6353448033332825,-0.16035152971744537,-0.1768438071012497,-0.12162724137306213,-0.002356340177357197,0.34809258580207825,0.688629686832428,-0.14408911764621735,-0.16684317588806152,-0.09970123320817947,0.010389352217316628,0.3391743004322052,0.6389474272727966,-0.04807505011558533,-0.07534753531217575,-0.03267041593790054,0.009829462505877018,0.13398373126983643,0.25002187490463257,0.16355301439762115,0.2359524816274643,0.11024220287799835,-0.008425520732998848,-0.40126949548721313,-0.8084657788276672,-0.33923885226249695,-0.2034686803817749,-0.3384210765361786,-0.15468959510326385,0.25549525022506714,0.6251497268676758,0.19193491339683533,0.16749005019664764,0.15252962708473206,0.075452521443367,-0.21181200444698334,-0.3912357985973358,0.2553708255290985,0.205717995762825,0.28056079149246216,0.05236024409532547,-0.3288089632987976,-0.5270744562149048,-0.1702525019645691,-0.24794121086597443,-0.12256213277578354,-0.0860290378332138,0.19056956470012665,0.5152596831321716,0.23062066733837128,0.2999825179576874,0.2538178265094757,0.11813925206661224,-0.30323535203933716,-0.5561526417732239,0.15190763771533966,0.26818954944610596,0.17765510082244873,0.10672548413276672,-0.23406702280044556,-0.44726863503456116,0.10495960712432861,0.08403388410806656,0.07011962682008743,0.015321205370128155,-0.10330726951360703,-0.20474499464035034,-0.2342272400856018,-0.29432612657546997,-0.2178887277841568,0.008299601264297962,0.15945352613925934,0.6278958320617676,-0.0059206229634583,-0.15397733449935913,0.13532091677188873,0.11278944462537766,-0.018226174637675285,0.21533656120300293,-0.14123910665512085,0.5734551548957825,-0.01121464092284441,0.08397475630044937,-0.39534780383110046,-0.0940430760383606,0.020442143082618713,-0.8062251210212708,0.5968456268310547,-0.21952183544635773,-0.02625054307281971,-0.14417868852615356,0.2245415598154068,-0.15881232917308807,-0.031093716621398926,0.4970831274986267,-0.6442520022392273,0.5532757043838501,0.04994813725352287,0.41248130798339844,-0.6049526333808899,-0.253282368183136,0.06892772018909454,-0.9229233264923096,1.2375531196594238,-0.011512749828398228,0.031224649399518967,0.6906029582023621,-0.1676395684480667,-0.4021105468273163,0.0452188216149807,-0.3265276253223419,0.6795313358306885,-0.12018236517906189,-0.0010329406941309571,-0.372653067111969,0.21250146627426147,-0.059256456792354584,-0.01855330914258957,-0.07623443752527237,-0.20145247876644135,-0.2121296375989914,0.019872885197401047,0.006072149146348238,-0.23188620805740356,-0.10077086836099625,-0.004281274043023586,-0.17462392151355743,0.3456522524356842,-0.31364569067955017,0.016733473166823387,0.05721651762723923,-0.5640072822570801,-0.31921935081481934,0.015499633736908436,-0.8537059426307678,0.8418593406677246,-0.44851213693618774,-0.00355526152998209,0.4454911947250366,-0.0926581472158432,-0.3453078866004944,-0.015604039654135704,0.18643637001514435,-0.42529189586639404,0.4700900912284851,-0.0383007787168026,-0.3283631503582001,0.16173234581947327,0.09336846321821213,-0.03704991564154625,0.6543859839439392,-1.0920120477676392,-0.007253760006278753,-0.388923317193985,0.005992469377815723,0.28683552145957947,-0.27566227316856384,-0.28491199016571045,-0.2968733608722687,-0.24099107086658478,-0.48536819219589233,0.01671735756099224,0.06176357716321945,0.09257186949253082,-0.011216072365641594,0.1961139291524887,0.001531283720396459,0.07558295875787735,0.03767234459519386,0.01839909330010414,0.08602690696716309,-0.11614809185266495,-0.007759366650134325,0.03361544758081436,-0.04119519516825676,-0.04850197583436966,0.030912654474377632,-0.05480314418673515,0.059304408729076385,-0.012375143356621265,0.024884218350052834,0.02014109678566456,-0.0023581727873533964,-0.012789477594196796,-0.06560037285089493,0.004156098701059818,8.901669207261875e-05,0.0037612570449709892,0.03574274107813835,-0.045320410281419754,-0.0026014980394393206,0.11762697994709015,0.05922267958521843,0.02990509569644928,0.09805292636156082,-0.12659592926502228,-0.02666502073407173,0.0507286936044693,0.03397751972079277,0.030519386753439903,-0.023266779258847237,0.061572249978780746,0.004513496533036232,0.007689472287893295,0.05743740126490593,0.05200942978262901,-0.041574690490961075,0.10635819286108017,-0.05103112384676933,-0.09397557377815247,-0.12907321751117706,0.030066102743148804,0.05334176495671272,0.02363498881459236,-0.35540035367012024,-0.15253587067127228,-0.17835062742233276,-0.06366938352584839,-0.172315314412117,0.10495588928461075,0.4659712314605713,-0.12265897542238235,-0.13999652862548828,-0.018610069528222084,-0.01602994091808796,0.02821856178343296,0.256560355424881,-0.06123470515012741,-0.1071091741323471,-0.07033681124448776,0.0035160344559699297,0.04291835427284241,-0.22148467600345612,-0.10591140389442444,-0.10074524581432343,-0.041027747094631195,-0.2145809680223465,-0.19610756635665894,-0.013137800619006157,-0.06869187951087952,-0.13841612637043,-0.0540473647415638,-0.08517111092805862,0.0037892209365963936,0.1491401344537735,-0.43919864296913147,-0.11606945842504501,-0.09243464469909668,-0.15260161459445953,-0.13331317901611328,0.09969991445541382,0.028857208788394928,-0.02287720888853073,0.19746434688568115,0.08939436078071594,-0.05910190939903259,-0.42931002378463745,0.00859649945050478,0.06071055307984352,0.07014016807079315,0.008189585991203785,-0.07373469322919846,0.07922210544347763,-0.0160506758838892,-0.018148327246308327,-0.04000607505440712,0.05046712979674339,0.1170235201716423,-0.023961583152413368,-0.06878514587879181,-0.03708510100841522,-0.040572892874479294,0.0025610162410885096,0.06170102208852768,-0.02586347796022892,0.03269012272357941,-0.010249271988868713,0.005453358869999647,-0.0168931744992733,-0.012269830331206322,0.004465654958039522,0.012299200519919395,0.010183705016970634,-0.006961156148463488,0.00929503794759512,0.09150023758411407,0.005840563215315342,5.037813025410287e-05,-0.029195189476013184,-0.05711640417575836,4.8145593609660864e-05,0.148042231798172,-0.027741974219679832,-0.10108014196157455,0.004228118807077408,0.0046540736220777035,0.01769271120429039,-0.030020445585250854,0.05478322505950928,-0.0017457257490605116,0.004530992358922958,0.03564131632447243,-0.04752432554960251,-0.07964825630187988,0.053937751799821854,0.09443334490060806,-0.18829351663589478,-0.027323441579937935,-0.132404625415802,-0.03124210052192211,0.21216963231563568,0.1304091364145279,0.05783667787909508,-0.006009179167449474,0.06505470722913742,-0.016018208116292953,-0.3236023783683777,0.165941521525383,0.2163054496049881,0.20955879986286163,0.2248595952987671,0.08468297123908997,-0.2189478576183319,-0.20647811889648438,-0.13165590167045593,-0.17779521644115448,-0.1263318657875061,-0.01411525346338749,0.06016666442155838,0.009004662744700909,-0.066159687936306,0.03770207613706589,-0.09519249200820923,-0.11270695179700851,-0.1209762915968895,-0.09747524559497833,-0.0700671449303627,-0.017003551125526428,-0.08700615912675858,-0.002408405067399144,-0.3559064567089081,0.38206374645233154,0.12714852392673492,0.018895160406827927,0.10780483484268188,0.040944699198007584,-0.0642591193318367,-0.2157774120569229,-0.3020498752593994,-0.17793676257133484,-0.18784238398075104,-0.10400306433439255,0.4231954514980316,-0.000294152123387903,0.0006300618988461792,0.0004305029578972608,0.00026825579698197544,0.00016243947902694345,0.00010705285239964724,0.0006054156110621989,-0.0007268072222359478,-0.0006798469694331288,-0.0004699719720520079,-2.9640172215295024e-05,0.0002957381075248122,-0.0001498271303717047,-0.001181987696327269,-0.0013591874158009887,-0.0013809697702527046,-0.0010950122959911823,-0.0013865239452570677,0.0008473169291391969,0.0009849700145423412,0.0010898568434640765,0.0011160590220242739,0.0009460237342864275,0.0014293172862380743,0.0017643470782786608,0.0012443733867257833,0.0013475969899445772,0.0013693050714209676,0.0014409755822271109,0.0015834799269214272,0.00391932250931859,0.002634766511619091,0.002811016049236059,0.0031395009718835354,0.0036402472760528326,0.004343329928815365,-0.00032970382017083466,-5.795828838017769e-05,-0.0002086995664285496,-0.00037291907938197255,-0.00033627758966758847,-0.00018070489750243723,0.0006323399138636887,0.001864540739916265,0.0018266147235408425,0.0016301210271194577,0.0011358082992956042,0.0013438729802146554,-0.0051566907204687595,0.1404748558998108,0.15056847035884857,0.08275532722473145,0.022559957578778267,-0.37452664971351624,-0.1216825470328331,-0.18418146669864655,-0.08110414445400238,-0.10602259635925293,-0.14103639125823975,0.3238690495491028,-0.1052369624376297,0.0025019992608577013,0.00028890592511743307,0.020329535007476807,0.0718579888343811,0.284004271030426,0.012461197562515736,-0.08441285043954849,-0.038891952484846115,-0.0326850600540638,-0.1566188484430313,-0.2258622944355011,-0.09681341797113419,-0.07990109920501709,-0.1940959244966507,-0.11121398955583572,-0.08915156126022339,-0.008983917534351349,-0.23740597069263458,-0.10993089526891708,-0.3335903584957123,-0.19223156571388245,-0.22858445346355438,0.29265958070755005,-0.28826501965522766,0.02889983169734478,0.03882423788309097,-0.1409720480442047,0.12265006452798843,0.08071926981210709,-0.009660765528678894,0.009400520473718643,0.018417147919535637,-0.038190703839063644,0.11727260053157806,-0.342854768037796,-0.040822505950927734,0.0051960633136332035,0.033160220831632614,0.11073539406061172,0.15754173696041107,0.10382233560085297,-0.061702385544776917,0.04692671075463295,0.03159019723534584,0.04880473390221596,0.016711505129933357,-0.19149722158908844,0.08163660019636154,-0.026529759168624878,0.07010263204574585,0.09888721257448196,0.08485063910484314,-0.1157124936580658,-0.018462464213371277,-0.0673307478427887,-0.06015852838754654,-0.017700571566820145,-0.13750289380550385,-0.13570156693458557,0.045005448162555695,-0.07286074012517929,-0.055368874222040176,-0.04356183856725693,-0.05879174545407295,0.029787477105855942,-0.034241534769535065,-0.0002158792340196669,-0.048067159950733185,-0.037060368806123734,0.0028395934496074915,0.09157146513462067,0.029073376208543777,0.0918935239315033,0.21850304305553436,0.2570393979549408,0.27203553915023804,0.10973580181598663,0.04454120621085167,-0.019653426483273506,-0.021947238594293594,-0.010664386674761772,-0.009948051534593105,0.0987626239657402,-0.04307130351662636,-0.016418153420090675,0.006903953850269318,-0.014562714844942093,0.04002838581800461,0.07020139694213867,0.1258683204650879,0.07475505769252777,0.014304486103355885,0.028294727206230164,-0.0356714203953743,-0.09286336600780487,0.040604010224342346,0.02429606392979622,-0.01973036862909794,-0.010704790242016315,-0.047494206577539444,-0.07974839955568314,0.001500979415141046,0.0015129580860957503,0.03440569341182709,0.03365175426006317,0.04183359816670418,0.05164439231157303,0.08066762238740921,0.07123157382011414,0.05618549883365631,0.06580105423927307,0.042567722499370575,0.018636375665664673,0.12155824899673462,0.08731400221586227,0.05436716601252556,0.06718118488788605,0.01249144971370697,-0.03788569197058678,-0.03228972852230072,0.0073615750297904015,-0.016078969463706017,-0.029832862317562103,0.009290582500398159,0.020811470225453377,-0.09044414013624191,-0.03378421068191528,0.021715516224503517,0.003938682842999697,0.06444857269525528,0.11419308185577393,0.0017894390039145947,0.04462771862745285,-0.07245565205812454,0.08262426406145096,-0.0039152768440544605,0.19842135906219482,-0.030124995857477188,-0.1267433613538742,0.004348946735262871,0.3351999819278717,-0.005623926408588886,-0.3673456311225891,-6.410803325707093e-05,0.6640926599502563,-0.12481506913900375,-0.00444164639338851,-0.015928607434034348,-0.3064620792865753,-0.03344106301665306,0.2894522547721863,-0.0005709708202630281,-0.16813097894191742,0.24603839218616486,0.01061626709997654,0.09117615967988968,0.18128521740436554,0.11894391477108002,-0.09743382036685944,9.483896428719163e-05,0.24262049794197083,-0.27619364857673645,0.15535810589790344,-0.1869833916425705,0.3025248646736145,0.23178866505622864,-0.17950738966464996,-6.385172309819609e-05,-0.21588344871997833,-0.2814822793006897,0.12262426316738129,0.16349993646144867,0.076729416847229,0.0767659991979599,-0.3567904233932495,0.0009674248867668211,0.25308048725128174,0.06851716339588165,0.03089563176035881,0.14967910945415497,0.3211625814437866,-0.0005753624718636274,-0.27974995970726013,0.000887626432813704,0.28154709935188293,0.16805794835090637,-0.05646105483174324,0.2477494478225708,0.1891796588897705,0.0640111044049263,-0.221125066280365,0.00028423918411135674,0.2283799946308136,0.00725936284288764,0.019042009487748146,0.011297913268208504,-0.28863972425460815,0.10029296576976776,-0.007310154847800732,-0.0016831224784255028,-0.21823635697364807,0.1654759645462036,0.1089717298746109,-0.04176551103591919,0.022058235481381416,0.14484049379825592,0.2508445978164673,-0.0002037770755123347,-0.17966775596141815,-0.08358731120824814,-0.09356968849897385,-0.1700841635465622,0.01663350872695446,-0.21114733815193176,0.15972939133644104,-0.2165370136499405,-0.2434169203042984,-0.013812948018312454,-0.42748066782951355,0.0523671954870224,0.054377734661102295,0.037146031856536865,0.04002833366394043,0.0619649700820446,0.0385211743414402,-0.059768181294202805,-0.0583934411406517,-0.07037615031003952,-0.04489662125706673,-0.020285971462726593,0.275398313999176,0.03363176807761192,0.04261424019932747,-0.027923181653022766,-0.04611523449420929,-0.10699914395809174,-0.006728603038936853,0.024269118905067444,0.03853019326925278,0.010770815424621105,0.022118285298347473,0.032804373651742935,0.042554453015327454,0.09163503348827362,0.12531845271587372,0.06372274458408356,0.09367607533931732,0.1207931712269783,0.20004071295261383,0.040795378386974335,0.007941878400743008,0.016518469899892807,0.05376218631863594,0.062134694308042526,0.19554851949214935,0.019901540130376816,0.05834946036338806,0.008867520838975906,0.07958585023880005,-0.028471941128373146,0.012695408426225185,0.0843578577041626,0.09347549825906754,0.04109833762049675,0.012055070139467716,0.09953458607196808,0.09594827890396118,-0.07583725452423096,-0.018474778160452843,-0.007468302734196186,0.043171100318431854,0.06660676747560501,-0.41212624311447144,0.004809670615941286,-0.036360811442136765,-0.12711472809314728,0.0515279583632946,-0.12755164504051208,0.35075557231903076,-0.17895786464214325,-0.2336111068725586,-0.2110031694173813,-0.06676923483610153,-0.18691693246364594,0.2062385231256485,-0.07587884366512299,-0.030171187594532967,0.1495290845632553,0.08950269222259521,0.07708059251308441,-0.32924672961235046,-0.013658973388373852,-0.00931402388960123,-0.07580022513866425,-0.011942746117711067,0.05081188678741455,-0.06038709729909897,-0.0992460772395134,-0.07602903991937637,-0.12525172531604767,-0.13784553110599518,-0.06249382346868515,-0.07308287173509598,-0.12619973719120026,-0.1756342500448227,-0.270840585231781,-0.10027841478586197,-0.16374492645263672,0.008689376525580883,-0.1550237238407135,0.00026779400650411844,-0.07149972766637802,-0.09154235571622849,-0.16285130381584167,-0.06862877309322357,0.008635654114186764,0.010708088055253029,0.007718465756624937,0.015633409842848778,0.014170495793223381,-0.03412938490509987,0.0020433771423995495,-0.004499347880482674,0.034249626100063324,0.005255833268165588,0.007279494311660528,0.04123767465353012,0.01949581876397133,0.011652637273073196,0.021411290392279625,0.03580055013298988,0.02590886317193508,0.044920921325683594,-0.001118327840231359,0.014883507043123245,-0.015351954847574234,-0.007687628734856844,-0.004417939577251673,0.026402730494737625,0.020515503361821175,0.021560657769441605,0.04171721637248993,0.010585580952465534,0.02511487901210785,0.05368751287460327,0.037464965134859085,0.025885749608278275,0.04827757179737091,0.023892100900411606,0.020927701145410538,0.0752844288945198,0.010931667871773243,0.011449955403804779,-0.009349029511213303,0.02813384309411049,-0.003447741735726595,-0.010289013385772705,0.02542746439576149,0.03376775607466698,0.022295450791716576,0.0363890677690506,0.0480746291577816,-0.01896338164806366,0.034698788076639175,0.03850173577666283,0.02759825438261032,0.04962453246116638,-0.036568522453308105,0.08439334481954575,-0.0054175350815057755,-0.011165662668645382,-0.02090415358543396,0.011934877373278141,0.12687313556671143,0.05467475950717926,0.005408817902207375,-0.02448509819805622,-0.009562828578054905,-0.0095833083614707,0.08377265930175781,-0.06994747370481491,-0.027418918907642365,-0.010717041790485382,-0.004543830640614033,-0.0390525683760643,-0.04496122524142265,-0.002549197990447283,0.07022318243980408,0.06836364418268204,0.06759583950042725,0.08399451524019241,0.11191296577453613,0.08115581423044205,0.030886180698871613,0.03399934619665146,0.03396384045481682,-0.020664513111114502,0.11197930574417114,0.10990709066390991,-0.00099488056730479,-0.010907778516411781,-0.0175660140812397,-0.017512472346425056,-0.04087314382195473,-0.0034894465934485197,0.07816433161497116,0.06898213922977448,0.04981285706162453,0.05690738558769226,-0.024673186242580414,0.11953626573085785,0.00019903089560102671,-0.03217708319425583,0.025918971747159958,-0.08547508716583252,-0.006667228415608406,0.21781562268733978,-0.0030725242104381323,-0.0117790000513196,0.18505366146564484,0.013383954763412476,0.11626381427049637,-0.23511354625225067,0.013035294599831104,0.005897060967981815,0.0731019452214241,0.19465282559394836,0.121580570936203,-0.38036781549453735,0.012776757590472698,-0.1575668901205063,-0.10493676364421844,-0.09088645875453949,-0.10148728638887405,0.20445843040943146,-0.013937246054410934,-0.15578800439834595,-0.14129751920700073,-0.13613605499267578,-0.1961241513490677,0.05199126899242401,-0.09273052960634232,0.04113195091485977,0.0033197004813700914,-0.030299870297312737,0.044239409267902374,-0.15872301161289215,-0.12929369509220123,-0.10818168520927429,0.06027272716164589,-0.06214713305234909,0.016247570514678955,0.02781098708510399,-0.07098721712827682,-0.12982510030269623,-0.09571253508329391,-0.05650828778743744,-0.1777983009815216,0.03617636859416962,-0.0021940087899565697,-0.024523328989744186,0.046891018748283386,0.084752656519413,0.011830458417534828,-0.0013555999612435699,-0.010674274526536465,0.039334241300821304,0.04291641339659691,-0.031721558421850204,0.01590653508901596,-0.04558776319026947,0.04379231110215187,0.07732492685317993,0.04175667464733124,-0.06489399820566177,0.01977561041712761,0.0030831308104097843,0.011249246075749397,-0.027654001489281654,-0.05264374241232872,-0.04334403946995735,0.023776326328516006,0.09435819834470749,-0.02771688438951969,0.017914822325110435,0.02141532674431801,-0.03364897519350052,0.007538455072790384,0.024606481194496155,0.03024943359196186,0.05941232293844223,0.02387557551264763,0.016014374792575836,0.07532386481761932,0.036887120455503464,0.06899737566709518,0.036351051181554794,0.03441528230905533,0.00853211060166359,0.03595176711678505,0.020660683512687683,-0.007352238986641169,-0.0035285388585180044,0.052605364471673965,0.07819133251905441,0.034610141068696976,0.04194194823503494,0.028783636167645454,0.03529110550880432,0.011622321791946888,-0.045510195195674896,0.031188923865556717,0.012137295678257942,0.021600909531116486,0.0015005908207967877,0.032728660851716995,0.0325440876185894,0.010755767114460468,0.07453563064336777,-0.005346877500414848,-0.037023477256298065,-0.005629666615277529,0.04663420096039772,-0.03396358713507652,0.03839906305074692,-0.00023350465926341712,0.0201772041618824,0.014924861490726471,0.04754600301384926,-0.010788747109472752,0.012022429145872593,0.020939400419592857,0.024836692959070206,0.04655245691537857,0.0813441053032875,0.04568389058113098,0.07182585448026657,0.03617653623223305,0.034970302134752274,0.06671155244112015,0.059511058032512665,0.014645902439951897,0.06823325902223587,0.018221572041511536,-0.0048096696846187115,-0.014992598444223404,0.003104773350059986,-0.06296943128108978,0.011763323098421097,0.0211641825735569,0.0408158116042614,0.05370762199163437,-0.039014872163534164,0.05117574706673622,0.02803357131779194,-0.026971960440278053,0.026483049616217613,-0.05663062632083893,-0.1598261296749115,-0.23553557693958282,0.24874956905841827,0.11702996492385864,0.1648595929145813,0.23406477272510529,0.10364668816328049,0.1197642907500267,-0.38680827617645264,-0.10108324885368347,0.026386912912130356,0.16826708614826202,0.01184565108269453,0.11161427944898605,-0.4136316776275635,-0.03168490529060364,-0.08300041407346725,-0.20110705494880676,-0.24642351269721985,-0.20690152049064636,0.28356191515922546,0.008458222262561321,0.058623794466257095,-0.01688445918262005,-0.11053680628538132,-0.14598609507083893,0.06398764997720718,-0.16113512217998505,-0.13770325481891632,-0.11491525918245316,0.022440822795033455,-0.0694180503487587,-0.15644140541553497,-0.0572284460067749,-0.0031222286634147167,0.11310567706823349,-0.08115889877080917,-0.06035425886511803,-0.12927353382110596,-0.11887788772583008,-0.042523059993982315,-0.08742079883813858,-0.09448093920946121,-0.21814271807670593,0.06663951277732849,-0.1745867282152176,0.13855895400047302,-0.04950752109289169,-0.10279973596334457,0.16483499109745026,-0.017586560919880867,-0.03894122689962387,0.014245161786675453,-0.017486289143562317,0.3068403899669647,0.0028079391922801733,-0.0009640166535973549,-0.2948325276374817,-0.006986032240092754,-0.0029079276137053967,-0.20947615802288055,0.23589956760406494,-0.15965814888477325,-0.09113407880067825,-0.07632257044315338,0.08539314568042755,0.17495885491371155,-0.03405684605240822,-0.026288378983736038,-0.4687057137489319,-0.02838587760925293,-0.035240620374679565,-0.042636580765247345,0.04977845028042793,0.0186173003166914,-0.011378702707588673,-0.06569356471300125,0.0495651513338089,0.24896171689033508,-0.026817817240953445,0.28438088297843933,-0.05251812934875488,-0.10766613483428955,0.060513678938150406,-0.08213762193918228,-0.010235300287604332,0.1128019392490387,0.03766458109021187,-0.03605806082487106,-0.08645742386579514,-0.09445487707853317,0.058590780943632126,-0.17179390788078308,-0.28182822465896606,0.08183522522449493,-0.0497114323079586,-0.0868896022439003,-0.2655007541179657,-0.09879906475543976,0.10172862559556961,-0.059158805757761,0.033619362860918045,0.05481790378689766,-0.00027265032986178994,-0.05725462734699249,-0.17941810190677643,-0.08368373662233353,0.09153258800506592,-0.1175018772482872,0.05921661853790283,-0.14738360047340393,-0.08480354398488998,0.17752119898796082,-0.03588859736919403,0.0018017677357420325,0.06418490409851074,0.397685170173645,0.4285106658935547,-0.21650998294353485,0.09549561142921448,0.010109075345098972,0.3622681200504303,0.013464853167533875,0.11120018362998962,0.2618601620197296,-0.0542166493833065,-0.22457009553909302,0.10143184661865234,-0.03974224627017975,0.07576480507850647,0.2814709544181824,0.07409278303384781,0.054549891501665115,0.011962908320128918,0.01168088149279356,0.01116303913295269,0.012452411465346813,0.009372062981128693,0.0123392753303051,0.002154232934117317,0.001237757271155715,0.0010000787442550063,5.434970807982609e-05,0.0014069706667214632,0.0016423302004113793,-0.0063824765384197235,-0.006594018545001745,-0.00501681724563241,-0.005036284681409597,-0.007364107761532068,-0.003581519238650799,0.005765901878476143,0.00632316991686821,0.005200889892876148,0.00909434910863638,0.009770887903869152,0.005606087855994701,0.0033716419711709023,0.0018058273708447814,0.0005642978940159082,-0.0002425648708594963,0.00030371505999937654,-0.00045247009256854653,0.013209923170506954,0.011907127685844898,0.012416686862707138,0.012295459397137165,0.012457812204957008,0.013338334858417511,0.0035846815444529057,0.006266904063522816,0.008037656545639038,0.008724838495254517,0.008109892718493938,0.011082351207733154,-0.0038092853501439095,-0.004773668013513088,-0.004001562017947435,-0.003936573397368193,-0.002829217351973057,-0.00411025108769536,0.12909850478172302,0.08692913502454758,0.08730274438858032,0.10930146276950836,0.12089847773313522,0.2192147821187973,-0.09466466307640076,-0.018489588052034378,-0.0156226372346282,0.011110604740679264,0.048806313425302505,-0.04509643465280533,0.027094244956970215,-0.003216381184756756,0.02726573869585991,0.02597147226333618,0.041957881301641464,-0.08336161822080612,0.002706762170419097,-0.008092505857348442,0.005752407014369965,-0.03555629029870033,-0.08140803128480911,-0.0929933488368988,0.10655923187732697,0.08618416637182236,0.09524797648191452,0.06543367356061935,0.0761793926358223,-0.012619884684681892,0.055261217057704926,0.022620683535933495,0.021847182884812355,0.08967307209968567,0.10562926530838013,-0.15576371550559998,0.004450745414942503,-0.01353090163320303,0.029185261577367783,0.0413433276116848,0.1037459522485733,0.18652763962745667,-0.03658280521631241,-0.00286339083686471,9.53976486925967e-05,-0.015642287209630013,-0.03031601570546627,0.24547043442726135,-0.01887286640703678,-0.013113261200487614,-0.08529169857501984,-0.09342736750841141,-0.14357900619506836,-0.008059951476752758,-0.018121175467967987,-0.03229586035013199,-0.03122078813612461,-0.011007159948348999,0.1460372507572174,-0.23556064069271088,0.02236752212047577,-0.004565653391182423,0.03633173927664757,0.05974516645073891,0.1444656252861023,-0.19998832046985626,0.020405594259500504,0.03399614989757538,-0.008417044766247272,-0.1095084622502327,-0.26380112767219543,0.14081238210201263,-0.04842190444469452,-0.020641524344682693,0.020943380892276764,-0.057767558842897415,-0.1613515466451645,0.008876554667949677,-0.05611741542816162,-0.027223894372582436,-0.0643998309969902,-0.015420571900904179,-0.013155593536794186,-0.14625225961208344,-0.09538555890321732,-0.07082431018352509,-0.03909013420343399,0.014154018834233284,0.18224096298217773,-0.20614637434482574,-0.018617570400238037,0.011396427638828754,0.050688933581113815,-0.028880475088953972,-0.1924402415752411,-0.16665758192539215,-0.025672949850559235,-0.003201801562681794,-0.06319621950387955,-0.05989924818277359,-0.060889605432748795,-0.12633554637432098,0.01588679850101471,0.0054619391448795795,-0.003554966766387224,-0.0606420673429966,-0.1831756830215454,0.13330310583114624,-0.0040900022722780704,0.02658306248486042,0.0013190784957259893,-0.0032488792203366756,-0.10866834223270416,-0.03231745958328247,-0.05992860347032547,-0.026471000164747238,-0.06053772196173668,-0.0033075350802391768,0.13105496764183044,-0.18857936561107635,0.011347758583724499,-0.005493626929819584,-0.0021895102690905333,0.016965294256806374,0.029894491657614708,-0.09091351926326752,-0.043138109147548676,-0.004523246083408594,-0.06455691903829575,-0.06085521727800369,-0.16389049589633942,0.009557751007378101,-0.04652474448084831,-0.004110026638954878,-0.046538058668375015,-0.054851751774549484,-0.19124644994735718,0.07874557375907898,0.0017829649150371552,0.0060291048139333725,0.007322156801819801,0.007697688415646553,-0.042638830840587616,0.02902526594698429,-0.03606262430548668,0.03431568294763565,-0.015885721892118454,-0.033159349113702774,-0.03614380955696106,0.05400572344660759,-0.0009507177164778113,0.001412928570061922,-0.017898032441735268,-0.013301283121109009,0.1010018140077591,-0.15506507456302643,-0.006195329129695892,0.00212924275547266,-0.0029798487666994333,-0.005293677095323801,0.14727917313575745,-0.09143264591693878,-0.011227048933506012,0.003804838052019477,0.004613082855939865,-0.04922519251704216,-0.1665089726448059,0.1963547319173813,-0.043420277535915375,-0.03343382477760315,0.02465178444981575,0.006145465187728405,-0.06085861101746559,0.028548533096909523,-0.03394890949130058,0.0527004636824131,-0.03333134576678276,-0.03135065361857414,0.02701612561941147,0.07040238380432129,-0.04464814066886902,-0.011746708303689957,0.0039555663242936134,-0.00752687593922019,0.11276738345623016,-0.1380392163991928,-0.023153379559516907,0.0020129806362092495,-0.008358621038496494,-0.06511514633893967,-0.04391409829258919,-0.036915749311447144,0.015984540805220604,0.02215067483484745,0.009586573578417301,0.03776748478412628,0.004767243284732103,-0.022249698638916016,0.015427946113049984,-0.007760830223560333,0.010698826052248478,0.00027191516710445285,-0.023059485480189323,0.05828193575143814,0.00846070796251297,-0.007240499369800091,0.014202374964952469,0.0034337742254137993,-0.023986533284187317,-0.023356758058071136,-0.015991516411304474,0.016185138374567032,0.002332740928977728,0.0010137205244973302,0.0406915508210659,0.055922314524650574,0.007992754690349102,0.020940791815519333,0.027756597846746445,0.03479727357625961,0.04592948406934738,0.05810113623738289,0.028231794014573097,0.022439058870077133,-0.0033925457391887903,-0.016415003687143326,-0.008130514062941074,0.08553842455148697,0.040416035801172256,0.009134011343121529,0.021934164687991142,0.021438097581267357,-0.03634916990995407,-0.0688432902097702,-0.013043832033872604,0.025448894128203392,-0.007640753872692585,0.03268642723560333,0.047593750059604645,0.02370806410908699,-0.04399384185671806,-0.011948148719966412,0.00317075802013278,-0.06658932566642761,-0.08103692531585693,-0.04337267950177193,-0.023365356028079987,-0.052465178072452545,-0.030956458300352097,-0.05088644102215767,0.0703824907541275,-0.2588604688644409,0.023407630622386932,0.03392265737056732,0.06753463298082352,0.08164247125387192,0.15510277450084686,0.0011452015023678541,-0.03879097104072571,-0.009392376989126205,-0.012997829355299473,-0.16343724727630615,-0.2353842705488205,0.0916450023651123,0.037001900374889374,-0.014430666342377663,-0.0011643551988527179,0.005117789842188358,-0.027942446991801262,0.014490150846540928,-0.03277868404984474,0.011851785704493523,-0.05282112956047058,0.014084258116781712,-0.01007429976016283,-0.12721730768680573,-0.003789447946473956,-0.01894785277545452,-0.023262031376361847,-0.025929678231477737,0.1534019261598587,-0.05066618695855141,0.008582773618400097,-0.0014871950261294842,-0.022923005744814873,-0.02590959146618843,-0.0809846892952919,-0.17623405158519745,-5.296582821756601e-05,-0.010371465235948563,0.01046698447316885,0.018660306930541992,-0.027369758114218712,-0.06596720963716507,0.00394004350528121,-0.0003436569531913847,0.015244418755173683,0.007451585028320551,-0.20645754039287567,0.0739370658993721,0.003440768923610449,-0.007658598478883505,0.01665991172194481,0.04277757555246353,-0.07102920860052109,-0.45434120297431946,-0.00031951398705132306,-0.03338506072759628,-0.003327078651636839,-0.023215793073177338,0.06005462259054184,0.3133610486984253,0.005246760789304972,0.00020440314256120473,-0.0030961160082370043,0.0617624931037426,0.08038321882486343,0.019375387579202652,-0.007900119759142399,0.005186088383197784,-0.008308961987495422,-0.024317370727658272,-0.0492163710296154,-0.3024464249610901,-0.006565870717167854,-0.017027761787176132,-0.012569603510200977,0.012271598912775517,-0.12424815446138382,-0.21630685031414032,0.006355376914143562,0.0015507243806496263,0.0028890392277389765,0.016685662791132927,-0.0009072396787814796,0.278278648853302,-0.012761072255671024,-0.3042920231819153,0.028039922937750816,0.045091696083545685,-0.0031253162305802107,-0.02635340578854084,0.15057329833507538,-0.004785833414644003,-0.0004970547743141651,0.015646932646632195,-0.09306631982326508,0.07830124348402023,-0.02687942050397396,-0.002228447003290057,-0.0009436106774955988,0.03676794096827507,0.002524727489799261,-0.1506466418504715,-0.005782684776932001,-0.3680035173892975,0.12511423230171204,0.036775361746549606,0.2711549401283264,-0.2344704270362854,0.0006696264026686549,0.30279073119163513,0.051748231053352356,0.09257961809635162,-0.000458657304989174,-0.1612149178981781,0.0007750566001050174,0.2346341460943222,-0.001092005637474358,0.004583359230309725,-0.09405200928449631,0.19261953234672546,0.03180311620235443,0.07911784201860428,-0.13232725858688354,0.21669380366802216,0.00026902765966951847,0.25169527530670166,-0.1456364393234253,-0.012061388231813908,-0.15985502302646637,0.1210893839597702,-0.07896390557289124,0.03925540670752525,-0.00018099798762705177,0.23339582979679108,0.016357263550162315,0.09504535049200058,-0.18607313930988312,0.029673222452402115,-0.13572274148464203,0.08992228657007217,-0.0011269589886069298,0.2093667984008789,-0.10966247320175171,0.056027814745903015,-0.16726382076740265,0.03395035117864609,0.015443289652466774,0.39030760526657104,0.002137497067451477,-0.19155019521713257,0.2582554817199707,0.00529828853905201,0.00395243801176548,-0.03599749132990837,-0.14827829599380493,0.3168904483318329,0.00386819988489151,-0.1431242823600769,0.19060319662094116,0.1602030247449875,-0.17516060173511505,0.006891300901770592,0.006364353932440281,-0.12534025311470032,0.17453676462173462,-0.24781173467636108,-0.2274036705493927,-0.07716944068670273,0.1364249289035797,-0.22001849114894867,-0.195912167429924,0.05082407221198082,-0.06157756596803665,-0.049609988927841187,-0.09645874053239822,-0.09845763444900513,-0.058724477887153625,-0.2910315990447998,0.05992788448929787,0.09997627139091492,0.06988915801048279,0.05733877792954445,0.07662343233823776,0.05876659229397774,-0.027041543275117874,-0.035558149218559265,-0.04862343892455101,-0.02315247245132923,-0.0737437829375267,-0.10528222471475601,-0.0407523475587368,-0.022975783795118332,-0.09554355591535568,-0.09753696620464325,0.050971951335668564,0.10738711804151535,-0.062209583818912506,-0.10057979077100754,-0.08776405453681946,-0.07698513567447662,-0.05568546801805496,0.005430766846984625,-0.06702272593975067,-0.057420603930950165,-0.013089178130030632,-0.06717434525489807,-0.07609580457210541,0.021132364869117737,-0.05051890388131142,-0.04687965288758278,-0.03363131359219551,-0.05710434168577194,-0.04783117026090622,-0.18991978466510773,-0.015309907495975494,-0.04459810629487038,-0.03981209173798561,-0.041416093707084656,0.011443929746747017,-0.359861820936203,0.015126680955290794,0.037950221449136734,0.03945678472518921,-0.010739504359662533,-0.06329593807458878,-0.03659160062670708,-0.0009168778196908534,0.010264221578836441,-0.0064476472325623035,0.0009197673061862588,0.020644301548600197,0.21476587653160095,-0.004168917890638113,0.018253212794661522,0.005428450647741556,-0.008239667862653732,-0.007943765260279179,0.16841907799243927,0.004009283147752285,0.03880644217133522,0.024656597524881363,0.02345881424844265,-0.024551549926400185,-0.22187010943889618,-0.0038578230887651443,-0.0061666835099458694,-0.005587984342128038,0.0018453766824677587,-0.010929650627076626,-0.061180803924798965,0.011982284486293793,0.007930515334010124,0.028988823294639587,0.01675187051296234,-0.003072875551879406,-0.02577994018793106,-0.002892138436436653,0.009060136042535305,0.01320548728108406,0.04464981332421303,0.03488237038254738,0.1432895064353943,-0.005590073298662901,0.0034555981401354074,-0.00804834347218275,-0.006901430897414684,0.00558615755289793,-0.02731095626950264,0.06491319090127945,0.0460221990942955,0.06545333564281464,0.07600213587284088,0.07684418559074402,-0.14747315645217896,0.0040442124009132385,0.005059823859483004,0.008940947242081165,0.022495364770293236,-0.012908759526908398,-0.09595384448766708,-0.003185620065778494,-0.01381529401987791,-0.02833646535873413,-0.027411142364144325,-0.018610501661896706,0.4955068826675415,-0.03680838271975517,-0.048237383365631104,-0.06346838921308517,-0.05696268752217293,-0.026799650862812996,0.3616291880607605,0.055949870496988297,0.03790578246116638,0.08075135946273804,0.09250252693891525,0.18799641728401184,0.6970909833908081,0.04255687817931175,0.0603620782494545,0.11556648463010788,0.07363750040531158,-0.019113963469862938,0.3265516459941864,0.003810744732618332,0.0022992687299847603,0.057988204061985016,0.057865943759679794,0.047047778964042664,0.05014481022953987,0.007962768897414207,0.010712034069001675,0.0161159485578537,-0.006972467992454767,0.015311154536902905,-0.013085782527923584,0.009560755454003811,0.00468518678098917,0.008349267765879631,0.007802641950547695,0.0067701456137001514,0.007483162451535463,0.005442208610475063,0.00213241227902472,4.239217741996981e-05,-0.002190625760704279,-0.0017774321604520082,-0.0008781905053183436,0.0038308114744722843,0.0009181617060676217,0.0056389449164271355,0.0020690420642495155,-0.00010446264786878601,0.00025002757320180535,-0.002759535564109683,0.005665327422320843,0.003953750245273113,0.008026551455259323,-0.0006373234209604561,0.007547320332378149,-0.001816987874917686,0.001469874638132751,-0.0008039652020670474,0.0006603687652386725,-0.0026205938775092363,0.00027394157950766385,0.009669439867138863,0.008217450231313705,0.007990799844264984,0.007284959312528372,0.0019996294286102057,0.008623096160590649,0.003010833403095603,-0.0017207895871251822,0.003569627646356821,0.0018618355970829725,0.004213252104818821,0.002797188237309456,-0.009942363016307354,-0.005406934302300215,-0.008276848122477531,-0.0055063399486243725,-0.0014002694515511394,-0.009442046284675598,-0.06432446092367172,-0.08929097652435303,-0.05199465900659561,-0.03929929435253143,-0.06181908771395683,-0.29216739535331726,0.01854567974805832,0.040646716952323914,0.07613226771354675,0.11495848000049591,0.06739386171102524,0.13051921129226685,-0.04068278893828392,-0.002201022580265999,-0.030943455174565315,0.037253424525260925,-0.058939408510923386,-0.09200569242238998,-0.06125922128558159,-0.0508534200489521,-0.07764054089784622,-0.0795878916978836,-0.021311486139893532,-0.11570984870195389,-0.020296219736337662,-0.05565427616238594,-0.025586072355508804,0.012524345889687538,0.04679558426141739,0.026460275053977966,-0.006594987120479345,-0.015441959723830223,-0.0009926656493917108,-0.07880520075559616,-0.11565646529197693,-0.10942148417234421,-0.00231841835193336,0.03065745159983635,0.016909891739487648,0.03281160071492195,-0.0002066885499516502,0.014521203935146332,-0.0009934678673744202,-0.009500415064394474,0.005318612791597843,-0.004327478818595409,0.027035946026444435,-0.0118045499548316,-0.029595565050840378,-0.06648647785186768,-0.0906176045536995,-0.09536019712686539,-0.022987639531493187,0.16578364372253418,-0.05715147405862808,-0.04012991860508919,-0.12436384707689285,-0.04936036840081215,-0.016090702265501022,-0.06462869048118591,0.007728311698883772,0.023551754653453827,0.004764785524457693,0.0023942063562572002,0.06662081182003021,-0.08532979339361191,-0.055295322090387344,-0.007275500800460577,-0.058010444045066833,-0.03233575448393822,0.029073011130094528,-0.11544042080640793,0.04459136351943016,0.007343911565840244,0.05636969208717346,0.014941293746232986,-0.004851503297686577,0.03395029157400131,-0.008499113842844963,-0.10668300837278366,-0.034300949424505234,-0.06803740561008453,-0.0742773488163948,-0.10931037366390228,-0.017125487327575684,-0.04512796923518181,-0.021469242870807648,-0.01802324876189232,-0.023960033431649208,0.09640704840421677,0.0028235684148967266,-0.008472803048789501,0.008178508840501308,0.005763400811702013,0.017590491101145744,-0.11370882391929626,-0.02043485827744007,-0.04773068055510521,-0.052825845777988434,-0.00025244904099963605,-0.008831620216369629,0.015651965513825417,0.03392547369003296,0.03173569217324257,0.03287873417139053,0.03619180992245674,0.032130107283592224,-0.011306362226605415,0.007267269771546125,0.006958526559174061,0.010454564355313778,0.016934921965003014,0.028693586587905884,0.00602825777605176,0.014752041548490524,0.0009245105320587754,0.02478424273431301,0.022086435928940773,0.009907713159918785,0.06687363237142563,-0.0038486451376229525,-0.006175129674375057,-0.01975204050540924,-0.027650389820337296,0.010037977248430252,-0.020818820223212242,0.0007990593439899385,-0.0351545512676239,-0.024667734280228615,0.02004738338291645,-0.012462334707379341,0.05567775294184685,0.02229267731308937,0.012979385443031788,0.022206388413906097,0.011836318299174309,0.0204591304063797,-0.03172696754336357,0.029953857883810997,0.021859997883439064,0.02669902890920639,0.04169936850667,0.0045562065206468105,0.021584587171673775,-0.07602842152118683,-0.09600120782852173,-0.16818730533123016,-0.052323002368211746,-0.05999279022216797,0.1293080598115921,-0.01640959456562996,-0.04595176503062248,-0.11202888190746307,-0.035003386437892914,-0.07958421111106873,-0.07762105762958527,0.0301678404211998,0.020115135237574577,0.027767080813646317,0.013779059052467346,0.017346110194921494,0.010373283177614212,-0.018533237278461456,0.014206326566636562,0.01581493206322193,0.011893676593899727,-0.05068211257457733,-0.08697935938835144,0.0197775699198246,0.029216302558779716,0.004591223783791065,0.04837799817323685,-0.005554300732910633,-0.11287916451692581,-0.09125594049692154,-0.10015533119440079,-0.11096338927745819,-0.06633907556533813,0.04030323028564453,-0.25569871068000793,-0.019004149362444878,-0.0005054314387962222,-0.026937438175082207,-0.023281028494238853,-0.00044196497765369713,0.05007847771048546,0.006916156969964504,0.0030350303277373314,0.02438214235007763,0.041907597333192825,-0.006019611842930317,-0.05775178223848343,0.04578792676329613,-0.06354375928640366,-0.40788108110427856,-0.006600453518331051,0.03025164268910885,0.054115381091833115,0.048155877739191055,0.11999569088220596,0.2036009430885315,0.0010909271659329534,-0.07039333134889603,0.0007010801346041262,0.04303617402911186,-0.0684196874499321,0.001527521526440978,-0.05880119279026985,0.019845711067318916,0.360970139503479,0.4622195363044739,-0.002268172800540924,0.049339424818754196,-0.07644987106323242,0.027917737141251564,-0.10681350529193878,0.13277238607406616,-0.2341519296169281,-0.14114779233932495,-0.002139428397640586,-0.17912988364696503,-0.0442054383456707,-0.026652516797184944,0.16574572026729584,0.0723133385181427,-0.19068360328674316,-0.3517877161502838,0.005713907536119223,0.2745997905731201,0.074082151055336,0.06941728293895721,-0.24736015498638153,0.09426059573888779,0.28834694623947144,-0.05880694091320038,-0.004469916690140963,0.20784859359264374,0.021492060273885727,-0.050359055399894714,-0.14829622209072113,0.24467001855373383,0.011162861250340939,-0.536612331867218,0.0009479831205680966,0.10707296431064606,-0.3203868567943573,0.03897557407617569,0.14941252768039703,-0.006176822818815708,0.15050196647644043,-0.0841926857829094,0.001436928054317832,0.18215157091617584,0.005407342221587896,0.05680146813392639,0.12489937245845795,-0.06785723567008972,-0.14878861606121063,-0.05590175464749336,0.001017837435938418,-0.08520101010799408,-0.00877400767058134,0.015181723050773144,0.1729975789785385,-0.07366211712360382,0.08724728226661682,0.3374062180519104,-0.005689971148967743,-0.08821878582239151,0.2531837821006775,-0.11063425242900848,0.21724645793437958,0.1862861067056656,-0.05009208619594574,0.33754247426986694,0.10525709390640259,0.4296097755432129,0.33775594830513,-0.19488966464996338,-0.14823494851589203,-0.01860814169049263,-0.016833091154694557,-0.028372375294566154,-0.021025149151682854,-0.03978007659316063,0.024270012974739075,0.013410355895757675,0.003393375314772129,0.009981946088373661,0.015116669237613678,-0.0009628817206248641,0.16167432069778442,-0.019004985690116882,-0.010371167212724686,-0.027646029368042946,-0.01687701605260372,-0.015540276654064655,0.13487409055233002,0.00818693544715643,-0.004664368461817503,0.0008040961110964417,0.011237806640565395,0.007410658057779074,-0.10289369523525238,-0.03578348830342293,-0.04414636641740799,-0.04631277918815613,-0.03832201287150383,-0.0686180591583252,-0.11908920854330063,0.0010883691720664501,0.0002564047754276544,0.016151273623108864,0.006090502720326185,0.010026785545051098,0.06913840025663376,0.0025610995944589376,-0.014307288452982903,-0.0029882730450481176,-0.0008001321111805737,-0.008594125509262085,-0.08142521232366562,-0.022067183628678322,-0.020098859444260597,-0.016951214522123337,-0.011837287805974483,-0.027384409680962563,-0.019458532333374023,-0.024814343079924583,-0.002281044377014041,0.05090751498937607,0.114610455930233,0.10128908604383469,-0.33807727694511414,-0.019723698496818542,-0.01838008128106594,-0.034148916602134705,-0.03452584892511368,-0.02412194199860096,-0.005897671915590763,0.0037611944135278463,0.017345134168863297,0.013207273557782173,0.07707484066486359,0.022679325193166733,0.07186967134475708,-0.048714376986026764,-0.02634131722152233,-0.04273952171206474,0.0022766804322600365,-0.10568215698003769,0.050680361688137054,0.02047356404364109,0.02181834913790226,0.055084023624658585,0.06125600263476372,0.08884141594171524,0.20060254633426666,-0.00116652250289917,-0.006875098217278719,0.01935185119509697,0.034849897027015686,0.016133809462189674,0.0379820242524147,-0.022916704416275024,-0.01521907839924097,-0.007712727412581444,0.020070454105734825,-0.02545625902712345,-0.12106851488351822,0.0069971452467143536,0.008861472830176353,0.001038735150359571,0.022271765395998955,-0.011266089044511318,-0.07612865418195724,0.006242228206247091,0.009349490515887737,0.005628910381346941,0.019917214289307594,0.013332701288163662,-0.049985121935606,-0.018641231581568718,0.014543609693646431,0.028980642557144165,0.013337572105228901,0.022357139736413956,0.06114953011274338,-0.001501290244050324,-0.0017929082969203591,0.001005501369945705,-0.013888407498598099,0.02441292814910412,-0.00757893780246377,-0.013889549300074577,-0.004949802998453379,-0.003512877505272627,0.006219217088073492,0.009919412434101105,0.0772988572716713,-0.011649354360997677,0.003145968308672309,0.005632378160953522,-0.020197173580527306,-0.03189452737569809,0.09706363081932068,0.03283818066120148,-3.5748027585214004e-05,-0.010410639457404613,-0.008934971876442432,0.014069204218685627,0.08001704514026642,-0.017615588381886482,-0.006001038942486048,-0.012394118122756481,-0.012678657658398151,-0.02338436245918274,-0.06339394301176071,0.0036917794495821,0.003410519566386938,0.013670184649527073,-0.014025796204805374,0.027416937053203583,-0.07876793295145035,-0.001968535128980875,0.016224762424826622,-0.012243582867085934,0.008430488407611847,-0.04917353019118309,-0.11799439787864685,0.003456472884863615,0.004492688458412886,-0.005937594920396805,0.013277958147227764,-0.023399870842695236,0.041871629655361176,0.01836906373500824,0.03415081650018692,0.03708576783537865,0.02640111744403839,-0.02581215277314186,-0.05928773805499077,0.006917813327163458,0.006076488643884659,0.022463878616690636,0.01566438004374504,0.04042171686887741,-0.00459289038553834,0.005736653693020344,0.024251746013760567,0.007134370040148497,-0.005344155710190535,0.01943145878612995,0.0038345414213836193,-0.003699151799082756,-0.015537766739726067,0.0012270106235519052,0.002562633715569973,-0.021950500085949898,-0.0194077268242836,0.005813048221170902,0.006526461802423,0.018930042162537575,0.010938763618469238,-0.01050648931413889,0.0014732804847881198,0.017288854345679283,0.025347962975502014,0.030423447489738464,0.028324030339717865,0.0260428786277771,-0.005618341267108917,0.020866110920906067,0.009614896960556507,0.0300692617893219,0.0549757219851017,0.12584275007247925,-0.30236539244651794,-0.0007584209088236094,0.0009703192044980824,0.010710547678172588,-0.027266530320048332,0.006027622148394585,-0.020152343437075615,0.010367927141487598,0.0030418497044593096,0.003376569366082549,0.07200530916452408,0.025403611361980438,-0.04665382206439972,0.018444858491420746,-0.02646319940686226,-0.023916488513350487,-0.04475677013397217,-0.06333761662244797,-0.07974414527416229,-0.010024141520261765,0.012496370822191238,0.02184184081852436,0.05194254592061043,0.03250065818428993,0.03257770836353302,0.0034553646109998226,-0.016148876398801804,-0.0019218785455450416,-0.010992632247507572,0.01834697835147381,0.0026562416460365057,-0.000533913669642061,-0.02957891672849655,-0.016232838854193687,-0.03588607907295227,-0.011057966388761997,-0.012827735394239426,0.008884166367352009,0.004861308261752129,0.0033386102877557278,0.025271909311413765,-0.023394614458084106,-0.03817884996533394,0.00987447053194046,0.006286920979619026,-0.0017946426523849368,-0.004759322851896286,-0.007556240074336529,0.13376060128211975,-0.013765363022685051,0.001524074818007648,-0.007506739813834429,-0.011266205459833145,0.0106890257447958,-0.25975215435028076,0.008066905662417412,0.013336144387722015,0.021975336596369743,0.020909929648041725,0.009435840882360935,0.008699169382452965,-0.0012372609926387668,-0.0007593536865897477,0.0033387921284884214,0.004462151322513819,-0.004945357795804739,-0.0007783150067552924,0.01425726804882288,0.03466477245092392,0.020240556448698044,0.023127857595682144,0.04703188315033913,0.22742553055286407,0.017213348299264908,-0.010867971926927567,-9.121960465563461e-05,0.007925871759653091,-0.025636887177824974,0.03134413808584213,0.008487475104629993,0.006983190309256315,0.016194552183151245,0.022567003965377808,0.011933554895222187,0.03902706131339073,0.013737647794187069,0.02005794644355774,0.015847517177462578,0.020516594871878624,0.025355365127325058,0.02699325792491436,0.0023456146009266376,0.003534374525770545,0.002291397424414754,-0.0011132345534861088,-0.00021099338482599705,0.001767497742548585,-0.0017209635116159916,0.000732846325263381,-0.0020939107052981853,-0.0012969155795872211,-0.0021205549128353596,0.004622600041329861,-0.003177026053890586,-0.0029006456024944782,-0.00030229275580495596,-0.0002724419755395502,-0.00029946863651275635,-0.008223564364016056,0.003612476633861661,0.0017561843851581216,0.0023694918490946293,0.00202890089713037,0.0029844576492905617,0.004275116138160229,-0.0012416709214448929,-0.001964209834113717,-0.0014412420568987727,-0.002155579626560211,-0.0019110642606392503,-0.001435203943401575,-0.0009523597545921803,0.00019007574883289635,-0.001430241740308702,0.0011090460466220975,0.0005674409912899137,0.0033162587787956,0.003655532142147422,0.0032306017819792032,0.0037755800876766443,0.005220312159508467,0.005132325924932957,0.0033382605761289597,-0.006373151671141386,-0.0057617477141320705,-0.007524011190980673,-0.006503239739686251,-0.006254561711102724,-0.007061397656798363,-0.03484179824590683,-0.015555502846837044,0.012132275849580765,0.03928210958838463,0.053219374269247055,0.1304119974374771,0.00768635468557477,0.002594209974631667,-0.00011258899030508474,-0.019195888191461563,-0.013637380674481392,-0.2633104622364044,-0.024698911234736443,-0.0028483348432928324,-0.02149086818099022,0.02207978069782257,0.013553948141634464,0.1545083224773407,-0.010752424597740173,-0.004049758892506361,-0.023398766294121742,-0.012434441596269608,-0.022791724652051926,0.030861902981996536,-0.010173416696488857,-0.010394014418125153,0.00785239040851593,0.027430664747953415,0.039288636296987534,0.1214815005660057,0.02104133926331997,0.005636829882860184,0.00952793937176466,-0.0008214336121454835,-0.005480497609823942,-0.029805079102516174,-0.0002927859022747725,0.0008699150639586151,-0.012235058471560478,-0.007804282009601593,-0.02079441025853157,0.0021960821468383074,0.007348956074565649,-0.0021685902029275894,-0.012761130928993225,-0.001736209262162447,0.005398925859481096,-0.008801481686532497,0.12745051085948944,0.2573653757572174,0.002128337975591421,-0.014018023386597633,0.15344694256782532,0.0839482918381691,-0.012146753259003162,-0.036794811487197876,0.006460408214479685,0.1885974407196045,-0.007822506129741669,0.014901570975780487,0.08206186443567276,-0.03905417397618294,-4.485156296141213e-06,-0.02938128262758255,0.028484215959906578,0.11763985455036163,0.09797674417495728,0.07128920406103134,-0.06079784780740738,-0.16530215740203857,-0.00023092326591722667,-0.125187486410141,-0.10782282799482346,0.04357478395104408,-0.0647103488445282,0.0622636079788208,0.07598234713077545,0.041107162833213806,0.00013669108739122748,-0.09392711520195007,-0.13477972149848938,0.2819616198539734,-0.08120597898960114,0.052223529666662216,0.16693376004695892,-0.2543966472148895,0.0003776351804845035,-0.19189976155757904,0.157841295003891,0.15878017246723175,-0.02790757827460766,-0.1087944284081459,0.15197744965553284,-0.13074420392513275,-0.00017231293895747513,0.03511018678545952,0.2639314830303192,0.050312381237745285,-0.03320528566837311,-0.07784096896648407,0.13187964260578156,-0.2863723933696747,-0.00043099085451103747,-0.19271676242351532,0.11683584004640579,0.01234407164156437,-0.029550353065133095,0.09072517603635788,-0.0508899949491024,-0.09828942269086838,-0.00016288823098875582,-0.03398805484175682,-0.06071007251739502,-0.10453396290540695,0.042993754148483276,-0.024001289159059525,-0.04378911852836609,0.0034463333431631327,-9.803317516343668e-05,0.017136117443442345,0.041832949966192245,-0.1465303897857666,0.002895959187299013,-0.04892216995358467,-0.14609958231449127,-0.19523154199123383,-4.4501215597847477e-05,-0.2038000375032425,-0.020752938464283943,-0.21087107062339783,0.039748869836330414,0.05646795406937599,-0.0022784217726439238,-0.15420937538146973,-0.10519378632307053,-0.22723576426506042,-0.0276349950581789,-0.005707947071641684,-0.010060635395348072,-0.01985171064734459,-0.03435448184609413,0.019230715930461884,0.03339696675539017,0.0036327987909317017,0.0028262585401535034,0.008432355709373951,-0.03662541136145592,0.0851462334394455,-0.013741662725806236,-0.002693138550966978,-0.0005618739523924887,-0.002579422201961279,0.019178781658411026,0.09849664568901062,0.006649029906839132,0.0031891765538603067,-0.007852827198803425,-0.011993123218417168,0.029482750222086906,-0.08435375988483429,0.01599232666194439,0.0013240852858871222,0.0051012057811021805,0.0026656126137822866,0.0005593307432718575,-0.06557809561491013,-0.00030112938839010894,-0.004174623638391495,-0.00823303870856762,-0.01971404440701008,-0.036672867834568024,0.0031936524901539087,0.022208809852600098,0.007356775458902121,0.0003297670336905867,-0.013179633766412735,-0.005567733198404312,0.0528072863817215,0.01855912059545517,-0.002984552877023816,-0.001965399831533432,0.005824321415275335,0.013449864462018013,-0.009072331711649895,-0.021617598831653595,-0.011463413015007973,-0.013678234070539474,-0.002839035354554653,0.07338939607143402,-0.22112777829170227,0.01399198267608881,-0.0007335224072448909,0.00914817489683628,-8.263098607130814e-06,0.02262033335864544,-0.026024606078863144,-0.005614751949906349,-0.0061329444870352745,-0.003950922284275293,0.004804844036698341,0.011241178028285503,0.11324433982372284,0.002107416046783328,-0.0024436721578240395,-0.00039775579352863133,-0.009494149126112461,0.001479227445088327,-0.011537540704011917,0.033048711717128754,0.009687463752925396,0.01996762491762638,0.028356442227959633,0.0026988727040588856,0.008013546466827393,0.0015844205627217889,0.005752845201641321,-0.00112936703953892,0.002338132355362177,-0.013164211995899677,-0.16390608251094818,0.017448175698518753,0.00937837827950716,0.010841389186680317,0.005406048148870468,0.009193913079798222,-0.17982977628707886,0.023922890424728394,0.0007036873139441013,0.012597600929439068,0.01728670299053192,-0.02901981957256794,-0.09962426871061325,0.013841294683516026,-0.010471299290657043,0.01246753428131342,-0.002176208421587944,-0.07646478712558746,-0.06473451107740402,-0.016358163207769394,-0.006553176790475845,-0.000176996472873725,0.015659336000680923,-0.06897132843732834,-0.04295092821121216,0.0017238035798072815,0.0022406051866710186,-0.002785556251183152,-0.005818480160087347,0.003867518389597535,-0.09177690744400024,0.006540913134813309,-0.0075669437646865845,0.01552874781191349,0.007236254867166281,0.073137067258358,0.03691287338733673,-0.04507960379123688,-0.014654047787189484,-0.008937043137848377,-0.017029622569680214,-0.014252875931560993,0.06683694571256638,-0.00852535106241703,-0.018229767680168152,-0.012499737553298473,-0.017285332083702087,-0.014078473672270775,-0.052428364753723145,-0.020910505205392838,-0.014070807956159115,-0.005129932891577482,-0.01661696285009384,-0.004258215427398682,-0.008492706343531609,-0.023236963897943497,-0.011277991347014904,-0.004487165249884129,0.0012026292970404029,0.023578321561217308,-0.06294575333595276,-0.013689886778593063,-0.0019399247830733657,0.024106096476316452,0.016170715913176537,0.08193232864141464,0.04756452143192291,0.015620890073478222,0.003609523642808199,0.011595671996474266,-0.01971970871090889,0.07666169852018356,0.08960895240306854,0.009357626549899578,0.018498029559850693,0.010680959559977055,0.03076089546084404,0.011092477478086948,0.019592713564634323,-0.006145718973129988,0.004843448288738728,-0.002088532317429781,-0.0007243769359774888,-0.059513844549655914,-0.03960555046796799,0.0010091557633131742,-0.013269531540572643,-0.004621906206011772,0.0018958612345159054,-0.0037038233131170273,-0.047317735850811005,-0.008626087568700314,0.005171171389520168,4.089415233465843e-05,-0.0025219430681318045,0.020558277145028114,0.057100214064121246,0.008703548461198807,0.006266864947974682,-0.0037064284551888704,-0.006889404729008675,0.02906021475791931,0.028478039428591728,0.0068532251752913,0.012218944728374481,0.005162563640624285,0.011799001134932041,-0.025451237335801125,-0.015290148556232452,0.027924712747335434,0.00827592983841896,-0.005950778257101774,0.03295321762561798,0.045068614184856415,-0.05399885028600693,-0.027386240661144257,-0.016215885058045387,0.0016407863004133105,-0.025026878342032433,0.03815924748778343,-0.06403820216655731,0.022557329386472702,0.01061945129185915,0.0019901711493730545,0.016054805368185043,0.0076987105421721935,-0.05122416466474533,0.013346776366233826,0.0008172622183337808,0.017638111487030983,0.011312361806631088,-0.022400876507163048,0.06335050612688065,-0.015355397947132587,-0.011596799828112125,0.0013431974221020937,-0.00847726222127676,0.011643596924841404,0.05347365140914917,0.010434486903250217,0.016848109662532806,0.006732464302331209,0.007507700473070145,0.016942759975790977,-0.0400577150285244,-0.006121583748608828,-0.0012084540212526917,0.00806424394249916,0.006786024197936058,0.0116852642968297,0.02053436078131199,-0.004062514286488295,-0.0004796432622242719,0.009088347665965557,-0.022793913260102272,-0.011920366436243057,-0.02681005373597145,0.005027020815759897,0.006635239813476801,0.0028679175302386284,-0.03432772681117058,-0.042419079691171646,-0.02814493700861931,0.017306510359048843,0.02056104689836502,0.010843065567314625,0.01892578974366188,-0.0031632648315280676,-0.0051155914552509785,0.00023194600362330675,0.0016124268295243382,0.009694898501038551,-0.010393169708549976,0.027763254940509796,0.011393793858587742,0.01642509736120701,0.02046395093202591,0.007283410057425499,-0.0016978011699393392,0.02848086692392826,-0.008311523124575615,-0.002197565510869026,-0.004970910027623177,-0.007865147665143013,-0.013305183500051498,0.0062459479086101055,-0.012493157759308815,0.01742441952228546,0.019221708178520203,0.0020455289632081985,0.021074913442134857,0.014363517984747887,0.006986967287957668,-0.00014383815869223326,-0.00404507527127862,0.010867769829928875,0.007619248237460852,-0.005480930674821138,0.030710294842720032,0.018511245027184486,0.023967724293470383,0.00098259502556175,0.0059007927775382996,-0.024633079767227173,-0.011961197480559349,0.005301324650645256,0.007394830696284771,0.008066954091191292,0.024757038801908493,0.06350794434547424,0.1072184219956398,-0.0027621302288025618,-0.008039865642786026,-0.005100382957607508,-0.012556646950542927,-0.0041697523556649685,0.039408400654792786,-0.002721700817346573,0.008535629138350487,0.0014668325893580914,-0.006210633087903261,0.006366613321006298,0.08422387391328812,0.02687140181660652,0.013657591305673122,0.01796390488743782,0.019732333719730377,0.005334788933396339,0.08622299134731293,0.03341282159090042,0.02382759004831314,0.028430277481675148,0.04487500339746475,0.041289620101451874,-0.037620559334754944,0.03213480859994888,0.015072232112288475,0.025158647447824478,0.04436074569821358,0.023399876430630684,0.20986023545265198,0.04689149558544159,0.01951630786061287,0.03213588520884514,0.05530892312526703,0.024238221347332,-0.06355420500040054,0.020657632499933243,0.01928671821951866,0.016384610906243324,0.00811236072331667,0.00033051855280064046,0.17039310932159424,0.014065878465771675,0.00033805359271354973,0.01550716906785965,0.008245271630585194,0.009564082138240337,0.009281759150326252,-0.01778441295027733,-0.007434742990881205,-0.014447739347815514,-0.006424386985599995,-0.009391921572387218,-0.007784420158714056,-0.004802433308213949,0.003026670543476939,0.0005000265664421022,-0.005539176054298878,-0.002488994738087058,-0.15989136695861816,0.0003222262021154165,0.008221893571317196,-0.00031988549744710326,0.0024983505718410015,0.007786872796714306,0.0500975027680397,0.018239924684166908,0.010919462889432907,0.020467014983296394,0.027757197618484497,0.016386443749070168,0.09925473481416702,0.014471546746790409,0.010550771839916706,0.013103239238262177,0.02087954245507717,0.01321299560368061,-0.03482825309038162,0.005207110662013292,0.013856885954737663,0.010461138561367989,0.012154066003859043,0.007617691066116095,-0.039720017462968826,0.006988046690821648,0.012767890468239784,0.005962492898106575,0.01521282084286213,0.008092476055026054,0.05423661321401596,0.032705213874578476,0.12163292616605759,-0.01764015480875969,0.0047594355419278145,-0.07893404364585876,-0.012751010246574879,-0.26200518012046814,-0.12093070149421692,-0.007864435203373432,0.08000782132148743,0.01190511416643858,0.00574525585398078,-0.00629684841260314,0.00019134400645270944,-0.003958191256970167,-0.008939872495830059,0.08531860262155533,0.11952407658100128,-0.05283540487289429,-0.010301629081368446,-0.06387005001306534,0.008961035870015621,0.02074357122182846,-0.02760336920619011,-0.05430535227060318,0.17234568297863007,0.09940506517887115,-0.09800131618976593,0.029386812821030617,0.04340991750359535,-0.09489671140909195,0.08528367429971695,-0.07594242691993713,-0.03585381805896759,0.04800359904766083,0.01160782016813755,0.06790266185998917,-0.06021445244550705,-0.06595179438591003,0.07337028533220291,0.034561045467853546,0.2761957347393036,-0.16304902732372284,0.13854612410068512,0.01715177670121193,-0.04998588562011719,-0.08162588626146317,0.0158248171210289,0.046994950622320175,0.15381644666194916,-0.024124959483742714,0.02382778562605381,-0.036266013979911804,0.017743704840540886,-0.19139929115772247,-0.10393698513507843,0.024562818929553032,0.059274934232234955,0.10298912972211838,-0.02949344925582409,-0.0063715060241520405,0.05825284868478775,-0.06764225661754608,0.02300546132028103,-0.08011709153652191,0.11529957503080368,0.07107049226760864,-0.0862356647849083,-0.017914658412337303,0.01670934446156025,0.00230672606267035,-0.03823602944612503,0.1970224529504776,-0.1037282645702362,0.04319211468100548,0.025574523955583572,-0.05647466331720352,0.028341282159090042,0.21100516617298126,-0.06501714885234833,-0.031102726235985756,-0.1470935344696045,0.04607786983251572,-0.18597406148910522,0.3044760823249817,-0.11714522540569305,-0.052136924117803574,0.18703682720661163,0.016565876081585884,-0.016370849683880806,0.047223493456840515,0.12453562021255493,-0.0947396531701088,0.06804384291172028,-0.026404930278658867,-0.0421702116727829,-0.012893985025584698,0.05401160567998886,0.23266488313674927,-0.0659346729516983,0.004093197174370289,0.0607774555683136,-0.03179023787379265,0.058059822767972946,0.14884668588638306,-0.15836618840694427,0.03679199144244194,0.002086648717522621,0.08182375133037567,0.02500399574637413,0.17675788700580597,0.06376516073942184,-0.0517842099070549,-0.07338108867406845,-0.04606150835752487,-0.1660141497850418,-0.027166396379470825,-0.09924393147230148,-0.0020055249333381653,-0.06195729225873947,-0.004115215037018061,0.1159985139966011,0.3394463360309601,-0.056668322533369064,-0.029000181704759598,0.11150865256786346,0.04449405148625374,-0.09391636401414871,-0.20572680234909058,0.07828065752983093,-0.07854954153299332,-0.004012946039438248,-0.012795798480510712,0.046957094222307205,-0.08687762171030045,0.059326350688934326,-0.02878211811184883,-0.052242446690797806,-0.07843931764364243,-0.02016940526664257,0.04298168793320656,0.13471485674381256,0.01107969880104065,0.08371403068304062,0.12316697835922241,0.07888603210449219,-0.09692102670669556,-0.1717015951871872,-0.026660442352294922,0.03747722506523132,0.12835510075092316,0.051231883466243744,-0.03934201970696449,-0.15567812323570251,-0.018204014748334885,0.02116173505783081,0.05712083727121353,0.03894875943660736,0.015916569158434868,-0.06759120523929596,0.06058935448527336,0.05245833843946457,0.07026316970586777,0.11732839792966843,-0.03233882412314415,-0.036665480583906174,-0.10972896963357925,-0.024228397756814957,0.04709567874670029,0.09830443561077118,-0.013891682960093021,-0.06637153774499893,0.052362337708473206,0.06670809537172318,-0.09765931963920593,0.017117908224463463,-0.011241002008318901,0.0481097474694252,0.063627690076828,-0.01350055169314146,-0.08429491519927979,-0.13852541148662567,0.06397639214992523,0.0738721415400505,0.022301435470581055,0.006365153472870588,-0.005867138970643282,-0.17337970435619354,-0.1615705043077469,-0.18812350928783417,-0.0981840267777443,-0.11870665103197098,-0.036018259823322296,0.031326960772275925,0.02329772524535656,0.03969481587409973,0.037849102169275284,0.03642219677567482,0.07036338746547699,0.16825424134731293,0.14929091930389404,0.3016076683998108,0.042377982288599014,0.06161079555749893,0.01844056323170662,0.07223266363143921,-0.030703391879796982,0.04164333641529083,-0.11910045146942139,-0.10339245200157166,0.04063047841191292,0.16206562519073486,0.07242107391357422,-0.02926163747906685,-0.006946404930204153,-0.009953820146620274,-0.17867411673069,-0.029377298429608345,-0.15746167302131653,0.026880215853452682,-0.14799335598945618,-0.15003462135791779,0.004466906189918518,-0.12682577967643738,-0.05042384937405586,-0.1518028974533081,0.009518844075500965,-0.06042925640940666,0.020267345011234283,-0.05581238865852356,-0.02674066461622715,0.010896708816289902,0.02604801207780838,-0.015221809968352318,-0.021442007273435593,-0.03740431368350983,-0.11084140092134476,-0.07580946385860443,0.07893288880586624,0.048250969499349594,0.041717950254678726,0.09244038164615631,0.04611574858427048,0.05535323545336723,0.037123240530490875,0.10142437368631363,0.05420973524451256,0.07640345394611359,0.08660498261451721,0.07099631428718567,-0.007184851914644241,0.003605975303798914,-0.027540817856788635,0.010596146807074547,-0.007782220374792814,-0.013129434548318386,-0.04114324599504471,-0.06215786561369896,-0.08883733302354813,-0.09034579992294312,-0.07970480620861053,-0.15944577753543854,-0.00732512166723609,-0.009369108825922012,-0.07072345912456512,0.0060222032479941845,-0.03157682716846466,-0.031926821917295456,0.020624440163373947,-0.03285244479775429,-0.02115623652935028,0.017478682100772858,-0.04449387267231941,-0.04200407862663269,0.03750857710838318,-0.006977111101150513,-0.03323286026716232,-0.005893507972359657,0.019489530473947525,0.11742949485778809,-0.0035874254535883665,-0.015641245990991592,-0.024614037945866585,0.01127716526389122,0.01883167214691639,-0.035671744495630264,-0.010295109823346138,-0.05454229563474655,-0.08182564377784729,-0.05676639452576637,-0.022146839648485184,-0.0100363465026021,-0.02173537388443947,-0.05074220895767212,-0.01922975294291973,-0.09396667778491974,-0.0883619636297226,-0.1646628975868225,0.0725354477763176,0.011612357571721077,0.1323462724685669,0.043716948479413986,0.1459580510854721,0.12850263714790344,0.10523061454296112,0.010691460222005844,0.06895067542791367,0.08994009345769882,0.09210821241140366,-0.00013807268987875432,-0.036310359835624695,-0.09185317903757095,0.005754078738391399,-0.021339280530810356,-0.040516093373298645,-0.026512710377573967,0.019850745797157288,-0.0030932961963117123,0.05632689222693443,-0.02593015506863594,-0.0598544105887413,0.03366759791970253,0.015242421068251133,0.08184660971164703,0.013073578476905823,0.0038578121457248926,0.018105272203683853,-0.0044570076279342175,0.08459098637104034,0.07197441905736923,0.012117483653128147,-0.05428324639797211,-0.08251345157623291,-0.13434268534183502,-0.0904269590973854,-0.04948589205741882,-0.12732578814029694,-0.082152359187603,0.11857886612415314,0.11292039602994919,-0.03326556459069252,-0.018894916400313377,-0.052222225815057755,-0.1304585188627243,0.00012197605974506587,0.0629713237285614,0.04575014114379883,-0.0023179040290415287,0.002905580447986722,-0.09577832370996475,-0.08858300000429153,-0.06975395232439041,0.029463892802596092,0.06450328230857849,0.014465367421507835,-0.06924916058778763,0.17193615436553955,0.09862048178911209,0.006619633641093969,-0.030915211886167526,0.013372521847486496,-0.1953902691602707,-0.11100370436906815,-0.03246927633881569,-0.014774863608181477,-0.06742914766073227,-0.0767289400100708,0.15803872048854828,-0.012111824005842209,-0.06530696898698807,-0.13844017684459686,-0.10944163799285889,-0.14259426295757294,-0.028227059170603752,-0.24349357187747955,-0.07277221232652664,-0.014545953832566738,0.046439480036497116,0.04509919881820679,-0.02607554756104946,0.24718299508094788,0.14343753457069397,-0.01293589361011982,0.04425528272986412,-0.0018187202513217926,0.06177736818790436,-0.03413061425089836,0.05746258422732353,0.010386677458882332,-0.11005841940641403,0.05176281929016113,-0.04230957478284836,-0.3091678321361542,-0.091904416680336,0.006105856969952583,0.060408417135477066,-0.007152164820581675,0.06662539392709732,-0.08796500414609909,-0.1891527771949768,0.04786508157849312,0.0683855190873146,-0.004524781368672848,-0.07841050624847412,-0.20234599709510803,-0.11760199815034866,0.03630883991718292,0.04065876081585884,0.0824335366487503,0.1584082692861557,-0.060351885855197906,0.06298317015171051,0.17741453647613525,0.09526931494474411,-0.017522450536489487,-0.03298521786928177,0.2654248774051666,0.22483815252780914,-0.023118149489164352,-0.04235503822565079,-0.007747250609099865,0.036402907222509384,0.28160908818244934,0.1535172462463379,0.019083794206380844,0.06665435433387756,0.040343277156353,0.04973504692316055,0.0385211817920208,0.03574690967798233,0.037077274173498154,0.010428912937641144,-0.028999855741858482,0.02608902007341385,0.001533069764263928,0.07708725333213806,0.03375444561243057,-0.06366394460201263,-0.057942185550928116,0.0008511418709531426,-0.043970681726932526,0.048385486006736755,0.025659779086709023,-0.00018465217726770788,-0.08330879360437393,-0.036241017282009125,0.0019551648292690516,0.07591289281845093,-0.024166764691472054,-0.04572447016835213,-0.09159506857395172,-0.03161883354187012,0.029452495276927948,0.14519502222537994,0.10525941848754883,0.008061286993324757,-0.04789050295948982,-0.01814308762550354,-0.06210850924253464,-0.004547127988189459,-0.07699038088321686,0.016185840591788292,0.060125380754470825,0.015525201335549355,0.04860684275627136,-0.033716391772031784,-0.044586095958948135,0.036564022302627563,0.04881097376346588,0.05261571332812309,0.05600806698203087,0.03198758512735367,-0.10306987166404724,-0.0162307508289814,0.10476817190647125,0.13501223921775818,-0.061936259269714355,0.11620824784040451,-0.5822197198867798,-0.012114688754081726,-0.020808767527341843,-0.03102140501141548,0.044857244938611984,0.005674312822520733,-0.002016895217821002,0.10646520555019379,0.021710867062211037,-0.004384246189147234,0.03104994259774685,0.021824121475219727,0.08433349430561066,0.17708167433738708,-0.19110475480556488,0.12938910722732544,-0.2724190354347229,0.02035687118768692,-0.26837706565856934,0.1304720640182495,0.27453380823135376,-0.004979434423148632,-0.009574315510690212,0.186019629240036,0.053065959364175797,-0.15477898716926575,-0.191025048494339,-0.29615193605422974,-0.1802576631307602,-0.15706481039524078,0.2297021448612213,0.203477144241333,0.5755424499511719,-0.07075967639684677,-0.133414626121521,-0.21891163289546967,0.08952058106660843,-0.04804720729589462,0.06322683393955231,0.15871672332286835,0.29333850741386414,0.1367144137620926,-0.1670621633529663,-0.08795080333948135,0.2297237515449524,0.1580786108970642,-0.14952827990055084,0.05003310367465019,0.26088517904281616,-0.015945689752697945,-0.22346298396587372,-0.07588586956262589,0.07761869579553604,-0.06617185473442078,-0.03236749395728111,0.22020642459392548,0.14045746624469757,0.22092020511627197,0.3384370505809784,-0.19654974341392517,0.2595534026622772,0.019771600142121315,0.11029312759637833,-0.23279979825019836,-0.3136809468269348,0.03194567561149597,0.4958762526512146,0.04139024391770363,-0.22647294402122498,0.1408761441707611,-0.1256798952817917,0.12624698877334595,-0.3934728801250458,-0.11788436770439148,0.13648708164691925,-0.31955286860466003,-0.23539192974567413,-0.23436221480369568,0.14403371512889862,0.21727396547794342,0.2570420205593109,0.23384489119052887,0.01635698787868023,-0.0025974183809012175,0.03167610988020897,0.035571809858083725,0.02698635868728161,-0.0056953951716423035,0.010679716244339943,0.07290482521057129,0.023324143141508102,0.037086449563503265,0.04194570705294609,-0.06547054648399353,0.016433684155344963,-0.023703021928668022,-0.0001395229046465829,-0.030705930665135384,-0.009235741570591927,0.039637841284275055,-0.01279559638351202,0.04145308583974838,-0.053978461772203445,0.04369623586535454,0.02713591791689396,0.06648332625627518,0.022315429523587227,-0.028878377750515938,-0.011980853043496609,0.03890518471598625,-0.031040038913488388,0.06602945923805237,0.0024006711319088936,0.1071651503443718,-0.019040144979953766,0.046466901898384094,-0.03701910749077797,-0.04589824378490448,-0.03388935700058937,0.04514236003160477,0.045478031039237976,0.09381946921348572,0.019335299730300903,-0.023519335314631462,0.1212797462940216,0.08889606595039368,0.13818371295928955,-0.06331761926412582,0.015858357772231102,0.016057316213846207,0.04328090324997902,-0.03717772290110588,-0.009613716043531895,0.10983377695083618,0.057700105011463165,0.03460211306810379,0.017774995416402817,0.00313973193988204,0.013015893287956715,0.09455522894859314,-0.024295281618833542,-0.05358615517616272,-0.012402256019413471,-0.006650728639215231,-0.057935941964387894,-0.11412815004587173,0.046035077422857285,-0.04144647344946861,0.016347607597708702,0.009067660197615623,0.030911998823285103,-0.016301026567816734,0.000734531378839165,0.0632239356637001,0.017984867095947266,0.018126966431736946,0.05695090442895889,0.010911332443356514,-0.053048647940158844,0.03679541498422623,-0.01185695081949234,0.06000106781721115,0.07434463500976562,-0.0035795317962765694,0.021431615576148033,-0.008251368999481201,-0.04610137268900871,0.06485873460769653,0.113835908472538,0.07949687540531158,0.05262986943125725,0.03544631227850914,0.1330934762954712,0.06425518542528152,-0.053677935153245926,-0.022778458893299103,0.026299111545085907,-0.07316708564758301,0.06138893961906433,0.015599582344293594,-0.012886229902505875,0.033341601490974426,0.09958359599113464,-0.06636148691177368,-0.011303802952170372,-0.009378333576023579,-0.034305207431316376,-0.06330399215221405,-0.11175953596830368,-0.0097859101369977,0.06658871471881866,0.019131958484649658,0.08998008817434311,-0.04520273953676224,-0.03284149244427681,0.0410529300570488,-0.08134046941995621,-0.05556904524564743,-0.018540486693382263,-0.10849515348672867,-0.03987133502960205,0.02516959421336651,0.006210220046341419,-0.012053669430315495,0.12970946729183197,0.0014609681675210595,0.11781346052885056,0.2721075415611267,0.018888210877776146,-0.056401848793029785,-0.010075903497636318,-0.08272384107112885,-0.15599662065505981,0.016165267676115036,0.024800797924399376,0.030935173854231834,0.0024840557016432285,0.05644623190164566,0.003188618691638112,-0.1222434788942337,-0.03973982110619545,-0.035049255937337875,-0.023314623162150383,-0.049450378865003586,0.07702665030956268,-0.08855025470256805,-0.00879074539989233,0.01138379331678152,0.020588111132383347,-0.015993665903806686,-0.05253526195883751,-0.16426552832126617,0.014358839020133018,-0.0636199340224266,-0.12135197967290878,-0.0875491201877594,-0.10817506909370422,0.1466933637857437,-0.01959720067679882,-0.03574463352560997,0.07906337082386017,0.07306846976280212,-0.017434073612093925,0.07614043354988098,0.0008692551054991782,0.07061222195625305,0.014370927587151527,-0.06636673212051392,0.04090317711234093,-0.08275328576564789,-0.025606853887438774,-0.059604573994874954,0.011451167985796928,0.0048378524370491505,0.12835551798343658,0.14099612832069397,0.06892091035842896,-0.04102691635489464,0.0033372296020388603,-0.0497116856276989,-0.07755158096551895,0.1275917887687683,0.010640362277626991,-0.029014453291893005,-0.09619209170341492,-0.12056326866149902,-0.04659326747059822,-0.09252747893333435,-0.035723976790905,-0.07857254892587662,-0.023625170812010765,0.005505668465048075,-0.04308690130710602,-0.13846243917942047,-0.035469409078359604,-0.015006974339485168,-0.010398943908512592,-0.04876837879419327,-0.06607270985841751,-0.03579200431704521,-0.030060624703764915,0.0038663793820887804,0.00999426655471325,-0.039492640644311905,0.05378318578004837,-0.05247202143073082,0.008463628590106964,-0.011255880817770958,-0.012120524421334267,-0.01663309521973133,0.016921356320381165,-0.12124049663543701,-0.018573015928268433,-0.03416843339800835,-0.01500276941806078,-0.04917960986495018,0.09563113749027252,-0.1975991129875183,-0.031017262488603592,-0.0039003374986350536,-0.02432643435895443,-0.09158772230148315,0.12502172589302063,-0.1694665253162384,0.006126489490270615,-0.013243480585515499,0.012089317664504051,-0.006745121907442808,0.06617631763219833,-0.12464923411607742,-0.031989935785532,-0.004516884684562683,-0.06498641520738602,-0.056024741381406784,-0.05456527695059776,0.00511553417891264,-0.06994733214378357,-0.01771339774131775,-0.04479273408651352,0.022783732041716576,-0.11856793612241745,0.031801458448171616,0.017861975356936455,-0.012113025411963463,0.011184824630618095,0.025595344603061676,0.025904366746544838,-0.0396004356443882,-0.044502876698970795,-0.030356815084815025,-0.09122686088085175,0.02174633927643299,-0.011316980235278606,0.06193608045578003,-0.02418464794754982,0.0007789732189849019,-0.09195757657289505,0.10834653675556183,-0.06841973215341568,0.05340573564171791,-0.005233679432421923,0.019758768379688263,-0.03487256541848183,0.062042057514190674,0.01847301609814167,-0.018286652863025665,-0.011429582722485065,0.036111630499362946,-0.04246358200907707,0.006089791655540466,0.022329283878207207,0.03966307267546654,0.010424396023154259,0.047281209379434586,0.023510685190558434,0.034825317561626434,0.014461481012403965,-0.03957975655794144,0.03025084175169468,-0.09581615030765533,-0.05994673818349838,-0.12577813863754272,-0.027237599715590477,0.026675118133425713,-0.04914272576570511,-0.051162656396627426,-0.014525331556797028,-0.023626048117876053,0.031352605670690536,0.004425545688718557,-0.002262543886899948,0.03383719548583031,0.025236977264285088,-0.025934772565960884,0.004928139504045248,-0.0011571210343390703,0.019217394292354584,0.015725214034318924,-0.0016965538961812854,-0.024875862523913383,-0.062007687985897064,0.11756701022386551,-0.05405079573392868,0.031403400003910065,0.058699216693639755,0.04915747791528702,-0.039361126720905304,0.07513219118118286,0.02644360437989235,0.029633304104208946,0.024020414799451828,0.026841923594474792,-0.07030186057090759,0.011636711657047272,-0.04209152236580849,-0.06112553924322128,-0.0568150170147419,-0.06194240227341652,-0.06317328661680222,-0.04120243340730667,0.03876779228448868,0.04551095888018608,0.032194141298532486,-0.055402688682079315,-0.1358901560306549,0.10542215406894684,0.05595214292407036,-0.049286022782325745,-0.04144373908638954,-0.08364438265562057,0.0722450315952301,-0.0584244579076767,0.010108571499586105,0.02113063633441925,0.00094483548309654,0.09089481085538864,0.05331201106309891,-0.030287712812423706,-0.038798049092292786,-0.021869849413633347,-0.04220254719257355,-0.11586184799671173,-0.14618639647960663,-0.022671878337860107,0.010849189944565296,-0.017948457971215248,-0.03753725066781044,-0.03415240719914436,0.026000693440437317,0.009784734807908535,-0.0035707640927284956,0.02371242083609104,0.04935620725154877,0.05407010018825531,0.15872475504875183,0.030926205217838287,-0.006268047261983156,0.02726241759955883,0.01793217472732067,-0.018901173025369644,0.05884953588247299,-0.05521726235747337,-0.06073581427335739,-0.0075376261956989765,-0.05867849290370941,-0.12714427709579468,0.053261179476976395,-0.13584919273853302,0.02560943178832531,-0.014833295717835426,0.00691424822434783,-0.04790825396776199,-0.033081602305173874,-0.02081124484539032,-0.027574719861149788,-0.08801498264074326,-0.05679338797926903,-0.09174614399671555,-0.03314061090350151,-0.020046455785632133,0.0030851815827190876,-0.03256654739379883,0.03608047962188721,-0.003428220748901367,-0.09160961955785751,-0.1723024696111679,0.04429299011826515,-0.06381706148386002,0.012959246523678303,-0.011570130474865437,-0.027875671163201332,0.012400572188198566,-0.06903690099716187,0.004244004376232624,-0.013867867179214954,-0.009530328214168549,-0.0035843774676322937,0.07272769510746002,0.02013309672474861,0.026244251057505608,0.006924567744135857,0.01156124472618103,0.024541033431887627,0.01025326270610094,-0.06532396376132965,0.15887613594532013,0.10497421771287918,-0.026010919362306595,0.05833110958337784,0.1327156126499176,-0.0678795725107193,-0.073076993227005,-0.062164999544620514,0.2319113165140152,0.19395065307617188,0.21126878261566162,-0.00866692140698433,0.20268741250038147,-0.13317787647247314,-0.16517142951488495,0.025119798257946968,0.1850389540195465,-0.12383300065994263,0.010342257097363472,0.2042207419872284,0.21019452810287476,-0.10240092128515244,-0.10827936232089996,0.2754465937614441,0.2796039283275604,0.007641127333045006,-0.08989249169826508,0.13537056744098663,-0.017713090404868126,-0.09452234208583832,-0.11262716352939606,-0.058249715715646744,0.029217587783932686,0.14966443181037903,0.07140937447547913,0.06682658940553665,0.17270587384700775,-0.14333418011665344,-0.1917029470205307,0.1115349605679512,0.04363594204187393,0.16216501593589783,0.11639150232076645,-0.008430268615484238,-0.07501814514398575,0.1530643105506897,0.08368663489818573,0.27556392550468445,-0.07838069647550583,-0.2362992912530899,-0.09262967854738235,-0.23996108770370483,-0.18216632306575775,0.03407234326004982,0.15881414711475372,0.15090928971767426,0.1103942021727562,0.00850534439086914,-0.01727660559117794,-0.105401910841465,-0.025064535439014435,0.207062229514122,-0.25282949209213257,0.017604202032089233,0.05408588796854019,0.28620976209640503,-0.08336206525564194,-0.1413872092962265,0.00883965753018856,0.009708653204143047,0.10499986261129379,-0.05389411374926567,-0.02436693012714386,-0.18361903727054596,0.16830173134803772,-0.015814803540706635,-0.05392041429877281,-0.09487640857696533,-0.15580537915229797,-0.12919233739376068,0.2644427716732025,0.04496938735246658,0.016110273078083992,0.08714481443166733,0.04554268717765808,0.00984109379351139,0.2679480016231537,0.013255326077342033,0.04785946384072304,0.09719541668891907,0.16133105754852295,0.05017276108264923,0.04328865557909012,0.013973535969853401,0.027295876294374466,-0.03438001498579979,0.0023513678461313248,0.03756497800350189,0.4216269254684448,-0.03793877735733986,-0.05304295942187309,-0.06641528010368347,-0.006599195767194033,-0.08024319261312485,0.2993588447570801,0.02355785109102726,-0.015296454541385174,-0.03841632232069969,-0.014091984368860722,-0.021625645458698273,-0.1494770497083664,0.0028195292688906193,-0.015256653539836407,-0.02588263340294361,0.11173483729362488,0.010023842565715313,-0.14871585369110107,0.011446432210505009,-0.03281797841191292,-0.02064361236989498,-0.054927945137023926,-0.002191904466599226,-0.0036473588552325964,-0.010269270278513432,-0.011980528943240643,0.07381930202245712,0.04962092265486717,0.00705846818163991,-0.2137126326560974,0.020224960520863533,0.00903127621859312,0.014782779850065708,0.040872812271118164,0.029420031234622,-0.15649479627609253,-7.037684554234147e-05,-0.003931264393031597,-0.007746538147330284,-0.05158991739153862,-0.03845414146780968,-0.30996719002723694,-0.013664505444467068,-0.014716897159814835,0.03484766557812691,0.0456589013338089,-0.06132819503545761,-0.30154454708099365,-0.0033045518212020397,0.024005556479096413,-0.019125862047076225,0.07114080339670181,0.04721949249505997,-0.1366240680217743,-0.010713504627346992,0.019299117848277092,0.05899392068386078,0.0012391390046104789,-0.021581746637821198,0.11690468341112137,-0.011266382411122322,-0.0009851237991824746,0.06338829547166824,0.11475168913602829,0.03777994588017464,0.2583865225315094,-0.025615563616156578,-0.03145597502589226,-0.005006607621908188,0.02602469176054001,0.07071004062891006,-0.06972284615039825,-0.014027547091245651,0.0011021561222150922,0.01660172827541828,0.005495518445968628,0.07957131415605545,-0.03157396242022514,0.0023172965738922358,0.05583943426609039,0.0679989829659462,0.007397620007395744,0.04209393635392189,0.004593467339873314,0.04980049654841423,-0.04818281531333923,-0.06463588774204254,-0.003585473168641329,-0.055586304515600204,0.038724664598703384,0.004484930541366339,0.02912459522485733,0.024960311129689217,-0.012578100897371769,-0.009255505166947842,-0.0938715711236,-0.021205367520451546,-0.07112132757902145,-0.04620100557804108,-0.004160509444773197,-0.04203586280345917,0.05971437320113182,0.006160689517855644,-0.051190003752708435,0.018505403771996498,-0.03528041020035744,-0.07025990635156631,-0.09134504199028015,0.01320500299334526,0.024817388504743576,-0.023892071098089218,0.01345887128263712,0.002783101284876466,-0.02546784095466137,0.011387109756469727,-0.02840087190270424,-0.04923955351114273,-0.0372733511030674,-0.05800221860408783,0.0042753373272717,0.017283441498875618,0.008226130157709122,-0.0060755605809390545,-0.0006702296086587012,-0.009981612674891949,0.003262670710682869,0.03846284747123718,0.055883828550577164,-0.017179008573293686,-0.03110869601368904,-0.029582621529698372,-0.13265256583690643,-0.01574028842151165,0.007401286158710718,-0.012360083870589733,-0.0039245509542524815,0.008357971906661987,-0.10303306579589844,-0.02135605178773403,0.012395436875522137,-0.016031859442591667,-0.0018409674521535635,0.015952518209815025,-0.042818937450647354,0.023104950785636902,-0.036844104528427124,0.0029677567072212696,0.0012632940197363496,0.018349619582295418,0.008924543857574463,0.015763064846396446,0.04411347210407257,-0.0009238386410288513,-0.013067418709397316,-0.004641887731850147,0.005456381943076849,0.0036510752979665995,0.0026010230649262667,-0.030768902972340584,-0.006126063875854015,-0.046288736164569855,-0.010136779397726059,-0.004594739060848951,-0.019276339560747147,-0.04189266636967659,-0.02827327512204647,-0.036530714482069016,-0.008375723846256733,0.02307966537773609,-0.003910561557859182,-0.0026134802028536797,-0.00898828450590372,-0.021255122497677803,0.02764795534312725,0.0027016224339604378,0.014587233774363995,-0.025225237011909485,-0.054526425898075104,-0.020860645920038223,-0.007756691426038742,-0.018074968829751015,0.061471953988075256,0.037185657769441605,0.016017349436879158,0.02840600162744522,-0.04071745276451111,-0.020445290952920914,-0.011707429774105549,-0.014783419668674469,-0.03149424120783806,-0.051995791494846344,-0.03813716769218445,0.010104380548000336,-0.027328243479132652,-0.046011947095394135,-0.025085223838686943,0.02255261316895485,0.10300146043300629,0.011540054343640804,0.023092832416296005,0.04085602983832359,0.020388484001159668,-0.00390661833807826,-0.014421625062823296,-0.024694442749023438,0.017233919352293015,-0.012727584689855576,-0.02867792546749115,0.020049097016453743,0.05086226016283035,0.028582945466041565,0.045511238276958466,0.03082006610929966,0.017973702400922775,-0.07117453962564468,-0.06575281918048859,-0.0235992930829525,-0.06715473532676697,-0.07912005484104156,-0.04344988986849785,-0.10621320456266403,-0.031666334718465805,0.035982005298137665,-0.025668105110526085,-0.03272957727313042,-0.038759563118219376,-0.03173001855611801,-0.065443255007267,0.0007957794587127864,0.05362705886363983,0.08321051299571991,0.07215892523527145,0.12105834484100342,-0.008922199718654156,0.015066824853420258,-0.0008977591642178595,-0.016456272453069687,0.003211460541933775,-0.05661286413669586,-0.027298958972096443,-0.027855725958943367,-0.021997813135385513,-0.018729209899902344,-0.05721083655953407,-0.050199851393699646,-0.14120925962924957,0.001386547228321433,-0.016820108518004417,-0.05249881371855736,-0.03449896723031998,-0.07501670718193054,0.02681763842701912,-0.003522244282066822,-0.0211764108389616,-0.016892075538635254,0.031168881803750992,-0.01976669579744339,-0.04605674371123314,0.030342306941747665,0.0003911071107722819,-0.057895857840776443,0.010657798498868942,-0.012152360752224922,0.039510492235422134,0.018202930688858032,-0.007971891202032566,-0.027069540694355965,0.0368281714618206,0.0033808911684900522,-0.07257061451673508,0.04130469262599945,0.0035338029265403748,0.022290432825684547,-0.005506410729140043,0.017741847783327103,-0.10923159867525101,-0.01558088418096304,0.02536584809422493,0.02782558649778366,0.08936235308647156,0.09243134409189224,-0.16822735965251923,-0.04141257703304291,-0.030593283474445343,-0.0465930812060833,0.018819134682416916,0.024823570623993874,-0.14848920702934265,0.012064945884048939,0.026626646518707275,0.031227275729179382,0.0921349748969078,0.013600917533040047,-0.02619628608226776,0.026663968339562416,-0.007707123178988695,0.010027733631432056,-0.05085579305887222,-0.00025607561110518873,-0.0018018961418420076,0.016152935102581978,-0.0036466429010033607,-0.014069365337491035,0.011924530379474163,-0.023275135084986687,0.0526529885828495,0.02800046093761921,0.008248819969594479,0.07191342860460281,0.03255216032266617,0.10475650429725647,0.007159733679145575,0.003572821384295821,0.05127925053238869,0.09106400609016418,0.031080039218068123,0.033774398267269135,-0.031231163069605827,0.023863332346081734,0.07585986703634262,0.053149979561567307,0.05995570868253708,0.06757515668869019,0.02001543529331684,-0.002410219283774495,-0.03827187418937683,-0.09167781472206116,-0.09949976950883865,-0.05523030087351799,-0.05147944763302803,-0.020076341927051544,0.004441834520548582,0.00974587257951498,-0.01892356015741825,-0.01731269061565399,0.0020752723794430494,0.015825891867280006,-0.04178515449166298,-0.02300211228430271,0.024524301290512085,-0.00020285202481318265,0.010716147720813751,0.023692570626735687,-0.023461151868104935,0.022895732894539833,0.040310267359018326,0.012028972618281841,0.05486277490854263,0.010050774551928043,0.024718452244997025,0.05456969141960144,-0.033355824649333954,-0.002495300956070423,-0.0551062636077404,0.32384034991264343,0.05740165337920189,0.2288072556257248,-0.018799839541316032,-0.06008131429553032,0.09304626286029816,-0.034943610429763794,0.0021431229542940855,0.05812867730855942,-0.011449260637164116,-0.008923539891839027,-0.0003702272952068597,0.0017744494834914804,0.020470216870307922,0.002747128950431943,-0.0019427669467404485,-0.0447828508913517,0.10887808352708817,-0.01037090178579092,0.01047907117754221,-0.028654618188738823,-0.22154347598552704,-0.0589231513440609,0.034075573086738586,-0.06296106427907944,-0.16690383851528168,-0.10937616229057312,-0.059911079704761505,0.026183458045125008,0.29675260186195374,0.13925997912883759,-0.18089263141155243,0.28849953413009644,-0.4951404631137848,-0.07139864563941956,-0.1333490014076233,0.03282565250992775,0.18493101000785828,-0.17741818726062775,-0.06619701534509659,0.3115416169166565,0.1357364058494568,-0.02973918989300728,-0.03221359848976135,-0.046417370438575745,-0.19089193642139435,-0.02632455714046955,0.07870367169380188,-0.006930837873369455,-0.02814146876335144,0.09166011214256287,-0.02625725418329239,-0.03311838209629059,0.1577124446630478,0.06606908142566681,-0.07975238561630249,-0.09934893250465393,0.23962658643722534,-0.018103670328855515,0.06380940973758698,-0.0519668310880661,0.15514658391475677,0.19743973016738892,-0.006666420493274927,-0.1443309187889099,-0.056091759353876114,0.016904862597584724,-0.022156786173582077,-0.020759714767336845,0.012911469675600529,0.02168758027255535,0.15169799327850342,-0.2438567727804184,0.27096524834632874,0.11692735552787781,0.01668950542807579,-0.0025650160387158394,-0.04375584051012993,-0.10728627443313599,0.08916842192411423,0.1760166436433792,-0.46055474877357483,-0.44149264693260193,-0.4355415999889374,0.45008814334869385,0.12597042322158813,0.29080426692962646,0.12967431545257568,-0.0007177169900387526,-0.0015023433370515704,-0.0014871293678879738,-0.0011666457867249846,-0.001184009131975472,-0.001046700868755579,0.004357173107564449,0.003780151018872857,0.001407284289598465,0.0024313104804605246,0.003968934994190931,0.006011130288243294,-9.671866428107023e-05,0.0012445738539099693,-0.0001742750609992072,-0.0002588293282315135,-0.0014517673989757895,-0.00021682106307707727,-0.004804810043424368,-0.0026264030020684004,-0.00027317384956404567,0.002135993679985404,-0.001100421417504549,-0.0036930604837834835,-0.01283601950854063,-0.011893738992512226,-0.01298411376774311,-0.011282398365437984,-0.011287236586213112,-0.010997200384736061,-0.0008270450052805245,-0.0016836656723171473,-0.002867428818717599,-0.0020054862834513187,-0.0012785206781700253,-0.0007689651683904231,0.0008072691271081567,0.0002123953017871827,2.7885849704034626e-05,-0.0007423987262882292,0.0010557310888543725,0.0020169515628367662,-0.007053320296108723,-0.006052385084331036,-0.004679244477301836,-0.005232995375990868,-0.006047294940799475,-0.007322330493479967,0.036131225526332855,0.06754235178232193,0.03844776004552841,-0.0018937467830255628,-0.06530072540044785,-0.0009940976742655039,-0.03042731247842312,-0.026897303760051727,0.01937989704310894,0.02102578617632389,-0.11091316491365433,0.18038421869277954,0.016734614968299866,0.04742571339011192,-0.007891274057328701,-0.0025615792255848646,0.0019663621205836535,-0.011178247630596161,0.0010205438593402505,-0.02130780555307865,0.006220218259841204,0.04573565348982811,0.028288569301366806,0.024123936891555786,0.0496029406785965,-0.02272183634340763,0.08263137936592102,-0.034752506762742996,0.06491058319807053,0.21433031558990479,-0.009537843987345695,0.06174289062619209,-0.015455246903002262,-0.025339970365166664,-0.10193605720996857,0.03053002431988716,0.009406039491295815,0.019382771104574203,0.01680668257176876,0.021772217005491257,-0.029114343225955963,-0.13597165048122406,0.010820207186043262,-0.012096704915165901,0.03682258352637291,0.0008326457464136183,-0.011295560747385025,-0.07066529989242554,-0.009724373929202557,-0.007454246282577515,-0.002133263973519206,0.02207436040043831,-0.009454749524593353,0.07045317441225052,0.010986887849867344,0.009830099530518055,0.03808224946260452,0.01579977758228779,0.027650726959109306,0.10280333459377289,-0.005181522108614445,0.004327430855482817,-0.009334816597402096,0.019124675542116165,-0.003158650128170848,-0.04276302456855774,0.004914674907922745,0.002887718379497528,-0.005407662130892277,0.011022885330021381,0.029977954924106598,-0.003631043480709195,-0.03694247454404831,-0.034033965319395065,-0.030526485294103622,-0.03515869006514549,-0.03764266520738602,-0.031660836189985275,0.017256613820791245,-0.0140593983232975,0.01624342054128647,-0.031796373426914215,0.0167215708643198,0.0428328774869442,0.0038845243398100138,-0.01389898918569088,-0.008809769526124,-0.02478811703622341,-0.000280623760772869,-0.055443961173295975,-0.0020865562837570906,-0.0073470743373036385,4.5130807848181576e-05,0.030507463961839676,0.02599925361573696,0.048393648117780685,-0.024643000215291977,-0.031369078904390335,-0.021871542558073997,-0.02013094536960125,-0.024272536858916283,-0.03719685226678848,0.016427068039774895,0.032079968601465225,0.029536858201026917,0.044212706387043,0.05975734815001488,0.060693297535181046,-0.0022368845529854298,-0.010687095113098621,-0.0023534693755209446,-0.0008057219674810767,0.038103435188531876,-0.007154753431677818,0.015459882095456123,0.0052326335571706295,0.002485196804627776,-0.012349443510174751,-0.007943346165120602,0.04628206789493561,-0.049883242696523666,-0.043723221868276596,-0.05466083437204361,-0.06307879090309143,-0.07310879230499268,-0.09837859123945236,-0.007716701831668615,-0.012342182919383049,-0.00023893508478067815,0.011135012842714787,0.037814993411302567,0.09703768044710159,0.003248135559260845,-0.005667502526193857,-7.6119131335872225e-06,-0.02124187909066677,-0.0007359978626482189,-0.07926775515079498,-0.004689761437475681,-0.005603042896836996,-0.00021149574604351074,-0.0028025219216942787,0.025676634162664413,0.026296675205230713,0.013384179212152958,-0.03337012603878975,0.0003898422000929713,-0.0017859019571915269,0.002872214885428548,-0.05115750432014465,-0.00759153114631772,-0.014773867093026638,-0.006537782493978739,-0.016623197123408318,0.056965067982673645,-0.0711263045668602,-0.03278637304902077,-0.01656954362988472,0.00588449090719223,0.0019851813558489084,0.06629464775323868,-0.1473049372434616,0.0188283734023571,0.01025963295251131,-0.01355651579797268,-0.014349636621773243,-0.03048151545226574,-0.16898693144321442,-0.01960139349102974,-0.024465689435601234,-0.045816369354724884,0.017076916992664337,0.044562701135873795,-0.10304863005876541,-0.0183319840580225,0.011461992748081684,0.03344066068530083,0.025354385375976562,0.04862924665212631,-0.08014227449893951,-0.03011087141931057,-0.011816299520432949,-0.06530847400426865,-0.04441852122545242,-0.03359405696392059,0.04476216435432434,-0.02751282975077629,-0.009054088033735752,-0.014054201543331146,-0.013118612580001354,-0.036157041788101196,0.0705127939581871,0.03379319980740547,0.004779744427651167,0.018961116671562195,-0.011819004081189632,-0.00776713527739048,-0.03355734422802925,0.022431576624512672,0.02136053331196308,-0.03317442908883095,0.0167554821819067,-0.040335189551115036,-0.008922550827264786,0.006586425006389618,-0.00943142268806696,0.009597190655767918,-0.0820479616522789,-0.014260801486670971,0.016911836341023445,-0.03155089542269707,-0.046122681349515915,0.008328182622790337,-0.002950396854430437,-0.00331140891648829,-0.0025645324494689703,-0.012971608899533749,0.0033982382155954838,-0.035885922610759735,0.028008297085762024,-0.018198827281594276,0.018747340887784958,0.01353017520159483,-0.013156246393918991,-0.04771474003791809,-0.03846358135342598,-0.024182239547371864,0.010285215452313423,0.012799020856618881,0.007010072935372591,-0.017355622723698616,-0.008302450180053711,-0.010411073453724384,0.023745287209749222,-0.0031134190503507853,-0.00696741184219718,-0.008795435540378094,-0.01889779046177864,-0.017010394483804703,-0.047319814562797546,0.03929676488041878,0.09865379333496094,0.03079524263739586,-0.005985569208860397,-0.02597947046160698,0.006006944924592972,0.04322362691164017,0.05639735981822014,0.01304059848189354,0.0020685307681560516,-0.06522033363580704,0.01658301241695881,0.023779219016432762,0.0757085457444191,0.00854966789484024,-0.05748409032821655,-0.049294132739305496,0.023520538583397865,-0.01794280670583248,-0.043002814054489136,0.013365553691983223,0.09163524955511093,0.011998986825346947,-0.09225005656480789,-0.036135777831077576,-0.018003838136792183,-0.019605321809649467,-0.00517710717394948,-0.0020943735726177692,-0.01091750804334879,0.010227608494460583,0.015683168545365334,-0.06401423364877701,-0.05807584896683693,-0.05547381937503815,-3.041302079509478e-05,-0.00903942808508873,0.02499009482562542,-0.06851929426193237,-0.009981339797377586,-0.06303192675113678,0.05058113485574722,0.01002067606896162,-0.0025288863107562065,-0.009922904893755913,0.006438370794057846,-0.05701608210802078,-0.06350012123584747,-0.011756557039916515,-0.030582547187805176,-0.020709367468953133,-0.10613734275102615,-0.11775720119476318,-0.04509371519088745,0.05415428429841995,0.06494639813899994,0.07623860239982605,0.068124920129776,0.05138740316033363,0.21359124779701233,0.010959080420434475,0.004506295546889305,0.023977970704436302,0.008749152533710003,0.03223064914345741,0.07248643040657043,0.010802638716995716,0.023750370368361473,0.036196883767843246,0.054507799446582794,0.05563441663980484,0.16233976185321808,-0.09079817682504654,-0.11197284609079361,-0.12738293409347534,-0.15363238751888275,-0.1595509946346283,-0.043661344796419144,0.0027601458132267,-0.02477247826755047,-0.009025928564369678,-0.047184236347675323,-0.0485471747815609,0.15893088281154633,0.023323386907577515,0.015898600220680237,0.03902299329638481,0.015181075781583786,0.014060739427804947,-0.07002290338277817,-0.011577568016946316,-0.0164553914219141,-0.015601128339767456,-0.00936522614210844,0.001701227854937315,0.03567945584654808,-0.006332076620310545,0.05542360618710518,-0.034595951437950134,-0.0492388978600502,0.053190335631370544,0.017435956746339798,-0.018759507685899734,-0.22466257214546204,6.986634980421513e-05,0.033401262015104294,0.0010269680060446262,0.0006762804696336389,0.002849629847332835,0.012069636024534702,0.008682779967784882,-0.007065874990075827,-0.0004754163674078882,0.10349714010953903,-0.037319350987672806,-0.04857685789465904,0.061387915164232254,0.0032060437370091677,-0.03297891840338707,-0.08000912517309189,0.001330091035924852,-0.03731777146458626,0.054287832230329514,0.016867462545633316,0.08188928663730621,-0.01886831782758236,0.05386931449174881,-0.18328730762004852,0.0008138661505654454,0.17021824419498444,0.06802884489297867,0.0563177727162838,-0.07594571262598038,0.020835058763623238,0.06567062437534332,0.015323125757277012,0.001247580861672759,0.2124485820531845,0.0270035769790411,-0.00463011022657156,0.056474748998880386,0.05736837536096573,0.13539747893810272,-0.15721645951271057,-0.0010172014590352774,-0.14546255767345428,-0.06950214505195618,-0.07987423241138458,0.012380760163068771,0.009133999235928059,0.11570782214403152,-0.16118581593036652,2.0291014152462594e-05,0.10616502165794373,-0.0710366815328598,-0.14344899356365204,0.11781319230794907,0.08445493131875992,0.0614783950150013,-0.3548583388328552,-0.0005394205218181014,0.04311952739953995,-0.012677731923758984,-0.04862979054450989,0.06437526643276215,0.03581008315086365,-0.0400652177631855,-0.09869465976953506,-1.5382069250335917e-05,-0.06306391209363937,0.08985069394111633,0.09737201780080795,0.15738922357559204,-0.09249473363161087,-0.23127955198287964,0.26733583211898804,0.0360252782702446,-0.1093512699007988,-0.40473464131355286,-0.34347063302993774,0.5204105377197266,-0.02695140428841114,0.1077716276049614,-0.13752277195453644,-0.012027970515191555,-0.0009656342444941401,-0.007294847629964352,-0.005928021855652332,-0.02127714455127716,-0.01139768585562706,0.008933060802519321,0.024999519810080528,0.018202204257249832,0.013219671323895454,-0.0025374984834343195,-0.011544888839125633,0.000950838380958885,0.008451850153505802,0.006041769869625568,0.012599455192685127,0.000535289291292429,-0.008287016302347183,0.002433371264487505,-0.017209501937031746,-0.007355046924203634,-0.010319184511899948,0.007780869025737047,0.017187507823109627,-0.013380125164985657,-0.01673216186463833,-0.014641203917562962,-0.018708130344748497,-0.018299534916877747,-0.019254084676504135,-0.014168894849717617,0.0036340453661978245,0.0026603781152516603,0.005404083523899317,-0.005960721988230944,-0.03663785010576248,0.012585502117872238,0.008928160183131695,0.004413208924233913,0.014566916972398758,0.000640926300548017,0.01881290040910244,-0.0034847718197852373,0.0023004314862191677,-0.0030329639557749033,-0.0033241775818169117,-0.009665943682193756,0.01169495563954115,0.045006949454545975,0.044832970947027206,0.06865164637565613,0.08859699964523315,0.04272197559475899,-0.08359025418758392,-0.07341466844081879,-0.08136450499296188,-0.07250934839248657,-0.029633430764079094,-0.07515868544578552,0.10920011252164841,-0.009228693321347237,0.013355782255530357,0.0196871105581522,-0.00364358676597476,-0.014109908603131771,0.10017065703868866,-0.019905390217900276,-0.032605499029159546,-0.017650963738560677,-0.008306660689413548,-0.012748136185109615,0.1179981604218483,0.09879148006439209,0.0820140466094017,0.04080786556005478,0.023983566090464592,0.029468988999724388,0.23846709728240967,-0.009172619320452213,0.026927104219794273,0.02548927627503872,0.007279256824404001,-0.03432939574122429,-0.05702497065067291,-0.014710525050759315,-0.020854230970144272,0.012894706800580025,-0.011827139183878899,-0.046417489647865295,-0.13724079728126526,-0.013913856819272041,-0.008107266388833523,-0.019823087379336357,-0.025154100731015205,-0.047817736864089966,-0.4362754821777344,-0.0029319687746465206,-0.007316363975405693,-0.001535222982056439,0.002743668854236603,-0.004569237120449543,0.036284297704696655,0.014335150830447674,0.008211868815124035,0.013603364117443562,0.015782561153173447,0.01986389420926571,0.03185148537158966,0.0033366840798407793,0.019871054217219353,0.010030549019575119,0.019365951418876648,0.00024782904074527323,-0.026958193629980087,-0.0010177107760682702,-0.004168285522609949,-0.0015560690080747008,-0.008135901764035225,-0.004323793575167656,-0.015914740040898323,-0.028452429920434952,-0.03274759650230408,-0.031196309253573418,-0.0310064647346735,-0.02506825514137745,-0.03646254539489746,-0.00926891528069973,-0.0003607178805395961,-0.006501144263893366,-0.008697712793946266,0.0011150272330269217,-0.015453745611011982,0.004952720832079649,0.011132892221212387,0.0024368988815695047,0.00020679431327152997,-0.00027657512691803277,0.0158024150878191,-0.006564124021679163,0.0012549017556011677,0.0021123234182596207,0.00098882254678756,-0.001028920989483595,0.026913804933428764,-0.012420705519616604,-0.03993004187941551,-0.034949254244565964,-0.04911472648382187,-0.05129415914416313,0.07714542746543884,0.043638925999403,0.050678789615631104,0.050752196460962296,0.04578956961631775,0.05354948341846466,0.023321473971009254,0.009613347239792347,0.00749908247962594,0.006008389871567488,0.005525137763470411,0.011904650367796421,-0.04502938687801361,0.003286748891696334,0.006206225603818893,0.011143133975565434,0.025895945727825165,-0.002356864744797349,-0.032967690378427505,-0.07392466068267822,-0.06337420642375946,-0.06335271894931793,-0.05469824746251106,-0.06691723316907883,-0.1756727695465088,-0.0277230367064476,-0.0027444225270301104,-0.02110311947762966,-0.022616585716605186,-0.0177915021777153,-0.10282685607671738,0.0073113287799060345,0.015120670199394226,0.009179254062473774,0.024174826219677925,-0.0027747848071157932,0.0694555938243866,-0.013317924924194813,0.005363975185900927,-0.005443797912448645,0.004399384371936321,-0.004284627735614777,0.2826424837112427,-0.008879391476511955,-0.019379062578082085,-0.023846453055739403,0.005295868963003159,0.024855153635144234,-0.02059110254049301,-0.018324295058846474,-0.005811769515275955,-0.018583577126264572,-0.08788924664258957,0.0041537173092365265,-0.1966736763715744,-0.010861099697649479,-0.03872577100992203,-0.009548051282763481,-0.013626058585941792,0.026083985343575478,-0.08443372696638107,-0.015330503694713116,-0.004479409195482731,-0.011504552327096462,-0.019413897767663002,-0.054316867142915726,-0.18256531655788422,0.06806981563568115,0.051231469959020615,0.014543664641678333,0.0577910840511322,-0.014499184675514698,0.08665765821933746,0.02149626426398754,0.020070137456059456,0.027138831093907356,0.06965509057044983,0.014862318523228168,0.0332174226641655,-0.028740981593728065,-0.01024872437119484,-0.031202783808112144,-0.0032953682821244,-0.0441431887447834,0.055769722908735275,0.031481727957725525,0.006952918600291014,0.009926814585924149,-0.01748296245932579,-0.013675786554813385,0.09201657772064209,0.024998119100928307,-0.015799004584550858,-0.00909122359007597,-0.01298392191529274,0.010862947441637516,-0.012651165015995502,-0.0004886696697212756,-0.020956015214323997,0.0017585533205419779,-0.005487194284796715,0.025542056187987328,0.0065523176454007626,-0.009373046457767487,-0.008649931289255619,-0.010156149044632912,-0.015214857645332813,0.015813926234841347,-0.02686590701341629,0.007045924197882414,0.03938356041908264,0.032408278435468674,0.021287869662046432,0.010954983532428741,-0.08196472376585007,-0.014722101390361786,-0.02973288670182228,-0.011363928206264973,0.011591416783630848,0.0071730646304786205,-0.022715888917446136,-0.005037238355726004,-0.018489038571715355,-0.021995360031723976,-0.01992650516331196,-0.014724666252732277,0.013180569745600224,-0.007525261491537094,-0.020493600517511368,-0.024687448516488075,0.004081751685589552,0.010941972024738789,-0.015409472398459911,-0.003875367110595107,0.004947967361658812,0.010511724278330803,-0.001933566527441144,-0.011150209233164787,0.005757358856499195,-0.03244985267519951,-0.01536429114639759,-0.010610855184495449,-0.03429598733782768,0.028887391090393066,0.17890600860118866,0.04546188935637474,0.04548240080475807,0.02455967850983143,0.029625272378325462,0.032891444861888885,-0.025972837582230568,-0.004811686463654041,-0.015900909900665283,-0.0020362879149615765,0.004055106081068516,0.04353160038590431,-0.18396307528018951,0.03339555114507675,0.017821790650486946,0.02306828275322914,0.02920634113252163,-0.01965559646487236,-0.08696206659078598,-0.01791800744831562,-0.01840970665216446,-0.04350195825099945,-0.03306206688284874,-0.06263533234596252,-0.47869107127189636,-0.018432488664984703,-0.005862923804670572,-0.0050299265421926975,-0.00045519685954786837,0.05213655158877373,-0.10759004950523376,0.017039278522133827,0.01854037307202816,0.003984678536653519,0.011192177422344685,0.008680981583893299,0.2686369717121124,0.023699576035141945,0.009327944368124008,0.0022592602763324976,0.02411695383489132,0.0011177699780091643,0.42475396394729614,-0.019755983725190163,-0.015827201306819916,-0.025880590081214905,-0.03230320289731026,-0.10247258841991425,-0.22532197833061218,0.06812979280948639,0.07029152661561966,0.08247550576925278,0.08732536435127258,0.1244267001748085,0.4034559428691864,0.00303309946320951,0.008681987412273884,0.013035147450864315,0.009126292541623116,-0.014143147505819798,0.3006867468357086,0.027675345540046692,0.02509661391377449,0.022843671962618828,0.024481644853949547,0.05425669997930527,0.5114333033561707,-0.13945667445659637,-0.14238597452640533,-0.16858607530593872,-0.1738055944442749,-0.1967606097459793,-0.2161826491355896,-0.017418285831809044,-0.016791438683867455,-0.02565300278365612,-0.021597320213913918,-0.0466265007853508,0.06264626979827881,0.03444599360227585,0.033084459602832794,0.031164545565843582,0.03190530464053154,0.034543607383966446,-0.22097395360469818,-0.032145630568265915,-0.0329652763903141,-0.03655678406357765,-0.03675997257232666,-0.02131369151175022,-0.22273191809654236,-0.010670570656657219,-0.002251485362648964,-0.016729604452848434,-0.1299920529127121,0.05099371820688248,-0.0015923093305900693,-0.09839268773794174,-0.49201035499572754,-0.007650628685951233,0.02866385132074356,-0.0005278514581732452,0.00921799149364233,-0.028258105739951134,-0.0008993669762276113,0.014354047365486622,-0.03215295448899269,-0.004482197109609842,-0.04635161906480789,-0.0292667206376791,-0.10245048254728317,-0.09897272288799286,0.04831860959529877,-0.13185229897499084,0.10509029030799866,0.015846654772758484,0.2757180333137512,0.045333512127399445,0.21684114634990692,-0.11512438207864761,-0.031805846840143204,0.19252294301986694,-0.05307061970233917,-0.00033737948979251087,0.19266913831233978,0.0611385703086853,0.10612770915031433,0.013855021446943283,-0.07304496318101883,0.2759372591972351,-0.07874979078769684,0.021693363785743713,0.313888818025589,0.03541017323732376,0.25033095479011536,-0.09081139415502548,0.04633219167590141,0.4282185733318329,-0.2538493871688843,0.015894832089543343,-0.03511681407690048,-0.026888763532042503,0.00514141283929348,-0.2329917997121811,0.03389052674174309,0.13370902836322784,-0.2330711930990219,0.016321439296007156,-0.033705487847328186,0.02216581255197525,0.12234263122081757,-0.2427009493112564,0.04020953178405762,0.017350230365991592,-0.7607975006103516,-0.011648779734969139,-0.1153607964515686,-0.018579166382551193,-0.13795948028564453,0.07298921793699265,0.024401739239692688,-0.2646816372871399,0.08563793450593948,-0.019630707800388336,0.14424510300159454,-0.023996589705348015,-0.22164395451545715,0.08962574601173401,0.030287668108940125,-0.0014492584159597754,0.3086436688899994,0.08149601519107819,0.1026722639799118,-0.08894044160842896,0.18849089741706848,0.10177157819271088,0.022631747648119926,0.2977038323879242,-0.11729063838720322,-0.026545511558651924,-0.020205268636345863,-0.011338689364492893,0.0004439836193341762,0.0070957583375275135,-0.0007829495007172227,-0.009377414360642433,0.0013728301273658872,0.002318847458809614,-0.005788236390799284,0.0029422189109027386,-0.007775275502353907,0.006428872235119343,0.0031921544577926397,0.004110562149435282,0.0005393375759012997,0.0064140199683606625,-0.0006685293628834188,-0.0002129747881554067,-0.0033688670955598354,-0.018475988879799843,-0.017805781215429306,-0.012759219855070114,-0.013948505744338036,0.0008233948610723019,-0.00017973042849916965,-0.0015773362247273326,-0.0058532459661364555,-0.006847324315458536,-0.0071027204394340515,0.003438417799770832,0.008762193843722343,0.015477228909730911,0.009630244225263596,0.009796480648219585,0.007122197188436985,0.018318375572562218,0.011298125609755516,0.008382638916373253,0.011347067542374134,0.007453115191310644,0.011949761770665646,-0.016023939475417137,-0.008600503206253052,-0.008764756843447685,-0.005555711220949888,-0.005020061507821083,-0.010729702189564705,0.003082503331825137,0.01678338274359703,0.0239393413066864,-0.0004175352514721453,0.007295905612409115,0.03130064532160759,0.009366114623844624,-0.008174014277756214,-0.006067280657589436,0.005655215587466955,0.020751548931002617,-0.018977032974362373,-0.00790982786566019,-0.004751404747366905,-0.017592865973711014,-0.018237341195344925,0.0003384941373951733,0.11559604853391647,-0.008682508021593094,0.028567111119627953,0.023563958704471588,0.01996820606291294,0.010978175327181816,-0.11087924242019653,0.03842242807149887,0.002141364151611924,0.045718882232904434,0.02538074180483818,0.013722001574933529,0.13459357619285583,0.0012463204329833388,0.020150206983089447,0.002023976529017091,0.030550803989171982,-0.007263506297022104,0.07290609180927277,-0.01274342741817236,0.01748267561197281,0.02401459775865078,0.022549016401171684,0.030786940827965736,-0.05140236020088196,0.01815005950629711,-0.00044093444012105465,0.006989309098571539,0.014824247919023037,0.016302259638905525,-0.03393807262182236,-0.021953480318188667,-0.010870929807424545,0.011033615097403526,0.00027169176610186696,-0.001733611454255879,-0.013515914790332317,0.061514150351285934,-0.00441261800006032,0.011432921513915062,0.010666067712008953,-0.0052563962526619434,0.0014709393726661801,0.060549039393663406,-0.0014684913912788033,0.0011008535511791706,0.009044084697961807,0.014662775211036205,-0.02411401830613613,0.003962303511798382,-0.007147164084017277,-0.006356744095683098,-0.023011093959212303,-0.016394231468439102,-0.0366431288421154,-0.045287832617759705,-0.0186262596398592,-0.020534783601760864,-0.01772458478808403,-0.020463017746806145,-0.007220660801976919,-0.05451728031039238,0.0026900211814790964,-0.0052290381863713264,-0.013959884643554688,0.005679448135197163,0.016353538259863853,-0.06874381005764008,0.0018881511641666293,-0.012368062511086464,-0.010610383003950119,-0.005117003805935383,-0.02256612852215767,-0.007147676777094603,-0.007391055580228567,-0.0003587240935303271,-0.010723447427153587,-0.011171354912221432,-0.02168293483555317,-0.023696165531873703,-0.0353531651198864,-0.04137608781456947,-0.03455103933811188,-0.039840735495090485,-0.06979554891586304,0.041917137801647186,0.04220709949731827,0.055400945246219635,0.033107053488492966,0.0540478453040123,0.29550856351852417,-0.01826162077486515,-0.013502314686775208,-0.017340850085020065,-0.0052286963909864426,-0.010385899804532528,0.3395656943321228,0.011286926455795765,0.017177052795886993,0.022408556193113327,0.016161639243364334,0.016339005902409554,0.18795683979988098,-0.09077873080968857,-0.1247495636343956,-0.13312454521656036,-0.11443488299846649,-0.1621071994304657,-0.33122536540031433,0.0056897616013884544,0.018688494339585304,0.021786199882626534,0.03755834698677063,0.03413363918662071,-0.2619089186191559,0.0075889467261731625,0.012009995058178902,0.02198474481701851,0.01438799686729908,0.012775738723576069,-0.20163442194461823,-0.028571683913469315,-0.03571575507521629,-0.027075527235865593,-0.021629683673381805,-0.04080863296985626,0.0036792417522519827,0.02429022453725338,0.02520567551255226,0.03558589518070221,0.00381839438341558,0.012416169047355652,0.11329364776611328,-0.024684779345989227,-0.04764651507139206,-0.059654463082551956,-0.030782010406255722,-0.020594296976923943,-0.15515677630901337,-0.003253053640946746,0.016305187717080116,0.019418152049183846,0.019070357084274292,-0.002989896573126316,-0.18096058070659637,-0.01741752028465271,-0.02189628593623638,-0.015994790941476822,-0.014528991654515266,-0.005054633133113384,-0.21104280650615692,0.10518760234117508,0.10236150026321411,0.1332811862230301,0.09158821403980255,0.07002231478691101,0.07106218487024307,-0.026071328669786453,-0.02379642240703106,-0.029491353780031204,-0.014186069369316101,-0.02261546440422535,-0.07480907440185547,-0.003701298963278532,-0.02109985612332821,-0.0007796994759701192,-0.0024311980232596397,0.004564558155834675,0.3259615898132324,0.017332278192043304,0.015105688944458961,0.021366730332374573,0.024706419557332993,-0.0046970960684120655,-0.041616249829530716,0.0018916052067652345,0.004431080538779497,-0.007134702522307634,-0.017598647624254227,-0.008114156313240528,0.06789607554674149,0.004934767726808786,0.015018780715763569,0.020986342802643776,0.015878520905971527,0.016868675127625465,-0.11467868089675903,0.00418785959482193,-0.010310572572052479,-0.025179829448461533,0.011077314615249634,-0.03171389549970627,-0.15072868764400482,0.006566979922354221,0.007934977300465107,0.016651488840579987,-0.013659263961017132,-0.015020935796201229,-0.051258914172649384,0.008523311465978622,-0.02891879715025425,-0.003026098944246769,0.057212237268686295,0.10032673180103302,0.2117156982421875,0.013949035666882992,0.008905455470085144,-0.0031755694653838873,-0.018109427765011787,-0.03348146378993988,0.11250492185354233,0.024130869656801224,0.008029126562178135,0.005082415882498026,0.014627039432525635,-0.016922475770115852,0.18250590562820435,0.014445389620959759,-0.00509860273450613,0.013043373823165894,0.03556514158844948,0.03592145815491676,0.05319822207093239,-0.014749848283827305,-0.018045106902718544,-0.003533882787451148,-0.036141037940979004,-0.033029742538928986,0.04970943555235863,0.007825423963367939,0.00527762807905674,0.026150988414883614,0.007450041361153126,0.018844641745090485,-0.10057497024536133,0.0012480939039960504,0.005365450866520405,-0.02563183568418026,-0.01687581278383732,-0.042634572833776474,-0.085434190928936,0.0053018187172710896,-0.00420865835621953,-0.010201070457696915,-0.00835945550352335,0.003383565228432417,-0.06482721120119095,-0.00847671739757061,0.02404898591339588,0.040657833218574524,0.03422173485159874,-0.00613854406401515,0.09829875826835632,0.0031380264554172754,0.012728487141430378,-0.01681172475218773,0.014011838473379612,-0.014345948584377766,0.047280360013246536,0.002313393633812666,0.012525878846645355,-0.018862247467041016,-0.016388963907957077,-0.03670124337077141,0.10266240686178207,0.006851153448224068,0.020624808967113495,0.014458376914262772,0.029903870075941086,0.003212863579392433,-0.02130332589149475,-0.001632216852158308,-0.005078581627458334,-0.002252056961879134,-0.01051438506692648,0.004270539153367281,0.009181133471429348,0.0025647575967013836,-0.013229059055447578,-0.01623939722776413,0.004010828677564859,-0.0012463410384953022,-0.03013615310192108,0.024342099204659462,0.0009989055106416345,0.005801463034003973,0.0033691718708723783,-0.00862880703061819,0.020273033529520035,-0.016042031347751617,0.001380637171678245,0.008111122995615005,-0.008150926791131496,-4.836211883230135e-05,-0.031267717480659485,-0.02827337011694908,8.75612113304669e-06,-0.003987882286310196,-0.011696937493979931,0.009118126705288887,0.05452730134129524,-0.02648676186800003,0.006016386207193136,0.013025335036218166,-0.014785119332373142,-0.008545378223061562,0.0426894873380661,-0.03374844789505005,-0.016786420717835426,-0.010796577669680119,-0.03577887639403343,-0.01640980876982212,0.03571911156177521,-0.0029665224719792604,-0.0027208449319005013,-0.004239776637405157,-0.006307879462838173,0.0017785732634365559,-0.06357885897159576,-0.004398129880428314,0.08649712055921555,-0.006843851879239082,-0.40418511629104614,0.2193661630153656,0.02143564447760582,0.04007400572299957,0.009117241017520428,0.00039674845174886286,0.010600625537335873,0.0024327703285962343,0.004289061762392521,-0.03467223793268204,-0.015875760465860367,-0.00502247828990221,-0.014967819675803185,-0.0032376362942159176,0.07231656461954117,0.07791513949632645,-0.0171032901853323,-0.005609450861811638,-0.09499490261077881,-0.0904051810503006,0.012418505735695362,-0.01614142209291458,0.1069798618555069,0.004986390471458435,-0.20627719163894653,-0.01265384629368782,-0.07440222799777985,-0.14791545271873474,-0.014991261065006256,-0.006613737903535366,-0.13716739416122437,-0.10392069071531296,-0.3266824185848236,-0.07710051536560059,-0.24621011316776276,0.10539334267377853,-0.036993201822042465,-0.02672642469406128,0.2950358986854553,0.0072238389402627945,-0.3043290376663208,-0.2950894832611084,-0.12129011750221252,-0.097808338701725,-0.009252864867448807,0.016628451645374298,-0.008804102428257465,-0.0060527147725224495,-0.2312251776456833,-0.31871548295021057,-0.13646791875362396,-0.33482980728149414,0.00891477894037962,-0.016581224277615547,-0.057967495173215866,-0.007429846096783876,-0.2608758807182312,-0.3134474754333496,-0.04957820102572441,-0.08766614645719528,-0.01623961143195629,-0.002170978346839547,-0.05478057265281677,0.0373687706887722,0.23878084123134613,-0.30586785078048706,0.041466087102890015,-0.03043214976787567,0.0038139193784445524,0.01899062469601631,0.0615614578127861,-0.063657246530056,0.2118789553642273,0.03783612698316574,0.06571612507104874,-0.07368813455104828,0.0032133387867361307,0.15195757150650024,0.26315367221832275,-0.15021435916423798,-0.23178720474243164,0.3314002454280853,0.20011559128761292,0.20511016249656677,-0.011754634790122509,-0.02712460421025753,-0.028812522068619728,-0.021774670109152794,-0.013428599573671818,-0.009802744723856449,-0.18908104300498962,0.0017674177652224898,-0.007167277857661247,-0.0020905614364892244,-0.016774088144302368,-0.015188619494438171,0.16581647098064423,-0.01896098628640175,-0.01935132034122944,-0.021983902901411057,-0.007928888313472271,-0.007642799522727728,0.20991303026676178,0.012094804085791111,0.014259996823966503,0.0015117130242288113,0.0011873382609337568,0.010910194367170334,0.28119394183158875,-0.02773914858698845,0.0006176666938699782,0.012787437997758389,0.016479289159178734,0.003247660817578435,0.1994500458240509,0.01972213201224804,0.022902270779013634,0.02369515970349312,0.014304774813354015,0.03044845536351204,0.21903188526630402,-0.006891601253300905,0.011405831202864647,-0.009470164775848389,-0.004464953672140837,0.01117696799337864,-0.06863971799612045,-0.012828823179006577,0.0035318585578352213,0.008391150273382664,0.0008358244085684419,0.012343944050371647,-0.2810300290584564,0.015868758782744408,0.026619266718626022,0.009403449483215809,0.005343059543520212,0.0056792753748595715,0.008483323268592358,-0.011557596735656261,-0.018489088863134384,-0.002326695714145899,0.0008874799241311848,-0.016832849010825157,0.2097475230693817,-0.006169695872813463,0.004429447930306196,0.004026907030493021,0.0051796650514006615,0.01216746587306261,0.07925064116716385,-0.011045442894101143,-0.017470598220825195,-0.0016359760193154216,-0.0051810601726174355,-0.01272442378103733,0.20559509098529816,0.029847102239727974,0.03648819029331207,-0.00964722316712141,-0.00021859616390429437,0.028679631650447845,0.12505532801151276,0.023006154224276543,0.009344608522951603,0.00581847969442606,0.004247021861374378,-0.00661345012485981,0.2493021935224533,-0.016130926087498665,-0.003154967911541462,-0.007759385742247105,-0.006037765182554722,-0.013714863918721676,0.05571872740983963,0.009774582460522652,0.0018669009441509843,-0.005493384785950184,0.0011501424014568329,0.009508872404694557,-0.21854469180107117,-0.015909958630800247,-0.022559359669685364,-0.014203714206814766,-0.01975480280816555,-0.04161371290683746,-0.034520138055086136,0.009031765162944794,0.016205141320824623,0.006721631158143282,0.016246089711785316,0.013799998909235,-0.0370243676006794,0.0002908736059907824,-0.0015163321513682604,-0.002630413742735982,-0.0057387687265872955,0.00016646803123876452,0.10852034389972687,0.01412018109112978,0.009310293011367321,0.010060546919703484,0.016037415713071823,0.009027758613228798,-0.02587748132646084,-0.020045971497893333,-0.0341462641954422,-0.027751753106713295,-0.025602128356695175,-0.0031191406305879354,-0.3371497094631195,0.021000035107135773,0.012655777856707573,0.016702987253665924,0.022641245275735855,-0.0027121305465698242,-0.000925111526157707,0.01760009489953518,0.008007078431546688,0.008706634864211082,0.03106098249554634,0.02700568176805973,0.10771982371807098,0.012405438348650932,-0.003943834453821182,-0.003227428300306201,0.0077136000618338585,0.007753213867545128,0.17200404405593872,-0.001162484404630959,0.005536541808396578,-0.003831373294815421,-0.010984768159687519,-0.01164997462183237,-0.0625501498579979,0.016895735636353493,-0.0031558037735521793,0.018566789105534554,-0.0003228500427212566,0.008415203541517258,0.11723798513412476,-0.011091604828834534,0.00671829329803586,-0.015131352469325066,0.006399251986294985,-0.006945624481886625,0.10241631418466568,0.013210863806307316,0.004969676956534386,0.018634669482707977,0.0028749865014106035,0.01602715626358986,0.042615365236997604,-0.05793879181146622,-0.046614766120910645,-0.04390506073832512,-0.044991206377744675,-0.055667050182819366,-0.26896730065345764,0.018125474452972412,0.007031367626041174,0.002954442286863923,0.005321734584867954,0.010989156551659107,-0.07558699697256088,0.006365771871060133,0.003044563578441739,0.02728951722383499,-0.001977144740521908,0.004478458780795336,-0.12304619699716568,-0.023697834461927414,-0.02103637345135212,-0.019493967294692993,-0.015956595540046692,-0.02241840772330761,-0.009477013722062111,-0.005011569708585739,-0.0023715277202427387,-0.0014585006283596158,-0.010064738802611828,-0.00986920390278101,-0.02051685005426407,0.005138781853020191,0.010240563191473484,0.005455935839563608,0.010998550802469254,0.011339549906551838,0.0019034616416320205,0.006479279603809118,0.007766073569655418,0.0012409116607159376,-0.011701599694788456,0.0006066272617317736,-0.013624866493046284,0.010124250315129757,-0.014169740490615368,0.0022415239363908768,-0.019203422591090202,-0.009560960344970226,-0.015505583956837654,-0.01008171122521162,-0.022156640887260437,-0.014230460859835148,-0.015163511969149113,-0.007973117753863335,0.003611486405134201,-0.010560947470366955,-0.0006890548975206912,-0.009137864224612713,0.010710394009947777,-0.006270510144531727,0.014693278819322586,0.014756498858332634,-0.01897304691374302,-0.005935434252023697,-0.0017419991781935096,-0.0028464868664741516,0.0068581136874854565,0.0050703296437859535,0.0031739219557493925,-0.004787822254002094,-0.003244978841394186,-0.012686926871538162,-0.01965964399278164,0.020933043211698532,0.00567541504278779,0.012125859037041664,0.028207147493958473,-0.017141083255410194,-0.1744791716337204,-0.004629814997315407,0.002988577587530017,0.0017846679547801614,-0.02428993582725525,-0.003939291462302208,0.07348188757896423,0.026423748582601547,0.024881502613425255,-0.0008840751252137125,0.0013406961224973202,-0.012079261243343353,0.2032223641872406,0.008108788169920444,-0.002477293135598302,0.005997294094413519,-0.008516641333699226,-0.011907004751265049,0.08527792245149612,0.009548635222017765,0.0027526109479367733,-0.008250858634710312,0.04971688240766525,0.06959973275661469,0.19321444630622864,-0.034941669553518295,-0.04325171560049057,-0.00942303892225027,-0.013472477905452251,-0.05151617154479027,0.1851106584072113,0.013038337230682373,-0.03663887828588486,0.0020501618273556232,-0.011638915166258812,-0.018556702882051468,-0.03308527171611786,0.01969805918633938,0.016460347920656204,-0.008670840412378311,0.00517646549269557,-0.005023967009037733,-0.22728928923606873,0.0025120978243649006,-0.004916861653327942,-0.003276708535850048,-0.00045036798110231757,-0.0018983223708346486,-0.0106576606631279,0.00037586045800708234,-0.00036815786734223366,0.0014337713364511728,-0.0017369374400004745,0.0037869936786592007,-0.003508207853883505,0.0008699579630047083,-0.0007891279528848827,3.9895039662951604e-05,-0.002189223887398839,-0.0016127126291394234,-0.005674536805599928,-0.004505063407123089,-0.0004095376643817872,0.002598297316581011,-0.0024647084064781666,-0.005666812881827354,-0.00012533363769762218,-0.008478887379169464,-0.006575928535312414,-0.005766304209828377,-0.008005916140973568,-0.005704541690647602,-0.0049801417626440525,0.001319232047535479,0.0028044991195201874,0.0022306658793240786,0.002130963373929262,0.006082391366362572,0.0005748647381551564,0.002788594691082835,0.0012097223661839962,0.0004747257044073194,7.829568176020985e-07,0.0008170068613253534,0.0012021177681162953,-0.0067033967934548855,-0.004339857492595911,-0.0027932506054639816,-0.00040530681144446135,-0.005727042444050312,-0.0042037381790578365,-0.0325690321624279,-0.033025167882442474,-0.029095236212015152,-0.004359835758805275,0.02585461363196373,0.1558196097612381,0.0010718766134232283,0.0012304730480536819,-0.00017594416567590088,-0.00788195338100195,0.0002681251789908856,-0.006937244441360235,0.0037248735316097736,0.008109713904559612,0.0009705563425086439,0.005135017447173595,0.004966109059751034,-0.051635850220918655,-0.0070120664313435555,-0.00814203079789877,0.001815221505239606,0.0011182436719536781,0.00020766216039191931,-0.16653338074684143,-0.008197431452572346,-0.00458485959097743,-0.00043527010711841285,-0.005758611019700766,-0.004412022419273853,0.08292457461357117,-0.019680948927998543,-0.008029911667108536,-0.0037175267934799194,-0.006192241795361042,0.007771321106702089,-0.04915972799062729,-0.0031528864055871964,0.0031356739345937967,0.0022447877563536167,0.005897652357816696,0.019741566851735115,-0.007092892192304134,-0.019770152866840363,-0.0066357264295220375,0.0009242536034435034,-0.011237076483666897,-0.00031291061895899475,-0.036054544150829315,-0.009490390308201313,0.04454319179058075,-0.14344407618045807,-0.09262167662382126,-0.005439334083348513,0.10316701978445053,-0.005402575712651014,0.04045027494430542,-0.03417664393782616,-0.03589441999793053,0.006944905035197735,0.01600528694689274,-0.0005000384408049285,-0.009232064709067345,-0.0012836161768063903,-0.032565172761678696,0.31944796442985535,0.1147516593337059,-0.07885687053203583,-0.03683696687221527,0.0021758023649454117,0.21625493466854095,-0.0007823983323760331,0.00016299735580105335,-0.3866589665412903,-0.2701027989387512,0.16677051782608032,-0.07996061444282532,0.001010906184092164,-0.20125874876976013,-2.494525870133657e-05,-0.07684823125600815,0.1413268744945526,-0.0676613599061966,0.1448773741722107,-0.11269301921129227,-0.0010270284255966544,-0.036474477499723434,0.001020739902742207,-0.17534609138965607,0.046931322664022446,0.049699816852808,0.2522965371608734,-0.19420622289180756,9.36902069952339e-05,0.21413934230804443,-0.002592954318970442,-0.08578614890575409,-0.006470244377851486,0.25084051489830017,0.023197438567876816,-0.16868534684181213,-0.0004041173669975251,0.07769311219453812,-0.0005570069770328701,0.0672110840678215,-0.01035800389945507,0.2645187973976135,0.028372682631015778,-0.156899556517601,0.014177107252180576,0.18452279269695282,-0.0012914196122437716,-0.07305104285478592,-0.21380624175071716,0.006759717594832182,-0.09076949954032898,0.10561054199934006,-0.00971096009016037,-0.0037739642430096865,-4.345384513726458e-05,-0.06348466128110886,0.20926183462142944,0.3075309693813324,-0.19104304909706116,0.12277371436357498,0.009178010746836662,0.0408317930996418,0.0022792562376707792,0.03203177452087402,0.17176684737205505,-0.11259521543979645,-0.35137367248535156,0.3232383728027344,0.00489308126270771,-0.1087421104311943,-0.01968010514974594,-0.06790177524089813,0.009032142348587513,-0.0010736305266618729,-0.0044992826879024506,0.020921872928738594,0.03549931198358536,0.02440084144473076,-0.02335752360522747,0.008662139065563679,0.06109528988599777,0.04475764185190201,0.02334425412118435,0.00391411455348134,-0.016076333820819855,-0.017760412767529488,0.09866464883089066,0.019785793498158455,-0.0022279610857367516,-0.0070306663401424885,-0.0628795400261879,-0.050468478351831436,-0.014871526509523392,-0.04147215187549591,0.0015238169580698013,0.021420201286673546,-0.10442133992910385,-0.06293515115976334,0.04419853165745735,0.07801366597414017,0.1634724736213684,0.003523360239341855,-0.029901638627052307,0.015971986576914787,-0.04433447867631912,-0.07145191729068756,-0.06343861669301987,-0.0579061396420002,-0.03743641451001167,-0.024524236097931862,-0.04228759557008743,-0.09967479854822159,-0.08292784541845322,-0.004007382318377495,-0.06218444183468819,0.20631438493728638,0.04738672077655792,0.005453793797641993,0.05283493548631668,-0.04213472828269005,0.007118762005120516,0.04866643249988556,-0.0015765646239742637,0.05832501873373985,0.04906352609395981,0.031333401799201965,-0.02684597671031952,-0.07600664347410202,0.09984074532985687,-0.05275941267609596,0.1638685166835785,0.022765235975384712,0.05180586129426956,0.07494926452636719,0.01657692901790142,0.04393694922327995,0.08670371770858765,0.07473915815353394,0.02959679812192917,0.09771443903446198,0.06123519316315651,0.097243532538414,0.06319811195135117,0.1348593384027481,0.13716758787631989,0.17297229170799255,0.15238867700099945,-0.0552583746612072,-0.050015632063150406,-0.017484428361058235,-0.10974915325641632,-0.059621699154376984,-0.042973875999450684,-0.06142856553196907,-0.07937204092741013,-0.05644693225622177,-0.03106319159269333,-0.05056711286306381,0.06844779849052429,0.008712275885045528,0.2854726016521454,0.21227024495601654,0.06990769505500793,-0.024497751146554947,-0.021146951243281364,-0.011892599053680897,-0.04243822395801544,-0.007158999796956778,0.11200359463691711,0.10727641731500626,0.011202416382730007,0.005258957855403423,-0.008463370613753796,-0.06377919018268585,0.08844880014657974,-0.03498105704784393,-0.045316025614738464,-0.010135403834283352,-0.06366336345672607,0.024836590513586998,0.009311939589679241,-0.11339234560728073,-0.044270195066928864,-0.02035589888691902,-0.02228844352066517,-0.02046523615717888,0.04320363700389862,-0.04934864491224289,-0.0020315430592745543,-0.039478424936532974,-0.4484596848487854,-0.3573192358016968,0.02072061225771904,0.006241136230528355,-0.008643066510558128,-0.025653984397649765,-0.018307527527213097,-0.022895287722349167,-0.02992766909301281,0.0031919570174068213,0.013418409042060375,0.015336349606513977,0.0005665046628564596,-0.003987452480942011,-0.13899953663349152,-0.027835221961140633,-0.0677252933382988,-0.005235329270362854,0.03666262701153755,0.07221091538667679,0.03147352486848831,0.06416007876396179,0.052701227366924286,0.06690747290849686,0.049457814544439316,-0.006865810602903366,-0.014562240801751614,-0.09796477854251862,0.0978560745716095,0.02087939716875553,-0.007793975528329611,0.0409303680062294,-0.0708586648106575,-0.014540131203830242,0.053385376930236816,-0.015604247339069843,-0.005711701232939959,0.03823912888765335,0.031079387292265892,0.014079133979976177,-0.012754845432937145,-0.0061613707803189754,0.030126867815852165,0.23909008502960205,-0.129447802901268,-0.1386338323354721,-0.000952943111769855,0.047946881502866745,-0.012241201475262642,-0.02218584343791008,0.0059920260682702065,0.0020224680192768574,0.011887815780937672,0.004770399536937475,-0.005474959034472704,-0.00043587147956714034,0.06177964806556702,0.01637204922735691,0.2738669216632843,-0.3320842981338501,0.0031582233496010303,0.05219680443406105,-0.032030247151851654,-0.02462816797196865,-0.027628418058156967,-0.033883754163980484,-0.046397339552640915,-0.05268830060958862,-0.0546773262321949,-0.03871842846274376,-0.07464800029993057,-0.0745149701833725,0.04939286410808563,0.030866943299770355,-0.2965913414955139,-0.21161960065364838,-0.10504398494958878,0.040216974914073944,0.017849313095211983,0.004633635748177767,0.04347923398017883,0.050261471420526505,0.03196004405617714,-0.027668051421642303,-0.0031497592572122812,-0.0049618822522461414,-0.007638953626155853,0.19530925154685974,0.09323450177907944,-0.20537935197353363,-0.048278726637363434,-0.050208285450935364,-0.001740866107866168,0.018940821290016174,-0.0009784211870282888,0.017967935651540756,0.006626039277762175,0.023416684940457344,0.007757667917758226,-0.020815156400203705,-0.014285615645349026,-0.02369057387113571,0.058063484728336334,-0.06452993303537369,0.09408998489379883,0.006946200039237738,-0.023007625713944435,0.04033752903342247,-0.03714264929294586,-0.04481453076004982,0.009066510945558548,-0.00040030138916336,-0.019821930676698685,-0.026695851236581802,-0.03309103474020958,-0.0017334340373054147,0.025602271780371666,-0.03848261386156082,-0.03630199283361435,0.016636217013001442,0.13935358822345734,0.04890183359384537,0.0012612721184268594,-0.004774066619575024,-0.0005703559145331383,-0.009764838963747025,-0.017928414046764374,0.015366952866315842,0.05131206661462784,0.02827977016568184,-0.10013136267662048,0.014069626107811928,0.06587398052215576,-0.14478835463523865,0.14574749767780304,0.06318755447864532,-0.0669894814491272,0.04084852337837219,-0.02597675286233425,-0.049144063144922256,-0.01007337961345911,0.05210178717970848,0.0622822642326355,0.030390089377760887,0.0813068151473999,0.14136768877506256,0.013456535525619984,-0.2909480631351471,-0.19324445724487305,0.11431511491537094,0.01592925563454628,0.009437940083444118,0.0337473563849926,-0.012585349380970001,-0.0005188602954149246,-0.01649763248860836,0.018301691859960556,0.01680050790309906,-0.015308761037886143,-0.0010623064590618014,0.0974615216255188,0.04129894822835922,-0.3627839982509613,-0.4354844093322754,0.15604080259799957,0.17937228083610535,0.07686243951320648,0.007965468801558018,0.033357977867126465,-0.008714400231838226,-0.036039941012859344,0.022189460694789886,-0.020821934565901756,0.08385638892650604,0.10400684922933578,0.050793860107660294,0.23719479143619537,-0.28524214029312134,-0.4119008183479309,0.5429198145866394,-0.030542928725481033,-0.22438956797122955,0.033297453075647354,-0.011585349217057228,-0.04379564896225929,-0.05596126988530159,-0.057471297681331635,-0.00046743417624384165,0.030463404953479767,-0.028187084943056107,-0.11465198546648026,-0.022184589877724648,-0.11953360587358475,-0.016353251412510872,0.4190024733543396,-0.011828912422060966,-0.1333330273628235,-0.03817526251077652,5.9401485486887395e-05,0.0456552691757679,0.047559674829244614,0.013654332607984543,0.014564408920705318,0.08535926789045334,0.0463232658803463,-0.124600350856781,0.01939363405108452,-0.03799408674240112,0.04599132761359215,0.47953081130981445,-0.1458190530538559,-0.25140658020973206,0.06757404655218124,0.0002977115800604224,-0.0427696630358696,-0.07034855335950851,0.024735113605856895,0.10000304877758026,0.058994270861148834,0.07467655837535858,0.20012643933296204,0.039805080741643906,-0.04274270683526993,0.16805370151996613,-0.23343956470489502,-0.22763675451278687,-0.08497442305088043,0.02571697346866131,-0.009990670718252659,-0.04910166561603546,0.001011169864796102,0.05508054420351982,0.05911380425095558,0.04101235792040825,-0.03248968347907066,-0.1652161329984665,0.002780792536213994,0.15065132081508636,0.017432963475584984,-0.13848745822906494,-0.066679447889328,-0.10550049692392349,-0.018008798360824585,-0.0032373478170484304,-0.025009015575051308,0.07517451047897339,0.0662408247590065,0.031286634504795074,-0.024363284930586815,-0.04761592298746109,0.014824174344539642,0.044735562056303024,0.019028816372156143,0.023057846352458,-0.07302039116621017,-0.06771501898765564,-0.029278989881277084,-0.012796374969184399,0.0341913141310215,0.06591010093688965,-0.03583098575472832,-0.07230281829833984,0.02351371943950653,0.04465736448764801,0.01112980954349041,-0.10281161963939667,-0.07660385221242905,-0.030231960117816925,-0.006218943744897842,0.04852260276675224,0.063754603266716,0.08438859134912491,0.018505584448575974,0.015647923573851585,-0.05036013573408127,0.036249324679374695,0.07818640023469925,0.0730864405632019,0.10652021318674088,-0.11422695964574814,-0.19125017523765564,-0.20144563913345337,-0.02459464594721794,0.04983546957373619,-0.020247293636202812,-0.07012714445590973,0.003715149825438857,0.030206948518753052,0.028968634083867073,-0.05560540035367012,0.03325316682457924,0.03036307543516159,0.07673677057027817,0.021585695445537567,-0.18551118671894073,-0.2453170269727707,-0.13148623704910278,0.049141280353069305,0.06955021619796753,-0.017402783036231995,-0.04410203918814659,0.0674619972705841,-0.06303410232067108,-0.005741264205425978,-0.06784141063690186,-0.043253783136606216,-0.02450767531991005,-0.02995321713387966,0.020319299772381783,-0.14590339362621307,-0.11425599455833435,-0.11929789185523987,0.007796081248670816,0.015909399837255478,-0.07660529017448425,-0.0708458423614502,-0.001827541971579194,0.011993465945124626,0.03579680621623993,-0.035290710628032684,-0.03996037691831589,-0.04494910314679146,-0.0034410813823342323,0.011134175583720207,-0.0658465102314949,0.01949402317404747,0.009507790207862854,0.07534515857696533,0.022317523136734962,-0.046019863337278366,-0.026888662949204445,0.02148324064910412,-0.0014706864021718502,-0.01903780922293663,0.031454525887966156,-0.05312711372971535,-0.013759756460785866,0.019229883328080177,0.039509762078523636,0.0864214077591896,0.04582773894071579,-0.016579005867242813,-0.00839849840849638,0.03596969693899155,0.04963548853993416,0.024528425186872482,-0.01069438923150301,-0.00786446500569582,-0.03637250140309334,-0.02981105074286461,0.06362726539373398,0.06973766535520554,0.0011685066856443882,-0.1089792475104332,0.04811510071158409,0.04886971786618233,0.13326309621334076,-0.009313655085861683,-0.04743654280900955,-0.07157152146100998,-0.039685577154159546,-0.06384340673685074,-0.00024580300669185817,0.016786562278866768,-0.015596971847116947,0.018841631710529327,0.05203045904636383,0.051604900509119034,0.030530165880918503,0.007500890642404556,0.016652308404445648,-0.020670665428042412,0.005907850805670023,-0.02250882424414158,-0.03313060849905014,0.01867230050265789,0.04082389920949936,-0.013324730098247528,0.004006627481430769,0.015934959053993225,0.023635251447558403,-0.019274886697530746,-0.06736381351947784,-0.021377213299274445,0.01993658021092415,0.03192127123475075,0.06933468580245972,0.039277564734220505,-0.07556113600730896,-0.10821374505758286,-0.09461396187543869,-0.05615644529461861,-0.021018803119659424,0.03846155107021332,0.04051230847835541,-0.007221213076263666,0.006562618538737297,-0.011734415777027607,0.003233977360650897,0.020078057423233986,-0.002210065023973584,-0.03174963220953941,-0.04490583389997482,-0.10057878494262695,-0.1608307659626007,-0.11870212852954865,-0.11101749539375305,-0.04356009513139725,-0.02293519116938114,-0.0014875628985464573,0.018159329891204834,-0.014651189558207989,-0.060725193470716476,-0.06771345436573029,-0.0019134461181238294,0.01821543090045452,-0.03513655066490173,-0.014890899881720543,-0.08227479457855225,-0.2179698944091797,-0.014995794743299484,0.09565306454896927,-0.01781008206307888,0.005745338276028633,0.03293602541089058,0.03692299872636795,0.04352165013551712,0.024333856999874115,-0.0026488967705518007,0.03867173194885254,0.07888875901699066,0.08927752077579498,0.06338834017515182,0.03224058076739311,-0.05573195964097977,0.12459956109523773,0.21874654293060303,-0.028447655960917473,0.014270147308707237,0.011074233800172806,0.02254507690668106,-0.0076225497759878635,-0.08385786414146423,-0.08700849115848541,-0.01631256751716137,-0.023499870672822,-0.017961563542485237,-0.033163636922836304,-0.01873963698744774,-0.09129147231578827,0.0618085078895092,0.05292261391878128,0.007982275448739529,-0.028624355792999268,0.028765948489308357,0.020351851359009743,-0.0046743168495595455,-0.02633654698729515,0.000594956858549267,0.0004047682450618595,0.03745286166667938,0.08647754788398743,0.0827285647392273,0.11986818164587021,0.05994819104671478,0.16583578288555145,0.202241450548172,0.034128349274396896,0.012136561796069145,-0.05208297818899155,-0.02291422337293625,-1.2383115972625092e-05,0.005632825195789337,-0.011708274483680725,0.01269055437296629,-0.011183304712176323,-0.05243301019072533,-0.08782224357128143,-0.034690022468566895,0.08260893076658249,0.09464582055807114,-0.03661808371543884,-0.012633657082915306,-0.016706425696611404,0.005854385439306498,-0.0068300762213766575,-0.03366502746939659,-0.019697822630405426,-0.06350100785493851,-0.05017933249473572,-0.018156686797738075,0.0003148428222630173,0.012922445312142372,0.013956605456769466,0.012912306003272533,-0.053699515759944916,0.09405864775180817,0.039684001356363297,-0.015801001340150833,0.009704023599624634,0.0042159175500273705,0.014777122065424919,0.027026081457734108,0.04424402490258217,0.045260775834321976,0.06699145585298538,0.05103264003992081,0.014280534349381924,-0.009232155978679657,0.0010084682144224644,0.009837860241532326,0.02278927154839039,-0.005587826017290354,0.021435828879475594,-0.03320985659956932,-0.01755640096962452,0.031800806522369385,-0.005420750472694635,0.019485102966427803,0.008618392050266266,0.039455756545066833,0.027855364605784416,0.04003989323973656,0.013352463021874428,-0.06448543816804886,0.012387103401124477,0.025186294689774513,0.01887282356619835,0.0055850353091955185,0.0064216116443276405,0.01768563687801361,0.005419604945927858,0.033780355006456375,0.008476964198052883,0.006264130584895611,0.03602079302072525,0.02377491444349289,0.02969076670706272,-0.060399819165468216,-0.061967916786670685,-0.030044790357351303,0.12130268663167953,0.004048209637403488,0.0005669641541317105,-0.005859861616045237,0.022400377318263054,0.006137377116829157,-0.010772798210382462,-0.010255779139697552,-0.03633556142449379,-0.025082161650061607,-0.03046177141368389,-0.029147394001483917,-0.09032991528511047,-0.09360188245773315,0.07119237631559372,0.07445354014635086,0.01884239912033081,0.010007091797888279,-0.002526373602449894,-0.024889769032597542,-0.054089341312646866,0.000303660926874727,-0.004426295403391123,-0.016387417912483215,-0.034753259271383286,-0.05119795724749565,-0.037566620856523514,-0.09905236214399338,-0.07116705179214478,0.11598580330610275,0.11676919460296631,0.033301614224910736,0.008951899595558643,-0.00658645061776042,0.004004702437669039,0.009917089715600014,-0.01305923517793417,0.024062365293502808,0.0019422938348725438,-0.0357896164059639,-0.05199192836880684,-0.05427829548716545,-0.041898854076862335,-0.016568053513765335,0.09062937647104263,-0.027781851589679718,0.028614675626158714,-0.0025802634190768003,0.055289726704359055,0.021618248894810677,-0.03622308373451233,0.013063323684036732,-0.027594275772571564,-0.015261244028806686,-0.025161802768707275,-0.014438553713262081,0.002057356759905815,0.022751281037926674,0.04322964325547218,-0.12369180470705032,0.0004996996140107512,0.0679653137922287,0.04244536906480789,0.046459048986434937,0.02114293724298477,0.04213527590036392,0.04220610857009888,0.054431889206171036,0.06018141657114029,-0.036999646574258804,0.04904709756374359,0.10271641612052917,-0.021155616268515587,-0.04905085638165474,0.035493288189172745,0.026236414909362793,-0.0032747434452176094,0.018729032948613167,0.020846085622906685,-0.042867206037044525,-0.07425066828727722,-0.0814436748623848,-0.012460040859878063,0.0699155330657959,0.2008974403142929,0.12282922863960266,-0.07610638439655304,-0.05940327048301697,-0.02799377590417862,0.07609880715608597,0.024624023586511612,0.0390436127781868,0.06569073349237442,-0.06228114664554596,-0.03337765112519264,0.025706972926855087,-0.02793310210108757,0.000516651023644954,-0.01038855966180563,0.01579216867685318,-0.03184516355395317,-0.003883598605170846,-0.17484639585018158,-0.2129206359386444,-0.06466397643089294,-0.12815213203430176,-0.009006532840430737,-0.013226493261754513,0.032145772129297256,-0.030728012323379517,-0.024075858294963837,-0.04648258164525032,0.018244339153170586,0.007451681420207024,-0.015204407274723053,-0.012223713099956512,0.02526276931166649,-0.1999179720878601,-0.20385116338729858,0.1431330144405365,0.09569183737039566,0.015940265730023384,-0.00285102310590446,-0.018519267439842224,0.03052337095141411,0.016334807500243187,0.022306108847260475,0.09334888309240341,0.08500320464372635,0.07247290760278702,0.022227196022868156,0.06653016060590744,-0.053318463265895844,-0.04445794224739075,0.02111450396478176,0.12009390443563461,-0.008502312004566193,0.009139233268797398,-0.006169437896460295,-0.03152458369731903,0.0027251343708485365,-0.01781635358929634,-0.015766555443406105,0.0018789094174280763,-0.09404724836349487,-0.09405659139156342,-0.0699772834777832,-0.09640995413064957,-0.04486609622836113,0.023850636556744576,-0.01541417371481657,0.016061559319496155,-0.02034919336438179,0.006783000193536282,-0.011188683100044727,-0.05702543631196022,0.046288833022117615,0.04722260683774948,0.04800772666931152,0.0343734435737133,-0.0038690457586199045,-0.00759310694411397,0.0636598989367485,0.0765727236866951,-0.05137530714273453,-0.024569885805249214,-0.012633733451366425,-0.012409688904881477,0.09112878888845444,-0.012641570530831814,0.014558717608451843,0.04239673167467117,0.05225488170981407,-0.005417924374341965,-0.13778358697891235,-0.08684936165809631,0.0032129769679158926,0.07715631276369095,0.01697571948170662,0.1605330854654312,0.026043904945254326,-0.13702772557735443,-0.04258185252547264,-0.04151517525315285,-0.10592132806777954,-0.04305613413453102,-0.011044962331652641,-0.04383455216884613,-0.10017465054988861,-0.13250820338726044,-0.08641991764307022,0.0617302767932415,0.059902217239141464,0.02439969964325428,-0.014207038097083569,-0.05324753373861313,0.047247789800167084,-0.06331691890954971,-0.05171172693371773,0.028109483420848846,0.04918011277914047,-0.013023610226809978,0.0016022383933886886,0.08567860722541809,0.06047052517533302,0.15614987909793854,0.11588876694440842,-0.04927179589867592,0.04212775453925133,-0.04933364689350128,0.03351037576794624,0.02549860253930092,-0.03446432575583458,0.014097166247665882,0.04593803733587265,0.014710024930536747,-0.01584213599562645,0.05309710651636124,0.07702634483575821,-0.015780985355377197,-0.01050171721726656,0.08830628544092178,0.21947738528251648,0.3123164474964142,-0.11436212062835693,0.05577157437801361,0.043960824608802795,0.0276560727506876,0.05580577626824379,-0.005114040803164244,0.029845068231225014,0.029703408479690552,0.03437308594584465,0.000780405243858695,-0.0838000625371933,0.05060361698269844,0.15637202560901642,0.1858912855386734,0.26653367280960083,-0.12905724346637726,-0.09841093420982361,-0.08133187144994736,-0.041622862219810486,-0.06524843722581863,-0.0908282920718193,-0.044962383806705475,0.003943218383938074,-0.021545754745602608,-0.1285324990749359,-0.15172791481018066,-0.033817827701568604,0.09565208852291107,0.0643923431634903,0.12148914486169815,-0.12518411874771118,-0.2028277963399887,-0.02647521160542965,0.10882695019245148,0.054854393005371094,-0.012065844610333443,0.03310234472155571,-0.00814872607588768,-0.022957991808652878,-0.04424378648400307,-0.09600616991519928,-0.018952419981360435,0.1111828088760376,0.2265302687883377,0.24209192395210266,-0.07326038926839828,-0.12453185021877289,-0.02009468898177147,-0.03720021992921829,-0.06758967787027359,0.016413716599345207,0.06116345524787903,0.04543657228350639,-0.025114785879850388,-0.03491205722093582,0.04209785908460617,0.005828904453665018,-0.07594149559736252,-0.13668090105056763,-0.11088578402996063,0.03374417871236801,-0.17912907898426056,-0.036145299673080444,-0.00362368649803102,0.0389019139111042,-0.05541828274726868,0.013143360614776611,0.0773455947637558,-0.048265304416418076,-0.034097377210855484,-0.07553543150424957,-0.24215096235275269,-0.07372858375310898,-0.047844935208559036,-0.2474737912416458,0.03295346349477768,0.02314911037683487,0.15025638043880463,-0.07346706092357635,0.018693849444389343,-0.020299552008509636,0.03951149061322212,0.012035111896693707,0.015556097030639648,-0.08958655595779419,0.03867821767926216,-0.006065695080906153,-0.011436272412538528,-0.0016457809833809733,-0.005105283576995134,0.001913450425490737,0.01898857206106186,0.006782500538975,-0.015379663556814194,-0.1911546289920807,0.4989303648471832,-0.2583763599395752,-0.17385053634643555,-0.16686387360095978,-0.16974705457687378,-0.007702530361711979,0.3958096206188202,0.33749911189079285,-0.050441037863492966,0.10230699181556702,-0.15661895275115967,0.12893156707286835,0.059168703854084015,-0.16845910251140594,0.18968869745731354,0.527280330657959,-0.27073952555656433,-0.10973590612411499,0.1198183223605156,0.16820381581783295,0.34342682361602783,-0.5733219981193542,0.1922038495540619,0.0660361796617508,0.06643257290124893,-0.12720750272274017,-0.23949991166591644,0.019153030589222908,-0.20022505521774292,-0.05184537544846535,0.0036745339166373014,0.15908488631248474,0.14605511724948883,0.009559208527207375,-0.24306952953338623,-0.10886012017726898,-0.375943660736084,-0.13521762192249298,0.07959312945604324,0.14437739551067352,-0.22665832936763763,0.16893106698989868,0.06274231523275375,0.14294983446598053,0.25122877955436707,0.1299838125705719,0.05796927958726883,-0.042061273008584976,0.6377348303794861,-0.32525357604026794,-0.2281484454870224,-0.21313656866550446,0.08129087090492249,-0.13185204565525055,0.11930844187736511,0.26630428433418274,-0.5213972330093384,0.1533105969429016,-0.05467516556382179,0.008790643885731697,-0.15712371468544006,-0.5434730052947998,0.13307052850723267,-0.2698586881160736,-0.167836531996727,0.03053177148103714,-0.11220739781856537,0.10381901264190674,0.17233292758464813,0.22368469834327698,-0.02731272764503956,0.0003847597981803119,0.0002025673456955701,-0.0005968130426481366,-0.005686185322701931,-0.015172899700701237,-0.03256845101714134,-0.028957830742001534,-0.028850452974438667,-0.005380199756473303,-0.014252711087465286,-0.04855092987418175,-0.03707082197070122,-0.06950711458921432,0.03610139340162277,-0.052632562816143036,-0.03819306567311287,-0.04373105242848396,-0.0576605424284935,-0.032655179500579834,-0.04069929197430611,-0.035724908113479614,-0.03608335554599762,-0.029499046504497528,-0.030341703444719315,-0.021372584626078606,-0.00931282714009285,-0.0419493094086647,-0.04062116518616676,0.06385928392410278,-0.035421986132860184,0.0011597559787333012,-0.0016503602964803576,-0.0087974164634943,-0.027519021183252335,-0.019804049283266068,-0.017613694071769714,-0.012969267554581165,-0.03149194270372391,-0.043276961892843246,-0.041496120393276215,-0.030537910759449005,-0.013657603412866592,-0.0527033731341362,0.07405754178762436,-0.04531211405992508,-0.06597504019737244,-0.04976290836930275,-0.05495257303118706,-0.03965674340724945,-0.04251749441027641,-0.04614423215389252,-0.058051347732543945,-0.03588768094778061,-0.047998737543821335,-0.051947589963674545,-0.03170478716492653,-0.08568184077739716,-0.08915887773036957,0.16121618449687958,-0.027529286220669746,-0.022070836275815964,-0.015872035175561905,-0.015007054433226585,-0.035404372960329056,-0.023025711998343468,-0.012977361679077148,-0.027432965114712715,-0.027789216488599777,-0.026996392756700516,-0.053266339004039764,-0.03632933273911476,-0.048963166773319244,-0.014102356508374214,0.09886056929826736,-0.015669139102101326,-0.026960112154483795,-0.030136022716760635,-0.01363524328917265,-0.03529401496052742,-0.03934936970472336,-0.02377624437212944,-0.0433259978890419,-0.02631218731403351,-0.02498074620962143,-0.013204936869442463,-0.021256649866700172,-0.027059873566031456,-0.03442781791090965,0.14240235090255737,-0.020315831527113914,-0.013099526055157185,0.0020409259013831615,0.00341456918977201,-0.008121552877128124,0.011321792379021645,0.005491833668202162,-0.007060042582452297,-0.006039062049239874,-0.016253432258963585,-0.016273727640509605,0.01885836198925972,0.06692717969417572,0.004178709350526333,-0.038882289081811905,-0.026844242587685585,-0.01642383635044098,-0.008139500394463539,-0.02561834640800953,-0.021818330511450768,-0.0046693915501236916,-0.01769920624792576,-0.02436772547662258,0.023495234549045563,0.0022939718328416348,-0.01364198699593544,0.007850591093301773,-0.0032618239056319,0.0036786627024412155,-0.018461918458342552,-0.02675444632768631,-0.01623074896633625,0.01631895639002323,0.0018762757536023855,-0.008690256625413895,-0.017751086503267288,-0.03924376890063286,-0.034709203988313675,-0.024643464013934135,-0.023173600435256958,-0.020854774862527847,-0.037344127893447876,-0.03339289873838425,-0.02421504072844982,0.005039446987211704,-0.07206223160028458,-0.07489041239023209,-0.06903980672359467,-0.07199300825595856,-0.06057683378458023,-0.043299827724695206,-0.03790188953280449,-0.033530283719301224,-0.06127605214715004,-0.050769656896591187,-0.011942666955292225,-0.030923768877983093,-0.05754953250288963,-0.03166356310248375,0.13157591223716736,-0.011857160367071629,-0.000439525640103966,-0.00550020532682538,-0.0010504539823159575,0.0013633273774757981,-0.038157835602760315,-0.0046639516949653625,-0.01323426328599453,-0.027473757043480873,-0.021949904039502144,-0.026838546618819237,-0.020833957940340042,-0.04354383423924446,-0.018903544172644615,0.02343989722430706,-0.04210372641682625,-0.04370225593447685,-0.04066857323050499,-0.05767348036170006,-0.04601634293794632,-0.022545550018548965,-0.043853286653757095,-0.08569125831127167,-0.05402249097824097,-0.045231111347675323,-0.07879488170146942,-0.05780917778611183,-0.05943315476179123,-0.03879617899656296,0.10452710092067719,-0.02859879471361637,-0.007192287594079971,-0.010390875861048698,0.0022069441620260477,-0.022593822330236435,-0.028780095279216766,-0.01096052210777998,-0.013212976045906544,0.0057943714782595634,-0.027974456548690796,-0.032183002680540085,-0.009649860672652721,-0.013386651873588562,-0.045400381088256836,0.11289513111114502,-0.02718566544353962,-0.03610913082957268,-0.05012928694486618,-0.03507176786661148,-0.03715195879340172,-0.05835316330194473,-0.028984570875763893,-0.027351178228855133,-0.014961129985749722,-0.0032146438024938107,-0.022233257070183754,0.004894159268587828,-0.06694118678569794,-0.02058415487408638,0.13342593610286713,-0.008008631877601147,0.034543078392744064,-0.0026641099248081446,0.01592221111059189,0.03213882073760033,-0.006013339385390282,0.015453863888978958,0.013343056663870811,0.014837611466646194,-0.00729095796123147,-0.017350956797599792,0.002532800193876028,0.056589748710393906,0.027629278600215912,-0.05129623785614967,-0.021835485473275185,0.0030281522776931524,-0.019259091466665268,-0.018240947276353836,-0.010227302089333534,0.01782318763434887,0.0004870293487329036,-0.00697367126122117,0.024058492854237556,-0.014804973267018795,0.025725599378347397,0.025434650480747223,0.013501688838005066,0.022357085719704628,0.035747744143009186,-0.0187082402408123,-0.007090229541063309,-0.015489708632230759,-0.005930323153734207,-0.009893577545881271,-0.016656989231705666,-0.013684076257050037,-0.01799461990594864,-0.025248082354664803,-0.013497349806129932,0.002766847610473633,0.010732556693255901,0.10283198207616806,0.23426543176174164,0.12085016071796417,-0.023600907996296883,-0.021887587383389473,-0.021187670528888702,0.004582571797072887,-0.013224197551608086,-0.013537846505641937,-0.029975665733218193,-0.02562306448817253,-0.005572075489908457,-0.0216959398239851,-0.007683716248720884,-0.013692203909158707,-0.0602504201233387,-0.06055460870265961,-0.15699099004268646,0.01621411368250847,0.0010148726869374514,0.010232468135654926,0.009275061078369617,0.006710771471261978,0.005752404220402241,0.01694796048104763,-0.002099730772897601,0.010323612950742245,0.02363624982535839,-0.002303563989698887,0.01608552783727646,0.0034289455506950617,-0.1651889532804489,0.029756808653473854,0.011290284804999828,0.011514165438711643,0.0010427591623738408,-0.005329023580998182,-0.0014707937370985746,0.0025837128050625324,0.013085602782666683,0.006951478309929371,0.008580491878092289,0.006736882962286472,0.001162096974439919,-0.0051905312575399876,-0.008348710834980011,0.053918465971946716,0.008262988179922104,0.013472695834934711,0.0032401399221271276,0.01821018196642399,0.004564602859318256,0.021090151742100716,0.01567355915904045,0.0009987782686948776,0.017092376947402954,0.004795118235051632,0.011495530605316162,0.008582878857851028,0.01306413859128952,-0.0018734729383140802,0.00017919241508934647,-0.10702482610940933,-0.02381746470928192,-0.0009900458389893174,-0.01153947226703167,0.005601809825748205,-0.023384416475892067,-0.0037469060625880957,-0.007688973098993301,-0.0029587873723357916,-0.024404898285865784,-0.03113248012959957,0.004413330927491188,-0.023771092295646667,0.012886451557278633,-0.07442435622215271,0.05949133262038231,-0.003248096676543355,-0.007962902076542377,0.002947211964055896,0.00017728078819345683,-0.033616479486227036,-0.007869131863117218,-0.0015245111426338553,0.0010132294846698642,-0.014664769172668457,0.006215312052518129,-0.00957330409437418,-0.012930012308061123,0.06783817708492279,0.13573575019836426,0.016209017485380173,-0.008563359268009663,0.0023956706281751394,-0.0013433793792501092,0.008027750067412853,0.004846462979912758,-0.0038076755590736866,-0.005459052510559559,0.001565398182719946,-0.00798715278506279,0.007037699688225985,0.027249416336417198,-0.009840189479291439,-0.11476525664329529,-0.0385754369199276,-0.033146489411592484,-0.021479230374097824,-0.010632001794874668,-0.032680753618478775,-0.00649596331641078,-0.009979445487260818,-0.02590847760438919,0.009996488690376282,0.018927203491330147,0.01747755892574787,-0.043302346020936966,-0.047527920454740524,-0.021573452278971672,-0.011092747561633587,0.06522026658058167,-0.07668203860521317,-0.01982824318110943,-0.042141687124967575,-0.03318380191922188,-0.009215221740305424,0.009058685973286629,-0.007061122916638851,-0.03050808049738407,0.003331966931000352,-0.0249672532081604,0.0018638280453160405,-0.003491292940452695,-0.030272575095295906,-0.06943131238222122,0.00990203395485878,-0.03665294870734215,0.005463126581162214,-0.02151774987578392,0.0049711368046700954,-0.017091065645217896,-0.0009261462255381048,0.003348472062498331,-0.0038761093746870756,-0.0036901128478348255,-0.0027396974619477987,0.0006772379856556654,-0.017539165914058685,-0.010630329139530659,-0.08630333840847015,-0.020765110850334167,-0.0568663589656353,-0.006309622898697853,0.006583129521459341,0.0024862890131771564,-0.0033704452216625214,-0.004135778173804283,-0.012062369845807552,-0.0031130488496273756,-0.016269436106085777,-0.020478732883930206,0.004236252047121525,0.002425196347758174,-0.01891494356095791,-0.031249480322003365,-0.00960059929639101,0.03899283707141876,0.011315407231450081,0.020529676228761673,0.00525312777608633,0.017735300585627556,0.011845353990793228,0.02839638851583004,0.00889117456972599,-0.006202235817909241,0.03460259735584259,0.030002973973751068,0.031243739649653435,0.033286478370428085,0.13416188955307007,-0.020779283717274666,0.05182620882987976,0.01230640709400177,0.010728895664215088,0.008124131709337234,-0.006554721388965845,-0.020749621093273163,0.017154105007648468,0.014091124758124352,0.014297105371952057,-0.002931946190074086,0.0037846574559807777,0.007457979954779148,0.06242837756872177,0.030248790979385376,0.06988177448511124,0.0909806489944458,-0.011507382616400719,-0.005351864267140627,0.020795611664652824,-0.007800856605172157,-0.006110112648457289,0.007186198607087135,0.0014472565380856395,0.010239195078611374,-0.002988530322909355,0.022676972672343254,0.038249142467975616,0.01856035739183426,-0.07838299125432968,0.10080520808696747,0.09526824951171875,-0.014563439413905144,-0.007195707876235247,-0.017990393564105034,0.005208006594330072,0.01850605197250843,-0.000771355873439461,-0.025064056739211082,-0.006196053232997656,0.0018822563579306006,0.0010325758485123515,0.007867231033742428,0.011473516933619976,-0.014755173586308956,0.035857539623975754,0.006009452976286411,0.08627159893512726,0.06745606660842896,0.07253469526767731,0.07222799211740494,0.07431495934724808,0.055014804005622864,0.09598129242658615,0.09747850894927979,0.07112286239862442,0.0718500167131424,0.05899898335337639,0.003182103391736746,0.03584073483943939,0.023813525214791298,-0.06296523660421371,0.07003229111433029,0.07055546343326569,0.05517553165555,0.06383330374956131,0.045088030397892,0.039829760789871216,0.06191256642341614,0.06332709640264511,0.07595151662826538,0.053753919899463654,0.06357062608003616,0.058142125606536865,0.06497615575790405,0.06259091198444366,-0.14117318391799927,0.044648535549640656,-0.003615713445469737,0.01843852549791336,0.009551444090902805,0.02479306049644947,0.06129780039191246,0.034249063581228256,0.03858552873134613,0.03096405789256096,0.02092999592423439,0.028630739077925682,0.01368743646889925,0.042285364121198654,0.09027989953756332,-0.12364166229963303,0.09074427932500839,0.10535120964050293,0.09725087881088257,0.10152319073677063,0.11007987707853317,0.09024103730916977,0.07714708894491196,0.11812634766101837,0.07563850283622742,0.09586028754711151,0.0830521211028099,0.10033673793077469,0.17766207456588745,0.1420292854309082,-0.2981865704059601,0.02303401567041874,0.04299771413207054,0.0265569556504488,0.045949552208185196,0.03836693987250328,0.027077892795205116,0.022025909274816513,0.013712608255445957,0.043567027896642685,0.034888364374637604,0.05677158385515213,0.04940364137291908,0.07527627050876617,0.054112572222948074,-0.19416770339012146,0.09508663415908813,0.05846070498228073,0.03534848242998123,0.018779629841446877,0.025040047243237495,0.06710489094257355,0.04315168038010597,0.045862309634685516,0.055998917669057846,0.04966014251112938,0.07062020152807236,0.016607660800218582,0.04443657398223877,0.058354735374450684,-0.10460290312767029,0.0187988318502903,0.014010211452841759,0.013714514672756195,0.011816812679171562,0.018158018589019775,-0.012439040467143059,-0.04165252298116684,-0.00984395295381546,0.0046908180229365826,0.03983664885163307,0.0011761768255382776,-0.022419480606913567,-0.019595975056290627,-0.03286297619342804,0.011234648525714874,0.026810627430677414,0.009541536681354046,-0.014442714862525463,0.003552571637555957,-0.005390006583184004,-0.025542540475726128,-0.02603515051305294,-0.010267399251461029,-0.04581262916326523,-0.050101447850465775,-0.03177414834499359,-0.04205179959535599,-0.05169513076543808,-0.05356000363826752,0.04400889202952385,-0.0035512966569513083,-0.011839860118925571,0.0056259045377373695,-0.0025706165470182896,-0.041096869856119156,-0.011778882704675198,0.002750223269686103,0.004920111503452063,-0.011527449823915958,-0.001656086533330381,-0.009909897111356258,0.008105712942779064,-0.04982231557369232,0.0005202456377446651,-0.002691322937607765,0.010809521190822124,0.010715518146753311,0.011588584631681442,0.010222634300589561,-0.005378421396017075,0.010059910826385021,0.023528702557086945,0.026470964774489403,0.015007609501481056,0.017520349472761154,0.013911415822803974,0.012365186586976051,0.05055542290210724,-0.015517384745180607,0.07088581472635269,-0.007653749547898769,-0.01284310407936573,-0.018865449354052544,-0.0042510563507676125,0.004097046796232462,-0.017164109274744987,-0.009486259892582893,-0.01010268833488226,-0.001803897088393569,0.002870628610253334,-0.01703825779259205,-0.04423189163208008,-0.0064401463605463505,-0.0057853031903505325,-0.004858046304434538,0.015385570004582405,0.013641302473843098,0.01762508414685726,0.015251281671226025,0.021358788013458252,0.010911543853580952,-0.00699745723977685,0.003260399214923382,0.014075626619160175,0.013286224566400051,0.02024507336318493,0.014568010345101357,0.011525824666023254,0.04872698709368706,0.129710853099823,-0.03189850598573685,-0.023216869682073593,-0.028819700703024864,-0.029774215072393417,-0.025467069819569588,-0.019970519468188286,-0.011906404048204422,-0.02992674708366394,-0.029268227517604828,-0.0365402027964592,-0.03215337544679642,-0.018015528097748756,-0.015576152130961418,-0.02626838907599449,0.07831315696239471,-0.0007418134482577443,0.018958892673254013,-0.00558440899476409,0.0037025369238108397,0.021537352353334427,0.010980640538036823,0.009567439556121826,0.01329229399561882,0.01152786798775196,0.0053076837211847305,0.017027802765369415,0.0030939707066863775,-0.05318405479192734,-0.07017862051725388,0.1566140055656433,0.0008659044397063553,0.007232097443193197,0.007578469812870026,0.008040889166295528,0.02478254772722721,-0.002619935665279627,0.0038225383032113314,0.009506349451839924,0.00933376420289278,-0.009667580015957355,0.0041231317445635796,0.021887756884098053,0.03990713506937027,0.08575630187988281,0.0551738440990448,-0.023393718525767326,-0.003544947598129511,-0.02756371907889843,-0.011876633390784264,0.005013743415474892,-0.028805488720536232,-0.02328234352171421,0.0025009645614773035,-0.0283858310431242,-0.010016931220889091,-0.006802803836762905,-0.017142891883850098,-0.05545145645737648,-0.082811638712883,-0.05218878760933876,-0.019175153225660324,-0.0013957791961729527,-0.015384470112621784,-0.011126967146992683,-0.01842436008155346,0.000350983755197376,0.00655706599354744,-0.008625992573797703,0.008657550439238548,-0.01092017162591219,-0.0020047477446496487,-0.005397455301135778,-0.0037358447443693876,-0.001214560936205089,0.0010000297334045172,2.368380137340864e-06,-0.0018671222496777773,0.0017684487393125892,0.01915445737540722,0.004877548664808273,0.013918246142566204,0.011342065408825874,0.013560655526816845,-0.00045665056677535176,-0.0011384588433429599,0.006744893733412027,0.006981884595006704,0.005668338388204575,0.002586015500128269,0.012560553848743439,0.0007320769364014268,-0.019191492348909378,0.01663522608578205,0.0007043377845548093,0.008737033233046532,-0.020282894372940063,-0.011916604824364185,-0.017358867451548576,0.004353290889412165,-0.012527208775281906,-0.008082372136414051,-0.019682221114635468,-0.024260949343442917,-0.020332103595137596,-0.026288239285349846,-0.013297206722199917,-0.007617452181875706,-0.004996770527213812,0.0004218959074933082,0.0018992128316313028,0.006687580142170191,-0.0024659710470587015,0.007441244088113308,0.013196246698498726,0.010393060743808746,-0.0009641062351875007,0.002343326574191451,0.015011911280453205,0.008740976452827454,0.0028699873946607113,-0.0012276602210476995,-0.0076031810604035854,-0.010455182753503323,-0.009554600343108177,-0.02186726965010166,-0.0035022441297769547,-0.01329139992594719,-0.0092370780184865,-0.008279227651655674,-0.010439637117087841,-0.004966130014508963,-0.012263912707567215,-0.018501410260796547,-0.02294134348630905,-0.028565283864736557,-0.013187538832426071,-0.025707479566335678,-0.011947031132876873,0.0015117865987122059,0.013263773173093796,-0.015496068634092808,-0.021185334771871567,-0.003382994793355465,-0.02081403136253357,0.001980019034817815,-0.014786259271204472,-0.0193796269595623,-0.03509021922945976,-0.018628794699907303,-0.028818506747484207,-0.0013151022139936686,0.024175364524126053,-0.005194886587560177,-0.00756129901856184,-0.01084953173995018,-0.0054831127636134624,0.02083835005760193,-0.013524601235985756,0.03633461892604828,-0.005265649873763323,0.014511690475046635,-0.004140202887356281,0.03370695561170578,0.024020995944738388,-0.02437051571905613,-0.02829836867749691,-0.030423421412706375,-0.02185841277241707,-0.008348610252141953,-0.027320703491568565,-0.011096795089542866,-0.01720258966088295,0.006105839740484953,-0.03094041347503662,-0.020877692848443985,-0.017847901210188866,-0.010820724070072174,0.016367396339774132,0.005545654334127903,-0.021762052550911903,-0.014747104607522488,-0.022099949419498444,-0.004252780694514513,-0.002329729963093996,-0.005199038423597813,-0.032265789806842804,-0.017397256568074226,-0.029825538396835327,-0.021392790600657463,-0.016278037801384926,0.004944225773215294,0.010056893341243267,0.005894689820706844,0.05128176882863045,0.09718065708875656,0.008210553787648678,0.015229745768010616,0.026019461452960968,0.02175156958401203,0.020754562690854073,-0.003812138456851244,0.01730261743068695,0.011226369999349117,0.0017469091108068824,0.040378473699092865,0.03457770496606827,0.030286187306046486,0.03981789946556091,0.040071215480566025,-0.0024496293626725674,-0.014528642408549786,-0.0016853369306772947,-0.020762184634804726,-0.005422373302280903,-0.007278650067746639,-0.016478698700666428,-0.012816423550248146,-0.010039297863841057,-0.018400562927126884,0.003624122589826584,-0.007866711355745792,-0.034341923892498016,-0.00517020421102643,-0.030180536210536957,0.013246046379208565,0.02251942828297615,0.00959511287510395,0.01931171491742134,0.005323116667568684,0.005446222145110369,0.024870283901691437,0.025215845555067062,0.020104849711060524,0.03289607912302017,0.005684259347617626,0.0010129540460184216,0.006057118531316519,0.003377840854227543,-0.0032008171547204256,0.15327122807502747,-0.02846387028694153,-0.02011715993285179,-0.028656238690018654,-0.02847893536090851,-0.01737876608967781,-0.0123314019292593,-0.0411161445081234,-0.02542373724281788,-0.03599000349640846,-0.036322519183158875,-0.036599691957235336,-0.01891503669321537,1.3289813978190068e-05,-0.03275509923696518,-0.017082421109080315,0.0012681351508945227,0.0002881360414903611,-0.00036323058884590864,0.014920596964657307,0.009305908344686031,-0.005726468283683062,0.005862466059625149,0.011510231532156467,0.004412891808897257,0.0029051578603684902,0.01863343082368374,0.014824766665697098,-0.03942729905247688,-0.017938263714313507,-0.08620977401733398,0.017780637368559837,0.015606525354087353,0.011502441018819809,0.01468151155859232,0.009498391300439835,0.004913792014122009,0.015796752646565437,0.03645903989672661,0.030574653297662735,0.0002637552679516375,0.010073996149003506,0.013499573804438114,0.027150457724928856,0.07049630582332611,-0.11439332365989685,-0.02444012276828289,-0.0063120038248598576,-0.009455211460590363,-0.013528442941606045,-0.004379219841212034,-0.03255321830511093,-0.019362103193998337,3.395947351236828e-05,-0.007625954691320658,-0.016914134845137596,-0.002263459609821439,-0.00422428036108613,-0.015243341214954853,-0.07964702695608139,-0.07896038889884949,0.061642687767744064,0.024442942813038826,0.14105361700057983,0.023637155070900917,-0.03337330371141434,-0.10956141352653503,-0.031005946919322014,-0.1831607073545456,-0.0007502210210077465,-0.004264688119292259,0.012955387122929096,0.008327063173055649,0.009250442497432232,-0.0035307754296809435,0.002313669305294752,0.006468155886977911,0.25671640038490295,0.237216979265213,0.06882791966199875,0.12190469354391098,-0.28149527311325073,-0.006519104819744825,0.007304952014237642,0.18087682127952576,-0.08410964161157608,-0.055838171392679214,-0.08557451516389847,-0.053994353860616684,0.08776145428419113,-0.12935473024845123,-0.015488713048398495,-0.08749568462371826,0.04846764728426933,0.1413854956626892,-0.08934034407138824,0.1397477686405182,-0.10239721089601517,0.180202916264534,-0.0033381925895810127,-0.10081317275762558,-0.2633430063724518,-0.1487269550561905,0.1365818828344345,-0.0743863508105278,0.19604019820690155,0.04290677607059479,-0.054780591279268265,0.07700330018997192,-0.033838193863630295,-0.21244403719902039,-0.09219452738761902,-0.05042899027466774,0.14213277399539948,-0.16267870366573334,-0.052660778164863586,0.03816885128617287,0.0014109337935224175,-0.05781257525086403,-0.05514617636799812,-0.033848900347948074,0.04184583202004433,0.04424460604786873,-0.018208811059594154,-0.2133096605539322,-0.043474696576595306,-0.08155197650194168,0.23998355865478516,0.02152159810066223,0.06806764751672745,-0.07373303174972534,0.010673865675926208,0.23070329427719116,-0.03557707741856575,0.056290142238140106,-0.08162044733762741,-0.07129549235105515,-0.017193136736750603,-0.01809229515492916,0.03389119356870651,-0.05930395796895027,-0.006247926969081163,-0.16257193684577942,-0.3435472846031189,0.17376840114593506,0.019070785492658615,0.045757368206977844,0.004400050733238459,0.34262773394584656,-0.0016028628451749682,0.011093361303210258,-0.008423924446105957,0.019545625895261765,0.11421450972557068,-0.008549237623810768,-0.015857413411140442,-0.033273737877607346,0.02962474152445793,-0.058064572513103485,-0.0019788402132689953,-0.007519792299717665,0.06581265479326248,0.07630176097154617,0.08867353945970535,-0.048511747270822525,-0.05227435380220413,0.019695965573191643,-6.951155228307471e-05,0.010658757761120796,0.025942876935005188,0.010171902365982533,0.025642458349466324,0.000639565521851182,0.028321733698248863,0.07753625512123108,0.0760558694601059,-0.0437285341322422,0.09406787902116776,0.0333808995783329,0.03427211567759514,0.0605788454413414,0.0044797626323997974,0.0028861970640718937,0.01941077969968319,-0.022091569378972054,-0.034456245601177216,-0.018863068893551826,-0.05649777874350548,0.003423749003559351,0.043792515993118286,0.09170754253864288,-0.002428269712254405,-0.008047527633607388,0.05650623142719269,-0.0744512602686882,-0.16558632254600525,-0.005757224280387163,0.0071420748718082905,-0.024433551356196404,0.03744414821267128,0.06845684349536896,0.06415406614542007,0.013302006758749485,-0.12072259187698364,-0.03512592986226082,-0.09196485579013824,-0.0032059785444289446,-0.18498069047927856,-0.1504044085741043,0.03877934440970421,0.10693338513374329,0.03395048901438713,0.1174534261226654,-0.05525593459606171,-0.07920310646295547,0.05279180780053139,-0.10462719202041626,0.07334540039300919,0.21392251551151276,-0.04884985834360123,-0.032504137605428696,0.07407324016094208,-0.0021985098719596863,0.14624811708927155,0.24771028757095337,0.08478032052516937,-0.03411414101719856,0.04566739872097969,0.13312318921089172,0.10122982412576675,0.08269346505403519,0.08945026993751526,0.005522702354937792,-0.07562074810266495,0.03100632317364216,-0.07502780854701996,-0.10622989386320114,0.14936567842960358,-0.003951127640902996,-0.05916174128651619,-0.05657517910003662,-0.03601321950554848,0.10450855642557144,-0.07494261860847473,-0.10162454843521118,0.12840856611728668,0.10196354240179062,-0.10971882939338684,-0.0735720843076706,-0.01724441535770893,0.17116422951221466,0.020717475563287735,-0.11827795207500458,-0.11795785278081894,-0.08940500766038895,0.10092516243457794,0.11678051203489304,0.09691791981458664,-0.03678121045231819,-0.20323562622070312,-0.19397272169589996,0.12802660465240479,-0.11540794372558594,0.04495599865913391,-0.00771827157586813,-0.07501312345266342,0.01284127589315176,-0.02885255590081215,0.12720423936843872,0.20014746487140656,0.09949366748332977,-0.11612594127655029,-0.11115755885839462,-0.008100227452814579,-0.04573100432753563,-0.06948578357696533,-0.03616661578416824,0.19502761960029602,0.24984857439994812,0.06757368892431259,-0.021231167018413544,0.11739377677440643,0.1359819918870926,-0.14263516664505005,-0.3096913993358612,-0.046340085566043854,0.15737268328666687,0.28272947669029236,0.06390459835529327,-0.30296266078948975,-0.0937785878777504,0.232670858502388,-0.006055028643459082,-0.024172062054276466,0.030204907059669495,0.04482339322566986,0.0831451416015625,0.12618803977966309,0.07706788927316666,-0.022595105692744255,-0.12286429107189178,-0.0731736272573471,-0.04421606287360191,0.06095113977789879,0.07221612334251404,-0.04474908858537674,-0.09579788148403168,-0.14643605053424835,-0.23735757172107697,-0.010164075531065464,-0.03157355263829231,-0.09314022213220596,-0.07378066331148148,-0.04537404701113701,-0.06708940118551254,-0.008365224115550518,0.06038881838321686,0.12679162621498108,0.14093467593193054,0.1016363799571991,-0.05389147251844406,0.1713719218969345,0.13566406071186066,0.03561747446656227,-0.03623921051621437,0.05785944685339928,-0.20803555846214294,-0.057656191289424896,0.04246002808213234,-0.005732519086450338,0.11117588728666306,-0.03153832629323006,0.048831433057785034,0.2754836976528168,0.013667874969542027,0.07715897262096405,0.17460884153842926,0.08835051953792572,0.04305007681250572,-0.07165481150150299,0.015535549260675907,0.0710982233285904,0.018689988180994987,0.03434889391064644,0.029833214357495308,0.06801806390285492,0.015164712443947792,-0.03364141657948494,-0.07830510288476944,-0.01941516436636448,-0.13917383551597595,0.061811260879039764,0.06674932688474655,-0.08989470452070236,-0.03681601956486702,-0.13788537681102753,-0.23077936470508575,0.06616293638944626,0.02797566168010235,-0.0004603115376085043,0.129752978682518,-0.11337825655937195,-0.09291861951351166,-0.017379838973283768,-0.03213529288768768,0.01140605378895998,0.09519656002521515,0.036732982844114304,0.10684508085250854,0.052078720182180405,0.05383756384253502,-0.06482657790184021,-0.009518987499177456,0.04071537405252457,-0.0733882337808609,0.06650599092245102,0.027972808107733727,-0.022834379225969315,0.031204497441649437,0.10107965022325516,0.08919941633939743,-0.015793975442647934,0.07079973816871643,0.10922934114933014,0.10138913989067078,0.011725726537406445,0.0867946445941925,0.11358314007520676,-0.07174186408519745,-0.0028755266685038805,-0.046640198677778244,0.007236640900373459,-0.02588953822851181,-0.023995984345674515,0.11057834327220917,0.024844830855727196,0.09610400348901749,0.02478271909058094,0.031433820724487305,0.007496674545109272,-0.025089627131819725,-0.10807503759860992,-0.04003845155239105,-0.07228554785251617,-0.04106764867901802,0.1419648975133896,0.030156945809721947,-0.054986607283353806,-0.10162670910358429,-0.04966449365019798,-0.023719049990177155,0.08983146399259567,0.21262697875499725,-0.057143330574035645,-0.05826783552765846,-0.048631276935338974,-0.04370821267366409,0.011232774704694748,-0.003681453410536051,0.014525237493216991,0.00449365796521306,0.02142420969903469,0.006539961788803339,0.0024417478125542402,0.04070639982819557,0.0389762818813324,0.027716422453522682,0.1414526104927063,-0.005688903387635946,-0.0669151246547699,-0.06689692288637161,0.005643829237669706,0.01589290425181389,-0.0731162428855896,0.12495144456624985,0.11376602202653885,0.07517790049314499,-0.024185894057154655,0.022717434912919998,-0.0002334756136406213,0.032467108219861984,0.10817878693342209,0.10047568380832672,-0.07592149078845978,-0.04699881374835968,0.017104173079133034,-0.17198660969734192,-0.2104315161705017,0.08769699931144714,0.02874135598540306,-0.05537935718894005,-0.05302293598651886,-0.22367745637893677,-0.2434222549200058,0.02666609175503254,0.031505633145570755,-0.06991641223430634,-0.04858087748289108,-0.10867191106081009,-0.07724127918481827,0.010599788278341293,-0.07418258488178253,0.1140265092253685,-0.001295661204494536,0.029288439080119133,0.20203903317451477,0.14271295070648193,-0.006059643812477589,0.09718652069568634,0.04940718412399292,0.08594366163015366,-0.05547688156366348,0.1845308095216751,-0.0035977482330054045,0.03894748538732529,0.07296036183834076,0.08965934067964554,0.11886852234601974,0.04135220870375633,-0.03933762013912201,0.15815624594688416,0.009497220627963543,0.11700505018234253,0.22597269713878632,0.06130187585949898,0.04917217791080475,0.01695866882801056,-0.08484011888504028,0.033624131232500076,-0.19918470084667206,0.2028811275959015,0.05903993546962738,0.24882207810878754,0.19817982614040375,0.13919468224048615,-0.12428746372461319,-0.25427761673927307,-0.12410831451416016,0.011134062893688679,0.1673841029405594,0.193581223487854,-0.20412036776542664,-0.08325064182281494,-0.08198650926351547,0.07390214502811432,-0.11290635913610458,0.0986378937959671,-0.3305661678314209,0.28990060091018677,-0.04004834592342377,0.05144284665584564,-0.031272225081920624,0.012628492899239063,-0.14499394595623016,0.14335140585899353,-0.03343139961361885,-0.0341191366314888,0.027043744921684265,-0.022252779453992844,-0.08049309253692627,-0.20043091475963593,0.03734199330210686,-0.0481884628534317,0.1488109678030014,-0.17891235649585724,0.07088196277618408,0.0650949478149414,0.03267790749669075,0.02410992793738842,-0.04046798497438431,0.07323727011680603,0.002364127663895488,-0.06967154145240784,-0.031460072845220566,0.32943397760391235,-0.11949893087148666,0.2451779693365097,0.18545101583003998,-0.05840824544429779,0.3241754174232483,-0.1968894600868225,-0.010468798689544201,0.030045874416828156,-0.003292028559371829,0.023835593834519386,0.002025305526331067,-0.0012293874751776457,-0.018943455070257187,-0.013621630147099495,-0.12367576360702515,-0.15976248681545258,0.34613174200057983,0.028717422857880592,-0.08387225866317749,0.19834411144256592,-0.22780291736125946,0.2835622727870941,-0.13188476860523224,0.18185333907604218,-0.03904455527663231,0.07278301566839218,-0.03842273727059364,-0.09206788241863251,-0.11046791821718216,-0.027072550728917122,0.11095515638589859,-0.3987448513507843,0.15507447719573975,0.3865607976913452,0.22766493260860443,-0.12223134934902191,0.15370376408100128,-0.5919889211654663,-0.14825887978076935,0.1328139752149582,-0.027630742639303207,0.022213466465473175,0.17568963766098022,-0.005445271730422974,-0.19978217780590057,0.0392245277762413,0.20598484575748444,0.22150732576847076,0.166180819272995,-0.36119401454925537,-0.3944496810436249,0.2546522915363312,-0.3888353705406189,0.07672042399644852,-0.11503329873085022,-0.118575319647789,-0.23246850073337555,0.14507734775543213,0.05791623517870903,0.0923364907503128,-0.1737484633922577,-0.06168600916862488,-0.34944674372673035,-0.15037356317043304,0.5217400789260864,0.01487204059958458,-0.07032483071088791,0.21572494506835938,0.07017668336629868,0.2669895887374878,0.0956084132194519,-0.12362423539161682,-0.33914509415626526,-0.07973150908946991,-0.1189228892326355,-0.08785392343997955,0.02474178932607174,-0.30968761444091797,0.17182539403438568,0.17243948578834534,-0.15098656713962555,-0.026244670152664185,0.36132919788360596,0.08702090382575989,0.04548963904380798,-0.12039526551961899,-0.05606228858232498,-0.007979126647114754,0.012713217176496983,0.0031267241574823856,-0.0026457190979272127,-0.058966562151908875,-0.10209119319915771,-0.04254074767231941,-0.0040117958560585976,0.0379180833697319,-0.014285169541835785,0.13414131104946136,0.047455795109272,-0.01739862561225891,0.026552122086286545,0.0804593563079834,0.034723617136478424,0.08055513352155685,-0.1212739422917366,-0.10695593059062958,0.020106175914406776,-0.0011782676447182894,0.12369012832641602,0.18164759874343872,-0.08933009207248688,-0.035388294607400894,-0.12729217112064362,0.06406820565462112,-0.026017075404524803,0.03791746124625206,0.005303739104419947,-0.00039789179572835565,0.020975202322006226,0.03262675553560257,0.04855440929532051,0.10654101520776749,0.03025694750249386,0.06284435838460922,0.029755650088191032,0.006844032555818558,0.020832141861319542,-0.05040333792567253,0.008666476234793663,0.03375934064388275,-0.04452652111649513,0.035567544400691986,0.005683983210474253,-0.04892709478735924,-0.06030179187655449,-0.0552535243332386,-0.06752559542655945,-0.051797423511743546,0.16472725570201874,0.101371631026268,-0.09755919128656387,-0.11105722188949585,-0.07805934548377991,0.004376168362796307,0.04982968419790268,0.2163010984659195,-0.06661344319581985,-0.04445938393473625,-0.0029737488366663456,0.10464824736118317,-0.03713272884488106,-0.0827505961060524,-0.14892661571502686,-0.10362518578767776,-0.08561635762453079,-0.017475072294473648,0.1826981157064438,0.167080357670784,-0.17536617815494537,-0.1156359538435936,-0.09060465544462204,-0.0018521909369155765,0.20511379837989807,0.19689317047595978,-0.05274170637130737,-0.03475916385650635,-0.03703708574175835,-0.001512897782959044,0.0980939194560051,0.08424720168113708,0.07629912346601486,0.08368703722953796,-0.003239694982767105,-0.018627803772687912,3.34090473188553e-05,-0.052615776658058167,-0.017653921619057655,-0.0314563512802124,0.04646718502044678,0.057060420513153076,-0.023545093834400177,-0.08286931365728378,0.0047780293971300125,-0.051126908510923386,-0.044978756457567215,0.08893504738807678,-0.13652212917804718,0.20680472254753113,-0.06353926658630371,0.05390481278300285,-0.07821480929851532,-0.017301112413406372,-0.02904447168111801,-0.04647429659962654,-0.04613146558403969,0.004722855519503355,0.057775601744651794,-0.05861474573612213,0.10034767538309097,-0.19285449385643005,-0.015792984515428543,-0.044442422688007355,0.04666414484381676,0.015094825997948647,0.062233686447143555,0.10212287306785583,0.008181021548807621,0.008758053183555603,-0.06543956696987152,0.14732427895069122,-0.15273672342300415,0.042230308055877686,0.07445540279150009,0.058347705751657486,0.12672275304794312,0.035143665969371796,0.10861314088106155,0.09070361405611038,-0.09555061906576157,-0.07828608900308609,-0.09993616491556168,-0.04842441901564598,-0.17568881809711456,0.09599117934703827,0.03101121447980404,0.09975674748420715,-0.08983203768730164,0.203499898314476,0.04119249805808067,-0.028016066178679466,-0.05758758634328842,-0.0521724671125412,0.06243205815553665,-0.0844510942697525,0.012949001044034958,0.20903092622756958,0.02005906216800213,0.10279221087694168,0.011387674137949944,-0.01064244657754898,-0.13483506441116333,-0.0031662448309361935,0.0227355919778347,0.07453830540180206,0.008584178052842617,-0.10455437004566193,0.043171048164367676,-0.17168907821178436,-0.05129942297935486,0.015091033652424812,0.07814451307058334,-0.04056928679347038,-0.12066596746444702,0.11315778642892838,0.030175574123859406,-0.03817393630743027,0.017036072909832,-0.08568614721298218,0.005587523803114891,0.1490015685558319,0.04026900976896286,-0.010463850572705269,-0.01983306184411049,0.01502299215644598,-0.01579088531434536,-0.040246766060590744,-0.030818728730082512,-0.05570031329989433,-0.0008489086758345366,-0.061662424355745316,0.1478387713432312,0.11007408797740936,-0.032918233424425125,0.02826067805290222,0.0895703062415123,-0.06801752001047134,0.13535556197166443,-0.06255000829696655,0.0851919874548912,0.062380559742450714,0.0077525777742266655,-0.008205164223909378,-0.01755145750939846,-0.06862236559391022,0.08912468701601028,0.07842981815338135,0.03289465233683586,0.0010514275636523962,-0.08593849092721939,-0.15481042861938477,-0.002998416544869542,-0.012016315944492817,0.0010723861632868648,-0.09981343895196915,-0.08797671645879745,-0.03537830337882042,0.17051243782043457,0.09615624696016312,0.026748914271593094,-0.007283469662070274,-0.23220299184322357,-0.239592045545578,0.27822017669677734,0.11223391443490982,0.1419556885957718,0.023525672033429146,-0.14524376392364502,-0.1622362732887268,0.07915276288986206,0.017682380974292755,0.010553369298577309,-0.05802208557724953,-0.06886496394872665,-0.022291945293545723,-0.027534954249858856,-0.01641860604286194,-0.05868702381849289,0.028889672830700874,-0.03165736049413681,-0.048243358731269836,-0.12216760218143463,-0.008684114553034306,0.007617112249135971,0.024905851110816002,0.11017915606498718,0.04344264790415764,0.0016361016314476728,-0.0242876335978508,0.040024351328611374,0.015224405564367771,-0.05029769986867905,0.022229455411434174,-0.049662016332149506,0.044671107083559036,0.03631770238280296,0.18197381496429443,-0.15542832016944885,0.15985924005508423,0.005293930880725384,0.0029218445997685194,0.04166838899254799,-0.10314658284187317,-0.3231969177722931,0.4147873520851135,-0.001485871965996921,0.016761554405093193,0.07865100353956223,0.06756877899169922,-0.18166780471801758,0.2485102117061615,-0.04675065353512764,-0.026888806372880936,0.06957825273275375,-0.02591560408473015,-0.2447858452796936,-0.15797705948352814,0.039340708404779434,0.10518161952495575,0.05139920115470886,-0.01918615587055683,-0.12420152872800827,0.15132559835910797,-0.02478879503905773,-0.07418747246265411,0.02929801307618618,-0.07681243121623993,-0.051418475806713104,0.05507054552435875,0.10267405956983566,-0.027292560786008835,0.0970320776104927,-0.03408001735806465,0.010726104490458965,0.07837668806314468,0.06280417740345001,0.027830176055431366,0.08946218341588974,-0.03385962173342705,-0.06562492251396179,0.031694378703832626,0.007638639770448208,0.059484273195266724,0.05502070486545563,0.007213515695184469,-0.05984830856323242,-0.102488674223423,0.026393016800284386,-0.013143016025424004,0.04209347441792488,-0.021337131038308144,-0.13966216146945953,0.020331664010882378,0.14508725702762604,0.06867209821939468,0.05016394332051277,-0.0043486375361680984,-0.1293262243270874,-0.13969159126281738,0.06926373392343521,0.0452880896627903,0.11376591771841049,0.09774547070264816,-0.19855374097824097,-0.09407376497983932,0.0916314423084259,-0.006393276154994965,0.03762590140104294,-0.08063963800668716,-0.08231417089700699,-0.04680606350302696,-0.10157208889722824,-0.039582617580890656,0.011772923171520233,-0.00692460872232914,0.02283279411494732,0.09905073046684265,0.035204604268074036,0.03838939219713211,-0.0066110603511333466,-0.01709665171802044,-0.021509991958737373,-0.029531879350543022,-0.015357360243797302,0.019038280472159386,0.007642277982085943,-0.021428070962429047,0.14374279975891113,0.02645229361951351,0.034648679196834564,-0.08023668825626373,-0.015437930822372437,-0.13455498218536377,0.11886652559041977,-0.019947776570916176,-0.03569324314594269,0.02980290725827217,-0.042202968150377274,0.03680342808365822,0.18115736544132233,-0.5409073233604431,0.055754415690898895,-0.03930310532450676,-0.006760180462151766,0.016833439469337463,0.054319027811288834,-0.12737247347831726,0.03349810838699341,-0.008279925212264061,-0.08460435271263123,-0.04384570196270943,0.14009475708007812,0.06524797528982162,0.022202594205737114,0.019729699939489365,-0.027603251859545708,-0.018577711656689644,0.17687180638313293,-0.10630589723587036,0.093966543674469,-0.014269810169935226,-0.005620808806270361,0.08150898665189743,-0.08383667469024658,-0.021102573722600937,-0.07841968536376953,-0.0391065888106823,0.0058632660657167435,0.02389136701822281,-0.0771651640534401,-0.034529611468315125,0.02573847956955433,-0.004577124025672674,0.15390647947788239,0.0943274274468422,-0.06189951300621033,-0.010760082863271236,-0.016185881569981575,0.03450249508023262,0.0028496438171714544,-0.010763058438897133,7.12506371201016e-05,0.007624304853379726,0.0033060763962566853,0.019510382786393166,-0.013703156262636185,0.006139456294476986,-0.07434424757957458,0.04311070963740349,0.185553178191185,0.20929443836212158,-0.028949428349733353,-0.038959961384534836,0.03230825439095497,0.0897824689745903,0.06684699654579163,0.05316077545285225,-0.047029171139001846,0.02996465191245079,-0.127297043800354,-0.1999741792678833,-0.10137974470853806,0.009561349637806416,0.02227046713232994,0.04662364348769188,-0.07887226343154907,-0.2063315361738205,-0.023852743208408356,0.4077996611595154,0.03205499053001404,-0.45312997698783875,0.028283091261982918,-0.1100432425737381,-0.02679917961359024,-0.08038093149662018,0.11837651580572128,-0.11231421679258347,-0.06164029613137245,-0.0743376687169075,-0.07223541289567947,0.1938290297985077,0.06594832241535187,0.14357125759124756,0.010005077347159386,-0.010458637960255146,-0.13992741703987122,0.10051469504833221,0.08192860335111618,0.07770876586437225,-0.0585847832262516,-0.19544848799705505,-0.07849343121051788,0.3659873604774475,0.087118960916996,-0.12970353662967682,-0.008199057541787624,-0.04287317022681236,0.16630294919013977,0.2401617020368576,-0.06380768120288849,0.35533565282821655,0.08581890165805817,-0.047010887414216995,-0.027305692434310913,0.011342447251081467,-0.3152892291545868,-0.20595747232437134,0.06487634778022766,-0.2704029977321625,-0.08766630291938782,-0.11248455941677094,0.09936373680830002,-0.03507976979017258,-0.05357476323843002,0.04404434561729431,0.1402427703142166,-0.08731293678283691,0.06583549827337265,-0.01053420640528202,0.054868631064891815,0.04871583357453346,-0.014973737299442291,0.004980355966836214,-0.06046511232852936,-0.030901141464710236,-0.03663152828812599,-0.019728533923625946,-0.042514968663454056,-0.006133725866675377,-0.1247425228357315,0.035443250089883804,-0.009045708924531937,0.0027369530871510506,-0.02509661763906479,-0.12479390949010849,-0.039567455649375916,0.01125265285372734,-0.02198559045791626,-0.04619959741830826,0.023727726191282272,-0.007487806491553783,-0.0848863273859024,0.11542718857526779,-0.020661739632487297,0.028082210570573807,-0.012888040393590927,0.005270063877105713,-0.08423205465078354,0.25703859329223633,0.01414104737341404,0.06764886528253555,0.08069465309381485,0.00901078712195158,0.020107436925172806,0.10676918923854828,-0.0028520135674625635,0.05997929349541664,-0.039567481726408005,0.04888833686709404,-0.062485404312610626,0.009979275986552238,-0.0004270410572644323,-0.0680277869105339,-0.023117145523428917,-0.0500926598906517,0.12058588862419128,0.014507047832012177,-0.030251512303948402,-0.10675348341464996,-0.06295597553253174,-0.15951313078403473,0.0019785533659160137,-0.0029638756532222033,-0.07184936106204987,-0.11943349242210388,-0.16669423878192902,-0.06505939364433289,-0.004297972656786442,0.16879047453403473,0.013561761938035488,-0.035956159234046936,0.024434352293610573,0.03591843321919441,0.08065618574619293,-0.015010563656687737,-0.04160074144601822,-0.0846044197678566,-0.14540670812129974,0.013004307635128498,0.18267977237701416,0.2204279601573944,-0.0420110858976841,-0.11980009078979492,-0.15874840319156647,-0.14287178218364716,0.01380270067602396,0.23839569091796875,-0.02184469811618328,-0.04239538684487343,-0.06138725206255913,0.03728101775050163,0.03412193804979324,0.04288776218891144,0.043113041669130325,0.067380391061306,0.06887921690940857,0.08525992184877396,-0.009514594450592995,-0.07626283913850784,0.008779174648225307,0.025769099593162537,0.13025280833244324,0.022622788324952126,0.009390070103108883,-0.007590544410049915,0.02174696885049343,0.0043756538070738316,-0.02880469523370266,-0.1369248330593109,0.008072265423834324,0.06261072307825089,-0.023110659793019295,0.03247147426009178,0.008827995508909225,0.043270472437143326,0.17423222959041595,-0.07995716482400894,-0.01908266916871071,0.018978139385581017,0.0028037854935973883,-0.029589327052235603,0.14491130411624908,0.1669759899377823,0.02661280892789364,0.014861827716231346,-0.0052615804597735405,0.004824329167604446,0.06631354242563248,0.15947072207927704,-0.02521195448935032,-0.042613521218299866,-0.012716575525701046,0.021148940548300743,-0.10614725202322006,-0.2834508717060089,0.019881242886185646,-0.011367138475179672,0.04828272759914398,-0.09195908159017563,0.043327897787094116,-0.013096516951918602,0.007081272546201944,-0.003558774944394827,-0.02383282221853733,0.08542532473802567,-0.10121548920869827,0.12273678928613663,-0.0021346532739698887,0.024031411856412888,-0.006243601907044649,0.000901074381545186,-0.09210999310016632,-0.10836631804704666,-0.020443076267838478,-0.024555684998631477,-0.012668173760175705,0.03969403728842735,0.006925093475729227,-0.1742669939994812,-0.061569370329380035,0.03871878236532211,-0.0467146560549736,-0.09783535450696945,-0.06882651895284653,-0.054332029074430466,-0.016940850764513016,0.05938238650560379,0.10997752100229263,-0.030079053714871407,0.039999328553676605,0.2103765606880188,0.002028533723205328,-0.04210924357175827,-0.04054379090666771,0.01595815271139145,0.0384182445704937,-0.0187357310205698,0.030864909291267395,0.048834607005119324,0.06758115440607071,0.03360501304268837,0.033945098519325256,0.0402987040579319,0.004038442391902208,-0.08154717087745667,-0.07675798237323761,0.026530086994171143,-0.11242823302745819,0.21715384721755981,-0.013299345038831234,0.0504768006503582,0.0838959664106369,-0.03951500356197357,-0.06938599050045013,-0.04901747778058052,0.006731668021529913,-0.03470015898346901,0.02839958854019642,-0.0010275576496496797,0.04403059184551239,0.04939568042755127,0.007119723130017519,0.15680551528930664,0.13192760944366455,0.14661051332950592,0.014411723241209984,-0.060479022562503815,0.06079957261681557,0.08311419188976288,0.10443662852048874,0.12428507953882217,-0.01274164766073227,-0.18555288016796112,-0.019674139097332954,0.032487500458955765,0.011080494150519371,-0.05439477786421776,-0.14006710052490234,0.03297272324562073,0.0626242607831955,0.09232043474912643,0.08087893575429916,-0.004937467165291309,-0.12341073155403137,-0.27471378445625305,0.06198152154684067,0.13332206010818481,0.22248800098896027,0.12170597165822983,-0.0460214838385582,-0.2024478316307068,0.040700145065784454,0.050506431609392166,0.0904531478881836,0.007009605877101421,-0.02573687769472599,-0.043811604380607605,-0.03603794425725937,-0.07020803540945053,-0.07545386254787445,-0.09273596853017807,0.01945112831890583,0.057140178978443146,0.013325628824532032,-0.028051944449543953,-0.06929661333560944,-0.06363111734390259,-0.013113203458487988,0.064604751765728,-0.03874428570270538,-0.058261677622795105,-0.06503787636756897,-0.015761476010084152,0.010058019310235977,0.018974177539348602,0.03251243382692337,0.08544233441352844,-0.022025378420948982,0.01287812553346157,0.005239854566752911,-0.026657357811927795,0.0046572997234761715,0.027221618220210075,0.07129790633916855,0.03789498656988144,-0.06224605441093445,-0.03941495344042778,0.0233834870159626,-0.06061480939388275,-0.05690031126141548,0.008458316326141357,-0.04113250970840454,0.014187066815793514,0.006451786495745182,-0.05069576948881149,-0.15694402158260345,0.041200488805770874,-0.0010649287141859531,0.05159318447113037,0.005299580283463001,-0.0837540552020073,-0.0030578316655009985,-0.039986301213502884,-0.03349955752491951,-0.027156466618180275,-0.012141726911067963,0.07451675087213516,0.08638167381286621,-0.014388586394488811,0.05424110218882561,0.015373571775853634,0.013598550111055374,0.07273586839437485,-0.0618169866502285,0.08170843124389648,-0.018795032054185867,-0.018273010849952698,0.03988388925790787,0.07801112532615662,0.06356708705425262,-0.06222284585237503,0.04028943181037903,0.04219895601272583,-0.05847320333123207,-0.044219911098480225,0.05165202170610428,0.0743463933467865,-0.04792221263051033,0.009192095138132572,-0.0395587682723999,-0.012644791044294834,0.04573189839720726,0.03239474445581436,0.010095563717186451,0.08279852569103241,0.01774727925658226,-0.006519337184727192,0.13912731409072876,0.007592401932924986,0.027171270921826363,-0.07481078803539276,0.03183187171816826,0.04808533191680908,0.22231411933898926,0.042532309889793396,0.03921863064169884,-0.03294070437550545,-0.0062544746324419975,-0.022715063765645027,0.07486535608768463,-0.007439624518156052,-0.06655765324831009,-0.02950242906808853,0.07276470214128494,-0.011249789036810398,-0.06850259751081467,-0.12680889666080475,0.01757253147661686,-0.04131767898797989,-0.05708610638976097,0.02097473479807377,0.03483998030424118,0.07278371602296829,0.04734567552804947,-0.0097691984847188,-0.012072619050741196,0.006907458417117596,-0.022629790008068085,0.002691514091566205,-0.01860055699944496,0.2716912031173706,0.0062679327093064785,-4.169622479821555e-05,-0.0018842853605747223,0.04799901321530342,-0.12218884378671646,0.10903505980968475,0.02430250309407711,0.003049340797588229,-0.012353911995887756,0.05647456645965576,-0.18211281299591064,-0.5057637095451355,0.007179556880146265,0.007009692024439573,0.005029099993407726,-0.047397419810295105,-0.15887881815433502,-0.04491753131151199,0.006497408729046583,-0.004374926444143057,-0.01836118847131729,0.04189715161919594,0.0006400799029506743,0.25291526317596436,-0.03331969305872917,0.007976270280778408,0.04912320896983147,0.04961012676358223,-0.050558533519506454,-0.3286236822605133,0.011615517549216747,0.00016078428598120809,-0.016987789422273636,-0.015393263660371304,0.023448973894119263,0.2321317046880722,-0.013539700768887997,-0.0022569941356778145,-0.058690473437309265,-0.07296396791934967,0.04597051069140434,-0.25575730204582214,-0.06730262190103531,-0.0044295103289186954,-0.16025161743164062,0.10245054960250854,-0.029265563935041428,-0.052344221621751785,0.11998771876096725,-0.10720325261354446,0.003828606102615595,-0.007445781957358122,-0.0006448819767683744,0.009867398999631405,-0.0033150336239486933,0.0030222234781831503,0.0022359092254191637,0.012772978283464909,-0.19658422470092773,0.17675437033176422,-0.057242076843976974,0.00394738232716918,-0.12070812284946442,0.00766207417473197,0.050903692841529846,0.20398929715156555,-0.15500256419181824,0.14604540169239044,-0.07899981737136841,0.0973801538348198,-0.15014143288135529,0.03905915096402168,-0.04529019072651863,0.03438127785921097,0.29482394456863403,-0.09929828345775604,0.02011536806821823,-0.002140755532309413,0.1213839203119278,0.13478095829486847,-0.0780622661113739,-0.28032493591308594,-0.20087239146232605,-0.23372776806354523,-0.03635315224528313,-0.011806159280240536,0.13705356419086456,0.08738706260919571,0.025984641164541245,-0.038641951978206635,0.022223087027668953,0.09742450714111328,-0.0766834244132042,-0.06068303808569908,0.004482374060899019,0.0004267839540261775,0.11020374298095703,0.16255418956279755,-0.10613268613815308,0.12717124819755554,-0.006707143969833851,0.042863838374614716,-0.017114222049713135,0.10222504287958145,-0.07377202063798904,-0.04973180964589119,0.06747984141111374,-0.07204543054103851,0.2930537462234497,-0.17970429360866547,0.05702183023095131,-0.014621410518884659,-0.09888558834791183,0.3008023500442505,-0.08110709488391876,0.019689194858074188,-0.18531380593776703,0.13626134395599365,0.03465369716286659,0.0178112480789423,0.09185317158699036,-0.3000105917453766,0.1181536316871643,0.1096944808959961,0.059753596782684326,0.05563477799296379,0.3112551271915436,-0.07338059693574905,0.02662121132016182,0.006192430388182402,-0.004951909184455872,0.021254392340779305,-0.014670108444988728,0.014451988972723484,-0.04528065770864487,0.2006101757287979,-0.0034944931976497173,-0.0026904612313956022,-0.027397019788622856,-0.0342656709253788,-0.04339589551091194,-0.013380476273596287,0.01489167008548975,0.014835563488304615,0.002884790301322937,-0.01298222690820694,0.012546141631901264,-0.01864139921963215,-0.010470140725374222,-0.03843013569712639,-0.00022819543664809316,-0.01777123287320137,-0.03435760736465454,0.06366635113954544,0.028508776798844337,0.015298877842724323,0.021875018253922462,0.013986802659928799,0.03267659246921539,0.2303493618965149,-0.030970420688390732,-0.048083361238241196,-0.036105088889598846,-0.04462100937962532,-0.03826150298118591,0.03875735029578209,0.019939258694648743,-0.003361293114721775,-0.004006677772849798,0.015730932354927063,0.03576775640249252,0.06355700641870499,0.01792951673269272,-0.0037184590473771095,0.019694028422236443,0.03263738751411438,0.09075096994638443,-0.05632621422410011,-0.004373012110590935,-0.013637233525514603,-0.03061104379594326,-0.047413650900125504,-0.0643952265381813,0.14497493207454681,-0.0031952194403856993,-0.01603119820356369,-0.042802851647138596,-0.009140620008111,-0.06015237420797348,-0.03174988180398941,0.02384427934885025,0.032058656215667725,0.014097800478339195,0.03485243767499924,0.006955413147807121,-0.14512024819850922,-0.04748820140957832,-0.023381799459457397,-0.020661281421780586,-0.02086193859577179,0.027048248797655106,0.09718991070985794,0.019921613857150078,0.013295389711856842,0.025921784341335297,-0.009321870282292366,-0.08140112459659576,0.22188091278076172,-0.012445572763681412,-0.016774926334619522,-0.03391590341925621,-0.03463952988386154,-0.028523627668619156,0.01881370320916176,0.0073534962721168995,-0.00989818386733532,0.019065631553530693,0.009472152218222618,0.007591093424707651,0.06572773307561874,-0.004635029938071966,0.00989461038261652,0.028920015320181847,-0.0035308534279465675,0.026242192834615707,-0.036353468894958496,-0.0066913338378071785,-0.01773710735142231,-0.019660023972392082,0.006395101081579924,-0.008443810045719147,0.24089990556240082,-0.02947668917477131,-0.035742390900850296,-0.028803110122680664,-0.03674484044313431,-0.09725234657526016,-0.13119657337665558,0.0049036466516554356,0.0027225830126553774,-0.003751816228032112,-0.013929316774010658,-0.1432885378599167,-0.006678557954728603,-0.007010793313384056,0.016569895669817924,0.012524553574621677,-0.013286309316754341,-0.017126036807894707,0.11225549131631851,0.00876504834741354,0.014402809552848339,0.023786459118127823,0.013712325133383274,0.032571032643318176,-0.022883949801325798,-0.004468244034796953,0.008498693816363811,-0.023985430598258972,-0.007052906788885593,0.05568218231201172,0.02187148481607437,-0.010606181807816029,0.00023102923296391964,-0.011061369441449642,-0.016053995117545128,0.033170703798532486,0.5556071996688843,-0.03858882188796997,-0.013710994273424149,-0.05429702252149582,-0.05588201433420181,-0.15941789746284485,-0.262715607881546,0.0018391433404758573,-0.0004557025677058846,-0.0013988602440804243,-0.029834270477294922,-0.010248411446809769,-0.0018634528387337923,0.009950068779289722,0.010858642868697643,0.013632682152092457,0.022120516747236252,-0.07336550951004028,0.004594400059431791,0.0005505456356331706,0.002873888937756419,0.009861002676188946,-0.005273778922855854,-0.10360968112945557,-0.014090449549257755,0.011145791970193386,0.009095738641917706,-0.007088849786669016,-0.0069394600577652454,-0.03879232332110405,0.00023300913744606078,-0.006525329779833555,-0.0021058693528175354,-0.005118425004184246,0.001051443163305521,-0.17451243102550507,0.0032799725886434317,-0.004703314043581486,0.0060063316486775875,0.0020635738037526608,0.00201305840164423,0.07939934730529785,0.03195629641413689,0.00834212638437748,0.015490113757550716,-0.00897516030818224,-0.013845957815647125,0.03846129775047302,0.010732090100646019,0.004788190126419067,0.003843985963612795,-0.0069630988873541355,-0.0015118583105504513,0.05849394574761391,0.03211989998817444,-0.041848476976156235,0.0210672989487648,0.044638264924287796,0.046263813972473145,0.0870155468583107,0.0233796127140522,0.026598649099469185,-0.03553019464015961,-0.037956658750772476,0.03234612196683884,0.0741683766245842,-0.1439150869846344,0.00719872023910284,-0.013220153748989105,0.05210452154278755,-0.013264729641377926,-0.04941054806113243,0.013517258688807487,0.03789355233311653,0.0046334597282111645,-0.009770304895937443,0.08380117267370224,-0.030418626964092255,-0.1896195411682129,0.06954430788755417,0.14645953476428986,0.19095340371131897,0.1380070447921753,0.02479124628007412,-0.1509941816329956,0.012940899468958378,-0.006024377420544624,0.016626792028546333,-0.04035947844386101,-0.03489656373858452,-0.021994005888700485,-0.022073877975344658,-0.035922251641750336,-0.04090884327888489,0.014338538981974125,-0.01282384805381298,-0.021899400278925896,-0.026975169777870178,0.034462373703718185,0.006732690613716841,-0.01844770275056362,-0.09613842517137527,0.030168849974870682,0.04999779909849167,0.041961390525102615,0.0026571403723210096,0.027728689834475517,0.005800939165055752,0.009009601548314095,-0.07331099361181259,-0.04827601835131645,-0.018862886354327202,-0.03799434378743172,0.013817772269248962,0.062271952629089355,0.030338821932673454,0.055107828229665756,0.001703150337561965,0.018739236518740654,0.02954413741827011,0.039876945316791534,-0.04127480834722519,-0.07040026038885117,-0.07347475737333298,-0.09938744455575943,-0.02503090538084507,0.0666290894150734,-0.06089073419570923,-0.07631700485944748,-0.09065548330545425,-0.06937438994646072,-0.038877103477716446,0.13381992280483246,-0.03390873596072197,-0.016095826402306557,-0.041282352060079575,-0.008546175435185432,0.04857373610138893,0.10911121219396591,-0.026454772800207138,-0.011454449035227299,-0.0023329749237746,0.06409244239330292,0.002564018592238426,-0.0033579159062355757,0.01770266704261303,-0.01097017526626587,0.0224591176956892,-0.052034277468919754,-0.027944032102823257,-0.02635445073246956,0.019052887335419655,0.017842071130871773,0.017414921894669533,-0.024629289284348488,-0.01846500299870968,0.01579936034977436,0.00923833902925253,0.01767665520310402,0.015263131819665432,0.016860825940966606,0.05995962768793106,0.02274188958108425,-0.0016736635006964207,0.0016355984844267368,0.0045699444599449635,0.01999226212501526,0.04489950090646744,-0.09095194935798645,-0.01537228748202324,0.01078183762729168,0.010832514613866806,0.022846776992082596,-0.023743439465761185,0.014697926118969917,-0.011814904399216175,-0.03336676210165024,-0.024325426667928696,-0.027463706210255623,-0.03928457200527191,-0.07156910747289658,-0.01296036783605814,-0.008615691214799881,-0.018550390377640724,-0.0189194455742836,-0.029360191896557808,-0.05552315711975098,0.002544247778132558,0.00746040977537632,0.004100501537322998,-0.009867154993116856,-0.0331307128071785,-0.1245141476392746,-0.02809918485581875,-0.013163984753191471,-0.013026374392211437,-0.03621744364500046,0.02834121137857437,-0.020626835525035858,0.000864903733599931,0.011351537890732288,-0.001190836657769978,-0.03369950130581856,0.01702287420630455,0.2091432362794876,-0.012023059651255608,-0.004398711025714874,-0.00884163100272417,0.005726051516830921,0.0011452906765043736,-0.040782541036605835,-0.002147400751709938,-0.008656165562570095,-0.0008460938115604222,0.010372436605393887,-0.01847548969089985,-0.5059886574745178,-0.0027394972275942564,0.011615410447120667,-0.0027547208592295647,0.01685914769768715,0.056284479796886444,-0.0809868797659874,0.00019839515152852982,-0.012411970645189285,-0.000300252519082278,0.02534269168972969,-0.013935910537838936,-0.00669061066582799,-0.0049889106303453445,-0.023287659510970116,-0.01860496960580349,0.0017816569888964295,0.05599157512187958,-0.14942502975463867,0.0018869576742872596,0.002638286678120494,0.003841526573523879,0.00032635321258567274,-0.019791336730122566,0.3353439271450043,-0.03078017756342888,-0.030047541484236717,-0.03746933862566948,-0.023553993552923203,-0.042418546974658966,-0.4062248170375824,0.23050102591514587,0.10009860247373581,-0.044005777686834335,0.00909569300711155,0.07492736726999283,-0.04409612715244293,-0.08325572311878204,-0.10509004443883896,0.0007333567482419312,0.005051909945905209,-0.0014124242588877678,0.004342180211097002,0.0038413526490330696,-0.002112558577209711,0.0010413479758426547,0.0073057543486356735,-0.275392085313797,-0.024535654112696648,0.04839122295379639,0.03631570562720299,-0.15082576870918274,0.173878014087677,0.010466182604432106,0.02304045669734478,-0.11567411571741104,-0.14419569075107574,-0.01993751712143421,0.021624885499477386,0.038444384932518005,0.17193427681922913,-0.009944669902324677,0.011007755063474178,0.09889166802167892,0.0414707250893116,0.1311590075492859,0.05223849043250084,-0.02985435537993908,-0.001301532262004912,0.002763835247606039,0.011024664156138897,-0.06601396948099136,0.07517395913600922,-0.003911333158612251,-0.01684488169848919,0.2450936734676361,-0.23876139521598816,-0.038080088794231415,-0.09163588285446167,-0.15474097430706024,0.03794793039560318,0.015365003608167171,0.12660899758338928,0.3292219638824463,0.14394880831241608,-0.04785642772912979,0.16177357733249664,-0.09408446401357651,-0.11093313992023468,-0.049285437911748886,0.08938370645046234,0.19826216995716095,-0.11570803821086884,-0.04607071727514267,-0.0227857306599617,0.22327940165996552,0.018957551568746567,0.18933866918087006,0.058310385793447495,-0.005957947112619877,-0.18845395743846893,-0.020145051181316376,0.23323549330234528,-0.08117760717868805,-0.1369916796684265,-0.1940271407365799,-0.04127868637442589,-0.009719450026750565,0.05875490978360176,0.009299562312662601,-0.18223083019256592,-0.05808820202946663,0.12146685272455215,-0.04341721907258034,0.08897563815116882,0.0479716882109642,-0.07948017120361328,-0.19427672028541565,-0.01015518419444561,0.01724299043416977,0.042263876646757126,0.015159904025495052,-0.03569834306836128,-0.04693872854113579,0.11534754931926727,0.027963465079665184,-0.006211601663380861,0.008407356217503548,0.02890140190720558,0.017622945830225945,0.044218458235263824,-0.014650625176727772,-0.015581237152218819,-0.021398425102233887,-0.027017908170819283,-0.035533614456653595,-0.04051538556814194,-0.017999622970819473,0.036440975964069366,0.04105057567358017,0.03286414593458176,-0.008876404725015163,0.13037584722042084,-0.0385039784014225,-0.020965704694390297,-0.02848326414823532,-0.008283278904855251,-0.03483182191848755,0.19584591686725616,0.007886474952101707,0.006140772718936205,0.012904857285320759,0.038567036390304565,0.02947608008980751,0.059351351112127304,-0.012625642120838165,0.00909190159291029,0.02687462419271469,0.018851637840270996,0.036484260112047195,0.009368100203573704,-0.006803339347243309,0.006508287508040667,0.007849220186471939,-0.003555749310180545,0.006191655062139034,-0.1677422672510147,0.017682507634162903,0.004314655903726816,-0.0029630998615175486,-0.0030395444482564926,0.01040071900933981,0.010005087591707706,-0.01058922614902258,-0.027782121673226357,-0.014993177726864815,0.0019441194599494338,-0.006646219175308943,0.10894609242677689,0.0231398306787014,0.011339278891682625,0.04713118076324463,0.0011034581111744046,0.05146549269556999,-0.07199906557798386,-0.012064899317920208,-0.030631570145487785,-0.0729966089129448,-0.03977666795253754,-0.11793777346611023,0.1320350617170334,0.022271744906902313,0.02701277658343315,0.039566271007061005,-0.01018631923943758,-0.0026437928900122643,0.3431195318698883,-0.015195832587778568,-0.016806645318865776,-0.03550601378083229,-0.009579414501786232,-0.00926705077290535,0.001853586407378316,0.004663275089114904,-0.014561458490788937,-0.009767596609890461,-0.009015575051307678,-0.007968675345182419,0.027319051325321198,0.020714756101369858,0.019444728270173073,0.027983766049146652,0.01186680793762207,0.016486922279000282,-0.09707916527986526,0.011085133999586105,-0.0044005378149449825,-0.026134537532925606,-0.004434176255017519,0.020794633775949478,0.010693946853280067,-0.020345283672213554,-0.01474146731197834,-0.010924456641077995,-0.019152594730257988,-0.04427171126008034,-0.11121316999197006,0.009642243385314941,-0.007111645303666592,0.0053225005976855755,0.0036616430152207613,-0.007300041150301695,0.13126026093959808,-0.009303873404860497,0.011158652603626251,-0.004905465990304947,0.002908842172473669,-0.017313869670033455,-0.05114299803972244,-0.006417714990675449,0.005327598657459021,0.02574833296239376,0.020999906584620476,0.0251547172665596,0.034895747900009155,-0.005435539875179529,0.010535869747400284,0.0009073077235370874,0.004543366841971874,0.013456917367875576,0.09119337797164917,-0.00477601308375597,-0.0024310906883329153,-0.016300657764077187,-0.003786597866564989,-0.025640428066253662,0.19620008766651154,-0.02994534559547901,-0.005274175200611353,-0.018597010523080826,-0.00983359944075346,-0.011168025434017181,-0.18385452032089233,-0.0034060871694236994,0.0170842744410038,-0.004943564534187317,0.000519079971127212,-0.013651848770678043,-0.06300520896911621,-0.01427283976227045,-0.005832212045788765,-0.0029291100800037384,-0.0060793026350438595,-0.02714797668159008,-0.0705958902835846,0.013643954880535603,-0.017904940992593765,0.010790546424686909,0.004560125060379505,-0.0001039734052028507,0.08781176805496216,-0.008193905465304852,0.0150002995505929,0.002440237905830145,-0.0035718369763344526,-0.018833275884389877,-0.10840674489736557,0.005495637655258179,-0.0019194368505850434,0.00841866061091423,0.012449887581169605,0.046700749546289444,-0.06309395283460617,-0.0249144546687603,0.014968800358474255,-0.024087633937597275,-0.02625073865056038,-0.026033101603388786,0.034669868648052216,-0.021042607724666595,0.01630007103085518,-0.00692060636356473,0.0008879808592610061,0.001285730628296733,0.1256706565618515,-0.016706472262740135,0.016328351572155952,0.0027431563939899206,0.018690360710024834,0.01568589173257351,-0.0426686592400074,-0.11013001203536987,-0.11718959361314774,-0.13932691514492035,-0.05557222664356232,-0.0867435559630394,0.11705099791288376,0.046261128038167953,-0.03678567335009575,-0.06094730645418167,-0.05533411726355553,-0.059664659202098846,-0.11398161202669144,-0.03545654937624931,0.07224413007497787,-0.01276514120399952,0.028232159093022346,-0.008347864262759686,0.1547425538301468,0.06055193766951561,0.06919635087251663,0.0895194560289383,0.06917379051446915,0.04736671596765518,-0.12217619270086288,0.0909496620297432,0.1129753366112709,0.18094348907470703,0.10514158010482788,0.18189659714698792,-0.09986882656812668,-0.02476632036268711,0.023835262283682823,-0.0030356321949511766,-0.050924018025398254,-0.07884053140878677,-0.2820051610469818,-0.05973457917571068,6.913243851158768e-05,0.025470538064837456,0.004406035877764225,-0.047152742743492126,-0.17243850231170654,-0.016205819323658943,-0.022742919623851776,0.03271816670894623,0.013815398328006268,-0.01083410531282425,-0.0739288479089737,-0.011886403895914555,0.0029942188411951065,-9.167320058622863e-06,0.017927374690771103,-0.011696030385792255,-0.0506887286901474,0.012593647465109825,0.008583742193877697,0.014982873573899269,-0.0008682282641530037,0.01261997688561678,-0.0016271329950541258,0.0017698407173156738,0.010563709773123264,-0.01277267374098301,-0.013953592628240585,0.0018247927073389292,0.09535188227891922,0.006349275819957256,0.00763245252892375,0.009770205244421959,0.01272447220981121,0.012976045720279217,0.03614890202879906,-0.015372815541923046,-0.014021149836480618,-0.013163008727133274,-0.022240910679101944,-0.016251442953944206,-0.06764762103557587,0.011176686733961105,0.012415510602295399,0.0013008639216423035,-0.010886664502322674,0.008275353349745274,0.03825487568974495,-0.0038903416134417057,0.005439141299575567,-0.0006672628223896027,0.010169538669288158,-0.007907489314675331,-0.0029931182507425547,0.0017740243347361684,0.0100462157279253,0.010067782364785671,-0.004805375821888447,-0.01013159565627575,0.005408281926065683,-0.011014205403625965,-0.03941473737359047,-0.02210618369281292,0.011793435551226139,0.043130986392498016,-0.07345002144575119,-0.025973381474614143,0.028636394068598747,0.024754170328378677,-0.013895412907004356,0.004359425511211157,-0.14542293548583984,-0.007002905942499638,0.013520041480660439,-0.010052036494016647,0.03168641775846481,0.006690339185297489,0.06734947860240936,0.01842753030359745,-0.0402647890150547,-0.023494895547628403,-0.01337770652025938,0.030024845153093338,-0.18135443329811096,0.029754742980003357,0.008585725910961628,0.002453901804983616,0.004200720228254795,0.04097062349319458,-0.19909894466400146,0.004376091528683901,-0.008473061956465244,0.009900901466608047,-0.028421808034181595,-0.032426659017801285,0.013683685101568699,0.0008977777324616909,-0.007169677410274744,-0.02208080142736435,-0.0085725924000144,-0.03760578855872154,0.0038352764677256346,-0.00039519101846963167,0.005718492902815342,-0.004002552013844252,0.012713485397398472,0.01781505160033703,0.07685776054859161,-0.0019522493239492178,-0.013388345018029213,-0.00980647373944521,-0.008310369215905666,-0.011612096801400185,-0.020650452002882957,0.012549135833978653,0.013126389123499393,0.007081991992890835,0.004834112245589495,0.008338079787790775,0.04215119034051895,0.008927673101425171,0.008195534348487854,0.008983736857771873,0.004960441030561924,0.011182352900505066,0.07077065110206604,-0.013723624870181084,0.000565190042834729,-0.0018409285694360733,-0.006119679659605026,-0.0031458097510039806,0.03661063313484192,0.0021464810706675053,0.0021615310106426477,0.002807852579280734,0.008933259174227715,0.003637404413893819,-0.0708063468337059,-0.012055350467562675,-0.010391667485237122,-0.008633742108941078,-0.00876812543720007,-0.008354545570909977,0.02799099311232567,-0.012715989723801613,-0.01483831275254488,-0.019320065155625343,-0.018411310389637947,-0.022007398307323456,-0.0690794512629509,0.020997339859604836,0.03338547796010971,0.0267240758985281,0.019338075071573257,0.028635531663894653,0.03165632486343384,9.792916534934193e-05,0.12545299530029297,0.0377853661775589,0.09074932336807251,0.13942459225654602,-0.013528630137443542,0.0157326590269804,0.02544596418738365,-0.0022191640455275774,-0.001601475989446044,0.006202720571309328,0.0041688974015414715,0.010178646072745323,0.0025294534862041473,-0.0035215243697166443,-0.032528311014175415,-0.0942002609372139,-0.04238176345825195,0.033168110996484756,0.07309529185295105,-0.13011743128299713,0.015291470102965832,0.073981873691082,0.013548502698540688,-0.11160364001989365,-0.08628488332033157,-0.06509514153003693,-0.02106185257434845,0.13635753095149994,0.019639533013105392,0.010698235593736172,-0.0025883398484438658,-0.05462116003036499,0.07468706369400024,0.048207081854343414,0.06718871742486954,-0.2451208084821701,-0.0009594136499799788,0.07413079589605331,0.012392736971378326,0.1209949254989624,-0.08628728240728378,-0.04848284274339676,-0.002237459411844611,0.12068767845630646,0.023400520905852318,-0.051988258957862854,-0.0062837982550263405,-0.03499460965394974,-0.08345954120159149,0.015148344449698925,0.05958547815680504,0.25748592615127563,0.03330545499920845,0.10161624103784561,0.0029108570888638496,-0.0007281217258423567,0.07392366230487823,0.011205444112420082,0.015176712535321712,0.46066519618034363,0.038724999874830246,-0.0797390267252922,-0.0046591744758188725,0.30032679438591003,0.025655191391706467,0.05383702367544174,0.054368384182453156,0.22733436524868011,0.03315075859427452,-0.1482374668121338,0.009623775258660316,0.03719234839081764,-0.04660721868276596,-0.07960981130599976,-0.02078857272863388,-0.010492061264812946,-0.015367396175861359,0.016949228942394257,0.005338029004633427,0.11179859191179276,0.050914764404296875,0.054633282124996185,-0.12890930473804474,-0.03050544671714306,-0.08904657512903214,-0.24523650109767914,0.010333511047065258,0.11448045074939728,0.12425045669078827,0.12160679697990417,0.09364241361618042,0.04268277436494827,-0.05782640725374222,0.0003247462445870042,-0.006744183599948883,-0.027757830917835236,0.004633112344890833,-0.006186287384480238,0.1460389792919159,0.06868583709001541,0.04746004566550255,0.06535546481609344,0.011057439260184765,0.029907336458563805,-0.05536673963069916,-0.09185267984867096,-0.11266034841537476,-0.08599455654621124,-0.06912866234779358,-0.06904056668281555,0.11338517814874649,-0.06466536223888397,-0.05438216030597687,-0.0321606807410717,-0.03835387900471687,0.0006681310478597879,0.1676410287618637,-0.09132909774780273,-0.1039084792137146,-0.12346851080656052,-0.06503429263830185,-0.07617665082216263,0.24360615015029907,-0.06178978458046913,-0.05921035632491112,-0.05982942134141922,-0.06863819807767868,-0.04319486767053604,0.07135019451379776,0.0177970789372921,-0.009828266687691212,-0.019116122275590897,-0.019724421203136444,-0.016682233661413193,-0.008967449888586998,0.010328744538128376,-0.015505097806453705,0.030859265476465225,0.0611283965408802,0.054512251168489456,0.05441056936979294,-0.04027821496129036,0.0019416416762396693,-0.026843534782528877,-0.02136397734284401,-0.026433123275637627,0.1346115618944168,0.015965139493346214,0.001097888802178204,0.036647405475378036,0.023849602788686752,0.04586226865649223,-0.19679982960224152,-0.024925492703914642,-0.057726532220840454,-0.07792245596647263,-0.10822901874780655,-0.06688018143177032,0.22370684146881104,0.042763449251651764,0.04252224788069725,0.017373977228999138,0.002639883430674672,0.018446438014507294,0.23975352942943573,-0.05270219221711159,-0.0630129799246788,-0.09367179870605469,-0.030099568888545036,-0.08669130504131317,0.15294969081878662,0.009849626570940018,-0.023792780935764313,-0.03904944658279419,-0.019963338971138,-0.016613097861409187,0.034087326377630234,0.007186091039329767,0.014379080384969711,0.030559832230210304,0.013410639949142933,0.016382433474063873,-0.0011626173509284854,-0.004025137517601252,0.008222192525863647,-0.022648947313427925,-0.0055024889297783375,-0.019363775849342346,0.1313306838274002,-0.011952945962548256,-0.0063632093369960785,-0.009746734984219074,-0.030895689502358437,-0.03526097536087036,-0.25344809889793396,-0.003575482638552785,-0.005897478200495243,0.001964441267773509,-0.005356703884899616,0.006487804464995861,0.12416578084230423,-0.0031918580643832684,0.014155126176774502,0.011516734026372433,-0.005875203292816877,0.03442119061946869,0.008126339875161648,0.0026107740122824907,-0.01892644353210926,0.006364447996020317,0.00019012064149137586,-0.0064878617413342,0.05756644904613495,0.020335864275693893,0.027734391391277313,0.02140646241605282,0.0007464279187843204,0.02554033324122429,-0.06700784713029861,-0.0017386064864695072,0.007982168346643448,0.004461427219212055,-0.016738668084144592,-0.0008030344033613801,0.4914581775665283,-0.019557863473892212,-0.016960302367806435,-0.017586302012205124,-0.04936058819293976,-0.036390338093042374,-0.4075899124145508,-0.023662541061639786,0.0028370146173983812,-0.023271964862942696,-0.029639508575201035,0.011128012090921402,-0.0869663655757904,-0.0006923466571606696,0.00889733899384737,-0.006394500378519297,-0.0198917668312788,0.011178779415786266,-0.02860669605433941,0.0073844389989972115,-0.00895628985017538,-0.005004040896892548,0.01723422482609749,0.01170434895902872,0.1879182755947113,0.006393710151314735,0.0059425244107842445,0.026800842955708504,-0.0052437931299209595,-0.020735645666718483,-0.0019130909349769354,0.020468711853027344,-7.458893378498033e-06,0.026824137195944786,0.030678218230605125,0.009541845880448818,-0.006116844248026609,0.013326662592589855,0.02552269957959652,0.028669502586126328,0.010966461151838303,0.02914271131157875,0.21747271716594696,-0.0007507265545427799,0.01649029739201069,0.00740209361538291,-0.004641481209546328,0.011169585399329662,-0.0777994766831398,0.017017725855112076,0.015664150938391685,0.028619250282645226,0.01078046765178442,0.039544906467199326,0.04870909824967384,-0.003485326422378421,0.010308867320418358,-0.029851987957954407,-0.07150302082300186,-0.04844297468662262,-0.023005083203315735,0.028458410874009132,-0.00015102769248187542,0.03804453834891319,0.03867438808083534,0.013918516226112843,-0.154847651720047,-0.026058634743094444,0.003907315898686647,-0.020016860216856003,-0.018813135102391243,-0.012658033519983292,0.02487216517329216,0.025076543912291527,0.041734080761671066,0.06809235364198685,0.10148991644382477,0.03978944569826126,-0.22921648621559143,-0.039258334785699844,-0.022150376811623573,-0.021312205120921135,-0.014171822927892208,0.0009829486953094602,-0.20310580730438232,0.051731374114751816,0.05383731797337532,0.060763534158468246,0.034381985664367676,0.04232524707913399,-0.08187706768512726,-0.02529793232679367,0.027310045436024666,0.0340648777782917,0.03396442532539368,0.004772127140313387,0.029454626142978668,-0.000803868577349931,0.009846664033830166,-0.024375731125473976,-0.014721965417265892,-0.014966744929552078,0.016035171225667,0.011300701647996902,-0.011527211405336857,0.0012701291125267744,-0.010072462260723114,-0.011735544539988041,-0.046225834637880325,-0.020304283127188683,0.003796136472374201,-0.01203887164592743,0.014595548622310162,0.01737922430038452,-0.0013355989940464497,-0.009925000369548798,0.00710147712379694,-0.01899125427007675,0.01516297459602356,-0.019011588767170906,0.11313976347446442,0.00039441982517018914,0.0029847656842321157,0.00020179289276711643,-0.007716394029557705,0.008294861763715744,0.037582434713840485,0.007129215635359287,-0.004180521704256535,0.007692186161875725,-0.0020526214502751827,-0.004952394403517246,-0.07296004891395569,-0.01437404379248619,-0.0034208556171506643,-0.01792566291987896,-0.017220791429281235,-0.002654721960425377,0.04250367730855942,0.05825848877429962,-0.012537665665149689,0.002834965707734227,0.01600404642522335,0.005755062215030193,-0.055185724049806595,-0.06006703898310661,-0.012503367848694324,-0.031975045800209045,0.009225625544786453,-0.024982409551739693,-0.012010078877210617,-0.10782475769519806,-0.11852604895830154,-0.14168183505535126,-0.08233456313610077,-0.10249018669128418,0.013714071363210678,-0.010652037337422371,0.023197485134005547,0.03073493018746376,-0.018167106434702873,0.035124268382787704,-0.2727327346801758,-0.05867573246359825,-0.0663982629776001,-0.09811539947986603,-0.05107228830456734,-0.030411621555685997,0.14302967488765717,0.0927843376994133,0.12232039868831635,0.11072792857885361,0.11899751424789429,0.10100807249546051,-0.1523439884185791,0.0997471958398819,0.041176337748765945,0.03685615956783295,0.029126785695552826,0.017879320308566093,-0.1897706836462021,0.11063782125711441,0.1678255945444107,0.2118827998638153,0.16349606215953827,0.10618198662996292,-0.3277449607849121,0.15463051199913025,0.08202524483203888,0.0709569975733757,0.10596568137407303,0.08182942122220993,-0.17228735983371735,-0.0032996765803545713,0.019773194566369057,0.036960095167160034,0.04275963827967644,0.013740533962845802,-0.06960142403841019,0.0023783401120454073,-1.7576161553733982e-05,-0.0007651306223124266,-0.004746968857944012,0.0030756837222725153,-0.0021406440064311028,0.01425481028854847,0.009646417573094368,0.010907648131251335,0.005429358687251806,-0.0013157789362594485,-0.0010092910379171371,0.01703874208033085,0.010033070109784603,0.014507213607430458,0.007728327065706253,0.019186463207006454,0.059578318148851395,-0.011272978037595749,-0.011535115540027618,-0.013469075784087181,-0.004261546768248081,-0.013857733458280563,0.10193337500095367,0.00025765824830159545,0.0062568592838943005,0.004198969341814518,0.006065502297133207,0.005933748092502356,-0.13601060211658478,-0.022967956960201263,-0.005845923908054829,-0.017358142882585526,-0.011271278373897076,-0.018378641456365585,0.042040981352329254,0.0027185254730284214,0.002053867094218731,0.0030683360528200865,-0.0027860510163009167,-0.008125360123813152,0.05257219448685646,0.006448062136769295,0.006156533490866423,-0.004340403247624636,0.00236941734328866,0.0019137135241180658,0.0031253660563379526,-0.06218269094824791,0.041191767901182175,-0.05338490754365921,0.07199648767709732,0.011205380782485008,0.06515459716320038,0.05349208787083626,-0.02578979730606079,0.006080546882003546,-0.00700960960239172,0.0015848121838644147,0.0069084507413208485,-0.003563456004485488,0.0026795503217726946,0.006155161652714014,-0.010146548971533775,0.029626281931996346,0.028663694858551025,0.11745745688676834,-0.003066814038902521,-0.017439208924770355,0.08046802133321762,0.08971207588911057,0.005303369369357824,-0.21921251714229584,-0.14221709966659546,-0.061608318239450455,0.038519248366355896,0.08189882338047028,0.05959154665470123,0.29974234104156494,-0.0013076213654130697,0.20427945256233215,0.07673569023609161,-0.007614254951477051,0.016920147463679314,-0.07006558775901794,-0.0368339903652668,-0.1483737677335739,-0.018699556589126587,-0.2426563799381256,-0.1364288628101349,-0.003359416965395212,-0.010709855705499649,0.08923576772212982,0.01235034130513668,0.15950210392475128,0.00040370551869273186,-0.23388700187206268,-0.13660027086734772,0.06707119196653366,0.04306285083293915,0.1597978174686432,-0.00958574190735817,0.3723754286766052,0.013985920697450638,-0.3663635551929474,-0.21961575746536255,0.010979163460433483,0.04053511470556259,0.14264464378356934,-0.0034465172793716192,0.3525424599647522,-0.006113084033131599,-0.18489545583724976,-0.03509024158120155,0.10686521232128143,-0.12555387616157532,0.04537711665034294,0.08385082334280014,0.2103237807750702,0.005713424179702997,-0.231953963637352,-0.0723176822066307,-0.04219165816903114,0.060241684317588806,0.06123334914445877,0.0304730162024498,0.32447293400764465,-0.010102994740009308,-0.022992756217718124,0.012807494029402733,0.010546107776463032,-0.014394705183804035,-0.03718918561935425,-0.018680864945054054,0.04187014698982239,0.027599090710282326,-0.00268796319141984,0.017776217311620712,0.04090813919901848,-0.005364298354834318,0.04947458952665329,-0.0435117669403553,-0.003140015061944723,0.007458202075213194,-0.014580128714442253,-0.018526526167988777,-0.03440629690885544,0.17879796028137207,0.036154236644506454,0.03282518684864044,0.05309383198618889,0.04055217653512955,0.03083103336393833,-0.11129281669855118,-0.03664407506585121,-0.016583049669861794,-0.03298044949769974,-0.009288783185184002,-0.06988170742988586,0.19744177162647247,-0.11299306154251099,-0.07852417230606079,-0.09976125508546829,-0.08324194699525833,-0.05236712843179703,0.17944534122943878,-0.0901859849691391,-0.06356828659772873,-0.08646915853023529,-0.07168365269899368,-0.05384772643446922,0.28777825832366943,-0.0637948289513588,-0.03568407520651817,-0.04617100581526756,-0.04284103214740753,-0.04665651544928551,0.011630275286734104,-0.08474361151456833,-0.031095042824745178,-0.03687189519405365,-0.07007712125778198,-0.013424440287053585,0.03329426050186157,-0.019392216578125954,-0.01915808580815792,-0.02461007609963417,0.011779531836509705,0.049158208072185516,-0.0937436893582344,-0.06681003421545029,-0.05457579717040062,-0.04437149316072464,-0.05250117927789688,-0.04939209297299385,0.1448129266500473,0.031421829015016556,0.03163162246346474,0.03377465903759003,0.04529856890439987,0.048649441450834274,-0.10359629988670349,-0.03788068890571594,-0.04155135899782181,-0.04519006609916687,-0.06850611418485641,-0.07284943014383316,0.12104155123233795,0.052874043583869934,0.07108011096715927,0.05715523287653923,0.04519796743988991,-0.024671634659171104,0.30100417137145996,-0.03224984556436539,-0.03180563077330589,-0.027995530515909195,-0.038809943944215775,-0.07244554907083511,0.2234567552804947,-0.018064530566334724,0.0018378415843471885,-0.016708044335246086,-0.027551904320716858,-0.03837365284562111,-0.05775393918156624,-0.020284250378608704,0.01230517216026783,-0.0002422887773718685,0.018687142059206963,-0.013151231221854687,0.17799830436706543,0.028615009039640427,-0.005177528131753206,0.0019185804994776845,0.002243173774331808,-0.02142484299838543,0.25282764434814453,0.021161528304219246,-0.004439355805516243,0.013507930561900139,0.014012801460921764,0.013345506973564625,-0.23197774589061737,-0.044103577733039856,-0.004091175738722086,-0.009212651289999485,-0.0088458601385355,-0.013163934461772442,-0.0224470142275095,0.02718370594084263,0.005059719085693359,0.013153038918972015,0.030817069113254547,0.033478207886219025,0.05224055051803589,0.018229778856039047,-0.020828833803534508,-0.01730961911380291,-0.030988123267889023,-0.016684046015143394,0.034094344824552536,0.04013039171695709,-0.014460685662925243,-0.009738052263855934,-0.003923170734196901,0.01914525032043457,-0.09994383901357651,0.06455761939287186,0.02226618491113186,0.021331243216991425,0.013412284664809704,0.0252443365752697,0.650275468826294,0.027769815176725388,-0.049601662904024124,-0.016668379306793213,-0.017407502979040146,-0.016928087919950485,-0.3887535035610199,-0.02497466467320919,-0.02298174798488617,-0.01940763182938099,-0.011202111840248108,-0.023101994767785072,-0.019845493137836456,-0.0008675153949297965,0.0007249414338730276,0.0001379223249386996,-0.004124574363231659,-0.006267320830374956,-0.003943646792322397,0.020823510363698006,0.010490359738469124,0.007365534082055092,0.012333299964666367,-0.0013451679842546582,0.29167675971984863,-0.014208385720849037,-0.008551781065762043,-0.005099347326904535,-0.01107007171958685,0.010967493988573551,-0.03818399831652641,0.010381338186562061,0.02315337583422661,0.02219008281826973,0.02024601213634014,0.014508624561131,-0.04982247203588486,-0.011351985856890678,0.0007650617626495659,0.003634457243606448,-0.005387656390666962,0.015027535147964954,0.11331234872341156,-0.017208417877554893,-0.006872826721519232,-0.001532002817839384,-0.013867226429283619,-0.000458324997453019,0.04920386150479317,-0.013509901240468025,0.005840553902089596,0.005864591337740421,-0.004296083003282547,0.0037982144858688116,0.057409144937992096,-0.009502925910055637,-0.0084905456751585,-0.011469290591776371,0.00013988451974000782,-0.040056295692920685,-0.004871339071542025,-0.001586836646310985,-0.0039975508116185665,-0.0010785851627588272,0.01231304556131363,0.012568820267915726,0.001853279653005302,0.029399771243333817,0.023136718198657036,0.003499030601233244,-0.00374264782294631,-0.0012380329426378012,0.006518077105283737,-0.028183365240693092,-0.01766122691333294,-0.00679159676656127,-0.0010886085219681263,-0.014969906769692898,-0.0693328008055687,-0.01130959577858448,-0.017198488116264343,-0.013031206093728542,-0.011451133526861668,0.036406274884939194,-0.013588925823569298,-0.046877309679985046,-0.04810356721282005,-0.04015170782804489,-0.019236981868743896,1.040046936395811e-05,-0.04444758966565132,-0.0003033973334822804,-0.020912863314151764,-0.013585967943072319,0.0009578438475728035,0.004389723297208548,0.02001989632844925,0.0044698226265609264,-0.0045863124541938305,-0.0025854306295514107,0.013531316071748734,0.007958807051181793,-0.003631585044786334,0.0013225217116996646,-0.005828208290040493,-0.007292077876627445,-0.0027333195321261883,0.00889604538679123,-0.005092503968626261,-0.006708827335387468,-0.004461392294615507,-0.0017096595838665962,-0.0028510787524282932,-0.023244859650731087,0.009590051136910915,-0.0018788465531542897,0.006183380261063576,-0.001674738246947527,-0.005257157608866692,-0.0034137971233576536,0.13734930753707886,-0.001287175458855927,-0.008780678734183311,-0.00726293446496129,-0.0009037030395120382,-0.016552193090319633,0.04935471713542938,0.01024599839001894,0.013996490277349949,0.014689105562865734,0.0031431487295776606,0.009564066305756569,-0.1248442530632019,0.004291372373700142,-0.010165131650865078,-0.0018595920410007238,-0.00018473766976967454,-0.014919230714440346,0.0650278776884079,0.022121747955679893,0.024386735633015633,0.015694720670580864,0.012430085800588131,0.009554327465593815,-0.003830188186839223,-0.00627843989059329,-0.009250467643141747,-0.008496110327541828,-0.0106721306219697,-0.030782105401158333,-0.004007773008197546,0.0052697849459946156,-0.01678869128227234,-0.04048793762922287,0.0068121254444122314,-0.06833252310752869,0.022042235359549522,0.005216095596551895,-0.009248810820281506,0.019922195002436638,0.015171191655099392,0.03969822824001312,-0.23031845688819885,-0.05681830644607544,-0.038170721381902695,-0.05162934213876724,-0.05146745964884758,-0.05280754715204239,0.15611065924167633,0.04507214203476906,0.020026054233312607,0.03474081680178642,0.021402794867753983,0.10327979922294617,-0.242219939827919,0.11729465425014496,0.09550047665834427,0.1027306541800499,0.10605807602405548,0.057209312915802,-0.19955278933048248,0.09520876407623291,0.07036363333463669,0.09256355464458466,0.10352896898984909,0.09294313192367554,-0.3544035851955414,0.08021260052919388,0.046312540769577026,0.051001034677028656,0.05788564682006836,0.060675933957099915,-0.007136733736842871,0.07942794263362885,0.04167438670992851,0.04052775725722313,0.0807342529296875,-0.0029306812211871147,-0.07263459265232086,0.05608174204826355,0.015759622678160667,0.004749262239784002,0.01040139514952898,0.013737790286540985,0.12561920285224915,0.10969765484333038,0.01379009336233139,0.04429234191775322,0.05205632373690605,0.07666847109794617,-0.04819454252719879,-0.07074148207902908,0.007557679433375597,-0.017074845731258392,-0.012819772586226463,0.01363682933151722,-0.3081345856189728,0.0669664517045021,0.01812131702899933,0.03532964363694191,0.035604264587163925,0.009379168972373009,-0.03437117487192154,-0.0014802617952227592,-0.0798124223947525,-0.07646110653877258,-0.06655829399824142,-0.0706103965640068,-0.13985086977481842,0.07387411594390869,-0.07837601006031036,-0.024035325273871422,-0.01601932756602764,-0.039086636155843735,-0.2149616926908493,0.12320676445960999,0.017768843099474907,0.023789698258042336,0.032352957874536514,0.02798447385430336,0.3077956736087799,0.08096340298652649,-0.04702970013022423,-0.03741580992937088,-0.013677145354449749,-0.022937078028917313,-0.26344144344329834,-0.06279647350311279,0.07123956829309464,-0.0857352465391159,0.061965297907590866,0.035018354654312134,0.03431125730276108,0.09075736254453659,0.08306217938661575,0.0019967108964920044,-0.012413154356181622,-0.008779071271419525,0.010217228904366493,-0.011091870255768299,0.00729399174451828,0.0015728480648249388,-0.00474591227248311,1.0186072168340239e-38,1.0746695250156526e-38,3.9963406696325166e-39,2.0052447901134021e-38,1.6378453522444e-38,-1.648565005236792e-38,-1.0874842593420566e-38,-1.4295978269417032e-38,-1.4096024188945595e-38,7.714728183672348e-39,-1.8433121408744906e-38,1.9817957418710835e-38,3.9770644079572644e-39,-1.0004056109510624e-38,1.501600325544566e-38,1.678350585004941e-38,-7.890808342207083e-39,1.70995126667393e-38,-1.5935745502505253e-38,-1.241019767663348e-38,-1.5899821815073822e-38,4.914165940392914e-39,-1.1304039293566293e-38,2.7186465389503987e-39,7.468263605871507e-39,-6.399347532090679e-39,4.736963341788255e-40,1.5665568152091107e-38,-1.4570295055494794e-39,-1.1151098776572954e-38,1.9565230438072926e-38,-8.433224353076398e-40,1.1287906144346521e-38,-1.07216988881499e-39,7.694201963766919e-39,3.4659590099748236e-39,5.2535646323477776e-39,-7.87002848727961e-39,3.3715269077624385e-39,-5.048253387847227e-39,3.029008925425988e-39,1.1600639527823748e-38,2.6553484860183824e-39,7.311554997307598e-40,4.193132019469972e-39,1.527973743552084e-38,-3.5078312093873135e-39,-1.4847927313739148e-38,2.3181049911016503e-39,-1.2615506118537103e-38,-1.1998744217643035e-39,2.5405681288055366e-40,1.5232908843440034e-38,-1.0054690628220537e-38,3.625030207749584e-39,1.1239475868120991e-38,5.516507078790615e-39,9.939100320495312e-39,-1.8163770824424711e-40,-3.322140945984239e-39,7.212784475049663e-39,-2.1211188607776534e-39,-2.423367729144802e-39,1.5250405456065593e-38,1.7394344462334776e-38,-8.018582940079613e-39,-1.6592822760217946e-38,-1.7013016117730386e-38,-3.278023866431901e-39,1.66615438382069e-38,-1.9026712839531363e-38,3.75875892279703e-39,0.002105402760207653,0.0021524899639189243,0.0066046882420778275,0.015485798008739948,0.02048439346253872,0.026210831478238106,0.035964831709861755,0.04133031144738197,0.042900849133729935,0.04532887786626816,0.04158208146691322,0.0289973895996809,0.019111942499876022,0.01040467619895935,0.0017236554995179176,-0.001677026622928679,0.00133381353225559,0.002751134568825364,0.004642687272280455,0.009968235157430172,0.017373664304614067,0.023292331025004387,0.029760872945189476,0.03677051141858101,0.039935242384672165,0.040587227791547775,0.0409630686044693,0.03615052253007889,0.027798913419246674,0.019182933494448662,0.011166532523930073,0.0033989297226071358,0.004674665629863739,0.002508667530491948,0.0027805003337562084,0.009089983068406582,0.005192168056964874,0.007154455874115229,0.010722311213612556,0.010130348615348339,0.01292505580931902,0.0033223216887563467,0.006331641227006912,-0.005408005323261023,-0.023166000843048096,-0.035156331956386566,-0.01459423266351223,-7.866859959904104e-05,0.005719808395951986,0.008986661210656166,0.014288077130913734,0.01909140683710575,0.017126472666859627,0.01810835301876068,0.020928019657731056,0.012994794175028801,0.0015449938364326954,-0.011299903504550457,-0.031168563291430473,-0.04373184219002724,-0.03867826983332634,-0.03130693361163139,-0.022020764648914337,-0.008578051812946796,-0.0013187170261517167,-0.00410851463675499,-0.007373117841780186,-0.013011344708502293,-0.020997747778892517,-0.027860520407557487,-0.03435580059885979,-0.04086102917790413,-0.043405186384916306,-0.0424937941133976,-0.04021992161870003,-0.03365069627761841,-0.02410084567964077,-0.015562263317406178,-0.008522171527147293,-0.002084582345560193,-0.00025239394744858146,0.001326124300248921,4.0544804505771026e-05,0.0005001064273528755,0.004275680519640446,0.005808766465634108,0.007585893385112286,0.011135957203805447,0.007796776480972767,-0.0012274776818230748,-0.005804082378745079,-0.008575319312512875,-0.01106658112257719,-0.006893262267112732,-0.0007629985921084881,0.0004665535525418818,-0.002140045864507556,-0.006125465501099825,-0.007649112958461046,-0.013374852016568184,-0.022036204114556313,-0.024686619639396667,-0.03255839645862579,-0.04130873084068298,-0.04154517501592636,-0.04029945656657219,-0.039622511714696884,-0.030358489602804184,-0.018562018871307373,-0.011499913409352303,-0.00608939491212368,0.00029405858367681503,-0.001983680995181203,-0.007027945015579462,-0.008903254754841328,-0.014589381404221058,-0.02141682058572769,-0.019293010234832764,-0.025776918977499008,-0.032348040491342545,-0.010892681777477264,-0.009521812200546265,0.01932375505566597,0.036592770367860794,0.052840705960989,0.02423705905675888,0.015388608910143375,0.0015029978239908814,0.08536387234926224,0.013119162060320377],"sample_rate":48000.0}}]},"weights":[],"sample_rate":48000.0,"metadata":{"date":{"year":2026,"month":5,"day":17,"hour":9,"minute":1,"second":28},"loudness":-12.049386978149414,"gain":0.612789648007463,"gear_make":"BOSS SD-1 Distortion","gear_model":"BOSS SD-1 Distortion","tone_type":"distortion","gear_type":"pedal"}} \ No newline at end of file From da5500f67583ec63bf352cb4260f881fedc64541 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 10 Jun 2026 08:09:59 -0700 Subject: [PATCH 068/162] Fix compiler warnings. Add A2 to default ModelTest tests --- NeuralAudio/LSTMDynamic.h | 2 +- NeuralAudio/WaveNetDynamic.h | 20 +++----------------- Utils/ModelTest/ModelTest.cpp | 14 +++++++++++++- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/NeuralAudio/LSTMDynamic.h b/NeuralAudio/LSTMDynamic.h index e9884bc..3205259 100644 --- a/NeuralAudio/LSTMDynamic.h +++ b/NeuralAudio/LSTMDynamic.h @@ -142,7 +142,7 @@ namespace NeuralAudio layer.SetNAMWeights(it); } - for (int i = 0; i < hiddenSize; i++) + for (size_t i = 0; i < hiddenSize; i++) headWeights[i] = *(it++); headBias = *(it++); diff --git a/NeuralAudio/WaveNetDynamic.h b/NeuralAudio/WaveNetDynamic.h index 948ce89..6ce7818 100644 --- a/NeuralAudio/WaveNetDynamic.h +++ b/NeuralAudio/WaveNetDynamic.h @@ -108,13 +108,13 @@ namespace NeuralAudio void SetWeights(std::vector::iterator& inWeights) { - for (auto i = 0; i < outSize; i++) - for (auto j = 0; j < inSize; j++) + for (size_t i = 0; i < outSize; i++) + for (size_t j = 0; j < inSize; j++) weights(i, j) = *(inWeights++); if (doBias) { - for (auto i = 0; i < outSize; i++) + for (size_t i = 0; i < outSize; i++) bias(i) = *(inWeights++); } } @@ -147,10 +147,7 @@ namespace NeuralAudio class WaveNetLayer { private: - size_t conditionSize; size_t channels; - size_t kernelSize; - size_t dilation; Conv1D conv1D; DenseLayer inputMixin; DenseLayer oneByOne; @@ -162,10 +159,7 @@ namespace NeuralAudio size_t bufferStart; WaveNetLayer(size_t conditionSize, size_t channels, size_t kernelSize, size_t dilation) : - conditionSize(conditionSize), channels(channels), - kernelSize(kernelSize), - dilation(dilation), conv1D(channels, channels, kernelSize, true, dilation), inputMixin(conditionSize, channels, false), oneByOne(channels, channels, true), @@ -263,11 +257,7 @@ namespace NeuralAudio class WaveNetLayerArray { private: - size_t inputSize; - size_t conditionSize; - size_t headSize; size_t channels; - size_t kernelSize; std::vector layers; DenseLayer rechannel; DenseLayer headRechannel; @@ -278,11 +268,7 @@ namespace NeuralAudio public: WaveNetLayerArray(size_t inputSize, size_t conditionSize, size_t headSize, size_t channels, size_t kernelSize, bool hasHeadBias, std::vector dilations) : - inputSize(inputSize), - conditionSize(conditionSize), - headSize(headSize), channels(channels), - kernelSize(kernelSize), rechannel(inputSize, channels, false), headRechannel(channels, headSize, hasHeadBias), arrayOutputs(channels, WAVENET_MAX_NUM_FRAMES), diff --git a/Utils/ModelTest/ModelTest.cpp b/Utils/ModelTest/ModelTest.cpp index 8681d57..5237e6c 100644 --- a/Utils/ModelTest/ModelTest.cpp +++ b/Utils/ModelTest/ModelTest.cpp @@ -252,7 +252,19 @@ int RunDefaultTests(NeuralModelLoader& loader, int blockSize) std::cout << "Loading models from: " << modelPath << std::endl << std::endl; - std::cout << "WaveNet (Standard) Test" << std::endl; + std::cout << "WaveNet (A2 Full) Test" << std::endl; + loader.SetDefaultQualityScaleFactor(1.0f); + RunNAMTests(modelPath / "BossWN-a2.nam", loader, blockSize); + + std::cout << std::endl; + + std::cout << "WaveNet (A2 Lite) Test" << std::endl; + loader.SetDefaultQualityScaleFactor(0.0f); + RunNAMTests(modelPath / "BossWN-a2.nam", loader, blockSize); + + std::cout << std::endl; + + std::cout << "WaveNet (A1 Standard) Test" << std::endl; RunNAMTests(modelPath / "BossWN-standard.nam", loader, blockSize); std::cout << std::endl; From 305d866418d36f46300f8a6a8c25c1f74c1ea4c9 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 10 Jun 2026 08:45:28 -0700 Subject: [PATCH 069/162] Compiler warning fix --- NeuralAudio/WaveNetDynamic.h | 1 + 1 file changed, 1 insertion(+) diff --git a/NeuralAudio/WaveNetDynamic.h b/NeuralAudio/WaveNetDynamic.h index 6ce7818..de84ea7 100644 --- a/NeuralAudio/WaveNetDynamic.h +++ b/NeuralAudio/WaveNetDynamic.h @@ -159,6 +159,7 @@ namespace NeuralAudio size_t bufferStart; WaveNetLayer(size_t conditionSize, size_t channels, size_t kernelSize, size_t dilation) : + bufferStart(0), channels(channels), conv1D(channels, channels, kernelSize, true, dilation), inputMixin(conditionSize, channels, false), From 7310b7f726a90ce740738cd058cba3d1f149e385 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 10 Jun 2026 08:46:57 -0700 Subject: [PATCH 070/162] More compiler fix --- NeuralAudio/WaveNetDynamic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NeuralAudio/WaveNetDynamic.h b/NeuralAudio/WaveNetDynamic.h index de84ea7..6fa77e7 100644 --- a/NeuralAudio/WaveNetDynamic.h +++ b/NeuralAudio/WaveNetDynamic.h @@ -159,13 +159,13 @@ namespace NeuralAudio size_t bufferStart; WaveNetLayer(size_t conditionSize, size_t channels, size_t kernelSize, size_t dilation) : - bufferStart(0), channels(channels), conv1D(channels, channels, kernelSize, true, dilation), inputMixin(conditionSize, channels, false), oneByOne(channels, channels, true), state(channels, WAVENET_MAX_NUM_FRAMES), - ReceptiveFieldSize((kernelSize - 1) * dilation) + ReceptiveFieldSize((kernelSize - 1) * dilation), + bufferStart(0) { state.setZero(); } From 6192f14cdb73495e78b887c5157c0439496a6b70 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 10 Jun 2026 17:29:03 -0700 Subject: [PATCH 071/162] Expose receptive field size from NAM Core models --- NeuralAudio/NAMModel.h | 5 +++++ deps/NeuralAmpModelerCore | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index 9fbfdfb..324bcaf 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -29,6 +29,11 @@ namespace NeuralAudio return EModelLoadMode::NAMCore; } + int GetReceptiveFieldSize() override + { + return namModel->GetPrewarmSamples(); + } + bool LoadFromJson(const nlohmann::json& modelJson) { if (namModel) diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index d5e93a7..c1ac112 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit d5e93a7e11f75affe85ccf8b82c0ff2a1105133a +Subproject commit c1ac112ece8c3cba9b5a59fad0468a43a47307ba From aaa178b8239e30bd2a892ec49c8280666e67fec3 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 11 Jun 2026 06:41:12 -0700 Subject: [PATCH 072/162] Update NAM Core (revert A2 full optimization) --- deps/NeuralAmpModelerCore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index c1ac112..2f5a2f5 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit c1ac112ece8c3cba9b5a59fad0468a43a47307ba +Subproject commit 2f5a2f505307745311175c7aacef4e3b8cbe8666 From 951c7afcf03de037487f99684aee5746d4800f42 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 11 Jun 2026 06:51:06 -0700 Subject: [PATCH 073/162] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7572eb4..cb81f26 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ int receptiveFieldSamples = model->GetReceptiveFieldSize(); Note that this can return -1, which means that the receptive field size is unknown, or not fixed (ie: LSTM models technically have an infinite tail because of their internal feedback loop). -This method is only supported for "internal" models. If you are using NAM Core or RTNeural it will always return -1. +This method is only supported for "internal" and NAM Core models. For RTNeural it will always return -1. # Building From 539d9fe75ea031bc6f161c4bb7e4f10312664265 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 11 Jun 2026 09:32:56 -0700 Subject: [PATCH 074/162] Handle loading of composite models (A2 slimmable containers) internally --- NeuralAudio/CMakeLists.txt | 1 + NeuralAudio/CompositeModel.h | 156 ++++++++++++++++++++++++++++++++++ NeuralAudio/NAMModel.h | 3 - NeuralAudio/NeuralModel.cpp | 56 ++++++++---- NeuralAudio/NeuralModel.h | 6 +- NeuralAudio/NeuralModelImpl.h | 1 - NeuralAudio/WaveNet.h | 1 - NeuralAudio/WaveNetDynamic.h | 1 - 8 files changed, 199 insertions(+), 26 deletions(-) create mode 100644 NeuralAudio/CompositeModel.h diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 0046129..27f600e 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -95,6 +95,7 @@ set(SOURCES LSTM.h LSTMDynamic.h InternalModel.h + CompositeModel.h TemplateHelper.h) if(BUILD_NAMCORE) diff --git a/NeuralAudio/CompositeModel.h b/NeuralAudio/CompositeModel.h new file mode 100644 index 0000000..29647a9 --- /dev/null +++ b/NeuralAudio/CompositeModel.h @@ -0,0 +1,156 @@ +#pragma once + +#include +#include +#include "NeuralModel.h" +#include "NeuralModelImpl.h" + +namespace NeuralAudio +{ + class CompositeModel : public NeuralModelImpl + { + public: + CompositeModel() + { + } + + ~CompositeModel() + { + for (auto& model : models) + { + delete model; + } + } + + void AddModel(NeuralModel* model) + { + if (currentModelIndex.load() == -1) + currentModelIndex.store(0); + + models.push_back(model); + } + + int GetModelCount() + { + return (int)models.size(); + } + + int GetCurrentModelIndex() + { + return currentModelIndex.load(); + } + + void SetCurrentModelIndex(int index) + { + currentModelIndex.store(index); + } + + void SetMaxAudioBufferSize(const int maxSize) override + { + for (auto& model : models) + { + model->SetMaxAudioBufferSize(maxSize); + } + } + + void Process(float* input, float* output, size_t numSamples) override + { + if (currentModelIndex == -1) + return; + + models[currentModelIndex.load()]->Process(input, output, numSamples); + } + + void Prewarm() override + { + for (auto& model : models) + { + model->Prewarm(); + } + } + + protected: + std::vector models; + std::atomic currentModelIndex = -1; + }; + + + class ScalableCompositeModel : public CompositeModel + { + public: + EModelLoadMode GetLoadMode() override + { + return EModelLoadMode::NAMCore; + } + + bool LoadFromJson(const nlohmann::json& modelJson) + { + ReadNAMConfig(modelJson); + + return CreateModelFromNAMJson(modelJson); + } + + virtual bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + { + nlohmann::json config = modelJson.at("config"); + + nlohmann::json subModels = config.at("submodels"); + + for (auto& submodelJson : subModels) + { + NeuralModel* submodel = loader->CreateFromJson(submodelJson.at("model"), ".nam", false); + + AddModel(submodelJson.at("max_value"), submodel); + } + + SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); + SetQualityScaleFactor(loader->GetDefaultQualityScaleFactor()); + + return true; + } + + bool HasQualityScaling() override + { + return true; + } + + float GetQualityScaleFactor() override + { + return currentQualityLevel.load(); + } + + void SetQualityScaleFactor(float scaleFactor) override + { + currentQualityLevel.store(scaleFactor); + + int modelIndex = 0; + + for (auto& level : qualityLevels) + { + if (std::get<0>(level) > scaleFactor) + break; + + modelIndex = std::get<1>(level); + } + + SetCurrentModelIndex(modelIndex); + } + + void AddModel(float scaleFactor, NeuralModel* model) + { + CompositeModel::AddModel(model); + + qualityLevels.emplace_back(scaleFactor, GetModelCount() - 1); + + // Sort by quality level + std::sort(qualityLevels.begin(), qualityLevels.end(), [](const auto& a, const auto& b) + { + return std::get<0>(a) < std::get<0>(b); + }); + } + + protected: + std::atomic currentQualityLevel = 1.0f; + std::vector> qualityLevels; + }; +} \ No newline at end of file diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index 324bcaf..b21a2c0 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -36,9 +36,6 @@ namespace NeuralAudio bool LoadFromJson(const nlohmann::json& modelJson) { - if (namModel) - namModel.reset(); - slimmableSize = loader->GetDefaultQualityScaleFactor(); ReadNAMConfig(modelJson); diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 5a2d324..ffada5e 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -8,6 +8,7 @@ #endif #include "RTNeuralModel.h" #include "InternalModel.h" +#include "CompositeModel.h" namespace NeuralAudio { @@ -116,16 +117,6 @@ namespace NeuralAudio return true; } - NeuralModel* NeuralModelLoader::CreateFromFile(std::filesystem::path modelPath) - { - if (!std::filesystem::exists(modelPath)) - return nullptr; - - std::ifstream jsonStream(modelPath, std::ifstream::binary); - - return CreateFromStream(jsonStream, modelPath.extension()); - } - bool NAMIsA2(std::string version) { int major = 0, minor = 0, patch = 0; @@ -137,32 +128,61 @@ namespace NeuralAudio return (major > 0) || (minor > 5) || ((minor == 5) && (patch > 4)); } - NeuralModel* NeuralModelLoader::CreateFromStream(std::basic_istream& jsonStream, std::filesystem::path extension) + NeuralModel* NeuralModelLoader::CreateFromFile(const std::filesystem::path& modelPath, bool doPrewarm) { - EnsureModelDefsAreLoaded(); + if (!std::filesystem::exists(modelPath)) + return nullptr; + + std::ifstream jsonStream(modelPath, std::ifstream::binary); + + return CreateFromStream(jsonStream, modelPath.extension(), doPrewarm); + } + + NeuralModel* NeuralModelLoader::CreateFromStream(std::basic_istream& jsonStream, const std::filesystem::path& extension, bool doPrewarm) + { nlohmann::json modelJson; jsonStream >> modelJson; + return CreateFromJson(modelJson, extension, doPrewarm); + } + + NeuralModel* NeuralModelLoader::CreateFromJson(const nlohmann::json modelJson, const std::filesystem::path& extension, bool doPrewarm) + { + EnsureModelDefsAreLoaded(); + NeuralModel* newModel = nullptr; if (extension == ".nam") { + std::string arch = modelJson.at("architecture"); + #ifdef BUILD_NAMCORE std::string version = modelJson.at("version"); if ((wavenetLoadMode == EModelLoadMode::NAMCore) || NAMIsA2(version)) { - NAMModel* model = new NAMModel; + if (arch == "SlimmableContainer") // Packed A2 multi-model file + { + ScalableCompositeModel* model = new ScalableCompositeModel; - model->SetModelLoader(this); - model->LoadFromJson(modelJson); + model->SetModelLoader(this); + model->LoadFromJson(modelJson); - newModel = model; + newModel = model; + } + else + { + NAMModel* model = new NAMModel; + + model->SetModelLoader(this); + model->LoadFromJson(modelJson); + + newModel = model; + } } #endif - std::string arch = modelJson.at("architecture"); nlohmann::json config = modelJson.at("config"); if (newModel == nullptr) @@ -328,7 +348,7 @@ namespace NeuralAudio } } - if (newModel != nullptr) + if ((newModel != nullptr) && doPrewarm) { newModel->Prewarm(); } diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index ef3559c..0bf2234 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -5,6 +5,7 @@ #include #include #include +#include "json.hpp" #ifndef DEFAULT_QUALITY_SCALE #define DEFAULT_QUALITY_SCALE 1.0 @@ -120,8 +121,9 @@ namespace NeuralAudio class NeuralModelLoader { public: - NeuralModel* CreateFromFile(std::filesystem::path modelPath); - NeuralModel* CreateFromStream(std::basic_istream& stream, std::filesystem::path extension); + NeuralModel* CreateFromFile(const std::filesystem::path& modelPath, bool doPrewarm = true); + NeuralModel* CreateFromStream(std::basic_istream& stream, const std::filesystem::path& extension, bool doPrewarm = true); + NeuralModel* CreateFromJson(const nlohmann::json modelJson, const std::filesystem::path& extension, bool doPrewarm = true); bool SetLSTMLoadMode(EModelLoadMode val) { diff --git a/NeuralAudio/NeuralModelImpl.h b/NeuralAudio/NeuralModelImpl.h index d8856e6..4e51820 100644 --- a/NeuralAudio/NeuralModelImpl.h +++ b/NeuralAudio/NeuralModelImpl.h @@ -1,6 +1,5 @@ #pragma once -#include "json.hpp" #include "NeuralModel.h" namespace NeuralAudio diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 81da190..8580503 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -6,7 +6,6 @@ #include #include #include "TemplateHelper.h" -#include "NeuralModel.h" #include "Activation.h" #ifndef WAVENET_MAX_NUM_FRAMES diff --git a/NeuralAudio/WaveNetDynamic.h b/NeuralAudio/WaveNetDynamic.h index 6fa77e7..2179c90 100644 --- a/NeuralAudio/WaveNetDynamic.h +++ b/NeuralAudio/WaveNetDynamic.h @@ -5,7 +5,6 @@ #include #include -#include "NeuralModel.h" #include "Activation.h" #ifndef WAVENET_MAX_NUM_FRAMES From 94555774d2f55a8c74c3f3bf6634b8b7d845c6c4 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 11 Jun 2026 10:02:20 -0700 Subject: [PATCH 075/162] Fix quality scaling model selection --- NeuralAudio/CompositeModel.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NeuralAudio/CompositeModel.h b/NeuralAudio/CompositeModel.h index 29647a9..7e6e113 100644 --- a/NeuralAudio/CompositeModel.h +++ b/NeuralAudio/CompositeModel.h @@ -127,10 +127,10 @@ namespace NeuralAudio for (auto& level : qualityLevels) { - if (std::get<0>(level) > scaleFactor) - break; - modelIndex = std::get<1>(level); + + if (scaleFactor <= std::get<0>(level)) + break; } SetCurrentModelIndex(modelIndex); From 6c3d909736eb6d0cc38df9500600ade6c038c21c Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 12 Jun 2026 07:54:24 -0700 Subject: [PATCH 076/162] Switch from prewarm workaround to new NAM Core method --- NeuralAudio/CompositeModel.h | 5 ++++- NeuralAudio/NAMModel.h | 6 +++--- deps/NeuralAmpModelerCore | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/NeuralAudio/CompositeModel.h b/NeuralAudio/CompositeModel.h index 7e6e113..0d2d54d 100644 --- a/NeuralAudio/CompositeModel.h +++ b/NeuralAudio/CompositeModel.h @@ -103,7 +103,10 @@ namespace NeuralAudio AddModel(submodelJson.at("max_value"), submodel); } - SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); + // Don't call SetMaxAudioBufferSize because it has already been done by individual models + // (this can be re-added for completeness if NAM Core GetMaxBufferSize() is made public + //SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); + SetQualityScaleFactor(loader->GetDefaultQualityScaleFactor()); return true; diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index b21a2c0..ffc1e7e 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -40,9 +40,9 @@ namespace NeuralAudio ReadNAMConfig(modelJson); - namModel = nam::get_dsp(modelJson); + nam::ScopedPrewarmOnResetDefault scoped_prewarm_default(false); - namModel->SuppressPrewarm(true); + namModel = nam::get_dsp(modelJson); auto* slim = dynamic_cast(namModel.get()); @@ -55,7 +55,7 @@ namespace NeuralAudio SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); - namModel->SuppressPrewarm(false); + namModel->SetPrewarmOnReset(true); return true; } diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index 2f5a2f5..fea0ac0 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit 2f5a2f505307745311175c7aacef4e3b8cbe8666 +Subproject commit fea0ac03d2ac65c924ed41d61400740413b5edc6 From f08ea65239e2e5271a1dc9902a2addb0bcd6e862 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 12 Jun 2026 09:26:48 -0700 Subject: [PATCH 077/162] Add composite model load mode and method to check quality scale RT safety --- NeuralAudio/CompositeModel.h | 77 +++++++++++++++++++++++++++-------- NeuralAudio/NeuralModel.cpp | 2 +- NeuralAudio/NeuralModel.h | 22 ++++++++++ NeuralAudio/NeuralModelImpl.h | 11 +++++ 4 files changed, 94 insertions(+), 18 deletions(-) diff --git a/NeuralAudio/CompositeModel.h b/NeuralAudio/CompositeModel.h index 0d2d54d..0b5d150 100644 --- a/NeuralAudio/CompositeModel.h +++ b/NeuralAudio/CompositeModel.h @@ -22,7 +22,7 @@ namespace NeuralAudio } } - void AddModel(NeuralModel* model) + void AddModel(NeuralModelImpl* model) { if (currentModelIndex.load() == -1) currentModelIndex.store(0); @@ -40,9 +40,29 @@ namespace NeuralAudio return currentModelIndex.load(); } + bool IsModelChangeRealtimeSafe(int newIndex) + { + if (newIndex == currentModelIndex.load()) + return true; + + return models[newIndex]->HadInitialPrewarm(); + } + void SetCurrentModelIndex(int index) { - currentModelIndex.store(index); + if (index != currentModelIndex.load()) + { + currentModelIndex.store(index); + + if (compositeLoadMode == ECompositeModelLoadMode::OnDemand) + { + // For on-demand loading we need to check if the model we are switching to has been prewarmed + if (!models[currentModelIndex.load()]->HadInitialPrewarm()) + { + Prewarm(); + } + } + } } void SetMaxAudioBufferSize(const int maxSize) override @@ -63,15 +83,26 @@ namespace NeuralAudio void Prewarm() override { - for (auto& model : models) + if (compositeLoadMode == ECompositeModelLoadMode::OnDemand) { + auto& model = models[currentModelIndex.load()]; model->Prewarm(); + model->SetHadInitialPrewarm(); + } + else + { + for (auto& model : models) + { + model->Prewarm(); + model->SetHadInitialPrewarm(); + } } } protected: - std::vector models; + std::vector models; std::atomic currentModelIndex = -1; + ECompositeModelLoadMode compositeLoadMode = ECompositeModelLoadMode::LoadAll; }; @@ -92,13 +123,15 @@ namespace NeuralAudio virtual bool CreateModelFromNAMJson(const nlohmann::json& modelJson) { + compositeLoadMode = loader->GetCompositeModelLoadMode(); + nlohmann::json config = modelJson.at("config"); nlohmann::json subModels = config.at("submodels"); for (auto& submodelJson : subModels) { - NeuralModel* submodel = loader->CreateFromJson(submodelJson.at("model"), ".nam", false); + NeuralModelImpl* submodel = dynamic_cast(loader->CreateFromJson(submodelJson.at("model"), ".nam", false)); AddModel(submodelJson.at("max_value"), submodel); } @@ -122,24 +155,19 @@ namespace NeuralAudio return currentQualityLevel.load(); } + bool IsSetQualityChangeRealtimeSafe(float newScaleFactor) override + { + return IsModelChangeRealtimeSafe(GetModelIndexFromQualityScale(newScaleFactor)); + } + void SetQualityScaleFactor(float scaleFactor) override { currentQualityLevel.store(scaleFactor); - int modelIndex = 0; - - for (auto& level : qualityLevels) - { - modelIndex = std::get<1>(level); - - if (scaleFactor <= std::get<0>(level)) - break; - } - - SetCurrentModelIndex(modelIndex); + SetCurrentModelIndex(GetModelIndexFromQualityScale(scaleFactor)); } - void AddModel(float scaleFactor, NeuralModel* model) + void AddModel(float scaleFactor, NeuralModelImpl* model) { CompositeModel::AddModel(model); @@ -155,5 +183,20 @@ namespace NeuralAudio protected: std::atomic currentQualityLevel = 1.0f; std::vector> qualityLevels; + + int GetModelIndexFromQualityScale(float qualityScale) + { + int modelIndex = 0; + + for (auto& level : qualityLevels) + { + modelIndex = std::get<1>(level); + + if (qualityScale <= std::get<0>(level)) + break; + } + + return modelIndex; + } }; } \ No newline at end of file diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index ffada5e..ea14525 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -151,7 +151,7 @@ namespace NeuralAudio { EnsureModelDefsAreLoaded(); - NeuralModel* newModel = nullptr; + NeuralModelImpl* newModel = nullptr; if (extension == ".nam") { diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index 0bf2234..d58cd5b 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -20,6 +20,12 @@ namespace NeuralAudio NAMCore }; + enum ECompositeModelLoadMode + { + LoadAll, + OnDemand + }; + class NeuralModel { public: @@ -42,6 +48,11 @@ namespace NeuralAudio return 1.0f; } + virtual bool IsSetQualityChangeRealtimeSafe(float newScaleFactor) + { + return true; + } + virtual void SetQualityScaleFactor(float scaleFactor) { (void)scaleFactor; @@ -145,6 +156,16 @@ namespace NeuralAudio return true; } + ECompositeModelLoadMode GetCompositeModelLoadMode() + { + return compositeLoadMode; + } + + void SetCompositeModelLoadMode(ECompositeModelLoadMode loadMode) + { + compositeLoadMode = loadMode; + } + bool SupportsWaveNetLoadMode(EModelLoadMode mode); bool SupportsLSTMLoadMode(EModelLoadMode mode); @@ -181,6 +202,7 @@ namespace NeuralAudio protected: EModelLoadMode lstmLoadMode = EModelLoadMode::Internal; EModelLoadMode wavenetLoadMode = EModelLoadMode::Internal; + ECompositeModelLoadMode compositeLoadMode = ECompositeModelLoadMode::LoadAll; float audioInputLevelDBu = 12; int defaultMaxAudioBufferSize = 128; float defaultQualityScaleFactor = DEFAULT_QUALITY_SCALE; diff --git a/NeuralAudio/NeuralModelImpl.h b/NeuralAudio/NeuralModelImpl.h index 4e51820..1886cb0 100644 --- a/NeuralAudio/NeuralModelImpl.h +++ b/NeuralAudio/NeuralModelImpl.h @@ -14,6 +14,16 @@ namespace NeuralAudio this->loader = modelLoader; } + bool HadInitialPrewarm() + { + return hadInitialPrewarm; + } + + void SetHadInitialPrewarm() + { + hadInitialPrewarm = true; + } + protected: void ReadNAMConfig(const nlohmann::json& modelJson) { @@ -102,5 +112,6 @@ namespace NeuralAudio } NeuralModelLoader *loader; + bool hadInitialPrewarm = false; }; } \ No newline at end of file From c6b95756417e57ddb94f93e8066a30c1b4c0abd2 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 12 Jun 2026 11:03:19 -0700 Subject: [PATCH 078/162] Fix static RTNeural --- NeuralAudio/RTNeuralLoader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NeuralAudio/RTNeuralLoader.cpp b/NeuralAudio/RTNeuralLoader.cpp index 2d3fda8..28f3249 100644 --- a/NeuralAudio/RTNeuralLoader.cpp +++ b/NeuralAudio/RTNeuralLoader.cpp @@ -45,7 +45,7 @@ namespace NeuralAudio return nullptr; } - NeuralModel* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson, NeuralModelLoader *loader) + NeuralModelImpl* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson, NeuralModelLoader *loader) { nlohmann::json config = modelJson.at("config"); From bd2190a2346a16ac614f941afcadf752e9920a90 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 12 Jun 2026 11:09:03 -0700 Subject: [PATCH 079/162] More RTNeural fixes --- NeuralAudio/NeuralModel.h | 2 +- NeuralAudio/RTNeuralLoader.cpp | 4 ++-- NeuralAudio/RTNeuralLoader.h | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index d58cd5b..93b899b 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -205,7 +205,7 @@ namespace NeuralAudio ECompositeModelLoadMode compositeLoadMode = ECompositeModelLoadMode::LoadAll; float audioInputLevelDBu = 12; int defaultMaxAudioBufferSize = 128; - float defaultQualityScaleFactor = DEFAULT_QUALITY_SCALE; + float defaultQualityScaleFactor = (float)DEFAULT_QUALITY_SCALE; }; } \ No newline at end of file diff --git a/NeuralAudio/RTNeuralLoader.cpp b/NeuralAudio/RTNeuralLoader.cpp index 28f3249..bceffa4 100644 --- a/NeuralAudio/RTNeuralLoader.cpp +++ b/NeuralAudio/RTNeuralLoader.cpp @@ -67,7 +67,7 @@ namespace NeuralAudio return nullptr; } - NeuralModel* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader *loader) + NeuralModelImpl* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader *loader) { nlohmann::json config = modelJson.at("config"); @@ -94,7 +94,7 @@ namespace NeuralAudio return nullptr; } - NeuralModel* RTNeuralLoadKeras(const nlohmann::json& modelJson, NeuralModelLoader* loader) + NeuralModelImpl* RTNeuralLoadKeras(const nlohmann::json& modelJson, NeuralModelLoader* loader) { const auto layers = modelJson.at("layers"); const size_t numLayers = layers.size() - 1; diff --git a/NeuralAudio/RTNeuralLoader.h b/NeuralAudio/RTNeuralLoader.h index b8b1886..72f0d21 100644 --- a/NeuralAudio/RTNeuralLoader.h +++ b/NeuralAudio/RTNeuralLoader.h @@ -6,7 +6,7 @@ namespace NeuralAudio { extern void EnsureRTNeuralModelDefsAreLoaded(); - extern NeuralModel* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson, NeuralModelLoader* loader); - extern NeuralModel* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader* loader); - extern NeuralModel* RTNeuralLoadKeras(const nlohmann::json& modelJson, NeuralModelLoader* loader); + extern NeuralModelImpl* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson, NeuralModelLoader* loader); + extern NeuralModelImpl* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader* loader); + extern NeuralModelImpl* RTNeuralLoadKeras(const nlohmann::json& modelJson, NeuralModelLoader* loader); } \ No newline at end of file From d36ea893659e2e33df939d5682283381a556d264 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 12 Jun 2026 11:37:23 -0700 Subject: [PATCH 080/162] IsSetQualityChangeRealtimeSafe -> IsQualityChangeRealtimeSafe --- NeuralAudio/CompositeModel.h | 2 +- NeuralAudio/NeuralModel.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NeuralAudio/CompositeModel.h b/NeuralAudio/CompositeModel.h index 0b5d150..7a7ea96 100644 --- a/NeuralAudio/CompositeModel.h +++ b/NeuralAudio/CompositeModel.h @@ -155,7 +155,7 @@ namespace NeuralAudio return currentQualityLevel.load(); } - bool IsSetQualityChangeRealtimeSafe(float newScaleFactor) override + bool IsQualityChangeRealtimeSafe(float newScaleFactor) override { return IsModelChangeRealtimeSafe(GetModelIndexFromQualityScale(newScaleFactor)); } diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index 93b899b..30e7d55 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -48,7 +48,7 @@ namespace NeuralAudio return 1.0f; } - virtual bool IsSetQualityChangeRealtimeSafe(float newScaleFactor) + virtual bool IsQualityChangeRealtimeSafe(float newScaleFactor) { return true; } From c90ceff9b7843f089f7ac3c0b543910e6333ec48 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 12 Jun 2026 11:40:06 -0700 Subject: [PATCH 081/162] Add composite model loading information --- README.md | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cb81f26..0fddae7 100644 --- a/README.md +++ b/README.md @@ -76,13 +76,13 @@ To set a known audio input level (ie: from an audio interface), use ```loader.Se By default, models are loaded using the internal NeuralAudio implementation. If you would like to force the use of the NAM Core or RTNeural implementations, you can use: ``` -loader.SetWaveNetLoadMode(loadMode) +loader.SetWaveNetLoadMode(loadMode); ``` and ``` -loader.SetLSTMLoadMode(loadMode) +loader.SetLSTMLoadMode(loadMode); ``` where "loadMode" is one of: @@ -97,6 +97,18 @@ You can check which implementation was actually used to load the model with ```m **NOTE:** Because of compile time and executable size considerations, only the internal, NAM Core and dynamic RTNeural implementations are built by default. If you want to use RTNeural, it is recommended that you add ```-DBUILD_STATIC_RTNEURAL=ON``` to your cmake commandline. This will create static model implmentations for the same sets of WaveNet and LSTM models as the internal implmentation, and results in increased performance. Interal static LSTM model support is also off by default - to turn it on use ```-DBUILD_INTERNAL_STATIC_LSTM=ON```. +### Composite model load behavior + +Some models (notably NAM A2 models) are comprised of multiple sub-models. By default, all sub-models will be fully loaded and initialized when the model is loaded. + +If you wish to avoid the overhead of initializing unused models and only initialize the active model on load, you can do: + +``` +loader->SetCompositeModelLoadMode(ECompositeModelLoadMode::OnDemand); +``` + +Note that this means that switching to a different model for the first time via quality scaling will ***not be realtime safe***. + ## Setting model quality scaling factor Some models (notably, slimmable NAM A2 models) support quality scaling - trading off quality for performance. @@ -121,9 +133,16 @@ To set the quality scaling factor for a loaded model, do: model->SetQualityScaleFactor(scaleFactor); ``` -***Note: This operation is not real-time safe if the quality scale factor results in a change to the model*** +***Note: This operation is not real-time safe if the quality scale factor results in switching to an uninitialized model.*** If you are using the default composite model loading behavior, setting the quality scale factor is always real-time safe. If you are using "OnDemand" composite model loading, you can check whether a quality scale change is real-time safe by doing: + +``` +if (!model->IsQualityChangeRealtimeSafe(newScaleFactor) +{ + (call SetQualityScaleFactor(), but ensure it is not done in a real-time context) +} +``` -To get the quality scaling factor for a model, do: +To get the current quality scaling factor for a model, do: ``` float scaleFactor = model->GetQualityScaleFactor(); From 0943ba73a8540c06be69f9ea5fe66a6af5397314 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 12 Jun 2026 12:59:43 -0700 Subject: [PATCH 082/162] Add DEFAULT_INPUT_DBU cmake option --- NeuralAudio/CMakeLists.txt | 6 ++++++ NeuralAudio/NeuralModel.h | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 27f600e..6daf964 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -65,6 +65,12 @@ add_definitions(-DDEFAULT_QUALITY_SCALE=${DEFAULT_QUALITY_SCALE}) message(STATUS "Default model quality scale factor is: ${DEFAULT_QUALITY_SCALE}") +set(DEFAULT_INPUT_DBU "12" CACHE STRING "Default dBu level for input scaling") + +add_definitions(-DDEFAULT_INPUT_DBU=${DEFAULT_INPUT_DBU}) + +message(STATUS "Default input dBu is: ${DEFAULT_INPUT_DBU}") + set(WAVENET_FRAMES "64" CACHE STRING "WaveNet frame size") add_definitions(-DWAVENET_MAX_NUM_FRAMES=${WAVENET_FRAMES}) diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index 30e7d55..180dfb8 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -11,6 +11,10 @@ #define DEFAULT_QUALITY_SCALE 1.0 #endif +#ifndef DEFAULT_INPUT_DBU +#define DEFAULT_INPUT_DBU 12 +#endif + namespace NeuralAudio { enum EModelLoadMode @@ -203,7 +207,7 @@ namespace NeuralAudio EModelLoadMode lstmLoadMode = EModelLoadMode::Internal; EModelLoadMode wavenetLoadMode = EModelLoadMode::Internal; ECompositeModelLoadMode compositeLoadMode = ECompositeModelLoadMode::LoadAll; - float audioInputLevelDBu = 12; + float audioInputLevelDBu = (float)DEFAULT_INPUT_DBU; int defaultMaxAudioBufferSize = 128; float defaultQualityScaleFactor = (float)DEFAULT_QUALITY_SCALE; }; From a3b0fa71e9d0f784ee7a255fe551a4f553412439 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 12 Jun 2026 13:01:19 -0700 Subject: [PATCH 083/162] Add DEFAULT_INPUT_DBU --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0fddae7..c3a969f 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,8 @@ Note - you'll have to change the Visual Studio version if you are using a differ ```-DDEFAULT_QUALITY_SCALE="X.X"```: Default model quality scale factor (0.0 to 1.0). Be sure to use quotes around value. Defaults to "1.0". +```-DDEFAULT_INPUT_DBU="XX"```: Default dBu level for model input calibration. + ```-DWAVENET_FRAMES=XXX```: Sample buffer size for the internal WaveNet implementation. Defaults to **64**. If you know you are going to be using a fixed sample buffer smaller or larger than this, use that instead. Note that the model will still be able to process any buffer size - it is just optimized for this size. ```-DBUFFER_PADDING=XXX```: Amount of padding to convolution layer buffers. This allows ring buffer resets to be staggered accross layers to improve performance. It also uses a significant amount of memory. It is set to **24** by default. It can be set all the way down to 0 to reduce memory usage. From 29071b1d407d0c9fa115a39478d2827802c91dc3 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 12 Jun 2026 14:33:39 -0700 Subject: [PATCH 084/162] Fix compiler warning --- NeuralAudio/NeuralModel.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index 180dfb8..0a7a0b1 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -54,6 +54,8 @@ namespace NeuralAudio virtual bool IsQualityChangeRealtimeSafe(float newScaleFactor) { + (void)newScaleFactor; + return true; } From 530f1ffda3e36787913919d84ad41b66e21cf4e7 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 12 Jun 2026 15:53:52 -0700 Subject: [PATCH 085/162] Extra gating check on NAM Core SetMaxAudioBufferSize() --- NeuralAudio/NAMModel.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index ffc1e7e..cca196d 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -90,7 +90,12 @@ namespace NeuralAudio void SetMaxAudioBufferSize(const int maxSize) override { - namModel->Reset(namModel->GetExpectedSampleRate(), maxSize); + if (maxSize != currentMaxSize) // Don't reset if we don't need to + { + namModel->Reset(namModel->GetExpectedSampleRate(), maxSize); + + currentMaxSize = maxSize; + } } void Process(float* input, float* output, size_t numSamples) override @@ -105,6 +110,7 @@ namespace NeuralAudio private: std::unique_ptr namModel = nullptr; + int currentMaxSize = -1; float slimmableSize = 1.0f; bool isSlimmable = false; }; From 401727d38b19e1985de57499e978dbfb8a248048 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 13 Jun 2026 12:25:32 -0700 Subject: [PATCH 086/162] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c3a969f..1996db4 100644 --- a/README.md +++ b/README.md @@ -249,4 +249,6 @@ The following applications and devices are using the NeuralAudio library for mod - [stompbox](https://github.com/mikeoliphant/stompbox): Guitar amplification and effects pedalboard simulation. - [NeuralRack](https://github.com/brummer10/NeuralRack): Neural Model and Impulse Response File loader for Linux/Windows. - [Darkglass Anagram](https://www.darkglass.com/creation/anagram): Bass guitar effects unit. -- [neural_tilde](https://github.com/apresta/neural_tilde): Max/MSP external for running neural amplifier captures +- [neural_tilde](https://github.com/apresta/neural_tilde): Max/MSP external for running neural amplifier captures. +- [Pi Pedal](https://github.com/rerdavies/pipedal): Guitar Effect Pedal for Raspberry Pi. +- [Casette](https://github.com/pawelKapl/Casette): Raspberry Pie guitar processor. From 3ab4cda220ff7d55662da8a8326c6b6a03c454d9 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sun, 14 Jun 2026 09:18:54 -0700 Subject: [PATCH 087/162] Add WaveNet oversampling if external sample rate is an even multiple of model sample rate --- NeuralAudio/NeuralModel.cpp | 37 ++++++++++++++++++++++++++++++++++++- NeuralAudio/NeuralModel.h | 8 +++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index ea14525..f9c3bbd 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -86,6 +86,39 @@ namespace NeuralAudio return true; } + static void OversampleNAMConfig(nlohmann::json& modelJson, int externalSampleRate) + { + std::string arch = modelJson.at("architecture"); + + if (arch != "WaveNet") + return; + + int modelSampleRate = 48000; + + if (modelJson.contains("sample_rate") && modelJson.at("sample_rate").is_number()) + { + modelSampleRate = (int)modelJson.at("sample_rate").get(); + } + + if (modelSampleRate == externalSampleRate) + return; // No oversampling required + + if ((externalSampleRate % modelSampleRate) != 0) + return; // Not an integer multiple + + int oversampleFactor = externalSampleRate / modelSampleRate; + + auto& config = modelJson.at("config"); + + for (auto& layer : config.at("layers")) + { + for (auto& dilation : layer.at("dilations")) + { + dilation = dilation.get() * oversampleFactor; + } + } + } + bool NeuralModelLoader::SupportsWaveNetLoadMode(EModelLoadMode mode) { if (mode == EModelLoadMode::NAMCore) @@ -147,7 +180,7 @@ namespace NeuralAudio return CreateFromJson(modelJson, extension, doPrewarm); } - NeuralModel* NeuralModelLoader::CreateFromJson(const nlohmann::json modelJson, const std::filesystem::path& extension, bool doPrewarm) + NeuralModel* NeuralModelLoader::CreateFromJson(nlohmann::json& modelJson, const std::filesystem::path& extension, bool doPrewarm) { EnsureModelDefsAreLoaded(); @@ -155,6 +188,8 @@ namespace NeuralAudio if (extension == ".nam") { + OversampleNAMConfig(modelJson, externalSampleRate); + std::string arch = modelJson.at("architecture"); #ifdef BUILD_NAMCORE diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index 0a7a0b1..81580f1 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -140,7 +140,7 @@ namespace NeuralAudio public: NeuralModel* CreateFromFile(const std::filesystem::path& modelPath, bool doPrewarm = true); NeuralModel* CreateFromStream(std::basic_istream& stream, const std::filesystem::path& extension, bool doPrewarm = true); - NeuralModel* CreateFromJson(const nlohmann::json modelJson, const std::filesystem::path& extension, bool doPrewarm = true); + NeuralModel* CreateFromJson(nlohmann::json& modelJson, const std::filesystem::path& extension, bool doPrewarm = true); bool SetLSTMLoadMode(EModelLoadMode val) { @@ -205,6 +205,11 @@ namespace NeuralAudio return defaultQualityScaleFactor; } + void SetExternalSampleRate(int sampleRate) + { + this->externalSampleRate = sampleRate; + } + protected: EModelLoadMode lstmLoadMode = EModelLoadMode::Internal; EModelLoadMode wavenetLoadMode = EModelLoadMode::Internal; @@ -212,6 +217,7 @@ namespace NeuralAudio float audioInputLevelDBu = (float)DEFAULT_INPUT_DBU; int defaultMaxAudioBufferSize = 128; float defaultQualityScaleFactor = (float)DEFAULT_QUALITY_SCALE; + int externalSampleRate = 48000; }; } \ No newline at end of file From 0966d5d350c6945103ebd87e8d8a5ac730afa12d Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sun, 14 Jun 2026 09:24:28 -0700 Subject: [PATCH 088/162] Visual Studio -> 2026 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4954104..098cbc1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build run: | - cmake -G "Visual Studio 17 2022" -A x64 -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_STATIC_RTNEURAL=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -T ClangCL .. + cmake -G "Visual Studio 18 2026" -A x64 -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_STATIC_RTNEURAL=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -T ClangCL .. cmake --build . --config=release -j4 - name: Run ModelTest From 7b1aea836bf50df067c8c7613a14a9127a7657a8 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sun, 14 Jun 2026 09:30:51 -0700 Subject: [PATCH 089/162] Visual Studio -> 2026 --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8cd180d..f5f2f16 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -128,7 +128,7 @@ jobs: env: CXXFLAGS: "/O2 -march=${{ matrix.native_arch }}" run: | - cmake.exe -G "Visual Studio 17 2022" -A x64 -T ClangCL -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON .. + cmake.exe -G "Visual Studio 18 2026" -A x64 -T ClangCL -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON .. cmake --build . --config=release -j4 - name: Add Archive From 38af41741b529c540b287e40956cf5b5079ce36c Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sun, 14 Jun 2026 09:40:37 -0700 Subject: [PATCH 090/162] Add oversampling docs --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 1996db4..b01d33e 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,18 @@ To get the current quality scaling factor for a model, do: float scaleFactor = model->GetQualityScaleFactor(); ``` +## Model oversampling + +WaveNet models are altered (via scaling the convolution dilation sizes) to produce the correct output when the external sample rate is an even multiple of the model sample rate. + +By default, the external sample rate is set to 48kHz. To change it, do: + +``` +loader.SetExternalSampleRate(sampleRate); +``` + +Note that ***models are only altered on load***. If you change the external sample rate you will need to reload any existing models where you want the change to take effect. + ## Getting model metadata To retrieve arbitrary metadata fields from models that contain them, do: From da199a7724aeb7fffd161f67c5be2f45bd14dc3b Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sun, 14 Jun 2026 09:59:06 -0700 Subject: [PATCH 091/162] Avoid deep copies of json data --- NeuralAudio/CompositeModel.h | 8 ++++---- NeuralAudio/InternalModel.h | 10 +++++----- NeuralAudio/NeuralModel.cpp | 8 ++++---- NeuralAudio/NeuralModelImpl.h | 23 +++++++++-------------- NeuralAudio/RTNeuralLoader.cpp | 8 ++++---- NeuralAudio/RTNeuralModel.h | 6 +++--- 6 files changed, 29 insertions(+), 34 deletions(-) diff --git a/NeuralAudio/CompositeModel.h b/NeuralAudio/CompositeModel.h index 7a7ea96..3b25869 100644 --- a/NeuralAudio/CompositeModel.h +++ b/NeuralAudio/CompositeModel.h @@ -114,20 +114,20 @@ namespace NeuralAudio return EModelLoadMode::NAMCore; } - bool LoadFromJson(const nlohmann::json& modelJson) + bool LoadFromJson(nlohmann::json& modelJson) { ReadNAMConfig(modelJson); return CreateModelFromNAMJson(modelJson); } - virtual bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + virtual bool CreateModelFromNAMJson(nlohmann::json& modelJson) { compositeLoadMode = loader->GetCompositeModelLoadMode(); - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); - nlohmann::json subModels = config.at("submodels"); + auto& subModels = config.at("submodels"); for (auto& submodelJson : subModels) { diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index 80f4eb7..ded5bc3 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -88,7 +88,7 @@ namespace NeuralAudio model = new ModelType; - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); model->SetWeights(modelJson.at("weights")); @@ -194,13 +194,13 @@ namespace NeuralAudio bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); std::vector layerArrays; for (size_t i = 0; i < config.at("layers").size(); i++) { - nlohmann::json layerConfig = config.at("layers").at(i); + auto& layerConfig = config.at("layers").at(i); layerArrays.push_back(WaveNetLayerArray(layerConfig.at("input_size"), layerConfig.at("condition_size"), layerConfig.at("head_size"), layerConfig.at("channels"), layerConfig.at("kernel_size"), layerConfig.at("head_bias"), layerConfig.at("dilations"))); @@ -278,7 +278,7 @@ namespace NeuralAudio model = new LSTMModelT; - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); model->SetNAMWeights(modelJson.at("weights")); @@ -438,7 +438,7 @@ namespace NeuralAudio model = nullptr; } - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); model = new LSTMModel(config.at("num_layers"), config.at("hidden_size")); diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index f9c3bbd..64433e9 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -72,7 +72,7 @@ namespace NeuralAudio static std::vector liteDilations = { 1, 2, 4, 8, 16, 32, 64 }; static std::vector liteDilations2 = { 128, 256, 512, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 }; - static bool CheckDilations(const nlohmann::json dilationJson, std::vector& checkDilations) + static bool CheckDilations(const nlohmann::json& dilationJson, std::vector& checkDilations) { if (dilationJson.size() != checkDilations.size()) return false; @@ -218,7 +218,7 @@ namespace NeuralAudio } #endif - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); if (newModel == nullptr) { @@ -226,8 +226,8 @@ namespace NeuralAudio { if (config.at("layers").size() == 2) { - nlohmann::json firstLayerConfig = config.at("layers").at(0); - nlohmann::json secondLayerConfig = config.at("layers").at(1); + auto& firstLayerConfig = config.at("layers").at(0); + auto& secondLayerConfig = config.at("layers").at(1); if (!firstLayerConfig.at("gated") && !secondLayerConfig.at("gated") && !firstLayerConfig.at("head_bias") && secondLayerConfig.at("head_bias")) { diff --git a/NeuralAudio/NeuralModelImpl.h b/NeuralAudio/NeuralModelImpl.h index 1886cb0..8b1b1a2 100644 --- a/NeuralAudio/NeuralModelImpl.h +++ b/NeuralAudio/NeuralModelImpl.h @@ -29,13 +29,6 @@ namespace NeuralAudio { modelVersion = modelJson.at("version"); - if (modelJson.contains("metadata")) - { - nlohmann::json metadataJson = modelJson.at("metadata"); - - AddMetadata(metadataJson); - } - if (modelJson.contains("sample_rate") && modelJson.at("sample_rate").is_number()) { sampleRate = modelJson.at("sample_rate"); @@ -43,21 +36,23 @@ namespace NeuralAudio if (modelJson.contains("metadata")) { - nlohmann::json metaData = modelJson.at("metadata"); + auto& metadataJson = modelJson.at("metadata"); + + AddMetadata(metadataJson); - if (metaData.contains("loudness") && metaData.at("loudness").is_number()) + if (metadataJson.contains("loudness") && metadataJson.at("loudness").is_number()) { - modelLoudnessDB = (float)metaData.at("loudness"); + modelLoudnessDB = (float)metadataJson.at("loudness"); } - if (metaData.contains("input_level_dbu") && metaData.at("input_level_dbu").is_number()) + if (metadataJson.contains("input_level_dbu") && metadataJson.at("input_level_dbu").is_number()) { - modelInputLevelDBu = metaData.at("input_level_dbu"); + modelInputLevelDBu = metadataJson.at("input_level_dbu"); } - if (metaData.contains("output_level_dbu") && metaData.at("output_level_dbu").is_number()) + if (metadataJson.contains("output_level_dbu") && metadataJson.at("output_level_dbu").is_number()) { - modelOutputLevelDBu = metaData.at("output_level_dbu"); + modelOutputLevelDBu = metadataJson.at("output_level_dbu"); } } } diff --git a/NeuralAudio/RTNeuralLoader.cpp b/NeuralAudio/RTNeuralLoader.cpp index bceffa4..bab5a86 100644 --- a/NeuralAudio/RTNeuralLoader.cpp +++ b/NeuralAudio/RTNeuralLoader.cpp @@ -47,10 +47,10 @@ namespace NeuralAudio NeuralModelImpl* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson, NeuralModelLoader *loader) { - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); - nlohmann::json firstLayerConfig = config.at("layers").at(0); - nlohmann::json secondLayerConfig = config.at("layers").at(1); + auto& firstLayerConfig = config.at("layers").at(0); + auto& secondLayerConfig = config.at("layers").at(1); auto modelDef = FindRTNeuralWaveNetDefinition(firstLayerConfig.at("channels"), firstLayerConfig.at("head_size")); @@ -69,7 +69,7 @@ namespace NeuralAudio NeuralModelImpl* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader *loader) { - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); auto modelDef = FindRTNeuralLSTMDefinition(config.at("num_layers"), config.at("hidden_size")); diff --git a/NeuralAudio/RTNeuralModel.h b/NeuralAudio/RTNeuralModel.h index 123fbb9..b1391fd 100644 --- a/NeuralAudio/RTNeuralModel.h +++ b/NeuralAudio/RTNeuralModel.h @@ -126,7 +126,7 @@ namespace NeuralAudio model = new ModelType; - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); std::vector weights = modelJson.at("weights"); @@ -278,7 +278,7 @@ namespace NeuralAudio model = new ModelType; - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); model->load_weights(modelJson); @@ -428,7 +428,7 @@ namespace NeuralAudio { model = std::make_unique>(1); - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); const size_t numLayers = config.at("num_layers"); const size_t inputSize = config.at("input_size"); From cb9a03a2d09e1377d7314bffce65c735d8ed9ec5 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sun, 14 Jun 2026 13:02:01 -0700 Subject: [PATCH 092/162] Remove unused config access --- NeuralAudio/InternalModel.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index ded5bc3..882b040 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -88,8 +88,6 @@ namespace NeuralAudio model = new ModelType; - auto& config = modelJson.at("config"); - model->SetWeights(modelJson.at("weights")); SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); From 3fc1d259ba45ecdbc7ab10ca7a247e829ea86f7c Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sun, 14 Jun 2026 13:25:25 -0700 Subject: [PATCH 093/162] Fix composite model receptive field --- NeuralAudio/CompositeModel.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/NeuralAudio/CompositeModel.h b/NeuralAudio/CompositeModel.h index 3b25869..368b44f 100644 --- a/NeuralAudio/CompositeModel.h +++ b/NeuralAudio/CompositeModel.h @@ -65,6 +65,14 @@ namespace NeuralAudio } } + int GetReceptiveFieldSize() override + { + if (currentModelIndex == -1) + return -1; + + return models[currentModelIndex.load()]->GetReceptiveFieldSize(); + } + void SetMaxAudioBufferSize(const int maxSize) override { for (auto& model : models) From 7a7be0df3ef8df6eef2ca2b80abff8bec4e183c3 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 15 Jun 2026 09:29:28 -0700 Subject: [PATCH 094/162] Update NAM Core. Fix WaveNet layer array head oversampling. --- NeuralAudio/NeuralModel.cpp | 7 +++++++ deps/NeuralAmpModelerCore | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 64433e9..3d7b251 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -116,6 +116,13 @@ namespace NeuralAudio { dilation = dilation.get() * oversampleFactor; } + + if (layer.contains("head")) + { + auto& head = layer.at("head"); + + head["head_dilation"] = oversampleFactor; + } } } diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index fea0ac0..1c8d83c 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit fea0ac03d2ac65c924ed41d61400740413b5edc6 +Subproject commit 1c8d83cd5d632ee4d4665b4277222003ba809345 From 6ccd786bd393c0103f83b576c19bcae3105c15ee Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 17 Jun 2026 07:32:52 -0700 Subject: [PATCH 095/162] Remove another unused config access --- NeuralAudio/InternalModel.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index 882b040..aa79de3 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -276,8 +276,6 @@ namespace NeuralAudio model = new LSTMModelT; - auto& config = modelJson.at("config"); - model->SetNAMWeights(modelJson.at("weights")); SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); From f6a7f43257e6fe3f095d76d46a0bc0848362cee9 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 23 Jun 2026 08:12:28 -0700 Subject: [PATCH 096/162] Update README.md --- README.md | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b01d33e..876a871 100644 --- a/README.md +++ b/README.md @@ -13,24 +13,18 @@ NeuralAudio currently supports the following model types: This repository is licensed under the [MIT license](https://github.com/mikeoliphant/NeuralAudio/blob/main/LICENSE). It is a liberal license, but please make sure that you comply with the terms - as well as the terms of [this project's dependencies](https://github.com/mikeoliphant/NeuralAudio/blob/main/CREDITS.md). I would also appreciate it if you would let me know if you are using this library. -# Underlying Libraries and Performance +# Underlying Libraries -By default, NeuralAudio uses its own implementation of WaveNet and LSTM network models. This implementation has been designed to produce **exactly** the same output as the [NAM Core library](https://github.com/sdatkinson/NeuralAmpModelerCore), but with increased performance and reduced memory usage. +NAM A2 models currently use the NAM Core implementation (and consequently require building with NAM Core enabled). -For completeness, and to facilitate accuracy and performance benchmarking, it can also load models using the [NAM Core implementation](https://github.com/sdatkinson/NeuralAmpModelerCore) and [RTNeural](https://github.com/jatinchowdhury18/RTNeural). +For A1 WaveNet, the internal implmeentation supports optimized static models of the offical NAM A1 network architectures: "Standard", "Lite", "Feather", "Nano". -The internal NeuralAudio implmentation currently outperforms the other implementations on all tested platforms (Windows x64, Linux x64/Arm64). It also uses significantly less memory than the NAM Core WaveNet implementation (which, for example, uses about 10x as much memory for a "standard" WaveNet model). +For A1 LSTM, the internal implementation supports optimized static models architectures for 1x8, 1x12, 1x16, 1x24, 2x8, 2x12, and 2x16 models. -For WaveNet, the internal implmeentation supports optimized static models of the offical NAM A1 network architectures: "Standard", "Lite", "Feather", "Nano". - -For LSTM, the internal implementation supports optimized static models architectures for 1x8, 1x12, 1x16, 1x24, 2x8, 2x12, and 2x16 models. - -All NAM files with WaveNet and LSTM architectures not supported internally will fall back on a less performant dynamic implementation (although still faster than NAM Core). +All A1 NAM files with WaveNet and LSTM architectures not supported internally will fall back on a less performant dynamic implementation (although still faster than NAM Core). All keras models not supported internally will fall back to the RTNeural implmentation. -NAM A2 models currently use the NAM Core implementation. - # API overview Models are loaded with a model loader: From d9a641b8b31397a2f361cbedb711549c063b74b3 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 25 Jun 2026 15:25:53 -0700 Subject: [PATCH 097/162] Update NAM Core --- deps/NeuralAmpModelerCore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index 1c8d83c..4c0ee78 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit 1c8d83cd5d632ee4d4665b4277222003ba809345 +Subproject commit 4c0ee78b71abd5eb20aec58562e7540f43caac3b From cf8d4d696f9c6e5bde96c8f45b6e0fb9b1decbcb Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 26 Jun 2026 08:37:10 -0700 Subject: [PATCH 098/162] Remove RTNeural NAM implementation --- NeuralAudio/CMakeLists.txt | 10 +- NeuralAudio/NeuralModel.cpp | 4 - NeuralAudio/RTNeuralLoader.cpp | 38 ---- NeuralAudio/RTNeuralModel.h | 119 ------------ deps/RTNeural-NAM/LICENSE | 29 --- deps/RTNeural-NAM/wavenet/arena.hpp | 146 -------------- deps/RTNeural-NAM/wavenet/wavenet_layer.hpp | 178 ----------------- .../wavenet/wavenet_layer_array.hpp | 179 ------------------ deps/RTNeural-NAM/wavenet/wavenet_model.hpp | 162 ---------------- 9 files changed, 1 insertion(+), 864 deletions(-) delete mode 100644 deps/RTNeural-NAM/LICENSE delete mode 100644 deps/RTNeural-NAM/wavenet/arena.hpp delete mode 100644 deps/RTNeural-NAM/wavenet/wavenet_layer.hpp delete mode 100644 deps/RTNeural-NAM/wavenet/wavenet_layer_array.hpp delete mode 100644 deps/RTNeural-NAM/wavenet/wavenet_model.hpp diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 6daf964..4638fe8 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -132,20 +132,12 @@ if(BUILD_NAMCORE) ../deps/NeuralAmpModelerCore/NAM/wavenet/a2_fast.cpp) endif() -if(BUILD_STATIC_RTNEURAL) -set(RTNEURAL_WN_SOURCES ../deps/RTNeural-NAM/wavenet/wavenet_layer.hpp - ../deps/RTNeural-NAM/wavenet/wavenet_layer_array.hpp - ../deps/RTNeural-NAM/wavenet/wavenet_model.hpp - ../deps/RTNeural-NAM/wavenet/arena.hpp) -endif() - -add_library(NeuralAudio OBJECT ${SOURCES} ${NAM_SOURCES} ${RTNEURAL_WN_SOURCES}) +add_library(NeuralAudio OBJECT ${SOURCES} ${NAM_SOURCES}) target_include_directories(NeuralAudio PUBLIC ..) target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/NeuralAmpModelerCore) target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/RTNeural) target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/math_approx) -target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/RTNeural-NAM/wavenet) target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/RTNeural/modules/Eigen) target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/RTNeural/modules/json) diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 3d7b251..8805fa5 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -136,11 +136,7 @@ namespace NeuralAudio #endif if (mode == EModelLoadMode::RTNeural) -#ifdef BUILD_STATIC_RTNEURAL - return true; -#else return false; -#endif return true; } diff --git a/NeuralAudio/RTNeuralLoader.cpp b/NeuralAudio/RTNeuralLoader.cpp index bab5a86..e54dad2 100644 --- a/NeuralAudio/RTNeuralLoader.cpp +++ b/NeuralAudio/RTNeuralLoader.cpp @@ -16,11 +16,6 @@ namespace NeuralAudio rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 8>); rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 12>); rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 16>); - - rtNeuralWaveNetModelDefs.push_back(new RTNeuralWaveNetDefinitionT<16, 8>); // Standard - rtNeuralWaveNetModelDefs.push_back(new RTNeuralWaveNetDefinitionT<12, 6>); // Lite - rtNeuralWaveNetModelDefs.push_back(new RTNeuralWaveNetDefinitionT<8, 4>); // Feather - rtNeuralWaveNetModelDefs.push_back(new RTNeuralWaveNetDefinitionT<4, 2>); // Nano } RTNeuralLSTMDefinitionBase* FindRTNeuralLSTMDefinition(size_t numLayers, size_t hiddenSize) @@ -34,39 +29,6 @@ namespace NeuralAudio return nullptr; } - RTNeuralWaveNetDefinitionBase* FindRTNeuralWaveNetDefinition(size_t numChannels, size_t headSize) - { - for (auto const& model : rtNeuralWaveNetModelDefs) - { - if ((numChannels == model->GetNumChannels()) && (headSize == model->GetHeadSize())) - return model; - } - - return nullptr; - } - - NeuralModelImpl* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson, NeuralModelLoader *loader) - { - auto& config = modelJson.at("config"); - - auto& firstLayerConfig = config.at("layers").at(0); - auto& secondLayerConfig = config.at("layers").at(1); - - auto modelDef = FindRTNeuralWaveNetDefinition(firstLayerConfig.at("channels"), firstLayerConfig.at("head_size")); - - if (modelDef != nullptr) - { - auto model = modelDef->CreateModel(); - - model->SetModelLoader(loader); - model->LoadFromNAMJson(modelJson); - - return model; - } - - return nullptr; - } - NeuralModelImpl* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader *loader) { auto& config = modelJson.at("config"); diff --git a/NeuralAudio/RTNeuralModel.h b/NeuralAudio/RTNeuralModel.h index b1391fd..dbb2240 100644 --- a/NeuralAudio/RTNeuralModel.h +++ b/NeuralAudio/RTNeuralModel.h @@ -3,9 +3,6 @@ #include "NeuralModel.h" #include "NeuralModelImpl.h" #include -#ifdef BUILD_STATIC_RTNEURAL -#include "wavenet_model.hpp" -#endif #include "TemplateHelper.h" namespace NeuralAudio @@ -232,83 +229,6 @@ namespace NeuralAudio ModelType* model = nullptr; }; - using StdDilations = wavenet::Dilations<1, 2, 4, 8, 16, 32, 64, 128, 256, 512>; - using LiteDilations1 = wavenet::Dilations<1, 2, 4, 8, 16, 32, 64>; - using LiteDilations2 = wavenet::Dilations<128, 256, 512, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512>; - - template - class RTNeuralWaveNetModelT : public RTNeuralModel - { - using ModelType = typename std::conditional, - wavenet::Layer_Array>, - wavenet::Wavenet_Model, - wavenet::Layer_Array> - >::type; - - public: - RTNeuralWaveNetModelT() - : model(nullptr) - { - } - - ~RTNeuralWaveNetModelT() - { - if (model != nullptr) - { - delete model; - model = nullptr; - } - } - - bool IsStatic() override - { - return true; - } - - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override - { - if (model != nullptr) - { - delete model; - model = nullptr; - } - - model = new ModelType; - - auto& config = modelJson.at("config"); - - model->load_weights(modelJson); - - SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); - - return true; - } - - void SetMaxAudioBufferSize(int maxSize) override - { - model->prepare(maxSize); - } - - void Process(float* input, float* output, size_t numSamples) override - { - model->forward(input, output, (int)numSamples); - } - - void Prewarm() override - { - float sample = 0; - - for (size_t i = 0; i < 2048; i++) - model->forward(sample); - } - - private: - ModelType* model = nullptr; - }; - class RTNeuralModelDefinitionBase { public: @@ -357,45 +277,6 @@ namespace NeuralAudio } }; - class RTNeuralWaveNetDefinitionBase : public RTNeuralModelDefinitionBase - { - public: - virtual RTNeuralModel* CreateModel() override - { - return nullptr; - } - - virtual size_t GetNumChannels() - { - return 0; - } - - virtual size_t GetHeadSize() - { - return 0; - } - }; - - template - class RTNeuralWaveNetDefinitionT : public RTNeuralWaveNetDefinitionBase - { - public: - RTNeuralModel* CreateModel() override - { - return new RTNeuralWaveNetModelT; - } - - size_t GetNumChannels() override - { - return numChannels; - } - - size_t GetHeadSize() override - { - return headSize; - } - }; - #endif class RTNeuralModelDyn : public RTNeuralModel diff --git a/deps/RTNeural-NAM/LICENSE b/deps/RTNeural-NAM/LICENSE deleted file mode 100644 index c761450..0000000 --- a/deps/RTNeural-NAM/LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2024, jatinchowdhury18 -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/deps/RTNeural-NAM/wavenet/arena.hpp b/deps/RTNeural-NAM/wavenet/arena.hpp deleted file mode 100644 index 1aa03ca..0000000 --- a/deps/RTNeural-NAM/wavenet/arena.hpp +++ /dev/null @@ -1,146 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace wavenet -{ -/** - * Returns the next pointer with a given byte alignment, - * or the base pointer if it is already aligned. - */ -template -Type* snap_pointer_to_alignment (Type* base_pointer, - Integer_Type alignment_bytes) noexcept -{ - return (Type*) ((((size_t) base_pointer) + (alignment_bytes - 1)) & ~(alignment_bytes - 1)); -} - -static constexpr int default_byte_alignment = 16; - -/** - * A simple memory arena. By default the arena will be - * backed with a vector of bytes, but the underlying - * memory resource can be changed via the template argument. - */ -template > -class Memory_Arena -{ -public: - Memory_Arena() = default; - - /** Constructs the arena with an initial allocated size. */ - explicit Memory_Arena (size_t size_in_bytes) { resize (size_in_bytes); } - - Memory_Arena (const Memory_Arena&) = delete; - Memory_Arena& operator= (const Memory_Arena&) = delete; - - Memory_Arena (Memory_Arena&&) noexcept = default; - Memory_Arena& operator= (Memory_Arena&&) noexcept = default; - - /** Re-allocates the internal buffer with a given number of bytes */ - void resize (size_t new_size_bytes) - { - clear(); - raw_data.resize (new_size_bytes, std::byte {}); - } - - /** - * Moves the allocator "stack pointer" back to zero, - * effectively "reclaiming" all allocated memory. - */ - void clear() noexcept - { -#if DEBUG - std::fill (raw_data.begin(), raw_data.begin() + bytes_used, std::byte { 0xDD }); -#endif - bytes_used = 0; - } - - /** Returns the number of bytes currently being used */ - [[nodiscard]] size_t get_bytes_used() const noexcept { return bytes_used; } - - /** - * Allocates a given number of bytes. - * The returned memory will be un-initialized, so be sure to clear it manually - * if needed. - */ - void* allocate_bytes (size_t num_bytes, size_t alignment = 1) - { - auto* pointer = snap_pointer_to_alignment (raw_data.data() + bytes_used, alignment); - const auto bytes_increment = static_cast (std::distance (raw_data.data() + bytes_used, pointer + num_bytes)); - - if (bytes_used + bytes_increment > raw_data.size()) - { - assert (false); - return nullptr; - } - - bytes_used += bytes_increment; - return pointer; - } - - /** - * Allocates space for some number of objects of type T - * The returned memory will be un-initialized, so be sure to clear it manually - * if needed. - */ - template - T* allocate (IntType num_Ts, size_t alignment = alignof (T)) - { - return static_cast (allocate_bytes ((size_t) num_Ts * sizeof (T), alignment)); - } - - /** - * Returns a span of type T, and size count. - * The returned memory will be un-initialized, so be sure to clear it manually - * if needed. - */ - template - auto make_span (IntType count, size_t alignment = default_byte_alignment) - { - return std::span { allocate (count, alignment), - static_cast (count) }; - } - - /** Returns a pointer to the internal buffer with a given offset in bytes */ - template - T* data (IntType offset_bytes) noexcept - { - return reinterpret_cast (raw_data.data() + offset_bytes); - } - - /** - * Creates a "frame" for the allocator. - * Once the frame goes out of scope, the allocator will be reset - * to whatever it's state was at the beginning of the frame. - */ - struct Frame - { - Frame() = default; - explicit Frame (Memory_Arena& allocator) - : alloc (&allocator), bytes_used_at_start (alloc->bytes_used) {} - - ~Frame() { alloc->bytes_used = bytes_used_at_start; } - - Memory_Arena* alloc = nullptr; - size_t bytes_used_at_start = 0; - }; - - /** Creates a frame for this allocator */ - auto create_frame() { return Frame { *this }; } - - void reset_to_frame (const Frame& frame) - { - assert (frame.alloc == this); - bytes_used = frame.bytes_used_at_start; - } - -private: - MemoryResourceType raw_data {}; - size_t bytes_used = 0; -}; -} // namespace wavenet diff --git a/deps/RTNeural-NAM/wavenet/wavenet_layer.hpp b/deps/RTNeural-NAM/wavenet/wavenet_layer.hpp deleted file mode 100644 index 3d81fe9..0000000 --- a/deps/RTNeural-NAM/wavenet/wavenet_layer.hpp +++ /dev/null @@ -1,178 +0,0 @@ -#pragma once - -#include - -namespace wavenet -{ -template > -// TODO: gated? -struct Wavenet_Layer -{ - RTNeural::Conv1DT conv; - RTNeural::DenseT input_mixin; - RTNeural::DenseT _1x1; - Activation activation; - -#if RTNEURAL_USE_EIGEN - Eigen::Matrix outs; -#elif RTNEURAL_USE_XSIMD - xsimd::batch outs[RTNeural::ceil_div (channels, (int) xsimd::batch::size)]; -#endif - - void reset() - { - conv.reset(); - } - - void load_weights (std::vector::iterator& weights) - { - conv.reset(); - - std::vector>> conv_weights (channels, std::vector> (channels, std::vector (kernel_size))); - for (int i = 0; i < channels; ++i) - for (int j = 0; j < channels; ++j) - for (int k = 0; k < kernel_size; k++) - conv_weights[i][j][k] = *(weights++); - RTNeural::torch_helpers::detail::reverseKernels (conv_weights); - conv.setWeights (conv_weights); - - std::vector conv_bias (channels); - for (int i = 0; i < channels; ++i) - conv_bias[i] = *(weights++); - conv.setBias (conv_bias); - - std::vector> input_mixin_weights (channels, std::vector (condition_size)); - for (int i = 0; i < channels; i++) - for (int j = 0; j < condition_size; j++) - input_mixin_weights[i][j] = *(weights++); - input_mixin.setWeights (input_mixin_weights); - - std::vector> _1x1_weights (channels, std::vector (channels)); - for (int i = 0; i < channels; i++) - for (int j = 0; j < channels; j++) - _1x1_weights[i][j] = *(weights++); - _1x1.setWeights (_1x1_weights); - - std::vector _1x1_bias (channels); - for (int i = 0; i < channels; i++) - _1x1_bias[i] = *(weights++); - _1x1.setBias (_1x1_bias.data()); - } - -#if RTNEURAL_USE_EIGEN - void forward (const Eigen::Matrix& ins, - const Eigen::Matrix& condition, - Eigen::Matrix& head_io) -#elif RTNEURAL_USE_XSIMD - void forward (const xsimd::batch (&ins)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)], - const xsimd::batch (&condition)[RTNeural::ceil_div (condition_size, (int) xsimd::batch::size)], - xsimd::batch (&head_io)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)]) -#endif - { - conv.forward (ins); - input_mixin.forward (condition); - -#if RTNEURAL_USE_EIGEN - outs = conv.outs + input_mixin.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (outs); ++i) - outs[i] = conv.outs[i] + input_mixin.outs[i]; -#endif - - activation.forward (outs); - -#if RTNEURAL_USE_EIGEN - head_io.noalias() += activation.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (head_io); ++i) - head_io[i] += activation.outs[i]; -#endif - - _1x1.forward (activation.outs); - -#if RTNEURAL_USE_EIGEN - outs = ins + _1x1.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (outs); ++i) - outs[i] = ins[i] + _1x1.outs[i]; -#endif - } - -#if RTNEURAL_USE_EIGEN - void forward (const Eigen::Matrix* ins, - const Eigen::Matrix* condition, - Eigen::Matrix* head_io, - Eigen::Matrix* layer_outputs, - int N, - Memory_Arena<>& arena) -#elif RTNEURAL_USE_XSIMD - void forward (const xsimd::batch (*ins)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)], - const xsimd::batch (*condition)[RTNeural::ceil_div (condition_size, (int) xsimd::batch::size)], - xsimd::batch (*head_io)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)], - xsimd::batch (*layer_outputs)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)], - int N, - Memory_Arena<>& arena) -#endif - { - const auto _ = arena.create_frame(); -#if RTNEURAL_USE_EIGEN - auto* temp_outs = arena.allocate> (N, RTNEURAL_DEFAULT_ALIGNMENT); -#elif RTNEURAL_USE_XSIMD - auto* temp_outs = arena.allocate[RTNeural::ceil_div (channels, (int) xsimd::batch::size)]> (N, RTNEURAL_DEFAULT_ALIGNMENT); -#endif - - for (int n = 0; n < N; ++n) - { - conv.forward (ins[n]); -#if RTNEURAL_USE_EIGEN - temp_outs[n].noalias() = conv.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (conv.outs); ++i) - temp_outs[n][i] = conv.outs[i]; -#endif - } - - for (int n = 0; n < N; ++n) - { - input_mixin.forward (condition[n]); -#if RTNEURAL_USE_EIGEN - temp_outs[n].noalias() += input_mixin.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (input_mixin.outs); ++i) - temp_outs[n][i] += input_mixin.outs[i]; -#endif - } - - for (int n = 0; n < N; ++n) - { - activation.forward (temp_outs[n]); -#if RTNEURAL_USE_EIGEN - temp_outs[n].noalias() = activation.outs; - head_io[n].noalias() += activation.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (activation.outs); ++i) - temp_outs[n][i] = activation.outs[i]; - for (int i = 0; i < std::size (activation.outs); ++i) - head_io[n][i] += activation.outs[i]; -#endif - } - - for (int n = 0; n < N; ++n) - { - _1x1.forward (temp_outs[n]); -#if RTNEURAL_USE_EIGEN - layer_outputs[n].noalias() = ins[n] + _1x1.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (_1x1.outs); ++i) - layer_outputs[n][i] = ins[n][i] + _1x1.outs[i]; -#endif - } - } -}; -} // namespace wavenet diff --git a/deps/RTNeural-NAM/wavenet/wavenet_layer_array.hpp b/deps/RTNeural-NAM/wavenet/wavenet_layer_array.hpp deleted file mode 100644 index 37ced02..0000000 --- a/deps/RTNeural-NAM/wavenet/wavenet_layer_array.hpp +++ /dev/null @@ -1,179 +0,0 @@ -#pragma once - -#include "wavenet_layer.hpp" - -namespace wavenet -{ -template -using Dilations = std::integer_sequence; - -template > -struct Layer_Array -{ - template - struct Layers_Helper - { - }; - - template - struct Layers_Helper> - { - using type = std::tuple...>; - }; - - using Layers = typename Layers_Helper::type; - - static constexpr auto n_channels = channels; - - RTNeural::DenseT rechannel; // no bias! - Layers layers; - static constexpr auto num_layers = std::tuple_size_v; - RTNeural::DenseT head_rechannel; - - using Last_Layer_Type = std::remove_reference_t - 1> (layers))>; - decltype (Last_Layer_Type::outs)& layer_outputs { std::get - 1> (layers).outs }; - -#if RTNEURAL_USE_EIGEN - Eigen::Matrix head_outputs {}; - Eigen::Matrix* layer_outputs_arr; - Eigen::Matrix* head_outputs_arr; -#elif RTNEURAL_USE_XSIMD - decltype (RTNeural::DenseT::outs)& head_outputs { head_rechannel.outs }; - using Layer_Out = xsimd::batch[RTNeural::ceil_div (channels, (int) xsimd::batch::size)]; - Layer_Out* layer_outputs_arr; - using Head_Out = xsimd::batch[RTNeural::ceil_div (head_size, (int) xsimd::batch::size)]; - Head_Out* head_outputs_arr; -#endif - - void reset() - { - RTNeural::modelt_detail::forEachInTuple ([] (auto& layer, size_t) - { layer.reset(); }, - layers); - } - - static size_t get_arena_bytes_needed (int N) - { -#if RTNEURAL_USE_EIGEN - return 2 * sizeof (Eigen::Matrix) * N + sizeof (Eigen::Matrix) * N; -#elif RTNEURAL_USE_XSIMD - return 2 * sizeof (Layer_Out) * N + sizeof (Head_Out) * N; -#endif - } - - void load_weights (std::vector::iterator& weights) - { - std::vector> rechannel_weights (channels, std::vector (in_size)); - for (int i = 0; i < channels; i++) - for (int j = 0; j < in_size; j++) - rechannel_weights[i][j] = *(weights++); - rechannel.setWeights (rechannel_weights); - - RTNeural::modelt_detail::forEachInTuple ([&weights] (auto& layer, size_t) - { layer.load_weights (weights); }, - layers); - - std::vector> head_rechannel_weights (head_size, std::vector (channels)); - for (int i = 0; i < head_size; i++) - for (int j = 0; j < channels; j++) - head_rechannel_weights[i][j] = *(weights++); - head_rechannel.setWeights (head_rechannel_weights); - - if constexpr (has_head_bias) - { - std::vector head_rechannel_bias (head_size); - for (int i = 0; i < head_size; i++) - head_rechannel_bias[i] = *(weights++); - head_rechannel.setBias (head_rechannel_bias.data()); - } - } - -#if RTNEURAL_USE_EIGEN - void forward (const Eigen::Matrix& ins, - const Eigen::Matrix& condition, - Eigen::Matrix& head_io) -#elif RTNEURAL_USE_XSIMD - void forward (const xsimd::batch (&ins)[RTNeural::ceil_div (in_size, (int) xsimd::batch::size)], - const xsimd::batch (&condition)[RTNeural::ceil_div (condition_size, (int) xsimd::batch::size)], - xsimd::batch (&head_io)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)]) -#endif - { - rechannel.forward (ins); - - RTNeural::modelt_detail::forEachInTuple ( - [&] (auto& layer, auto index_t) - { - static constexpr size_t index = index_t; - if constexpr (index == 0) - layer.forward (rechannel.outs, condition, head_io); - else - layer.forward (std::get (layers).outs, condition, head_io); - }, - layers); - - head_rechannel.forward (head_io); -#if RTNEURAL_USE_EIGEN - head_outputs = head_rechannel.outs; -#endif - } - -#if RTNEURAL_USE_EIGEN - void forward (const Eigen::Matrix* ins, - const Eigen::Matrix* condition, - Eigen::Matrix* head_io, - int N, - Memory_Arena<>& arena) -#elif RTNEURAL_USE_XSIMD - void forward (const xsimd::batch (*ins)[RTNeural::ceil_div (in_size, (int) xsimd::batch::size)], - const xsimd::batch (*condition)[RTNeural::ceil_div (condition_size, (int) xsimd::batch::size)], - xsimd::batch (*head_io)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)], - int N, - Memory_Arena<>& arena) -#endif - { -#if RTNEURAL_USE_EIGEN - layer_outputs_arr = arena.allocate> (N, RTNEURAL_DEFAULT_ALIGNMENT); - head_outputs_arr = arena.allocate> (N, RTNEURAL_DEFAULT_ALIGNMENT); -#elif RTNEURAL_USE_XSIMD - layer_outputs_arr = arena.allocate (N, RTNEURAL_DEFAULT_ALIGNMENT); - head_outputs_arr = arena.allocate (N, RTNEURAL_DEFAULT_ALIGNMENT); -#endif - - for (int n = 0; n < N; ++n) - { - rechannel.forward (ins[n]); -#if RTNEURAL_USE_EIGEN - layer_outputs_arr[n] = rechannel.outs; -#elif RTNEURAL_USE_XSIMD - std::copy (std::begin (rechannel.outs), std::end (rechannel.outs), std::begin (layer_outputs_arr[n])); -#endif - } - - RTNeural::modelt_detail::forEachInTuple ( - [&] (auto& layer, auto) - { - layer.forward (layer_outputs_arr, condition, head_io, layer_outputs_arr, N, arena); - }, - layers); - - for (int n = 0; n < N; ++n) - { - head_rechannel.forward (head_io[n]); -#if RTNEURAL_USE_EIGEN - head_outputs_arr[n] = head_rechannel.outs; -#elif RTNEURAL_USE_XSIMD - std::copy (std::begin (head_rechannel.outs), std::end (head_rechannel.outs), std::begin (head_outputs_arr[n])); -#endif - } - } -}; -} // namespace wavenet diff --git a/deps/RTNeural-NAM/wavenet/wavenet_model.hpp b/deps/RTNeural-NAM/wavenet/wavenet_model.hpp deleted file mode 100644 index cecefce..0000000 --- a/deps/RTNeural-NAM/wavenet/wavenet_model.hpp +++ /dev/null @@ -1,162 +0,0 @@ -#pragma once - -#include "arena.hpp" -#include "wavenet_layer_array.hpp" - -namespace wavenet -{ -template -struct Wavenet_Model -{ - std::tuple layer_arrays; - - static constexpr auto head_layer_n_channels = std::tuple_element_t<0, std::tuple>::n_channels; - -#if RTNEURAL_USE_EIGEN - Eigen::Matrix head_input {}; -#elif RTNEURAL_USE_XSIMD - xsimd::batch head_input[RTNeural::ceil_div (head_layer_n_channels, (int) xsimd::batch::size)]; -#endif - T head_scale = (T) 0; - - Memory_Arena<> arena {}; - - Wavenet_Model() = default; - - void prepare (int block_size) - { -#if RTNEURAL_USE_EIGEN - size_t arena_bytes_needed = sizeof (Eigen::Matrix) * block_size; -#elif RTNEURAL_USE_XSIMD - size_t arena_bytes_needed = sizeof (xsimd::batch[RTNeural::ceil_div (head_layer_n_channels, (int) xsimd::batch::size)]) * block_size - + sizeof (xsimd::batch) * block_size; -#endif - RTNeural::modelt_detail::forEachInTuple ( - [&arena_bytes_needed, block_size] (auto& layer_array, auto) - { - arena_bytes_needed += layer_array.get_arena_bytes_needed (block_size); - }, - layer_arrays); - - arena.resize (arena_bytes_needed + 256); - // prewarm(); - } - - void prewarm() - { - RTNeural::modelt_detail::forEachInTuple ( - [] (auto& layer, size_t) - { - layer.reset(); - }, - layer_arrays); - for (int i = 0; i < 1 << 14; ++i) - forward (0.0f); - } - - void load_weights (const nlohmann::json& model_json) - { - std::vector model_weights = model_json.at ("weights"); - auto weights_iterator = model_weights.begin(); - RTNeural::modelt_detail::forEachInTuple ( - [&weights_iterator] (auto& layer, size_t) - { - layer.load_weights (weights_iterator); - }, - layer_arrays); - - head_scale = *weights_iterator++; - - // Make sure we use the all of the weights exactly - assert (std::distance (model_weights.begin(), weights_iterator) == model_weights.size()); - } - - T forward (T input) noexcept - { -#if RTNEURAL_USE_EIGEN - const auto v_ins = Eigen::Matrix::Constant (input); -#elif RTNEURAL_USE_XSIMD - xsimd::batch v_ins[1]; - v_ins[0] = RTNeural::set_value (v_ins[0], 0, input); -#endif - - RTNeural::modelt_detail::forEachInTuple ( - [this, v_ins] (auto& layer_array, auto index_t) - { - static constexpr size_t index = index_t; - if constexpr (index == 0) - { -#if RTNEURAL_USE_EIGEN - head_input.setZero(); - std::get<0> (layer_arrays).forward (v_ins, v_ins, head_input); -#elif RTNEURAL_USE_XSIMD - std::fill (std::begin (head_input), std::end (head_input), xsimd::batch {}); - std::get<0> (layer_arrays).forward (v_ins, v_ins, head_input); -#endif - } - else - { - std::get (layer_arrays).forward (std::get (layer_arrays).layer_outputs, v_ins, std::get (layer_arrays).head_outputs); - } - }, - layer_arrays); - -#if RTNEURAL_USE_EIGEN - return std::get - 1> (layer_arrays).head_outputs[0] * head_scale; -#elif RTNEURAL_USE_XSIMD - return std::get - 1> (layer_arrays).head_outputs[0].get (0) * head_scale; -#endif - } - - void forward (const T* input, T* output, int N) noexcept - { -#if RTNEURAL_USE_EIGEN - const auto* v_ins = reinterpret_cast*> (input); -#elif RTNEURAL_USE_XSIMD - auto* v_ins = arena.allocate[1]> (N, RTNEURAL_DEFAULT_ALIGNMENT); - for (int n = 0; n < N; ++n) - v_ins[n][0] = RTNeural::set_value (v_ins[n][0], 0, input[n]); -#endif - - RTNeural::modelt_detail::forEachInTuple ( - [this, v_ins, N, output] (auto& layer_array, auto index_t) - { - static constexpr size_t index = index_t; - if constexpr (index == 0) - { -#if RTNEURAL_USE_EIGEN - auto* head_inputs = arena.allocate> (N, RTNEURAL_DEFAULT_ALIGNMENT); - for (int n = 0; n < N; ++n) - head_inputs[n].setZero(); - std::get<0> (layer_arrays).forward (v_ins, v_ins, head_inputs, N, arena); -#elif RTNEURAL_USE_XSIMD - auto* head_inputs = arena.allocate[RTNeural::ceil_div (head_layer_n_channels, (int) xsimd::batch::size)]> (N, RTNEURAL_DEFAULT_ALIGNMENT); - for (int n = 0; n < N; ++n) - std::fill (std::begin (head_inputs[n]), std::end (head_inputs[n]), xsimd::batch {}); - std::get<0> (layer_arrays).forward (v_ins, v_ins, head_inputs, N, arena); -#endif - } - else - { - auto& prev_layer_array = std::get (layer_arrays); - std::get (layer_arrays).forward (prev_layer_array.layer_outputs_arr, v_ins, prev_layer_array.head_outputs_arr, N, arena); - } - }, - layer_arrays); - - auto& last_layer_array = std::get - 1> (layer_arrays); - for (int n = 0; n < N; ++n) - { -#if RTNEURAL_USE_EIGEN - output[n] = last_layer_array.head_outputs_arr[n][0] * head_scale; -#elif RTNEURAL_USE_XSIMD - output[n] = last_layer_array.head_outputs_arr[n][0].get (0) * head_scale; -#endif - } - - arena.clear(); - } -}; -} // namespace wavenet From 847f804bf7aa03334d343ce576c3d7eb7cad7966 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 26 Jun 2026 08:39:43 -0700 Subject: [PATCH 099/162] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 876a871..a0f9740 100644 --- a/README.md +++ b/README.md @@ -84,12 +84,12 @@ where "loadMode" is one of: ``` NeuralAudio::EModelLoadMode::Internal NeuralAudio::EModelLoadMode::NAMCore -NeuralAudio::EModelLoadMode::RTNeural +NeuralAudio::EModelLoadMode::RTNeural (only supported for LSTM) ``` You can check which implementation was actually used to load the model with ```model->GetLoadMode()```. -**NOTE:** Because of compile time and executable size considerations, only the internal, NAM Core and dynamic RTNeural implementations are built by default. If you want to use RTNeural, it is recommended that you add ```-DBUILD_STATIC_RTNEURAL=ON``` to your cmake commandline. This will create static model implmentations for the same sets of WaveNet and LSTM models as the internal implmentation, and results in increased performance. Interal static LSTM model support is also off by default - to turn it on use ```-DBUILD_INTERNAL_STATIC_LSTM=ON```. +**NOTE:** Because of compile time and executable size considerations, only the internal, NAM Core and dynamic RTNeural implementations are built by default. If you want to use RTNeural for LSTM models, it is recommended that you add ```-DBUILD_STATIC_RTNEURAL=ON``` to your cmake commandline. This will create static model implmentations for the same set of LSTM models as the internal implmentation, and results in increased performance. Interal static LSTM model support is also off by default - to turn it on use ```-DBUILD_INTERNAL_STATIC_LSTM=ON```. ### Composite model load behavior From 72ee608d0bdcfde5bf4ea5a68a5a632adbd7cdd9 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 26 Jun 2026 08:47:10 -0700 Subject: [PATCH 100/162] Fix FTNeural build --- NeuralAudio/NeuralModel.cpp | 7 ------- NeuralAudio/RTNeuralLoader.cpp | 1 - NeuralAudio/RTNeuralModel.h | 2 -- 3 files changed, 10 deletions(-) diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 8805fa5..0199435 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -253,13 +253,6 @@ namespace NeuralAudio if (isOfficialArchitecture) { - if (wavenetLoadMode == EModelLoadMode::RTNeural) - { -#ifdef BUILD_STATIC_RTNEURAL - newModel = RTNeuralLoadNAMWaveNet(modelJson, this); -#endif - } - if (newModel == nullptr) { auto modelDef = FindInternalWaveNetDefinition(firstLayerConfig.at("channels"), firstLayerConfig.at("head_size")); diff --git a/NeuralAudio/RTNeuralLoader.cpp b/NeuralAudio/RTNeuralLoader.cpp index e54dad2..4b3f637 100644 --- a/NeuralAudio/RTNeuralLoader.cpp +++ b/NeuralAudio/RTNeuralLoader.cpp @@ -5,7 +5,6 @@ namespace NeuralAudio { #ifdef BUILD_STATIC_RTNEURAL std::list rtNeuralLSTMModelDefs; - std::list rtNeuralWaveNetModelDefs; void EnsureRTNeuralModelDefsAreLoaded() { diff --git a/NeuralAudio/RTNeuralModel.h b/NeuralAudio/RTNeuralModel.h index dbb2240..899a49f 100644 --- a/NeuralAudio/RTNeuralModel.h +++ b/NeuralAudio/RTNeuralModel.h @@ -123,8 +123,6 @@ namespace NeuralAudio model = new ModelType; - auto& config = modelJson.at("config"); - std::vector weights = modelJson.at("weights"); const int networkInputSize = 1; From b17078080d6a6efa5f68c23483ae27b262bbf33e Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 26 Jun 2026 09:15:43 -0700 Subject: [PATCH 101/162] Use gcc15 for linux builds (#57) --- .github/workflows/build.yml | 7 ++++++- .github/workflows/release.yml | 13 ++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 098cbc1..5a0e71f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,11 +40,16 @@ jobs: - uses: actions/checkout@v3.3.0 with: submodules: recursive + + - name: Setup GCC 15 + uses: egor-tensin/setup-gcc@v2 + with: + version: '15' - name: Build working-directory: ${{github.workspace}}/build env: - CXX: g++-13 + CXX: g++-15 run: | cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_STATIC_RTNEURAL=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON cmake --build . --config $BUILD_TYPE -j4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f5f2f16..0dcf07b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,12 +41,17 @@ jobs: - uses: actions/checkout@v3.3.0 with: submodules: recursive + + - name: Setup GCC 15 + uses: egor-tensin/setup-gcc@v2 + with: + version: '15' - name: Build working-directory: ${{github.workspace}}/build env: - CXX: clang++ - CXXFLAGS: "-Wall -O3 -march=${{ matrix.native_arch }}" + CXX: g++-15 + CXXFLAGS: "-Wall -O3 -march=${{ matrix.native_arch }} -static-libstdc++ -static-libgcc" run: | cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON cmake --build . --config $BUILD_TYPE -j4 @@ -77,7 +82,7 @@ jobs: native_arch: [rpi4, rpi5] runs-on: ubuntu-latest container: - image: ghcr.io/tttapa/docker-arm-cross-toolchain:aarch64-rpi3-linux-gnu-gcc12 + image: ghcr.io/tttapa/docker-arm-cross-toolchain:aarch64-rpi3-linux-gnu-gcc15 env: ARCHIVE_NAME: ModelTest_${{ matrix.native_arch }}.tgz steps: @@ -87,6 +92,8 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build + env: + CXXFLAGS: "-static-libstdc++ -static-libgcc" run: | cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DCMAKE_TOOLCHAIN_FILE=/home/develop/opt/x-tools/aarch64-rpi3-linux-gnu/aarch64-${{ matrix.native_arch }}-linux-gnu.toolchain.cmake cmake --build . --config $BUILD_TYPE -j4 From 81f31ecfe8a8a8efbf21e84a43a3ed0115b3478d Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 27 Jun 2026 09:26:12 -0700 Subject: [PATCH 102/162] Make layer kernel sizes configurable. Add LeakyReLU activation. Move convolution buffer into separate class. --- NeuralAudio/Activation.h | 20 +++++ NeuralAudio/InternalModel.h | 11 ++- NeuralAudio/WaveNet.h | 161 ++++++++++++++++++++--------------- NeuralAudio/WaveNetDynamic.h | 18 ++-- 4 files changed, 126 insertions(+), 84 deletions(-) diff --git a/NeuralAudio/Activation.h b/NeuralAudio/Activation.h index a04af9b..edcafa3 100644 --- a/NeuralAudio/Activation.h +++ b/NeuralAudio/Activation.h @@ -37,6 +37,16 @@ namespace NeuralAudio { return 1.0f / (1.0f + std::exp(-x)); } + + static inline float LeakyRelu(float x, float negativeSlope) + { + return x > 0.0f ? x : negativeSlope * x; + } + + static inline float LeakyRelu(float x) + { + return LeakyRelu(x, 0.01f); + } }; struct FastMath @@ -74,6 +84,16 @@ namespace NeuralAudio //return 1.0f / (1.0f + std::exp(-x)); return 0.5f * (Tanh(x * 0.5f) + 1); } + + static inline float LeakyRelu(float x, float negativeSlope) + { + return x > 0.0f ? x : negativeSlope * x; + } + + static inline float LeakyRelu(float x) + { + return LeakyRelu(x, 0.01f); + } }; struct EigenMath diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index aa79de3..d851b95 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -10,8 +10,11 @@ namespace NeuralAudio { using IStdDilations = NeuralAudio::Dilations<1, 2, 4, 8, 16, 32, 64, 128, 256, 512>; + using IStdKernelSizes = NeuralAudio::KernelSizes<3, 3, 3, 3, 3, 3, 3, 3, 3, 3>; using ILiteDilations1 = NeuralAudio::Dilations<1, 2, 4, 8, 16, 32, 64>; + using ILiteKernelSizes1 = NeuralAudio::KernelSizes<3, 3, 3, 3, 3, 3, 3>; using ILiteDilations2 = NeuralAudio::Dilations<128, 256, 512, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512>; + using ILiteKernelSizes2 = NeuralAudio::KernelSizes<3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3>; class InternalModel : public NeuralModelImpl { @@ -51,11 +54,11 @@ namespace NeuralAudio { using ModelType = typename std::conditional, - NeuralAudio::WaveNetLayerArrayT>, + NeuralAudio::WaveNetLayerArrayT<1, 1, headSize, numChannels, IStdKernelSizes, IStdDilations, false>, + NeuralAudio::WaveNetLayerArrayT>, NeuralAudio::WaveNetModelT< - NeuralAudio::WaveNetLayerArrayT<1, 1, headSize, numChannels, 3, ILiteDilations1, false>, - NeuralAudio::WaveNetLayerArrayT> + NeuralAudio::WaveNetLayerArrayT<1, 1, headSize, numChannels, ILiteKernelSizes1, ILiteDilations1, false>, + NeuralAudio::WaveNetLayerArrayT> >::type; public: diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 8580503..378e9d4 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -123,6 +123,74 @@ namespace NeuralAudio Eigen::Vector bias; }; + template + class ChannelBuffer + { + public: + static constexpr auto BufferSize = ReceptiveFieldSize + ((LAYER_ARRAY_BUFFER_PADDING + 1) * WAVENET_MAX_NUM_FRAMES); + static constexpr bool TooBigForStatic = ((Channels * BufferSize) * 4) > EIGEN_STACK_ALLOCATION_LIMIT; + + using ChannelBufferType = typename std::conditional, + Eigen::Matrix>::type; + + ChannelBufferType buffer; + //Eigen::Matrix layerBuffer; + size_t bufferStart; + + void AllocBuffer(int allocNum) + { + long size = BufferSize; + + if constexpr (TooBigForStatic) + { + buffer.resize(Channels, size); + } + + buffer.setZero(); + + //if (offset > (size - (ReceptiveFieldSize + WAVENET_MAX_NUM_FRAMES))) + //{ + // bufferStart = ReceptiveFieldSize; + //} + //else + //{ + // bufferStart = size - offset; + //} + +#if (LAYER_ARRAY_BUFFER_PADDING == 0) + bufferStart = ReceptiveFieldSize; +#else + bufferStart = size - (WAVENET_MAX_NUM_FRAMES * ((allocNum % LAYER_ARRAY_BUFFER_PADDING) + 1)); // Do the modulo to handle cases where LAYER_ARRAY_BUFFER_PADDING is not big enough to handle offset +#endif + } + + void AdvanceFrames(const size_t numFrames) + { + bufferStart += numFrames; + + if ((bufferStart + WAVENET_MAX_NUM_FRAMES) > (size_t)buffer.cols()) + RewindBuffer(); + } + + void RewindBuffer() + { + size_t start = ReceptiveFieldSize; + + buffer.middleCols(start - ReceptiveFieldSize, ReceptiveFieldSize) = buffer.middleCols(bufferStart - ReceptiveFieldSize, ReceptiveFieldSize); + + bufferStart = start; + } + + void CopyBuffer() + { + for (size_t offset = 1; offset < ReceptiveFieldSize + 1; offset++) + { + buffer.col(bufferStart - offset) = buffer.col(bufferStart); + } + } + }; + template class WaveNetLayerT { @@ -134,16 +202,8 @@ namespace NeuralAudio public: static constexpr auto ReceptiveFieldSize = (KernelSize - 1) * Dilation; - static constexpr auto BufferSize = ReceptiveFieldSize + ((LAYER_ARRAY_BUFFER_PADDING + 1) * WAVENET_MAX_NUM_FRAMES); - static constexpr bool TooBigForStatic = ((Channels * BufferSize) * 4) > EIGEN_STACK_ALLOCATION_LIMIT; - - using LayerBufferType = typename std::conditional, - Eigen::Matrix>::type; - LayerBufferType layerBuffer; - //Eigen::Matrix layerBuffer; - size_t bufferStart; + ChannelBuffer channelBuffer; WaveNetLayerT() { @@ -152,29 +212,12 @@ namespace NeuralAudio void AllocBuffer(int allocNum) { - long size = BufferSize; - - if constexpr(TooBigForStatic) - { - layerBuffer.resize(Channels, size); - } - - layerBuffer.setZero(); - - //if (offset > (size - (ReceptiveFieldSize + WAVENET_MAX_NUM_FRAMES))) - //{ - // bufferStart = ReceptiveFieldSize; - //} - //else - //{ - // bufferStart = size - offset; - //} + channelBuffer.AllocBuffer(allocNum); + } -#if (LAYER_ARRAY_BUFFER_PADDING == 0) - bufferStart = ReceptiveFieldSize; -#else - bufferStart = size - (WAVENET_MAX_NUM_FRAMES * ((allocNum % LAYER_ARRAY_BUFFER_PADDING) + 1)); // Do the modulo to handle cases where LAYER_ARRAY_BUFFER_PADDING is not big enough to handle offset -#endif + void AdvanceFrames(const size_t numFrames) + { + channelBuffer.AdvanceFrames(numFrames); } size_t GetNumWeights() @@ -189,39 +232,12 @@ namespace NeuralAudio oneByOne.SetWeights(weights); } - void AdvanceFrames(const size_t numFrames) - { - bufferStart += numFrames; - - if ((bufferStart + WAVENET_MAX_NUM_FRAMES) > (size_t)layerBuffer.cols()) - RewindBuffer(); - } - - void RewindBuffer() - { - //numRewinds++; - - size_t start = ReceptiveFieldSize; - - layerBuffer.middleCols(start - ReceptiveFieldSize, ReceptiveFieldSize) = layerBuffer.middleCols(bufferStart - ReceptiveFieldSize, ReceptiveFieldSize); - - bufferStart = start; - } - - void CopyBuffer() - { - for (size_t offset = 1; offset < ReceptiveFieldSize + 1; offset++) - { - layerBuffer.col(bufferStart - offset) = layerBuffer.col(bufferStart); - } - } - template void Process(const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInput, Eigen::MatrixBase const& output, const size_t outputStart, const size_t numFrames) { auto block = state.leftCols(numFrames); - conv1D.Process(layerBuffer, block, bufferStart, numFrames); + conv1D.Process(channelBuffer.buffer, block, channelBuffer.bufferStart, numFrames); inputMixin.ProcessAcc(condition, block); @@ -231,28 +247,31 @@ namespace NeuralAudio oneByOne.Process(block, const_cast&>(output).middleCols(outputStart, numFrames)); - const_cast&>(output).middleCols(outputStart, numFrames).noalias() += layerBuffer.middleCols(bufferStart, numFrames); + const_cast&>(output).middleCols(outputStart, numFrames).noalias() += channelBuffer.buffer.middleCols(channelBuffer.bufferStart, numFrames); } }; template using Dilations = std::integer_sequence; - template + template + using KernelSizes = std::integer_sequence; + + template class WaveNetLayerArrayT { - template + template struct LayersHelper { }; - template - struct LayersHelper> + template + struct LayersHelper, Dilations> { - using type = std::tuple...>; + using type = std::tuple...>; }; - using Layers = typename LayersHelper::type; + using Layers = typename LayersHelper::type; private: Layers layers; @@ -317,11 +336,11 @@ namespace NeuralAudio template void Prewarm(const Eigen::MatrixBase& layerInputs, const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInputs) { - rechannel.Process(layerInputs.leftCols(1), std::get<0>(layers).layerBuffer.middleCols(std::get<0>(layers).bufferStart, 1)); + rechannel.Process(layerInputs.leftCols(1), std::get<0>(layers).channelBuffer.buffer.middleCols(std::get<0>(layers).channelBuffer.bufferStart, 1)); ForEachIndex([&](auto layerIndex) { - std::get(layers).CopyBuffer(); + std::get(layers).channelBuffer.CopyBuffer(); if constexpr (layerIndex == lastLayer) { @@ -329,7 +348,7 @@ namespace NeuralAudio } else { - std::get(layers).Process(condition, headInputs, std::get(layers).layerBuffer, std::get(layers).bufferStart, 1); + std::get(layers).Process(condition, headInputs, std::get(layers).channelBuffer.buffer, std::get(layers).channelBuffer.bufferStart, 1); } }); @@ -339,7 +358,7 @@ namespace NeuralAudio template void Process(const Eigen::MatrixBase& layerInputs, const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInputs, const size_t numFrames) { - rechannel.Process(layerInputs.leftCols(numFrames), std::get<0>(layers).layerBuffer.middleCols(std::get<0>(layers).bufferStart, numFrames)); + rechannel.Process(layerInputs.leftCols(numFrames), std::get<0>(layers).channelBuffer.buffer.middleCols(std::get<0>(layers).channelBuffer.bufferStart, numFrames)); ForEachIndex([&](auto layerIndex) { @@ -349,7 +368,7 @@ namespace NeuralAudio } else { - std::get(layers).Process(condition, headInputs, std::get(layers).layerBuffer, std::get(layers).bufferStart, numFrames); + std::get(layers).Process(condition, headInputs, std::get(layers).channelBuffer.buffer, std::get(layers).channelBuffer.bufferStart, numFrames); } std::get(layers).AdvanceFrames(numFrames); diff --git a/NeuralAudio/WaveNetDynamic.h b/NeuralAudio/WaveNetDynamic.h index 2179c90..3049b6f 100644 --- a/NeuralAudio/WaveNetDynamic.h +++ b/NeuralAudio/WaveNetDynamic.h @@ -151,7 +151,7 @@ namespace NeuralAudio DenseLayer inputMixin; DenseLayer oneByOne; Eigen::MatrixXf state; - Eigen::MatrixXf layerBuffer; + Eigen::MatrixXf buffer; public: size_t ReceptiveFieldSize; @@ -171,15 +171,15 @@ namespace NeuralAudio Eigen::MatrixXf& GetLayerBuffer() { - return layerBuffer; + return buffer; } void AllocBuffer(size_t allocNum) { size_t size = ReceptiveFieldSize + ((LAYER_ARRAY_BUFFER_PADDING + 1) * WAVENET_MAX_NUM_FRAMES); - layerBuffer.resize(channels, size); - layerBuffer.setZero(); + buffer.resize(channels, size); + buffer.setZero(); // offset prevents buffer rewinds of various layers from happening at the same time #if (LAYER_ARRAY_BUFFER_PADDING == 0) @@ -205,7 +205,7 @@ namespace NeuralAudio { bufferStart += numFrames; - if ((int)(bufferStart + WAVENET_MAX_NUM_FRAMES) > layerBuffer.cols()) + if ((int)(bufferStart + WAVENET_MAX_NUM_FRAMES) > buffer.cols()) RewindBuffer(); } @@ -213,7 +213,7 @@ namespace NeuralAudio { size_t start = ReceptiveFieldSize; - layerBuffer.middleCols(start - ReceptiveFieldSize, ReceptiveFieldSize) = layerBuffer.middleCols(bufferStart - ReceptiveFieldSize, ReceptiveFieldSize); + buffer.middleCols(start - ReceptiveFieldSize, ReceptiveFieldSize) = buffer.middleCols(bufferStart - ReceptiveFieldSize, ReceptiveFieldSize); bufferStart = start; } @@ -222,7 +222,7 @@ namespace NeuralAudio { for (size_t offset = 1; offset < ReceptiveFieldSize + 1; offset++) { - layerBuffer.col(bufferStart - offset) = layerBuffer.col(bufferStart); + buffer.col(bufferStart - offset) = buffer.col(bufferStart); } } @@ -230,7 +230,7 @@ namespace NeuralAudio { auto block = state.leftCols(numFrames); - conv1D.Process(layerBuffer, block, bufferStart, numFrames); + conv1D.Process(buffer, block, bufferStart, numFrames); inputMixin.ProcessAcc(condition, block); @@ -248,7 +248,7 @@ namespace NeuralAudio oneByOne.Process(block.topRows(channels), output.middleCols(outputStart, numFrames)); - output.middleCols(outputStart, numFrames).noalias() += layerBuffer.middleCols(bufferStart, numFrames); + output.middleCols(outputStart, numFrames).noalias() += buffer.middleCols(bufferStart, numFrames); AdvanceFrames(numFrames); } From c39f8f54769cce7e8548dd5c6abcca41ea321532 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 27 Jun 2026 09:49:15 -0700 Subject: [PATCH 103/162] Move convolution buffer into Conv1D class --- NeuralAudio/WaveNet.h | 177 +++++++++++++++++++++--------------------- 1 file changed, 90 insertions(+), 87 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 378e9d4..5f006a2 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -18,13 +18,81 @@ namespace NeuralAudio { - //int numRewinds = 0; - //int maxRewinds = 0; + template + class ChannelBuffer + { + public: + static constexpr auto BufferSize = ReceptiveFieldSize + ((LAYER_ARRAY_BUFFER_PADDING + 1) * WAVENET_MAX_NUM_FRAMES); + static constexpr bool TooBigForStatic = ((Channels * BufferSize) * 4) > EIGEN_STACK_ALLOCATION_LIMIT; + + using ChannelBufferType = typename std::conditional, + Eigen::Matrix>::type; + + ChannelBufferType buffer; + //Eigen::Matrix layerBuffer; + size_t bufferStart; + + void AllocBuffer(int allocNum) + { + long size = BufferSize; + + if constexpr (TooBigForStatic) + { + buffer.resize(Channels, size); + } + + buffer.setZero(); + + //if (offset > (size - (ReceptiveFieldSize + WAVENET_MAX_NUM_FRAMES))) + //{ + // bufferStart = ReceptiveFieldSize; + //} + //else + //{ + // bufferStart = size - offset; + //} + +#if (LAYER_ARRAY_BUFFER_PADDING == 0) + bufferStart = ReceptiveFieldSize; +#else + bufferStart = size - (WAVENET_MAX_NUM_FRAMES * ((allocNum % LAYER_ARRAY_BUFFER_PADDING) + 1)); // Do the modulo to handle cases where LAYER_ARRAY_BUFFER_PADDING is not big enough to handle offset +#endif + } + + void AdvanceFrames(const size_t numFrames) + { + bufferStart += numFrames; + + if ((bufferStart + WAVENET_MAX_NUM_FRAMES) > (size_t)buffer.cols()) + RewindBuffer(); + } + + void RewindBuffer() + { + size_t start = ReceptiveFieldSize; + + buffer.middleCols(start - ReceptiveFieldSize, ReceptiveFieldSize) = buffer.middleCols(bufferStart - ReceptiveFieldSize, ReceptiveFieldSize); + + bufferStart = start; + } + + void CopyBuffer() + { + for (size_t offset = 1; offset < ReceptiveFieldSize + 1; offset++) + { + buffer.col(bufferStart - offset) = buffer.col(bufferStart); + } + } + }; template class Conv1DT { public: + static constexpr auto ReceptiveFieldSize = (KernelSize - 1) * Dilation; + ChannelBuffer channelBuffer; + size_t GetNumWeights() { return OutChannels * InChannels * KernelSize + (DoBias ? OutChannels : 0); @@ -46,23 +114,23 @@ namespace NeuralAudio } } - template - inline void Process(const Eigen::MatrixBase& input, Eigen::MatrixBase const & output, const size_t iStart, const size_t nCols) const + template + inline void Process(Eigen::MatrixBase const & output, const size_t numFrames) const { for (size_t k = 0; k < KernelSize; k++) { auto offset = Dilation * ((int)k + 1 - KernelSize); - auto inBlock = input.middleCols(iStart + offset, nCols); + auto inBlock = channelBuffer.buffer.middleCols(channelBuffer.bufferStart + offset, numFrames); if (k == 0) - const_cast&>(output).noalias() = weights[k] * inBlock; + const_cast&>(output).noalias() = weights[k] * inBlock; else - const_cast&>(output).noalias() += weights[k] * inBlock; + const_cast&>(output).noalias() += weights[k] * inBlock; } if constexpr (DoBias) - const_cast&>(output).colwise() += bias; + const_cast&>(output).colwise() += bias; } private: @@ -123,74 +191,6 @@ namespace NeuralAudio Eigen::Vector bias; }; - template - class ChannelBuffer - { - public: - static constexpr auto BufferSize = ReceptiveFieldSize + ((LAYER_ARRAY_BUFFER_PADDING + 1) * WAVENET_MAX_NUM_FRAMES); - static constexpr bool TooBigForStatic = ((Channels * BufferSize) * 4) > EIGEN_STACK_ALLOCATION_LIMIT; - - using ChannelBufferType = typename std::conditional, - Eigen::Matrix>::type; - - ChannelBufferType buffer; - //Eigen::Matrix layerBuffer; - size_t bufferStart; - - void AllocBuffer(int allocNum) - { - long size = BufferSize; - - if constexpr (TooBigForStatic) - { - buffer.resize(Channels, size); - } - - buffer.setZero(); - - //if (offset > (size - (ReceptiveFieldSize + WAVENET_MAX_NUM_FRAMES))) - //{ - // bufferStart = ReceptiveFieldSize; - //} - //else - //{ - // bufferStart = size - offset; - //} - -#if (LAYER_ARRAY_BUFFER_PADDING == 0) - bufferStart = ReceptiveFieldSize; -#else - bufferStart = size - (WAVENET_MAX_NUM_FRAMES * ((allocNum % LAYER_ARRAY_BUFFER_PADDING) + 1)); // Do the modulo to handle cases where LAYER_ARRAY_BUFFER_PADDING is not big enough to handle offset -#endif - } - - void AdvanceFrames(const size_t numFrames) - { - bufferStart += numFrames; - - if ((bufferStart + WAVENET_MAX_NUM_FRAMES) > (size_t)buffer.cols()) - RewindBuffer(); - } - - void RewindBuffer() - { - size_t start = ReceptiveFieldSize; - - buffer.middleCols(start - ReceptiveFieldSize, ReceptiveFieldSize) = buffer.middleCols(bufferStart - ReceptiveFieldSize, ReceptiveFieldSize); - - bufferStart = start; - } - - void CopyBuffer() - { - for (size_t offset = 1; offset < ReceptiveFieldSize + 1; offset++) - { - buffer.col(bufferStart - offset) = buffer.col(bufferStart); - } - } - }; - template class WaveNetLayerT { @@ -203,8 +203,6 @@ namespace NeuralAudio public: static constexpr auto ReceptiveFieldSize = (KernelSize - 1) * Dilation; - ChannelBuffer channelBuffer; - WaveNetLayerT() { state.setZero(); @@ -212,12 +210,12 @@ namespace NeuralAudio void AllocBuffer(int allocNum) { - channelBuffer.AllocBuffer(allocNum); + conv1D.channelBuffer.AllocBuffer(allocNum); } void AdvanceFrames(const size_t numFrames) { - channelBuffer.AdvanceFrames(numFrames); + conv1D.channelBuffer.AdvanceFrames(numFrames); } size_t GetNumWeights() @@ -225,6 +223,11 @@ namespace NeuralAudio return conv1D.GetNumWeights() + inputMixin.GetNumWeights() + oneByOne.GetNumWeights(); } + auto& GetInputBuffer() + { + return conv1D.channelBuffer; + } + void SetWeights(std::vector::iterator& weights) { conv1D.SetWeights(weights); @@ -237,7 +240,7 @@ namespace NeuralAudio { auto block = state.leftCols(numFrames); - conv1D.Process(channelBuffer.buffer, block, channelBuffer.bufferStart, numFrames); + conv1D.Process(block, numFrames); inputMixin.ProcessAcc(condition, block); @@ -247,7 +250,7 @@ namespace NeuralAudio oneByOne.Process(block, const_cast&>(output).middleCols(outputStart, numFrames)); - const_cast&>(output).middleCols(outputStart, numFrames).noalias() += channelBuffer.buffer.middleCols(channelBuffer.bufferStart, numFrames); + const_cast&>(output).middleCols(outputStart, numFrames).noalias() += conv1D.channelBuffer.buffer.middleCols(conv1D.channelBuffer.bufferStart, numFrames); } }; @@ -336,11 +339,11 @@ namespace NeuralAudio template void Prewarm(const Eigen::MatrixBase& layerInputs, const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInputs) { - rechannel.Process(layerInputs.leftCols(1), std::get<0>(layers).channelBuffer.buffer.middleCols(std::get<0>(layers).channelBuffer.bufferStart, 1)); + rechannel.Process(layerInputs.leftCols(1), std::get<0>(layers).GetInputBuffer().buffer.middleCols(std::get<0>(layers).GetInputBuffer().bufferStart, 1)); ForEachIndex([&](auto layerIndex) { - std::get(layers).channelBuffer.CopyBuffer(); + std::get(layers).GetInputBuffer().CopyBuffer(); if constexpr (layerIndex == lastLayer) { @@ -348,7 +351,7 @@ namespace NeuralAudio } else { - std::get(layers).Process(condition, headInputs, std::get(layers).channelBuffer.buffer, std::get(layers).channelBuffer.bufferStart, 1); + std::get(layers).Process(condition, headInputs, std::get(layers).GetInputBuffer().buffer, std::get(layers).GetInputBuffer().bufferStart, 1); } }); @@ -358,7 +361,7 @@ namespace NeuralAudio template void Process(const Eigen::MatrixBase& layerInputs, const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInputs, const size_t numFrames) { - rechannel.Process(layerInputs.leftCols(numFrames), std::get<0>(layers).channelBuffer.buffer.middleCols(std::get<0>(layers).channelBuffer.bufferStart, numFrames)); + rechannel.Process(layerInputs.leftCols(numFrames), std::get<0>(layers).GetInputBuffer().buffer.middleCols(std::get<0>(layers).GetInputBuffer().bufferStart, numFrames)); ForEachIndex([&](auto layerIndex) { @@ -368,7 +371,7 @@ namespace NeuralAudio } else { - std::get(layers).Process(condition, headInputs, std::get(layers).channelBuffer.buffer, std::get(layers).channelBuffer.bufferStart, numFrames); + std::get(layers).Process(condition, headInputs, std::get(layers).GetInputBuffer().buffer, std::get(layers).GetInputBuffer().bufferStart, numFrames); } std::get(layers).AdvanceFrames(numFrames); From 776bd71ca804d1c0d07ebdd739b920cba4dd8e85 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 27 Jun 2026 10:00:15 -0700 Subject: [PATCH 104/162] Make WaveNet head rechannel a generalized convolution --- NeuralAudio/WaveNet.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 5f006a2..2ebe075 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -279,7 +279,7 @@ namespace NeuralAudio private: Layers layers; DenseLayerT rechannel; - DenseLayerT headRechannel; + Conv1DT headRechannel; static constexpr auto numLayers = std::tuple_size_v; static constexpr auto lastLayer = numLayers - 1; @@ -355,7 +355,8 @@ namespace NeuralAudio } }); - headRechannel.Process(headInputs.leftCols(1), headOutputs.leftCols(1)); + headRechannel.channelBuffer.buffer.middleCols(headRechannel.channelBuffer.bufferStart, 1).noalias() = headInputs.leftCols(1); // Should be able to avoid this copy + headRechannel.Process(headOutputs.leftCols(1), 1); } template @@ -377,7 +378,8 @@ namespace NeuralAudio std::get(layers).AdvanceFrames(numFrames); }); - headRechannel.Process(headInputs.leftCols(numFrames), headOutputs.leftCols(numFrames)); + headRechannel.channelBuffer.buffer.middleCols(headRechannel.channelBuffer.bufferStart, numFrames).noalias() = headInputs.leftCols(numFrames); // Should be able to avoid this copy + headRechannel.Process(headOutputs.leftCols(numFrames), numFrames); } }; From dc4c022de3bf2d3c0e58b0df71c38659c58cc201 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 27 Jun 2026 12:04:22 -0700 Subject: [PATCH 105/162] First pass at working internal A2 --- NeuralAudio/Activation.h | 12 +++++++ NeuralAudio/CompositeModel.h | 15 +++++--- NeuralAudio/InternalModel.h | 34 +++++++++--------- NeuralAudio/NeuralModel.cpp | 67 +++++++++++++++++++++++++++--------- NeuralAudio/WaveNet.h | 10 ++++-- NeuralAudio/WaveNetDynamic.h | 6 ++-- 6 files changed, 101 insertions(+), 43 deletions(-) diff --git a/NeuralAudio/Activation.h b/NeuralAudio/Activation.h index edcafa3..e348d4b 100644 --- a/NeuralAudio/Activation.h +++ b/NeuralAudio/Activation.h @@ -85,6 +85,18 @@ namespace NeuralAudio return 0.5f * (Tanh(x * 0.5f) + 1); } + template + static void LeakyRelu(Matrix* x) + { + float* data = x->data(); + size_t size = x->rows() * x->cols(); + + for (size_t pos = 0; pos < size; pos++) + { + data[pos] = LeakyRelu(data[pos]); + } + } + static inline float LeakyRelu(float x, float negativeSlope) { return x > 0.0f ? x : negativeSlope * x; diff --git a/NeuralAudio/CompositeModel.h b/NeuralAudio/CompositeModel.h index 368b44f..7a54de7 100644 --- a/NeuralAudio/CompositeModel.h +++ b/NeuralAudio/CompositeModel.h @@ -65,6 +65,16 @@ namespace NeuralAudio } } + EModelLoadMode GetLoadMode() override + { + return models[currentModelIndex.load()]->GetLoadMode(); + } + + bool IsStatic() override + { + return models[currentModelIndex.load()]->IsStatic(); + } + int GetReceptiveFieldSize() override { if (currentModelIndex == -1) @@ -117,11 +127,6 @@ namespace NeuralAudio class ScalableCompositeModel : public CompositeModel { public: - EModelLoadMode GetLoadMode() override - { - return EModelLoadMode::NAMCore; - } - bool LoadFromJson(nlohmann::json& modelJson) { ReadNAMConfig(modelJson); diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index d851b95..ef803f5 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -16,6 +16,9 @@ namespace NeuralAudio using ILiteDilations2 = NeuralAudio::Dilations<128, 256, 512, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512>; using ILiteKernelSizes2 = NeuralAudio::KernelSizes<3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3>; + using A2KernelSizes = NeuralAudio::KernelSizes<6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 15, 15, 6, 6, 6, 6, 6, 6, 6>; + using A2Dilations = NeuralAudio::Dilations<1, 3, 7, 17, 41, 101, 239, 1, 3, 7, 17, 41, 101, 239, 1, 13, 1, 3, 7, 17, 41, 101, 239>; + class InternalModel : public NeuralModelImpl { public: @@ -48,19 +51,9 @@ namespace NeuralAudio } }; - - template + template class InternalWaveNetModelT : public InternalModel { - using ModelType = typename std::conditional, - NeuralAudio::WaveNetLayerArrayT>, - NeuralAudio::WaveNetModelT< - NeuralAudio::WaveNetLayerArrayT<1, 1, headSize, numChannels, ILiteKernelSizes1, ILiteDilations1, false>, - NeuralAudio::WaveNetLayerArrayT> - >::type; - public: InternalWaveNetModelT() : model(nullptr) @@ -152,23 +145,32 @@ namespace NeuralAudio } }; - template - class InternalWaveNetDefinitionT : public InternalWaveNetDefinitionBase + template + class InternalA1WaveNetDefinitionT : public InternalWaveNetDefinitionBase { public: + using ModelType = typename std::conditional, + NeuralAudio::WaveNetLayerArrayT>, + NeuralAudio::WaveNetModelT< + NeuralAudio::WaveNetLayerArrayT<1, 1, HeadSize, 1, 1, NumChannels, ILiteKernelSizes1, ILiteDilations1, false>, + NeuralAudio::WaveNetLayerArrayT> + >::type; + InternalModel* CreateModel() override { - return new InternalWaveNetModelT; + return new InternalWaveNetModelT; } virtual size_t GetNumChannels() override { - return numChannels; + return NumChannels; } virtual size_t GetHeadSize() override { - return headSize; + return HeadSize; } }; diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 0199435..53314cb 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -22,10 +22,10 @@ namespace NeuralAudio if (!modelDefsAreLoaded) { #ifdef BUILD_INTERNAL_STATIC_WAVENET - internalWavenetModelDefs.push_back(new InternalWaveNetDefinitionT<16, 8>); // Standard - internalWavenetModelDefs.push_back(new InternalWaveNetDefinitionT<12, 6>); // Lite - internalWavenetModelDefs.push_back(new InternalWaveNetDefinitionT<8, 4>); // Feather - internalWavenetModelDefs.push_back(new InternalWaveNetDefinitionT<4, 2>); // Nano + internalWavenetModelDefs.push_back(new InternalA1WaveNetDefinitionT<16, 8>); // Standard + internalWavenetModelDefs.push_back(new InternalA1WaveNetDefinitionT<12, 6>); // Lite + internalWavenetModelDefs.push_back(new InternalA1WaveNetDefinitionT<8, 4>); // Feather + internalWavenetModelDefs.push_back(new InternalA1WaveNetDefinitionT<4, 2>); // Nano #endif #ifdef BUILD_INTERNAL_STATIC_LSTM @@ -46,11 +46,11 @@ namespace NeuralAudio } } - static InternalWaveNetDefinitionBase* FindInternalWaveNetDefinition(size_t numChannels, size_t headSize) + static InternalWaveNetDefinitionBase* FindInternalWaveNetDefinition(size_t NumChannels, size_t HeadSize) { for (auto const& model : internalWavenetModelDefs) { - if ((numChannels == model->GetNumChannels()) && (headSize == model->GetHeadSize())) + if ((NumChannels == model->GetNumChannels()) && (HeadSize == model->GetHeadSize())) return model; } @@ -72,6 +72,9 @@ namespace NeuralAudio static std::vector liteDilations = { 1, 2, 4, 8, 16, 32, 64 }; static std::vector liteDilations2 = { 128, 256, 512, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 }; + static std::vector a2KernelSizes = { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 15, 15, 6, 6, 6, 6, 6, 6, 6 }; + static std::vector a2Dilations = { 1, 3, 7, 17, 41, 101, 239, 1, 3, 7, 17, 41, 101, 239, 1, 13, 1, 3, 7, 17, 41, 101, 239 }; + static bool CheckDilations(const nlohmann::json& dilationJson, std::vector& checkDilations) { if (dilationJson.size() != checkDilations.size()) @@ -198,18 +201,18 @@ namespace NeuralAudio #ifdef BUILD_NAMCORE std::string version = modelJson.at("version"); - if ((wavenetLoadMode == EModelLoadMode::NAMCore) || NAMIsA2(version)) + if (arch == "SlimmableContainer") // Packed A2 multi-model file { - if (arch == "SlimmableContainer") // Packed A2 multi-model file - { - ScalableCompositeModel* model = new ScalableCompositeModel; + ScalableCompositeModel* model = new ScalableCompositeModel; - model->SetModelLoader(this); - model->LoadFromJson(modelJson); + model->SetModelLoader(this); + model->LoadFromJson(modelJson); - newModel = model; - } - else + newModel = model; + } + else + { + if ((wavenetLoadMode == EModelLoadMode::NAMCore)) // || NAMIsA2(version)) { NAMModel* model = new NAMModel; @@ -227,7 +230,39 @@ namespace NeuralAudio { if (arch == "WaveNet") { - if (config.at("layers").size() == 2) + if (config.at("layers").size() == 1) + { + auto& layerConfig = config.at("layers").at(0); + + if (CheckDilations(layerConfig.at("dilations"), a2Dilations)) + { + if (layerConfig.at("channels") == 3) + { + auto model = new InternalWaveNetModelT>>(); + + if (model != nullptr) + { + model->SetModelLoader(this); + model->LoadFromNAMJson(modelJson); + + newModel = model; + } + } + else if (layerConfig.at("channels") == 8) + { + auto model = new InternalWaveNetModelT >>(); + + if (model != nullptr) + { + model->SetModelLoader(this); + model->LoadFromNAMJson(modelJson); + + newModel = model; + } + } + } + } + else if (config.at("layers").size() == 2) { auto& firstLayerConfig = config.at("layers").at(0); auto& secondLayerConfig = config.at("layers").at(1); diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 2ebe075..900801b 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -244,7 +244,7 @@ namespace NeuralAudio inputMixin.ProcessAcc(condition, block); - WAVENET_MATH::Tanh(&block); + WAVENET_MATH::LeakyRelu(&block); const_cast&>(headInput).leftCols(numFrames).noalias() += block; @@ -260,7 +260,7 @@ namespace NeuralAudio template using KernelSizes = std::integer_sequence; - template + template class WaveNetLayerArrayT { template @@ -279,7 +279,7 @@ namespace NeuralAudio private: Layers layers; DenseLayerT rechannel; - Conv1DT headRechannel; + Conv1DT headRechannel; static constexpr auto numLayers = std::tuple_size_v; static constexpr auto lastLayer = numLayers - 1; @@ -307,6 +307,8 @@ namespace NeuralAudio std::get(layers).AllocBuffer(allocNum++); }); + headRechannel.channelBuffer.AllocBuffer(allocNum++); + return allocNum; } @@ -356,6 +358,7 @@ namespace NeuralAudio }); headRechannel.channelBuffer.buffer.middleCols(headRechannel.channelBuffer.bufferStart, 1).noalias() = headInputs.leftCols(1); // Should be able to avoid this copy + headRechannel.channelBuffer.CopyBuffer(); headRechannel.Process(headOutputs.leftCols(1), 1); } @@ -380,6 +383,7 @@ namespace NeuralAudio headRechannel.channelBuffer.buffer.middleCols(headRechannel.channelBuffer.bufferStart, numFrames).noalias() = headInputs.leftCols(numFrames); // Should be able to avoid this copy headRechannel.Process(headOutputs.leftCols(numFrames), numFrames); + headRechannel.channelBuffer.AdvanceFrames(numFrames); } }; diff --git a/NeuralAudio/WaveNetDynamic.h b/NeuralAudio/WaveNetDynamic.h index 3049b6f..ea56b56 100644 --- a/NeuralAudio/WaveNetDynamic.h +++ b/NeuralAudio/WaveNetDynamic.h @@ -267,12 +267,12 @@ namespace NeuralAudio public: - WaveNetLayerArray(size_t inputSize, size_t conditionSize, size_t headSize, size_t channels, size_t kernelSize, bool hasHeadBias, std::vector dilations) : + WaveNetLayerArray(size_t inputSize, size_t conditionSize, size_t HeadSize, size_t channels, size_t kernelSize, bool hasHeadBias, std::vector dilations) : channels(channels), rechannel(inputSize, channels, false), - headRechannel(channels, headSize, hasHeadBias), + headRechannel(channels, HeadSize, hasHeadBias), arrayOutputs(channels, WAVENET_MAX_NUM_FRAMES), - headOutputs(headSize, WAVENET_MAX_NUM_FRAMES) + headOutputs(HeadSize, WAVENET_MAX_NUM_FRAMES) { for (auto dilation : dilations) { From ec79bfea1f2580e831bf8b062a4950c28c4f6d19 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 27 Jun 2026 14:19:14 -0700 Subject: [PATCH 106/162] Gating checks for standard A2. Compiler flag for internal NAM A2 support. --- NeuralAudio/Activation.h | 28 ++++-- NeuralAudio/CMakeLists.txt | 8 ++ NeuralAudio/InternalModel.h | 8 +- NeuralAudio/NeuralModel.cpp | 186 +++++++++++++++++++++++++++++++++--- NeuralAudio/WaveNet.h | 21 +++- 5 files changed, 221 insertions(+), 30 deletions(-) diff --git a/NeuralAudio/Activation.h b/NeuralAudio/Activation.h index e348d4b..e69b405 100644 --- a/NeuralAudio/Activation.h +++ b/NeuralAudio/Activation.h @@ -38,14 +38,26 @@ namespace NeuralAudio return 1.0f / (1.0f + std::exp(-x)); } - static inline float LeakyRelu(float x, float negativeSlope) + template + static void LeakyReLU(Matrix* x) + { + float* data = x->data(); + size_t size = x->rows() * x->cols(); + + for (size_t pos = 0; pos < size; pos++) + { + data[pos] = LeakyReLU(data[pos]); + } + } + + static inline float LeakyReLU(float x, float negativeSlope) { return x > 0.0f ? x : negativeSlope * x; } - static inline float LeakyRelu(float x) + static inline float LeakyReLU(float x) { - return LeakyRelu(x, 0.01f); + return LeakyReLU(x, 0.01f); } }; @@ -86,25 +98,25 @@ namespace NeuralAudio } template - static void LeakyRelu(Matrix* x) + static void LeakyReLU(Matrix* x) { float* data = x->data(); size_t size = x->rows() * x->cols(); for (size_t pos = 0; pos < size; pos++) { - data[pos] = LeakyRelu(data[pos]); + data[pos] = LeakyReLU(data[pos]); } } - static inline float LeakyRelu(float x, float negativeSlope) + static inline float LeakyReLU(float x, float negativeSlope) { return x > 0.0f ? x : negativeSlope * x; } - static inline float LeakyRelu(float x) + static inline float LeakyReLU(float x) { - return LeakyRelu(x, 0.01f); + return LeakyReLU(x, 0.01f); } }; diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 4638fe8..fa25902 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -43,6 +43,14 @@ else() message(STATUS "NOT building Internal static WaveNet models") endif() +option(BUILD_STATIC_INTERNAL_NAMA2 "Build Internal static A2 WaveNet models" OFF) +if(BUILD_STATIC_INTERNAL_NAMA2) + message(STATUS "Building Internal static A2 WaveNet models") + add_definitions(-DBUILD_STATIC_INTERNAL_NAMA2) +else() + message(STATUS "NOT building Internal static A2 WaveNet models") +endif() + set(WAVENET_MATH "FastMath" CACHE STRING "WaveNet math functions") add_definitions(-DWAVENET_MATH=${WAVENET_MATH}) message(STATUS "WaveNet math is: ${WAVENET_MATH}") diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index ef803f5..02c4552 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -151,11 +151,11 @@ namespace NeuralAudio public: using ModelType = typename std::conditional, - NeuralAudio::WaveNetLayerArrayT>, + NeuralAudio::WaveNetLayerArrayT<1, 1, HeadSize, 1, 1, NumChannels, IStdKernelSizes, IStdDilations, false, EActivationType::Tanh>, + NeuralAudio::WaveNetLayerArrayT>, NeuralAudio::WaveNetModelT< - NeuralAudio::WaveNetLayerArrayT<1, 1, HeadSize, 1, 1, NumChannels, ILiteKernelSizes1, ILiteDilations1, false>, - NeuralAudio::WaveNetLayerArrayT> + NeuralAudio::WaveNetLayerArrayT<1, 1, HeadSize, 1, 1, NumChannels, ILiteKernelSizes1, ILiteDilations1, false, EActivationType::Tanh>, + NeuralAudio::WaveNetLayerArrayT> >::type; InternalModel* CreateModel() override diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 53314cb..41e8f24 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -75,14 +75,14 @@ namespace NeuralAudio static std::vector a2KernelSizes = { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 15, 15, 6, 6, 6, 6, 6, 6, 6 }; static std::vector a2Dilations = { 1, 3, 7, 17, 41, 101, 239, 1, 3, 7, 17, 41, 101, 239, 1, 13, 1, 3, 7, 17, 41, 101, 239 }; - static bool CheckDilations(const nlohmann::json& dilationJson, std::vector& checkDilations) + static bool CheckIntegerSequence(const nlohmann::json& sequenceJson, std::vector& sequenceVector) { - if (dilationJson.size() != checkDilations.size()) + if (sequenceJson.size() != sequenceVector.size()) return false; - for (size_t i = 0; i < dilationJson.size(); i++) + for (size_t i = 0; i < sequenceJson.size(); i++) { - if (dilationJson[i] != checkDilations[i]) + if (sequenceJson[i] != sequenceVector[i]) return false; } @@ -167,6 +167,155 @@ namespace NeuralAudio return (major > 0) || (minor > 5) || ((minor == 5) && (patch > 4)); } + bool ApproxEqual(float a, float b, float epsilon = 1e-5f) + { + return std::fabs(a - b) <= epsilon; + } + + bool IsActive(const nlohmann::json& json, const char* name) + { + if (!json.contains(name)) + return true; + + return json.at(name).value("active", false); + } + + bool HasNonNull(const nlohmann::json& json, const char* name) + { + return json.contains(name) && !json.at(name).is_null(); + } + + bool NAMIsA2Standard(const nlohmann::json& modelJson) + { + if (!modelJson.contains("architecture")) + return false; + + if (modelJson.at("architecture") != "WaveNet") + return false; + + if (!modelJson.contains("config")) + return false; + + auto& config = modelJson.at("config"); + + if (HasNonNull(config, "head")) + return false; + + if (config.contains("condition_dsp")) + return false; + + if (config.value("in_channels", 1) != 1) + return false; + + if (!config.contains("layers")) + return false; + + if (config.at("layers").size() != 1) + return false; + + const auto& layerConfig = config.at("layers").at(0); + + if (layerConfig.value("input_size", 0) != 1) + return false; + + if (layerConfig.value("condition_size", 0) != 1) + return false; + + const int channels = layerConfig.value("channels", 0); + + if ((channels != 3) && (channels != 8)) + return false; + + if (layerConfig.value("bottleneck", channels) != channels) + return false; + + if (!layerConfig.contains("kernel_sizes")) + return false; + + if (!CheckIntegerSequence(layerConfig.at("kernel_sizes"), a2KernelSizes)) + return false; + + if (!layerConfig.contains("dilations")) + return false; + + if (!CheckIntegerSequence(layerConfig.at("dilations"), a2Dilations)) + return false; + + if (!layerConfig.contains("activation")) + return false; + + for (const auto& activation : layerConfig.at("activation")) + { + if (!activation.contains("type")) + return false; + + if (activation.at("type") != "LeakyReLU") + return false; + + if (!ApproxEqual(activation.value("negative_slope", 0.01f), 0.01f)) + return false; + } + + if (layerConfig.contains("secondary_activation")) + { + for (const auto& activation : layerConfig.at("secondary_activation")) + { + if (!activation.is_null()) + return false; + } + } + + if (layerConfig.contains("gating_mode")) + { + for (const auto& gating : layerConfig.at("gating_mode")) + { + if (!gating.is_null() && (gating != "none")) + return false; + } + } + + if (!layerConfig.contains("head")) + return false; + + const auto& head = layerConfig.at("head"); + + if (head.value("out_channels", 1) != 1) + return false; + + if (head.value("kernel_size", 16) != 16) + return false; + + if (head.value("head_dilation", 1) != 1) + return false; + + if (!head.value("bias", true)) + return false; + + if (!IsActive(layerConfig, "layer1x1")) + return false; + + if (layerConfig.at("layer1x1").value("groups", 1) != 1) + return false; + + for (auto& key : { "head1x1", "conv_pre_film", "conv_post_film", "input_mixin_pre_film", "input_mixin_post_film", + "activation_pre_film", "activation_post_film", "layer1x1_post_film", "head1x1_post_film" }) + { + if (IsActive(layerConfig, key)) + return false; + } + + if (layerConfig.value("groups_input", 1) != 1) + return false; + + if (layerConfig.value("groups_input_mixin", 1) != 1) + return false; + + if (HasNonNull(layerConfig, "slimmable")) + return false; + + return true; + } + NeuralModel* NeuralModelLoader::CreateFromFile(const std::filesystem::path& modelPath, bool doPrewarm) { if (!std::filesystem::exists(modelPath)) @@ -198,9 +347,6 @@ namespace NeuralAudio std::string arch = modelJson.at("architecture"); -#ifdef BUILD_NAMCORE - std::string version = modelJson.at("version"); - if (arch == "SlimmableContainer") // Packed A2 multi-model file { ScalableCompositeModel* model = new ScalableCompositeModel; @@ -210,9 +356,19 @@ namespace NeuralAudio newModel = model; } - else + + + if (newModel == nullptr) { - if ((wavenetLoadMode == EModelLoadMode::NAMCore)) // || NAMIsA2(version)) +#ifdef BUILD_NAMCORE + std::string version = modelJson.at("version"); + + bool loadA2WithNAMCore = true; + +#ifdef BUILD_STATIC_INTERNAL_NAMA2 + loadA2WithNAMCore = false; +#endif + if ((wavenetLoadMode == EModelLoadMode::NAMCore) || (NAMIsA2(version) && (loadA2WithNAMCore || !NAMIsA2Standard(modelJson)))) { NAMModel* model = new NAMModel; @@ -232,13 +388,14 @@ namespace NeuralAudio { if (config.at("layers").size() == 1) { +#ifdef BUILD_STATIC_INTERNAL_NAMA2 auto& layerConfig = config.at("layers").at(0); - if (CheckDilations(layerConfig.at("dilations"), a2Dilations)) + if (CheckIntegerSequence(layerConfig.at("dilations"), a2Dilations)) { if (layerConfig.at("channels") == 3) { - auto model = new InternalWaveNetModelT>>(); + auto model = new InternalWaveNetModelT>>(); if (model != nullptr) { @@ -250,7 +407,7 @@ namespace NeuralAudio } else if (layerConfig.at("channels") == 8) { - auto model = new InternalWaveNetModelT >>(); + auto model = new InternalWaveNetModelT >>(); if (model != nullptr) { @@ -261,6 +418,7 @@ namespace NeuralAudio } } } +#endif } else if (config.at("layers").size() == 2) { @@ -273,14 +431,14 @@ namespace NeuralAudio if (firstLayerConfig.at("channels") == 16) { - if (CheckDilations(firstLayerConfig.at("dilations"), stdDilations) && CheckDilations(secondLayerConfig.at("dilations"), stdDilations)) + if (CheckIntegerSequence(firstLayerConfig.at("dilations"), stdDilations) && CheckIntegerSequence(secondLayerConfig.at("dilations"), stdDilations)) { isOfficialArchitecture = true; } } else { - if (CheckDilations(firstLayerConfig.at("dilations"), liteDilations) && CheckDilations(secondLayerConfig.at("dilations"), liteDilations2)) + if (CheckIntegerSequence(firstLayerConfig.at("dilations"), liteDilations) && CheckIntegerSequence(secondLayerConfig.at("dilations"), liteDilations2)) { isOfficialArchitecture = true; } diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 900801b..dff2c7a 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -16,6 +16,12 @@ #define LAYER_ARRAY_BUFFER_PADDING 24 #endif +enum EActivationType +{ + Tanh, + LeakyReLU +}; + namespace NeuralAudio { template @@ -191,7 +197,7 @@ namespace NeuralAudio Eigen::Vector bias; }; - template + template class WaveNetLayerT { private: @@ -244,7 +250,14 @@ namespace NeuralAudio inputMixin.ProcessAcc(condition, block); - WAVENET_MATH::LeakyRelu(&block); + if constexpr (Activation == EActivationType::Tanh) + { + WAVENET_MATH::Tanh(&block); + } + else if constexpr (Activation == EActivationType::LeakyReLU) + { + WAVENET_MATH::LeakyReLU(&block); + } const_cast&>(headInput).leftCols(numFrames).noalias() += block; @@ -260,7 +273,7 @@ namespace NeuralAudio template using KernelSizes = std::integer_sequence; - template + template class WaveNetLayerArrayT { template @@ -271,7 +284,7 @@ namespace NeuralAudio template struct LayersHelper, Dilations> { - using type = std::tuple...>; + using type = std::tuple...>; }; using Layers = typename LayersHelper::type; From 900c4b4eeecf3defb387cbc7e7251c7e8e131fbe Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 27 Jun 2026 14:33:40 -0700 Subject: [PATCH 107/162] Fix layer array receptive field size for convolution head rechannel --- NeuralAudio/WaveNet.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index dff2c7a..a56375a 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -311,6 +311,8 @@ namespace NeuralAudio { ReceptiveFieldSize += std::get(layers).ReceptiveFieldSize; }); + + ReceptiveFieldSize += headRechannel.ReceptiveFieldSize; } int AllocBuffers(int allocNum) From df230ab284770e18014d5f7de5059b062a20a8b0 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 6 Jul 2026 07:42:33 -0700 Subject: [PATCH 108/162] Optimized paths for specific size conv/matmul (#59) --- .github/workflows/release.yml | 6 +- NeuralAudio/Activation.h | 8 ++ NeuralAudio/CMakeLists.txt | 11 ++- NeuralAudio/MatMul.h | 154 ++++++++++++++++++++++++++++++++++ NeuralAudio/WaveNet.h | 140 +++++++++++++++++++++++++++---- README.md | 4 + 6 files changed, 305 insertions(+), 18 deletions(-) create mode 100644 NeuralAudio/MatMul.h diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0dcf07b..1fc471c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,7 +53,7 @@ jobs: CXX: g++-15 CXXFLAGS: "-Wall -O3 -march=${{ matrix.native_arch }} -static-libstdc++ -static-libgcc" run: | - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=ON cmake --build . --config $BUILD_TYPE -j4 - name: Add Archive @@ -95,7 +95,7 @@ jobs: env: CXXFLAGS: "-static-libstdc++ -static-libgcc" run: | - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DCMAKE_TOOLCHAIN_FILE=/home/develop/opt/x-tools/aarch64-rpi3-linux-gnu/aarch64-${{ matrix.native_arch }}-linux-gnu.toolchain.cmake + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=ON -DCMAKE_TOOLCHAIN_FILE=/home/develop/opt/x-tools/aarch64-rpi3-linux-gnu/aarch64-${{ matrix.native_arch }}-linux-gnu.toolchain.cmake cmake --build . --config $BUILD_TYPE -j4 - name: Add Archive @@ -135,7 +135,7 @@ jobs: env: CXXFLAGS: "/O2 -march=${{ matrix.native_arch }}" run: | - cmake.exe -G "Visual Studio 18 2026" -A x64 -T ClangCL -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON .. + cmake.exe -G "Visual Studio 18 2026" -A x64 -T ClangCL -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=ON .. cmake --build . --config=release -j4 - name: Add Archive diff --git a/NeuralAudio/Activation.h b/NeuralAudio/Activation.h index e69b405..0a62efa 100644 --- a/NeuralAudio/Activation.h +++ b/NeuralAudio/Activation.h @@ -100,6 +100,8 @@ namespace NeuralAudio template static void LeakyReLU(Matrix* x) { + //*x = (x->array() < 0.0f).select(x->array() * 0.01f, x->array()); + float* data = x->data(); size_t size = x->rows() * x->cols(); @@ -133,6 +135,12 @@ namespace NeuralAudio return Eigen::numext::tanh(x); } + template + static void LeakyReLU(Matrix* x) + { + *x = (x->array() < 0.0f).select(x->array() * 0.01f, x->array()); + } + static inline float Sigmoid(float x) { return 0.5f * (Tanh(x * 0.5f) + 1); diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index fa25902..6240237 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -43,7 +43,7 @@ else() message(STATUS "NOT building Internal static WaveNet models") endif() -option(BUILD_STATIC_INTERNAL_NAMA2 "Build Internal static A2 WaveNet models" OFF) +option(BUILD_STATIC_INTERNAL_NAMA2 "Build Internal static A2 WaveNet models" ON) if(BUILD_STATIC_INTERNAL_NAMA2) message(STATUS "Building Internal static A2 WaveNet models") add_definitions(-DBUILD_STATIC_INTERNAL_NAMA2) @@ -51,6 +51,14 @@ else() message(STATUS "NOT building Internal static A2 WaveNet models") endif() +option(ENABLE_MULTIFRAME_8X8_CONVOLUTION "Enable multi-frame 8x8 convolution" OFF) +if(ENABLE_MULTIFRAME_8X8_CONVOLUTION) + message(STATUS "Using multi-frame 8x8 convolution") + add_definitions(-DENABLE_MULTIFRAME_8X8_CONVOLUTION) +else() + message(STATUS "NOT using multi-frame 8x8 convolution") +endif() + set(WAVENET_MATH "FastMath" CACHE STRING "WaveNet math functions") add_definitions(-DWAVENET_MATH=${WAVENET_MATH}) message(STATUS "WaveNet math is: ${WAVENET_MATH}") @@ -104,6 +112,7 @@ set(SOURCES RTNeuralLoader.cpp RTNeuralLoader.h Activation.h + MatMul.h WaveNet.h WaveNetDynamic.h LSTM.h diff --git a/NeuralAudio/MatMul.h b/NeuralAudio/MatMul.h new file mode 100644 index 0000000..433a4ea --- /dev/null +++ b/NeuralAudio/MatMul.h @@ -0,0 +1,154 @@ +#pragma once + +namespace NeuralAudio +{ + template + struct MatMul + { + static constexpr bool HasKernel() { return ((InChannels == 3) && ((OutChannels == 3) || (OutChannels == 1))); } + + // 3x3 implementation inspired by @jfsantos NAM Core a2fast - https://github.com/sdatkinson/NeuralAmpModelerCore/blob/main/NAM/wavenet/a2_fast.cpp + + static inline void MultiplyInitZero(const float* inData, float* outData, const float* weights, size_t numFrames) + { + static_assert(HasKernel(), "Multiplication not implemented for InChannel/OutChannel combination"); + + if constexpr ((InChannels == 3) && (OutChannels == 3)) + { + const float w00 = weights[0], w10 = weights[1], w20 = weights[2]; + const float w01 = weights[3], w11 = weights[4], w21 = weights[5]; + const float w02 = weights[6], w12 = weights[7], w22 = weights[8]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + const float* in = &inData[offset]; + float* out = &outData[offset]; + + float a0 = w00 * in[0]; + float a1 = w10 * in[0]; + float a2 = w20 * in[0]; + a0 += w01 * in[1]; + a1 += w11 * in[1]; + a2 += w21 * in[1]; + a0 += w02 * in[2]; + a1 += w12 * in[2]; + a2 += w22 * in[2]; + out[0] = a0; + out[1] = a1; + out[2] = a2; + } + } + else if constexpr ((InChannels == 3) && (OutChannels == 1)) + { + const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + const float* in = &inData[offset]; + + float a0 = w0 * in[frame * 3]; + a0 += w1 * in[1]; + a0 += w2 * in[2]; + outData[frame] = a0; + } + } + } + + static inline void MultiplyInitColwise(const float* inData, float* outData, const float* weights, const float* initData, size_t numFrames) + { + static_assert(HasKernel(), "Multiplication not implemented for InChannel/OutChannel combination"); + + if constexpr ((InChannels == 3) && (OutChannels == 3)) + { + const float w00 = weights[0], w10 = weights[1], w20 = weights[2]; + const float w01 = weights[3], w11 = weights[4], w21 = weights[5]; + const float w02 = weights[6], w12 = weights[7], w22 = weights[8]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + const float* in = &inData[offset]; + float* out = &outData[offset]; + + float a0 = initData[0] + w00 * in[0]; + float a1 = initData[1] + w10 * in[0]; + float a2 = initData[2] + w20 * in[0]; + a0 += w01 * in[1]; + a1 += w11 * in[1]; + a2 += w21 * in[1]; + a0 += w02 * in[2]; + a1 += w12 * in[2]; + a2 += w22 * in[2]; + out[0] = a0; + out[1] = a1; + out[2] = a2; + } + } + else if constexpr ((InChannels == 3) && (OutChannels == 1)) + { + const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + const float* in = &inData[offset]; + + float a0 = initData[0] + w0 * in[0]; + a0 += w1 * in[1]; + a0 += w2 * in[2]; + outData[frame] = a0; + } + } + } + + static inline void MultiplyAccumlulate(const float* inData, float* outData, const float* weights, size_t numFrames) + { + static_assert(HasKernel(), "Multiplication not implemented for InChannel/OutChannel combination"); + + if constexpr ((InChannels == 3) && (OutChannels == 3)) + { + const float w00 = weights[0], w10 = weights[1], w20 = weights[2]; + const float w01 = weights[3], w11 = weights[4], w21 = weights[5]; + const float w02 = weights[6], w12 = weights[7], w22 = weights[8]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + const float* in = &inData[offset]; + float* out = &outData[offset]; + + float a0 = out[0] + w00 * in[0]; + float a1 = out[1] + w10 * in[0]; + float a2 = out[2] + w20 * in[0]; + a0 += w01 * in[1]; + a1 += w11 * in[1]; + a2 += w21 * in[1]; + a0 += w02 * in[2]; + a1 += w12 * in[2]; + a2 += w22 * in[2]; + out[0] = a0; + out[1] = a1; + out[2] = a2; + } + } + else if constexpr ((InChannels == 3) && (OutChannels == 1)) + { + const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; + + for (int frame = 0; frame < numFrames; frame++) + { + const int offset = frame * 3; + const float* in = &inData[offset]; + + float a0 = outData[frame] + w0 * in[0]; + a0 += w1 * in[1]; + a0 += w2 * in[2]; + outData[frame] = a0; + } + } + } + }; + +} \ No newline at end of file diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index a56375a..3d3db93 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -7,6 +7,7 @@ #include #include "TemplateHelper.h" #include "Activation.h" +#include "MatMul.h" #ifndef WAVENET_MAX_NUM_FRAMES #define WAVENET_MAX_NUM_FRAMES 64 @@ -123,19 +124,107 @@ namespace NeuralAudio template inline void Process(Eigen::MatrixBase const & output, const size_t numFrames) const { - for (size_t k = 0; k < KernelSize; k++) + float* outputPtr = &const_cast&>(output)(0, 0); + const float* biasPtr = bias.data(); + +#if ENABLE_MULTIFRAME_8X8_CONVOLUTION + if constexpr ((InChannels == 8) && (OutChannels == 8)) { - auto offset = Dilation * ((int)k + 1 - KernelSize); + // Based on @jfsantos NAM Core implementation - https://github.com/sdatkinson/NeuralAmpModelerCore/pull/277 - auto inBlock = channelBuffer.buffer.middleCols(channelBuffer.bufferStart + offset, numFrames); + constexpr size_t T = 4; + const size_t nF4 = (numFrames / T) * T; - if (k == 0) - const_cast&>(output).noalias() = weights[k] * inBlock; - else - const_cast&>(output).noalias() += weights[k] * inBlock; + for (size_t f = 0; f < nF4; f += T) + { + float a[T][InChannels]{}; + + for (size_t k = 0; k < KernelSize; k++) + { + const float* W = this->weights[k].data(); + const auto offset = Dilation * (k + 1 - KernelSize); + const auto inBlock = channelBuffer.buffer.middleCols(channelBuffer.bufferStart + offset + f, T); + const float* hb = inBlock.data(); + + for (size_t cp = 0; cp < InChannels; cp++) + { + const float* Wcol = W + cp * InChannels; + const float h0 = hb[cp], h1 = hb[InChannels + cp], h2 = hb[2 * InChannels + cp], h3 = hb[3 * InChannels + cp]; + + for (size_t o = 0; o < InChannels; o++) + { + a[0][o] += Wcol[o] * h0; + a[1][o] += Wcol[o] * h1; + a[2][o] += Wcol[o] * h2; + a[3][o] += Wcol[o] * h3; + } + } + } + + for (size_t ti = 0; ti < T; ti++) + std::memcpy(outputPtr + static_cast(f + ti) * InChannels, a[ti], InChannels * sizeof(float)); + } + + // Scalar tail for any frames past the T-aligned boundary. + for (size_t f = nF4; f < numFrames; f++) + { + float* zf = outputPtr + static_cast(f) * InChannels; + + for (size_t o = 0; o < InChannels; o++) + zf[o] = 0.0f; + + for (size_t k = 0; k < KernelSize; k++) + { + const float* W = this->weights[k].data(); + const auto offset = Dilation * (k + 1 - KernelSize); + const auto inBlock = channelBuffer.buffer.middleCols(channelBuffer.bufferStart + offset + f, T); + const float* h = inBlock.data(); + + for (int cp = 0; cp < InChannels; cp++) + { + const float hv = h[cp]; + const float* Wcol = W + cp * InChannels; + + for (size_t o = 0; o < InChannels; o++) + zf[o] += Wcol[o] * hv; + } + } + } } + else +#endif + { + for (size_t k = 0; k < KernelSize; k++) + { + const float* weightPtr = this->weights[k].data(); - if constexpr (DoBias) + const auto offset = Dilation * ((int)k + 1 - KernelSize); + + const auto inBlock = channelBuffer.buffer.middleCols(channelBuffer.bufferStart + offset, numFrames); + const float* inputPtr = inBlock.data(); + + if constexpr (MatMul::HasKernel()) + { + if (k == 0) // Maybe move this out of loop? + { + MatMul::MultiplyInitColwise(inputPtr, outputPtr, weightPtr, biasPtr, numFrames); + } + else + { + MatMul::MultiplyAccumlulate(inputPtr, outputPtr, weightPtr, numFrames); + } + } + else + { + if (k == 0) + const_cast&>(output).noalias() = weights[k] * inBlock; + else + const_cast&>(output).noalias() += weights[k] * inBlock; + } + } + } + + if constexpr (DoBias && !MatMul::HasKernel()) const_cast&>(output).colwise() += bias; } @@ -169,26 +258,47 @@ namespace NeuralAudio template void Process(const Eigen::MatrixBase& input, Eigen::MatrixBase const& output) const { - if constexpr (DoBias) + if constexpr (MatMul::HasKernel()) { - const_cast&>(output).noalias() = (weights * input).colwise() + bias; + if constexpr (DoBias) + { + MatMul::MultiplyInitColwise(&input(0, 0), &const_cast&>(output)(0, 0), weights.data(), bias.data(), output.cols()); + } + else + { + MatMul::MultiplyInitZero(&input(0, 0), &const_cast&>(output)(0, 0), weights.data(), output.cols()); + } } else { - const_cast&>(output).noalias() = weights * input; + if constexpr (DoBias) + { + const_cast&>(output).noalias() = (weights * input).colwise() + bias; + } + else + { + const_cast&>(output).noalias() = weights * input; + } } } template void ProcessAcc(const Eigen::MatrixBase& input, Eigen::MatrixBase const& output) const { - if constexpr (DoBias) + if constexpr (!DoBias && MatMul::HasKernel()) { - const_cast&>(output).noalias() += (weights * input).colwise() + bias; + MatMul::MultiplyAccumlulate(&input(0, 0), &const_cast&>(output)(0, 0), weights.data(), output.cols()); } else { - const_cast&>(output).noalias() += weights * input; + if constexpr (DoBias) + { + const_cast&>(output).noalias() += (weights * input).colwise() + bias; + } + else + { + const_cast&>(output).noalias() += weights * input; + } } } @@ -466,6 +576,8 @@ namespace NeuralAudio auto condition = Eigen::Map>(&input, 1, 1); + headArray.setZero(); + ForEachIndex([&](auto layerIndex) { if constexpr (layerIndex == 0) diff --git a/README.md b/README.md index a0f9740..75903d2 100644 --- a/README.md +++ b/README.md @@ -230,6 +230,10 @@ Note - you'll have to change the Visual Studio version if you are using a differ ```-DBUILD_INTERNAL_STATIC_LSTM=ON|OFF```: Build internal static LSTM model architectures (faster internal LSTM, but slower compile, larger size). +```-DBUILD_STATIC_INTERNAL_NAMA2=ON|OFF```: Build internal static A2 implementation. + +```-DENABLE_MULTIFRAME_8X8_CONVOLUTION=ON|OFF```: Use optimized multiframe 8x8 convolution. Much faster on very modern compilers. Much slower on older compilers. + ```-DDEFAULT_QUALITY_SCALE="X.X"```: Default model quality scale factor (0.0 to 1.0). Be sure to use quotes around value. Defaults to "1.0". ```-DDEFAULT_INPUT_DBU="XX"```: Default dBu level for model input calibration. From c70ca178586407742114040855f30842f2684767 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 6 Jul 2026 07:50:35 -0700 Subject: [PATCH 109/162] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 75903d2..a393425 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,13 @@ This repository is licensed under the [MIT license](https://github.com/mikeoliph # Underlying Libraries -NAM A2 models currently use the NAM Core implementation (and consequently require building with NAM Core enabled). - -For A1 WaveNet, the internal implmeentation supports optimized static models of the offical NAM A1 network architectures: "Standard", "Lite", "Feather", "Nano". +For WaveNet, the internal implmeentation supports optimized static implemenationas the offical NAM A1 and A2 network architectures: A1 "Standard", "Lite", "Feather", "Nano" and A2 "Lite" and "Full". For A1 LSTM, the internal implementation supports optimized static models architectures for 1x8, 1x12, 1x16, 1x24, 2x8, 2x12, and 2x16 models. -All A1 NAM files with WaveNet and LSTM architectures not supported internally will fall back on a less performant dynamic implementation (although still faster than NAM Core). +All A1 NAM files with WaveNet and LSTM architectures not supported statically will fall back on a less performant dynamic implementation. + +All non-standard A2 models currently use the NAM Core implementation (and consequently require building with NAM Core enabled). All keras models not supported internally will fall back to the RTNeural implmentation. From 9a6a36689b4808f5ea695ba019de16cdd50535fb Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 6 Jul 2026 08:16:32 -0700 Subject: [PATCH 110/162] Update README.md --- README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a393425..99fe167 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,17 @@ NeuralAudio is a C++ library designed to make it easy to use neural network machine learning models (ie: guitar amplifier captures/profiles) in real-time audio applications. -# Supported Models +# License + +This repository is licensed under the [MIT license](https://github.com/mikeoliphant/NeuralAudio/blob/main/LICENSE). It is a liberal license, but please make sure that you comply with the terms - as well as the terms of [this project's dependencies](https://github.com/mikeoliphant/NeuralAudio/blob/main/CREDITS.md). I would also appreciate it if you would let me know if you are using this library. + +# Supported Models and Underlying Libraries NeuralAudio currently supports the following model types: - [Neural Amp Modeler](https://github.com/sdatkinson/neural-amp-modeler) (NAM) WaveNet and LSTM models, A1 and A2 support - [RTNeural](https://github.com/jatinchowdhury18/RTNeural) keras models (LSTM, GRU) -# License - -This repository is licensed under the [MIT license](https://github.com/mikeoliphant/NeuralAudio/blob/main/LICENSE). It is a liberal license, but please make sure that you comply with the terms - as well as the terms of [this project's dependencies](https://github.com/mikeoliphant/NeuralAudio/blob/main/CREDITS.md). I would also appreciate it if you would let me know if you are using this library. - -# Underlying Libraries - For WaveNet, the internal implmeentation supports optimized static implemenationas the offical NAM A1 and A2 network architectures: A1 "Standard", "Lite", "Feather", "Nano" and A2 "Lite" and "Full". For A1 LSTM, the internal implementation supports optimized static models architectures for 1x8, 1x12, 1x16, 1x24, 2x8, 2x12, and 2x16 models. @@ -67,7 +65,7 @@ To set a known audio input level (ie: from an audio interface), use ```loader.Se ## Model load behavior -By default, models are loaded using the internal NeuralAudio implementation. If you would like to force the use of the NAM Core or RTNeural implementations, you can use: +By default, models are loaded using the internal NeuralAudio implementation (if possible). If you would like to force the use of the NAM Core or RTNeural implementations, you can use: ``` loader.SetWaveNetLoadMode(loadMode); From 6216e57b5f0ad0e9a5ee5fe8a8c9cc8d5ac4b567 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 6 Jul 2026 08:17:48 -0700 Subject: [PATCH 111/162] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 99fe167..46ef1f3 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ NeuralAudio currently supports the following model types: For WaveNet, the internal implmeentation supports optimized static implemenationas the offical NAM A1 and A2 network architectures: A1 "Standard", "Lite", "Feather", "Nano" and A2 "Lite" and "Full". -For A1 LSTM, the internal implementation supports optimized static models architectures for 1x8, 1x12, 1x16, 1x24, 2x8, 2x12, and 2x16 models. +For LSTM, the internal implementation supports optimized static models architectures for 1x8, 1x12, 1x16, 1x24, 2x8, 2x12, and 2x16 models. All A1 NAM files with WaveNet and LSTM architectures not supported statically will fall back on a less performant dynamic implementation. From e4c9fec4b9fad74dc6452ecd730c115981166bd0 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 8 Jul 2026 17:55:50 -0700 Subject: [PATCH 112/162] Default optimized 8x8 convolution based on compiler/version --- NeuralAudio/CMakeLists.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 6240237..7226e2e 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -51,7 +51,14 @@ else() message(STATUS "NOT building Internal static A2 WaveNet models") endif() -option(ENABLE_MULTIFRAME_8X8_CONVOLUTION "Enable multi-frame 8x8 convolution" OFF) +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "15.0") + option(ENABLE_MULTIFRAME_8X8_CONVOLUTION "Enable multi-frame 8x8 convolution" ON) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") + option(ENABLE_MULTIFRAME_8X8_CONVOLUTION "Enable multi-frame 8x8 convolution" ON) +else() + option(ENABLE_MULTIFRAME_8X8_CONVOLUTION "Enable multi-frame 8x8 convolution" OFF) +endif() + if(ENABLE_MULTIFRAME_8X8_CONVOLUTION) message(STATUS "Using multi-frame 8x8 convolution") add_definitions(-DENABLE_MULTIFRAME_8X8_CONVOLUTION) From 41d0cf6dddfbda55f2b5c7e02b8e0feb9fe58155 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 8 Jul 2026 18:07:01 -0700 Subject: [PATCH 113/162] one more int->size_t --- NeuralAudio/MatMul.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NeuralAudio/MatMul.h b/NeuralAudio/MatMul.h index 433a4ea..6eff44c 100644 --- a/NeuralAudio/MatMul.h +++ b/NeuralAudio/MatMul.h @@ -137,7 +137,7 @@ namespace NeuralAudio { const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; - for (int frame = 0; frame < numFrames; frame++) + for (size_t frame = 0; frame < numFrames; frame++) { const int offset = frame * 3; const float* in = &inData[offset]; From b83c69f07ca3ecbf9c532e0eed3f4127f0e210c4 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 13 Jul 2026 10:08:17 -0700 Subject: [PATCH 114/162] Show xRT ModelTest --- Utils/ModelTest/ModelTest.cpp | 49 ++++++++++++++--------------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/Utils/ModelTest/ModelTest.cpp b/Utils/ModelTest/ModelTest.cpp index 5237e6c..b62345a 100644 --- a/Utils/ModelTest/ModelTest.cpp +++ b/Utils/ModelTest/ModelTest.cpp @@ -56,7 +56,7 @@ NeuralModel* LoadModel(std::filesystem::path modelPath, NeuralModelLoader& loade return nullptr; } -static std::tuple BenchModel(NeuralModel* model, int blockSize, int numBlocks) +static double BenchModel(NeuralModel* model, int blockSize, int numBlocks) { std::vector inData; inData.resize(blockSize); @@ -66,26 +66,16 @@ static std::tuple BenchModel(NeuralModel* model, int blockSize, auto start = std::chrono::high_resolution_clock::now(); - double maxBlock = 0; - for (int block = 0; block < numBlocks; block++) { - auto blockStart = std::chrono::high_resolution_clock::now(); - model->Process(inData.data(), outData.data(), blockSize); - - auto blockEnd = std::chrono::high_resolution_clock::now(); - - maxBlock = std::max(maxBlock, std::chrono::duration_cast> (blockEnd - blockStart).count()); } auto end = std::chrono::high_resolution_clock::now(); double tot = std::chrono::duration_cast> (end - start).count(); - maxBlock *= numBlocks; - - return std::tie(tot, maxBlock); + return tot; } static double ComputeError(NeuralModel* model1, NeuralModel* model2, int blockSize, int numBlocks) @@ -127,6 +117,11 @@ static double ComputeError(NeuralModel* model1, NeuralModel* model2, int blockSi return sqrt(totErr / (double)(blockSize * numBlocks)); } +void PrintBench(std::string name, double time, int dataSize) +{ + std::cout << name << ": " << time << " (" << (((float)dataSize / 48000.0f) / time) << "xRT)" << std::endl; +} + void RunNAMTests(std::filesystem::path modelPath, NeuralModelLoader& loader, int blockSize) { std::cout << "Model: " << modelPath << std::endl; @@ -144,15 +139,15 @@ void RunNAMTests(std::filesystem::path modelPath, NeuralModelLoader& loader, int double rms; - std::tuple internal; - std::tuple rtNeural; - std::tuple namCore; + double internal; + double rtNeural; + double namCore; if (internalModel != nullptr) { internal = BenchModel(internalModel, blockSize, numBlocks); - std::cout << "Internal: " << std::get<0>(internal) << " (" << std::get<1>(internal) << ")" << std::endl; + PrintBench("Internal", internal, dataSize); } else { @@ -161,28 +156,24 @@ void RunNAMTests(std::filesystem::path modelPath, NeuralModelLoader& loader, int if (namCoreModel != nullptr) { - std::cout << std::endl; - namCore = BenchModel(namCoreModel, blockSize, numBlocks); - std::cout << "NAM Core: " << std::get<0>(namCore) << " (" << std::get<1>(namCore) << ")" << std::endl; + PrintBench("NAM Core", namCore, dataSize); if (internalModel != nullptr) { rms = ComputeError(namCoreModel, internalModel, blockSize, numBlocks); std::cout << "NAM vs Internal RMS err: " << rms << std::endl; - std::cout << "Internal is: " << (std::get<0>(namCore) / std::get<0>(internal)) << "x NAM" << std::endl; + std::cout << "Internal is: " << (namCore / internal) << "x NAM" << std::endl; } } if (rtNeuralModel != nullptr) { - std::cout << std::endl; - rtNeural = BenchModel(rtNeuralModel, blockSize, numBlocks); - std::cout << "RTNeural: " << std::get<0>(rtNeural) << " (" << std::get<1>(rtNeural) << ")" << std::endl; + PrintBench("RTNeural", rtNeural, dataSize); if (namCoreModel != nullptr) { @@ -191,7 +182,7 @@ void RunNAMTests(std::filesystem::path modelPath, NeuralModelLoader& loader, int if (namCoreModel != nullptr) { - std::cout << "RTNeural is: " << (std::get<0>(namCore) / std::get<0>(rtNeural)) << "x NAM" << std::endl; + std::cout << "RTNeural is: " << (namCore / rtNeural) << "x NAM" << std::endl; } } } @@ -216,12 +207,12 @@ void RunKerasTests(std::filesystem::path modelPath, NeuralModelLoader& loader, i std::cout << "Internal vs RTNeural RMS err: " << rms << std::endl; std::cout << std::endl; - auto internal = BenchModel(internalModel, blockSize, numBlocks); - auto rt = BenchModel(rtNeuralModel, blockSize, numBlocks); + double internal = BenchModel(internalModel, blockSize, numBlocks); + double rt = BenchModel(rtNeuralModel, blockSize, numBlocks); - std::cout << "RTNeural: " << std::get<0>(rt) << " (" << std::get<1>(rt) << ")" << std::endl; - std::cout << "Internal: " << std::get<0>(internal) << " (" << std::get<1>(internal) << ")" << std::endl; - std::cout << "Internal is: " << (std::get<0>(rt) / std::get<0>(internal)) << "x RTNeural" << std::endl; + PrintBench("Internal", internal, dataSize); + PrintBench("RTNeural", rt, dataSize); + std::cout << "Internal is: " << (rt / internal) << "x RTNeural" << std::endl; std::cout << std::endl; } From b5a714da3cf17865390d80b206388be69909904c Mon Sep 17 00:00:00 2001 From: Alessandro Presta Date: Mon, 13 Jul 2026 15:48:08 -0700 Subject: [PATCH 115/162] Forward audio input level to model NeuralModelLoader::SetAudioInputLevelDBu() is currently a no-op. With this change, it gets forwarded to the model in NeuralModelImpl::SetModelLoader(). --- NeuralAudio/NeuralModelImpl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/NeuralAudio/NeuralModelImpl.h b/NeuralAudio/NeuralModelImpl.h index 8b1b1a2..0983867 100644 --- a/NeuralAudio/NeuralModelImpl.h +++ b/NeuralAudio/NeuralModelImpl.h @@ -12,6 +12,7 @@ namespace NeuralAudio void SetModelLoader(NeuralModelLoader* modelLoader) { this->loader = modelLoader; + audioInputLevelDBu = modelLoader->GetAudioInputLevelDBu(); } bool HadInitialPrewarm() From d6592670124a6b4ca8cbdd247ea6cdc7d7459a65 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 15 Jul 2026 09:05:11 -0700 Subject: [PATCH 116/162] Add ChannelBuffer classes (#62) --- NeuralAudio/Activation.h | 13 +- NeuralAudio/CMakeLists.txt | 1 + NeuralAudio/ChannelBuffer.h | 219 ++++++++++++++++++++++++++++ NeuralAudio/MatMul.h | 132 ++++++++++++++++- NeuralAudio/WaveNet.h | 281 +++++++++++++++++++++--------------- 5 files changed, 509 insertions(+), 137 deletions(-) create mode 100644 NeuralAudio/ChannelBuffer.h diff --git a/NeuralAudio/Activation.h b/NeuralAudio/Activation.h index 0a62efa..bfdd654 100644 --- a/NeuralAudio/Activation.h +++ b/NeuralAudio/Activation.h @@ -63,12 +63,8 @@ namespace NeuralAudio struct FastMath { - template - static void Tanh(Matrix *x) + static void Tanh(float* data, size_t size) { - float* data = x->data(); - size_t size = x->rows() * x->cols(); - for (size_t pos = 0; pos < size; pos++) { data[pos] = Tanh(data[pos]); @@ -97,13 +93,12 @@ namespace NeuralAudio return 0.5f * (Tanh(x * 0.5f) + 1); } - template - static void LeakyReLU(Matrix* x) + static void LeakyReLU(float *data, size_t size) { //*x = (x->array() < 0.0f).select(x->array() * 0.01f, x->array()); - float* data = x->data(); - size_t size = x->rows() * x->cols(); + //float* data = channelData.GetData(); + //size_t size = Channels * channelData.NumCols(); for (size_t pos = 0; pos < size; pos++) { diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 7226e2e..f25486a 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -119,6 +119,7 @@ set(SOURCES RTNeuralLoader.cpp RTNeuralLoader.h Activation.h + ChannelBuffer.h MatMul.h WaveNet.h WaveNetDynamic.h diff --git a/NeuralAudio/ChannelBuffer.h b/NeuralAudio/ChannelBuffer.h new file mode 100644 index 0000000..1d23028 --- /dev/null +++ b/NeuralAudio/ChannelBuffer.h @@ -0,0 +1,219 @@ +#pragma once + +#include +#include +#include + +namespace NeuralAudio +{ + template + class ChannelRowSpan; + + template + class ChannelBufferBase + { + public: + virtual size_t GetNumCols() const { return 0; } + size_t GetNumChannels() const { return Channels; } + virtual T* GetData() { return nullptr; } + virtual T* GetData(size_t startCol) + { + (void)startCol; + + return nullptr; + } + virtual const T* GetDataConst() const { return nullptr; } + virtual const T* GetDataConst(size_t startCol) const + { + (void)startCol; + + return nullptr; + } + virtual void SetZero() {} + }; + + template + class ChannelBuffer : public ChannelBufferBase + { + public: + ChannelBuffer() + { + } + + size_t GetNumCols() const override + { + return Cols; + } + + T* GetData() override + { + return data[0].data(); + } + + const T* GetDataConst() const override + { + return data[0].data(); + } + + T* GetData(size_t startCol) override + { + return data[startCol].data(); + } + + const T* GetDataConst(size_t startCol) const override + { + return data[startCol].data(); + } + + void SetZero() override + { + for (auto& col : data) { + col.fill(0); + } + } + + T& operator()(size_t row, size_t col) + { + return data[col][row]; + } + + const T& operator()(size_t row, size_t col) const + { + return data[col][row]; + } + + const ChannelRowSpan Slice(size_t startCol, size_t numCols) + { + return ChannelRowSpan(this, startCol, numCols); + } + + const ChannelRowSpan Slice(size_t numCols) + { + return ChannelRowSpan(this, numCols); + } + + const Eigen::Map> GetEigenMap() + { + return Eigen::Map>(GetData(), Channels, GetNumCols()); + } + + const Eigen::Map> GetEigenMapConst() const + { + return Eigen::Map>(GetDataConst(), Channels, GetNumCols()); + } + + private: + alignas(32) std::array, Cols> data; + }; + + template + class ChannelRowSpan + { + public: + ChannelRowSpan(ChannelBufferBase& baseBuffer) : + buffer(&baseBuffer), + startCol(0), + numCols(baseBuffer.GetNumCols()) + { + + } + + ChannelRowSpan(ChannelBufferBase* baseBuffer) : + buffer(baseBuffer), + startCol(0), + numCols(baseBuffer->GetNumCols()) + { + + } + + ChannelRowSpan(ChannelBufferBase* baseBuffer, size_t startCol, size_t numCols) : + buffer(baseBuffer), + startCol(startCol), + numCols(numCols) + { + + } + + ChannelRowSpan(ChannelBufferBase* baseBuffer, size_t numCols) : + buffer(baseBuffer), + startCol(0), + numCols(numCols) + { + + } + + const ChannelRowSpan Slice(size_t startCol, size_t numCols) const + { + return ChannelRowSpan(buffer, this->startCol + startCol, numCols); + } + + const ChannelRowSpan Slice(size_t numCols) const + { + return ChannelRowSpan(buffer, this->startCol, numCols); + } + + size_t GetNumCols() const + { + return numCols; + } + + size_t GetNumChannels() const + { + return Channels; + } + + T* GetData() const + { + return buffer->GetData(startCol); + } + + const T* GetDataConst() const + { + return buffer->GetDataConst(startCol); + } + + T* GetData(size_t startCol) const + { + return buffer->GetData(this->startCol + startCol); + } + + const T* GetDataConst(size_t startCol) const + { + return buffer->GetDataConst(this->startCol + startCol); + } + + Eigen::Map> GetEigenMap() const + { + return Eigen::Map>(GetData(), Channels, numCols); + } + + const Eigen::Map> GetEigenMapConst() const + { + return Eigen::Map>(GetDataConst(), Channels, numCols); + } + + void CopyData(const ChannelRowSpan& srcSpan) const + { + const T* srcPtr = srcSpan.GetDataConst(); + T* destPtr = GetData(); + + memmove(destPtr, srcPtr, Channels * numCols * sizeof(T)); + } + + void AddData(const ChannelRowSpan& srcSpan) const + { + const T* srcPtr = srcSpan.GetDataConst(); + T* destPtr = GetData(); + + for (size_t count = 0; count < Channels * numCols; count++) + { + *destPtr++ += *srcPtr++; + } + } + + private: + ChannelBufferBase* const buffer; + const size_t startCol; + const size_t numCols; + }; +} \ No newline at end of file diff --git a/NeuralAudio/MatMul.h b/NeuralAudio/MatMul.h index 6eff44c..b87a334 100644 --- a/NeuralAudio/MatMul.h +++ b/NeuralAudio/MatMul.h @@ -5,7 +5,9 @@ namespace NeuralAudio template struct MatMul { - static constexpr bool HasKernel() { return ((InChannels == 3) && ((OutChannels == 3) || (OutChannels == 1))); } + #define IsKernel(In, Out) ((InChannels == In) && (OutChannels == Out)) + + static constexpr bool HasKernel() { return (IsKernel(3, 3) || IsKernel(8, 1) || IsKernel(3, 1) || IsKernel(1, 3)); } // 3x3 implementation inspired by @jfsantos NAM Core a2fast - https://github.com/sdatkinson/NeuralAmpModelerCore/blob/main/NAM/wavenet/a2_fast.cpp @@ -13,7 +15,7 @@ namespace NeuralAudio { static_assert(HasKernel(), "Multiplication not implemented for InChannel/OutChannel combination"); - if constexpr ((InChannels == 3) && (OutChannels == 3)) + if constexpr (IsKernel(3, 3)) { const float w00 = weights[0], w10 = weights[1], w20 = weights[2]; const float w01 = weights[3], w11 = weights[4], w21 = weights[5]; @@ -39,7 +41,28 @@ namespace NeuralAudio out[2] = a2; } } - else if constexpr ((InChannels == 3) && (OutChannels == 1)) + else if constexpr (IsKernel(8, 1)) + { + const float w0 = weights[0], w1 = weights[1], w2 = weights[2], w3 = weights[3]; + const float w4 = weights[4], w5 = weights[5], w6 = weights[6], w7 = weights[7]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * InChannels; + const float* in = &inData[offset]; + + float a0 = w0 * in[0]; + a0 += w1 * in[1]; + a0 += w2 * in[2]; + a0 += w3 * in[3]; + a0 += w4 * in[4]; + a0 += w5 * in[5]; + a0 += w6 * in[6]; + a0 += w7 * in[7]; + outData[frame] = a0; + } + } + else if constexpr (IsKernel(3, 1)) { const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; @@ -48,19 +71,35 @@ namespace NeuralAudio const size_t offset = frame * 3; const float* in = &inData[offset]; - float a0 = w0 * in[frame * 3]; + float a0 = w0 * in[0]; a0 += w1 * in[1]; a0 += w2 * in[2]; outData[frame] = a0; } } + else if constexpr (IsKernel(1, 3)) + { + const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + float* out = &outData[offset]; + + const float in = inData[frame]; + + out[0] = w0 * in; + out[1] = w1 * in; + out[2] = w2 * in; + } + } } static inline void MultiplyInitColwise(const float* inData, float* outData, const float* weights, const float* initData, size_t numFrames) { static_assert(HasKernel(), "Multiplication not implemented for InChannel/OutChannel combination"); - if constexpr ((InChannels == 3) && (OutChannels == 3)) + if constexpr (IsKernel(3, 3)) { const float w00 = weights[0], w10 = weights[1], w20 = weights[2]; const float w01 = weights[3], w11 = weights[4], w21 = weights[5]; @@ -86,7 +125,30 @@ namespace NeuralAudio out[2] = a2; } } - else if constexpr ((InChannels == 3) && (OutChannels == 1)) + else if constexpr (IsKernel(8, 1)) + { + const float w0 = weights[0], w1 = weights[1], w2 = weights[2], w3 = weights[3]; + const float w4 = weights[4], w5 = weights[5], w6 = weights[6], w7 = weights[7]; + + const float init = initData[0]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * InChannels; + const float* in = &inData[offset]; + + float a0 = init + w0 * in[0]; + a0 += w1 * in[1]; + a0 += w2 * in[2]; + a0 += w3 * in[3]; + a0 += w4 * in[4]; + a0 += w5 * in[5]; + a0 += w6 * in[6]; + a0 += w7 * in[7]; + outData[frame] = a0; + } + } + else if constexpr (IsKernel(3, 1)) { const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; @@ -101,13 +163,30 @@ namespace NeuralAudio outData[frame] = a0; } } + else if constexpr (IsKernel(1, 3)) + { + const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; + const float init0 = inData[0], init1 = inData[1], init2 = inData[2]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + float* out = &outData[offset]; + + const float in = inData[frame]; + + out[0] = init0 + (w0 * in); + out[1] = init1 + (w1 * in); + out[2] = init2 + (w2 * in); + } + } } static inline void MultiplyAccumlulate(const float* inData, float* outData, const float* weights, size_t numFrames) { static_assert(HasKernel(), "Multiplication not implemented for InChannel/OutChannel combination"); - if constexpr ((InChannels == 3) && (OutChannels == 3)) + if constexpr (IsKernel(3, 3)) { const float w00 = weights[0], w10 = weights[1], w20 = weights[2]; const float w01 = weights[3], w11 = weights[4], w21 = weights[5]; @@ -133,7 +212,28 @@ namespace NeuralAudio out[2] = a2; } } - else if constexpr ((InChannels == 3) && (OutChannels == 1)) + else if constexpr (IsKernel(8, 1)) + { + const float w0 = weights[0], w1 = weights[1], w2 = weights[2], w3 = weights[3]; + const float w4 = weights[4], w5 = weights[5], w6 = weights[6], w7 = weights[7]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * InChannels; + const float* in = &inData[offset]; + + float a0 = outData[frame] + w0 * in[0]; + a0 += w1 * in[1]; + a0 += w2 * in[2]; + a0 += w3 * in[3]; + a0 += w4 * in[4]; + a0 += w5 * in[5]; + a0 += w6 * in[6]; + a0 += w7 * in[7]; + outData[frame] = a0; + } + } + else if constexpr (IsKernel(3, 1)) { const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; @@ -148,6 +248,22 @@ namespace NeuralAudio outData[frame] = a0; } } + else if constexpr (IsKernel(1, 3)) + { + const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + float* out = &outData[offset]; + + const float in = inData[frame]; + + out[0] += w0 * in; + out[1] += w1 * in; + out[2] += w2 * in; + } + } } }; diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 3d3db93..3cdbada 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -3,10 +3,12 @@ // Based on WaveNet model structure from https://github.com/sdatkinson/NeuralAmpModelerCore // with some template ideas from https://github.com/jatinchowdhury18/RTNeural-NAM +#include #include #include #include "TemplateHelper.h" #include "Activation.h" +#include "ChannelBuffer.h" #include "MatMul.h" #ifndef WAVENET_MAX_NUM_FRAMES @@ -26,30 +28,17 @@ enum EActivationType namespace NeuralAudio { template - class ChannelBuffer + class ChannelHistoryBuffer { public: static constexpr auto BufferSize = ReceptiveFieldSize + ((LAYER_ARRAY_BUFFER_PADDING + 1) * WAVENET_MAX_NUM_FRAMES); - static constexpr bool TooBigForStatic = ((Channels * BufferSize) * 4) > EIGEN_STACK_ALLOCATION_LIMIT; - using ChannelBufferType = typename std::conditional, - Eigen::Matrix>::type; - - ChannelBufferType buffer; - //Eigen::Matrix layerBuffer; + ChannelBuffer buffer; size_t bufferStart; void AllocBuffer(int allocNum) { - long size = BufferSize; - - if constexpr (TooBigForStatic) - { - buffer.resize(Channels, size); - } - - buffer.setZero(); + buffer.SetZero(); //if (offset > (size - (ReceptiveFieldSize + WAVENET_MAX_NUM_FRAMES))) //{ @@ -63,7 +52,7 @@ namespace NeuralAudio #if (LAYER_ARRAY_BUFFER_PADDING == 0) bufferStart = ReceptiveFieldSize; #else - bufferStart = size - (WAVENET_MAX_NUM_FRAMES * ((allocNum % LAYER_ARRAY_BUFFER_PADDING) + 1)); // Do the modulo to handle cases where LAYER_ARRAY_BUFFER_PADDING is not big enough to handle offset + bufferStart = BufferSize - (WAVENET_MAX_NUM_FRAMES * ((allocNum % LAYER_ARRAY_BUFFER_PADDING) + 1)); // Do the modulo to handle cases where LAYER_ARRAY_BUFFER_PADDING is not big enough to handle offset #endif } @@ -71,34 +60,36 @@ namespace NeuralAudio { bufferStart += numFrames; - if ((bufferStart + WAVENET_MAX_NUM_FRAMES) > (size_t)buffer.cols()) + if ((bufferStart + WAVENET_MAX_NUM_FRAMES) > (size_t)buffer.GetNumCols()) RewindBuffer(); } void RewindBuffer() { - size_t start = ReceptiveFieldSize; - - buffer.middleCols(start - ReceptiveFieldSize, ReceptiveFieldSize) = buffer.middleCols(bufferStart - ReceptiveFieldSize, ReceptiveFieldSize); + buffer.Slice(0, ReceptiveFieldSize).CopyData(buffer.Slice(bufferStart - ReceptiveFieldSize, ReceptiveFieldSize)); - bufferStart = start; + bufferStart = ReceptiveFieldSize; } void CopyBuffer() { + auto slice = buffer.Slice(bufferStart, 1); + for (size_t offset = 1; offset < ReceptiveFieldSize + 1; offset++) { - buffer.col(bufferStart - offset) = buffer.col(bufferStart); + buffer.Slice(bufferStart - offset, 1).CopyData(slice); } } }; + struct Empty {}; + template class Conv1DT { public: static constexpr auto ReceptiveFieldSize = (KernelSize - 1) * Dilation; - ChannelBuffer channelBuffer; + ChannelHistoryBuffer channelBuffer; size_t GetNumWeights() { @@ -107,44 +98,51 @@ namespace NeuralAudio void SetWeights(std::vector::iterator& inWeights) { - weights.resize(KernelSize); - for (size_t i = 0; i < OutChannels; i++) for (size_t j = 0; j < InChannels; j++) for (size_t k = 0; k < KernelSize; k++) weights[k](i, j) = *(inWeights++); - if (DoBias) + if constexpr (DoBias) { for (size_t i = 0; i < OutChannels; i++) bias(i) = *(inWeights++); } } - template - inline void Process(Eigen::MatrixBase const & output, const size_t numFrames) const + Conv1DT() + { + for (int k = 0; k < KernelSize; k++) + weightPtrs[k] = weights[k].GetData(); + } + + auto GetInputBuffer(size_t numFrames) + { + return channelBuffer.buffer.Slice(channelBuffer.bufferStart, numFrames); + } + + inline void Process(const ChannelRowSpan& output) { - float* outputPtr = &const_cast&>(output)(0, 0); - const float* biasPtr = bias.data(); + const size_t numFrames = output.GetNumCols(); + float* outputPtr = output.GetData(); #if ENABLE_MULTIFRAME_8X8_CONVOLUTION - if constexpr ((InChannels == 8) && (OutChannels == 8)) + if constexpr ((InChannels == 8) && (OutChannels == 8) && DoBias) { // Based on @jfsantos NAM Core implementation - https://github.com/sdatkinson/NeuralAmpModelerCore/pull/277 - constexpr size_t T = 4; - const size_t nF4 = (numFrames / T) * T; + constexpr size_t tileSize = 4; + const size_t nF4 = (numFrames / tileSize) * tileSize; - for (size_t f = 0; f < nF4; f += T) + for (size_t f = 0; f < nF4; f += tileSize) { - float a[T][InChannels]{}; + float a[tileSize][InChannels]{}; for (size_t k = 0; k < KernelSize; k++) { - const float* W = this->weights[k].data(); + const float* W = weightPtrs[k]; const auto offset = Dilation * (k + 1 - KernelSize); - const auto inBlock = channelBuffer.buffer.middleCols(channelBuffer.bufferStart + offset + f, T); - const float* hb = inBlock.data(); + const float* hb = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset + f); for (size_t cp = 0; cp < InChannels; cp++) { @@ -161,11 +159,11 @@ namespace NeuralAudio } } - for (size_t ti = 0; ti < T; ti++) + for (size_t ti = 0; ti < tileSize; ti++) std::memcpy(outputPtr + static_cast(f + ti) * InChannels, a[ti], InChannels * sizeof(float)); } - // Scalar tail for any frames past the T-aligned boundary. + // Scalar tail for any frames past the tile-aligned boundary. for (size_t f = nF4; f < numFrames; f++) { float* zf = outputPtr + static_cast(f) * InChannels; @@ -175,10 +173,9 @@ namespace NeuralAudio for (size_t k = 0; k < KernelSize; k++) { - const float* W = this->weights[k].data(); + const float* W = weightPtrs[k]; const auto offset = Dilation * (k + 1 - KernelSize); - const auto inBlock = channelBuffer.buffer.middleCols(channelBuffer.bufferStart + offset + f, T); - const float* h = inBlock.data(); + const float* h = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset + f); for (int cp = 0; cp < InChannels; cp++) { @@ -194,20 +191,33 @@ namespace NeuralAudio else #endif { + const float* biasPtr = nullptr; + + if constexpr (DoBias) + { + biasPtr = bias.data(); + } + for (size_t k = 0; k < KernelSize; k++) { - const float* weightPtr = this->weights[k].data(); + const float* weightPtr = this->weights[k].GetDataConst(); const auto offset = Dilation * ((int)k + 1 - KernelSize); - const auto inBlock = channelBuffer.buffer.middleCols(channelBuffer.bufferStart + offset, numFrames); - const float* inputPtr = inBlock.data(); - - if constexpr (MatMul::HasKernel()) + if constexpr (DoBias && MatMul::HasKernel()) { + const float* inputPtr = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset); + if (k == 0) // Maybe move this out of loop? { - MatMul::MultiplyInitColwise(inputPtr, outputPtr, weightPtr, biasPtr, numFrames); + if constexpr (DoBias) + { + MatMul::MultiplyInitColwise(inputPtr, outputPtr, weightPtr, biasPtr, numFrames); + } + else + { + MatMul::MultiplyInitZero(inputPtr, outputPtr, weightPtr, numFrames); + } } else { @@ -216,21 +226,30 @@ namespace NeuralAudio } else { + const auto inBlock = channelBuffer.buffer.Slice(channelBuffer.bufferStart + offset, numFrames); + if (k == 0) - const_cast&>(output).noalias() = weights[k] * inBlock; + output.GetEigenMap().noalias() = weights[k].GetEigenMapConst() * inBlock.GetEigenMapConst(); else - const_cast&>(output).noalias() += weights[k] * inBlock; + output.GetEigenMap().noalias() += weights[k].GetEigenMapConst() * inBlock.GetEigenMapConst(); } } } if constexpr (DoBias && !MatMul::HasKernel()) - const_cast&>(output).colwise() += bias; + output.GetEigenMap().colwise() += bias; } private: - std::vector> weights; - Eigen::Vector bias; + alignas(32) std::array, KernelSize> weights; // consider making this a contiguous block of data instead of block of ChannelBuffers + std::array weightPtrs; + + // Avoid allocation for unused bias + using BiasType = typename std::conditional, + Empty>::type; + + BiasType bias; }; template @@ -255,56 +274,64 @@ namespace NeuralAudio } } - template - void Process(const Eigen::MatrixBase& input, Eigen::MatrixBase const& output) const + void Process(const ChannelRowSpan& input, const ChannelRowSpan& output) const { + size_t numFrames = output.GetNumCols(); + if constexpr (MatMul::HasKernel()) { if constexpr (DoBias) { - MatMul::MultiplyInitColwise(&input(0, 0), &const_cast&>(output)(0, 0), weights.data(), bias.data(), output.cols()); + MatMul::MultiplyInitColwise(input.GetDataConst(), output.GetData(), weights.GetDataConst(), bias.data(), numFrames); } else { - MatMul::MultiplyInitZero(&input(0, 0), &const_cast&>(output)(0, 0), weights.data(), output.cols()); + MatMul::MultiplyInitZero(input.GetDataConst(), output.GetData(), weights.GetDataConst(), numFrames); } } else { if constexpr (DoBias) { - const_cast&>(output).noalias() = (weights * input).colwise() + bias; + output.GetEigenMap().noalias() = (weights.GetEigenMapConst() * input.GetEigenMapConst()).colwise() + bias; } else { - const_cast&>(output).noalias() = weights * input; + output.GetEigenMap().noalias() = weights.GetEigenMapConst() * input.GetEigenMapConst(); } } } - template - void ProcessAcc(const Eigen::MatrixBase& input, Eigen::MatrixBase const& output) const + void ProcessAcc(const ChannelRowSpan& input, const ChannelRowSpan& output) const { + size_t numFrames = output.GetNumCols(); + if constexpr (!DoBias && MatMul::HasKernel()) { - MatMul::MultiplyAccumlulate(&input(0, 0), &const_cast&>(output)(0, 0), weights.data(), output.cols()); + MatMul::MultiplyAccumlulate(input.GetDataConst(), output.GetData(), weights.GetDataConst(), numFrames); } else { if constexpr (DoBias) { - const_cast&>(output).noalias() += (weights * input).colwise() + bias; + output.GetEigenMap().noalias() += (weights.GetEigenMapConst() * input.GetEigenMapConst()) + bias; } else { - const_cast&>(output).noalias() += weights * input; + output.GetEigenMap().noalias() += weights.GetEigenMapConst() * input.GetEigenMapConst(); } } } private: - Eigen::Matrix weights; - Eigen::Vector bias; + ChannelBuffer weights; + + // Avoid allocation for unused bias + using BiasType = typename std::conditional, + Empty>::type; + + BiasType bias; }; template @@ -314,14 +341,14 @@ namespace NeuralAudio Conv1DT conv1D; DenseLayerT inputMixin; DenseLayerT oneByOne; - Eigen::Matrix state; + ChannelBuffer state; public: static constexpr auto ReceptiveFieldSize = (KernelSize - 1) * Dilation; WaveNetLayerT() { - state.setZero(); + state.SetZero(); } void AllocBuffer(int allocNum) @@ -339,9 +366,14 @@ namespace NeuralAudio return conv1D.GetNumWeights() + inputMixin.GetNumWeights() + oneByOne.GetNumWeights(); } - auto& GetInputBuffer() + auto GetInputBuffer(size_t numFrames) { - return conv1D.channelBuffer; + return conv1D.channelBuffer.buffer.Slice(conv1D.channelBuffer.bufferStart, numFrames); + } + + void CopyBuffer() + { + conv1D.channelBuffer.CopyBuffer(); } void SetWeights(std::vector::iterator& weights) @@ -351,29 +383,34 @@ namespace NeuralAudio oneByOne.SetWeights(weights); } - template - void Process(const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInput, Eigen::MatrixBase const& output, const size_t outputStart, const size_t numFrames) + void Process(const ChannelRowSpan& condition, const ChannelRowSpan& headInput, const ChannelRowSpan& output) { - auto block = state.leftCols(numFrames); + size_t numFrames = output.GetNumCols(); - conv1D.Process(block, numFrames); + auto block = state.Slice(numFrames); + + conv1D.Process(block); inputMixin.ProcessAcc(condition, block); if constexpr (Activation == EActivationType::Tanh) { - WAVENET_MATH::Tanh(&block); + WAVENET_MATH::Tanh(block.GetData(), block.GetNumChannels() * block.GetNumCols()); } else if constexpr (Activation == EActivationType::LeakyReLU) { - WAVENET_MATH::LeakyReLU(&block); + WAVENET_MATH::LeakyReLU(block.GetData(), block.GetNumChannels() * block.GetNumCols()); } - const_cast&>(headInput).leftCols(numFrames).noalias() += block; + headInput.GetEigenMap().noalias() += block.GetEigenMapConst(); + + //headInput.AddData(block); + + oneByOne.Process(block, output); - oneByOne.Process(block, const_cast&>(output).middleCols(outputStart, numFrames)); + output.GetEigenMap().noalias() += conv1D.GetInputBuffer(numFrames).GetEigenMapConst(); - const_cast&>(output).middleCols(outputStart, numFrames).noalias() += conv1D.channelBuffer.buffer.middleCols(conv1D.channelBuffer.bufferStart, numFrames); + //output.AddData(conv1D.GetInputBuffer(numFrames)); } }; @@ -411,8 +448,8 @@ namespace NeuralAudio static constexpr auto NumChannelsP = Channels; static constexpr auto HeadSizeP = HeadSize; - Eigen::Matrix arrayOutputs; - Eigen::Matrix headOutputs; + ChannelBuffer arrayOutputs; + ChannelBuffer headOutputs; int ReceptiveFieldSize = 0; // This should be a static constexpr, but I haven't sorted out the right template magic WaveNetLayerArrayT() @@ -463,51 +500,58 @@ namespace NeuralAudio headRechannel.SetWeights(weights); } - template - void Prewarm(const Eigen::MatrixBase& layerInputs, const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInputs) + void Prewarm(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) { - rechannel.Process(layerInputs.leftCols(1), std::get<0>(layers).GetInputBuffer().buffer.middleCols(std::get<0>(layers).GetInputBuffer().bufferStart, 1)); + rechannel.Process(layerInputs, std::get<0>(layers).GetInputBuffer(1)); ForEachIndex([&](auto layerIndex) { - std::get(layers).GetInputBuffer().CopyBuffer(); + std::get(layers).CopyBuffer(); if constexpr (layerIndex == lastLayer) { - std::get(layers).Process(condition, headInputs, arrayOutputs, 0, 1); + std::get(layers).Process(condition, headInputs, arrayOutputs.Slice(1)); } else { - std::get(layers).Process(condition, headInputs, std::get(layers).GetInputBuffer().buffer, std::get(layers).GetInputBuffer().bufferStart, 1); + std::get(layers).Process(condition, headInputs, std::get(layers).GetInputBuffer(1)); } }); - headRechannel.channelBuffer.buffer.middleCols(headRechannel.channelBuffer.bufferStart, 1).noalias() = headInputs.leftCols(1); // Should be able to avoid this copy + //headRechannel.channelBuffer.buffer.middleCols(headRechannel.channelBuffer.bufferStart, 1).noalias() = headInputs.leftCols(1); // Should be able to avoid this copy + + headRechannel.GetInputBuffer(1).CopyData(headInputs.Slice(1)); headRechannel.channelBuffer.CopyBuffer(); - headRechannel.Process(headOutputs.leftCols(1), 1); + headRechannel.Process(headOutputs.Slice(1)); } - template - void Process(const Eigen::MatrixBase& layerInputs, const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInputs, const size_t numFrames) + void Process(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) { - rechannel.Process(layerInputs.leftCols(numFrames), std::get<0>(layers).GetInputBuffer().buffer.middleCols(std::get<0>(layers).GetInputBuffer().bufferStart, numFrames)); + size_t numFrames = condition.GetNumCols(); + + rechannel.Process(layerInputs, std::get<0>(layers).GetInputBuffer(numFrames)); ForEachIndex([&](auto layerIndex) { if constexpr (layerIndex == lastLayer) { - std::get(layers).Process(condition, headInputs, arrayOutputs, 0, numFrames); + std::get(layers).Process(condition, headInputs, arrayOutputs.Slice(numFrames)); } else { - std::get(layers).Process(condition, headInputs, std::get(layers).GetInputBuffer().buffer, std::get(layers).GetInputBuffer().bufferStart, numFrames); + std::get(layers).Process(condition, headInputs, std::get(layers).GetInputBuffer(numFrames)); } std::get(layers).AdvanceFrames(numFrames); }); - headRechannel.channelBuffer.buffer.middleCols(headRechannel.channelBuffer.bufferStart, numFrames).noalias() = headInputs.leftCols(numFrames); // Should be able to avoid this copy - headRechannel.Process(headOutputs.leftCols(numFrames), numFrames); + + //headRechannel.channelBuffer.buffer.middleCols(headRechannel.channelBuffer.bufferStart, numFrames).noalias() = headInputs.leftCols(numFrames); // Should be able to avoid this copy + + //headRechannel.GetInputBuffer(numFrames).CopyData(headInputs); + + headRechannel.GetInputBuffer(numFrames).GetEigenMap().noalias() = headInputs.GetEigenMapConst(); + headRechannel.Process(headOutputs.Slice(numFrames)); headRechannel.channelBuffer.AdvanceFrames(numFrames); } }; @@ -572,64 +616,61 @@ namespace NeuralAudio void Prewarm() { - float input = 0; + condition.GetData()[0] = 0; - auto condition = Eigen::Map>(&input, 1, 1); + headArray.SetZero(); - headArray.setZero(); + auto conditionSpan = condition.Slice(1); + auto headArraySpan = headArray.Slice(1); ForEachIndex([&](auto layerIndex) { if constexpr (layerIndex == 0) { - std::get(layerArrays).Prewarm(condition, condition, headArray); + std::get(layerArrays).Prewarm(conditionSpan, conditionSpan, headArraySpan); } else { - std::get(layerArrays).Prewarm(std::get(layerArrays).arrayOutputs, condition, std::get(layerArrays).headOutputs); + std::get(layerArrays).Prewarm(std::get(layerArrays).arrayOutputs.Slice(1), conditionSpan, std::get(layerArrays).headOutputs.Slice(1)); } }); } void Process(const float* input, float* output, const size_t numFrames) { - //numRewinds = 0; + std::memcpy(condition.GetData(), input, numFrames * sizeof(float)); - auto condition = Eigen::Map>(input, 1, numFrames); + headArray.SetZero(); - headArray.setZero(); + auto conditionSpan = condition.Slice(numFrames); + auto headArraySpan = headArray.Slice(numFrames); ForEachIndex([&](auto layerIndex) { if constexpr (layerIndex == 0) { - std::get(layerArrays).Process(condition, condition, headArray, numFrames); + std::get(layerArrays).Process(conditionSpan, conditionSpan, headArraySpan); } else { - std::get(layerArrays).Process(std::get(layerArrays).arrayOutputs, condition, std::get(layerArrays).headOutputs, numFrames); + std::get(layerArrays).Process(std::get(layerArrays).arrayOutputs.Slice(numFrames), conditionSpan, std::get(layerArrays).headOutputs.Slice(numFrames)); } }); - const auto finalHeadArray = std::get(layerArrays).headOutputs; - - auto out = Eigen::Map>(output, 1, numFrames); + const float* finalHeadArray = std::get(layerArrays).headOutputs.GetData(); - out.noalias() = headScale * finalHeadArray.leftCols(numFrames); - - //if (numRewinds > maxRewinds) - //{ - // maxRewinds = numRewinds; - - // std::cout << "New Max Rewinds: " << maxRewinds << std::endl; - //} + for (size_t i = 0; i < numFrames; i++) + { + output[i] = headScale * finalHeadArray[i]; + } } private: static constexpr auto headLayerChannels = std::tuple_element_t<0, std::tuple>::NumChannelsP; std::tuple layerArrays; - Eigen::Matrix headArray; + ChannelBuffer condition; + ChannelBuffer headArray; float headScale; }; } From 16e785873d3ba89cd5756c7f2020e740d4812347 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 15 Jul 2026 09:15:34 -0700 Subject: [PATCH 117/162] Add -O3 to rpi build --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1fc471c..40cd1f3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -93,7 +93,7 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build env: - CXXFLAGS: "-static-libstdc++ -static-libgcc" + CXXFLAGS: "-Wall -O3 -static-libstdc++ -static-libgcc" run: | cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=ON -DCMAKE_TOOLCHAIN_FILE=/home/develop/opt/x-tools/aarch64-rpi3-linux-gnu/aarch64-${{ matrix.native_arch }}-linux-gnu.toolchain.cmake cmake --build . --config $BUILD_TYPE -j4 From 2fd2a744696430b97745211ba95d8a8a33cc46ef Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 15 Jul 2026 09:22:17 -0700 Subject: [PATCH 118/162] Add -ffast-math for gcc --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40cd1f3..161d362 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,7 +51,7 @@ jobs: working-directory: ${{github.workspace}}/build env: CXX: g++-15 - CXXFLAGS: "-Wall -O3 -march=${{ matrix.native_arch }} -static-libstdc++ -static-libgcc" + CXXFLAGS: "-Wall -O3 -ffast-math -march=${{ matrix.native_arch }} -static-libstdc++ -static-libgcc" run: | cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=ON cmake --build . --config $BUILD_TYPE -j4 @@ -93,7 +93,7 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build env: - CXXFLAGS: "-Wall -O3 -static-libstdc++ -static-libgcc" + CXXFLAGS: "-Wall -O3 -ffast-math -static-libstdc++ -static-libgcc" run: | cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=ON -DCMAKE_TOOLCHAIN_FILE=/home/develop/opt/x-tools/aarch64-rpi3-linux-gnu/aarch64-${{ matrix.native_arch }}-linux-gnu.toolchain.cmake cmake --build . --config $BUILD_TYPE -j4 From 7ac6a397b22c70d8039787c3053996170ec7afb4 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 15 Jul 2026 10:13:08 -0700 Subject: [PATCH 119/162] Fix build when not using NAM Core --- NeuralAudio/NeuralModel.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 41e8f24..2afc3fb 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -360,9 +360,9 @@ namespace NeuralAudio if (newModel == nullptr) { -#ifdef BUILD_NAMCORE std::string version = modelJson.at("version"); +#ifdef BUILD_NAMCORE bool loadA2WithNAMCore = true; #ifdef BUILD_STATIC_INTERNAL_NAMA2 @@ -377,8 +377,8 @@ namespace NeuralAudio newModel = model; } - } #endif + } auto& config = modelJson.at("config"); From 280c91d5093c3c6f795d3db1bd61a2ffc46320cf Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 15 Jul 2026 11:45:37 -0700 Subject: [PATCH 120/162] int -> size_t --- NeuralAudio/MatMul.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NeuralAudio/MatMul.h b/NeuralAudio/MatMul.h index b87a334..66dd748 100644 --- a/NeuralAudio/MatMul.h +++ b/NeuralAudio/MatMul.h @@ -2,7 +2,7 @@ namespace NeuralAudio { - template + template struct MatMul { #define IsKernel(In, Out) ((InChannels == In) && (OutChannels == Out)) @@ -239,7 +239,7 @@ namespace NeuralAudio for (size_t frame = 0; frame < numFrames; frame++) { - const int offset = frame * 3; + const size_t offset = frame * InChannels; const float* in = &inData[offset]; float a0 = outData[frame] + w0 * in[0]; @@ -254,7 +254,7 @@ namespace NeuralAudio for (size_t frame = 0; frame < numFrames; frame++) { - const size_t offset = frame * 3; + const size_t offset = frame * InChannels; float* out = &outData[offset]; const float in = inData[frame]; From 3526fcb0a9af04dc099f6228014d2b0081995e08 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Wed, 15 Jul 2026 16:46:44 -0700 Subject: [PATCH 121/162] Better compiler hints for vectorization of 8x8 conv --- NeuralAudio/WaveNet.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 3cdbada..d116bf1 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -124,7 +124,7 @@ namespace NeuralAudio inline void Process(const ChannelRowSpan& output) { const size_t numFrames = output.GetNumCols(); - float* outputPtr = output.GetData(); + float* __restrict outputPtr = output.GetData(); #if ENABLE_MULTIFRAME_8X8_CONVOLUTION if constexpr ((InChannels == 8) && (OutChannels == 8) && DoBias) @@ -136,17 +136,17 @@ namespace NeuralAudio for (size_t f = 0; f < nF4; f += tileSize) { - float a[tileSize][InChannels]{}; + alignas(32) float a[tileSize][InChannels]{}; for (size_t k = 0; k < KernelSize; k++) { - const float* W = weightPtrs[k]; + const float* __restrict W = weightPtrs[k]; const auto offset = Dilation * (k + 1 - KernelSize); - const float* hb = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset + f); + const float* __restrict hb = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset + f); for (size_t cp = 0; cp < InChannels; cp++) { - const float* Wcol = W + cp * InChannels; + const float* __restrict Wcol = W + cp * InChannels; const float h0 = hb[cp], h1 = hb[InChannels + cp], h2 = hb[2 * InChannels + cp], h3 = hb[3 * InChannels + cp]; for (size_t o = 0; o < InChannels; o++) From c16296a7978a0e72e32dbf630305dd24e77808ed Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 16 Jul 2026 15:23:50 -0700 Subject: [PATCH 122/162] Compiler vectorization tweaks (#63) --- .github/workflows/release.yml | 18 +++++++++--------- NeuralAudio/MatMul.h | 6 +++--- NeuralAudio/WaveNet.h | 9 +++++---- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 161d362..3a44e66 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,18 +42,18 @@ jobs: with: submodules: recursive - - name: Setup GCC 15 - uses: egor-tensin/setup-gcc@v2 + - name: Setup Clang 21 + uses: egor-tensin/setup-clang@v2 with: - version: '15' + version: '22' - name: Build working-directory: ${{github.workspace}}/build env: - CXX: g++-15 - CXXFLAGS: "-Wall -O3 -ffast-math -march=${{ matrix.native_arch }} -static-libstdc++ -static-libgcc" + CXX: clang++ + CXXFLAGS: "-Wall -O3 -march=${{ matrix.native_arch }} -static-libstdc++ -static-libgcc" run: | - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=ON + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=${{ matrix.native_arch == 'x86-64-v3' && 'ON' || 'OFF' }} cmake --build . --config $BUILD_TYPE -j4 - name: Add Archive @@ -82,7 +82,7 @@ jobs: native_arch: [rpi4, rpi5] runs-on: ubuntu-latest container: - image: ghcr.io/tttapa/docker-arm-cross-toolchain:aarch64-rpi3-linux-gnu-gcc15 + image: ghcr.io/tttapa/docker-arm-cross-toolchain:aarch64-rpi3-linux-gnu-gcc16 env: ARCHIVE_NAME: ModelTest_${{ matrix.native_arch }}.tgz steps: @@ -93,7 +93,7 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build env: - CXXFLAGS: "-Wall -O3 -ffast-math -static-libstdc++ -static-libgcc" + CXXFLAGS: "-Wall -O3 -static-libstdc++ -static-libgcc" run: | cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=ON -DCMAKE_TOOLCHAIN_FILE=/home/develop/opt/x-tools/aarch64-rpi3-linux-gnu/aarch64-${{ matrix.native_arch }}-linux-gnu.toolchain.cmake cmake --build . --config $BUILD_TYPE -j4 @@ -135,7 +135,7 @@ jobs: env: CXXFLAGS: "/O2 -march=${{ matrix.native_arch }}" run: | - cmake.exe -G "Visual Studio 18 2026" -A x64 -T ClangCL -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=ON .. + cmake.exe -G "Visual Studio 18 2026" -A x64 -T ClangCL -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=${{ matrix.native_arch == 'x86-64-v3' && 'ON' || 'OFF' }} .. cmake --build . --config=release -j4 - name: Add Archive diff --git a/NeuralAudio/MatMul.h b/NeuralAudio/MatMul.h index 66dd748..b87a334 100644 --- a/NeuralAudio/MatMul.h +++ b/NeuralAudio/MatMul.h @@ -2,7 +2,7 @@ namespace NeuralAudio { - template + template struct MatMul { #define IsKernel(In, Out) ((InChannels == In) && (OutChannels == Out)) @@ -239,7 +239,7 @@ namespace NeuralAudio for (size_t frame = 0; frame < numFrames; frame++) { - const size_t offset = frame * InChannels; + const int offset = frame * 3; const float* in = &inData[offset]; float a0 = outData[frame] + w0 * in[0]; @@ -254,7 +254,7 @@ namespace NeuralAudio for (size_t frame = 0; frame < numFrames; frame++) { - const size_t offset = frame * InChannels; + const size_t offset = frame * 3; float* out = &outData[offset]; const float in = inData[frame]; diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index d116bf1..55ccda7 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -151,10 +151,11 @@ namespace NeuralAudio for (size_t o = 0; o < InChannels; o++) { - a[0][o] += Wcol[o] * h0; - a[1][o] += Wcol[o] * h1; - a[2][o] += Wcol[o] * h2; - a[3][o] += Wcol[o] * h3; + const float wo = Wcol[o]; + a[0][o] += wo * h0; + a[1][o] += wo * h1; + a[2][o] += wo * h2; + a[3][o] += wo * h3; } } } From 5faf32fab51da33a60de6d0bbf9af3d5a372c5be Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 16 Jul 2026 16:19:06 -0700 Subject: [PATCH 123/162] cmake cleanup. Fix EigenMath and StdMath. Add model SetAudioInputLevelDBu() --- CMakeLists.txt | 4 +-- NeuralAudio/Activation.h | 57 ++++++++++++++++++----------------- NeuralAudio/CMakeLists.txt | 2 ++ NeuralAudio/ChannelBuffer.h | 11 +++++++ NeuralAudio/NeuralModel.h | 12 +++++++- NeuralAudio/NeuralModelImpl.h | 3 +- NeuralAudio/WaveNet.h | 4 +-- 7 files changed, 59 insertions(+), 34 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 76df4e7..07ac5e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.10) -project(NeuralAudio VERSION 0.0.1) +project(NeuralAudio VERSION 0.1.3) add_subdirectory(NeuralAudio) add_subdirectory(NeuralAudioCAPI) diff --git a/NeuralAudio/Activation.h b/NeuralAudio/Activation.h index bfdd654..ed698fd 100644 --- a/NeuralAudio/Activation.h +++ b/NeuralAudio/Activation.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace NeuralAudio { @@ -16,11 +17,11 @@ namespace NeuralAudio struct StdMath { - template - static void Tanh(Matrix *x) + template + static void Tanh(Buffer& channelBuffer) { - float* data = x->data(); - size_t size = x->rows() * x->cols(); + float* data = channelBuffer.GetData(); + size_t size = channelBuffer.GetSize(); for (size_t pos = 0; pos < size; pos++) { @@ -38,11 +39,11 @@ namespace NeuralAudio return 1.0f / (1.0f + std::exp(-x)); } - template - static void LeakyReLU(Matrix* x) + template + static void LeakyReLU(Buffer& channelBuffer) { - float* data = x->data(); - size_t size = x->rows() * x->cols(); + float* data = channelBuffer.GetData(); + size_t size = channelBuffer.GetSize(); for (size_t pos = 0; pos < size; pos++) { @@ -63,8 +64,12 @@ namespace NeuralAudio struct FastMath { - static void Tanh(float* data, size_t size) - { + template + static void Tanh(Buffer& channelBuffer) + { + float* data = channelBuffer.GetData(); + size_t size = channelBuffer.GetSize(); + for (size_t pos = 0; pos < size; pos++) { data[pos] = Tanh(data[pos]); @@ -73,10 +78,6 @@ namespace NeuralAudio static inline float Tanh(const float x) { - //return std::tanh(x); - - //return math_approx::tanh<5>(x); - const float ax = fabsf(x); const float x2 = x * x; @@ -87,18 +88,14 @@ namespace NeuralAudio static inline float Sigmoid(float x) { - //return math_approx::sigmoid_exp<5>(x); - - //return 1.0f / (1.0f + std::exp(-x)); return 0.5f * (Tanh(x * 0.5f) + 1); } - static void LeakyReLU(float *data, size_t size) + template + static void LeakyReLU(Buffer& channelBuffer) { - //*x = (x->array() < 0.0f).select(x->array() * 0.01f, x->array()); - - //float* data = channelData.GetData(); - //size_t size = Channels * channelData.NumCols(); + float* data = channelBuffer.GetData(); + size_t size = channelBuffer.GetSize(); for (size_t pos = 0; pos < size; pos++) { @@ -119,10 +116,12 @@ namespace NeuralAudio struct EigenMath { - template - static void Tanh(Matrix* x) + template + static void Tanh(Buffer& channelBuffer) { - *x = x->array().tanh(); + auto map = channelBuffer.GetEigenMap(); + + map = map.array().tanh(); } static inline float Tanh(const float x) @@ -130,10 +129,12 @@ namespace NeuralAudio return Eigen::numext::tanh(x); } - template - static void LeakyReLU(Matrix* x) + template + static void LeakyReLU(Buffer& channelBuffer) { - *x = (x->array() < 0.0f).select(x->array() * 0.01f, x->array()); + auto map = channelBuffer.GetEigenMap(); + + map = (map.array() < 0.0f).select(map.array() * 0.01f, map.array()); } static inline float Sigmoid(float x) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index f25486a..3d1f5e8 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -168,6 +168,8 @@ target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/RTNeural/modules/j set_property(TARGET NeuralAudio PROPERTY POSITION_INDEPENDENT_CODE ON) +set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "" FORCE) + add_subdirectory(../deps/RTNeural RTNeural) add_subdirectory(../deps/math_approx math_approx) target_link_libraries(NeuralAudio LINK_PUBLIC RTNeural math_approx) diff --git a/NeuralAudio/ChannelBuffer.h b/NeuralAudio/ChannelBuffer.h index 1d23028..cf65580 100644 --- a/NeuralAudio/ChannelBuffer.h +++ b/NeuralAudio/ChannelBuffer.h @@ -13,6 +13,7 @@ namespace NeuralAudio class ChannelBufferBase { public: + virtual size_t GetSize() const { return 0; } virtual size_t GetNumCols() const { return 0; } size_t GetNumChannels() const { return Channels; } virtual T* GetData() { return nullptr; } @@ -40,6 +41,11 @@ namespace NeuralAudio { } + size_t GetSize() const override + { + return (size_t)(Channels * Cols); + } + size_t GetNumCols() const override { return Cols; @@ -152,6 +158,11 @@ namespace NeuralAudio return ChannelRowSpan(buffer, this->startCol, numCols); } + size_t GetSize() const + { + return GetNumChannels() * GetNumCols(); + } + size_t GetNumCols() const { return numCols; diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index 81580f1..ede8572 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -74,6 +74,16 @@ namespace NeuralAudio (void)maxSize; } + virtual void SetAudioInputLevelDBu(float audioDBu) + { + audioInputLevelDBu = audioDBu; + } + + virtual float GetAudioInputLevelDBu() + { + return audioInputLevelDBu; + } + virtual float GetRecommendedInputDBAdjustment() { return audioInputLevelDBu - modelInputLevelDBu; @@ -126,7 +136,7 @@ namespace NeuralAudio } protected: - float audioInputLevelDBu = 12; + float audioInputLevelDBu = (float)DEFAULT_INPUT_DBU; float modelInputLevelDBu = 12; float modelOutputLevelDBu = 12; float modelLoudnessDB = -18; diff --git a/NeuralAudio/NeuralModelImpl.h b/NeuralAudio/NeuralModelImpl.h index 0983867..160581f 100644 --- a/NeuralAudio/NeuralModelImpl.h +++ b/NeuralAudio/NeuralModelImpl.h @@ -12,7 +12,8 @@ namespace NeuralAudio void SetModelLoader(NeuralModelLoader* modelLoader) { this->loader = modelLoader; - audioInputLevelDBu = modelLoader->GetAudioInputLevelDBu(); + + SetAudioInputLevelDBu(modelLoader->GetAudioInputLevelDBu()); } bool HadInitialPrewarm() diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 55ccda7..729ab38 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -396,11 +396,11 @@ namespace NeuralAudio if constexpr (Activation == EActivationType::Tanh) { - WAVENET_MATH::Tanh(block.GetData(), block.GetNumChannels() * block.GetNumCols()); + WAVENET_MATH::Tanh(block); } else if constexpr (Activation == EActivationType::LeakyReLU) { - WAVENET_MATH::LeakyReLU(block.GetData(), block.GetNumChannels() * block.GetNumCols()); + WAVENET_MATH::LeakyReLU(block); } headInput.GetEigenMap().noalias() += block.GetEigenMapConst(); From 5cd91f496dccc2af26842b7b36f1b812f28366af Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 17 Jul 2026 09:22:02 -0700 Subject: [PATCH 124/162] update 8x8 conv docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 46ef1f3..a86d752 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ Note - you'll have to change the Visual Studio version if you are using a differ ```-DBUILD_STATIC_INTERNAL_NAMA2=ON|OFF```: Build internal static A2 implementation. -```-DENABLE_MULTIFRAME_8X8_CONVOLUTION=ON|OFF```: Use optimized multiframe 8x8 convolution. Much faster on very modern compilers. Much slower on older compilers. +```-DMULTIFRAME_8X8_CONVOLUTION="0"|"4"||"8"```: Use optimized multiframe 8x8 convolution. Much faster on very modern compilers. Much slower on older compilers. Be sure to use quotes around value. Defaults to "0" (disabled). ```-DDEFAULT_QUALITY_SCALE="X.X"```: Default model quality scale factor (0.0 to 1.0). Be sure to use quotes around value. Defaults to "1.0". From 22e4a0f9607fb358ac53afd3de520b38cd4bc6a7 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 17 Jul 2026 09:59:19 -0700 Subject: [PATCH 125/162] Multiframe convolution improvements/docs (#64) --- .github/workflows/release.yml | 6 +++--- NeuralAudio/CMakeLists.txt | 29 +++++++++++++++++++++-------- NeuralAudio/WaveNet.h | 20 +++++++++++++++----- README.md | 13 +++++++++++++ 4 files changed, 52 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3a44e66..7219970 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,7 +53,7 @@ jobs: CXX: clang++ CXXFLAGS: "-Wall -O3 -march=${{ matrix.native_arch }} -static-libstdc++ -static-libgcc" run: | - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=${{ matrix.native_arch == 'x86-64-v3' && 'ON' || 'OFF' }} + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON ${{ matrix.native_arch == 'x86-64' && '-DMULTIFRAME_8X8_CONVOLUTION=0' || '' }} cmake --build . --config $BUILD_TYPE -j4 - name: Add Archive @@ -95,7 +95,7 @@ jobs: env: CXXFLAGS: "-Wall -O3 -static-libstdc++ -static-libgcc" run: | - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=ON -DCMAKE_TOOLCHAIN_FILE=/home/develop/opt/x-tools/aarch64-rpi3-linux-gnu/aarch64-${{ matrix.native_arch }}-linux-gnu.toolchain.cmake + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DCMAKE_TOOLCHAIN_FILE=/home/develop/opt/x-tools/aarch64-rpi3-linux-gnu/aarch64-${{ matrix.native_arch }}-linux-gnu.toolchain.cmake cmake --build . --config $BUILD_TYPE -j4 - name: Add Archive @@ -135,7 +135,7 @@ jobs: env: CXXFLAGS: "/O2 -march=${{ matrix.native_arch }}" run: | - cmake.exe -G "Visual Studio 18 2026" -A x64 -T ClangCL -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DENABLE_MULTIFRAME_8X8_CONVOLUTION=${{ matrix.native_arch == 'x86-64-v3' && 'ON' || 'OFF' }} .. + cmake.exe -G "Visual Studio 18 2026" -A x64 -T ClangCL -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON ${{ matrix.native_arch == 'x86-64' && '-DMULTIFRAME_8X8_CONVOLUTION=0' || '' }} .. cmake --build . --config=release -j4 - name: Add Archive diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 3d1f5e8..4626e7a 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -51,17 +51,30 @@ else() message(STATUS "NOT building Internal static A2 WaveNet models") endif() -if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "15.0") - option(ENABLE_MULTIFRAME_8X8_CONVOLUTION "Enable multi-frame 8x8 convolution" ON) -elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") - option(ENABLE_MULTIFRAME_8X8_CONVOLUTION "Enable multi-frame 8x8 convolution" ON) +if(MSVC) + set(MULTIFRAME_8X8_CONVOLUTION "8" CACHE STRING "Multi-frame 8x8 convolution") else() - option(ENABLE_MULTIFRAME_8X8_CONVOLUTION "Enable multi-frame 8x8 convolution" OFF) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64|arm64)") + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "15.0") + set(MULTIFRAME_8X8_CONVOLUTION "4" CACHE STRING "Multi-frame 8x8 convolution") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") + set(MULTIFRAME_8X8_CONVOLUTION "4" CACHE STRING "Multi-frame 8x8 convolution") + endif() + else() + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "16.0") + set(MULTIFRAME_8X8_CONVOLUTION "4" CACHE STRING "Multi-frame 8x8 convolution") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") + set(MULTIFRAME_8X8_CONVOLUTION "8" CACHE STRING "Multi-frame 8x8 convolution") + endif() + endif() endif() -if(ENABLE_MULTIFRAME_8X8_CONVOLUTION) - message(STATUS "Using multi-frame 8x8 convolution") - add_definitions(-DENABLE_MULTIFRAME_8X8_CONVOLUTION) +# Set default to 0 if it isn't already set +set(MULTIFRAME_8X8_CONVOLUTION "0" CACHE STRING "Multi-frame 8x8 convolution") + +if(MULTIFRAME_8X8_CONVOLUTION GREATER 0) + message(STATUS "Using multi-frame 8x8 convolution: ${MULTIFRAME_8X8_CONVOLUTION}") + add_definitions(-DMULTIFRAME_8X8_CONVOLUTION=${MULTIFRAME_8X8_CONVOLUTION}) else() message(STATUS "NOT using multi-frame 8x8 convolution") endif() diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 729ab38..ecae93e 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -126,15 +126,15 @@ namespace NeuralAudio const size_t numFrames = output.GetNumCols(); float* __restrict outputPtr = output.GetData(); -#if ENABLE_MULTIFRAME_8X8_CONVOLUTION +#if MULTIFRAME_8X8_CONVOLUTION != 0 if constexpr ((InChannels == 8) && (OutChannels == 8) && DoBias) { // Based on @jfsantos NAM Core implementation - https://github.com/sdatkinson/NeuralAmpModelerCore/pull/277 - constexpr size_t tileSize = 4; - const size_t nF4 = (numFrames / tileSize) * tileSize; + constexpr size_t tileSize = MULTIFRAME_8X8_CONVOLUTION; + const size_t nFTile = (numFrames / tileSize) * tileSize; - for (size_t f = 0; f < nF4; f += tileSize) + for (size_t f = 0; f < nFTile; f += tileSize) { alignas(32) float a[tileSize][InChannels]{}; @@ -149,6 +149,10 @@ namespace NeuralAudio const float* __restrict Wcol = W + cp * InChannels; const float h0 = hb[cp], h1 = hb[InChannels + cp], h2 = hb[2 * InChannels + cp], h3 = hb[3 * InChannels + cp]; +#if MULTIFRAME_8X8_CONVOLUTION == 8 + const float h4 = hb[4 * InChannels + cp], h5 = hb[5 * InChannels + cp], h6 = hb[6 * InChannels + cp], h7 = hb[7 * InChannels + cp]; +#endif + for (size_t o = 0; o < InChannels; o++) { const float wo = Wcol[o]; @@ -156,6 +160,12 @@ namespace NeuralAudio a[1][o] += wo * h1; a[2][o] += wo * h2; a[3][o] += wo * h3; +#if MULTIFRAME_8X8_CONVOLUTION == 8 + a[4][o] += wo * h4; + a[5][o] += wo * h5; + a[6][o] += wo * h6; + a[7][o] += wo * h7; +#endif } } } @@ -165,7 +175,7 @@ namespace NeuralAudio } // Scalar tail for any frames past the tile-aligned boundary. - for (size_t f = nF4; f < numFrames; f++) + for (size_t f = nFTile; f < numFrames; f++) { float* zf = outputPtr + static_cast(f) * InChannels; diff --git a/README.md b/README.md index a86d752..32fb7b0 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,19 @@ cmake --build . --config=release -j4 Note - you'll have to change the Visual Studio version if you are using a different one. +## Performance considerations + +Optimization for this library is highly dependent on the details of the compiler you are using and the architecture of the system you intend to run on. In particular, getting the best performance relies on critical paths in the code being vectorized efficiently by the compiler for the target CPU. + +The CMake setup for the library *does not* specify any optimization compiler flags - you are responsible for doing that yourself. Specifically, that means doing the highest possible compiler optimization (ie: "-O3") +and making sure that the compiler is targeting your CPU properly (ie: "-march=native" for GCC/Clang, "/arch:AVX2" for MSVC). + +One more specific note - the ```MULTIFRAME_8X8_CONVOLUTION``` option described in the next section enables very impactful performance increases when set to "4" or "8" ("0" is the default) on appropriate hardware and compilers. +It also significantly *slows* performance if the optimizations are not properly supported. The best way to know which is the case in your scenario is to test it. The CMake config for this library does its best to default to the correct setting +based on architecture and compiler. + +The "ModelTest" application binaries provided in the [Releases section](https://github.com/mikeoliphant/NeuralAudio/releases) have been optimized for various specific platforms and can be used as a basis for comparison. + ## CMake Options ```-DBUILD_NAMCORE=ON|OFF```: Support loading models using the NAM Core implemenations. From fec4ea2ca369f58343d005b5e13094556e4fd977 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 17 Jul 2026 10:00:46 -0700 Subject: [PATCH 126/162] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 32fb7b0..67bb34b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ NeuralAudio is a C++ library designed to make it easy to use neural network machine learning models (ie: guitar amplifier captures/profiles) in real-time audio applications. +Priorities are a simple, but powerful API, a clean, maintainable codebase, and highly optimized performance across platforms. + # License This repository is licensed under the [MIT license](https://github.com/mikeoliphant/NeuralAudio/blob/main/LICENSE). It is a liberal license, but please make sure that you comply with the terms - as well as the terms of [this project's dependencies](https://github.com/mikeoliphant/NeuralAudio/blob/main/CREDITS.md). I would also appreciate it if you would let me know if you are using this library. From b1137e1db2d43670e7dcf6345ade7808613cac20 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 17 Jul 2026 16:43:33 -0700 Subject: [PATCH 127/162] Use 8-frame convolution for RPi5 (#66) --- .github/workflows/release.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7219970..f6b14a4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -95,7 +95,7 @@ jobs: env: CXXFLAGS: "-Wall -O3 -static-libstdc++ -static-libgcc" run: | - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -DCMAKE_TOOLCHAIN_FILE=/home/develop/opt/x-tools/aarch64-rpi3-linux-gnu/aarch64-${{ matrix.native_arch }}-linux-gnu.toolchain.cmake + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON ${{ matrix.native_arch == 'rpi5' && '-DMULTIFRAME_8X8_CONVOLUTION=8' || '' }} -DCMAKE_TOOLCHAIN_FILE=/home/develop/opt/x-tools/aarch64-rpi3-linux-gnu/aarch64-${{ matrix.native_arch }}-linux-gnu.toolchain.cmake cmake --build . --config $BUILD_TYPE -j4 - name: Add Archive diff --git a/README.md b/README.md index 67bb34b..89bc066 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,7 @@ The CMake setup for the library *does not* specify any optimization compiler fla and making sure that the compiler is targeting your CPU properly (ie: "-march=native" for GCC/Clang, "/arch:AVX2" for MSVC). One more specific note - the ```MULTIFRAME_8X8_CONVOLUTION``` option described in the next section enables very impactful performance increases when set to "4" or "8" ("0" is the default) on appropriate hardware and compilers. -It also significantly *slows* performance if the optimizations are not properly supported. The best way to know which is the case in your scenario is to test it. The CMake config for this library does its best to default to the correct setting +It also significantly *slows* performance if the optimizations are not properly supported. In general, assuming you have a very recent compiler you should be able to set it to "4" for systems with 128bit intrinsics (ie: Raspberry Pi 4) and "8" for systems with 256bit intrinsics (ie: Raspberry Pi 5, most x64 PCs). The best way to know what works in your scenario is to test it. The CMake config for this library does its best to default to the correct setting based on architecture and compiler. The "ModelTest" application binaries provided in the [Releases section](https://github.com/mikeoliphant/NeuralAudio/releases) have been optimized for various specific platforms and can be used as a basis for comparison. From bf18e101d2fbf2087d171c361d1f4dfc04efe257 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 21 Jul 2026 08:54:38 -0700 Subject: [PATCH 128/162] Use template type instead of float for activations --- NeuralAudio/Activation.h | 79 +++++++++++++++++++----------------- NeuralAudio/LSTM.h | 6 +-- NeuralAudio/LSTMDynamic.h | 6 +-- NeuralAudio/WaveNet.h | 4 +- NeuralAudio/WaveNetDynamic.h | 2 +- 5 files changed, 51 insertions(+), 46 deletions(-) diff --git a/NeuralAudio/Activation.h b/NeuralAudio/Activation.h index ed698fd..7e9d49c 100644 --- a/NeuralAudio/Activation.h +++ b/NeuralAudio/Activation.h @@ -5,6 +5,8 @@ #include #include +#define TCONST(num) static_cast(num) + namespace NeuralAudio { #ifndef LSTM_MATH @@ -15,12 +17,13 @@ namespace NeuralAudio #define WAVENET_MATH FastMath #endif + template struct StdMath { - template - static void Tanh(Buffer& channelBuffer) + template + static void Tanh(ChannelRowSpan& channelBuffer) { - float* data = channelBuffer.GetData(); + T* data = channelBuffer.GetData(); size_t size = channelBuffer.GetSize(); for (size_t pos = 0; pos < size; pos++) @@ -29,20 +32,20 @@ namespace NeuralAudio } } - static inline float Tanh(const float x) + static inline T Tanh(const T x) { return std::tanh(x); } - static inline float Sigmoid(float x) + static inline T Sigmoid(T x) { - return 1.0f / (1.0f + std::exp(-x)); + return TCONST(1.0) / (TCONST(1.0)+ std::exp(-x)); } - template - static void LeakyReLU(Buffer& channelBuffer) + template + static void LeakyReLU(ChannelRowSpan channelBuffer) { - float* data = channelBuffer.GetData(); + T* data = channelBuffer.GetData(); size_t size = channelBuffer.GetSize(); for (size_t pos = 0; pos < size; pos++) @@ -51,23 +54,24 @@ namespace NeuralAudio } } - static inline float LeakyReLU(float x, float negativeSlope) + static inline T LeakyReLU(T x, T negativeSlope) { - return x > 0.0f ? x : negativeSlope * x; + return x > TCONST(0) ? x : negativeSlope * x; } - static inline float LeakyReLU(float x) + static inline T LeakyReLU(T x) { - return LeakyReLU(x, 0.01f); + return LeakyReLU(x, TCONST(0.01)); } }; + template struct FastMath { - template - static void Tanh(Buffer& channelBuffer) + template + static void Tanh(ChannelRowSpan& channelBuffer) { - float* data = channelBuffer.GetData(); + T* data = channelBuffer.GetData(); size_t size = channelBuffer.GetSize(); for (size_t pos = 0; pos < size; pos++) @@ -76,25 +80,25 @@ namespace NeuralAudio } } - static inline float Tanh(const float x) + static inline T Tanh(const T x) { - const float ax = fabsf(x); + const T ax = std::abs(x); - const float x2 = x * x; + const T x2 = x * x; - return (x * (2.45550750702956f + 2.45550750702956f * ax + (0.893229853513558f + 0.821226666969744f * ax) * x2) - / (2.44506634652299f + (2.44506634652299f + x2) * fabsf(x + 0.814642734961073f * x * ax))); + return (x * (TCONST(2.45550750702956) + TCONST(2.45550750702956) * ax + (TCONST(0.893229853513558) + TCONST(0.821226666969744) * ax) * x2) + / (TCONST(2.44506634652299) + (TCONST(2.44506634652299) + x2) * std::abs(x + TCONST(0.814642734961073) * x * ax))); } - static inline float Sigmoid(float x) + static inline T Sigmoid(T x) { - return 0.5f * (Tanh(x * 0.5f) + 1); + return TCONST(0.5) * (Tanh(x * TCONST(0.5) + TCONST(1))); } - template - static void LeakyReLU(Buffer& channelBuffer) + template + static void LeakyReLU(ChannelRowSpan channelBuffer) { - float* data = channelBuffer.GetData(); + T* data = channelBuffer.GetData(); size_t size = channelBuffer.GetSize(); for (size_t pos = 0; pos < size; pos++) @@ -103,21 +107,22 @@ namespace NeuralAudio } } - static inline float LeakyReLU(float x, float negativeSlope) + static inline T LeakyReLU(T x, T negativeSlope) { - return x > 0.0f ? x : negativeSlope * x; + return x > TCONST(0.0) ? x : negativeSlope * x; } - static inline float LeakyReLU(float x) + static inline T LeakyReLU(T x) { - return LeakyReLU(x, 0.01f); + return LeakyReLU(x, TCONST(0.01)); } }; + template struct EigenMath { - template - static void Tanh(Buffer& channelBuffer) + template + static void Tanh(ChannelRowSpan& channelBuffer) { auto map = channelBuffer.GetEigenMap(); @@ -129,17 +134,17 @@ namespace NeuralAudio return Eigen::numext::tanh(x); } - template - static void LeakyReLU(Buffer& channelBuffer) + template + static void LeakyReLU(ChannelRowSpan channelBuffer) { auto map = channelBuffer.GetEigenMap(); - map = (map.array() < 0.0f).select(map.array() * 0.01f, map.array()); + map = (map.array() < TCONST(0.0)).select(map.array() * TCONST(0.01), map.array()); } - static inline float Sigmoid(float x) + static inline T Sigmoid(T x) { - return 0.5f * (Tanh(x * 0.5f) + 1); + return TCONST(0.5) * (Tanh(x * TCONST(0.5)) + TCONST(1)); } }; } \ No newline at end of file diff --git a/NeuralAudio/LSTM.h b/NeuralAudio/LSTM.h index c4c70b4..4536ee8 100644 --- a/NeuralAudio/LSTM.h +++ b/NeuralAudio/LSTM.h @@ -92,11 +92,11 @@ namespace NeuralAudio gates = (inputHiddenWeights * state) + bias; for (auto i = 0; i < HiddenSize; i++) - cellState[i] = (LSTM_MATH::Sigmoid(gates[i + fOffset]) * cellState[i]) + - (LSTM_MATH::Sigmoid(gates[i + iOffset]) * LSTM_MATH::Tanh(gates[i + gOffset])); + cellState[i] = (LSTM_MATH::Sigmoid(gates[i + fOffset]) * cellState[i]) + + (LSTM_MATH::Sigmoid(gates[i + iOffset]) * LSTM_MATH::Tanh(gates[i + gOffset])); for (int i = 0; i < HiddenSize; i++) - state[i + hOffset] = LSTM_MATH::Sigmoid(gates[i + oOffset]) * LSTM_MATH::Tanh(cellState[i]); + state[i + hOffset] = LSTM_MATH::Sigmoid(gates[i + oOffset]) * LSTM_MATH::Tanh(cellState[i]); } }; diff --git a/NeuralAudio/LSTMDynamic.h b/NeuralAudio/LSTMDynamic.h index 3205259..e90d9b8 100644 --- a/NeuralAudio/LSTMDynamic.h +++ b/NeuralAudio/LSTMDynamic.h @@ -100,11 +100,11 @@ namespace NeuralAudio gates = (inputHiddenWeights * state) + bias; for (size_t i = 0; i < hiddenSize; i++) - cellState[i] = (LSTM_MATH::Sigmoid(gates[i + fOffset]) * cellState[i]) + (LSTM_MATH::Sigmoid(gates[i + iOffset]) * - LSTM_MATH::Tanh(gates[i + gOffset])); + cellState[i] = (LSTM_MATH::Sigmoid(gates[i + fOffset]) * cellState[i]) + (LSTM_MATH::Sigmoid(gates[i + iOffset]) * + LSTM_MATH::Tanh(gates[i + gOffset])); for (size_t i = 0; i < hiddenSize; i++) - state[i + hOffset] = LSTM_MATH::Sigmoid(gates[i + oOffset]) * LSTM_MATH::Tanh(cellState[i]); + state[i + hOffset] = LSTM_MATH::Sigmoid(gates[i + oOffset]) * LSTM_MATH::Tanh(cellState[i]); } }; diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index ecae93e..f34e105 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -406,11 +406,11 @@ namespace NeuralAudio if constexpr (Activation == EActivationType::Tanh) { - WAVENET_MATH::Tanh(block); + WAVENET_MATH::Tanh(block); } else if constexpr (Activation == EActivationType::LeakyReLU) { - WAVENET_MATH::LeakyReLU(block); + WAVENET_MATH::LeakyReLU(block); } headInput.GetEigenMap().noalias() += block.GetEigenMapConst(); diff --git a/NeuralAudio/WaveNetDynamic.h b/NeuralAudio/WaveNetDynamic.h index ea56b56..b16f548 100644 --- a/NeuralAudio/WaveNetDynamic.h +++ b/NeuralAudio/WaveNetDynamic.h @@ -241,7 +241,7 @@ namespace NeuralAudio for (auto pos = 0; pos < size; pos++) { - data[pos] = WAVENET_MATH::Tanh(data[pos]); + data[pos] = WAVENET_MATH::Tanh(data[pos]); } headInput.noalias() += block.topRows(channels); From f19b5b97b6f03025007077ef1ddcc7516f6c42bc Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 21 Jul 2026 09:02:02 -0700 Subject: [PATCH 129/162] Fix fastmath sigmoid typo --- NeuralAudio/Activation.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NeuralAudio/Activation.h b/NeuralAudio/Activation.h index 7e9d49c..076ddc0 100644 --- a/NeuralAudio/Activation.h +++ b/NeuralAudio/Activation.h @@ -92,7 +92,7 @@ namespace NeuralAudio static inline T Sigmoid(T x) { - return TCONST(0.5) * (Tanh(x * TCONST(0.5) + TCONST(1))); + return TCONST(0.5) * (Tanh(x * TCONST(0.5)) + TCONST(1)); } template From 766c1ba19f2282ec2e806d6a537f8755510d45ed Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 21 Jul 2026 09:36:14 -0700 Subject: [PATCH 130/162] use template type instead of float for WaveNet samples --- NeuralAudio/ChannelBuffer.h | 16 ++--- NeuralAudio/InternalModel.h | 12 ++-- NeuralAudio/MatMul.h | 120 ++++++++++++++++----------------- NeuralAudio/NeuralModel.cpp | 4 +- NeuralAudio/WaveNet.h | 128 ++++++++++++++++++------------------ 5 files changed, 140 insertions(+), 140 deletions(-) diff --git a/NeuralAudio/ChannelBuffer.h b/NeuralAudio/ChannelBuffer.h index cf65580..7101aa2 100644 --- a/NeuralAudio/ChannelBuffer.h +++ b/NeuralAudio/ChannelBuffer.h @@ -98,14 +98,14 @@ namespace NeuralAudio return ChannelRowSpan(this, numCols); } - const Eigen::Map> GetEigenMap() + const Eigen::Map> GetEigenMap() { - return Eigen::Map>(GetData(), Channels, GetNumCols()); + return Eigen::Map>(GetData(), Channels, GetNumCols()); } - const Eigen::Map> GetEigenMapConst() const + const Eigen::Map> GetEigenMapConst() const { - return Eigen::Map>(GetDataConst(), Channels, GetNumCols()); + return Eigen::Map>(GetDataConst(), Channels, GetNumCols()); } private: @@ -193,14 +193,14 @@ namespace NeuralAudio return buffer->GetDataConst(this->startCol + startCol); } - Eigen::Map> GetEigenMap() const + Eigen::Map> GetEigenMap() const { - return Eigen::Map>(GetData(), Channels, numCols); + return Eigen::Map>(GetData(), Channels, numCols); } - const Eigen::Map> GetEigenMapConst() const + const Eigen::Map> GetEigenMapConst() const { - return Eigen::Map>(GetDataConst(), Channels, numCols); + return Eigen::Map>(GetDataConst(), Channels, numCols); } void CopyData(const ChannelRowSpan& srcSpan) const diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index 02c4552..3e0444a 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -150,12 +150,12 @@ namespace NeuralAudio { public: using ModelType = typename std::conditional, - NeuralAudio::WaveNetLayerArrayT>, - NeuralAudio::WaveNetModelT< - NeuralAudio::WaveNetLayerArrayT<1, 1, HeadSize, 1, 1, NumChannels, ILiteKernelSizes1, ILiteDilations1, false, EActivationType::Tanh>, - NeuralAudio::WaveNetLayerArrayT> + NeuralAudio::WaveNetModelT, + NeuralAudio::WaveNetLayerArrayT>, + NeuralAudio::WaveNetModelT, + NeuralAudio::WaveNetLayerArrayT> >::type; InternalModel* CreateModel() override diff --git a/NeuralAudio/MatMul.h b/NeuralAudio/MatMul.h index b87a334..17810e9 100644 --- a/NeuralAudio/MatMul.h +++ b/NeuralAudio/MatMul.h @@ -2,7 +2,7 @@ namespace NeuralAudio { - template + template struct MatMul { #define IsKernel(In, Out) ((InChannels == In) && (OutChannels == Out)) @@ -11,25 +11,25 @@ namespace NeuralAudio // 3x3 implementation inspired by @jfsantos NAM Core a2fast - https://github.com/sdatkinson/NeuralAmpModelerCore/blob/main/NAM/wavenet/a2_fast.cpp - static inline void MultiplyInitZero(const float* inData, float* outData, const float* weights, size_t numFrames) + static inline void MultiplyInitZero(const T* inData, T* outData, const T* weights, size_t numFrames) { static_assert(HasKernel(), "Multiplication not implemented for InChannel/OutChannel combination"); if constexpr (IsKernel(3, 3)) { - const float w00 = weights[0], w10 = weights[1], w20 = weights[2]; - const float w01 = weights[3], w11 = weights[4], w21 = weights[5]; - const float w02 = weights[6], w12 = weights[7], w22 = weights[8]; + const T w00 = weights[0], w10 = weights[1], w20 = weights[2]; + const T w01 = weights[3], w11 = weights[4], w21 = weights[5]; + const T w02 = weights[6], w12 = weights[7], w22 = weights[8]; for (size_t frame = 0; frame < numFrames; frame++) { const size_t offset = frame * 3; - const float* in = &inData[offset]; - float* out = &outData[offset]; + const T* in = &inData[offset]; + T* out = &outData[offset]; - float a0 = w00 * in[0]; - float a1 = w10 * in[0]; - float a2 = w20 * in[0]; + T a0 = w00 * in[0]; + T a1 = w10 * in[0]; + T a2 = w20 * in[0]; a0 += w01 * in[1]; a1 += w11 * in[1]; a2 += w21 * in[1]; @@ -43,15 +43,15 @@ namespace NeuralAudio } else if constexpr (IsKernel(8, 1)) { - const float w0 = weights[0], w1 = weights[1], w2 = weights[2], w3 = weights[3]; - const float w4 = weights[4], w5 = weights[5], w6 = weights[6], w7 = weights[7]; + const T w0 = weights[0], w1 = weights[1], w2 = weights[2], w3 = weights[3]; + const T w4 = weights[4], w5 = weights[5], w6 = weights[6], w7 = weights[7]; for (size_t frame = 0; frame < numFrames; frame++) { const size_t offset = frame * InChannels; - const float* in = &inData[offset]; + const T* in = &inData[offset]; - float a0 = w0 * in[0]; + T a0 = w0 * in[0]; a0 += w1 * in[1]; a0 += w2 * in[2]; a0 += w3 * in[3]; @@ -64,14 +64,14 @@ namespace NeuralAudio } else if constexpr (IsKernel(3, 1)) { - const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; + const T w0 = weights[0], w1 = weights[1], w2 = weights[2]; for (size_t frame = 0; frame < numFrames; frame++) { const size_t offset = frame * 3; - const float* in = &inData[offset]; + const T* in = &inData[offset]; - float a0 = w0 * in[0]; + T a0 = w0 * in[0]; a0 += w1 * in[1]; a0 += w2 * in[2]; outData[frame] = a0; @@ -79,14 +79,14 @@ namespace NeuralAudio } else if constexpr (IsKernel(1, 3)) { - const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; + const T w0 = weights[0], w1 = weights[1], w2 = weights[2]; for (size_t frame = 0; frame < numFrames; frame++) { const size_t offset = frame * 3; - float* out = &outData[offset]; + T* out = &outData[offset]; - const float in = inData[frame]; + const T in = inData[frame]; out[0] = w0 * in; out[1] = w1 * in; @@ -95,25 +95,25 @@ namespace NeuralAudio } } - static inline void MultiplyInitColwise(const float* inData, float* outData, const float* weights, const float* initData, size_t numFrames) + static inline void MultiplyInitColwise(const T* inData, T* outData, const T* weights, const T* initData, size_t numFrames) { static_assert(HasKernel(), "Multiplication not implemented for InChannel/OutChannel combination"); if constexpr (IsKernel(3, 3)) { - const float w00 = weights[0], w10 = weights[1], w20 = weights[2]; - const float w01 = weights[3], w11 = weights[4], w21 = weights[5]; - const float w02 = weights[6], w12 = weights[7], w22 = weights[8]; + const T w00 = weights[0], w10 = weights[1], w20 = weights[2]; + const T w01 = weights[3], w11 = weights[4], w21 = weights[5]; + const T w02 = weights[6], w12 = weights[7], w22 = weights[8]; for (size_t frame = 0; frame < numFrames; frame++) { const size_t offset = frame * 3; - const float* in = &inData[offset]; - float* out = &outData[offset]; + const T* in = &inData[offset]; + T* out = &outData[offset]; - float a0 = initData[0] + w00 * in[0]; - float a1 = initData[1] + w10 * in[0]; - float a2 = initData[2] + w20 * in[0]; + T a0 = initData[0] + w00 * in[0]; + T a1 = initData[1] + w10 * in[0]; + T a2 = initData[2] + w20 * in[0]; a0 += w01 * in[1]; a1 += w11 * in[1]; a2 += w21 * in[1]; @@ -127,17 +127,17 @@ namespace NeuralAudio } else if constexpr (IsKernel(8, 1)) { - const float w0 = weights[0], w1 = weights[1], w2 = weights[2], w3 = weights[3]; - const float w4 = weights[4], w5 = weights[5], w6 = weights[6], w7 = weights[7]; + const T w0 = weights[0], w1 = weights[1], w2 = weights[2], w3 = weights[3]; + const T w4 = weights[4], w5 = weights[5], w6 = weights[6], w7 = weights[7]; - const float init = initData[0]; + const T init = initData[0]; for (size_t frame = 0; frame < numFrames; frame++) { const size_t offset = frame * InChannels; - const float* in = &inData[offset]; + const T* in = &inData[offset]; - float a0 = init + w0 * in[0]; + T a0 = init + w0 * in[0]; a0 += w1 * in[1]; a0 += w2 * in[2]; a0 += w3 * in[3]; @@ -150,14 +150,14 @@ namespace NeuralAudio } else if constexpr (IsKernel(3, 1)) { - const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; + const T w0 = weights[0], w1 = weights[1], w2 = weights[2]; for (size_t frame = 0; frame < numFrames; frame++) { const size_t offset = frame * 3; - const float* in = &inData[offset]; + const T* in = &inData[offset]; - float a0 = initData[0] + w0 * in[0]; + T a0 = initData[0] + w0 * in[0]; a0 += w1 * in[1]; a0 += w2 * in[2]; outData[frame] = a0; @@ -165,15 +165,15 @@ namespace NeuralAudio } else if constexpr (IsKernel(1, 3)) { - const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; - const float init0 = inData[0], init1 = inData[1], init2 = inData[2]; + const T w0 = weights[0], w1 = weights[1], w2 = weights[2]; + const T init0 = inData[0], init1 = inData[1], init2 = inData[2]; for (size_t frame = 0; frame < numFrames; frame++) { const size_t offset = frame * 3; - float* out = &outData[offset]; + T* out = &outData[offset]; - const float in = inData[frame]; + const T in = inData[frame]; out[0] = init0 + (w0 * in); out[1] = init1 + (w1 * in); @@ -182,25 +182,25 @@ namespace NeuralAudio } } - static inline void MultiplyAccumlulate(const float* inData, float* outData, const float* weights, size_t numFrames) + static inline void MultiplyAccumlulate(const T* inData, T* outData, const T* weights, size_t numFrames) { static_assert(HasKernel(), "Multiplication not implemented for InChannel/OutChannel combination"); if constexpr (IsKernel(3, 3)) { - const float w00 = weights[0], w10 = weights[1], w20 = weights[2]; - const float w01 = weights[3], w11 = weights[4], w21 = weights[5]; - const float w02 = weights[6], w12 = weights[7], w22 = weights[8]; + const T w00 = weights[0], w10 = weights[1], w20 = weights[2]; + const T w01 = weights[3], w11 = weights[4], w21 = weights[5]; + const T w02 = weights[6], w12 = weights[7], w22 = weights[8]; for (size_t frame = 0; frame < numFrames; frame++) { const size_t offset = frame * 3; - const float* in = &inData[offset]; - float* out = &outData[offset]; + const T* in = &inData[offset]; + T* out = &outData[offset]; - float a0 = out[0] + w00 * in[0]; - float a1 = out[1] + w10 * in[0]; - float a2 = out[2] + w20 * in[0]; + T a0 = out[0] + w00 * in[0]; + T a1 = out[1] + w10 * in[0]; + T a2 = out[2] + w20 * in[0]; a0 += w01 * in[1]; a1 += w11 * in[1]; a2 += w21 * in[1]; @@ -214,15 +214,15 @@ namespace NeuralAudio } else if constexpr (IsKernel(8, 1)) { - const float w0 = weights[0], w1 = weights[1], w2 = weights[2], w3 = weights[3]; - const float w4 = weights[4], w5 = weights[5], w6 = weights[6], w7 = weights[7]; + const T w0 = weights[0], w1 = weights[1], w2 = weights[2], w3 = weights[3]; + const T w4 = weights[4], w5 = weights[5], w6 = weights[6], w7 = weights[7]; for (size_t frame = 0; frame < numFrames; frame++) { const size_t offset = frame * InChannels; - const float* in = &inData[offset]; + const T* in = &inData[offset]; - float a0 = outData[frame] + w0 * in[0]; + T a0 = outData[frame] + w0 * in[0]; a0 += w1 * in[1]; a0 += w2 * in[2]; a0 += w3 * in[3]; @@ -235,14 +235,14 @@ namespace NeuralAudio } else if constexpr (IsKernel(3, 1)) { - const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; + const T w0 = weights[0], w1 = weights[1], w2 = weights[2]; for (size_t frame = 0; frame < numFrames; frame++) { const int offset = frame * 3; - const float* in = &inData[offset]; + const T* in = &inData[offset]; - float a0 = outData[frame] + w0 * in[0]; + T a0 = outData[frame] + w0 * in[0]; a0 += w1 * in[1]; a0 += w2 * in[2]; outData[frame] = a0; @@ -250,14 +250,14 @@ namespace NeuralAudio } else if constexpr (IsKernel(1, 3)) { - const float w0 = weights[0], w1 = weights[1], w2 = weights[2]; + const T w0 = weights[0], w1 = weights[1], w2 = weights[2]; for (size_t frame = 0; frame < numFrames; frame++) { const size_t offset = frame * 3; - float* out = &outData[offset]; + T* out = &outData[offset]; - const float in = inData[frame]; + const T in = inData[frame]; out[0] += w0 * in; out[1] += w1 * in; diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 2afc3fb..04d21b4 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -395,7 +395,7 @@ namespace NeuralAudio { if (layerConfig.at("channels") == 3) { - auto model = new InternalWaveNetModelT>>(); + auto model = new InternalWaveNetModelT>>(); if (model != nullptr) { @@ -407,7 +407,7 @@ namespace NeuralAudio } else if (layerConfig.at("channels") == 8) { - auto model = new InternalWaveNetModelT >>(); + auto model = new InternalWaveNetModelT >>(); if (model != nullptr) { diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index f34e105..02822b6 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -27,13 +27,13 @@ enum EActivationType namespace NeuralAudio { - template + template class ChannelHistoryBuffer { public: static constexpr auto BufferSize = ReceptiveFieldSize + ((LAYER_ARRAY_BUFFER_PADDING + 1) * WAVENET_MAX_NUM_FRAMES); - ChannelBuffer buffer; + ChannelBuffer buffer; size_t bufferStart; void AllocBuffer(int allocNum) @@ -84,12 +84,12 @@ namespace NeuralAudio struct Empty {}; - template + template class Conv1DT { public: static constexpr auto ReceptiveFieldSize = (KernelSize - 1) * Dilation; - ChannelHistoryBuffer channelBuffer; + ChannelHistoryBuffer channelBuffer; size_t GetNumWeights() { @@ -121,10 +121,10 @@ namespace NeuralAudio return channelBuffer.buffer.Slice(channelBuffer.bufferStart, numFrames); } - inline void Process(const ChannelRowSpan& output) + inline void Process(const ChannelRowSpan& output) { const size_t numFrames = output.GetNumCols(); - float* __restrict outputPtr = output.GetData(); + T* __restrict outputPtr = output.GetData(); #if MULTIFRAME_8X8_CONVOLUTION != 0 if constexpr ((InChannels == 8) && (OutChannels == 8) && DoBias) @@ -136,26 +136,26 @@ namespace NeuralAudio for (size_t f = 0; f < nFTile; f += tileSize) { - alignas(32) float a[tileSize][InChannels]{}; + alignas(32) T a[tileSize][InChannels]{}; for (size_t k = 0; k < KernelSize; k++) { - const float* __restrict W = weightPtrs[k]; + const T* __restrict W = weightPtrs[k]; const auto offset = Dilation * (k + 1 - KernelSize); - const float* __restrict hb = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset + f); + const T* __restrict hb = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset + f); for (size_t cp = 0; cp < InChannels; cp++) { - const float* __restrict Wcol = W + cp * InChannels; - const float h0 = hb[cp], h1 = hb[InChannels + cp], h2 = hb[2 * InChannels + cp], h3 = hb[3 * InChannels + cp]; + const T* __restrict Wcol = W + cp * InChannels; + const T h0 = hb[cp], h1 = hb[InChannels + cp], h2 = hb[2 * InChannels + cp], h3 = hb[3 * InChannels + cp]; #if MULTIFRAME_8X8_CONVOLUTION == 8 - const float h4 = hb[4 * InChannels + cp], h5 = hb[5 * InChannels + cp], h6 = hb[6 * InChannels + cp], h7 = hb[7 * InChannels + cp]; + const T h4 = hb[4 * InChannels + cp], h5 = hb[5 * InChannels + cp], h6 = hb[6 * InChannels + cp], h7 = hb[7 * InChannels + cp]; #endif for (size_t o = 0; o < InChannels; o++) { - const float wo = Wcol[o]; + const T wo = Wcol[o]; a[0][o] += wo * h0; a[1][o] += wo * h1; a[2][o] += wo * h2; @@ -171,27 +171,27 @@ namespace NeuralAudio } for (size_t ti = 0; ti < tileSize; ti++) - std::memcpy(outputPtr + static_cast(f + ti) * InChannels, a[ti], InChannels * sizeof(float)); + std::memcpy(outputPtr + static_cast(f + ti) * InChannels, a[ti], InChannels * sizeof(T)); } // Scalar tail for any frames past the tile-aligned boundary. for (size_t f = nFTile; f < numFrames; f++) { - float* zf = outputPtr + static_cast(f) * InChannels; + T* zf = outputPtr + static_cast(f) * InChannels; for (size_t o = 0; o < InChannels; o++) - zf[o] = 0.0f; + zf[o] = TCONST(0.0); for (size_t k = 0; k < KernelSize; k++) { - const float* W = weightPtrs[k]; + const T* W = weightPtrs[k]; const auto offset = Dilation * (k + 1 - KernelSize); - const float* h = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset + f); + const T* h = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset + f); for (int cp = 0; cp < InChannels; cp++) { - const float hv = h[cp]; - const float* Wcol = W + cp * InChannels; + const T hv = h[cp]; + const T* Wcol = W + cp * InChannels; for (size_t o = 0; o < InChannels; o++) zf[o] += Wcol[o] * hv; @@ -202,7 +202,7 @@ namespace NeuralAudio else #endif { - const float* biasPtr = nullptr; + const T* biasPtr = nullptr; if constexpr (DoBias) { @@ -211,28 +211,28 @@ namespace NeuralAudio for (size_t k = 0; k < KernelSize; k++) { - const float* weightPtr = this->weights[k].GetDataConst(); + const T* weightPtr = this->weights[k].GetDataConst(); const auto offset = Dilation * ((int)k + 1 - KernelSize); - if constexpr (DoBias && MatMul::HasKernel()) + if constexpr (DoBias && MatMul::HasKernel()) { - const float* inputPtr = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset); + const T* inputPtr = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset); if (k == 0) // Maybe move this out of loop? { if constexpr (DoBias) { - MatMul::MultiplyInitColwise(inputPtr, outputPtr, weightPtr, biasPtr, numFrames); + MatMul::MultiplyInitColwise(inputPtr, outputPtr, weightPtr, biasPtr, numFrames); } else { - MatMul::MultiplyInitZero(inputPtr, outputPtr, weightPtr, numFrames); + MatMul::MultiplyInitZero(inputPtr, outputPtr, weightPtr, numFrames); } } else { - MatMul::MultiplyAccumlulate(inputPtr, outputPtr, weightPtr, numFrames); + MatMul::MultiplyAccumlulate(inputPtr, outputPtr, weightPtr, numFrames); } } else @@ -247,23 +247,23 @@ namespace NeuralAudio } } - if constexpr (DoBias && !MatMul::HasKernel()) + if constexpr (DoBias && !MatMul::HasKernel()) output.GetEigenMap().colwise() += bias; } private: - alignas(32) std::array, KernelSize> weights; // consider making this a contiguous block of data instead of block of ChannelBuffers - std::array weightPtrs; + alignas(32) std::array, KernelSize> weights; // consider making this a contiguous block of data instead of block of ChannelBuffers + std::array weightPtrs; // Avoid allocation for unused bias using BiasType = typename std::conditional, + Eigen::Vector, Empty>::type; BiasType bias; }; - template + template class DenseLayerT { public: @@ -285,19 +285,19 @@ namespace NeuralAudio } } - void Process(const ChannelRowSpan& input, const ChannelRowSpan& output) const + void Process(const ChannelRowSpan& input, const ChannelRowSpan& output) const { size_t numFrames = output.GetNumCols(); - if constexpr (MatMul::HasKernel()) + if constexpr (MatMul::HasKernel()) { if constexpr (DoBias) { - MatMul::MultiplyInitColwise(input.GetDataConst(), output.GetData(), weights.GetDataConst(), bias.data(), numFrames); + MatMul::MultiplyInitColwise(input.GetDataConst(), output.GetData(), weights.GetDataConst(), bias.data(), numFrames); } else { - MatMul::MultiplyInitZero(input.GetDataConst(), output.GetData(), weights.GetDataConst(), numFrames); + MatMul::MultiplyInitZero(input.GetDataConst(), output.GetData(), weights.GetDataConst(), numFrames); } } else @@ -313,13 +313,13 @@ namespace NeuralAudio } } - void ProcessAcc(const ChannelRowSpan& input, const ChannelRowSpan& output) const + void ProcessAcc(const ChannelRowSpan& input, const ChannelRowSpan& output) const { size_t numFrames = output.GetNumCols(); - if constexpr (!DoBias && MatMul::HasKernel()) + if constexpr (!DoBias && MatMul::HasKernel()) { - MatMul::MultiplyAccumlulate(input.GetDataConst(), output.GetData(), weights.GetDataConst(), numFrames); + MatMul::MultiplyAccumlulate(input.GetDataConst(), output.GetData(), weights.GetDataConst(), numFrames); } else { @@ -335,24 +335,24 @@ namespace NeuralAudio } private: - ChannelBuffer weights; + ChannelBuffer weights; // Avoid allocation for unused bias using BiasType = typename std::conditional, + Eigen::Vector, Empty>::type; BiasType bias; }; - template + template class WaveNetLayerT { private: - Conv1DT conv1D; - DenseLayerT inputMixin; - DenseLayerT oneByOne; - ChannelBuffer state; + Conv1DT conv1D; + DenseLayerT inputMixin; + DenseLayerT oneByOne; + ChannelBuffer state; public: static constexpr auto ReceptiveFieldSize = (KernelSize - 1) * Dilation; @@ -394,7 +394,7 @@ namespace NeuralAudio oneByOne.SetWeights(weights); } - void Process(const ChannelRowSpan& condition, const ChannelRowSpan& headInput, const ChannelRowSpan& output) + void Process(const ChannelRowSpan& condition, const ChannelRowSpan& headInput, const ChannelRowSpan& output) { size_t numFrames = output.GetNumCols(); @@ -406,11 +406,11 @@ namespace NeuralAudio if constexpr (Activation == EActivationType::Tanh) { - WAVENET_MATH::Tanh(block); + WAVENET_MATH::template Tanh(block); } else if constexpr (Activation == EActivationType::LeakyReLU) { - WAVENET_MATH::LeakyReLU(block); + WAVENET_MATH::template LeakyReLU(block); } headInput.GetEigenMap().noalias() += block.GetEigenMapConst(); @@ -431,7 +431,7 @@ namespace NeuralAudio template using KernelSizes = std::integer_sequence; - template + template class WaveNetLayerArrayT { template @@ -442,15 +442,15 @@ namespace NeuralAudio template struct LayersHelper, Dilations> { - using type = std::tuple...>; + using type = std::tuple...>; }; using Layers = typename LayersHelper::type; private: Layers layers; - DenseLayerT rechannel; - Conv1DT headRechannel; + DenseLayerT rechannel; + Conv1DT headRechannel; static constexpr auto numLayers = std::tuple_size_v; static constexpr auto lastLayer = numLayers - 1; @@ -459,8 +459,8 @@ namespace NeuralAudio static constexpr auto NumChannelsP = Channels; static constexpr auto HeadSizeP = HeadSize; - ChannelBuffer arrayOutputs; - ChannelBuffer headOutputs; + ChannelBuffer arrayOutputs; + ChannelBuffer headOutputs; int ReceptiveFieldSize = 0; // This should be a static constexpr, but I haven't sorted out the right template magic WaveNetLayerArrayT() @@ -511,7 +511,7 @@ namespace NeuralAudio headRechannel.SetWeights(weights); } - void Prewarm(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) + void Prewarm(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) { rechannel.Process(layerInputs, std::get<0>(layers).GetInputBuffer(1)); @@ -536,7 +536,7 @@ namespace NeuralAudio headRechannel.Process(headOutputs.Slice(1)); } - void Process(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) + void Process(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) { size_t numFrames = condition.GetNumCols(); @@ -567,7 +567,7 @@ namespace NeuralAudio } }; - template + template class WaveNetModelT { public: @@ -647,9 +647,9 @@ namespace NeuralAudio }); } - void Process(const float* input, float* output, const size_t numFrames) + void Process(const T* input, T* output, const size_t numFrames) { - std::memcpy(condition.GetData(), input, numFrames * sizeof(float)); + std::memcpy(condition.GetData(), input, numFrames * sizeof(T)); headArray.SetZero(); @@ -668,7 +668,7 @@ namespace NeuralAudio } }); - const float* finalHeadArray = std::get(layerArrays).headOutputs.GetData(); + T* finalHeadArray = std::get(layerArrays).headOutputs.GetData(); for (size_t i = 0; i < numFrames; i++) { @@ -680,8 +680,8 @@ namespace NeuralAudio static constexpr auto headLayerChannels = std::tuple_element_t<0, std::tuple>::NumChannelsP; std::tuple layerArrays; - ChannelBuffer condition; - ChannelBuffer headArray; - float headScale; + ChannelBuffer condition; + ChannelBuffer headArray; + T headScale; }; } From a78d32f5b4c30c7c3a5094cf887c2d8ea461165d Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 21 Jul 2026 12:14:30 -0700 Subject: [PATCH 131/162] Faster A1 Standard --- NeuralAudio/WaveNet.h | 50 +++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 02822b6..7cfd3fa 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -127,11 +127,11 @@ namespace NeuralAudio T* __restrict outputPtr = output.GetData(); #if MULTIFRAME_8X8_CONVOLUTION != 0 - if constexpr ((InChannels == 8) && (OutChannels == 8) && DoBias) + if constexpr ((((InChannels == 8) && (OutChannels == 8)) || ((InChannels == 16) && (OutChannels == 16))) && DoBias) { // Based on @jfsantos NAM Core implementation - https://github.com/sdatkinson/NeuralAmpModelerCore/pull/277 - constexpr size_t tileSize = MULTIFRAME_8X8_CONVOLUTION; + constexpr size_t tileSize = (InChannels == 16) ? 4 : MULTIFRAME_8X8_CONVOLUTION; const size_t nFTile = (numFrames / tileSize) * tileSize; for (size_t f = 0; f < nFTile; f += tileSize) @@ -147,25 +147,37 @@ namespace NeuralAudio for (size_t cp = 0; cp < InChannels; cp++) { const T* __restrict Wcol = W + cp * InChannels; - const T h0 = hb[cp], h1 = hb[InChannels + cp], h2 = hb[2 * InChannels + cp], h3 = hb[3 * InChannels + cp]; -#if MULTIFRAME_8X8_CONVOLUTION == 8 - const T h4 = hb[4 * InChannels + cp], h5 = hb[5 * InChannels + cp], h6 = hb[6 * InChannels + cp], h7 = hb[7 * InChannels + cp]; -#endif - - for (size_t o = 0; o < InChannels; o++) + if constexpr (tileSize == 4) { - const T wo = Wcol[o]; - a[0][o] += wo * h0; - a[1][o] += wo * h1; - a[2][o] += wo * h2; - a[3][o] += wo * h3; -#if MULTIFRAME_8X8_CONVOLUTION == 8 - a[4][o] += wo * h4; - a[5][o] += wo * h5; - a[6][o] += wo * h6; - a[7][o] += wo * h7; -#endif + const T h0 = hb[cp], h1 = hb[InChannels + cp], h2 = hb[2 * InChannels + cp], h3 = hb[3 * InChannels + cp]; + + for (size_t o = 0; o < InChannels; o++) + { + const T wo = Wcol[o]; + a[0][o] += wo * h0; + a[1][o] += wo * h1; + a[2][o] += wo * h2; + a[3][o] += wo * h3; + } + } + else // tileSize == 8 + { + const T h0 = hb[cp], h1 = hb[InChannels + cp], h2 = hb[2 * InChannels + cp], h3 = hb[3 * InChannels + cp]; + const T h4 = hb[4 * InChannels + cp], h5 = hb[5 * InChannels + cp], h6 = hb[6 * InChannels + cp], h7 = hb[7 * InChannels + cp]; + + for (size_t o = 0; o < InChannels; o++) + { + const T wo = Wcol[o]; + a[0][o] += wo * h0; + a[1][o] += wo * h1; + a[2][o] += wo * h2; + a[3][o] += wo * h3; + a[4][o] += wo * h4; + a[5][o] += wo * h5; + a[6][o] += wo * h6; + a[7][o] += wo * h7; + } } } } From c3d7e60f3eeca4c38a4f48d5da1654fb93da3fc0 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 23 Jul 2026 07:18:08 -0700 Subject: [PATCH 132/162] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 89bc066..1c91cb7 100644 --- a/README.md +++ b/README.md @@ -245,11 +245,11 @@ The "ModelTest" application binaries provided in the [Releases section](https:// ```-DBUILD_STATIC_INTERNAL_NAMA2=ON|OFF```: Build internal static A2 implementation. -```-DMULTIFRAME_8X8_CONVOLUTION="0"|"4"||"8"```: Use optimized multiframe 8x8 convolution. Much faster on very modern compilers. Much slower on older compilers. Be sure to use quotes around value. Defaults to "0" (disabled). +```-DMULTIFRAME_8X8_CONVOLUTION=0|4|8```: Use optimized multiframe 8x8 convolution. Much faster on very modern compilers. Much slower on older compilers. Defaults to "0" (disabled). ```-DDEFAULT_QUALITY_SCALE="X.X"```: Default model quality scale factor (0.0 to 1.0). Be sure to use quotes around value. Defaults to "1.0". -```-DDEFAULT_INPUT_DBU="XX"```: Default dBu level for model input calibration. +```-DDEFAULT_INPUT_DBU="XX"```: Default dBu level for model input calibration. Be sure to use quotes around floating point values. Defaults to "12.0". ```-DWAVENET_FRAMES=XXX```: Sample buffer size for the internal WaveNet implementation. Defaults to **64**. If you know you are going to be using a fixed sample buffer smaller or larger than this, use that instead. Note that the model will still be able to process any buffer size - it is just optimized for this size. From 08c0adafdb18bd2c0359349602c3268330cbded7 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 23 Jul 2026 07:38:05 -0700 Subject: [PATCH 133/162] Weight accessors for conv/dense --- NeuralAudio/WaveNet.h | 62 ++++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 7cfd3fa..5e3cd31 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -110,6 +110,21 @@ namespace NeuralAudio } } + std::array, KernelSize>& GetWeights() + { + return weights; + } + + // Avoid allocation for unused bias + using BiasType = typename std::conditional, + Empty>::type; + + BiasType& GetBias() + { + return bias; + } + Conv1DT() { for (int k = 0; k < KernelSize; k++) @@ -267,11 +282,6 @@ namespace NeuralAudio alignas(32) std::array, KernelSize> weights; // consider making this a contiguous block of data instead of block of ChannelBuffers std::array weightPtrs; - // Avoid allocation for unused bias - using BiasType = typename std::conditional, - Empty>::type; - BiasType bias; }; @@ -297,6 +307,21 @@ namespace NeuralAudio } } + ChannelBuffer& GetWeights() + { + return weights; + } + + // Avoid allocation for unused bias + using BiasType = typename std::conditional, + Empty>::type; + + BiasType& GetBias() + { + return bias; + } + void Process(const ChannelRowSpan& input, const ChannelRowSpan& output) const { size_t numFrames = output.GetNumCols(); @@ -348,11 +373,6 @@ namespace NeuralAudio private: ChannelBuffer weights; - - // Avoid allocation for unused bias - using BiasType = typename std::conditional, - Empty>::type; BiasType bias; }; @@ -379,14 +399,21 @@ namespace NeuralAudio conv1D.channelBuffer.AllocBuffer(allocNum); } - void AdvanceFrames(const size_t numFrames) + size_t GetNumWeights() { - conv1D.channelBuffer.AdvanceFrames(numFrames); + return conv1D.GetNumWeights() + inputMixin.GetNumWeights() + oneByOne.GetNumWeights(); } - size_t GetNumWeights() + void SetWeights(std::vector::iterator& weights) { - return conv1D.GetNumWeights() + inputMixin.GetNumWeights() + oneByOne.GetNumWeights(); + conv1D.SetWeights(weights); + inputMixin.SetWeights(weights); + oneByOne.SetWeights(weights); + } + + void AdvanceFrames(const size_t numFrames) + { + conv1D.channelBuffer.AdvanceFrames(numFrames); } auto GetInputBuffer(size_t numFrames) @@ -399,13 +426,6 @@ namespace NeuralAudio conv1D.channelBuffer.CopyBuffer(); } - void SetWeights(std::vector::iterator& weights) - { - conv1D.SetWeights(weights); - inputMixin.SetWeights(weights); - oneByOne.SetWeights(weights); - } - void Process(const ChannelRowSpan& condition, const ChannelRowSpan& headInput, const ChannelRowSpan& output) { size_t numFrames = output.GetNumCols(); From 4dbfe29b206c28271d15abc7a95f9d5abca9a101 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 23 Jul 2026 09:55:11 -0700 Subject: [PATCH 134/162] Getters for WaveNet internal structures --- NeuralAudio/WaveNet.h | 69 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 5e3cd31..61e433c 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -411,6 +411,26 @@ namespace NeuralAudio oneByOne.SetWeights(weights); } + Conv1DT& GetConv1D() + { + return conv1D; + } + + DenseLayerT& GetInputMixin() + { + return inputMixin; + } + + DenseLayerT& GetOneByOne() + { + return oneByOne; + } + + ChannelBuffer& GetState() + { + return state; + } + void AdvanceFrames(const size_t numFrames) { conv1D.channelBuffer.AdvanceFrames(numFrames); @@ -543,6 +563,31 @@ namespace NeuralAudio headRechannel.SetWeights(weights); } + Layers& GetLayers() + { + return layers; + } + + DenseLayerT& GetRechannel() + { + return rechannel; + } + + Conv1DT& GetHeadRechannel() + { + return headRechannel; + } + + ChannelBuffer& GetArrayOutputs() + { + return arrayOutputs; + } + + ChannelBuffer& GetHeadOutputs() + { + return headOutputs; + } + void Prewarm(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) { rechannel.Process(layerInputs, std::get<0>(layers).GetInputBuffer(1)); @@ -605,6 +650,8 @@ namespace NeuralAudio public: int ReceptiveFieldSize = 0; // This should be a static constexpr, but I haven't sorted out the right template magic + static constexpr auto headLayerChannels = std::tuple_element_t<0, std::tuple>::NumChannelsP; + WaveNetModelT() { int allocNum = 0; @@ -657,6 +704,26 @@ namespace NeuralAudio return WAVENET_MAX_NUM_FRAMES; } + auto& GetLayerArrays() + { + return layerArrays; + } + + ChannelBuffer& GetCondition() + { + return condition; + } + + ChannelBuffer& GetHeadArray() + { + return headArray; + } + + T& GetHeadScale() + { + return headScale; + } + void Prewarm() { condition.GetData()[0] = 0; @@ -709,8 +776,6 @@ namespace NeuralAudio } private: - static constexpr auto headLayerChannels = std::tuple_element_t<0, std::tuple>::NumChannelsP; - std::tuple layerArrays; ChannelBuffer condition; ChannelBuffer headArray; From 606d57a57a6534218bb635ca778c85b8fdd3929a Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Thu, 23 Jul 2026 11:33:39 -0700 Subject: [PATCH 135/162] Expose some more internals --- NeuralAudio/WaveNet.h | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 61e433c..50cdd91 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -388,6 +388,8 @@ namespace NeuralAudio public: static constexpr auto ReceptiveFieldSize = (KernelSize - 1) * Dilation; + static constexpr auto KernelSizeP = KernelSize; + static constexpr auto DilationP = Dilation; WaveNetLayerT() { @@ -504,12 +506,11 @@ namespace NeuralAudio DenseLayerT rechannel; Conv1DT headRechannel; - static constexpr auto numLayers = std::tuple_size_v; - static constexpr auto lastLayer = numLayers - 1; - public: static constexpr auto NumChannelsP = Channels; static constexpr auto HeadSizeP = HeadSize; + static constexpr auto NumLayers = std::tuple_size_v; + static constexpr auto LastLayer = NumLayers - 1; ChannelBuffer arrayOutputs; ChannelBuffer headOutputs; @@ -517,7 +518,7 @@ namespace NeuralAudio WaveNetLayerArrayT() { - ForEachIndex([&](auto layerIndex) + ForEachIndex([&](auto layerIndex) { ReceptiveFieldSize += std::get(layers).ReceptiveFieldSize; }); @@ -527,7 +528,7 @@ namespace NeuralAudio int AllocBuffers(int allocNum) { - ForEachIndex([&](auto layerIndex) + ForEachIndex([&](auto layerIndex) { std::get(layers).AllocBuffer(allocNum++); }); @@ -541,7 +542,7 @@ namespace NeuralAudio { size_t numWeights = rechannel.GetNumWeights(); - ForEachIndex([&](auto layerIndex) + ForEachIndex([&](auto layerIndex) { numWeights += std::get(layers).GetNumWeights(); }); @@ -555,7 +556,7 @@ namespace NeuralAudio { rechannel.SetWeights(weights); - ForEachIndex([&](auto layerIndex) + ForEachIndex([&](auto layerIndex) { std::get(layers).SetWeights(weights); }); @@ -592,11 +593,11 @@ namespace NeuralAudio { rechannel.Process(layerInputs, std::get<0>(layers).GetInputBuffer(1)); - ForEachIndex([&](auto layerIndex) + ForEachIndex([&](auto layerIndex) { std::get(layers).CopyBuffer(); - if constexpr (layerIndex == lastLayer) + if constexpr (layerIndex == LastLayer) { std::get(layers).Process(condition, headInputs, arrayOutputs.Slice(1)); } @@ -619,9 +620,9 @@ namespace NeuralAudio rechannel.Process(layerInputs, std::get<0>(layers).GetInputBuffer(numFrames)); - ForEachIndex([&](auto layerIndex) + ForEachIndex([&](auto layerIndex) { - if constexpr (layerIndex == lastLayer) + if constexpr (layerIndex == LastLayer) { std::get(layers).Process(condition, headInputs, arrayOutputs.Slice(numFrames)); } @@ -719,7 +720,7 @@ namespace NeuralAudio return headArray; } - T& GetHeadScale() + T GetHeadScale() { return headScale; } From 3fb8bc36a6e8f35df2c650cb414e951dda20dd72 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 1 Aug 2026 09:26:05 -0700 Subject: [PATCH 136/162] Don't process the unused 1x1 for the last layer of the last layer array --- NeuralAudio/WaveNet.h | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 50cdd91..7c26ca0 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -448,6 +448,7 @@ namespace NeuralAudio conv1D.channelBuffer.CopyBuffer(); } + template void Process(const ChannelRowSpan& condition, const ChannelRowSpan& headInput, const ChannelRowSpan& output) { size_t numFrames = output.GetNumCols(); @@ -471,9 +472,12 @@ namespace NeuralAudio //headInput.AddData(block); - oneByOne.Process(block, output); + if constexpr (NeedOutput) + { + oneByOne.Process(block, output); - output.GetEigenMap().noalias() += conv1D.GetInputBuffer(numFrames).GetEigenMapConst(); + output.GetEigenMap().noalias() += conv1D.GetInputBuffer(numFrames).GetEigenMapConst(); + } //output.AddData(conv1D.GetInputBuffer(numFrames)); } @@ -614,6 +618,7 @@ namespace NeuralAudio headRechannel.Process(headOutputs.Slice(1)); } + template void Process(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) { size_t numFrames = condition.GetNumCols(); @@ -624,7 +629,7 @@ namespace NeuralAudio { if constexpr (layerIndex == LastLayer) { - std::get(layers).Process(condition, headInputs, arrayOutputs.Slice(numFrames)); + std::get(layers).template Process(condition, headInputs, arrayOutputs.Slice(numFrames)); } else { @@ -652,6 +657,8 @@ namespace NeuralAudio int ReceptiveFieldSize = 0; // This should be a static constexpr, but I haven't sorted out the right template magic static constexpr auto headLayerChannels = std::tuple_element_t<0, std::tuple>::NumChannelsP; + static constexpr auto NumLayerArrays = std::tuple_size_v>; + static constexpr auto LastLayerArray = NumLayerArrays - 1; WaveNetModelT() { @@ -762,6 +769,10 @@ namespace NeuralAudio { std::get(layerArrays).Process(conditionSpan, conditionSpan, headArraySpan); } + else if constexpr (layerIndex == LastLayerArray) + { + std::get(layerArrays).template Process(std::get(layerArrays).arrayOutputs.Slice(numFrames), conditionSpan, std::get(layerArrays).headOutputs.Slice(numFrames)); + } else { std::get(layerArrays).Process(std::get(layerArrays).arrayOutputs.Slice(numFrames), conditionSpan, std::get(layerArrays).headOutputs.Slice(numFrames)); From 40724d2ff4806784ca06c3ebea6fe93e3fa1a90a Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 1 Aug 2026 09:34:57 -0700 Subject: [PATCH 137/162] Use half instead of hardcode 4 for 16 channel multiframe conv --- NeuralAudio/WaveNet.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 7c26ca0..c4ea6bb 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -146,7 +146,7 @@ namespace NeuralAudio { // Based on @jfsantos NAM Core implementation - https://github.com/sdatkinson/NeuralAmpModelerCore/pull/277 - constexpr size_t tileSize = (InChannels == 16) ? 4 : MULTIFRAME_8X8_CONVOLUTION; + constexpr size_t tileSize = (InChannels == 16) ? (MULTIFRAME_8X8_CONVOLUTION / 2) : MULTIFRAME_8X8_CONVOLUTION; const size_t nFTile = (numFrames / tileSize) * tileSize; for (size_t f = 0; f < nFTile; f += tileSize) From 56f23e49882292a63c5b59a00109f580ae0342ba Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 1 Aug 2026 09:42:35 -0700 Subject: [PATCH 138/162] Add path for tileSize == 2 --- NeuralAudio/WaveNet.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index c4ea6bb..4558fc4 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -163,7 +163,18 @@ namespace NeuralAudio { const T* __restrict Wcol = W + cp * InChannels; - if constexpr (tileSize == 4) + if constexpr (tileSize == 2) + { + const T h0 = hb[cp], h1 = hb[InChannels + cp]; + + for (size_t o = 0; o < InChannels; o++) + { + const T wo = Wcol[o]; + a[0][o] += wo * h0; + a[1][o] += wo * h1; + } + } + else if constexpr (tileSize == 4) { const T h0 = hb[cp], h1 = hb[InChannels + cp], h2 = hb[2 * InChannels + cp], h3 = hb[3 * InChannels + cp]; From 49100f90603afc83d810a960faf30e8326edc4bc Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sat, 1 Aug 2026 10:19:23 -0700 Subject: [PATCH 139/162] Update CREDITS.md --- CREDITS.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CREDITS.md b/CREDITS.md index dcf2d70..9870641 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -6,8 +6,6 @@ https://github.com/sdatkinson/NeuralAmpModelerCore https://github.com/jatinchowdhury18/RTNeural -https://github.com/jatinchowdhury18/RTNeural-NAM - https://gitlab.com/libeigen/eigen https://github.com/nlohmann/json From a34cc858454fd3305288711ff730a21bc71d622c Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 14 Aug 2026 10:13:09 -0700 Subject: [PATCH 140/162] data access operator for ChannelRowSpan --- NeuralAudio/ChannelBuffer.h | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/NeuralAudio/ChannelBuffer.h b/NeuralAudio/ChannelBuffer.h index 7101aa2..7bc281c 100644 --- a/NeuralAudio/ChannelBuffer.h +++ b/NeuralAudio/ChannelBuffer.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -31,6 +32,9 @@ namespace NeuralAudio return nullptr; } virtual void SetZero() {} + virtual T& operator()(size_t row, size_t col) = 0; + + virtual const T& operator()(size_t row, size_t col) const = 0; }; template @@ -78,12 +82,12 @@ namespace NeuralAudio } } - T& operator()(size_t row, size_t col) + T& operator()(size_t row, size_t col) override { return data[col][row]; } - const T& operator()(size_t row, size_t col) const + const T& operator()(size_t row, size_t col) const override { return data[col][row]; } @@ -121,7 +125,6 @@ namespace NeuralAudio startCol(0), numCols(baseBuffer.GetNumCols()) { - } ChannelRowSpan(ChannelBufferBase* baseBuffer) : @@ -137,7 +140,8 @@ namespace NeuralAudio startCol(startCol), numCols(numCols) { - + assert(startCol >= 0); + assert(numCols <= (baseBuffer->GetNumCols() - startCol)); } ChannelRowSpan(ChannelBufferBase* baseBuffer, size_t numCols) : @@ -145,6 +149,7 @@ namespace NeuralAudio startCol(0), numCols(numCols) { + assert(numCols <= baseBuffer->GetNumCols()); } @@ -173,6 +178,16 @@ namespace NeuralAudio return Channels; } + T& operator()(size_t row, size_t col) + { + return (*buffer)(row, startCol + col); + } + + const T& operator()(size_t row, size_t col) const + { + return (*buffer)(row, startCol + col); + } + T* GetData() const { return buffer->GetData(startCol); From 87ebe0e3e32a9da8cc30c7849cab412903ef7899 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 21 Aug 2026 08:18:58 -0700 Subject: [PATCH 141/162] Add multiframe convolution for 4x4 and 2x2 (faster A1 nano and feather) --- NeuralAudio/WaveNet.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 4558fc4..5f411d0 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -142,7 +142,7 @@ namespace NeuralAudio T* __restrict outputPtr = output.GetData(); #if MULTIFRAME_8X8_CONVOLUTION != 0 - if constexpr ((((InChannels == 8) && (OutChannels == 8)) || ((InChannels == 16) && (OutChannels == 16))) && DoBias) + if constexpr ((((InChannels == 8) && (OutChannels == 8)) || ((InChannels == 16) && (OutChannels == 16)) || ((InChannels == 4) && (OutChannels == 4)) || ((InChannels == 2) && (OutChannels == 2))) && DoBias) { // Based on @jfsantos NAM Core implementation - https://github.com/sdatkinson/NeuralAmpModelerCore/pull/277 From 294cc6cf6e393e482214fbf001f8065049c84d7c Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 21 Aug 2026 08:32:31 -0700 Subject: [PATCH 142/162] Try to skip NAM core submodules --- .gitmodules | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitmodules b/.gitmodules index 6d369bc..d694ae5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,12 @@ [submodule "deps/NeuralAmpModelerCore"] path = deps/NeuralAmpModelerCore url = https://github.com/mikeoliphant/NeuralAmpModelerCore +[submodule "deps/NeuralAmpModelerCore/Dependencies/eigen"] + path = deps/NeuralAmpModelerCore/Dependencies/eigen + update = none +[submodule "deps/NeuralAmpModelerCore/Dependencies/AudioDSPTools"] + path = deps/NeuralAmpModelerCore/Dependencies/AudioDSPTools + update = none [submodule "deps/RTNeural"] path = deps/RTNeural url = https://github.com/mikeoliphant/RTNeural From ea3bfeeaaf0dc8019025703d29ef239df194eeaa Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 21 Aug 2026 08:53:29 -0700 Subject: [PATCH 143/162] another try at excluding sub-submodules --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index d694ae5..666c5dc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,10 +3,10 @@ url = https://github.com/mikeoliphant/NeuralAmpModelerCore [submodule "deps/NeuralAmpModelerCore/Dependencies/eigen"] path = deps/NeuralAmpModelerCore/Dependencies/eigen - update = none + active = false [submodule "deps/NeuralAmpModelerCore/Dependencies/AudioDSPTools"] path = deps/NeuralAmpModelerCore/Dependencies/AudioDSPTools - update = none + active = false [submodule "deps/RTNeural"] path = deps/RTNeural url = https://github.com/mikeoliphant/RTNeural From 59e8a63e77c177989a5d141409f3e9c389272fe7 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 21 Aug 2026 08:59:28 -0700 Subject: [PATCH 144/162] Another try at excluding submodules --- .gitmodules | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitmodules b/.gitmodules index 666c5dc..dbbc568 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,9 @@ +[submodule] + active = . + active = :(exclude)deps/NeuralAmpModelerCore/Dependencies/* [submodule "deps/NeuralAmpModelerCore"] path = deps/NeuralAmpModelerCore url = https://github.com/mikeoliphant/NeuralAmpModelerCore -[submodule "deps/NeuralAmpModelerCore/Dependencies/eigen"] - path = deps/NeuralAmpModelerCore/Dependencies/eigen - active = false -[submodule "deps/NeuralAmpModelerCore/Dependencies/AudioDSPTools"] - path = deps/NeuralAmpModelerCore/Dependencies/AudioDSPTools - active = false [submodule "deps/RTNeural"] path = deps/RTNeural url = https://github.com/mikeoliphant/RTNeural From af695278229db2bfb50ed264afa7b97618888ed5 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 21 Aug 2026 09:08:03 -0700 Subject: [PATCH 145/162] Revert sub-submodule stuff - doesn't work --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index dbbc568..6d369bc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule] - active = . - active = :(exclude)deps/NeuralAmpModelerCore/Dependencies/* [submodule "deps/NeuralAmpModelerCore"] path = deps/NeuralAmpModelerCore url = https://github.com/mikeoliphant/NeuralAmpModelerCore From 9bb47feb5dc833db8a9372c15a99fcaa3b3f48db Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 21 Aug 2026 09:25:51 -0700 Subject: [PATCH 146/162] Update NeuralAmpModelerCore (exclude submodules) --- deps/NeuralAmpModelerCore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index 4c0ee78..50210db 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit 4c0ee78b71abd5eb20aec58562e7540f43caac3b +Subproject commit 50210db7dc3331dbf89d358f866061460d8ad0a2 From e59cd5d473d5b5772c69e755d7c5bc1007cff9ab Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 21 Aug 2026 09:56:06 -0700 Subject: [PATCH 147/162] Update NeuralAmpModelerCore --- deps/NeuralAmpModelerCore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index 50210db..896c5f1 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit 50210db7dc3331dbf89d358f866061460d8ad0a2 +Subproject commit 896c5f17f0fd00c0af056e18a4fe66f19cfbeb52 From 9646564a11ec2cd3be8048a873579d545fc8529d Mon Sep 17 00:00:00 2001 From: fer <82901154+fragfz@users.noreply.github.com> Date: Tue, 1 Sep 2026 02:03:44 +0200 Subject: [PATCH 148/162] Add NAM_A2_RING_MODE option to CMakeLists.txt (#68) --- NeuralAudio/CMakeLists.txt | 3 +++ README.md | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index 4626e7a..e7197e8 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -31,6 +31,9 @@ option(NAM_ENABLE_A2_FAST "Use NAM A2 fast-path WaveNet" ON) if(NAM_ENABLE_A2_FAST) message(STATUS "Using NAM A2 fast-path WaveNet") add_definitions(-DNAM_ENABLE_A2_FAST) + set(NAM_A2_RING_MODE "1" CACHE STRING "Ring-buffer strategy") + add_definitions(-DNAM_A2_RING_MODE=${NAM_A2_RING_MODE}) + message(STATUS "Ring-buffer strategy is: ${NAM_A2_RING_MODE}") else() message(STATUS "NOT using NAM A2 fast-path WaveNet") endif() diff --git a/README.md b/README.md index 1c91cb7..54c4fd5 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,11 @@ The "ModelTest" application binaries provided in the [Releases section](https:// ```-DNAM_ENABLE_A2_FAST=ON|OFF```: Enable use of A2 fast path wavenet in NAM Core. +```-DNAM_A2_RING_MODE=0|1```: Ring-buffer strategy for A2 FAST. Defaults to "1" + + - ```0```: linear memmove-rewind (variable worst-case latency, sporadic spikes) + - ```1```(the default): pow2 + tail mirror (constant per-block work, branchless reads) + ```-DBUILD_STATIC_RTNEURAL=ON|OFF```: Build static RTNeural model architectures (slower compile, larger size - only use if you plan on forcing RTNeural model loading). ```-DBUILD_INTERNAL_STATIC_WAVENET=ON|OFF```: Build internal static WaveNet model architectures (faster internal WaveNet, but slower compile, larger size). From 9f5be0a1bb9646bc87b7205101da54a276e334d5 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 1 Sep 2026 06:42:56 -0700 Subject: [PATCH 149/162] Update README.md --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 54c4fd5..52416a4 100644 --- a/README.md +++ b/README.md @@ -231,19 +231,6 @@ The "ModelTest" application binaries provided in the [Releases section](https:// ## CMake Options -```-DBUILD_NAMCORE=ON|OFF```: Support loading models using the NAM Core implemenations. - -```-DNAM_USE_INLINE_GEMM=ON|OFF```: Enable use of inline matrix multiplication in NAM Core. - -```-DNAM_ENABLE_A2_FAST=ON|OFF```: Enable use of A2 fast path wavenet in NAM Core. - -```-DNAM_A2_RING_MODE=0|1```: Ring-buffer strategy for A2 FAST. Defaults to "1" - - - ```0```: linear memmove-rewind (variable worst-case latency, sporadic spikes) - - ```1```(the default): pow2 + tail mirror (constant per-block work, branchless reads) - -```-DBUILD_STATIC_RTNEURAL=ON|OFF```: Build static RTNeural model architectures (slower compile, larger size - only use if you plan on forcing RTNeural model loading). - ```-DBUILD_INTERNAL_STATIC_WAVENET=ON|OFF```: Build internal static WaveNet model architectures (faster internal WaveNet, but slower compile, larger size). ```-DBUILD_INTERNAL_STATIC_LSTM=ON|OFF```: Build internal static LSTM model architectures (faster internal LSTM, but slower compile, larger size). @@ -269,6 +256,19 @@ The "ModelTest" application binaries provided in the [Releases section](https:// ```-DBUILD_UTILS=ON|OFF```: Build performance/accuracy testing tools (located in the "Utils" folder). +```-DBUILD_NAMCORE=ON|OFF```: Support loading models using the NAM Core implemenations. + +```-DNAM_USE_INLINE_GEMM=ON|OFF```: Enable use of inline matrix multiplication in NAM Core. + +```-DNAM_ENABLE_A2_FAST=ON|OFF```: Enable use of A2 fast path wavenet in NAM Core. + +```-DNAM_A2_RING_MODE=0|1```: Ring-buffer strategy for A2 FAST. Defaults to "1" + + - ```0```: linear memmove-rewind (variable worst-case latency, sporadic spikes) + - ```1```(the default): pow2 + tail mirror (constant per-block work, branchless reads) + +```-DBUILD_STATIC_RTNEURAL=ON|OFF```: Build static RTNeural model architectures (slower compile, larger size - only use if you plan on forcing RTNeural model loading). + # Software/Hardware Using NeuralAudio The following applications and devices are using the NeuralAudio library for model processing: From 6db84454dbc3efbff5f3dd979a71010464c55544 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 1 Sep 2026 06:57:02 -0700 Subject: [PATCH 150/162] Rename Dense In/OutSize to In/OutChannels --- NeuralAudio/WaveNet.h | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 4558fc4..ba4a5bb 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -296,36 +296,36 @@ namespace NeuralAudio BiasType bias; }; - template + template class DenseLayerT { public: size_t GetNumWeights() { - return OutSize * InSize + (DoBias ? OutSize : 0); + return OutChannels * InChannels + (DoBias ? OutChannels : 0); } void SetWeights(std::vector::iterator& inWeights) { - for (size_t i = 0; i < OutSize; i++) - for (size_t j = 0; j < InSize; j++) + for (size_t i = 0; i < OutChannels; i++) + for (size_t j = 0; j < InChannels; j++) weights(i, j) = *(inWeights++); if constexpr (DoBias) { - for (size_t i = 0; i < OutSize; i++) + for (size_t i = 0; i < OutChannels; i++) bias(i) = *(inWeights++); } } - ChannelBuffer& GetWeights() + ChannelBuffer& GetWeights() { return weights; } // Avoid allocation for unused bias using BiasType = typename std::conditional, + Eigen::Vector, Empty>::type; BiasType& GetBias() @@ -333,19 +333,19 @@ namespace NeuralAudio return bias; } - void Process(const ChannelRowSpan& input, const ChannelRowSpan& output) const + void Process(const ChannelRowSpan& input, const ChannelRowSpan& output) const { size_t numFrames = output.GetNumCols(); - if constexpr (MatMul::HasKernel()) + if constexpr (MatMul::HasKernel()) { if constexpr (DoBias) { - MatMul::MultiplyInitColwise(input.GetDataConst(), output.GetData(), weights.GetDataConst(), bias.data(), numFrames); + MatMul::MultiplyInitColwise(input.GetDataConst(), output.GetData(), weights.GetDataConst(), bias.data(), numFrames); } else { - MatMul::MultiplyInitZero(input.GetDataConst(), output.GetData(), weights.GetDataConst(), numFrames); + MatMul::MultiplyInitZero(input.GetDataConst(), output.GetData(), weights.GetDataConst(), numFrames); } } else @@ -361,13 +361,13 @@ namespace NeuralAudio } } - void ProcessAcc(const ChannelRowSpan& input, const ChannelRowSpan& output) const + void ProcessAcc(const ChannelRowSpan& input, const ChannelRowSpan& output) const { size_t numFrames = output.GetNumCols(); - if constexpr (!DoBias && MatMul::HasKernel()) + if constexpr (!DoBias && MatMul::HasKernel()) { - MatMul::MultiplyAccumlulate(input.GetDataConst(), output.GetData(), weights.GetDataConst(), numFrames); + MatMul::MultiplyAccumlulate(input.GetDataConst(), output.GetData(), weights.GetDataConst(), numFrames); } else { @@ -383,7 +383,7 @@ namespace NeuralAudio } private: - ChannelBuffer weights; + ChannelBuffer weights; BiasType bias; }; From 3b36251128f06a714d9ca8cfcbc12a42ef93a59e Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 1 Sep 2026 06:58:14 -0700 Subject: [PATCH 151/162] Add transposed Eigen maps. Added ChannelBufferDynamic. --- NeuralAudio/ChannelBuffer.h | 113 +++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 1 deletion(-) diff --git a/NeuralAudio/ChannelBuffer.h b/NeuralAudio/ChannelBuffer.h index 7bc281c..055bb16 100644 --- a/NeuralAudio/ChannelBuffer.h +++ b/NeuralAudio/ChannelBuffer.h @@ -112,10 +112,111 @@ namespace NeuralAudio return Eigen::Map>(GetDataConst(), Channels, GetNumCols()); } - private: + Eigen::Map> GetEigenMapTransposed() + { + return Eigen::Map>(GetData(), Channels, GetNumCols()); + } + + const Eigen::Map> GetEigenMapTransposedConst() const + { + return Eigen::Map>(GetDataConst(), Channels, GetNumCols()); + } + + private: alignas(32) std::array, Cols> data; }; + + template + class ChannelBufferDynamic : public ChannelBufferBase + { + public: + ChannelBufferDynamic(T* data, size_t numCols) : + data(data), + numCols(numCols) + { + } + + size_t GetSize() const override + { + return (size_t)(Channels * numCols); + } + + size_t GetNumCols() const override + { + return numCols; + } + + T* GetData() override + { + return data; + } + + const T* GetDataConst() const override + { + return data; + } + + T* GetData(size_t startCol) override + { + return data + (startCol * Channels); + } + + const T* GetDataConst(size_t startCol) const override + { + return data + (startCol * Channels); + } + + void SetZero() override + { + std::fill(data, data + GetSize(), 0); + } + + T& operator()(size_t row, size_t col) override + { + return data[(col * Channels) + row]; + } + + const T& operator()(size_t row, size_t col) const override + { + return data[(col * Channels) + row]; + } + + const ChannelRowSpan Slice(size_t startCol, size_t numCols) + { + return ChannelRowSpan(this, startCol, numCols); + } + + const ChannelRowSpan Slice(size_t numCols) + { + return ChannelRowSpan(this, numCols); + } + + const Eigen::Map> GetEigenMap() + { + return Eigen::Map>(GetData(), Channels, GetNumCols()); + } + + const Eigen::Map> GetEigenMapConst() const + { + return Eigen::Map>(GetDataConst(), Channels, GetNumCols()); + } + + Eigen::Map> GetEigenMapTransposed() + { + return Eigen::Map>(GetData(), Channels, GetNumCols()); + } + + const Eigen::Map> GetEigenMapTransposedConst() const + { + return Eigen::Map>(GetDataConst(), Channels, GetNumCols()); + } + + private: + T* data; + size_t numCols; + }; + template class ChannelRowSpan { @@ -218,6 +319,16 @@ namespace NeuralAudio return Eigen::Map>(GetDataConst(), Channels, numCols); } + Eigen::Map> GetEigenMapTransposed() + { + return Eigen::Map>(GetData(), Channels, numCols); + } + + const Eigen::Map> GetEigenMapTransposedConst() const + { + return Eigen::Map>(GetDataConst(), Channels, numCols); + } + void CopyData(const ChannelRowSpan& srcSpan) const { const T* srcPtr = srcSpan.GetDataConst(); From 6b87ef3fa76f54a8678a610fc8bb3e39fd050e0c Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 1 Sep 2026 07:07:12 -0700 Subject: [PATCH 152/162] Add SetZero for ChannelRowSpan. Base override for cols. --- NeuralAudio/ChannelBuffer.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/NeuralAudio/ChannelBuffer.h b/NeuralAudio/ChannelBuffer.h index 055bb16..df5fa62 100644 --- a/NeuralAudio/ChannelBuffer.h +++ b/NeuralAudio/ChannelBuffer.h @@ -32,6 +32,7 @@ namespace NeuralAudio return nullptr; } virtual void SetZero() {} + virtual void SetZero(size_t startCol, size_t numCols) {} virtual T& operator()(size_t row, size_t col) = 0; virtual const T& operator()(size_t row, size_t col) const = 0; @@ -82,6 +83,12 @@ namespace NeuralAudio } } + void SetZero(size_t startCol, size_t numCols) override + { + for (size_t col = 0; col < numCols; col++) + data[startCol + col].fill(0); + } + T& operator()(size_t row, size_t col) override { return data[col][row]; @@ -172,6 +179,11 @@ namespace NeuralAudio std::fill(data, data + GetSize(), 0); } + void SetZero(size_t startCol, size_t numCols) override + { + std::fill(data + (startCol * this->numCols), data + ((startCol + numCols) * this->numCols), 0); + } + T& operator()(size_t row, size_t col) override { return data[(col * Channels) + row]; @@ -279,6 +291,11 @@ namespace NeuralAudio return Channels; } + void SetZero() const + { + buffer->SetZero(startCol, numCols); + } + T& operator()(size_t row, size_t col) { return (*buffer)(row, startCol + col); From 0f02753fc2ae75a16aefc310832f2af0c8d8d160 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 1 Sep 2026 07:38:10 -0700 Subject: [PATCH 153/162] Fix SetZero --- NeuralAudio/ChannelBuffer.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/NeuralAudio/ChannelBuffer.h b/NeuralAudio/ChannelBuffer.h index df5fa62..41b1569 100644 --- a/NeuralAudio/ChannelBuffer.h +++ b/NeuralAudio/ChannelBuffer.h @@ -32,7 +32,7 @@ namespace NeuralAudio return nullptr; } virtual void SetZero() {} - virtual void SetZero(size_t startCol, size_t numCols) {} + virtual void SetZero(size_t startCol, size_t numCols) = 0; virtual T& operator()(size_t row, size_t col) = 0; virtual const T& operator()(size_t row, size_t col) const = 0; @@ -138,6 +138,12 @@ namespace NeuralAudio class ChannelBufferDynamic : public ChannelBufferBase { public: + ChannelBufferDynamic() + { + numCols = 0; + data = nullptr; + } + ChannelBufferDynamic(T* data, size_t numCols) : data(data), numCols(numCols) @@ -181,7 +187,7 @@ namespace NeuralAudio void SetZero(size_t startCol, size_t numCols) override { - std::fill(data + (startCol * this->numCols), data + ((startCol + numCols) * this->numCols), 0); + std::fill(data + (startCol * Channels), data + ((startCol + numCols) * Channels), 0); } T& operator()(size_t row, size_t col) override @@ -263,7 +269,6 @@ namespace NeuralAudio numCols(numCols) { assert(numCols <= baseBuffer->GetNumCols()); - } const ChannelRowSpan Slice(size_t startCol, size_t numCols) const From e24943164651f726b874939de781a97e1d3fb9a7 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 4 Sep 2026 15:42:49 -0700 Subject: [PATCH 154/162] Use stack allocation for WaveNet layer state block --- NeuralAudio/WaveNet.h | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 5f411d0..b0e3b05 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -395,7 +395,6 @@ namespace NeuralAudio Conv1DT conv1D; DenseLayerT inputMixin; DenseLayerT oneByOne; - ChannelBuffer state; public: static constexpr auto ReceptiveFieldSize = (KernelSize - 1) * Dilation; @@ -404,7 +403,6 @@ namespace NeuralAudio WaveNetLayerT() { - state.SetZero(); } void AllocBuffer(int allocNum) @@ -439,11 +437,6 @@ namespace NeuralAudio return oneByOne; } - ChannelBuffer& GetState() - { - return state; - } - void AdvanceFrames(const size_t numFrames) { conv1D.channelBuffer.AdvanceFrames(numFrames); @@ -462,6 +455,8 @@ namespace NeuralAudio template void Process(const ChannelRowSpan& condition, const ChannelRowSpan& headInput, const ChannelRowSpan& output) { + ChannelBuffer state; + size_t numFrames = output.GetNumCols(); auto block = state.Slice(numFrames); From 204993a7df4e7eafca4ac5c4675f25140f006535 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 4 Sep 2026 15:59:12 -0700 Subject: [PATCH 155/162] Remove accessors --- NeuralAudio/WaveNet.h | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 42b4dd8..c69f156 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -589,16 +589,6 @@ namespace NeuralAudio return headRechannel; } - ChannelBuffer& GetArrayOutputs() - { - return arrayOutputs; - } - - ChannelBuffer& GetHeadOutputs() - { - return headOutputs; - } - void Prewarm(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) { rechannel.Process(layerInputs, std::get<0>(layers).GetInputBuffer(1)); @@ -723,16 +713,6 @@ namespace NeuralAudio return layerArrays; } - ChannelBuffer& GetCondition() - { - return condition; - } - - ChannelBuffer& GetHeadArray() - { - return headArray; - } - T GetHeadScale() { return headScale; @@ -740,30 +720,31 @@ namespace NeuralAudio void Prewarm() { + ChannelBuffer condition; condition.GetData()[0] = 0; + ChannelBuffer headArray; headArray.SetZero(); - auto conditionSpan = condition.Slice(1); - auto headArraySpan = headArray.Slice(1); - ForEachIndex([&](auto layerIndex) { if constexpr (layerIndex == 0) { - std::get(layerArrays).Prewarm(conditionSpan, conditionSpan, headArraySpan); + std::get(layerArrays).Prewarm(condition, condition, headArray); } else { - std::get(layerArrays).Prewarm(std::get(layerArrays).arrayOutputs.Slice(1), conditionSpan, std::get(layerArrays).headOutputs.Slice(1)); + std::get(layerArrays).Prewarm(std::get(layerArrays).arrayOutputs.Slice(1), condition, std::get(layerArrays).headOutputs.Slice(1)); } }); } void Process(const T* input, T* output, const size_t numFrames) { + ChannelBuffer condition; std::memcpy(condition.GetData(), input, numFrames * sizeof(T)); + ChannelBuffer headArray; headArray.SetZero(); auto conditionSpan = condition.Slice(numFrames); @@ -795,8 +776,6 @@ namespace NeuralAudio private: std::tuple layerArrays; - ChannelBuffer condition; - ChannelBuffer headArray; T headScale; }; } From 574c8df0b8d1abbcab0abd37451a6de7b95cc9e5 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Mon, 7 Sep 2026 10:23:38 -0700 Subject: [PATCH 156/162] define for alignas --- NeuralAudio/ChannelBuffer.h | 4 +++- NeuralAudio/WaveNet.h | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/NeuralAudio/ChannelBuffer.h b/NeuralAudio/ChannelBuffer.h index 41b1569..7e2ffcd 100644 --- a/NeuralAudio/ChannelBuffer.h +++ b/NeuralAudio/ChannelBuffer.h @@ -5,6 +5,8 @@ #include #include +constexpr size_t CHANNEL_BUFFER_ALIGN = 32; + namespace NeuralAudio { template @@ -130,7 +132,7 @@ namespace NeuralAudio } private: - alignas(32) std::array, Cols> data; + alignas(CHANNEL_BUFFER_ALIGN) std::array, Cols> data; }; diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 3c00735..751fd2b 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -151,7 +151,7 @@ namespace NeuralAudio for (size_t f = 0; f < nFTile; f += tileSize) { - alignas(32) T a[tileSize][InChannels]{}; + alignas(CHANNEL_BUFFER_ALIGN) T a[tileSize][InChannels]{}; for (size_t k = 0; k < KernelSize; k++) { @@ -290,7 +290,7 @@ namespace NeuralAudio } private: - alignas(32) std::array, KernelSize> weights; // consider making this a contiguous block of data instead of block of ChannelBuffers + alignas(CHANNEL_BUFFER_ALIGN) std::array, KernelSize> weights; // consider making this a contiguous block of data instead of block of ChannelBuffers std::array weightPtrs; BiasType bias; From c5275bdb39db5fa7f547f50535ac75cc45aa1185 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 8 Sep 2026 13:05:04 -0700 Subject: [PATCH 157/162] remove RTNeural-NAM source group --- NeuralAudio/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index e7197e8..a43ff97 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -192,4 +192,4 @@ target_link_libraries(NeuralAudio LINK_PUBLIC RTNeural math_approx) source_group(NeuralAudio ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES}) source_group(NAM ${CMAKE_CURRENT_SOURCE_DIR} FILES ${NAM_SOURCES}) -source_group(RTNeural-NAM ${CMAKE_CURRENT_SOURCE_DIR} FILES ${RTNEURAL_WN_SOURCES}) + From f3b6a9892f92ba93e14a3174052c23aa3725251d Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Sun, 13 Sep 2026 12:23:27 -0700 Subject: [PATCH 158/162] Make RTNeural completely optional --- NeuralAudio/CMakeLists.txt | 27 +++++--- NeuralAudio/NeuralModel.cpp | 4 ++ NeuralAudio/RTNeuralLoader.cpp | 112 ++++++++++++++++----------------- NeuralAudio/RTNeuralLoader.h | 8 ++- NeuralAudio/RTNeuralModel.h | 3 + 5 files changed, 88 insertions(+), 66 deletions(-) diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index a43ff97..452789b 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -3,12 +3,20 @@ set(CMAKE_CXX_STANDARD 20) add_definitions(-DNAM_SAMPLE_FLOAT) add_definitions(-DDSP_SAMPLE_FLOAT) -option(BUILD_STATIC_RTNEURAL "Build Static RTNeural" OFF) -if(BUILD_STATIC_RTNEURAL) - message(STATUS "Building static RTNeural models") - add_definitions(-DBUILD_STATIC_RTNEURAL) +option(BUILD_RTNEURAL "Build RTNeural" ON) +if(BUILD_RTNEURAL) + message(STATUS "Building RTNeural models") + add_definitions(-DBUILD_RTNEURAL) + + option(BUILD_STATIC_RTNEURAL "Build Static RTNeural" OFF) + if(BUILD_STATIC_RTNEURAL) + message(STATUS "Building static RTNeural models") + add_definitions(-DBUILD_STATIC_RTNEURAL) + else() + message(STATUS "NOT Building static RTNeural models") + endif() else() - message(STATUS "NOT Building static RTNeural models") + message(STATUS "NOT Building RTNeural models") endif() option(BUILD_NAMCORE "Build NAM Core" ON) @@ -180,15 +188,18 @@ target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/NeuralAmpModelerCo target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/RTNeural) target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/math_approx) target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/RTNeural/modules/Eigen) -target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/RTNeural/modules/json) +target_include_directories(NeuralAudio PUBLIC ../deps/RTNeural/modules/json) set_property(TARGET NeuralAudio PROPERTY POSITION_INDEPENDENT_CODE ON) set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "" FORCE) -add_subdirectory(../deps/RTNeural RTNeural) +if(BUILD_RTNEURAL) + add_subdirectory(../deps/RTNeural RTNeural) + target_link_libraries(NeuralAudio LINK_PUBLIC RTNeural) +endif() add_subdirectory(../deps/math_approx math_approx) -target_link_libraries(NeuralAudio LINK_PUBLIC RTNeural math_approx) +target_link_libraries(NeuralAudio LINK_PUBLIC math_approx) source_group(NeuralAudio ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES}) source_group(NAM ${CMAKE_CURRENT_SOURCE_DIR} FILES ${NAM_SOURCES}) diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 04d21b4..074ad3a 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -3,10 +3,12 @@ #ifdef BUILD_NAMCORE #include "NAMModel.h" #endif +#ifdef BUILD_RTNEURAL #ifdef BUILD_STATIC_RTNEURAL #include "RTNeuralLoader.h" #endif #include "RTNeuralModel.h" +#endif #include "InternalModel.h" #include "CompositeModel.h" @@ -562,6 +564,7 @@ namespace NeuralAudio } } +#ifdef BUILD_RTNEURAL if (newModel == nullptr) { // Use a dynamic model for other model types @@ -570,6 +573,7 @@ namespace NeuralAudio newModel = model; } +#endif } if ((newModel != nullptr) && doPrewarm) diff --git a/NeuralAudio/RTNeuralLoader.cpp b/NeuralAudio/RTNeuralLoader.cpp index 4b3f637..1ade6b3 100644 --- a/NeuralAudio/RTNeuralLoader.cpp +++ b/NeuralAudio/RTNeuralLoader.cpp @@ -1,81 +1,81 @@ +#ifdef BUILD_STATIC_RTNEURAL + #include #include "RTNeuralModel.h" namespace NeuralAudio { -#ifdef BUILD_STATIC_RTNEURAL - std::list rtNeuralLSTMModelDefs; - - void EnsureRTNeuralModelDefsAreLoaded() + std::list rtNeuralLSTMModelDefs; + + void EnsureRTNeuralModelDefsAreLoaded() + { + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 8>); + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 12>); + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 16>); + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 24>); + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 8>); + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 12>); + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 16>); + } + + RTNeuralLSTMDefinitionBase* FindRTNeuralLSTMDefinition(size_t numLayers, size_t hiddenSize) + { + for (auto const& model : rtNeuralLSTMModelDefs) { - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 8>); - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 12>); - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 16>); - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 24>); - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 8>); - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 12>); - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 16>); + if ((numLayers == model->GetNumLayers()) && (hiddenSize == model->GetHiddenSize())) + return model; } - RTNeuralLSTMDefinitionBase* FindRTNeuralLSTMDefinition(size_t numLayers, size_t hiddenSize) - { - for (auto const& model : rtNeuralLSTMModelDefs) - { - if ((numLayers == model->GetNumLayers()) && (hiddenSize == model->GetHiddenSize())) - return model; - } + return nullptr; + } - return nullptr; - } + NeuralModelImpl* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader *loader) + { + auto& config = modelJson.at("config"); + + auto modelDef = FindRTNeuralLSTMDefinition(config.at("num_layers"), config.at("hidden_size")); - NeuralModelImpl* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader *loader) + if (modelDef != nullptr) { - auto& config = modelJson.at("config"); + RTNeuralModel* model = modelDef->CreateModel(); - auto modelDef = FindRTNeuralLSTMDefinition(config.at("num_layers"), config.at("hidden_size")); + model->SetModelLoader(loader); + model->LoadFromNAMJson(modelJson); - if (modelDef != nullptr) - { - RTNeuralModel* model = modelDef->CreateModel(); + return model; + } - model->SetModelLoader(loader); - model->LoadFromNAMJson(modelJson); + // If we didn't have a static model that matched, use RTNeural's dynamic model + RTNeuralModelDyn* dynModel = new RTNeuralModelDyn; - return model; - } + dynModel->SetModelLoader(loader); + dynModel->LoadFromNAMJson(modelJson); - // If we didn't have a static model that matched, use RTNeural's dynamic model - RTNeuralModelDyn* dynModel = new RTNeuralModelDyn; + return dynModel; - dynModel->SetModelLoader(loader); - dynModel->LoadFromNAMJson(modelJson); + return nullptr; + } - return dynModel; + NeuralModelImpl* RTNeuralLoadKeras(const nlohmann::json& modelJson, NeuralModelLoader* loader) + { + const auto layers = modelJson.at("layers"); + const size_t numLayers = layers.size() - 1; + const std::string modelType = layers.at(0).at("type"); + const int hiddenSize = layers.at(0).at("shape").back(); - return nullptr; - } + auto modelDef = FindRTNeuralLSTMDefinition(numLayers, hiddenSize); - NeuralModelImpl* RTNeuralLoadKeras(const nlohmann::json& modelJson, NeuralModelLoader* loader) + if (modelDef != nullptr) { - const auto layers = modelJson.at("layers"); - const size_t numLayers = layers.size() - 1; - const std::string modelType = layers.at(0).at("type"); - const int hiddenSize = layers.at(0).at("shape").back(); - - auto modelDef = FindRTNeuralLSTMDefinition(numLayers, hiddenSize); - - if (modelDef != nullptr) - { - RTNeuralModel* model = modelDef->CreateModel(); + RTNeuralModel* model = modelDef->CreateModel(); - model->SetModelLoader(loader); - model->LoadFromKerasJson(modelJson); - - return model; - } + model->SetModelLoader(loader); + model->LoadFromKerasJson(modelJson); - return nullptr; + return model; } -#endif -} + return nullptr; + } +} +#endif diff --git a/NeuralAudio/RTNeuralLoader.h b/NeuralAudio/RTNeuralLoader.h index 72f0d21..f8b0156 100644 --- a/NeuralAudio/RTNeuralLoader.h +++ b/NeuralAudio/RTNeuralLoader.h @@ -1,12 +1,16 @@ #pragma once +#ifdef BUILD_RTNEURAL + #include "NeuralModel.h" #include "NeuralModelImpl.h" namespace NeuralAudio { +#ifdef BUILD_STATIC_RTNEURAL extern void EnsureRTNeuralModelDefsAreLoaded(); - extern NeuralModelImpl* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson, NeuralModelLoader* loader); extern NeuralModelImpl* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader* loader); +#endif extern NeuralModelImpl* RTNeuralLoadKeras(const nlohmann::json& modelJson, NeuralModelLoader* loader); -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/NeuralAudio/RTNeuralModel.h b/NeuralAudio/RTNeuralModel.h index 899a49f..855657a 100644 --- a/NeuralAudio/RTNeuralModel.h +++ b/NeuralAudio/RTNeuralModel.h @@ -1,5 +1,7 @@ #pragma once +#ifdef BUILD_RTNEURAL + #include "NeuralModel.h" #include "NeuralModelImpl.h" #include @@ -432,3 +434,4 @@ namespace NeuralAudio std::unique_ptr> model; }; } +#endif \ No newline at end of file From 8fd7137e9752fc8d6088864f37539a1e7780959b Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 15 Sep 2026 11:03:14 -0700 Subject: [PATCH 159/162] compiler fix. headRechannel -> oneByOne --- NeuralAudio/NeuralModel.cpp | 1 + NeuralAudio/WaveNet.h | 24 ++++++++++++------------ NeuralAudio/WaveNetDynamic.h | 10 +++++----- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 074ad3a..54331e7 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -1,4 +1,5 @@ #include +#include #include "NeuralModel.h" #ifdef BUILD_NAMCORE #include "NAMModel.h" diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 19142b8..65c1796 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -514,7 +514,7 @@ namespace NeuralAudio private: Layers layers; DenseLayerT rechannel; - Conv1DT headRechannel; + Conv1DT oneByOne; public: static constexpr auto NumChannelsP = Channels; @@ -533,7 +533,7 @@ namespace NeuralAudio ReceptiveFieldSize += std::get(layers).ReceptiveFieldSize; }); - ReceptiveFieldSize += headRechannel.ReceptiveFieldSize; + ReceptiveFieldSize += oneByOne.ReceptiveFieldSize; } int AllocBuffers(int allocNum) @@ -543,7 +543,7 @@ namespace NeuralAudio std::get(layers).AllocBuffer(allocNum++); }); - headRechannel.channelBuffer.AllocBuffer(allocNum++); + oneByOne.channelBuffer.AllocBuffer(allocNum++); return allocNum; } @@ -557,7 +557,7 @@ namespace NeuralAudio numWeights += std::get(layers).GetNumWeights(); }); - numWeights += headRechannel.GetNumWeights(); + numWeights += oneByOne.GetNumWeights(); return numWeights; } @@ -571,7 +571,7 @@ namespace NeuralAudio std::get(layers).SetWeights(weights); }); - headRechannel.SetWeights(weights); + oneByOne.SetWeights(weights); } Layers& GetLayers() @@ -586,7 +586,7 @@ namespace NeuralAudio Conv1DT& GetHeadRechannel() { - return headRechannel; + return oneByOne; } void Prewarm(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) @@ -609,9 +609,9 @@ namespace NeuralAudio //headRechannel.channelBuffer.buffer.middleCols(headRechannel.channelBuffer.bufferStart, 1).noalias() = headInputs.leftCols(1); // Should be able to avoid this copy - headRechannel.GetInputBuffer(1).CopyData(headInputs.Slice(1)); - headRechannel.channelBuffer.CopyBuffer(); - headRechannel.Process(headOutputs.Slice(1)); + oneByOne.GetInputBuffer(1).CopyData(headInputs.Slice(1)); + oneByOne.channelBuffer.CopyBuffer(); + oneByOne.Process(headOutputs.Slice(1)); } template @@ -640,9 +640,9 @@ namespace NeuralAudio //headRechannel.GetInputBuffer(numFrames).CopyData(headInputs); - headRechannel.GetInputBuffer(numFrames).GetEigenMap().noalias() = headInputs.GetEigenMapConst(); - headRechannel.Process(headOutputs.Slice(numFrames)); - headRechannel.channelBuffer.AdvanceFrames(numFrames); + oneByOne.GetInputBuffer(numFrames).GetEigenMap().noalias() = headInputs.GetEigenMapConst(); + oneByOne.Process(headOutputs.Slice(numFrames)); + oneByOne.channelBuffer.AdvanceFrames(numFrames); } }; diff --git a/NeuralAudio/WaveNetDynamic.h b/NeuralAudio/WaveNetDynamic.h index b16f548..5b850b2 100644 --- a/NeuralAudio/WaveNetDynamic.h +++ b/NeuralAudio/WaveNetDynamic.h @@ -260,7 +260,7 @@ namespace NeuralAudio size_t channels; std::vector layers; DenseLayer rechannel; - DenseLayer headRechannel; + DenseLayer oneByOne; size_t lastLayer; Eigen::MatrixXf arrayOutputs; Eigen::MatrixXf headOutputs; @@ -270,7 +270,7 @@ namespace NeuralAudio WaveNetLayerArray(size_t inputSize, size_t conditionSize, size_t HeadSize, size_t channels, size_t kernelSize, bool hasHeadBias, std::vector dilations) : channels(channels), rechannel(inputSize, channels, false), - headRechannel(channels, HeadSize, hasHeadBias), + oneByOne(channels, HeadSize, hasHeadBias), arrayOutputs(channels, WAVENET_MAX_NUM_FRAMES), headOutputs(HeadSize, WAVENET_MAX_NUM_FRAMES) { @@ -324,7 +324,7 @@ namespace NeuralAudio layer.SetWeights(weights); } - headRechannel.SetWeights(weights); + oneByOne.SetWeights(weights); } void Prewarm(const Eigen::MatrixXf& layerInputs, const Eigen::MatrixXf& condition, Eigen::Ref const& headInputs) @@ -345,7 +345,7 @@ namespace NeuralAudio } } - headRechannel.Process(headInputs, headOutputs.leftCols(1)); + oneByOne.Process(headInputs, headOutputs.leftCols(1)); } void Process(const Eigen::MatrixXf& layerInputs, const Eigen::MatrixXf& condition, Eigen::Ref headInputs, const size_t numFrames) @@ -364,7 +364,7 @@ namespace NeuralAudio } } - headRechannel.Process(headInputs, headOutputs.leftCols(numFrames)); + oneByOne.Process(headInputs, headOutputs.leftCols(numFrames)); } }; From a34cd7858ee68aa66b23f440be48d9d7c8779000 Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 29 Sep 2026 07:34:48 -0700 Subject: [PATCH 160/162] Use const for Process() input. const cast workaround for NAM Core process. --- NeuralAudio/CompositeModel.h | 2 +- NeuralAudio/InternalModel.h | 8 ++++---- NeuralAudio/NAMModel.h | 4 ++-- NeuralAudio/NeuralModel.h | 2 +- NeuralAudio/RTNeuralModel.h | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/NeuralAudio/CompositeModel.h b/NeuralAudio/CompositeModel.h index 7a54de7..0b68765 100644 --- a/NeuralAudio/CompositeModel.h +++ b/NeuralAudio/CompositeModel.h @@ -91,7 +91,7 @@ namespace NeuralAudio } } - void Process(float* input, float* output, size_t numSamples) override + void Process(const float* input, float* output, size_t numSamples) override { if (currentModelIndex == -1) return; diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index 3e0444a..69cc409 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -101,7 +101,7 @@ namespace NeuralAudio return model->ReceptiveFieldSize; } - void Process(float* input, float* output, size_t numSamples) override + void Process(const float* input, float* output, size_t numSamples) override { size_t offset = 0; @@ -223,7 +223,7 @@ namespace NeuralAudio model->SetMaxFrames(maxSize); } - void Process(float* input, float* output, size_t numSamples) override + void Process(const float* input, float* output, size_t numSamples) override { size_t offset = 0; @@ -360,7 +360,7 @@ namespace NeuralAudio (void)maxSize; } - void Process(float* input, float* output, size_t numSamples) override + void Process(const float* input, float* output, size_t numSamples) override { model->Process(input, output, numSamples); } @@ -523,7 +523,7 @@ namespace NeuralAudio (void)maxSize; } - void Process(float* input, float* output, size_t numSamples) override + void Process(const float* input, float* output, size_t numSamples) override { model->Process(input, output, numSamples); } diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index cca196d..a6fca73 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -98,9 +98,9 @@ namespace NeuralAudio } } - void Process(float* input, float* output, size_t numSamples) override + void Process(const float* input, float* output, size_t numSamples) override { - namModel->process(&input, &output, (int)numSamples); + namModel->process(const_cast(&input), &output, (int)numSamples); } void Prewarm() override diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index ede8572..5b2978e 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -124,7 +124,7 @@ namespace NeuralAudio return ""; } - virtual void Process(float* input, float* output, size_t numSamples) + virtual void Process(const float* input, float* output, size_t numSamples) { (void)input; (void)output; diff --git a/NeuralAudio/RTNeuralModel.h b/NeuralAudio/RTNeuralModel.h index 855657a..6177a78 100644 --- a/NeuralAudio/RTNeuralModel.h +++ b/NeuralAudio/RTNeuralModel.h @@ -416,7 +416,7 @@ namespace NeuralAudio return true; } - void Process(float* input, float* output, size_t numSamples) override + void Process(const float* input, float* output, size_t numSamples) override { for (size_t i = 0; i < numSamples; i++) output[i] = model->forward(input + i); From 048d195c5113116e07d5ac25844b21380323db6f Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Tue, 29 Sep 2026 07:41:18 -0700 Subject: [PATCH 161/162] fix Process() const input in RTNeural override --- NeuralAudio/RTNeuralModel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NeuralAudio/RTNeuralModel.h b/NeuralAudio/RTNeuralModel.h index 6177a78..3024cdb 100644 --- a/NeuralAudio/RTNeuralModel.h +++ b/NeuralAudio/RTNeuralModel.h @@ -211,7 +211,7 @@ namespace NeuralAudio return true; } - void Process(float* input, float* output, size_t numSamples) override + void Process(const float* input, float* output, size_t numSamples) override { for (size_t i = 0; i < numSamples; i++) output[i] = model->forward(input + i); From b98dfbdfdc7e7f845e3273aeeee2f3721960bbbc Mon Sep 17 00:00:00 2001 From: Mike Oliphant Date: Fri, 9 Oct 2026 09:46:40 -0700 Subject: [PATCH 162/162] Handle A1 head rechannel more cleanly (fixes A1 debug error) --- NeuralAudio/WaveNet.h | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index 65c1796..d8b4402 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -31,7 +31,7 @@ namespace NeuralAudio class ChannelHistoryBuffer { public: - static constexpr auto BufferSize = ReceptiveFieldSize + ((LAYER_ARRAY_BUFFER_PADDING + 1) * WAVENET_MAX_NUM_FRAMES); + static constexpr auto BufferSize = (ReceptiveFieldSize == 0) ? WAVENET_MAX_NUM_FRAMES : ReceptiveFieldSize + ((LAYER_ARRAY_BUFFER_PADDING + 1) * WAVENET_MAX_NUM_FRAMES); ChannelBuffer buffer; size_t bufferStart; @@ -52,16 +52,22 @@ namespace NeuralAudio #if (LAYER_ARRAY_BUFFER_PADDING == 0) bufferStart = ReceptiveFieldSize; #else - bufferStart = BufferSize - (WAVENET_MAX_NUM_FRAMES * ((allocNum % LAYER_ARRAY_BUFFER_PADDING) + 1)); // Do the modulo to handle cases where LAYER_ARRAY_BUFFER_PADDING is not big enough to handle offset + if constexpr (ReceptiveFieldSize == 0) + bufferStart = 0; + else + bufferStart = BufferSize - (WAVENET_MAX_NUM_FRAMES * ((allocNum % LAYER_ARRAY_BUFFER_PADDING) + 1)); // Do the modulo to handle cases where LAYER_ARRAY_BUFFER_PADDING is not big enough to handle offset #endif } void AdvanceFrames(const size_t numFrames) { - bufferStart += numFrames; + if constexpr (ReceptiveFieldSize > 0) + { + bufferStart += numFrames; - if ((bufferStart + WAVENET_MAX_NUM_FRAMES) > (size_t)buffer.GetNumCols()) - RewindBuffer(); + if ((bufferStart + WAVENET_MAX_NUM_FRAMES) > (size_t)buffer.GetNumCols()) + RewindBuffer(); + } } void RewindBuffer() @@ -514,7 +520,7 @@ namespace NeuralAudio private: Layers layers; DenseLayerT rechannel; - Conv1DT oneByOne; + Conv1DT headRechannel; public: static constexpr auto NumChannelsP = Channels; @@ -533,7 +539,7 @@ namespace NeuralAudio ReceptiveFieldSize += std::get(layers).ReceptiveFieldSize; }); - ReceptiveFieldSize += oneByOne.ReceptiveFieldSize; + ReceptiveFieldSize += headRechannel.ReceptiveFieldSize; } int AllocBuffers(int allocNum) @@ -543,7 +549,7 @@ namespace NeuralAudio std::get(layers).AllocBuffer(allocNum++); }); - oneByOne.channelBuffer.AllocBuffer(allocNum++); + headRechannel.channelBuffer.AllocBuffer(allocNum++); return allocNum; } @@ -557,7 +563,7 @@ namespace NeuralAudio numWeights += std::get(layers).GetNumWeights(); }); - numWeights += oneByOne.GetNumWeights(); + numWeights += headRechannel.GetNumWeights(); return numWeights; } @@ -571,7 +577,7 @@ namespace NeuralAudio std::get(layers).SetWeights(weights); }); - oneByOne.SetWeights(weights); + headRechannel.SetWeights(weights); } Layers& GetLayers() @@ -586,7 +592,7 @@ namespace NeuralAudio Conv1DT& GetHeadRechannel() { - return oneByOne; + return headRechannel; } void Prewarm(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) @@ -609,9 +615,9 @@ namespace NeuralAudio //headRechannel.channelBuffer.buffer.middleCols(headRechannel.channelBuffer.bufferStart, 1).noalias() = headInputs.leftCols(1); // Should be able to avoid this copy - oneByOne.GetInputBuffer(1).CopyData(headInputs.Slice(1)); - oneByOne.channelBuffer.CopyBuffer(); - oneByOne.Process(headOutputs.Slice(1)); + headRechannel.GetInputBuffer(1).CopyData(headInputs.Slice(1)); + headRechannel.channelBuffer.CopyBuffer(); + headRechannel.Process(headOutputs.Slice(1)); } template @@ -640,9 +646,9 @@ namespace NeuralAudio //headRechannel.GetInputBuffer(numFrames).CopyData(headInputs); - oneByOne.GetInputBuffer(numFrames).GetEigenMap().noalias() = headInputs.GetEigenMapConst(); - oneByOne.Process(headOutputs.Slice(numFrames)); - oneByOne.channelBuffer.AdvanceFrames(numFrames); + headRechannel.GetInputBuffer(numFrames).GetEigenMap().noalias() = headInputs.GetEigenMapConst(); + headRechannel.Process(headOutputs.Slice(numFrames)); + headRechannel.channelBuffer.AdvanceFrames(numFrames); } };