diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9f45ba0..5a0e71f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build run: | - cmake -G "Visual Studio 17 2022" -A x64 -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_STATIC_RTNEURAL=ON -T ClangCL .. + cmake -G "Visual Studio 18 2026" -A x64 -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_STATIC_RTNEURAL=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON -T ClangCL .. cmake --build . --config=release -j4 - name: Run ModelTest @@ -40,13 +40,18 @@ jobs: - uses: actions/checkout@v3.3.0 with: submodules: recursive + + - name: Setup GCC 15 + uses: egor-tensin/setup-gcc@v2 + with: + version: '15' - name: Build working-directory: ${{github.workspace}}/build env: - CXX: g++-13 + CXX: g++-15 run: | - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_STATIC_RTNEURAL=ON + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_STATIC_RTNEURAL=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON cmake --build . --config $BUILD_TYPE -j4 - name: Run ModelTest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f6b14a4 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,157 @@ +name: Release + +on: + workflow_dispatch: + +env: + BUILD_TYPE: Release + +jobs: + create_release: + name: Create release + runs-on: ubuntu-latest + outputs: + upload_url: ${{steps.create_release.outputs.upload_url}} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Create release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + with: + draft: true + tag_name: ${{github.ref}} + release_name: Release ${{github.ref}} + + build-linux-x64: + name: Build Linux x64 + needs: create_release + strategy: + matrix: + native_arch: [x86-64, x86-64-v3] + runs-on: ubuntu-latest + env: + ARCHIVE_NAME: ModelTest_linux_${{ matrix.native_arch }}.tgz + steps: + - uses: actions/checkout@v3.3.0 + with: + submodules: recursive + + - name: Setup Clang 21 + uses: egor-tensin/setup-clang@v2 + with: + version: '22' + + - name: Build + working-directory: ${{github.workspace}}/build + env: + CXX: clang++ + CXXFLAGS: "-Wall -O3 -march=${{ matrix.native_arch }} -static-libstdc++ -static-libgcc" + run: | + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON ${{ matrix.native_arch == 'x86-64' && '-DMULTIFRAME_8X8_CONVOLUTION=0' || '' }} + cmake --build . --config $BUILD_TYPE -j4 + + - name: Add Archive + working-directory: ${{github.workspace}}/build + run: | + mkdir ModelTest + cp Utils/ModelTest/ModelTest ModelTest + cp -r Utils/Models ModelTest + tar -czf ${{ env.ARCHIVE_NAME }} ModelTest + + - name: Upload Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: ./build/${{ env.ARCHIVE_NAME }} + asset_name: ${{ env.ARCHIVE_NAME }} + asset_content_type: application/tgz + + build-rpi: + name: Build Raspberry Pi + needs: create_release + strategy: + matrix: + native_arch: [rpi4, rpi5] + runs-on: ubuntu-latest + container: + image: ghcr.io/tttapa/docker-arm-cross-toolchain:aarch64-rpi3-linux-gnu-gcc16 + env: + ARCHIVE_NAME: ModelTest_${{ matrix.native_arch }}.tgz + steps: + - uses: actions/checkout@v3.3.0 + with: + submodules: recursive + + - name: Build + working-directory: ${{github.workspace}}/build + env: + CXXFLAGS: "-Wall -O3 -static-libstdc++ -static-libgcc" + run: | + cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON ${{ matrix.native_arch == 'rpi5' && '-DMULTIFRAME_8X8_CONVOLUTION=8' || '' }} -DCMAKE_TOOLCHAIN_FILE=/home/develop/opt/x-tools/aarch64-rpi3-linux-gnu/aarch64-${{ matrix.native_arch }}-linux-gnu.toolchain.cmake + cmake --build . --config $BUILD_TYPE -j4 + + - name: Add Archive + working-directory: ${{github.workspace}}/build + run: | + mkdir ModelTest + cp Utils/ModelTest/ModelTest ModelTest + cp -r Utils/Models ModelTest + tar -czf ${{ env.ARCHIVE_NAME }} ModelTest + + - name: Upload Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: ./build/${{ env.ARCHIVE_NAME }} + asset_name: ${{ env.ARCHIVE_NAME }} + asset_content_type: application/tgz + + build-windows: + name: Build Windows + needs: create_release + strategy: + matrix: + native_arch: [x86-64, x86-64-v3] + runs-on: windows-latest + env: + ARCHIVE_NAME: ModelTest_win_${{ matrix.native_arch }}.zip + steps: + - uses: actions/checkout@v3.3.0 + with: + submodules: recursive + + - name: Build + working-directory: ${{github.workspace}}/build + env: + CXXFLAGS: "/O2 -march=${{ matrix.native_arch }}" + run: | + cmake.exe -G "Visual Studio 18 2026" -A x64 -T ClangCL -DBUILD_UTILS=ON -DBUILD_NAMCORE=ON -DBUILD_INTERNAL_STATIC_WAVENET=ON -DBUILD_INTERNAL_STATIC_LSTM=ON ${{ matrix.native_arch == 'x86-64' && '-DMULTIFRAME_8X8_CONVOLUTION=0' || '' }} .. + cmake --build . --config=release -j4 + + - name: Add Archive + working-directory: ${{github.workspace}}/build + run: | + mkdir ModelTest + cp Utils/ModelTest/Release/ModelTest.exe ModelTest + cp -r Utils/Models ModelTest + Compress-Archive -Path ModelTest -Destination ${{ env.ARCHIVE_NAME }} + + - name: Upload Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: ./build/${{ env.ARCHIVE_NAME }} + asset_name: ${{ env.ARCHIVE_NAME }} + asset_content_type: application/zip diff --git a/CMakeLists.txt b/CMakeLists.txt index 76df4e7..07ac5e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.10) -project(NeuralAudio VERSION 0.0.1) +project(NeuralAudio VERSION 0.1.3) add_subdirectory(NeuralAudio) add_subdirectory(NeuralAudioCAPI) diff --git a/CREDITS.md b/CREDITS.md index dcf2d70..9870641 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -6,8 +6,6 @@ https://github.com/sdatkinson/NeuralAmpModelerCore https://github.com/jatinchowdhury18/RTNeural -https://github.com/jatinchowdhury18/RTNeural-NAM - https://gitlab.com/libeigen/eigen https://github.com/nlohmann/json diff --git a/NeuralAudio/Activation.h b/NeuralAudio/Activation.h index cd77a0c..076ddc0 100644 --- a/NeuralAudio/Activation.h +++ b/NeuralAudio/Activation.h @@ -3,6 +3,9 @@ #include #include #include +#include + +#define TCONST(num) static_cast(num) namespace NeuralAudio { @@ -14,78 +17,116 @@ namespace NeuralAudio #define WAVENET_MATH FastMath #endif + template struct StdMath { - template - static auto Tanh(Matrix& x) + template + static void Tanh(ChannelRowSpan& channelBuffer) { - float* data = x.data(); - size_t size = x.rows() * x.cols(); + T* data = channelBuffer.GetData(); + size_t size = channelBuffer.GetSize(); for (size_t pos = 0; pos < size; pos++) { data[pos] = Tanh(data[pos]); } - - return x; } - static inline float Tanh(const float x) + static inline T Tanh(const T x) { return std::tanh(x); } - static inline float Sigmoid(float x) + static inline T Sigmoid(T x) { - return 1.0f / (1.0f + std::exp(-x)); + return TCONST(1.0) / (TCONST(1.0)+ std::exp(-x)); + } + + template + static void LeakyReLU(ChannelRowSpan channelBuffer) + { + T* data = channelBuffer.GetData(); + size_t size = channelBuffer.GetSize(); + + for (size_t pos = 0; pos < size; pos++) + { + data[pos] = LeakyReLU(data[pos]); + } + } + + static inline T LeakyReLU(T x, T negativeSlope) + { + return x > TCONST(0) ? x : negativeSlope * x; + } + + static inline T LeakyReLU(T x) + { + return LeakyReLU(x, TCONST(0.01)); } }; + template struct FastMath { - template - static auto Tanh(Matrix& x) + template + static void Tanh(ChannelRowSpan& channelBuffer) { - float* data = x.data(); - size_t size = x.rows() * x.cols(); + T* data = channelBuffer.GetData(); + size_t size = channelBuffer.GetSize(); for (size_t pos = 0; pos < size; pos++) { data[pos] = Tanh(data[pos]); } - - return x; } - static inline float Tanh(const float x) + static inline T Tanh(const T x) { - //return std::tanh(x); + const T ax = std::abs(x); + + const T x2 = x * x; - //return math_approx::tanh<5>(x); + return (x * (TCONST(2.45550750702956) + TCONST(2.45550750702956) * ax + (TCONST(0.893229853513558) + TCONST(0.821226666969744) * ax) * x2) + / (TCONST(2.44506634652299) + (TCONST(2.44506634652299) + x2) * std::abs(x + TCONST(0.814642734961073) * x * ax))); + } - const float ax = fabsf(x); + static inline T Sigmoid(T x) + { + return TCONST(0.5) * (Tanh(x * TCONST(0.5)) + TCONST(1)); + } - const float x2 = x * x; + template + static void LeakyReLU(ChannelRowSpan channelBuffer) + { + T* data = channelBuffer.GetData(); + size_t size = channelBuffer.GetSize(); - return (x * (2.45550750702956f + 2.45550750702956f * ax + (0.893229853513558f + 0.821226666969744f * ax) * x2) - / (2.44506634652299f + (2.44506634652299f + x2) * fabsf(x + 0.814642734961073f * x * ax))); + for (size_t pos = 0; pos < size; pos++) + { + data[pos] = LeakyReLU(data[pos]); + } } - static inline float Sigmoid(float x) + static inline T LeakyReLU(T x, T negativeSlope) { - //return math_approx::sigmoid_exp<5>(x); + return x > TCONST(0.0) ? x : negativeSlope * x; + } - //return 1.0f / (1.0f + std::exp(-x)); - return 0.5f * (Tanh(x * 0.5f) + 1); + static inline T LeakyReLU(T x) + { + return LeakyReLU(x, TCONST(0.01)); } }; + template struct EigenMath { - template - static auto Tanh(const Matrix& x) + template + static void Tanh(ChannelRowSpan& channelBuffer) { - return x.array().tanh(); + auto map = channelBuffer.GetEigenMap(); + + map = map.array().tanh(); } static inline float Tanh(const float x) @@ -93,9 +134,17 @@ namespace NeuralAudio return Eigen::numext::tanh(x); } - static inline float Sigmoid(float x) + template + static void LeakyReLU(ChannelRowSpan channelBuffer) + { + auto map = channelBuffer.GetEigenMap(); + + map = (map.array() < TCONST(0.0)).select(map.array() * TCONST(0.01), map.array()); + } + + static inline T Sigmoid(T x) { - return 0.5f * (Tanh(x * 0.5f) + 1); + return TCONST(0.5) * (Tanh(x * TCONST(0.5)) + TCONST(1)); } }; } \ No newline at end of file diff --git a/NeuralAudio/CMakeLists.txt b/NeuralAudio/CMakeLists.txt index dbd5240..452789b 100644 --- a/NeuralAudio/CMakeLists.txt +++ b/NeuralAudio/CMakeLists.txt @@ -3,15 +3,23 @@ set(CMAKE_CXX_STANDARD 20) add_definitions(-DNAM_SAMPLE_FLOAT) add_definitions(-DDSP_SAMPLE_FLOAT) -option(BUILD_STATIC_RTNEURAL "Build Static RTNeural" OFF) -if(BUILD_STATIC_RTNEURAL) - message(STATUS "Building static RTNeural models") - add_definitions(-DBUILD_STATIC_RTNEURAL) +option(BUILD_RTNEURAL "Build RTNeural" ON) +if(BUILD_RTNEURAL) + message(STATUS "Building RTNeural models") + add_definitions(-DBUILD_RTNEURAL) + + option(BUILD_STATIC_RTNEURAL "Build Static RTNeural" OFF) + if(BUILD_STATIC_RTNEURAL) + message(STATUS "Building static RTNeural models") + add_definitions(-DBUILD_STATIC_RTNEURAL) + else() + message(STATUS "NOT Building static RTNeural models") + endif() else() - message(STATUS "NOT Building static RTNeural models") + message(STATUS "NOT Building RTNeural models") endif() -option(BUILD_NAMCORE "Build NAM Core" OFF) +option(BUILD_NAMCORE "Build NAM Core" ON) if(BUILD_NAMCORE) message(STATUS "Building NAM Core implementation") add_definitions(-DBUILD_NAMCORE) @@ -19,29 +27,96 @@ else() message(STATUS "NOT Building NAM Core implementation") endif() +option(NAM_USE_INLINE_GEMM "Use inline matrix multiplications in NAM Core" OFF) +if(NAM_USE_INLINE_GEMM) + message(STATUS "Using NAM Core inline matrix multiplications") + add_definitions(-DNAM_USE_INLINE_GEMM) +else() + message(STATUS "NOT using NAM Core inline matrix multiplications") +endif() + +option(NAM_ENABLE_A2_FAST "Use NAM A2 fast-path WaveNet" ON) +if(NAM_ENABLE_A2_FAST) + message(STATUS "Using NAM A2 fast-path WaveNet") + add_definitions(-DNAM_ENABLE_A2_FAST) + set(NAM_A2_RING_MODE "1" CACHE STRING "Ring-buffer strategy") + add_definitions(-DNAM_A2_RING_MODE=${NAM_A2_RING_MODE}) + message(STATUS "Ring-buffer strategy is: ${NAM_A2_RING_MODE}") +else() + message(STATUS "NOT using NAM A2 fast-path WaveNet") +endif() + option(BUILD_INTERNAL_STATIC_WAVENET "Build Internal static WaveNet models" ON) if(BUILD_INTERNAL_STATIC_WAVENET) message(STATUS "Building Internal static WaveNet models") add_definitions(-DBUILD_INTERNAL_STATIC_WAVENET) else() - message(STATUS "NOT Building Internal static WaveNet models") + message(STATUS "NOT building Internal static WaveNet models") +endif() + +option(BUILD_STATIC_INTERNAL_NAMA2 "Build Internal static A2 WaveNet models" ON) +if(BUILD_STATIC_INTERNAL_NAMA2) + message(STATUS "Building Internal static A2 WaveNet models") + add_definitions(-DBUILD_STATIC_INTERNAL_NAMA2) +else() + message(STATUS "NOT building Internal static A2 WaveNet models") endif() -option(BUILD_INTERNAL_STATIC_LSTM "Build Internal static LSTM models" ON) +if(MSVC) + set(MULTIFRAME_8X8_CONVOLUTION "8" CACHE STRING "Multi-frame 8x8 convolution") +else() + if (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64|arm64)") + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "15.0") + set(MULTIFRAME_8X8_CONVOLUTION "4" CACHE STRING "Multi-frame 8x8 convolution") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") + set(MULTIFRAME_8X8_CONVOLUTION "4" CACHE STRING "Multi-frame 8x8 convolution") + endif() + else() + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "16.0") + set(MULTIFRAME_8X8_CONVOLUTION "4" CACHE STRING "Multi-frame 8x8 convolution") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") + set(MULTIFRAME_8X8_CONVOLUTION "8" CACHE STRING "Multi-frame 8x8 convolution") + endif() + endif() +endif() + +# Set default to 0 if it isn't already set +set(MULTIFRAME_8X8_CONVOLUTION "0" CACHE STRING "Multi-frame 8x8 convolution") + +if(MULTIFRAME_8X8_CONVOLUTION GREATER 0) + message(STATUS "Using multi-frame 8x8 convolution: ${MULTIFRAME_8X8_CONVOLUTION}") + add_definitions(-DMULTIFRAME_8X8_CONVOLUTION=${MULTIFRAME_8X8_CONVOLUTION}) +else() + message(STATUS "NOT using multi-frame 8x8 convolution") +endif() + +set(WAVENET_MATH "FastMath" CACHE STRING "WaveNet math functions") +add_definitions(-DWAVENET_MATH=${WAVENET_MATH}) +message(STATUS "WaveNet math is: ${WAVENET_MATH}") + +option(BUILD_INTERNAL_STATIC_LSTM "Build Internal static LSTM models" OFF) if(BUILD_INTERNAL_STATIC_LSTM) message(STATUS "Building Internal static LSTM models") add_definitions(-DBUILD_INTERNAL_STATIC_LSTM) else() - message(STATUS "NOT Building Internal static LSTM models") + message(STATUS "NOT building Internal static LSTM models") endif() set(LSTM_MATH "FastMath" CACHE STRING "LSTM math functions") add_definitions(-DLSTM_MATH=${LSTM_MATH}) message(STATUS "LSTM math is: ${LSTM_MATH}") -set(WAVENET_MATH "FastMath" CACHE STRING "WaveNet math functions") -add_definitions(-DWAVENET_MATH=${WAVENET_MATH}) -message(STATUS "WaveNet math is: ${WAVENET_MATH}") +set(DEFAULT_QUALITY_SCALE "1.0" CACHE STRING "Default model quality scale factor") + +add_definitions(-DDEFAULT_QUALITY_SCALE=${DEFAULT_QUALITY_SCALE}) + +message(STATUS "Default model quality scale factor is: ${DEFAULT_QUALITY_SCALE}") + +set(DEFAULT_INPUT_DBU "12" CACHE STRING "Default dBu level for input scaling") + +add_definitions(-DDEFAULT_INPUT_DBU=${DEFAULT_INPUT_DBU}) + +message(STATUS "Default input dBu is: ${DEFAULT_INPUT_DBU}") set(WAVENET_FRAMES "64" CACHE STRING "WaveNet frame size") @@ -62,52 +137,70 @@ endif() set(SOURCES NeuralModel.h NeuralModel.cpp + NeuralModelImpl.h NAMModel.h RTNeuralModel.h RTNeuralLoader.cpp RTNeuralLoader.h Activation.h + ChannelBuffer.h + MatMul.h WaveNet.h WaveNetDynamic.h LSTM.h LSTMDynamic.h InternalModel.h + CompositeModel.h TemplateHelper.h) if(BUILD_NAMCORE) set(NAM_SOURCES ../deps/NeuralAmpModelerCore/NAM/activations.h ../deps/NeuralAmpModelerCore/NAM/activations.cpp + ../deps/NeuralAmpModelerCore/NAM/conv1d.cpp + ../deps/NeuralAmpModelerCore/NAM/conv1d.h + ../deps/NeuralAmpModelerCore/NAM/film.h + ../deps/NeuralAmpModelerCore/NAM/gating_activations.h + ../deps/NeuralAmpModelerCore/NAM/get_dsp.h + ../deps/NeuralAmpModelerCore/NAM/get_dsp.cpp ../deps/NeuralAmpModelerCore/NAM/lstm.h + ../deps/NeuralAmpModelerCore/NAM/registry.h + ../deps/NeuralAmpModelerCore/NAM/ring_buffer.h + ../deps/NeuralAmpModelerCore/NAM/ring_buffer.cpp + ../deps/NeuralAmpModelerCore/NAM/slimmable.h ../deps/NeuralAmpModelerCore/NAM/lstm.cpp ../deps/NeuralAmpModelerCore/NAM/dsp.h ../deps/NeuralAmpModelerCore/NAM/dsp.cpp - ../deps/NeuralAmpModelerCore/NAM/wavenet.cpp - ../deps/NeuralAmpModelerCore/NAM/wavenet.h) -endif() - -if(BUILD_STATIC_RTNEURAL) -set(RTNEURAL_WN_SOURCES ../deps/RTNeural-NAM/wavenet/wavenet_layer.hpp - ../deps/RTNeural-NAM/wavenet/wavenet_layer_array.hpp - ../deps/RTNeural-NAM/wavenet/wavenet_model.hpp - ../deps/RTNeural-NAM/wavenet/arena.hpp) + ../deps/NeuralAmpModelerCore/NAM/container.cpp + ../deps/NeuralAmpModelerCore/NAM/container.h + ../deps/NeuralAmpModelerCore/NAM/wavenet/slimmable.cpp + ../deps/NeuralAmpModelerCore/NAM/wavenet/detail.h + ../deps/NeuralAmpModelerCore/NAM/wavenet/params.h + ../deps/NeuralAmpModelerCore/NAM/wavenet/model.h + ../deps/NeuralAmpModelerCore/NAM/wavenet/model.cpp + ../deps/NeuralAmpModelerCore/NAM/wavenet/a2_fast.h + ../deps/NeuralAmpModelerCore/NAM/wavenet/a2_fast.cpp) endif() -add_library(NeuralAudio STATIC ${SOURCES} ${NAM_SOURCES} ${RTNEURAL_WN_SOURCES}) +add_library(NeuralAudio OBJECT ${SOURCES} ${NAM_SOURCES}) target_include_directories(NeuralAudio PUBLIC ..) target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/NeuralAmpModelerCore) target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/RTNeural) target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/math_approx) -target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/RTNeural-NAM/wavenet) target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/RTNeural/modules/Eigen) -target_include_directories(NeuralAudio SYSTEM PRIVATE ../deps/RTNeural/modules/json) +target_include_directories(NeuralAudio PUBLIC ../deps/RTNeural/modules/json) set_property(TARGET NeuralAudio PROPERTY POSITION_INDEPENDENT_CODE ON) -add_subdirectory(../deps/RTNeural RTNeural) +set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "" FORCE) + +if(BUILD_RTNEURAL) + add_subdirectory(../deps/RTNeural RTNeural) + target_link_libraries(NeuralAudio LINK_PUBLIC RTNeural) +endif() add_subdirectory(../deps/math_approx math_approx) -target_link_libraries(NeuralAudio LINK_PUBLIC RTNeural math_approx) +target_link_libraries(NeuralAudio LINK_PUBLIC math_approx) source_group(NeuralAudio ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES}) source_group(NAM ${CMAKE_CURRENT_SOURCE_DIR} FILES ${NAM_SOURCES}) -source_group(RTNeural-NAM ${CMAKE_CURRENT_SOURCE_DIR} FILES ${RTNEURAL_WN_SOURCES}) + diff --git a/NeuralAudio/ChannelBuffer.h b/NeuralAudio/ChannelBuffer.h new file mode 100644 index 0000000..7e2ffcd --- /dev/null +++ b/NeuralAudio/ChannelBuffer.h @@ -0,0 +1,380 @@ +#pragma once + +#include +#include +#include +#include + +constexpr size_t CHANNEL_BUFFER_ALIGN = 32; + +namespace NeuralAudio +{ + template + class ChannelRowSpan; + + template + class ChannelBufferBase + { + public: + virtual size_t GetSize() const { return 0; } + virtual size_t GetNumCols() const { return 0; } + size_t GetNumChannels() const { return Channels; } + virtual T* GetData() { return nullptr; } + virtual T* GetData(size_t startCol) + { + (void)startCol; + + return nullptr; + } + virtual const T* GetDataConst() const { return nullptr; } + virtual const T* GetDataConst(size_t startCol) const + { + (void)startCol; + + return nullptr; + } + virtual void SetZero() {} + virtual void SetZero(size_t startCol, size_t numCols) = 0; + virtual T& operator()(size_t row, size_t col) = 0; + + virtual const T& operator()(size_t row, size_t col) const = 0; + }; + + template + class ChannelBuffer : public ChannelBufferBase + { + public: + ChannelBuffer() + { + } + + size_t GetSize() const override + { + return (size_t)(Channels * Cols); + } + + size_t GetNumCols() const override + { + return Cols; + } + + T* GetData() override + { + return data[0].data(); + } + + const T* GetDataConst() const override + { + return data[0].data(); + } + + T* GetData(size_t startCol) override + { + return data[startCol].data(); + } + + const T* GetDataConst(size_t startCol) const override + { + return data[startCol].data(); + } + + void SetZero() override + { + for (auto& col : data) { + col.fill(0); + } + } + + void SetZero(size_t startCol, size_t numCols) override + { + for (size_t col = 0; col < numCols; col++) + data[startCol + col].fill(0); + } + + T& operator()(size_t row, size_t col) override + { + return data[col][row]; + } + + const T& operator()(size_t row, size_t col) const override + { + return data[col][row]; + } + + const ChannelRowSpan Slice(size_t startCol, size_t numCols) + { + return ChannelRowSpan(this, startCol, numCols); + } + + const ChannelRowSpan Slice(size_t numCols) + { + return ChannelRowSpan(this, numCols); + } + + const Eigen::Map> GetEigenMap() + { + return Eigen::Map>(GetData(), Channels, GetNumCols()); + } + + const Eigen::Map> GetEigenMapConst() const + { + return Eigen::Map>(GetDataConst(), Channels, GetNumCols()); + } + + Eigen::Map> GetEigenMapTransposed() + { + return Eigen::Map>(GetData(), Channels, GetNumCols()); + } + + const Eigen::Map> GetEigenMapTransposedConst() const + { + return Eigen::Map>(GetDataConst(), Channels, GetNumCols()); + } + + private: + alignas(CHANNEL_BUFFER_ALIGN) std::array, Cols> data; + }; + + + template + class ChannelBufferDynamic : public ChannelBufferBase + { + public: + ChannelBufferDynamic() + { + numCols = 0; + data = nullptr; + } + + ChannelBufferDynamic(T* data, size_t numCols) : + data(data), + numCols(numCols) + { + } + + size_t GetSize() const override + { + return (size_t)(Channels * numCols); + } + + size_t GetNumCols() const override + { + return numCols; + } + + T* GetData() override + { + return data; + } + + const T* GetDataConst() const override + { + return data; + } + + T* GetData(size_t startCol) override + { + return data + (startCol * Channels); + } + + const T* GetDataConst(size_t startCol) const override + { + return data + (startCol * Channels); + } + + void SetZero() override + { + std::fill(data, data + GetSize(), 0); + } + + void SetZero(size_t startCol, size_t numCols) override + { + std::fill(data + (startCol * Channels), data + ((startCol + numCols) * Channels), 0); + } + + T& operator()(size_t row, size_t col) override + { + return data[(col * Channels) + row]; + } + + const T& operator()(size_t row, size_t col) const override + { + return data[(col * Channels) + row]; + } + + const ChannelRowSpan Slice(size_t startCol, size_t numCols) + { + return ChannelRowSpan(this, startCol, numCols); + } + + const ChannelRowSpan Slice(size_t numCols) + { + return ChannelRowSpan(this, numCols); + } + + const Eigen::Map> GetEigenMap() + { + return Eigen::Map>(GetData(), Channels, GetNumCols()); + } + + const Eigen::Map> GetEigenMapConst() const + { + return Eigen::Map>(GetDataConst(), Channels, GetNumCols()); + } + + Eigen::Map> GetEigenMapTransposed() + { + return Eigen::Map>(GetData(), Channels, GetNumCols()); + } + + const Eigen::Map> GetEigenMapTransposedConst() const + { + return Eigen::Map>(GetDataConst(), Channels, GetNumCols()); + } + + private: + T* data; + size_t numCols; + }; + + template + class ChannelRowSpan + { + public: + ChannelRowSpan(ChannelBufferBase& baseBuffer) : + buffer(&baseBuffer), + startCol(0), + numCols(baseBuffer.GetNumCols()) + { + } + + ChannelRowSpan(ChannelBufferBase* baseBuffer) : + buffer(baseBuffer), + startCol(0), + numCols(baseBuffer->GetNumCols()) + { + + } + + ChannelRowSpan(ChannelBufferBase* baseBuffer, size_t startCol, size_t numCols) : + buffer(baseBuffer), + startCol(startCol), + numCols(numCols) + { + assert(startCol >= 0); + assert(numCols <= (baseBuffer->GetNumCols() - startCol)); + } + + ChannelRowSpan(ChannelBufferBase* baseBuffer, size_t numCols) : + buffer(baseBuffer), + startCol(0), + numCols(numCols) + { + assert(numCols <= baseBuffer->GetNumCols()); + } + + const ChannelRowSpan Slice(size_t startCol, size_t numCols) const + { + return ChannelRowSpan(buffer, this->startCol + startCol, numCols); + } + + const ChannelRowSpan Slice(size_t numCols) const + { + return ChannelRowSpan(buffer, this->startCol, numCols); + } + + size_t GetSize() const + { + return GetNumChannels() * GetNumCols(); + } + + size_t GetNumCols() const + { + return numCols; + } + + size_t GetNumChannels() const + { + return Channels; + } + + void SetZero() const + { + buffer->SetZero(startCol, numCols); + } + + T& operator()(size_t row, size_t col) + { + return (*buffer)(row, startCol + col); + } + + const T& operator()(size_t row, size_t col) const + { + return (*buffer)(row, startCol + col); + } + + T* GetData() const + { + return buffer->GetData(startCol); + } + + const T* GetDataConst() const + { + return buffer->GetDataConst(startCol); + } + + T* GetData(size_t startCol) const + { + return buffer->GetData(this->startCol + startCol); + } + + const T* GetDataConst(size_t startCol) const + { + return buffer->GetDataConst(this->startCol + startCol); + } + + Eigen::Map> GetEigenMap() const + { + return Eigen::Map>(GetData(), Channels, numCols); + } + + const Eigen::Map> GetEigenMapConst() const + { + return Eigen::Map>(GetDataConst(), Channels, numCols); + } + + Eigen::Map> GetEigenMapTransposed() + { + return Eigen::Map>(GetData(), Channels, numCols); + } + + const Eigen::Map> GetEigenMapTransposedConst() const + { + return Eigen::Map>(GetDataConst(), Channels, numCols); + } + + void CopyData(const ChannelRowSpan& srcSpan) const + { + const T* srcPtr = srcSpan.GetDataConst(); + T* destPtr = GetData(); + + memmove(destPtr, srcPtr, Channels * numCols * sizeof(T)); + } + + void AddData(const ChannelRowSpan& srcSpan) const + { + const T* srcPtr = srcSpan.GetDataConst(); + T* destPtr = GetData(); + + for (size_t count = 0; count < Channels * numCols; count++) + { + *destPtr++ += *srcPtr++; + } + } + + private: + ChannelBufferBase* const buffer; + const size_t startCol; + const size_t numCols; + }; +} \ No newline at end of file diff --git a/NeuralAudio/CompositeModel.h b/NeuralAudio/CompositeModel.h new file mode 100644 index 0000000..0b68765 --- /dev/null +++ b/NeuralAudio/CompositeModel.h @@ -0,0 +1,215 @@ +#pragma once + +#include +#include +#include "NeuralModel.h" +#include "NeuralModelImpl.h" + +namespace NeuralAudio +{ + class CompositeModel : public NeuralModelImpl + { + public: + CompositeModel() + { + } + + ~CompositeModel() + { + for (auto& model : models) + { + delete model; + } + } + + void AddModel(NeuralModelImpl* model) + { + if (currentModelIndex.load() == -1) + currentModelIndex.store(0); + + models.push_back(model); + } + + int GetModelCount() + { + return (int)models.size(); + } + + int GetCurrentModelIndex() + { + return currentModelIndex.load(); + } + + bool IsModelChangeRealtimeSafe(int newIndex) + { + if (newIndex == currentModelIndex.load()) + return true; + + return models[newIndex]->HadInitialPrewarm(); + } + + void SetCurrentModelIndex(int index) + { + if (index != currentModelIndex.load()) + { + currentModelIndex.store(index); + + if (compositeLoadMode == ECompositeModelLoadMode::OnDemand) + { + // For on-demand loading we need to check if the model we are switching to has been prewarmed + if (!models[currentModelIndex.load()]->HadInitialPrewarm()) + { + Prewarm(); + } + } + } + } + + EModelLoadMode GetLoadMode() override + { + return models[currentModelIndex.load()]->GetLoadMode(); + } + + bool IsStatic() override + { + return models[currentModelIndex.load()]->IsStatic(); + } + + int GetReceptiveFieldSize() override + { + if (currentModelIndex == -1) + return -1; + + return models[currentModelIndex.load()]->GetReceptiveFieldSize(); + } + + void SetMaxAudioBufferSize(const int maxSize) override + { + for (auto& model : models) + { + model->SetMaxAudioBufferSize(maxSize); + } + } + + void Process(const float* input, float* output, size_t numSamples) override + { + if (currentModelIndex == -1) + return; + + models[currentModelIndex.load()]->Process(input, output, numSamples); + } + + void Prewarm() override + { + if (compositeLoadMode == ECompositeModelLoadMode::OnDemand) + { + auto& model = models[currentModelIndex.load()]; + model->Prewarm(); + model->SetHadInitialPrewarm(); + } + else + { + for (auto& model : models) + { + model->Prewarm(); + model->SetHadInitialPrewarm(); + } + } + } + + protected: + std::vector models; + std::atomic currentModelIndex = -1; + ECompositeModelLoadMode compositeLoadMode = ECompositeModelLoadMode::LoadAll; + }; + + + class ScalableCompositeModel : public CompositeModel + { + public: + bool LoadFromJson(nlohmann::json& modelJson) + { + ReadNAMConfig(modelJson); + + return CreateModelFromNAMJson(modelJson); + } + + virtual bool CreateModelFromNAMJson(nlohmann::json& modelJson) + { + compositeLoadMode = loader->GetCompositeModelLoadMode(); + + auto& config = modelJson.at("config"); + + auto& subModels = config.at("submodels"); + + for (auto& submodelJson : subModels) + { + NeuralModelImpl* submodel = dynamic_cast(loader->CreateFromJson(submodelJson.at("model"), ".nam", false)); + + AddModel(submodelJson.at("max_value"), submodel); + } + + // Don't call SetMaxAudioBufferSize because it has already been done by individual models + // (this can be re-added for completeness if NAM Core GetMaxBufferSize() is made public + //SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); + + SetQualityScaleFactor(loader->GetDefaultQualityScaleFactor()); + + return true; + } + + bool HasQualityScaling() override + { + return true; + } + + float GetQualityScaleFactor() override + { + return currentQualityLevel.load(); + } + + bool IsQualityChangeRealtimeSafe(float newScaleFactor) override + { + return IsModelChangeRealtimeSafe(GetModelIndexFromQualityScale(newScaleFactor)); + } + + void SetQualityScaleFactor(float scaleFactor) override + { + currentQualityLevel.store(scaleFactor); + + SetCurrentModelIndex(GetModelIndexFromQualityScale(scaleFactor)); + } + + void AddModel(float scaleFactor, NeuralModelImpl* model) + { + CompositeModel::AddModel(model); + + qualityLevels.emplace_back(scaleFactor, GetModelCount() - 1); + + // Sort by quality level + std::sort(qualityLevels.begin(), qualityLevels.end(), [](const auto& a, const auto& b) + { + return std::get<0>(a) < std::get<0>(b); + }); + } + + protected: + std::atomic currentQualityLevel = 1.0f; + std::vector> qualityLevels; + + int GetModelIndexFromQualityScale(float qualityScale) + { + int modelIndex = 0; + + for (auto& level : qualityLevels) + { + modelIndex = std::get<1>(level); + + if (qualityScale <= std::get<0>(level)) + break; + } + + return modelIndex; + } + }; +} \ No newline at end of file diff --git a/NeuralAudio/InternalModel.h b/NeuralAudio/InternalModel.h index ee41d2a..69cc409 100644 --- a/NeuralAudio/InternalModel.h +++ b/NeuralAudio/InternalModel.h @@ -1,6 +1,7 @@ #pragma once #include "NeuralModel.h" +#include "NeuralModelImpl.h" #include "WaveNet.h" #include "WaveNetDynamic.h" #include "LSTM.h" @@ -9,10 +10,16 @@ namespace NeuralAudio { using IStdDilations = NeuralAudio::Dilations<1, 2, 4, 8, 16, 32, 64, 128, 256, 512>; + using IStdKernelSizes = NeuralAudio::KernelSizes<3, 3, 3, 3, 3, 3, 3, 3, 3, 3>; using ILiteDilations1 = NeuralAudio::Dilations<1, 2, 4, 8, 16, 32, 64>; + using ILiteKernelSizes1 = NeuralAudio::KernelSizes<3, 3, 3, 3, 3, 3, 3>; using ILiteDilations2 = NeuralAudio::Dilations<128, 256, 512, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512>; + using ILiteKernelSizes2 = NeuralAudio::KernelSizes<3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3>; - class InternalModel : public NeuralModel + using A2KernelSizes = NeuralAudio::KernelSizes<6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 15, 15, 6, 6, 6, 6, 6, 6, 6>; + using A2Dilations = NeuralAudio::Dilations<1, 3, 7, 17, 41, 101, 239, 1, 3, 7, 17, 41, 101, 239, 1, 13, 1, 3, 7, 17, 41, 101, 239>; + + class InternalModel : public NeuralModelImpl { public: bool LoadFromKerasJson(const nlohmann::json& modelJson) @@ -20,8 +27,6 @@ namespace NeuralAudio ReadKerasConfig(modelJson); return CreateModelFromKerasJson(modelJson); - - return true; } virtual bool CreateModelFromKerasJson(const nlohmann::json& modelJson) @@ -46,19 +51,9 @@ namespace NeuralAudio } }; - - template + template class InternalWaveNetModelT : public InternalModel { - using ModelType = typename std::conditional, - NeuralAudio::WaveNetLayerArrayT>, - NeuralAudio::WaveNetModelT< - NeuralAudio::WaveNetLayerArrayT<1, 1, headSize, numChannels, 3, ILiteDilations1, false>, - NeuralAudio::WaveNetLayerArrayT> - >::type; - public: InternalWaveNetModelT() : model(nullptr) @@ -74,12 +69,12 @@ namespace NeuralAudio } } - bool IsStatic() + bool IsStatic() override { return true; } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -89,26 +84,24 @@ namespace NeuralAudio model = new ModelType; - nlohmann::json config = modelJson.at("config"); - model->SetWeights(modelJson.at("weights")); - SetMaxAudioBufferSize(defaultMaxAudioBufferSize); + SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); return true; } - void SetMaxAudioBufferSize(int maxSize) + void SetMaxAudioBufferSize(const int maxSize) override { (void)maxSize; } - int GetReceptiveFieldSize() + int GetReceptiveFieldSize() override { return model->ReceptiveFieldSize; } - void Process(float* input, float* output, size_t numSamples) + void Process(const float* input, float* output, size_t numSamples) override { size_t offset = 0; @@ -123,7 +116,7 @@ namespace NeuralAudio } } - void Prewarm() + void Prewarm() override { model->Prewarm(); } @@ -152,23 +145,32 @@ namespace NeuralAudio } }; - template - class InternalWaveNetDefinitionT : public InternalWaveNetDefinitionBase + template + class InternalA1WaveNetDefinitionT : public InternalWaveNetDefinitionBase { public: - InternalModel* CreateModel() + using ModelType = typename std::conditional, + NeuralAudio::WaveNetLayerArrayT>, + NeuralAudio::WaveNetModelT, + NeuralAudio::WaveNetLayerArrayT> + >::type; + + InternalModel* CreateModel() override { - return new InternalWaveNetModelT; + return new InternalWaveNetModelT; } - virtual size_t GetNumChannels() + virtual size_t GetNumChannels() override { - return numChannels; + return NumChannels; } - virtual size_t GetHeadSize() + virtual size_t GetHeadSize() override { - return headSize; + return HeadSize; } }; @@ -188,20 +190,20 @@ namespace NeuralAudio } } - EModelLoadMode GetLoadMode() + EModelLoadMode GetLoadMode() override { return EModelLoadMode::Internal; } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); std::vector layerArrays; for (size_t i = 0; i < config.at("layers").size(); i++) { - nlohmann::json layerConfig = config.at("layers").at(i); + auto& layerConfig = config.at("layers").at(i); layerArrays.push_back(WaveNetLayerArray(layerConfig.at("input_size"), layerConfig.at("condition_size"), layerConfig.at("head_size"), layerConfig.at("channels"), layerConfig.at("kernel_size"), layerConfig.at("head_bias"), layerConfig.at("dilations"))); @@ -211,17 +213,17 @@ namespace NeuralAudio model->SetWeights(modelJson.at("weights")); - SetMaxAudioBufferSize(defaultMaxAudioBufferSize); + SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); return true; } - void SetMaxAudioBufferSize(int maxSize) + void SetMaxAudioBufferSize(const int maxSize) override { model->SetMaxFrames(maxSize); } - void Process(float* input, float* output, size_t numSamples) + void Process(const float* input, float* output, size_t numSamples) override { size_t offset = 0; @@ -236,7 +238,7 @@ namespace NeuralAudio } } - void Prewarm() + void Prewarm() override { model->Prewarm(); } @@ -264,12 +266,12 @@ namespace NeuralAudio } } - bool IsStatic() + bool IsStatic() override { return true; } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -279,11 +281,9 @@ namespace NeuralAudio model = new LSTMModelT; - nlohmann::json config = modelJson.at("config"); - model->SetNAMWeights(modelJson.at("weights")); - SetMaxAudioBufferSize(defaultMaxAudioBufferSize); + SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); return true; } @@ -308,7 +308,7 @@ namespace NeuralAudio return vec; } - bool CreateModelFromKerasJson(const nlohmann::json& modelJson) + bool CreateModelFromKerasJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -355,19 +355,19 @@ namespace NeuralAudio return true; } - void SetMaxAudioBufferSize(int maxSize) + void SetMaxAudioBufferSize(const int maxSize) override { (void)maxSize; } - void Process(float* input, float* output, size_t numSamples) + void Process(const float* input, float* output, size_t numSamples) override { model->Process(input, output, numSamples); } - void Prewarm() + void Prewarm() override { - NeuralModel::Prewarm(2048, 64); + NeuralModelImpl::Prewarm(2048, 64); } private: @@ -398,17 +398,17 @@ namespace NeuralAudio class InternalLSTMDefinitionT : public InternalLSTMDefinitionBase { public: - InternalModel* CreateModel() + InternalModel* CreateModel() override { return new InternalLSTMModelT; } - virtual size_t GetNumLayers() + virtual size_t GetNumLayers() override { return NumLayers; } - virtual size_t GetHiddenSize() + virtual size_t GetHiddenSize() override { return HiddenSize; } @@ -431,7 +431,7 @@ namespace NeuralAudio } } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -439,13 +439,13 @@ namespace NeuralAudio model = nullptr; } - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); model = new LSTMModel(config.at("num_layers"), config.at("hidden_size")); model->SetNAMWeights(modelJson.at("weights")); - SetMaxAudioBufferSize(defaultMaxAudioBufferSize); + SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); return true; } @@ -470,7 +470,7 @@ namespace NeuralAudio return vec; } - bool CreateModelFromKerasJson(const nlohmann::json& modelJson) + bool CreateModelFromKerasJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -518,19 +518,19 @@ namespace NeuralAudio return true; } - void SetMaxAudioBufferSize(int maxSize) + void SetMaxAudioBufferSize(const int maxSize) override { (void)maxSize; } - void Process(float* input, float* output, size_t numSamples) + void Process(const float* input, float* output, size_t numSamples) override { model->Process(input, output, numSamples); } - void Prewarm() + void Prewarm() override { - NeuralModel::Prewarm(2048, 64); + NeuralModelImpl::Prewarm(2048, 64); } private: diff --git a/NeuralAudio/LSTM.h b/NeuralAudio/LSTM.h index c4c70b4..4536ee8 100644 --- a/NeuralAudio/LSTM.h +++ b/NeuralAudio/LSTM.h @@ -92,11 +92,11 @@ namespace NeuralAudio gates = (inputHiddenWeights * state) + bias; for (auto i = 0; i < HiddenSize; i++) - cellState[i] = (LSTM_MATH::Sigmoid(gates[i + fOffset]) * cellState[i]) + - (LSTM_MATH::Sigmoid(gates[i + iOffset]) * LSTM_MATH::Tanh(gates[i + gOffset])); + cellState[i] = (LSTM_MATH::Sigmoid(gates[i + fOffset]) * cellState[i]) + + (LSTM_MATH::Sigmoid(gates[i + iOffset]) * LSTM_MATH::Tanh(gates[i + gOffset])); for (int i = 0; i < HiddenSize; i++) - state[i + hOffset] = LSTM_MATH::Sigmoid(gates[i + oOffset]) * LSTM_MATH::Tanh(cellState[i]); + state[i + hOffset] = LSTM_MATH::Sigmoid(gates[i + oOffset]) * LSTM_MATH::Tanh(cellState[i]); } }; diff --git a/NeuralAudio/LSTMDynamic.h b/NeuralAudio/LSTMDynamic.h index e9884bc..e90d9b8 100644 --- a/NeuralAudio/LSTMDynamic.h +++ b/NeuralAudio/LSTMDynamic.h @@ -100,11 +100,11 @@ namespace NeuralAudio gates = (inputHiddenWeights * state) + bias; for (size_t i = 0; i < hiddenSize; i++) - cellState[i] = (LSTM_MATH::Sigmoid(gates[i + fOffset]) * cellState[i]) + (LSTM_MATH::Sigmoid(gates[i + iOffset]) * - LSTM_MATH::Tanh(gates[i + gOffset])); + cellState[i] = (LSTM_MATH::Sigmoid(gates[i + fOffset]) * cellState[i]) + (LSTM_MATH::Sigmoid(gates[i + iOffset]) * + LSTM_MATH::Tanh(gates[i + gOffset])); for (size_t i = 0; i < hiddenSize; i++) - state[i + hOffset] = LSTM_MATH::Sigmoid(gates[i + oOffset]) * LSTM_MATH::Tanh(cellState[i]); + state[i + hOffset] = LSTM_MATH::Sigmoid(gates[i + oOffset]) * LSTM_MATH::Tanh(cellState[i]); } }; @@ -142,7 +142,7 @@ namespace NeuralAudio layer.SetNAMWeights(it); } - for (int i = 0; i < hiddenSize; i++) + for (size_t i = 0; i < hiddenSize; i++) headWeights[i] = *(it++); headBias = *(it++); diff --git a/NeuralAudio/MatMul.h b/NeuralAudio/MatMul.h new file mode 100644 index 0000000..17810e9 --- /dev/null +++ b/NeuralAudio/MatMul.h @@ -0,0 +1,270 @@ +#pragma once + +namespace NeuralAudio +{ + template + struct MatMul + { + #define IsKernel(In, Out) ((InChannels == In) && (OutChannels == Out)) + + static constexpr bool HasKernel() { return (IsKernel(3, 3) || IsKernel(8, 1) || IsKernel(3, 1) || IsKernel(1, 3)); } + + // 3x3 implementation inspired by @jfsantos NAM Core a2fast - https://github.com/sdatkinson/NeuralAmpModelerCore/blob/main/NAM/wavenet/a2_fast.cpp + + static inline void MultiplyInitZero(const T* inData, T* outData, const T* weights, size_t numFrames) + { + static_assert(HasKernel(), "Multiplication not implemented for InChannel/OutChannel combination"); + + if constexpr (IsKernel(3, 3)) + { + const T w00 = weights[0], w10 = weights[1], w20 = weights[2]; + const T w01 = weights[3], w11 = weights[4], w21 = weights[5]; + const T w02 = weights[6], w12 = weights[7], w22 = weights[8]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + const T* in = &inData[offset]; + T* out = &outData[offset]; + + T a0 = w00 * in[0]; + T a1 = w10 * in[0]; + T a2 = w20 * in[0]; + a0 += w01 * in[1]; + a1 += w11 * in[1]; + a2 += w21 * in[1]; + a0 += w02 * in[2]; + a1 += w12 * in[2]; + a2 += w22 * in[2]; + out[0] = a0; + out[1] = a1; + out[2] = a2; + } + } + else if constexpr (IsKernel(8, 1)) + { + const T w0 = weights[0], w1 = weights[1], w2 = weights[2], w3 = weights[3]; + const T w4 = weights[4], w5 = weights[5], w6 = weights[6], w7 = weights[7]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * InChannels; + const T* in = &inData[offset]; + + T a0 = w0 * in[0]; + a0 += w1 * in[1]; + a0 += w2 * in[2]; + a0 += w3 * in[3]; + a0 += w4 * in[4]; + a0 += w5 * in[5]; + a0 += w6 * in[6]; + a0 += w7 * in[7]; + outData[frame] = a0; + } + } + else if constexpr (IsKernel(3, 1)) + { + const T w0 = weights[0], w1 = weights[1], w2 = weights[2]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + const T* in = &inData[offset]; + + T a0 = w0 * in[0]; + a0 += w1 * in[1]; + a0 += w2 * in[2]; + outData[frame] = a0; + } + } + else if constexpr (IsKernel(1, 3)) + { + const T w0 = weights[0], w1 = weights[1], w2 = weights[2]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + T* out = &outData[offset]; + + const T in = inData[frame]; + + out[0] = w0 * in; + out[1] = w1 * in; + out[2] = w2 * in; + } + } + } + + static inline void MultiplyInitColwise(const T* inData, T* outData, const T* weights, const T* initData, size_t numFrames) + { + static_assert(HasKernel(), "Multiplication not implemented for InChannel/OutChannel combination"); + + if constexpr (IsKernel(3, 3)) + { + const T w00 = weights[0], w10 = weights[1], w20 = weights[2]; + const T w01 = weights[3], w11 = weights[4], w21 = weights[5]; + const T w02 = weights[6], w12 = weights[7], w22 = weights[8]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + const T* in = &inData[offset]; + T* out = &outData[offset]; + + T a0 = initData[0] + w00 * in[0]; + T a1 = initData[1] + w10 * in[0]; + T a2 = initData[2] + w20 * in[0]; + a0 += w01 * in[1]; + a1 += w11 * in[1]; + a2 += w21 * in[1]; + a0 += w02 * in[2]; + a1 += w12 * in[2]; + a2 += w22 * in[2]; + out[0] = a0; + out[1] = a1; + out[2] = a2; + } + } + else if constexpr (IsKernel(8, 1)) + { + const T w0 = weights[0], w1 = weights[1], w2 = weights[2], w3 = weights[3]; + const T w4 = weights[4], w5 = weights[5], w6 = weights[6], w7 = weights[7]; + + const T init = initData[0]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * InChannels; + const T* in = &inData[offset]; + + T a0 = init + w0 * in[0]; + a0 += w1 * in[1]; + a0 += w2 * in[2]; + a0 += w3 * in[3]; + a0 += w4 * in[4]; + a0 += w5 * in[5]; + a0 += w6 * in[6]; + a0 += w7 * in[7]; + outData[frame] = a0; + } + } + else if constexpr (IsKernel(3, 1)) + { + const T w0 = weights[0], w1 = weights[1], w2 = weights[2]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + const T* in = &inData[offset]; + + T a0 = initData[0] + w0 * in[0]; + a0 += w1 * in[1]; + a0 += w2 * in[2]; + outData[frame] = a0; + } + } + else if constexpr (IsKernel(1, 3)) + { + const T w0 = weights[0], w1 = weights[1], w2 = weights[2]; + const T init0 = inData[0], init1 = inData[1], init2 = inData[2]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + T* out = &outData[offset]; + + const T in = inData[frame]; + + out[0] = init0 + (w0 * in); + out[1] = init1 + (w1 * in); + out[2] = init2 + (w2 * in); + } + } + } + + static inline void MultiplyAccumlulate(const T* inData, T* outData, const T* weights, size_t numFrames) + { + static_assert(HasKernel(), "Multiplication not implemented for InChannel/OutChannel combination"); + + if constexpr (IsKernel(3, 3)) + { + const T w00 = weights[0], w10 = weights[1], w20 = weights[2]; + const T w01 = weights[3], w11 = weights[4], w21 = weights[5]; + const T w02 = weights[6], w12 = weights[7], w22 = weights[8]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + const T* in = &inData[offset]; + T* out = &outData[offset]; + + T a0 = out[0] + w00 * in[0]; + T a1 = out[1] + w10 * in[0]; + T a2 = out[2] + w20 * in[0]; + a0 += w01 * in[1]; + a1 += w11 * in[1]; + a2 += w21 * in[1]; + a0 += w02 * in[2]; + a1 += w12 * in[2]; + a2 += w22 * in[2]; + out[0] = a0; + out[1] = a1; + out[2] = a2; + } + } + else if constexpr (IsKernel(8, 1)) + { + const T w0 = weights[0], w1 = weights[1], w2 = weights[2], w3 = weights[3]; + const T w4 = weights[4], w5 = weights[5], w6 = weights[6], w7 = weights[7]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * InChannels; + const T* in = &inData[offset]; + + T a0 = outData[frame] + w0 * in[0]; + a0 += w1 * in[1]; + a0 += w2 * in[2]; + a0 += w3 * in[3]; + a0 += w4 * in[4]; + a0 += w5 * in[5]; + a0 += w6 * in[6]; + a0 += w7 * in[7]; + outData[frame] = a0; + } + } + else if constexpr (IsKernel(3, 1)) + { + const T w0 = weights[0], w1 = weights[1], w2 = weights[2]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const int offset = frame * 3; + const T* in = &inData[offset]; + + T a0 = outData[frame] + w0 * in[0]; + a0 += w1 * in[1]; + a0 += w2 * in[2]; + outData[frame] = a0; + } + } + else if constexpr (IsKernel(1, 3)) + { + const T w0 = weights[0], w1 = weights[1], w2 = weights[2]; + + for (size_t frame = 0; frame < numFrames; frame++) + { + const size_t offset = frame * 3; + T* out = &outData[offset]; + + const T in = inData[frame]; + + out[0] += w0 * in; + out[1] += w1 * in; + out[2] += w2 * in; + } + } + } + }; + +} \ No newline at end of file diff --git a/NeuralAudio/NAMModel.h b/NeuralAudio/NAMModel.h index 814ee33..a6fca73 100644 --- a/NeuralAudio/NAMModel.h +++ b/NeuralAudio/NAMModel.h @@ -1,14 +1,16 @@ #pragma once #include "NeuralModel.h" +#include "NeuralModelImpl.h" #include +#include #include -#include -#include +#include +#include namespace NeuralAudio { - class NAMModel : public NeuralModel + class NAMModel : public NeuralModelImpl { public: NAMModel() @@ -22,66 +24,94 @@ namespace NeuralAudio namModel.reset(); } - EModelLoadMode GetLoadMode() + EModelLoadMode GetLoadMode() override { return EModelLoadMode::NAMCore; } + int GetReceptiveFieldSize() override + { + return namModel->GetPrewarmSamples(); + } + bool LoadFromJson(const nlohmann::json& modelJson) { - if (namModel) - namModel.reset(); + slimmableSize = loader->GetDefaultQualityScaleFactor(); ReadNAMConfig(modelJson); - std::string arch = modelJson.at("architecture"); + nam::ScopedPrewarmOnResetDefault scoped_prewarm_default(false); - nlohmann::json config = modelJson.at("config"); + namModel = nam::get_dsp(modelJson); - std::vector weights = modelJson.at("weights"); + auto* slim = dynamic_cast(namModel.get()); - if (arch == "WaveNet") + if (slim != nullptr) { - std::vector layer_array_params; + isSlimmable = true; - for (size_t i = 0; i < config.at("layers").size(); i++) - { - nlohmann::json layerConfig = config.at("layers").at(i); + slim->SetSlimmableSize(slimmableSize); + } - layer_array_params.push_back( - nam::wavenet::LayerArrayParams(layerConfig.at("input_size"), layerConfig.at("condition_size"), layerConfig.at("head_size"), - layerConfig.at("channels"), layerConfig.at("kernel_size"), layerConfig.at("dilations"), - layerConfig.at("activation"), layerConfig.at("gated"), layerConfig.at("head_bias"))); - } + SetMaxAudioBufferSize(loader->GetDefaultMaxAudioBufferSize()); - const bool with_head = !config.at("head").is_null(); - const float head_scale = config.at("head_scale"); + namModel->SetPrewarmOnReset(true); - namModel = std::make_unique(layer_array_params, head_scale, with_head, weights, sampleRate); - } - else if (arch == "LSTM") + return true; + } + + bool HasQualityScaling() override + { + return isSlimmable; + } + + float GetQualityScaleFactor() override + { + return slimmableSize; + } + + void SetQualityScaleFactor(float scaleFactor) override + { + if (HasQualityScaling()) { - const int num_layers = config.at("num_layers"); - const int input_size = config.at("input_size"); - const int hidden_size = config.at("hidden_size"); + if (slimmableSize != scaleFactor) + { + slimmableSize = scaleFactor; + + if (namModel != nullptr) + { + auto* slim = dynamic_cast(namModel.get()); - namModel = std::make_unique(num_layers, input_size, hidden_size, weights, sampleRate); + slim->SetSlimmableSize(slimmableSize); + } + } } + } - return true; + void SetMaxAudioBufferSize(const int maxSize) override + { + if (maxSize != currentMaxSize) // Don't reset if we don't need to + { + namModel->Reset(namModel->GetExpectedSampleRate(), maxSize); + + currentMaxSize = maxSize; + } } - void Process(float* input, float* output, size_t numSamples) + void Process(const float* input, float* output, size_t numSamples) override { - namModel->process(input, output, (int)numSamples); + namModel->process(const_cast(&input), &output, (int)numSamples); } - void Prewarm() + void Prewarm() override { namModel->prewarm(); } private: std::unique_ptr namModel = nullptr; + int currentMaxSize = -1; + float slimmableSize = 1.0f; + bool isSlimmable = false; }; } \ No newline at end of file diff --git a/NeuralAudio/NeuralModel.cpp b/NeuralAudio/NeuralModel.cpp index 99562fb..54331e7 100644 --- a/NeuralAudio/NeuralModel.cpp +++ b/NeuralAudio/NeuralModel.cpp @@ -1,13 +1,17 @@ #include +#include #include "NeuralModel.h" #ifdef BUILD_NAMCORE #include "NAMModel.h" #endif +#ifdef BUILD_RTNEURAL #ifdef BUILD_STATIC_RTNEURAL #include "RTNeuralLoader.h" #endif #include "RTNeuralModel.h" +#endif #include "InternalModel.h" +#include "CompositeModel.h" namespace NeuralAudio { @@ -21,10 +25,10 @@ namespace NeuralAudio if (!modelDefsAreLoaded) { #ifdef BUILD_INTERNAL_STATIC_WAVENET - internalWavenetModelDefs.push_back(new InternalWaveNetDefinitionT<16, 8>); // Standard - internalWavenetModelDefs.push_back(new InternalWaveNetDefinitionT<12, 6>); // Lite - internalWavenetModelDefs.push_back(new InternalWaveNetDefinitionT<8, 4>); // Feather - internalWavenetModelDefs.push_back(new InternalWaveNetDefinitionT<4, 2>); // Nano + internalWavenetModelDefs.push_back(new InternalA1WaveNetDefinitionT<16, 8>); // Standard + internalWavenetModelDefs.push_back(new InternalA1WaveNetDefinitionT<12, 6>); // Lite + internalWavenetModelDefs.push_back(new InternalA1WaveNetDefinitionT<8, 4>); // Feather + internalWavenetModelDefs.push_back(new InternalA1WaveNetDefinitionT<4, 2>); // Nano #endif #ifdef BUILD_INTERNAL_STATIC_LSTM @@ -45,11 +49,11 @@ namespace NeuralAudio } } - static InternalWaveNetDefinitionBase* FindInternalWaveNetDefinition(size_t numChannels, size_t headSize) + static InternalWaveNetDefinitionBase* FindInternalWaveNetDefinition(size_t NumChannels, size_t HeadSize) { for (auto const& model : internalWavenetModelDefs) { - if ((numChannels == model->GetNumChannels()) && (headSize == model->GetHeadSize())) + if ((NumChannels == model->GetNumChannels()) && (HeadSize == model->GetHeadSize())) return model; } @@ -71,21 +75,64 @@ namespace NeuralAudio static std::vector liteDilations = { 1, 2, 4, 8, 16, 32, 64 }; static std::vector liteDilations2 = { 128, 256, 512, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 }; - static bool CheckDilations(const nlohmann::json dilationJson, std::vector& checkDilations) + static std::vector a2KernelSizes = { 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 15, 15, 6, 6, 6, 6, 6, 6, 6 }; + static std::vector a2Dilations = { 1, 3, 7, 17, 41, 101, 239, 1, 3, 7, 17, 41, 101, 239, 1, 13, 1, 3, 7, 17, 41, 101, 239 }; + + static bool CheckIntegerSequence(const nlohmann::json& sequenceJson, std::vector& sequenceVector) { - if (dilationJson.size() != checkDilations.size()) + if (sequenceJson.size() != sequenceVector.size()) return false; - for (size_t i = 0; i < dilationJson.size(); i++) + for (size_t i = 0; i < sequenceJson.size(); i++) { - if (dilationJson[i] != checkDilations[i]) + if (sequenceJson[i] != sequenceVector[i]) return false; } return true; } - bool NeuralModel::SupportsWaveNetLoadMode(EModelLoadMode mode) + static void OversampleNAMConfig(nlohmann::json& modelJson, int externalSampleRate) + { + std::string arch = modelJson.at("architecture"); + + if (arch != "WaveNet") + return; + + int modelSampleRate = 48000; + + if (modelJson.contains("sample_rate") && modelJson.at("sample_rate").is_number()) + { + modelSampleRate = (int)modelJson.at("sample_rate").get(); + } + + if (modelSampleRate == externalSampleRate) + return; // No oversampling required + + if ((externalSampleRate % modelSampleRate) != 0) + return; // Not an integer multiple + + int oversampleFactor = externalSampleRate / modelSampleRate; + + auto& config = modelJson.at("config"); + + for (auto& layer : config.at("layers")) + { + for (auto& dilation : layer.at("dilations")) + { + dilation = dilation.get() * oversampleFactor; + } + + if (layer.contains("head")) + { + auto& head = layer.at("head"); + + head["head_dilation"] = oversampleFactor; + } + } + } + + bool NeuralModelLoader::SupportsWaveNetLoadMode(EModelLoadMode mode) { if (mode == EModelLoadMode::NAMCore) #ifdef BUILD_NAMCORE @@ -95,16 +142,12 @@ namespace NeuralAudio #endif if (mode == EModelLoadMode::RTNeural) -#ifdef BUILD_STATIC_RTNEURAL - return true; -#else return false; -#endif return true; } - bool NeuralModel::SupportsLSTMLoadMode(EModelLoadMode mode) + bool NeuralModelLoader::SupportsLSTMLoadMode(EModelLoadMode mode) { if (mode == EModelLoadMode::NAMCore) #ifdef BUILD_NAMCORE @@ -116,50 +159,274 @@ namespace NeuralAudio return true; } - NeuralModel* NeuralModel::CreateFromFile(std::filesystem::path modelPath) + bool NAMIsA2(std::string version) + { + int major = 0, minor = 0, patch = 0; + char dot; + std::stringstream ss(version); + + ss >> major >> dot >> minor >> dot >> patch; + + return (major > 0) || (minor > 5) || ((minor == 5) && (patch > 4)); + } + + bool ApproxEqual(float a, float b, float epsilon = 1e-5f) + { + return std::fabs(a - b) <= epsilon; + } + + bool IsActive(const nlohmann::json& json, const char* name) + { + if (!json.contains(name)) + return true; + + return json.at(name).value("active", false); + } + + bool HasNonNull(const nlohmann::json& json, const char* name) + { + return json.contains(name) && !json.at(name).is_null(); + } + + bool NAMIsA2Standard(const nlohmann::json& modelJson) + { + if (!modelJson.contains("architecture")) + return false; + + if (modelJson.at("architecture") != "WaveNet") + return false; + + if (!modelJson.contains("config")) + return false; + + auto& config = modelJson.at("config"); + + if (HasNonNull(config, "head")) + return false; + + if (config.contains("condition_dsp")) + return false; + + if (config.value("in_channels", 1) != 1) + return false; + + if (!config.contains("layers")) + return false; + + if (config.at("layers").size() != 1) + return false; + + const auto& layerConfig = config.at("layers").at(0); + + if (layerConfig.value("input_size", 0) != 1) + return false; + + if (layerConfig.value("condition_size", 0) != 1) + return false; + + const int channels = layerConfig.value("channels", 0); + + if ((channels != 3) && (channels != 8)) + return false; + + if (layerConfig.value("bottleneck", channels) != channels) + return false; + + if (!layerConfig.contains("kernel_sizes")) + return false; + + if (!CheckIntegerSequence(layerConfig.at("kernel_sizes"), a2KernelSizes)) + return false; + + if (!layerConfig.contains("dilations")) + return false; + + if (!CheckIntegerSequence(layerConfig.at("dilations"), a2Dilations)) + return false; + + if (!layerConfig.contains("activation")) + return false; + + for (const auto& activation : layerConfig.at("activation")) + { + if (!activation.contains("type")) + return false; + + if (activation.at("type") != "LeakyReLU") + return false; + + if (!ApproxEqual(activation.value("negative_slope", 0.01f), 0.01f)) + return false; + } + + if (layerConfig.contains("secondary_activation")) + { + for (const auto& activation : layerConfig.at("secondary_activation")) + { + if (!activation.is_null()) + return false; + } + } + + if (layerConfig.contains("gating_mode")) + { + for (const auto& gating : layerConfig.at("gating_mode")) + { + if (!gating.is_null() && (gating != "none")) + return false; + } + } + + if (!layerConfig.contains("head")) + return false; + + const auto& head = layerConfig.at("head"); + + if (head.value("out_channels", 1) != 1) + return false; + + if (head.value("kernel_size", 16) != 16) + return false; + + if (head.value("head_dilation", 1) != 1) + return false; + + if (!head.value("bias", true)) + return false; + + if (!IsActive(layerConfig, "layer1x1")) + return false; + + if (layerConfig.at("layer1x1").value("groups", 1) != 1) + return false; + + for (auto& key : { "head1x1", "conv_pre_film", "conv_post_film", "input_mixin_pre_film", "input_mixin_post_film", + "activation_pre_film", "activation_post_film", "layer1x1_post_film", "head1x1_post_film" }) + { + if (IsActive(layerConfig, key)) + return false; + } + + if (layerConfig.value("groups_input", 1) != 1) + return false; + + if (layerConfig.value("groups_input_mixin", 1) != 1) + return false; + + if (HasNonNull(layerConfig, "slimmable")) + return false; + + return true; + } + + NeuralModel* NeuralModelLoader::CreateFromFile(const std::filesystem::path& modelPath, bool doPrewarm) { if (!std::filesystem::exists(modelPath)) return nullptr; std::ifstream jsonStream(modelPath, std::ifstream::binary); - return CreateFromStream(jsonStream, modelPath.extension()); + return CreateFromStream(jsonStream, modelPath.extension(), doPrewarm); } - NeuralModel* NeuralModel::CreateFromStream(std::basic_istream& jsonStream, std::filesystem::path extension) - { - EnsureModelDefsAreLoaded(); + NeuralModel* NeuralModelLoader::CreateFromStream(std::basic_istream& jsonStream, const std::filesystem::path& extension, bool doPrewarm) + { nlohmann::json modelJson; jsonStream >> modelJson; - NeuralModel* newModel = nullptr; + return CreateFromJson(modelJson, extension, doPrewarm); + } + + NeuralModel* NeuralModelLoader::CreateFromJson(nlohmann::json& modelJson, const std::filesystem::path& extension, bool doPrewarm) + { + EnsureModelDefsAreLoaded(); + + NeuralModelImpl* newModel = nullptr; if (extension == ".nam") { + OversampleNAMConfig(modelJson, externalSampleRate); + std::string arch = modelJson.at("architecture"); -#ifdef BUILD_NAMCORE - if (wavenetLoadMode == EModelLoadMode::NAMCore) + if (arch == "SlimmableContainer") // Packed A2 multi-model file { - NAMModel* model = new NAMModel; + ScalableCompositeModel* model = new ScalableCompositeModel; + model->SetModelLoader(this); model->LoadFromJson(modelJson); newModel = model; } + + + if (newModel == nullptr) + { + std::string version = modelJson.at("version"); + +#ifdef BUILD_NAMCORE + bool loadA2WithNAMCore = true; + +#ifdef BUILD_STATIC_INTERNAL_NAMA2 + loadA2WithNAMCore = false; #endif + if ((wavenetLoadMode == EModelLoadMode::NAMCore) || (NAMIsA2(version) && (loadA2WithNAMCore || !NAMIsA2Standard(modelJson)))) + { + NAMModel* model = new NAMModel; + + model->SetModelLoader(this); + model->LoadFromJson(modelJson); + + newModel = model; + } +#endif + } + + auto& config = modelJson.at("config"); if (newModel == nullptr) { if (arch == "WaveNet") { - nlohmann::json config = modelJson.at("config"); + if (config.at("layers").size() == 1) + { +#ifdef BUILD_STATIC_INTERNAL_NAMA2 + auto& layerConfig = config.at("layers").at(0); + + if (CheckIntegerSequence(layerConfig.at("dilations"), a2Dilations)) + { + if (layerConfig.at("channels") == 3) + { + auto model = new InternalWaveNetModelT>>(); - if (config.at("layers").size() == 2) + if (model != nullptr) + { + model->SetModelLoader(this); + model->LoadFromNAMJson(modelJson); + + newModel = model; + } + } + else if (layerConfig.at("channels") == 8) + { + auto model = new InternalWaveNetModelT >>(); + + if (model != nullptr) + { + model->SetModelLoader(this); + model->LoadFromNAMJson(modelJson); + + newModel = model; + } + } + } +#endif + } + else if (config.at("layers").size() == 2) { - nlohmann::json firstLayerConfig = config.at("layers").at(0); - nlohmann::json secondLayerConfig = config.at("layers").at(1); + auto& firstLayerConfig = config.at("layers").at(0); + auto& secondLayerConfig = config.at("layers").at(1); if (!firstLayerConfig.at("gated") && !secondLayerConfig.at("gated") && !firstLayerConfig.at("head_bias") && secondLayerConfig.at("head_bias")) { @@ -167,14 +434,14 @@ namespace NeuralAudio if (firstLayerConfig.at("channels") == 16) { - if (CheckDilations(firstLayerConfig.at("dilations"), stdDilations) && CheckDilations(secondLayerConfig.at("dilations"), stdDilations)) + if (CheckIntegerSequence(firstLayerConfig.at("dilations"), stdDilations) && CheckIntegerSequence(secondLayerConfig.at("dilations"), stdDilations)) { isOfficialArchitecture = true; } } else { - if (CheckDilations(firstLayerConfig.at("dilations"), liteDilations) && CheckDilations(secondLayerConfig.at("dilations"), liteDilations2)) + if (CheckIntegerSequence(firstLayerConfig.at("dilations"), liteDilations) && CheckIntegerSequence(secondLayerConfig.at("dilations"), liteDilations2)) { isOfficialArchitecture = true; } @@ -182,13 +449,6 @@ namespace NeuralAudio if (isOfficialArchitecture) { - if (wavenetLoadMode == EModelLoadMode::RTNeural) - { -#ifdef BUILD_STATIC_RTNEURAL - newModel = RTNeuralLoadNAMWaveNet(modelJson); -#endif - } - if (newModel == nullptr) { auto modelDef = FindInternalWaveNetDefinition(firstLayerConfig.at("channels"), firstLayerConfig.at("head_size")); @@ -197,6 +457,7 @@ namespace NeuralAudio { auto model = modelDef->CreateModel(); + model->SetModelLoader(this); model->LoadFromNAMJson(modelJson); newModel = model; @@ -211,6 +472,8 @@ namespace NeuralAudio // Use a dynamic model if we had no static definition InternalWaveNetModelDyn* model = new InternalWaveNetModelDyn; + model->SetModelLoader(this); + if (model->LoadFromNAMJson(modelJson)) { newModel = model; @@ -219,12 +482,10 @@ namespace NeuralAudio } else if (arch == "LSTM") { - nlohmann::json config = modelJson.at("config"); - #ifdef BUILD_STATIC_RTNEURAL if (lstmLoadMode == EModelLoadMode::RTNeural) { - newModel = RTNeuralLoadNAMLSTM(modelJson); + newModel = RTNeuralLoadNAMLSTM(modelJson, this); } #endif @@ -235,6 +496,8 @@ namespace NeuralAudio if (modelDef != nullptr) { auto model = modelDef->CreateModel(); + + model->SetModelLoader(this); model->LoadFromNAMJson(modelJson); newModel = model; @@ -245,6 +508,8 @@ namespace NeuralAudio { InternalLSTMModelDyn* model = new InternalLSTMModelDyn; + model->SetModelLoader(this); + if (model->LoadFromNAMJson(modelJson)) { newModel = model; @@ -266,7 +531,7 @@ namespace NeuralAudio #ifdef BUILD_STATIC_RTNEURAL if (lstmLoadMode == EModelLoadMode::RTNeural) { - newModel = RTNeuralLoadKeras(modelJson); + newModel = RTNeuralLoadKeras(modelJson, this); } #endif @@ -300,6 +565,7 @@ namespace NeuralAudio } } +#ifdef BUILD_RTNEURAL if (newModel == nullptr) { // Use a dynamic model for other model types @@ -308,59 +574,14 @@ namespace NeuralAudio newModel = model; } +#endif } - if (newModel != nullptr) + if ((newModel != nullptr) && doPrewarm) { newModel->Prewarm(); } return newModel; } - - void NeuralModel::ReadNAMConfig(const nlohmann::json& modelJson) - { - if (modelJson.contains("sample_rate") && modelJson.at("sample_rate").is_number()) - { - sampleRate = modelJson.at("sample_rate"); - } - - if (modelJson.contains("metadata")) - { - nlohmann::json metaData = modelJson.at("metadata"); - - if (metaData.contains("loudness") && metaData.at("loudness").is_number_float()) - { - modelLoudnessDB = (float)metaData.at("loudness"); - } - - if (metaData.contains("input_level_dbu") && metaData.at("input_level_dbu").is_number_float()) - { - modelInputLevelDBu = metaData.at("input_level_dbu"); - } - - if (metaData.contains("output_level_dbu") && metaData.at("output_level_dbu").is_number_float()) - { - modelOutputLevelDBu = metaData.at("output_level_dbu"); - } - } - } - - void NeuralModel::ReadKerasConfig(const nlohmann::json& modelJson) - { - if (modelJson.contains("samplerate") && modelJson.at("samplerate").is_number()) - { - sampleRate = modelJson.at("samplerate"); - } - - if (modelJson.contains("in_gain") && modelJson.at("in_gain").is_number_float()) - { - modelInputLevelDBu = modelJson.at("in_gain"); - } - - if (modelJson.contains("out_gain") && modelJson.at("out_gain").is_number_float()) - { - modelLoudnessDB = -18 - (float)modelJson.at("out_gain"); - } - } } \ No newline at end of file diff --git a/NeuralAudio/NeuralModel.h b/NeuralAudio/NeuralModel.h index 0e1483b..5b2978e 100644 --- a/NeuralAudio/NeuralModel.h +++ b/NeuralAudio/NeuralModel.h @@ -2,8 +2,19 @@ #include #include +#include +#include +#include #include "json.hpp" +#ifndef DEFAULT_QUALITY_SCALE +#define DEFAULT_QUALITY_SCALE 1.0 +#endif + +#ifndef DEFAULT_INPUT_DBU +#define DEFAULT_INPUT_DBU 12 +#endif + namespace NeuralAudio { enum EModelLoadMode @@ -13,62 +24,64 @@ namespace NeuralAudio NAMCore }; + enum ECompositeModelLoadMode + { + LoadAll, + OnDemand + }; + class NeuralModel { public: - static NeuralModel* CreateFromFile(std::filesystem::path modelPath); - static NeuralModel* CreateFromStream(std::basic_istream& stream, std::filesystem::path extension); - virtual ~NeuralModel() { } - static bool SetLSTMLoadMode(EModelLoadMode val) + virtual EModelLoadMode GetLoadMode() { - if (!SupportsLSTMLoadMode(val)) - return false; - - lstmLoadMode = val; + return EModelLoadMode::Internal; + } - return true; + virtual bool HasQualityScaling() + { + return false; } - static bool SetWaveNetLoadMode(EModelLoadMode val) + virtual float GetQualityScaleFactor() { - if (!SupportsWaveNetLoadMode(val)) - return false; + return 1.0f; + } - wavenetLoadMode = val; + virtual bool IsQualityChangeRealtimeSafe(float newScaleFactor) + { + (void)newScaleFactor; return true; } - static bool SupportsWaveNetLoadMode(EModelLoadMode mode); - static bool SupportsLSTMLoadMode(EModelLoadMode mode); - - static void SetAudioInputLevelDBu(float audioDBu) + virtual void SetQualityScaleFactor(float scaleFactor) { - audioInputLevelDBu = audioDBu; + (void)scaleFactor; } - static void SetDefaultMaxAudioBufferSize(int maxSize) + virtual bool IsStatic() { - defaultMaxAudioBufferSize = maxSize; + return false; } - virtual EModelLoadMode GetLoadMode() + virtual void SetMaxAudioBufferSize(const int maxSize) { - return EModelLoadMode::Internal; + (void)maxSize; } - virtual bool IsStatic() + virtual void SetAudioInputLevelDBu(float audioDBu) { - return false; + audioInputLevelDBu = audioDBu; } - virtual void SetMaxAudioBufferSize(int maxSize) + virtual float GetAudioInputLevelDBu() { - (void)maxSize; + return audioInputLevelDBu; } virtual float GetRecommendedInputDBAdjustment() @@ -91,7 +104,27 @@ namespace NeuralAudio return -1; // No fixed receptive field size (ie: for LSTM) } - virtual void Process(float* input, float* output, size_t numSamples) + virtual std::string GetModelVersion() + { + return modelVersion; + } + + virtual std::string GetMetadata(const std::string& fieldName) + { + auto it = std::find_if(metadata.begin(), metadata.end(), [&](const auto& pair) + { + return pair.first == fieldName; + }); + + if (it != metadata.end()) + { + return it->second; + } + + return ""; + } + + virtual void Process(const float* input, float* output, size_t numSamples) { (void)input; (void)output; @@ -103,32 +136,98 @@ namespace NeuralAudio } protected: - void ReadNAMConfig(const nlohmann::json& modelJson); - void ReadKerasConfig(const nlohmann::json& modelJson); - + float audioInputLevelDBu = (float)DEFAULT_INPUT_DBU; float modelInputLevelDBu = 12; float modelOutputLevelDBu = 12; float modelLoudnessDB = -18; float sampleRate = 48000; + std::string modelVersion = ""; + std::vector> metadata; + }; - inline static float audioInputLevelDBu = 12; - inline static EModelLoadMode lstmLoadMode = EModelLoadMode::Internal; - inline static EModelLoadMode wavenetLoadMode = EModelLoadMode::Internal; - inline static int defaultMaxAudioBufferSize = 128; + class NeuralModelLoader + { + public: + NeuralModel* CreateFromFile(const std::filesystem::path& modelPath, bool doPrewarm = true); + NeuralModel* CreateFromStream(std::basic_istream& stream, const std::filesystem::path& extension, bool doPrewarm = true); + NeuralModel* CreateFromJson(nlohmann::json& modelJson, const std::filesystem::path& extension, bool doPrewarm = true); - void Prewarm(size_t numSamples, size_t blockSize) - { - std::vector input; - input.resize(blockSize); - std::fill(input.begin(), input.end(), 0.0f); + bool SetLSTMLoadMode(EModelLoadMode val) + { + if (!SupportsLSTMLoadMode(val)) + return false; - std::vector output; - output.resize(blockSize); + lstmLoadMode = val; + + return true; + } - for (size_t block = 0; block < (numSamples / blockSize); block++) + bool SetWaveNetLoadMode(EModelLoadMode val) { - Process(input.data(), output.data(), blockSize); + if (!SupportsWaveNetLoadMode(val)) + return false; + + wavenetLoadMode = val; + + return true; } - } + + ECompositeModelLoadMode GetCompositeModelLoadMode() + { + return compositeLoadMode; + } + + void SetCompositeModelLoadMode(ECompositeModelLoadMode loadMode) + { + compositeLoadMode = loadMode; + } + + bool SupportsWaveNetLoadMode(EModelLoadMode mode); + bool SupportsLSTMLoadMode(EModelLoadMode mode); + + void SetAudioInputLevelDBu(float audioDBu) + { + audioInputLevelDBu = audioDBu; + } + + float GetAudioInputLevelDBu() + { + return audioInputLevelDBu; + } + + void SetDefaultMaxAudioBufferSize(int maxSize) + { + defaultMaxAudioBufferSize = maxSize; + } + + int GetDefaultMaxAudioBufferSize() + { + return defaultMaxAudioBufferSize; + } + + void SetDefaultQualityScaleFactor(float scaleFactor) + { + defaultQualityScaleFactor = scaleFactor; + } + + float GetDefaultQualityScaleFactor() + { + return defaultQualityScaleFactor; + } + + void SetExternalSampleRate(int sampleRate) + { + this->externalSampleRate = sampleRate; + } + + protected: + EModelLoadMode lstmLoadMode = EModelLoadMode::Internal; + EModelLoadMode wavenetLoadMode = EModelLoadMode::Internal; + ECompositeModelLoadMode compositeLoadMode = ECompositeModelLoadMode::LoadAll; + float audioInputLevelDBu = (float)DEFAULT_INPUT_DBU; + int defaultMaxAudioBufferSize = 128; + float defaultQualityScaleFactor = (float)DEFAULT_QUALITY_SCALE; + int externalSampleRate = 48000; }; + } \ No newline at end of file diff --git a/NeuralAudio/NeuralModelImpl.h b/NeuralAudio/NeuralModelImpl.h new file mode 100644 index 0000000..160581f --- /dev/null +++ b/NeuralAudio/NeuralModelImpl.h @@ -0,0 +1,114 @@ +#pragma once + +#include "NeuralModel.h" + +namespace NeuralAudio +{ + class NeuralModelImpl : public NeuralModel + { + public: + using NeuralModel::Prewarm; + + void SetModelLoader(NeuralModelLoader* modelLoader) + { + this->loader = modelLoader; + + SetAudioInputLevelDBu(modelLoader->GetAudioInputLevelDBu()); + } + + bool HadInitialPrewarm() + { + return hadInitialPrewarm; + } + + void SetHadInitialPrewarm() + { + hadInitialPrewarm = true; + } + + protected: + void ReadNAMConfig(const nlohmann::json& modelJson) + { + modelVersion = modelJson.at("version"); + + if (modelJson.contains("sample_rate") && modelJson.at("sample_rate").is_number()) + { + sampleRate = modelJson.at("sample_rate"); + } + + if (modelJson.contains("metadata")) + { + auto& metadataJson = modelJson.at("metadata"); + + AddMetadata(metadataJson); + + if (metadataJson.contains("loudness") && metadataJson.at("loudness").is_number()) + { + modelLoudnessDB = (float)metadataJson.at("loudness"); + } + + if (metadataJson.contains("input_level_dbu") && metadataJson.at("input_level_dbu").is_number()) + { + modelInputLevelDBu = metadataJson.at("input_level_dbu"); + } + + if (metadataJson.contains("output_level_dbu") && metadataJson.at("output_level_dbu").is_number()) + { + modelOutputLevelDBu = metadataJson.at("output_level_dbu"); + } + } + } + + void ReadKerasConfig(const nlohmann::json& modelJson) + { + if (modelJson.contains("samplerate") && modelJson.at("samplerate").is_number()) + { + sampleRate = modelJson.at("samplerate"); + } + + if (modelJson.contains("in_gain") && modelJson.at("in_gain").is_number()) + { + modelInputLevelDBu = modelJson.at("in_gain"); + } + + if (modelJson.contains("out_gain") && modelJson.at("out_gain").is_number()) + { + modelLoudnessDB = -18 - (float)modelJson.at("out_gain"); + } + } + + void AddMetadata(std::string fieldName, std::string fieldValue) + { + metadata.push_back({ fieldName, fieldValue }); + } + + void AddMetadata(const nlohmann::json& metadataJson) + { + for (auto& [key, value] : metadataJson.items()) + { + if (!value.is_null()) + { + AddMetadata(key, value.dump()); + } + } + } + + void Prewarm(size_t numSamples, size_t blockSize) + { + std::vector input; + input.resize(blockSize); + std::fill(input.begin(), input.end(), 0.0f); + + std::vector output; + output.resize(blockSize); + + for (size_t block = 0; block < (numSamples / blockSize); block++) + { + Process(input.data(), output.data(), blockSize); + } + } + + NeuralModelLoader *loader; + bool hadInitialPrewarm = false; + }; +} \ No newline at end of file diff --git a/NeuralAudio/RTNeuralLoader.cpp b/NeuralAudio/RTNeuralLoader.cpp index 3afde01..1ade6b3 100644 --- a/NeuralAudio/RTNeuralLoader.cpp +++ b/NeuralAudio/RTNeuralLoader.cpp @@ -1,115 +1,81 @@ +#ifdef BUILD_STATIC_RTNEURAL + #include #include "RTNeuralModel.h" namespace NeuralAudio { -#ifdef BUILD_STATIC_RTNEURAL - std::list rtNeuralLSTMModelDefs; - std::list rtNeuralWaveNetModelDefs; - - void EnsureRTNeuralModelDefsAreLoaded() + std::list rtNeuralLSTMModelDefs; + + void EnsureRTNeuralModelDefsAreLoaded() + { + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 8>); + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 12>); + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 16>); + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 24>); + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 8>); + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 12>); + rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 16>); + } + + RTNeuralLSTMDefinitionBase* FindRTNeuralLSTMDefinition(size_t numLayers, size_t hiddenSize) + { + for (auto const& model : rtNeuralLSTMModelDefs) { - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 8>); - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 12>); - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 16>); - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<1, 24>); - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 8>); - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 12>); - rtNeuralLSTMModelDefs.push_back(new RTNeuralLSTMDefinitionT<2, 16>); - - rtNeuralWaveNetModelDefs.push_back(new RTNeuralWaveNetDefinitionT<16, 8>); // Standard - rtNeuralWaveNetModelDefs.push_back(new RTNeuralWaveNetDefinitionT<12, 6>); // Lite - rtNeuralWaveNetModelDefs.push_back(new RTNeuralWaveNetDefinitionT<8, 4>); // Feather - rtNeuralWaveNetModelDefs.push_back(new RTNeuralWaveNetDefinitionT<4, 2>); // Nano + if ((numLayers == model->GetNumLayers()) && (hiddenSize == model->GetHiddenSize())) + return model; } - RTNeuralLSTMDefinitionBase* FindRTNeuralLSTMDefinition(size_t numLayers, size_t hiddenSize) - { - for (auto const& model : rtNeuralLSTMModelDefs) - { - if ((numLayers == model->GetNumLayers()) && (hiddenSize == model->GetHiddenSize())) - return model; - } + return nullptr; + } - return nullptr; - } + NeuralModelImpl* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader *loader) + { + auto& config = modelJson.at("config"); - RTNeuralWaveNetDefinitionBase* FindRTNeuralWaveNetDefinition(size_t numChannels, size_t headSize) - { - for (auto const& model : rtNeuralWaveNetModelDefs) - { - if ((numChannels == model->GetNumChannels()) && (headSize == model->GetHeadSize())) - return model; - } - - return nullptr; - } + auto modelDef = FindRTNeuralLSTMDefinition(config.at("num_layers"), config.at("hidden_size")); - NeuralModel* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson) + if (modelDef != nullptr) { - nlohmann::json config = modelJson.at("config"); + RTNeuralModel* model = modelDef->CreateModel(); - nlohmann::json firstLayerConfig = config.at("layers").at(0); - nlohmann::json secondLayerConfig = config.at("layers").at(1); - - auto modelDef = FindRTNeuralWaveNetDefinition(firstLayerConfig.at("channels"), firstLayerConfig.at("head_size")); + model->SetModelLoader(loader); + model->LoadFromNAMJson(modelJson); - if (modelDef != nullptr) - { - auto model = modelDef->CreateModel(); - - model->LoadFromNAMJson(modelJson); - - return model; - } - - return nullptr; + return model; } - NeuralModel* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson) - { - nlohmann::json config = modelJson.at("config"); - - auto modelDef = FindRTNeuralLSTMDefinition(config.at("num_layers"), config.at("hidden_size")); + // If we didn't have a static model that matched, use RTNeural's dynamic model + RTNeuralModelDyn* dynModel = new RTNeuralModelDyn; - if (modelDef != nullptr) - { - RTNeuralModel* model = modelDef->CreateModel(); - model->LoadFromNAMJson(modelJson); + dynModel->SetModelLoader(loader); + dynModel->LoadFromNAMJson(modelJson); - if (model != nullptr) - return model; + return dynModel; - // If we didn't have a static model that matched, use RTNeural's dynamic model - RTNeuralModelDyn* dynModel = new RTNeuralModelDyn; - dynModel->LoadFromNAMJson(modelJson); + return nullptr; + } - return dynModel; - } + NeuralModelImpl* RTNeuralLoadKeras(const nlohmann::json& modelJson, NeuralModelLoader* loader) + { + const auto layers = modelJson.at("layers"); + const size_t numLayers = layers.size() - 1; + const std::string modelType = layers.at(0).at("type"); + const int hiddenSize = layers.at(0).at("shape").back(); - return nullptr; - } + auto modelDef = FindRTNeuralLSTMDefinition(numLayers, hiddenSize); - NeuralModel* RTNeuralLoadKeras(const nlohmann::json& modelJson) + if (modelDef != nullptr) { - const auto layers = modelJson.at("layers"); - const size_t numLayers = layers.size() - 1; - const std::string modelType = layers.at(0).at("type"); - const int hiddenSize = layers.at(0).at("shape").back(); - - auto modelDef = FindRTNeuralLSTMDefinition(numLayers, hiddenSize); - - if (modelDef != nullptr) - { - RTNeuralModel* model = modelDef->CreateModel(); + RTNeuralModel* model = modelDef->CreateModel(); - model->LoadFromKerasJson(modelJson); + model->SetModelLoader(loader); + model->LoadFromKerasJson(modelJson); - return model; - } - - return nullptr; + return model; } -#endif -} + return nullptr; + } +} +#endif diff --git a/NeuralAudio/RTNeuralLoader.h b/NeuralAudio/RTNeuralLoader.h index 7e6cb23..f8b0156 100644 --- a/NeuralAudio/RTNeuralLoader.h +++ b/NeuralAudio/RTNeuralLoader.h @@ -1,11 +1,16 @@ #pragma once +#ifdef BUILD_RTNEURAL + #include "NeuralModel.h" +#include "NeuralModelImpl.h" namespace NeuralAudio { +#ifdef BUILD_STATIC_RTNEURAL extern void EnsureRTNeuralModelDefsAreLoaded(); - extern NeuralModel* RTNeuralLoadNAMWaveNet(const nlohmann::json& modelJson); - extern NeuralModel* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson); - extern NeuralModel* RTNeuralLoadKeras(const nlohmann::json& modelJson); -} \ No newline at end of file + extern NeuralModelImpl* RTNeuralLoadNAMLSTM(const nlohmann::json& modelJson, NeuralModelLoader* loader); +#endif + extern NeuralModelImpl* RTNeuralLoadKeras(const nlohmann::json& modelJson, NeuralModelLoader* loader); +} +#endif \ No newline at end of file diff --git a/NeuralAudio/RTNeuralModel.h b/NeuralAudio/RTNeuralModel.h index e01c9b9..f904daa 100644 --- a/NeuralAudio/RTNeuralModel.h +++ b/NeuralAudio/RTNeuralModel.h @@ -1,13 +1,13 @@ #pragma once +#ifdef BUILD_RTNEURAL + #include #include #include #include "NeuralModel.h" +#include "NeuralModelImpl.h" #include -#ifdef BUILD_STATIC_RTNEURAL -#include "wavenet_model.hpp" -#endif #include "TemplateHelper.h" namespace NeuralAudio @@ -46,21 +46,19 @@ namespace NeuralAudio } }; - class RTNeuralModel : public NeuralModel + class RTNeuralModel : public NeuralModelImpl { public: - EModelLoadMode GetLoadMode() + EModelLoadMode GetLoadMode() override { return EModelLoadMode::RTNeural; } - bool LoadFromKerasJson(const nlohmann::json& modelJson) + virtual bool LoadFromKerasJson(const nlohmann::json& modelJson) { ReadKerasConfig(modelJson); return CreateModelFromKerasJson(modelJson); - - return true; } virtual bool CreateModelFromKerasJson(const nlohmann::json& modelJson) @@ -110,12 +108,12 @@ namespace NeuralAudio } } - bool IsStatic() + bool IsStatic() override { return true; } - bool CreateModelFromKerasJson(const nlohmann::json& modelJson) + bool CreateModelFromKerasJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -131,7 +129,7 @@ namespace NeuralAudio return true; } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { if (model != nullptr) { @@ -141,8 +139,6 @@ namespace NeuralAudio model = new ModelType; - nlohmann::json config = modelJson.at("config"); - std::vector weights = modelJson.at("weights"); const int networkInputSize = 1; @@ -229,13 +225,13 @@ namespace NeuralAudio return true; } - void Process(float* input, float* output, size_t numSamples) + void Process(const float* input, float* output, size_t numSamples) override { for (size_t i = 0; i < numSamples; i++) output[i] = model->forward(input + i); } - void Prewarm() + void Prewarm() override { float sample = 0; @@ -247,83 +243,6 @@ namespace NeuralAudio ModelType* model = nullptr; }; - using StdDilations = wavenet::Dilations<1, 2, 4, 8, 16, 32, 64, 128, 256, 512>; - using LiteDilations1 = wavenet::Dilations<1, 2, 4, 8, 16, 32, 64>; - using LiteDilations2 = wavenet::Dilations<128, 256, 512, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512>; - - template - class RTNeuralWaveNetModelT : public RTNeuralModel - { - using ModelType = typename std::conditional, - wavenet::Layer_Array>, - wavenet::Wavenet_Model, - wavenet::Layer_Array> - >::type; - - public: - RTNeuralWaveNetModelT() - : model(nullptr) - { - } - - ~RTNeuralWaveNetModelT() - { - if (model != nullptr) - { - delete model; - model = nullptr; - } - } - - bool IsStatic() - { - return true; - } - - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) - { - if (model != nullptr) - { - delete model; - model = nullptr; - } - - model = new ModelType; - - nlohmann::json config = modelJson.at("config"); - - model->load_weights(modelJson); - - SetMaxAudioBufferSize(defaultMaxAudioBufferSize); - - return true; - } - - void SetMaxAudioBufferSize(int maxSize) - { - model->prepare(maxSize); - } - - void Process(float* input, float* output, size_t numSamples) - { - model->forward(input, output, (int)numSamples); - } - - void Prewarm() - { - float sample = 0; - - for (size_t i = 0; i < 2048; i++) - model->forward(sample); - } - - private: - ModelType* model = nullptr; - }; - class RTNeuralModelDefinitionBase { public: @@ -336,7 +255,7 @@ namespace NeuralAudio class RTNeuralLSTMDefinitionBase : public RTNeuralModelDefinitionBase { public: - virtual RTNeuralModel* CreateModel() + RTNeuralModel* CreateModel() override { return nullptr; } @@ -356,61 +275,22 @@ namespace NeuralAudio class RTNeuralLSTMDefinitionT : public RTNeuralLSTMDefinitionBase { public: - RTNeuralModel* CreateModel() + RTNeuralModel* CreateModel() override { return new RTNeuralLSTMModelT; } - size_t GetNumLayers() + size_t GetNumLayers() override { return numLayers; } - size_t GetHiddenSize() + size_t GetHiddenSize() override { return hiddenSize; } }; - class RTNeuralWaveNetDefinitionBase : public RTNeuralModelDefinitionBase - { - public: - virtual RTNeuralModel* CreateModel() - { - return nullptr; - } - - virtual size_t GetNumChannels() - { - return 0; - } - - virtual size_t GetHeadSize() - { - return 0; - } - }; - - template - class RTNeuralWaveNetDefinitionT : public RTNeuralWaveNetDefinitionBase - { - public: - RTNeuralModel* CreateModel() - { - return new RTNeuralWaveNetModelT; - } - - virtual size_t GetNumChannels() - { - return numChannels; - } - - virtual size_t GetHeadSize() - { - return headSize; - } - }; - #endif class RTNeuralModelDyn : public RTNeuralModel @@ -426,12 +306,12 @@ namespace NeuralAudio model.reset(); } - EModelLoadMode GetLoadMode() + EModelLoadMode GetLoadMode() override { return EModelLoadMode::RTNeural; } - bool CreateModelFromKerasJson(const nlohmann::json& modelJson) + bool CreateModelFromKerasJson(const nlohmann::json& modelJson) override { model = RTNeural::json_parser::parseJson(modelJson, false); model->reset(); @@ -439,11 +319,11 @@ namespace NeuralAudio return true; } - bool CreateModelFromNAMJson(const nlohmann::json& modelJson) + bool CreateModelFromNAMJson(const nlohmann::json& modelJson) override { model = std::make_unique>(1); - nlohmann::json config = modelJson.at("config"); + auto& config = modelJson.at("config"); const size_t numLayers = config.at("num_layers"); const size_t inputSize = config.at("input_size"); @@ -550,13 +430,13 @@ namespace NeuralAudio return true; } - void Process(float* input, float* output, size_t numSamples) + void Process(const float* input, float* output, size_t numSamples) override { for (size_t i = 0; i < numSamples; i++) output[i] = model->forward(input + i); } - void Prewarm() + void Prewarm() override { float sample = 0; @@ -568,3 +448,4 @@ namespace NeuralAudio std::unique_ptr> model; }; } +#endif diff --git a/NeuralAudio/WaveNet.h b/NeuralAudio/WaveNet.h index d8d2054..d8b4402 100644 --- a/NeuralAudio/WaveNet.h +++ b/NeuralAudio/WaveNet.h @@ -3,11 +3,13 @@ // Based on WaveNet model structure from https://github.com/sdatkinson/NeuralAmpModelerCore // with some template ideas from https://github.com/jatinchowdhury18/RTNeural-NAM +#include #include #include #include "TemplateHelper.h" -#include "NeuralModel.h" #include "Activation.h" +#include "ChannelBuffer.h" +#include "MatMul.h" #ifndef WAVENET_MAX_NUM_FRAMES #define WAVENET_MAX_NUM_FRAMES 64 @@ -17,155 +19,406 @@ #define LAYER_ARRAY_BUFFER_PADDING 24 #endif +enum EActivationType +{ + Tanh, + LeakyReLU +}; + namespace NeuralAudio { - //int numRewinds = 0; - //int maxRewinds = 0; + template + class ChannelHistoryBuffer + { + public: + static constexpr auto BufferSize = (ReceptiveFieldSize == 0) ? WAVENET_MAX_NUM_FRAMES : ReceptiveFieldSize + ((LAYER_ARRAY_BUFFER_PADDING + 1) * WAVENET_MAX_NUM_FRAMES); - template + ChannelBuffer buffer; + size_t bufferStart; + + void AllocBuffer(int allocNum) + { + buffer.SetZero(); + + //if (offset > (size - (ReceptiveFieldSize + WAVENET_MAX_NUM_FRAMES))) + //{ + // bufferStart = ReceptiveFieldSize; + //} + //else + //{ + // bufferStart = size - offset; + //} + +#if (LAYER_ARRAY_BUFFER_PADDING == 0) + bufferStart = ReceptiveFieldSize; +#else + if constexpr (ReceptiveFieldSize == 0) + bufferStart = 0; + else + bufferStart = BufferSize - (WAVENET_MAX_NUM_FRAMES * ((allocNum % LAYER_ARRAY_BUFFER_PADDING) + 1)); // Do the modulo to handle cases where LAYER_ARRAY_BUFFER_PADDING is not big enough to handle offset +#endif + } + + void AdvanceFrames(const size_t numFrames) + { + if constexpr (ReceptiveFieldSize > 0) + { + bufferStart += numFrames; + + if ((bufferStart + WAVENET_MAX_NUM_FRAMES) > (size_t)buffer.GetNumCols()) + RewindBuffer(); + } + } + + void RewindBuffer() + { + buffer.Slice(0, ReceptiveFieldSize).CopyData(buffer.Slice(bufferStart - ReceptiveFieldSize, ReceptiveFieldSize)); + + bufferStart = ReceptiveFieldSize; + } + + void CopyBuffer() + { + auto slice = buffer.Slice(bufferStart, 1); + + for (size_t offset = 1; offset < ReceptiveFieldSize + 1; offset++) + { + buffer.Slice(bufferStart - offset, 1).CopyData(slice); + } + } + }; + + struct Empty {}; + + template class Conv1DT { public: - void SetWeights(std::vector::iterator& inWeights) + static constexpr auto ReceptiveFieldSize = (KernelSize - 1) * Dilation; + ChannelHistoryBuffer channelBuffer; + + size_t GetNumWeights() { - weights.resize(KernelSize); + return OutChannels * InChannels * KernelSize + (DoBias ? OutChannels : 0); + } + void SetWeights(std::vector::iterator& inWeights) + { for (size_t i = 0; i < OutChannels; i++) for (size_t j = 0; j < InChannels; j++) for (size_t k = 0; k < KernelSize; k++) weights[k](i, j) = *(inWeights++); - if (DoBias) + if constexpr (DoBias) { for (size_t i = 0; i < OutChannels; i++) bias(i) = *(inWeights++); } } - template - inline void Process(const Eigen::MatrixBase& input, Eigen::MatrixBase const & output, const size_t iStart, const size_t nCols) const + std::array, KernelSize>& GetWeights() + { + return weights; + } + + // Avoid allocation for unused bias + using BiasType = typename std::conditional, + Empty>::type; + + BiasType& GetBias() + { + return bias; + } + + Conv1DT() + { + for (int k = 0; k < KernelSize; k++) + weightPtrs[k] = weights[k].GetData(); + } + + auto GetInputBuffer(size_t numFrames) { - for (size_t k = 0; k < KernelSize; k++) + return channelBuffer.buffer.Slice(channelBuffer.bufferStart, numFrames); + } + + inline void Process(const ChannelRowSpan& output) + { + const size_t numFrames = output.GetNumCols(); + T* __restrict outputPtr = output.GetData(); + +#if MULTIFRAME_8X8_CONVOLUTION != 0 + if constexpr ((((InChannels == 8) && (OutChannels == 8)) || ((InChannels == 16) && (OutChannels == 16)) || ((InChannels == 4) && (OutChannels == 4)) || ((InChannels == 2) && (OutChannels == 2))) && DoBias) { - auto offset = Dilation * ((int)k + 1 - KernelSize); + // Based on @jfsantos NAM Core implementation - https://github.com/sdatkinson/NeuralAmpModelerCore/pull/277 - auto inBlock = input.middleCols(iStart + offset, nCols); + constexpr size_t tileSize = (InChannels == 16) ? (MULTIFRAME_8X8_CONVOLUTION / 2) : MULTIFRAME_8X8_CONVOLUTION; + const size_t nFTile = (numFrames / tileSize) * tileSize; - if (k == 0) - const_cast&>(output).noalias() = weights[k] * inBlock; - else - const_cast&>(output).noalias() += weights[k] * inBlock; + for (size_t f = 0; f < nFTile; f += tileSize) + { + alignas(CHANNEL_BUFFER_ALIGN) T a[tileSize][InChannels]{}; + + for (size_t k = 0; k < KernelSize; k++) + { + const T* __restrict W = weightPtrs[k]; + const auto offset = Dilation * (k + 1 - KernelSize); + const T* __restrict hb = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset + f); + + for (size_t cp = 0; cp < InChannels; cp++) + { + const T* __restrict Wcol = W + cp * InChannels; + + if constexpr (tileSize == 2) + { + const T h0 = hb[cp], h1 = hb[InChannels + cp]; + + for (size_t o = 0; o < InChannels; o++) + { + const T wo = Wcol[o]; + a[0][o] += wo * h0; + a[1][o] += wo * h1; + } + } + else if constexpr (tileSize == 4) + { + const T h0 = hb[cp], h1 = hb[InChannels + cp], h2 = hb[2 * InChannels + cp], h3 = hb[3 * InChannels + cp]; + + for (size_t o = 0; o < InChannels; o++) + { + const T wo = Wcol[o]; + a[0][o] += wo * h0; + a[1][o] += wo * h1; + a[2][o] += wo * h2; + a[3][o] += wo * h3; + } + } + else // tileSize == 8 + { + const T h0 = hb[cp], h1 = hb[InChannels + cp], h2 = hb[2 * InChannels + cp], h3 = hb[3 * InChannels + cp]; + const T h4 = hb[4 * InChannels + cp], h5 = hb[5 * InChannels + cp], h6 = hb[6 * InChannels + cp], h7 = hb[7 * InChannels + cp]; + + for (size_t o = 0; o < InChannels; o++) + { + const T wo = Wcol[o]; + a[0][o] += wo * h0; + a[1][o] += wo * h1; + a[2][o] += wo * h2; + a[3][o] += wo * h3; + a[4][o] += wo * h4; + a[5][o] += wo * h5; + a[6][o] += wo * h6; + a[7][o] += wo * h7; + } + } + } + } + + for (size_t ti = 0; ti < tileSize; ti++) + std::memcpy(outputPtr + static_cast(f + ti) * InChannels, a[ti], InChannels * sizeof(T)); + } + + // Scalar tail for any frames past the tile-aligned boundary. + for (size_t f = nFTile; f < numFrames; f++) + { + T* zf = outputPtr + static_cast(f) * InChannels; + + for (size_t o = 0; o < InChannels; o++) + zf[o] = TCONST(0.0); + + for (size_t k = 0; k < KernelSize; k++) + { + const T* W = weightPtrs[k]; + const auto offset = Dilation * (k + 1 - KernelSize); + const T* h = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset + f); + + for (int cp = 0; cp < InChannels; cp++) + { + const T hv = h[cp]; + const T* Wcol = W + cp * InChannels; + + for (size_t o = 0; o < InChannels; o++) + zf[o] += Wcol[o] * hv; + } + } + } } + else +#endif + { + const T* biasPtr = nullptr; + + if constexpr (DoBias) + { + biasPtr = bias.data(); + } - if constexpr (DoBias) - const_cast&>(output).colwise() += bias; + for (size_t k = 0; k < KernelSize; k++) + { + const T* weightPtr = this->weights[k].GetDataConst(); + + const auto offset = Dilation * ((int)k + 1 - KernelSize); + + if constexpr (DoBias && MatMul::HasKernel()) + { + const T* inputPtr = channelBuffer.buffer.GetDataConst(channelBuffer.bufferStart + offset); + + if (k == 0) // Maybe move this out of loop? + { + if constexpr (DoBias) + { + MatMul::MultiplyInitColwise(inputPtr, outputPtr, weightPtr, biasPtr, numFrames); + } + else + { + MatMul::MultiplyInitZero(inputPtr, outputPtr, weightPtr, numFrames); + } + } + else + { + MatMul::MultiplyAccumlulate(inputPtr, outputPtr, weightPtr, numFrames); + } + } + else + { + const auto inBlock = channelBuffer.buffer.Slice(channelBuffer.bufferStart + offset, numFrames); + + if (k == 0) + output.GetEigenMap().noalias() = weights[k].GetEigenMapConst() * inBlock.GetEigenMapConst(); + else + output.GetEigenMap().noalias() += weights[k].GetEigenMapConst() * inBlock.GetEigenMapConst(); + } + } + } + + if constexpr (DoBias && !MatMul::HasKernel()) + output.GetEigenMap().colwise() += bias; } private: - std::vector> weights; - Eigen::Vector bias; + alignas(CHANNEL_BUFFER_ALIGN) std::array, KernelSize> weights; // consider making this a contiguous block of data instead of block of ChannelBuffers + std::array weightPtrs; + + BiasType bias; }; - template + template class DenseLayerT { public: + size_t GetNumWeights() + { + return OutChannels * InChannels + (DoBias ? OutChannels : 0); + } + void SetWeights(std::vector::iterator& inWeights) { - for (size_t i = 0; i < OutSize; i++) - for (size_t j = 0; j < InSize; j++) + for (size_t i = 0; i < OutChannels; i++) + for (size_t j = 0; j < InChannels; j++) weights(i, j) = *(inWeights++); if constexpr (DoBias) { - for (size_t i = 0; i < OutSize; i++) + for (size_t i = 0; i < OutChannels; i++) bias(i) = *(inWeights++); } } - template - void Process(const Eigen::MatrixBase& input, Eigen::MatrixBase const& output) const + ChannelBuffer& GetWeights() { - if constexpr (DoBias) + return weights; + } + + // Avoid allocation for unused bias + using BiasType = typename std::conditional, + Empty>::type; + + BiasType& GetBias() + { + return bias; + } + + void Process(const ChannelRowSpan& input, const ChannelRowSpan& output) const + { + size_t numFrames = output.GetNumCols(); + + if constexpr (MatMul::HasKernel()) { - const_cast&>(output).noalias() = (weights * input).colwise() + bias; + if constexpr (DoBias) + { + MatMul::MultiplyInitColwise(input.GetDataConst(), output.GetData(), weights.GetDataConst(), bias.data(), numFrames); + } + else + { + MatMul::MultiplyInitZero(input.GetDataConst(), output.GetData(), weights.GetDataConst(), numFrames); + } } else { - const_cast&>(output).noalias() = weights * input; + if constexpr (DoBias) + { + output.GetEigenMap().noalias() = (weights.GetEigenMapConst() * input.GetEigenMapConst()).colwise() + bias; + } + else + { + output.GetEigenMap().noalias() = weights.GetEigenMapConst() * input.GetEigenMapConst(); + } } } - template - void ProcessAcc(const Eigen::MatrixBase& input, Eigen::MatrixBase const& output) const + void ProcessAcc(const ChannelRowSpan& input, const ChannelRowSpan& output) const { - if constexpr (DoBias) + size_t numFrames = output.GetNumCols(); + + if constexpr (!DoBias && MatMul::HasKernel()) { - const_cast&>(output).noalias() += (weights * input).colwise() + bias; + MatMul::MultiplyAccumlulate(input.GetDataConst(), output.GetData(), weights.GetDataConst(), numFrames); } else { - const_cast&>(output).noalias() += weights * input; + if constexpr (DoBias) + { + output.GetEigenMap().noalias() += (weights.GetEigenMapConst() * input.GetEigenMapConst()) + bias; + } + else + { + output.GetEigenMap().noalias() += weights.GetEigenMapConst() * input.GetEigenMapConst(); + } } } private: - Eigen::Matrix weights; - Eigen::Vector bias; + ChannelBuffer weights; + + BiasType bias; }; - template + template class WaveNetLayerT { private: - Conv1DT conv1D; - DenseLayerT inputMixin; - DenseLayerT oneByOne; - Eigen::Matrix state; + Conv1DT conv1D; + DenseLayerT inputMixin; + DenseLayerT oneByOne; public: static constexpr auto ReceptiveFieldSize = (KernelSize - 1) * Dilation; - static constexpr auto BufferSize = ReceptiveFieldSize + ((LAYER_ARRAY_BUFFER_PADDING + 1) * WAVENET_MAX_NUM_FRAMES); - static constexpr bool TooBigForStatic = ((Channels * BufferSize) * 4) > EIGEN_STACK_ALLOCATION_LIMIT; - - using LayerBufferType = typename std::conditional, - Eigen::Matrix>::type; - - LayerBufferType layerBuffer; - //Eigen::Matrix layerBuffer; - size_t bufferStart; + static constexpr auto KernelSizeP = KernelSize; + static constexpr auto DilationP = Dilation; WaveNetLayerT() { - state.setZero(); } void AllocBuffer(int allocNum) { - long size = BufferSize; - - if constexpr(TooBigForStatic) - { - layerBuffer.resize(Channels, size); - } - - layerBuffer.setZero(); - - //if (offset > (size - (ReceptiveFieldSize + WAVENET_MAX_NUM_FRAMES))) - //{ - // bufferStart = ReceptiveFieldSize; - //} - //else - //{ - // bufferStart = size - offset; - //} + conv1D.channelBuffer.AllocBuffer(allocNum); + } -#if (LAYER_ARRAY_BUFFER_PADDING == 0) - bufferStart = ReceptiveFieldSize; -#else - bufferStart = size - (WAVENET_MAX_NUM_FRAMES * ((allocNum % LAYER_ARRAY_BUFFER_PADDING) + 1)); // Do the modulo to handle cases where LAYER_ARRAY_BUFFER_PADDING is not big enough to handle offset -#endif + size_t GetNumWeights() + { + return conv1D.GetNumWeights() + inputMixin.GetNumWeights() + oneByOne.GetNumWeights(); } void SetWeights(std::vector::iterator& weights) @@ -175,120 +428,151 @@ namespace NeuralAudio oneByOne.SetWeights(weights); } - void AdvanceFrames(const size_t numFrames) + Conv1DT& GetConv1D() { - bufferStart += numFrames; - - if ((bufferStart + WAVENET_MAX_NUM_FRAMES) > (size_t)layerBuffer.cols()) - RewindBuffer(); + return conv1D; } - void RewindBuffer() + DenseLayerT& GetInputMixin() { - //numRewinds++; + return inputMixin; + } - size_t start = ReceptiveFieldSize; + DenseLayerT& GetOneByOne() + { + return oneByOne; + } - layerBuffer.middleCols(start - ReceptiveFieldSize, ReceptiveFieldSize) = layerBuffer.middleCols(bufferStart - ReceptiveFieldSize, ReceptiveFieldSize); + void AdvanceFrames(const size_t numFrames) + { + conv1D.channelBuffer.AdvanceFrames(numFrames); + } - bufferStart = start; + auto GetInputBuffer(size_t numFrames) + { + return conv1D.channelBuffer.buffer.Slice(conv1D.channelBuffer.bufferStart, numFrames); } void CopyBuffer() { - for (size_t offset = 1; offset < ReceptiveFieldSize + 1; offset++) - { - layerBuffer.col(bufferStart - offset) = layerBuffer.col(bufferStart); - } + conv1D.channelBuffer.CopyBuffer(); } - template - void Process(const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInput, Eigen::MatrixBase const& output, const size_t outputStart, const size_t numFrames) + template + void Process(const ChannelRowSpan& condition, const ChannelRowSpan& headInput, const ChannelRowSpan& output) { - auto block = state.leftCols(numFrames); + ChannelBuffer state; - conv1D.Process(layerBuffer, block, bufferStart, numFrames); + size_t numFrames = output.GetNumCols(); - inputMixin.ProcessAcc(condition, block); + auto block = state.Slice(numFrames); - block = WAVENET_MATH::Tanh(block); + conv1D.Process(block); + + inputMixin.ProcessAcc(condition, block); - //block = block.array().tanh(); + if constexpr (Activation == EActivationType::Tanh) + { + WAVENET_MATH::template Tanh(block); + } + else if constexpr (Activation == EActivationType::LeakyReLU) + { + WAVENET_MATH::template LeakyReLU(block); + } - //float* data = block.data(); - //size_t size = block.rows() * block.cols(); + headInput.GetEigenMap().noalias() += block.GetEigenMapConst(); - //for (size_t pos = 0; pos < size; pos++) - //{ - // data[pos] = WAVENET_MATH::Tanh(data[pos]); - //} + //headInput.AddData(block); - const_cast&>(headInput).noalias() += block.topRows(Channels); + if constexpr (NeedOutput) + { + oneByOne.Process(block, output); - oneByOne.Process(block.topRows(Channels), const_cast&>(output).middleCols(outputStart, numFrames)); + output.GetEigenMap().noalias() += conv1D.GetInputBuffer(numFrames).GetEigenMapConst(); + } - const_cast&>(output).middleCols(outputStart, numFrames).noalias() += layerBuffer.middleCols(bufferStart, numFrames); + //output.AddData(conv1D.GetInputBuffer(numFrames)); } }; template using Dilations = std::integer_sequence; - template + template + using KernelSizes = std::integer_sequence; + + template class WaveNetLayerArrayT { - template + template struct LayersHelper { }; - template - struct LayersHelper> + template + struct LayersHelper, Dilations> { - using type = std::tuple...>; + using type = std::tuple...>; }; - using Layers = typename LayersHelper::type; + using Layers = typename LayersHelper::type; private: Layers layers; - DenseLayerT rechannel; - DenseLayerT headRechannel; - - static constexpr auto numLayers = std::tuple_size_v; - static constexpr auto lastLayer = numLayers - 1; + DenseLayerT rechannel; + Conv1DT headRechannel; public: static constexpr auto NumChannelsP = Channels; static constexpr auto HeadSizeP = HeadSize; + static constexpr auto NumLayers = std::tuple_size_v; + static constexpr auto LastLayer = NumLayers - 1; - Eigen::Matrix arrayOutputs; - Eigen::Matrix headOutputs; + ChannelBuffer arrayOutputs; + ChannelBuffer headOutputs; int ReceptiveFieldSize = 0; // This should be a static constexpr, but I haven't sorted out the right template magic WaveNetLayerArrayT() { - ForEachIndex([&](auto layerIndex) + ForEachIndex([&](auto layerIndex) { ReceptiveFieldSize += std::get(layers).ReceptiveFieldSize; }); + + ReceptiveFieldSize += headRechannel.ReceptiveFieldSize; } int AllocBuffers(int allocNum) { - ForEachIndex([&](auto layerIndex) + ForEachIndex([&](auto layerIndex) { std::get(layers).AllocBuffer(allocNum++); }); + headRechannel.channelBuffer.AllocBuffer(allocNum++); + return allocNum; } + size_t GetNumWeights() + { + size_t numWeights = rechannel.GetNumWeights(); + + ForEachIndex([&](auto layerIndex) + { + numWeights += std::get(layers).GetNumWeights(); + }); + + numWeights += headRechannel.GetNumWeights(); + + return numWeights; + } + void SetWeights(std::vector::iterator& weights) { rechannel.SetWeights(weights); - ForEachIndex([&](auto layerIndex) + ForEachIndex([&](auto layerIndex) { std::get(layers).SetWeights(weights); }); @@ -296,57 +580,88 @@ namespace NeuralAudio headRechannel.SetWeights(weights); } - template - void Prewarm(const Eigen::MatrixBase& layerInputs, const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInputs) + Layers& GetLayers() { - rechannel.Process(layerInputs, std::get<0>(layers).layerBuffer.middleCols(std::get<0>(layers).bufferStart, 1)); + return layers; + } + + DenseLayerT& GetRechannel() + { + return rechannel; + } - ForEachIndex([&](auto layerIndex) + Conv1DT& GetHeadRechannel() + { + return headRechannel; + } + + void Prewarm(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) + { + rechannel.Process(layerInputs, std::get<0>(layers).GetInputBuffer(1)); + + ForEachIndex([&](auto layerIndex) { std::get(layers).CopyBuffer(); - if constexpr (layerIndex == lastLayer) + if constexpr (layerIndex == LastLayer) { - std::get(layers).Process(condition, headInputs, arrayOutputs, 0, 1); + std::get(layers).Process(condition, headInputs, arrayOutputs.Slice(1)); } else { - std::get(layers).Process(condition, headInputs, std::get(layers).layerBuffer, std::get(layers).bufferStart, 1); + std::get(layers).Process(condition, headInputs, std::get(layers).GetInputBuffer(1)); } }); - headRechannel.Process(headInputs, headOutputs.leftCols(1)); + //headRechannel.channelBuffer.buffer.middleCols(headRechannel.channelBuffer.bufferStart, 1).noalias() = headInputs.leftCols(1); // Should be able to avoid this copy + + headRechannel.GetInputBuffer(1).CopyData(headInputs.Slice(1)); + headRechannel.channelBuffer.CopyBuffer(); + headRechannel.Process(headOutputs.Slice(1)); } - template - void Process(const Eigen::MatrixBase& layerInputs, const Eigen::MatrixBase& condition, Eigen::MatrixBase const& headInputs, const size_t numFrames) + template + void Process(const ChannelRowSpan& layerInputs, const ChannelRowSpan& condition, const ChannelRowSpan& headInputs) { - rechannel.Process(layerInputs, std::get<0>(layers).layerBuffer.middleCols(std::get<0>(layers).bufferStart, numFrames)); + size_t numFrames = condition.GetNumCols(); + + rechannel.Process(layerInputs, std::get<0>(layers).GetInputBuffer(numFrames)); - ForEachIndex([&](auto layerIndex) + ForEachIndex([&](auto layerIndex) { - if constexpr (layerIndex == lastLayer) + if constexpr (layerIndex == LastLayer) { - std::get(layers).Process(condition, headInputs, arrayOutputs, 0, numFrames); + std::get(layers).template Process(condition, headInputs, arrayOutputs.Slice(numFrames)); } else { - std::get(layers).Process(condition, headInputs, std::get(layers).layerBuffer, std::get(layers).bufferStart, numFrames); + std::get(layers).Process(condition, headInputs, std::get(layers).GetInputBuffer(numFrames)); } std::get(layers).AdvanceFrames(numFrames); }); - headRechannel.Process(headInputs, headOutputs.leftCols(numFrames)); + + //headRechannel.channelBuffer.buffer.middleCols(headRechannel.channelBuffer.bufferStart, numFrames).noalias() = headInputs.leftCols(numFrames); // Should be able to avoid this copy + + //headRechannel.GetInputBuffer(numFrames).CopyData(headInputs); + + headRechannel.GetInputBuffer(numFrames).GetEigenMap().noalias() = headInputs.GetEigenMapConst(); + headRechannel.Process(headOutputs.Slice(numFrames)); + headRechannel.channelBuffer.AdvanceFrames(numFrames); } }; - template + template class WaveNetModelT { public: int ReceptiveFieldSize = 0; // This should be a static constexpr, but I haven't sorted out the right template magic + static constexpr auto headLayerChannels = std::tuple_element_t<0, std::tuple>::NumChannelsP; + static constexpr auto NumLayerArrays = std::tuple_size_v>; + static constexpr auto LastLayerArray = NumLayerArrays - 1; + WaveNetModelT() { int allocNum = 0; @@ -357,11 +672,33 @@ namespace NeuralAudio allocNum = std::get(layerArrays).AllocBuffers(allocNum); }); + } + size_t GetNumWeights() + { + size_t numWeights = 0; + + ForEachIndex([&](auto layerIndex) + { + numWeights += std::get(layerArrays).GetNumWeights(); + }); + + numWeights++; // headScale; + + return numWeights; } void SetWeights(std::vector weights) { + size_t numWeights = GetNumWeights(); + + if (numWeights != weights.size()) + { + std::stringstream str; + str << "Wrong number of weights. Expected " << numWeights << " but got " << weights.size(); + throw std::runtime_error(str.str()); + } + std::vector::iterator it = weights.begin(); ForEachIndex([&](auto layerIndex) @@ -370,8 +707,6 @@ namespace NeuralAudio }); headScale = *(it++); - - assert(std::distance(weights.begin(), it) == (long)weights.size()); } size_t GetMaxFrames() @@ -379,64 +714,74 @@ namespace NeuralAudio return WAVENET_MAX_NUM_FRAMES; } + auto& GetLayerArrays() + { + return layerArrays; + } + + T GetHeadScale() + { + return headScale; + } + void Prewarm() { - float input = 0; + ChannelBuffer condition; + condition.GetData()[0] = 0; - auto condition = Eigen::Map>(&input, 1, 1); + ChannelBuffer headArray; + headArray.SetZero(); ForEachIndex([&](auto layerIndex) { if constexpr (layerIndex == 0) { - std::get(layerArrays).Prewarm(condition, condition, headArray.leftCols(1)); + std::get(layerArrays).Prewarm(condition, condition, headArray); } else { - std::get(layerArrays).Prewarm(std::get(layerArrays).arrayOutputs.leftCols(1), condition, std::get(layerArrays).headOutputs.leftCols(1)); + std::get(layerArrays).Prewarm(std::get(layerArrays).arrayOutputs.Slice(1), condition, std::get(layerArrays).headOutputs.Slice(1)); } }); } - void Process(const float* input, float* output, const size_t numFrames) + void Process(const T* input, T* output, const size_t numFrames) { - //numRewinds = 0; + ChannelBuffer condition; + std::memcpy(condition.GetData(), input, numFrames * sizeof(T)); - auto condition = Eigen::Map>(input, 1, numFrames); + ChannelBuffer headArray; + headArray.SetZero(); - headArray.setZero(); + auto conditionSpan = condition.Slice(numFrames); + auto headArraySpan = headArray.Slice(numFrames); ForEachIndex([&](auto layerIndex) { if constexpr (layerIndex == 0) { - std::get(layerArrays).Process(condition, condition, headArray.leftCols(numFrames), numFrames); + std::get(layerArrays).Process(conditionSpan, conditionSpan, headArraySpan); + } + else if constexpr (layerIndex == LastLayerArray) + { + std::get(layerArrays).template Process(std::get(layerArrays).arrayOutputs.Slice(numFrames), conditionSpan, std::get(layerArrays).headOutputs.Slice(numFrames)); } else { - std::get(layerArrays).Process(std::get(layerArrays).arrayOutputs.leftCols(numFrames), condition, std::get(layerArrays).headOutputs.leftCols(numFrames), numFrames); + std::get(layerArrays).Process(std::get(layerArrays).arrayOutputs.Slice(numFrames), conditionSpan, std::get(layerArrays).headOutputs.Slice(numFrames)); } }); - const auto finalHeadArray = std::get(layerArrays).headOutputs; - - auto out = Eigen::Map>(output, 1, numFrames); - - out.noalias() = headScale * finalHeadArray.leftCols(numFrames); + T* finalHeadArray = std::get(layerArrays).headOutputs.GetData(); - //if (numRewinds > maxRewinds) - //{ - // maxRewinds = numRewinds; - - // std::cout << "New Max Rewinds: " << maxRewinds << std::endl; - //} + for (size_t i = 0; i < numFrames; i++) + { + output[i] = headScale * finalHeadArray[i]; + } } private: - static constexpr auto headLayerChannels = std::tuple_element_t<0, std::tuple>::NumChannelsP; - std::tuple layerArrays; - Eigen::Matrix headArray; - float headScale; + T headScale; }; -} \ No newline at end of file +} diff --git a/NeuralAudio/WaveNetDynamic.h b/NeuralAudio/WaveNetDynamic.h index 948ce89..5b850b2 100644 --- a/NeuralAudio/WaveNetDynamic.h +++ b/NeuralAudio/WaveNetDynamic.h @@ -5,7 +5,6 @@ #include #include -#include "NeuralModel.h" #include "Activation.h" #ifndef WAVENET_MAX_NUM_FRAMES @@ -108,13 +107,13 @@ namespace NeuralAudio void SetWeights(std::vector::iterator& inWeights) { - for (auto i = 0; i < outSize; i++) - for (auto j = 0; j < inSize; j++) + for (size_t i = 0; i < outSize; i++) + for (size_t j = 0; j < inSize; j++) weights(i, j) = *(inWeights++); if (doBias) { - for (auto i = 0; i < outSize; i++) + for (size_t i = 0; i < outSize; i++) bias(i) = *(inWeights++); } } @@ -147,45 +146,40 @@ namespace NeuralAudio class WaveNetLayer { private: - size_t conditionSize; size_t channels; - size_t kernelSize; - size_t dilation; Conv1D conv1D; DenseLayer inputMixin; DenseLayer oneByOne; Eigen::MatrixXf state; - Eigen::MatrixXf layerBuffer; + Eigen::MatrixXf buffer; public: size_t ReceptiveFieldSize; size_t bufferStart; WaveNetLayer(size_t conditionSize, size_t channels, size_t kernelSize, size_t dilation) : - conditionSize(conditionSize), channels(channels), - kernelSize(kernelSize), - dilation(dilation), conv1D(channels, channels, kernelSize, true, dilation), inputMixin(conditionSize, channels, false), oneByOne(channels, channels, true), state(channels, WAVENET_MAX_NUM_FRAMES), - ReceptiveFieldSize((kernelSize - 1) * dilation) + ReceptiveFieldSize((kernelSize - 1) * dilation), + bufferStart(0) { state.setZero(); } Eigen::MatrixXf& GetLayerBuffer() { - return layerBuffer; + return buffer; } void AllocBuffer(size_t allocNum) { size_t size = ReceptiveFieldSize + ((LAYER_ARRAY_BUFFER_PADDING + 1) * WAVENET_MAX_NUM_FRAMES); - layerBuffer.resize(channels, size); - layerBuffer.setZero(); + buffer.resize(channels, size); + buffer.setZero(); // offset prevents buffer rewinds of various layers from happening at the same time #if (LAYER_ARRAY_BUFFER_PADDING == 0) @@ -211,7 +205,7 @@ namespace NeuralAudio { bufferStart += numFrames; - if ((int)(bufferStart + WAVENET_MAX_NUM_FRAMES) > layerBuffer.cols()) + if ((int)(bufferStart + WAVENET_MAX_NUM_FRAMES) > buffer.cols()) RewindBuffer(); } @@ -219,7 +213,7 @@ namespace NeuralAudio { size_t start = ReceptiveFieldSize; - layerBuffer.middleCols(start - ReceptiveFieldSize, ReceptiveFieldSize) = layerBuffer.middleCols(bufferStart - ReceptiveFieldSize, ReceptiveFieldSize); + buffer.middleCols(start - ReceptiveFieldSize, ReceptiveFieldSize) = buffer.middleCols(bufferStart - ReceptiveFieldSize, ReceptiveFieldSize); bufferStart = start; } @@ -228,7 +222,7 @@ namespace NeuralAudio { for (size_t offset = 1; offset < ReceptiveFieldSize + 1; offset++) { - layerBuffer.col(bufferStart - offset) = layerBuffer.col(bufferStart); + buffer.col(bufferStart - offset) = buffer.col(bufferStart); } } @@ -236,7 +230,7 @@ namespace NeuralAudio { auto block = state.leftCols(numFrames); - conv1D.Process(layerBuffer, block, bufferStart, numFrames); + conv1D.Process(buffer, block, bufferStart, numFrames); inputMixin.ProcessAcc(condition, block); @@ -247,14 +241,14 @@ namespace NeuralAudio for (auto pos = 0; pos < size; pos++) { - data[pos] = WAVENET_MATH::Tanh(data[pos]); + data[pos] = WAVENET_MATH::Tanh(data[pos]); } headInput.noalias() += block.topRows(channels); oneByOne.Process(block.topRows(channels), output.middleCols(outputStart, numFrames)); - output.middleCols(outputStart, numFrames).noalias() += layerBuffer.middleCols(bufferStart, numFrames); + output.middleCols(outputStart, numFrames).noalias() += buffer.middleCols(bufferStart, numFrames); AdvanceFrames(numFrames); } @@ -263,30 +257,22 @@ namespace NeuralAudio class WaveNetLayerArray { private: - size_t inputSize; - size_t conditionSize; - size_t headSize; size_t channels; - size_t kernelSize; std::vector layers; DenseLayer rechannel; - DenseLayer headRechannel; + DenseLayer oneByOne; size_t lastLayer; Eigen::MatrixXf arrayOutputs; Eigen::MatrixXf headOutputs; public: - WaveNetLayerArray(size_t inputSize, size_t conditionSize, size_t headSize, size_t channels, size_t kernelSize, bool hasHeadBias, std::vector dilations) : - inputSize(inputSize), - conditionSize(conditionSize), - headSize(headSize), + WaveNetLayerArray(size_t inputSize, size_t conditionSize, size_t HeadSize, size_t channels, size_t kernelSize, bool hasHeadBias, std::vector dilations) : channels(channels), - kernelSize(kernelSize), rechannel(inputSize, channels, false), - headRechannel(channels, headSize, hasHeadBias), + oneByOne(channels, HeadSize, hasHeadBias), arrayOutputs(channels, WAVENET_MAX_NUM_FRAMES), - headOutputs(headSize, WAVENET_MAX_NUM_FRAMES) + headOutputs(HeadSize, WAVENET_MAX_NUM_FRAMES) { for (auto dilation : dilations) { @@ -338,7 +324,7 @@ namespace NeuralAudio layer.SetWeights(weights); } - headRechannel.SetWeights(weights); + oneByOne.SetWeights(weights); } void Prewarm(const Eigen::MatrixXf& layerInputs, const Eigen::MatrixXf& condition, Eigen::Ref const& headInputs) @@ -359,7 +345,7 @@ namespace NeuralAudio } } - headRechannel.Process(headInputs, headOutputs.leftCols(1)); + oneByOne.Process(headInputs, headOutputs.leftCols(1)); } void Process(const Eigen::MatrixXf& layerInputs, const Eigen::MatrixXf& condition, Eigen::Ref headInputs, const size_t numFrames) @@ -378,7 +364,7 @@ namespace NeuralAudio } } - headRechannel.Process(headInputs, headOutputs.leftCols(numFrames)); + oneByOne.Process(headInputs, headOutputs.leftCols(numFrames)); } }; diff --git a/NeuralAudioCAPI/NeuralAudioCApi.cpp b/NeuralAudioCAPI/NeuralAudioCApi.cpp index 04d8cdd..0aac758 100644 --- a/NeuralAudioCAPI/NeuralAudioCApi.cpp +++ b/NeuralAudioCAPI/NeuralAudioCApi.cpp @@ -6,11 +6,31 @@ struct NeuralModel NeuralAudio::NeuralModel* model; }; -NeuralModel* CreateModelFromFile(const wchar_t* modelPath) +struct NeuralModelLoader +{ + NeuralAudio::NeuralModelLoader* loader; +}; + +NeuralModelLoader* CreateLoader() +{ + NeuralModelLoader* loader = new NeuralModelLoader(); + + loader->loader = new NeuralAudio::NeuralModelLoader(); + + return loader; +} + +void DeleteLoader(NeuralModelLoader* loader) +{ + delete loader->loader; + delete loader; +} + +NeuralModel* CreateModelFromFile(NeuralModelLoader* loader, const wchar_t* modelPath) { NeuralModel* model = new NeuralModel(); - model->model = NeuralAudio::NeuralModel::CreateFromFile(modelPath); + model->model = loader->loader->CreateFromFile(modelPath); return model; } @@ -21,24 +41,24 @@ void DeleteModel(NeuralModel* model) delete model; } -void SetLSTMLoadMode(int loadMode) +void SetLSTMLoadMode(NeuralModelLoader* loader, int loadMode) { - NeuralAudio::NeuralModel::SetLSTMLoadMode((NeuralAudio::EModelLoadMode)loadMode); + loader->loader->SetLSTMLoadMode((NeuralAudio::EModelLoadMode)loadMode); } -void SetWaveNetLoadMode(int loadMode) +void SetWaveNetLoadMode(NeuralModelLoader* loader, int loadMode) { - NeuralAudio::NeuralModel::SetWaveNetLoadMode((NeuralAudio::EModelLoadMode)loadMode); + loader->loader->SetWaveNetLoadMode((NeuralAudio::EModelLoadMode)loadMode); } -void SetAudioInputLevelDBu(float audioDBu) +void SetAudioInputLevelDBu(NeuralModelLoader* loader, float audioDBu) { - NeuralAudio::NeuralModel::SetAudioInputLevelDBu(audioDBu); + loader->loader->SetAudioInputLevelDBu(audioDBu); } -void SetDefaultMaxAudioBufferSize(int maxSize) +void SetDefaultMaxAudioBufferSize(NeuralModelLoader* loader, int maxSize) { - NeuralAudio::NeuralModel::SetDefaultMaxAudioBufferSize(maxSize); + loader->loader->SetDefaultMaxAudioBufferSize(maxSize); } int GetLoadMode(NeuralModel* model) diff --git a/NeuralAudioCAPI/NeuralAudioCApi.h b/NeuralAudioCAPI/NeuralAudioCApi.h index 28d8886..18a34b6 100644 --- a/NeuralAudioCAPI/NeuralAudioCApi.h +++ b/NeuralAudioCAPI/NeuralAudioCApi.h @@ -13,19 +13,23 @@ extern "C" { #endif struct NeuralModel; +struct NeuralModelLoader; +NA_EXTERN NeuralModelLoader* CreateLoader(); -NA_EXTERN NeuralModel* CreateModelFromFile(const wchar_t* modelPath); +NA_EXTERN void DeleteLoader(NeuralModelLoader* loader); + +NA_EXTERN NeuralModel* CreateModelFromFile(NeuralModelLoader *loader, const wchar_t* modelPath); NA_EXTERN void DeleteModel(NeuralModel* model); -NA_EXTERN void SetLSTMLoadMode(int loadMode); +NA_EXTERN void SetLSTMLoadMode(NeuralModelLoader* loader, int loadMode); -NA_EXTERN void SetWaveNetLoadMode(int loadMode); +NA_EXTERN void SetWaveNetLoadMode(NeuralModelLoader* loader, int loadMode); -NA_EXTERN void SetAudioInputLevelDBu(float audioDBu); +NA_EXTERN void SetAudioInputLevelDBu(NeuralModelLoader* loader, float audioDBu); -NA_EXTERN void SetDefaultMaxAudioBufferSize(int maxSize); +NA_EXTERN void SetDefaultMaxAudioBufferSize(NeuralModelLoader* loader, int maxSize); NA_EXTERN int GetLoadMode(NeuralModel* model); diff --git a/NeuralAudioCSharp/NeuralAudio/NativeApi.cs b/NeuralAudioCSharp/NeuralAudio/NativeApi.cs index a562e97..54f9393 100644 --- a/NeuralAudioCSharp/NeuralAudio/NativeApi.cs +++ b/NeuralAudioCSharp/NeuralAudio/NativeApi.cs @@ -9,22 +9,28 @@ static class NativeApi public const string NEURAL_AUDIO_LIB_NAME = "NeuralAudioCAPI"; [DllImport(NEURAL_AUDIO_LIB_NAME)] - public static extern IntPtr CreateModelFromFile([MarshalAs(UnmanagedType.LPWStr)]string modelPath); + public static extern IntPtr CreateLoader(); + + [DllImport(NEURAL_AUDIO_LIB_NAME)] + public static extern void DeleteLoader(IntPtr loader); + + [DllImport(NEURAL_AUDIO_LIB_NAME)] + public static extern IntPtr CreateModelFromFile(IntPtr loader, [MarshalAs(UnmanagedType.LPWStr)]string modelPath); [DllImport(NEURAL_AUDIO_LIB_NAME)] public static extern void DeleteModel(IntPtr model); [DllImport(NEURAL_AUDIO_LIB_NAME)] - public static extern void SetLSTMLoadMode(int loadMode); + public static extern void SetLSTMLoadMode(IntPtr loader, int loadMode); [DllImport(NEURAL_AUDIO_LIB_NAME)] - public static extern void SetWaveNetLoadMode(int loadMode); + public static extern void SetWaveNetLoadMode(IntPtr loader, int loadMode); [DllImport(NEURAL_AUDIO_LIB_NAME)] - public static extern void SetAudioInputLevelDBu(float audioDBu); + public static extern void SetAudioInputLevelDBu(IntPtr loader, float audioDBu); [DllImport(NEURAL_AUDIO_LIB_NAME)] - public static extern void SetDefaultMaxAudioBufferSize(int maxSize); + public static extern void SetDefaultMaxAudioBufferSize(IntPtr loader, int maxSize); [DllImport(NEURAL_AUDIO_LIB_NAME)] public static extern int GetLoadMode(IntPtr model); diff --git a/NeuralAudioCSharp/NeuralAudio/NeuralModel.cs b/NeuralAudioCSharp/NeuralAudio/NeuralModel.cs index 482ae99..215e60a 100644 --- a/NeuralAudioCSharp/NeuralAudio/NeuralModel.cs +++ b/NeuralAudioCSharp/NeuralAudio/NeuralModel.cs @@ -3,31 +3,58 @@ namespace NeuralAudio { - public class NeuralModel + public enum EModelLoadMode + { + Internal, + RTNeural, + NAMCore + }; + + public class NeuralModelLoader { - public enum EModelLoadMode + IntPtr nativeLoader; + + public NeuralModelLoader() + { + nativeLoader = NativeApi.CreateLoader(); + } + + ~NeuralModelLoader() { - Internal, - RTNeural, - NAMCore - }; + NativeApi.DeleteLoader(nativeLoader); + } - IntPtr nativeModel; + public void SetLSTMModelLoadMode(EModelLoadMode mode) + { + NativeApi.SetLSTMLoadMode(nativeLoader, (int)mode); + } - public static void SetLSTMModelLoadMode(EModelLoadMode mode) + public void SetWaveNetModelLoadMode(EModelLoadMode mode) { - NativeApi.SetLSTMLoadMode((int)mode); + NativeApi.SetWaveNetLoadMode(nativeLoader, (int)mode); } - public static void SetWaveNetModelLoadMode(EModelLoadMode mode) + public void SetDefaultMaxAudioBufferSize(int bufferSize) { - NativeApi.SetWaveNetLoadMode((int)mode); + NativeApi.SetDefaultMaxAudioBufferSize(nativeLoader, bufferSize); } - public static void SetDefaultMaxAudioBufferSize(int bufferSize) + public NeuralModel CreateModelFromFile(string modelPath) { - NativeApi.SetDefaultMaxAudioBufferSize(bufferSize); + NeuralModel model = new NeuralModel(); + + model.nativeModel = NativeApi.CreateModelFromFile(nativeLoader, modelPath); + + if (model.nativeModel == IntPtr.Zero) + return null; + + return model; } + } + + public class NeuralModel + { + internal IntPtr nativeModel; public bool IsStatic { get { return NativeApi.IsStatic(nativeModel); } } public EModelLoadMode LoadMode { get { return (EModelLoadMode)NativeApi.GetLoadMode(nativeModel); } } @@ -35,15 +62,9 @@ public static void SetDefaultMaxAudioBufferSize(int bufferSize) public float RecommendedInputDBAdjustment { get { return NativeApi.GetRecommendedInputDBAdjustment(nativeModel); } } public float RecommendedOutputDBAdjustment { get { return NativeApi.GetRecommendedOutputDBAdjustment(nativeModel); } } - public static NeuralModel FromFile(string modelPath) + ~NeuralModel() { - NeuralModel model = new NeuralModel(); - - IntPtr nativeModel = NativeApi.CreateModelFromFile(modelPath); - - model.nativeModel = nativeModel; - - return model; + NativeApi.DeleteModel(nativeModel); } public void SetMaxAudioBufferSize(int bufferSize) diff --git a/NeuralAudioCSharp/NeuralAudioTest/Program.cs b/NeuralAudioCSharp/NeuralAudioTest/Program.cs index 77f1b94..8058db6 100644 --- a/NeuralAudioCSharp/NeuralAudioTest/Program.cs +++ b/NeuralAudioCSharp/NeuralAudioTest/Program.cs @@ -6,9 +6,11 @@ internal class Program { static void Main(string[] args) { - NeuralModel.SetWaveNetModelLoadMode(NeuralModel.EModelLoadMode.Internal); + NeuralModelLoader loader = new(); - NeuralModel model = NeuralModel.FromFile("BossWN-standard.nam"); + loader.SetWaveNetModelLoadMode(EModelLoadMode.Internal); + + NeuralModel model = loader.CreateModelFromFile("BossWN-standard.nam"); var input = new float[1024]; var output = new float[1024]; diff --git a/README.md b/README.md index 149f311..52416a4 100644 --- a/README.md +++ b/README.md @@ -2,34 +2,37 @@ NeuralAudio is a C++ library designed to make it easy to use neural network machine learning models (ie: guitar amplifier captures/profiles) in real-time audio applications. -# Supported Models +Priorities are a simple, but powerful API, a clean, maintainable codebase, and highly optimized performance across platforms. -NeuralAudio currently supports the following model types: - -- [Neural Amp Modeler](https://github.com/sdatkinson/neural-amp-modeler) (NAM) WaveNet and LSTM models -- [RTNeural](https://github.com/jatinchowdhury18/RTNeural) keras models (LSTM, GRU) +# License -# Underlying Libraries and Performance +This repository is licensed under the [MIT license](https://github.com/mikeoliphant/NeuralAudio/blob/main/LICENSE). It is a liberal license, but please make sure that you comply with the terms - as well as the terms of [this project's dependencies](https://github.com/mikeoliphant/NeuralAudio/blob/main/CREDITS.md). I would also appreciate it if you would let me know if you are using this library. -By default, NeuralAudio uses its own implementation of WaveNet and LSTM network models. This implementation has been designed to produce **exactly** the same output as the [NAM Core library](https://github.com/sdatkinson/NeuralAmpModelerCore), but with increased performance and reduced memory usage. +# Supported Models and Underlying Libraries -For completeness, and to facilitate accuracy and performance benchmarking, it can also load models using the [NAM Core implementation](https://github.com/sdatkinson/NeuralAmpModelerCore) and [RTNeural](https://github.com/jatinchowdhury18/RTNeural). +NeuralAudio currently supports the following model types: -The internal NeuralAudio implmentation currently outperforms the other implementations on all tested platforms (Windows x64, Linux x64/Arm64). It also uses significantly less memory than the NAM Core WaveNet implementation (which, for example, uses about 10x as much memory for a "standard" WaveNet model). +- [Neural Amp Modeler](https://github.com/sdatkinson/neural-amp-modeler) (NAM) WaveNet and LSTM models, A1 and A2 support +- [RTNeural](https://github.com/jatinchowdhury18/RTNeural) keras models (LSTM, GRU) -For WaveNet, the internal implmeentation supports optimized static models of the offical NAM network architectures: "Standard", "Lite", "Feather", "Nano". +For WaveNet, the internal implmeentation supports optimized static implemenationas the offical NAM A1 and A2 network architectures: A1 "Standard", "Lite", "Feather", "Nano" and A2 "Lite" and "Full". For LSTM, the internal implementation supports optimized static models architectures for 1x8, 1x12, 1x16, 1x24, 2x8, 2x12, and 2x16 models. -All NAM files with WaveNet and LSTM architectures not supported internally will fall back on a less performant dynamic implementation (although still faster than NAM Core). +All A1 NAM files with WaveNet and LSTM architectures not supported statically will fall back on a less performant dynamic implementation. + +All non-standard A2 models currently use the NAM Core implementation (and consequently require building with NAM Core enabled). All keras models not supported internally will fall back to the RTNeural implmentation. # API overview -To load a model: +Models are loaded with a model loader: + ``` -NeuralModel* model = NeuralAudio::NeuralModel::CreateFromFile(""); +NeuralModelLoader loader; + +NeuralModel* model = loader.CreateFromFile(""); ``` To process a model: @@ -42,10 +45,10 @@ model->Process(pointerToFloatInputData, pointerToFloatOutputData, int numSamples Some models need to allocate memory based on the size of the audio buffers being used. You need to make sure that processing does not exceed the specified maximum buffer size. -The default maximum size is 128 samples. To change it, do: +The default maximum size is 128 samples. To change it, change the default size on the model loader: ``` -NeuralAudio::NeuralModel::SetDefaultMaxAudioBufferSize(maxSize); +loader.SetDefaultMaxAudioBufferSize(maxSize); ``` if you want to change the maximum buffer size of an already created model, do: @@ -60,20 +63,20 @@ model->SetMaxAudioBufferSize(int maxSize); Use ```model->GetRecommendedInputDBAdjustment()``` and ```model->GetRecommendedOutputDBAdjustment()``` to obtain the ideal input and output volume level adjustments in dB. -To set a known audio input level (ie: from an audio interface), use ```model->SetAudioInputLevelDBu(float audioDBu)```. This is set at 12DBu by default. +To set a known audio input level (ie: from an audio interface), use ```loader.SetAudioInputLevelDBu(float audioDBu)```. This is set at 12DBu by default. ## Model load behavior -By default, models are loaded using the internal NeuralAudio implementation. If you would like to force the use of the NAM Core or RTNeural implementations, you can use: +By default, models are loaded using the internal NeuralAudio implementation (if possible). If you would like to force the use of the NAM Core or RTNeural implementations, you can use: ``` -NeuralAudio::NeuralModel::SetWaveNetLoadMode(loadMode) +loader.SetWaveNetLoadMode(loadMode); ``` and ``` -NeuralAudio::NeuralModel::SetLSTMLoadMode(loadMode) +loader.SetLSTMLoadMode(loadMode); ``` where "loadMode" is one of: @@ -81,12 +84,99 @@ where "loadMode" is one of: ``` NeuralAudio::EModelLoadMode::Internal NeuralAudio::EModelLoadMode::NAMCore -NeuralAudio::EModelLoadMode::RTNeural +NeuralAudio::EModelLoadMode::RTNeural (only supported for LSTM) ``` You can check which implementation was actually used to load the model with ```model->GetLoadMode()```. -**NOTE:** Because of compile time and executable size considerations, only the internal and dynamic RTNeural implementations are built by default. If you want to use RTNeural, it is recommended that you add ```-DBUILD_STATIC_RTNEURAL=ON``` to your cmake commandline. This will create static model implmentations for the same sets of WaveNet and LSTM models as the internal implmentation, and results in increased performance. If you want to use NAM Core, add ```DBUILD_NAMCORE=ON``` to your cmake commandline. +**NOTE:** Because of compile time and executable size considerations, only the internal, NAM Core and dynamic RTNeural implementations are built by default. If you want to use RTNeural for LSTM models, it is recommended that you add ```-DBUILD_STATIC_RTNEURAL=ON``` to your cmake commandline. This will create static model implmentations for the same set of LSTM models as the internal implmentation, and results in increased performance. Interal static LSTM model support is also off by default - to turn it on use ```-DBUILD_INTERNAL_STATIC_LSTM=ON```. + +### Composite model load behavior + +Some models (notably NAM A2 models) are comprised of multiple sub-models. By default, all sub-models will be fully loaded and initialized when the model is loaded. + +If you wish to avoid the overhead of initializing unused models and only initialize the active model on load, you can do: + +``` +loader->SetCompositeModelLoadMode(ECompositeModelLoadMode::OnDemand); +``` + +Note that this means that switching to a different model for the first time via quality scaling will ***not be realtime safe***. + +## Setting model quality scaling factor + +Some models (notably, slimmable NAM A2 models) support quality scaling - trading off quality for performance. + +Quality scaling is a floating point range from 0.0 (highest performance) to 1.0 (highest quality). + +To set the default quality scaling factor, set it on the loader: + +``` +loader.SetDefaultQualityScaleFactor(scaleFactor); +``` + +To check if a model supports quality scaling, do: + +``` +if (model->HasQualityScaling()) ... +``` + +To set the quality scaling factor for a loaded model, do: + +``` +model->SetQualityScaleFactor(scaleFactor); +``` + +***Note: This operation is not real-time safe if the quality scale factor results in switching to an uninitialized model.*** If you are using the default composite model loading behavior, setting the quality scale factor is always real-time safe. If you are using "OnDemand" composite model loading, you can check whether a quality scale change is real-time safe by doing: + +``` +if (!model->IsQualityChangeRealtimeSafe(newScaleFactor) +{ + (call SetQualityScaleFactor(), but ensure it is not done in a real-time context) +} +``` + +To get the current quality scaling factor for a model, do: + +``` +float scaleFactor = model->GetQualityScaleFactor(); +``` + +## Model oversampling + +WaveNet models are altered (via scaling the convolution dilation sizes) to produce the correct output when the external sample rate is an even multiple of the model sample rate. + +By default, the external sample rate is set to 48kHz. To change it, do: + +``` +loader.SetExternalSampleRate(sampleRate); +``` + +Note that ***models are only altered on load***. If you change the external sample rate you will need to reload any existing models where you want the change to take effect. + +## Getting model metadata + +To retrieve arbitrary metadata fields from models that contain them, do: + +``` +std::string fieldName = "this_is_a_field_name"; + +std::string metadataValue = model->GetMetadata(fieldName); + +if (!metadataValue.empty()) +{ + // do something +} +``` +Results are always returned a strings. Field names are case sensitive. + +To get the model version string, do: + +``` +std::string version = model->GetModelVersion() +``` + +The string will be empty if no version information exists. ## Getting the model receptive field size @@ -100,7 +190,7 @@ int receptiveFieldSamples = model->GetReceptiveFieldSize(); Note that this can return -1, which means that the receptive field size is unknown, or not fixed (ie: LSTM models technically have an infinite tail because of their internal feedback loop). -This method is only supported for "internal" models. If you are using NAM Core or RTNeural it will always return -1. +This method is only supported for "internal" and NAM Core models. For RTNeural it will always return -1. # Building @@ -126,16 +216,33 @@ cmake --build . --config=release -j4 Note - you'll have to change the Visual Studio version if you are using a different one. -## CMake Options +## Performance considerations -```-DBUILD_NAMCORE=ON|OFF```: Support loading models using the NAM Core implemenations. +Optimization for this library is highly dependent on the details of the compiler you are using and the architecture of the system you intend to run on. In particular, getting the best performance relies on critical paths in the code being vectorized efficiently by the compiler for the target CPU. -```-DBUILD_STATIC_RTNEURAL=ON|OFF```: Build static RTNeural model architectures (slower compile, larger size - only use if you plan on forcing RTNeural model loading). +The CMake setup for the library *does not* specify any optimization compiler flags - you are responsible for doing that yourself. Specifically, that means doing the highest possible compiler optimization (ie: "-O3") +and making sure that the compiler is targeting your CPU properly (ie: "-march=native" for GCC/Clang, "/arch:AVX2" for MSVC). + +One more specific note - the ```MULTIFRAME_8X8_CONVOLUTION``` option described in the next section enables very impactful performance increases when set to "4" or "8" ("0" is the default) on appropriate hardware and compilers. +It also significantly *slows* performance if the optimizations are not properly supported. In general, assuming you have a very recent compiler you should be able to set it to "4" for systems with 128bit intrinsics (ie: Raspberry Pi 4) and "8" for systems with 256bit intrinsics (ie: Raspberry Pi 5, most x64 PCs). The best way to know what works in your scenario is to test it. The CMake config for this library does its best to default to the correct setting +based on architecture and compiler. + +The "ModelTest" application binaries provided in the [Releases section](https://github.com/mikeoliphant/NeuralAudio/releases) have been optimized for various specific platforms and can be used as a basis for comparison. + +## CMake Options ```-DBUILD_INTERNAL_STATIC_WAVENET=ON|OFF```: Build internal static WaveNet model architectures (faster internal WaveNet, but slower compile, larger size). ```-DBUILD_INTERNAL_STATIC_LSTM=ON|OFF```: Build internal static LSTM model architectures (faster internal LSTM, but slower compile, larger size). +```-DBUILD_STATIC_INTERNAL_NAMA2=ON|OFF```: Build internal static A2 implementation. + +```-DMULTIFRAME_8X8_CONVOLUTION=0|4|8```: Use optimized multiframe 8x8 convolution. Much faster on very modern compilers. Much slower on older compilers. Defaults to "0" (disabled). + +```-DDEFAULT_QUALITY_SCALE="X.X"```: Default model quality scale factor (0.0 to 1.0). Be sure to use quotes around value. Defaults to "1.0". + +```-DDEFAULT_INPUT_DBU="XX"```: Default dBu level for model input calibration. Be sure to use quotes around floating point values. Defaults to "12.0". + ```-DWAVENET_FRAMES=XXX```: Sample buffer size for the internal WaveNet implementation. Defaults to **64**. If you know you are going to be using a fixed sample buffer smaller or larger than this, use that instead. Note that the model will still be able to process any buffer size - it is just optimized for this size. ```-DBUFFER_PADDING=XXX```: Amount of padding to convolution layer buffers. This allows ring buffer resets to be staggered accross layers to improve performance. It also uses a significant amount of memory. It is set to **24** by default. It can be set all the way down to 0 to reduce memory usage. @@ -149,6 +256,19 @@ Note - you'll have to change the Visual Studio version if you are using a differ ```-DBUILD_UTILS=ON|OFF```: Build performance/accuracy testing tools (located in the "Utils" folder). +```-DBUILD_NAMCORE=ON|OFF```: Support loading models using the NAM Core implemenations. + +```-DNAM_USE_INLINE_GEMM=ON|OFF```: Enable use of inline matrix multiplication in NAM Core. + +```-DNAM_ENABLE_A2_FAST=ON|OFF```: Enable use of A2 fast path wavenet in NAM Core. + +```-DNAM_A2_RING_MODE=0|1```: Ring-buffer strategy for A2 FAST. Defaults to "1" + + - ```0```: linear memmove-rewind (variable worst-case latency, sporadic spikes) + - ```1```(the default): pow2 + tail mirror (constant per-block work, branchless reads) + +```-DBUILD_STATIC_RTNEURAL=ON|OFF```: Build static RTNeural model architectures (slower compile, larger size - only use if you plan on forcing RTNeural model loading). + # Software/Hardware Using NeuralAudio The following applications and devices are using the NeuralAudio library for model processing: @@ -157,3 +277,6 @@ The following applications and devices are using the NeuralAudio library for mod - [stompbox](https://github.com/mikeoliphant/stompbox): Guitar amplification and effects pedalboard simulation. - [NeuralRack](https://github.com/brummer10/NeuralRack): Neural Model and Impulse Response File loader for Linux/Windows. - [Darkglass Anagram](https://www.darkglass.com/creation/anagram): Bass guitar effects unit. +- [neural_tilde](https://github.com/apresta/neural_tilde): Max/MSP external for running neural amplifier captures. +- [Pi Pedal](https://github.com/rerdavies/pipedal): Guitar Effect Pedal for Raspberry Pi. +- [Casette](https://github.com/pawelKapl/Casette): Raspberry Pie guitar processor. diff --git a/Utils/ModelTest/ModelTest.cpp b/Utils/ModelTest/ModelTest.cpp index c4974db..b62345a 100644 --- a/Utils/ModelTest/ModelTest.cpp +++ b/Utils/ModelTest/ModelTest.cpp @@ -1,3 +1,4 @@ +#include #include #include #include "argparse.hpp" @@ -7,14 +8,21 @@ using namespace NeuralAudio; static std::string LoadModes[] = { "Internal", "RTNeural", "NAMCore" }; -NeuralModel* LoadModel(std::filesystem::path modelPath, EModelLoadMode loadMode) +NeuralModel* LoadModel(std::filesystem::path modelPath, NeuralModelLoader& loader, EModelLoadMode loadMode) { - NeuralModel::SetWaveNetLoadMode(loadMode); - NeuralModel::SetLSTMLoadMode(loadMode); + loader.SetWaveNetLoadMode(loadMode); + loader.SetLSTMLoadMode(loadMode); + + if (!std::filesystem::exists(modelPath)) + { + std::cout << "Model file does not exist: " << modelPath << std::endl; + + return nullptr; + } try { - auto model = NeuralAudio::NeuralModel::CreateFromFile(modelPath); + auto model = loader.CreateFromFile(modelPath); if (model == nullptr) { @@ -48,7 +56,7 @@ NeuralModel* LoadModel(std::filesystem::path modelPath, EModelLoadMode loadMode) return nullptr; } -static std::tuple BenchModel(NeuralModel* model, int blockSize, int numBlocks) +static double BenchModel(NeuralModel* model, int blockSize, int numBlocks) { std::vector inData; inData.resize(blockSize); @@ -58,24 +66,16 @@ static std::tuple BenchModel(NeuralModel* model, int blockSize, auto start = std::chrono::high_resolution_clock::now(); - double maxBlock = 0; - for (int block = 0; block < numBlocks; block++) { - auto blockStart = std::chrono::high_resolution_clock::now(); - model->Process(inData.data(), outData.data(), blockSize); - - auto blockEnd = std::chrono::high_resolution_clock::now(); - - maxBlock = std::max(maxBlock, std::chrono::duration_cast> (blockEnd - blockStart).count()); } auto end = std::chrono::high_resolution_clock::now(); double tot = std::chrono::duration_cast> (end - start).count(); - return std::tie(tot, maxBlock); + return tot; } static double ComputeError(NeuralModel* model1, NeuralModel* model2, int blockSize, int numBlocks) @@ -117,7 +117,12 @@ static double ComputeError(NeuralModel* model1, NeuralModel* model2, int blockSi return sqrt(totErr / (double)(blockSize * numBlocks)); } -void RunNAMTests(std::filesystem::path modelPath, int blockSize) +void PrintBench(std::string name, double time, int dataSize) +{ + std::cout << name << ": " << time << " (" << (((float)dataSize / 48000.0f) / time) << "xRT)" << std::endl; +} + +void RunNAMTests(std::filesystem::path modelPath, NeuralModelLoader& loader, int blockSize) { std::cout << "Model: " << modelPath << std::endl; std::cout << std::endl; @@ -126,59 +131,66 @@ void RunNAMTests(std::filesystem::path modelPath, int blockSize) int numBlocks = dataSize / blockSize; - NeuralModel::SetDefaultMaxAudioBufferSize(blockSize); + loader.SetDefaultMaxAudioBufferSize(blockSize); - NeuralModel* rtNeuralModel = LoadModel(modelPath, EModelLoadMode::RTNeural); - NeuralModel* namCoreModel = LoadModel(modelPath, EModelLoadMode::NAMCore); - NeuralModel* internalModel = LoadModel(modelPath, EModelLoadMode::Internal); + NeuralModel* rtNeuralModel = LoadModel(modelPath, loader, EModelLoadMode::RTNeural); + NeuralModel* namCoreModel = LoadModel(modelPath, loader, EModelLoadMode::NAMCore); + NeuralModel* internalModel = LoadModel(modelPath, loader, EModelLoadMode::Internal); double rms; - if (namCoreModel != nullptr) - { - } - - std::tuple internal; - std::tuple rtNeural; - std::tuple namCore; + double internal; + double rtNeural; + double namCore; - internal = BenchModel(internalModel, blockSize, numBlocks); + if (internalModel != nullptr) + { + internal = BenchModel(internalModel, blockSize, numBlocks); - std::cout << "Internal: " << std::get<0>(internal) << " (" << std::get<1>(internal) << ")" << std::endl; + PrintBench("Internal", internal, dataSize); + } + else + { + std::cout << "Model can't be loaded as internal model" << std::endl; + } if (namCoreModel != nullptr) { - std::cout << std::endl; - namCore = BenchModel(namCoreModel, blockSize, numBlocks); - rms = ComputeError(namCoreModel, internalModel, blockSize, numBlocks); + PrintBench("NAM Core", namCore, dataSize); - std::cout << "NAM Core: " << std::get<0>(namCore) << " (" << std::get<1>(namCore) << ")" << std::endl; - std::cout << "NAM vs Internal RMS err: " << rms << std::endl; - std::cout << "Internal is: " << (std::get<0>(namCore) / std::get<0>(internal)) << "x NAM" << std::endl; + if (internalModel != nullptr) + { + rms = ComputeError(namCoreModel, internalModel, blockSize, numBlocks); + + std::cout << "NAM vs Internal RMS err: " << rms << std::endl; + std::cout << "Internal is: " << (namCore / internal) << "x NAM" << std::endl; + } } if (rtNeuralModel != nullptr) { - std::cout << std::endl; - rtNeural = BenchModel(rtNeuralModel, blockSize, numBlocks); - std::cout << "RTNeural: " << std::get<0>(rtNeural) << " (" << std::get<1>(rtNeural) << ")" << std::endl; - rms = ComputeError(namCoreModel, rtNeuralModel, blockSize, numBlocks); - std::cout << "NAM vs RTNeural RMS err: " << rms << std::endl; + PrintBench("RTNeural", rtNeural, dataSize); if (namCoreModel != nullptr) { - std::cout << "RTNeural is: " << (std::get<0>(namCore) / std::get<0>(rtNeural)) << "x NAM" << std::endl; + rms = ComputeError(namCoreModel, rtNeuralModel, blockSize, numBlocks); + std::cout << "NAM vs RTNeural RMS err: " << rms << std::endl; + + if (namCoreModel != nullptr) + { + std::cout << "RTNeural is: " << (namCore / rtNeural) << "x NAM" << std::endl; + } } } std::cout << std::endl; } -void RunKerasTests(std::filesystem::path modelPath, int blockSize) +void RunKerasTests(std::filesystem::path modelPath, NeuralModelLoader& loader, int blockSize) { std::cout << "Model: " << modelPath << std::endl; @@ -186,53 +198,70 @@ void RunKerasTests(std::filesystem::path modelPath, int blockSize) int numBlocks = dataSize / blockSize; - NeuralAudio::NeuralModel::SetDefaultMaxAudioBufferSize(blockSize); + loader.SetDefaultMaxAudioBufferSize(blockSize); - auto internalModel = LoadModel(modelPath, EModelLoadMode::Internal); - auto rtNeuralModel = LoadModel(modelPath, EModelLoadMode::RTNeural); + auto internalModel = LoadModel(modelPath, loader, EModelLoadMode::Internal); + auto rtNeuralModel = LoadModel(modelPath, loader, EModelLoadMode::RTNeural); double rms = ComputeError(rtNeuralModel, internalModel, blockSize, numBlocks); std::cout << "Internal vs RTNeural RMS err: " << rms << std::endl; std::cout << std::endl; - auto internal = BenchModel(internalModel, blockSize, numBlocks); - auto rt = BenchModel(rtNeuralModel, blockSize, numBlocks); + double internal = BenchModel(internalModel, blockSize, numBlocks); + double rt = BenchModel(rtNeuralModel, blockSize, numBlocks); - std::cout << "RTNeural: " << std::get<0>(rt) << " (" << std::get<1>(rt) << ")" << std::endl; - std::cout << "Internal: " << std::get<0>(internal) << " (" << std::get<1>(internal) << ")" << std::endl; - std::cout << "Internal is: " << (std::get<0>(rt) / std::get<0>(internal)) << "x RTNeural" << std::endl; + PrintBench("Internal", internal, dataSize); + PrintBench("RTNeural", rt, dataSize); + std::cout << "Internal is: " << (rt / internal) << "x RTNeural" << std::endl; std::cout << std::endl; } -int RunDefaultTests(int blockSize) +int RunDefaultTests(NeuralModelLoader& loader, int blockSize) { std::filesystem::path modelPath = std::filesystem::current_path(); - while (modelPath.filename() != "Utils") + do { + if (std::filesystem::exists((modelPath / "Models"))) + break; + modelPath = modelPath.parent_path(); + } + while (modelPath != modelPath.root_path()); - if (modelPath == modelPath.root_path()) - { - std::cout << "Unable to find Models: " << std::filesystem::current_path() << std::endl; - std::cout << "ModelTest must be run from within the Utils subdirectory" << std::endl; + if (modelPath == modelPath.root_path()) + { + std::cout << "Unable to find Models: " << std::filesystem::current_path() << std::endl; + std::cout << "ModelTest looks for a \"Models\" folder in current folder or up the path." << std::endl; + std::cout << "You can also specify a specific model to test by passing the path on the commandline." << std::endl; - return -1; - } + return -1; } modelPath = modelPath / "Models"; std::cout << "Loading models from: " << modelPath << std::endl << std::endl; - std::cout << "WaveNet (Standard) Test" << std::endl; - RunNAMTests(modelPath / "BossWN-standard.nam", blockSize); + std::cout << "WaveNet (A2 Full) Test" << std::endl; + loader.SetDefaultQualityScaleFactor(1.0f); + RunNAMTests(modelPath / "BossWN-a2.nam", loader, blockSize); + + std::cout << std::endl; + + std::cout << "WaveNet (A2 Lite) Test" << std::endl; + loader.SetDefaultQualityScaleFactor(0.0f); + RunNAMTests(modelPath / "BossWN-a2.nam", loader, blockSize); + + std::cout << std::endl; + + std::cout << "WaveNet (A1 Standard) Test" << std::endl; + RunNAMTests(modelPath / "BossWN-standard.nam", loader, blockSize); std::cout << std::endl; std::cout << "LSTM (1x16) Test" << std::endl; - RunNAMTests(modelPath / "BossLSTM-1x16.nam", blockSize); + RunNAMTests(modelPath / "BossLSTM-1x16.nam", loader, blockSize); return 0; } @@ -252,7 +281,13 @@ int main(int argc, char* argv[]) .required() .help("Specify the number of iterations") .scan<'i', int>(); - + + program.add_argument("-q", "--quality_scale") + .default_value(1.0f) + .nargs(1) + .required() + .help("Quality scaling (0.0 is fasteset, 1.0 is highest quality") + .scan<'g', float>(); try { program.parse_args(argc, argv); @@ -271,22 +306,30 @@ int main(int argc, char* argv[]) blockSize = program.get("--block_size"); - std::cout << "Block size: " << blockSize << std::endl; + float qualityScale = 1.0f; + + qualityScale = program.get("--quality_scale"); + + NeuralModelLoader loader; + + loader.SetDefaultQualityScaleFactor(qualityScale); + + std::cout << "Block size: " << blockSize << " Quality Scale: " << qualityScale << std::endl; if (!modelPath.empty()) { if (modelPath.extension() == ".nam") { - RunNAMTests(modelPath, blockSize); + RunNAMTests(modelPath, loader, blockSize); } else { - RunKerasTests(modelPath, blockSize); + RunKerasTests(modelPath, loader, blockSize); } } else { - if (RunDefaultTests(blockSize) < 0) + if (RunDefaultTests(loader, blockSize) < 0) return -1; } diff --git a/Utils/Models/BossWN-a2.nam b/Utils/Models/BossWN-a2.nam new file mode 100644 index 0000000..1376177 --- /dev/null +++ b/Utils/Models/BossWN-a2.nam @@ -0,0 +1 @@ +{"version":"0.7.0","architecture":"SlimmableContainer","config":{"submodels":[{"max_value":0.5,"model":{"version":"0.7.0","metadata":{"date":{"year":2026,"month":5,"day":17,"hour":22,"minute":50,"second":14},"loudness":-12.17414665222168,"gain":0.6298792269436364},"architecture":"WaveNet","config":{"layers":[{"input_size":1,"condition_size":1,"head":{"out_channels":1,"kernel_size":16,"bias":true},"channels":3,"kernel_sizes":[6,6,6,6,6,6,6,6,6,6,6,6,6,6,15,15,6,6,6,6,6,6,6],"dilations":[1,3,7,17,41,101,239,1,3,7,17,41,101,239,1,13,1,3,7,17,41,101,239],"activation":[{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01}],"bottleneck":3,"head1x1":{"active":false,"out_channels":1,"groups":1},"layer1x1":{"active":true,"groups":1},"groups_input":1,"groups_input_mixin":1,"conv_pre_film":{"active":false,"shift":true,"groups":1},"conv_post_film":{"active":false,"shift":true,"groups":1},"input_mixin_pre_film":{"active":false,"shift":true,"groups":1},"input_mixin_post_film":{"active":false,"shift":true,"groups":1},"activation_pre_film":{"active":false,"shift":true,"groups":1},"activation_post_film":{"active":false,"shift":true,"groups":1},"layer1x1_post_film":{"active":false,"shift":true,"groups":1},"head1x1_post_film":{"active":false,"shift":true,"groups":1},"gating_mode":["none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none"],"secondary_activation":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"slimmable":null}],"head":null,"head_scale":0.013119162069029721},"weights":[-2.2014312744140625,-2.3644347190856934,4.10330057144165,0.013875482603907585,0.15356875956058502,0.3278008699417114,0.13442133367061615,-0.4063030779361725,-1.0289421081542969,0.20730212330818176,0.11322535574436188,0.3868657350540161,0.10031043738126755,-0.31691259145736694,-1.0278282165527344,-0.12845806777477264,-0.3099781572818756,-0.272651731967926,-0.056800853461027145,0.4446815252304077,1.354359745979309,0.15971176326274872,0.08661764115095139,-0.10098310559988022,0.18293239176273346,0.05380669981241226,0.23948296904563904,0.09054432064294815,-0.08613887429237366,-0.1300751268863678,0.15587326884269714,0.11223623901605606,0.19360223412513733,-0.16403675079345703,0.16310861706733704,-0.007865854538977146,0.08529536426067352,-0.34021446108818054,-0.3202733099460602,-0.16050001978874207,-0.15973332524299622,-0.1709902435541153,-0.2039112001657486,0.23347772657871246,0.7998841404914856,-0.15269169211387634,-0.08793331682682037,-0.39089450240135193,-0.1976887583732605,0.25692084431648254,0.9227266311645508,0.254395455121994,0.1028246060013771,0.2839694321155548,0.2400667518377304,-0.27344000339508057,-1.0547384023666382,1.6182467937469482,1.2084239721298218,1.7490651607513428,1.002180814743042,-0.7275207042694092,-1.1934421062469482,-0.25818192958831787,0.43737220764160156,0.17171142995357513,-0.3276022970676422,0.5688443779945374,0.4970926344394684,0.39238572120666504,0.3128047585487366,-0.44379669427871704,0.04468515142798424,0.41651737689971924,0.29364076256752014,-0.04451122134923935,-0.028112323954701424,-0.02097748965024948,-0.012702057138085365,0.06268917769193649,-0.16147184371948242,-0.29312142729759216,-0.09629882127046585,-0.02723022736608982,0.06206013262271881,0.16032196581363678,0.05538935586810112,0.23848466575145721,-0.27474716305732727,-0.33876675367355347,-0.033302415162324905,-0.10256210714578629,-0.1081211045384407,0.11435101181268692,0.2447659820318222,0.2783104181289673,0.2373391091823578,-0.159987673163414,-0.5479333996772766,0.2566385269165039,0.18790094554424286,0.2716050148010254,0.1309361457824707,0.18553760647773743,-0.14580664038658142,-0.014284756034612656,-0.09840688854455948,-0.16540786623954773,0.11237860471010208,0.14221222698688507,0.9043796062469482,-0.0736541599035263,-0.20429277420043945,0.09105397015810013,-0.04670644924044609,0.09283225983381271,0.4834168255329132,0.2033672034740448,-0.05711207911372185,-0.038311850279569626,-0.164163738489151,-0.08022404462099075,0.13044373691082,0.2241334766149521,0.4343363344669342,0.3230689465999603,0.26705202460289,0.26982206106185913,-0.20042891800403595,-0.05943610146641731,0.23236514627933502,0.2435539811849594,0.08832674473524094,1.2771165370941162,-0.19556638598442078,0.14746785163879395,0.06528852134943008,-0.07936146855354309,-0.4645145833492279,-0.6456254720687866,0.2770215570926666,0.003341606119647622,0.31739142537117004,-0.09175405651330948,-0.47012990713119507,-0.13660883903503418,0.20460069179534912,-0.10530567914247513,0.09297942370176315,0.5133674144744873,0.45620226860046387,0.4208831787109375,-0.32265979051589966,-0.03083098866045475,-0.00627914909273386,-0.05858776718378067,-0.11458605527877808,-0.10568328201770782,-0.38585251569747925,-0.056530069559812546,0.05505377799272537,0.030275659635663033,-0.38001692295074463,-0.5056266188621521,0.22018752992153168,-0.07126032561063766,-0.20173831284046173,-0.3261547386646271,-0.359453409910202,-0.015088825486600399,0.42483004927635193,-0.018563169986009598,-0.043394844979047775,0.0406346395611763,0.08621232956647873,0.053887128829956055,0.32103431224823,-0.034591853618621826,-0.04755638167262077,0.15458261966705322,0.3090440034866333,0.37724411487579346,-0.4374990463256836,0.2141016125679016,0.13493311405181885,0.26272568106651306,0.33496421575546265,0.15733586251735687,-0.5267455577850342,0.0708104819059372,0.04849817976355553,0.03587888926267624,-0.07463464140892029,0.001113251899369061,-0.030115514993667603,0.101874940097332,-0.028080172836780548,-0.11411042511463165,-0.0713108479976654,-0.10225676000118256,0.5586898326873779,-0.36880797147750854,0.23473967611789703,0.22187674045562744,-0.08366920053958893,-0.39052024483680725,0.5241316556930542,0.3194989562034607,-0.29806801676750183,0.2259378880262375,-0.8656169176101685,0.6054820418357849,-1.0572688579559326,0.10955116152763367,0.05774696171283722,0.19563937187194824,0.34555792808532715,0.3106386959552765,0.3284628391265869,0.1351257860660553,0.0653991848230362,0.056746706366539,0.115949347615242,-0.5801233649253845,-0.11422739923000336,0.07654698938131332,0.060470253229141235,0.014784484170377254,0.03117985650897026,0.05313900113105774,-0.3193320333957672,0.197667196393013,0.15637464821338654,0.019759058952331543,0.1039513498544693,0.4362679123878479,-0.008409534581005573,-0.11793550103902817,-0.14007437229156494,-0.1831897646188736,-0.01337550487369299,0.24091477692127228,0.1191105768084526,-0.04455364868044853,-0.08310576528310776,-0.08645974844694138,-0.06507641822099686,-0.05547230318188667,-0.7532620429992676,-0.12714485824108124,-0.21965573728084564,-0.2249358892440796,-0.1996520459651947,-0.3197749853134155,-0.08810564875602722,0.1319267600774765,0.20888161659240723,0.15505792200565338,-0.06444970518350601,-0.6391368508338928,0.16648919880390167,0.02057655341923237,0.043972697108983994,0.07323552668094635,0.05857202410697937,-0.05138305202126503,0.5299592614173889,0.0357242152094841,0.06755470484495163,0.1893053948879242,0.170505091547966,0.1796083301305771,-0.14555144309997559,0.17707335948944092,-0.38982537388801575,0.22848962247371674,-0.08042185008525848,-0.07579585909843445,-0.3839818835258484,-0.21525897085666656,-0.049253564327955246,0.23048636317253113,0.4585550129413605,0.6998566389083862,0.5793481469154358,-0.11139228194952011,-0.23135340213775635,-0.2864333987236023,-0.04127928987145424,-0.24587275087833405,0.17506276071071625,-0.024926166981458664,-0.09200415760278702,-0.09672544151544571,0.0023581082932651043,0.0559343658387661,0.20842275023460388,0.03211093321442604,0.04095561057329178,0.09505100548267365,0.08572191745042801,0.011949539184570312,-0.5728508830070496,0.0651869848370552,0.07643463462591171,0.18087291717529297,0.20300918817520142,-0.021814070641994476,0.33623814582824707,0.023220300674438477,0.09634418785572052,0.08783882111310959,0.03317289054393768,-0.013059227727353573,-0.14349544048309326,0.003293004585430026,-0.0003971911792177707,0.01377302035689354,0.004187173675745726,-0.00927561055868864,-0.10491470247507095,0.011596589349210262,0.004819414112716913,0.0596567839384079,0.025614546611905098,0.00022714523947797716,-0.2966059744358063,0.08415508270263672,-0.0512981191277504,-0.027162181213498116,-0.030133729800581932,-0.06166153401136398,0.14925351738929749,0.01606331579387188,-0.01696416363120079,-0.014036207459867,-0.04851203411817551,0.011458122171461582,-0.051346227526664734,0.07395359128713608,0.004695982672274113,0.06185265630483627,-0.04176633805036545,0.027011973783373833,0.27004680037498474,0.2184954732656479,0.08525806665420532,-0.04840027168393135,0.06630273163318634,-0.03354614973068237,0.12632602453231812,-0.020363574847579002,0.16040000319480896,-0.24998949468135834,-0.6133416891098022,0.3964405357837677,-0.08699718117713928,-0.05510358512401581,0.030894216150045395,-0.13260945677757263,-0.38290613889694214,-0.04843368008732796,0.541000485420227,0.019082050770521164,0.007336693350225687,-0.0015728918369859457,-0.05935760587453842,-0.14012505114078522,0.007425954565405846,-0.007963333278894424,-0.007962853647768497,-0.0028553702868521214,-0.016793249174952507,-0.014149646274745464,-0.6677324175834656,-0.029075751081109047,-0.027568113058805466,-0.020008312538266182,-0.08829547464847565,-0.12561483681201935,-0.2089890092611313,-0.0086365332826972,-0.02048121578991413,-0.04040398448705673,-0.07442653924226761,-0.11849841475486755,0.5839603543281555,-0.005353859625756741,-0.012812442146241665,-0.004449067171663046,-0.023512255400419235,-0.012884330004453659,-0.486892431974411,-0.029179057106375694,-0.03881366550922394,-0.018658852204680443,-0.11640860885381699,-0.05758708715438843,-0.02758430503308773,0.010116863064467907,0.03526017814874649,0.0957784578204155,0.18534670770168304,0.34974345564842224,-0.2780730426311493,0.009035254828631878,0.005115012172609568,0.00764759024605155,0.04431099444627762,0.010754058137536049,0.010473896749317646,0.027558807283639908,0.020677799358963966,0.045722510665655136,0.18427978456020355,0.20525576174259186,0.15279117226600647,-0.27704840898513794,-0.09763302654027939,0.1571602076292038,-0.06785169988870621,-0.12322850525379181,-0.04365439713001251,0.025742774829268456,-0.087237149477005,0.16346365213394165,1.0164475440979004,-0.3515949845314026,0.3112505078315735,-0.3225294351577759,-0.018905742093920708,-0.39615869522094727,-0.16421464085578918,-0.3041362762451172,0.3561646640300751,-0.02584652788937092,-0.03723614662885666,-0.0378250852227211,-0.043828085064888,-0.033122528344392776,-0.05960679054260254,-0.0123366117477417,-0.016761887818574905,-0.017051242291927338,-0.018122490495443344,-0.013254676945507526,-0.0065485830418765545,-0.06753022223711014,-0.08714424818754196,-0.07974662631750107,-0.10276509821414948,-0.08740398287773132,0.3400799334049225,-0.04502898454666138,-0.05914813652634621,-0.027337580919265747,-0.029985355213284492,-0.041764311492443085,-0.3453563153743744,-0.019694466143846512,-0.02872847020626068,-0.014559841714799404,-0.01689673773944378,-0.018111858516931534,0.1508011817932129,-0.10746829211711884,-0.14647206664085388,-0.07456900924444199,-0.09562823921442032,-0.06073016673326492,0.03304087743163109,0.0030416434165090322,0.01756305620074272,-0.005867429543286562,-0.0070260995998978615,0.09984858334064484,0.027783868834376335,0.0008591634687036276,0.005053444765508175,-0.0029418659396469593,-0.006573853082954884,-0.02672344446182251,0.4023105800151825,0.0064987437799572945,0.02932850830256939,-0.0160983819514513,-0.03192165121436119,-0.07437776029109955,0.24474897980690002,-0.2958236038684845,-0.3867059350013733,0.0019733794033527374,0.27910539507865906,0.07492859661579132,0.08461777120828629,-0.06402987986803055,-0.13830603659152985,0.09749722480773926,-0.22630615532398224,0.3731735348701477,0.22992894053459167,-0.34267500042915344,-0.5737828016281128,-0.4918070137500763,-0.45648831129074097,0.05826294422149658,0.420463889837265,0.16099809110164642,0.06086291745305061,-0.015186207368969917,0.03053053840994835,0.09299395978450775,0.17543672025203705,-0.1887088567018509,-0.0016415286809206009,0.08831943571567535,0.17218364775180817,0.21438223123550415,-0.25929197669029236,-0.32457491755485535,0.05628906562924385,0.023955170065164566,-0.09208360314369202,0.04444421827793121,0.03765854611992836,-0.022934360429644585,0.2666116952896118,-0.11071984469890594,-0.14637991786003113,-0.2116205096244812,-0.15522781014442444,0.032325442880392075,0.30661794543266296,0.034208860248327255,-0.05703316628932953,0.035322047770023346,-0.10946434736251831,-0.06372347474098206,0.1628868728876114,0.2534482777118683,0.055746499449014664,-0.07275748252868652,0.2312823385000229,-0.102630615234375,-0.15631455183029175,-0.24276049435138702,0.02730661630630493,0.06320495158433914,0.21693485975265503,0.0053083705715835094,0.05945627763867378,0.0044203768484294415,0.14401677250862122,0.6592717170715332,0.6577001810073853,0.06053007394075394,0.033865492790937424,0.0725623071193695,0.24501881003379822,-0.06038414686918259,0.16380807757377625,-0.09601780027151108,-0.060286134481430054,0.2112821340560913,-0.005282821599394083,0.2468477487564087,-0.6443699598312378,-0.4285217225551605,0.19263726472854614,0.6201222538948059,-0.17815150320529938,-0.4198221266269684,0.7243912220001221,0.5521109700202942,-0.28732725977897644,-0.042256489396095276,-0.15002775192260742,-0.37756964564323425,0.6629250645637512,0.1854286789894104,0.11356539279222488,0.17620515823364258,0.21373027563095093,-0.22099517285823822,-0.15375979244709015,-0.05918728932738304,0.011076151393353939,-0.04833417013287544,-0.04595540091395378,-0.2917470335960388,-0.13568267226219177,0.22090356051921844,0.1213298961520195,0.18485966324806213,0.21171437203884125,0.3657015562057495,0.05172756686806679,-0.17758505046367645,-0.18795205652713776,-0.1028355211019516,-0.06293430179357529,-0.006775418296456337,-0.23636171221733093,0.08007046580314636,0.05864149332046509,0.07869783788919449,0.12085473537445068,0.24944719672203064,0.16303913295269012,0.21768684685230255,0.08480014652013779,-0.022742707282304764,0.19164447486400604,0.15293581783771515,0.09564486891031265,0.19310149550437927,0.06410381197929382,0.09536483138799667,0.045794595032930374,-0.02825321815907955,-0.08688146620988846,-0.06949115544557571,-0.028912706300616264,-0.10171185433864594,0.02747916989028454,-0.3905165493488312,0.014015856198966503,0.04064818099141121,0.09153930842876434,0.1189781203866005,0.2024744749069214,0.25191307067871094,0.13821421563625336,0.2617477774620056,0.1649087518453598,0.1616164743900299,-0.13608133792877197,-0.25968968868255615,-0.03105182945728302,0.31666168570518494,-0.005689946934580803,0.15644478797912598,-0.5720689296722412,0.11084175854921341,-0.3403335213661194,-0.40763193368911743,0.26767855882644653,-0.441596657037735,0.2859145402908325,-0.5444028377532959,0.29211774468421936,-0.07738512754440308,-0.1804807186126709,-0.02522926777601242,-0.09450779855251312,-0.13646376132965088,0.29535430669784546,0.040519386529922485,0.04803149029612541,0.02461175061762333,0.016729354858398438,0.07254209369421005,-0.28055301308631897,-0.1413072645664215,-0.19903384149074554,-0.14780324697494507,-0.11357695609331131,-0.21471437811851501,-0.3246343433856964,-0.0510096475481987,-0.08354977518320084,0.12729278206825256,-0.10149084031581879,-0.05479840934276581,0.16306541860103607,-0.015471145510673523,-0.007160392589867115,-0.15260739624500275,-0.11011581867933273,-0.2112150341272354,0.1642652153968811,-0.041242215782403946,0.07066415250301361,0.22906073927879333,-0.012592502869665623,0.03516409546136856,0.00900303479284048,-0.06584323942661285,-0.03831200301647186,-0.21301425993442535,0.04159072786569595,-0.008085627108812332,0.12950187921524048,0.01966777630150318,0.007433672901242971,0.049937669187784195,-0.015088389627635479,0.009842945262789726,-0.3282533288002014,-0.044814977794885635,-0.02022007293999195,-0.16136394441127777,-0.0007228617905639112,0.025680389255285263,-0.4317805767059326,-0.2836703658103943,0.21167778968811035,-0.17584557831287384,-0.03210986778140068,0.007393332198262215,0.06711545586585999,-0.29140788316726685,0.09494075179100037,0.060016632080078125,0.6892462968826294,-0.12644565105438232,0.5437824726104736,0.3083539605140686,-0.09905592352151871,-0.07869111746549606,0.1842239499092102,0.11328814923763275,0.473525732755661,0.0931219682097435,0.04723215475678444,-0.00942918099462986,0.12776996195316315,0.10211355239152908,-0.13509821891784668,-0.026894384995102882,-0.003409402910619974,0.021159030497074127,-0.025657540187239647,0.03933345153927803,-0.10009203106164932,0.019600436091423035,0.009639068506658077,-0.0657072588801384,0.1254866123199463,0.0225848201662302,0.10728231817483902,-0.01374603621661663,-0.07630118727684021,0.11476660519838333,-0.0681256502866745,0.12623153626918793,-0.3740171790122986,-0.006826187949627638,0.02012096345424652,-0.039208441972732544,0.034855764359235764,-0.021081412211060524,-0.08178651332855225,0.04184757545590401,-0.02104531228542328,0.12483389675617218,-0.037485286593437195,0.12625505030155182,0.4177972674369812,0.005522859748452902,0.05570603534579277,0.12235534191131592,0.04680335894227028,0.29607969522476196,-0.28425854444503784,-0.0033091583754867315,-0.02145201526582241,-0.042982712388038635,-0.0012671989388763905,-0.07852332293987274,-0.3635716736316681,0.011116636916995049,0.06510531157255173,0.11740104854106903,-0.0036171970423310995,0.3098653554916382,0.17680472135543823,-0.05556192621588707,0.11135676503181458,0.09721453487873077,0.05879608169198036,-0.03277534246444702,-0.07436835020780563,-0.1520647257566452,0.0529743954539299,-0.09556834399700165,-0.3573068380355835,-0.4127682149410248,-0.5504496097564697,-0.058370739221572876,0.06758173555135727,-0.41957026720046997,0.5789896249771118,-0.5567089915275574,0.02437531016767025,0.01897905021905899,0.008736285381019115,0.13045452535152435,0.05155551806092262,0.03634728863835335,-0.07670553028583527,-0.014921480789780617,-0.0001275207760045305,-0.04321090504527092,-0.013399453833699226,-0.030683742836117744,-0.3170943856239319,0.02865612879395485,-0.02591852843761444,0.050633110105991364,0.03243114426732063,0.049704816192388535,-0.13640345633029938,-0.017414020374417305,-0.09700603038072586,-0.13105517625808716,-0.1012539342045784,-0.008842669427394867,-0.08316817134618759,0.0076532321982085705,0.026950640603899956,0.04738757014274597,0.027548838406801224,0.07576663792133331,0.05371657758951187,0.004232553765177727,-0.029722118750214577,-0.017079457640647888,-0.034212835133075714,0.0678187906742096,0.1804538369178772,-0.010413448326289654,-0.03706008940935135,-0.042372141033411026,0.005118170287460089,0.015261722728610039,-0.0745769813656807,-0.05799354612827301,0.01239084079861641,0.013459793291985989,0.0033835917711257935,-0.0006350994808599353,0.05803684517741203,0.004481020383536816,0.03986607864499092,0.06641774624586105,0.01457990426570177,0.04627985134720802,-0.014160661026835442,0.20562240481376648,0.029852863401174545,0.004694575443863869,-0.09294068068265915,-0.05241657420992851,0.006039710249751806,-0.17241093516349792,0.0891229659318924,0.1347063183784485,-0.23249280452728271,-0.10104064643383026,0.006114705931395292,-0.054887741804122925,0.1170739158987999,-0.021641410887241364,0.4166780412197113,-0.7408673167228699,-0.34534701704978943,-0.004390317015349865,-0.0133216492831707,-0.013962293975055218,-0.04001116007566452,-0.0017552917124703526,0.008528734557330608,0.03392954170703888,0.0360761359333992,0.06000314652919769,0.0953037291765213,0.039254967123270035,-0.015198169276118279,-0.13355867564678192,-0.12131726741790771,-0.20218780636787415,-0.16920284926891327,-0.24456675350666046,-0.04598214477300644,-0.06402838975191116,-0.05914926901459694,-0.11267099529504776,-0.09625673294067383,-0.018514737486839294,-0.09744048863649368,0.02030935324728489,0.02018473669886589,0.037463828921318054,0.03457596153020859,0.006644686684012413,0.825168788433075,-0.03622704744338989,-0.035193800926208496,-0.05690038949251175,-0.07324764132499695,-0.03475940600037575,0.07583461701869965,-0.07657330483198166,-0.08096276968717575,-0.1342262625694275,-0.16198377311229706,-0.19149421155452728,-0.43036723136901855,0.02622113563120365,0.02749551460146904,0.04527297616004944,0.05606325343251228,0.06924230605363846,-0.3003419041633606,-0.034038741141557693,-0.035073067992925644,-0.06426286697387695,-0.06528802216053009,-0.08003722876310349,-0.6930965781211853,-0.032366540282964706,-0.42031794786453247,-0.5722066760063171,0.0567447692155838,0.38014471530914307,-0.029751446098089218,-0.1923818439245224,-0.456892192363739,0.11181174218654633,0.1238681897521019,-1.2726902961730957,0.7340671420097351,0.16623497009277344,0.0877658799290657,0.6717463135719299,0.09809945523738861,0.11721499264240265,0.24386271834373474,-0.0468749925494194,-0.092890165746212,-0.06576604396104813,-0.11520735174417496,-0.02461431920528412,-0.08256372809410095,-0.004163934849202633,-0.0200889240950346,0.00592941464856267,-0.021959252655506134,0.032569244503974915,-0.035753559321165085,0.0015350664034485817,0.06677763164043427,-0.0041939979419112206,0.06927374005317688,-0.05020373314619064,0.05175931379199028,-0.024826476350426674,-0.018296018242836,-0.011916188523173332,-0.018089836463332176,-0.015844322741031647,-0.016764501109719276,0.001426394795998931,0.006738968193531036,0.0193238016217947,0.03287462517619133,0.005511725787073374,0.0040188804268836975,-0.009849431924521923,-0.008338280022144318,0.004751404281705618,0.011433483101427555,0.0008858044166117907,-0.0023382750805467367,-0.0011220235610380769,-0.0036734105087816715,-0.0037256337236613035,-0.0031578040216118097,-0.004243722651153803,-0.0014916519867256284,0.012692187912762165,0.0043168445117771626,0.00796261802315712,0.007874337024986744,0.009626087732613087,0.009238019585609436,-0.0077214716002345085,-0.0023708934895694256,-0.0053356848657131195,-0.007169315591454506,-0.008372701704502106,-0.01434654463082552,-0.0124212009832263,-0.00593726197257638,-0.0014124347362667322,0.014019911177456379,-0.0015306307468563318,-0.0009815986268222332,0.018811382353305817,0.004848717246204615,-0.0023553117644041777,0.13777117431163788,0.010609139688313007,-0.0008198893629014492,0.04232117533683777,0.01120645273476839,-0.003369665704667568,0.057212989777326584,0.2081596553325653,-0.06718017905950546,0.17269113659858704,0.009713338688015938,-0.03404852747917175,0.08491694927215576,0.11330597847700119,0.041843436658382416,0.045436955988407135,0.1359654664993286,-0.01210862398147583,0.10351971536874771,-0.09000325947999954,-0.13927295804023743,-0.15863150358200073,0.22291485965251923,0.03032924234867096,0.01198500581085682,-0.056901611387729645,0.022123921662569046,-0.010731500573456287,-0.051212161779403687,0.012613601051270962,0.02941378764808178,0.02553156204521656,0.1221400648355484,-0.07341241836547852,-0.5046965479850769,-0.6048814654350281,-0.47892361879348755,-0.15346777439117432,-0.03878682106733322,-0.0584254264831543,0.015974368900060654,0.15863056480884552,0.05177295580506325,-0.01242576353251934,-0.05994808301329613,0.10977894812822342,0.03320879116654396,0.039395589381456375,0.10629669576883316,0.058564599603414536,0.10299590975046158,0.03876424953341484,-0.12510058283805847,-0.05931894853711128,0.18011242151260376,0.0008341691573150456,0.0009805859299376607,0.16798320412635803,0.06688108295202255,0.029469454661011696,0.2033063918352127,0.24847964942455292,0.06179556995630264,-0.002073288196697831,-0.1087300106883049,0.10886983573436737,0.12217050790786743,-0.004577497951686382,-0.05989358201622963,-0.14923034608364105,-0.03575071692466736,0.05595117062330246,-0.08653093129396439,-0.0018496819538995624,0.011525840498507023,-0.02607526071369648,-0.09662096947431564,-0.004074508789926767,-0.15243199467658997,-0.015339260920882225,0.38580283522605896,0.5369555354118347,-0.015834636986255646,-0.28406691551208496,0.1948232203722,0.03356671333312988,0.04813729599118233,0.05075075104832649,0.06731343269348145,0.08344151824712753,0.015106960199773312,0.04820121079683304,0.12462487816810608,0.1424451470375061,-0.00261193560436368,0.15644453465938568,0.3097437620162964,0.21650174260139465,0.09976957738399506,-0.004249217454344034,-0.03585461899638176,-0.13884197175502777,0.03310080245137215,0.025318201631307602,0.01099180243909359,-0.0360720194876194,-0.06428101658821106,0.03864077851176262,0.07165437191724777,-0.017714744433760643,-0.1632019430398941,-0.06779570877552032,-0.02947402000427246,-0.0844537764787674,0.08557608723640442,-0.007783283945173025,-0.053134895861148834,-0.11975040286779404,-0.12594766914844513,-0.1929493397474289,-0.0352880097925663,0.04859646409749985,0.04121916741132736,0.09053142368793488,-0.13254739344120026,-0.13205386698246002,-0.1608976274728775,-0.12032034993171692,0.1704205572605133,-0.0199842881411314,0.06420713663101196,0.04240137338638306,0.026440057903528214,0.08491645008325577,0.06401057541370392,0.09507822245359421,0.004296616185456514,0.07048927247524261,0.13774053752422333,0.12633119523525238,0.05486954748630524,0.08617448061704636,0.028352098539471626,0.05629890784621239,0.2628430426120758,0.27364468574523926,0.12121216952800751,-0.02701500989496708,-0.10187099128961563,-0.0019906789530068636,0.08286310732364655,-0.5924199819564819,-0.06184062361717224,0.2982446849346161,-0.03407451510429382,-0.1747429519891739,-0.5763257145881653,0.637191653251648,-0.17938430607318878,-0.04755958169698715,0.038731761276721954,-0.6143099665641785,0.059068404138088226,0.11320030689239502,0.16534854471683502,0.11437245458364487,0.10673084110021591,0.07022818177938461,0.003832305548712611,0.05411868914961815,0.022144947201013565,0.05946105346083641,-0.011302797123789787,0.08274678140878677,0.1359892636537552,-0.17296907305717468,0.3382267951965332,0.01912323758006096,-0.029805863276124,-0.019274523481726646,-0.0012285838602110744,-0.05631855130195618,0.029644066467881203,-0.06144419312477112,0.0773959681391716,0.04722733423113823,0.013274071738123894,0.056786246597766876,-0.0013954784953966737,0.23763790726661682,0.008414988406002522,-0.41124045848846436,0.03231858089566231,0.10895248502492905,0.06642306596040726,0.06283331662416458,0.09885290265083313,0.028784483671188354,0.03015569970011711,0.07237100601196289,0.02827944979071617,0.08045416325330734,0.06920401751995087,0.07699686288833618,0.22619569301605225,0.47617295384407043,-0.535832405090332,0.01129608228802681,-0.04972101375460625,-0.08959131687879562,-0.05961409956216812,-0.1049358919262886,-0.16627180576324463,-0.08263284713029861,-0.16013680398464203,-0.13960857689380646,-0.14504225552082062,-0.07391935586929321,-0.0669969916343689,-0.06543442606925964,0.2150854617357254,-0.2799462080001831,-0.07504361867904663,0.002481478499248624,-0.019103670492768288,-0.028899382799863815,0.013751109130680561,-0.02258693426847458,0.08810557425022125,-0.04907526448369026,-0.036649320274591446,-0.016108041629195213,-0.026935124769806862,-0.012373811565339565,-0.22196374833583832,-0.08736121654510498,0.246938094496727,0.0024825208820402622,-0.0693335086107254,-0.03585716709494591,-0.030721642076969147,-0.07896463572978973,-0.04599202796816826,-0.054013997316360474,-0.10336010158061981,-0.05640396103262901,-0.09523042291402817,-0.09239568561315536,-0.052691757678985596,-0.18009594082832336,-0.4294753968715668,0.5425381064414978,-0.0372757725417614,-0.047823864966630936,-0.0104625029489398,-0.004671900998800993,-0.03973732516169548,-0.0021263654343783855,-0.023855945095419884,-0.0789547860622406,-0.10577692836523056,-0.08959463983774185,-0.03843184560537338,-0.06384005397558212,0.1419914811849594,-0.038728974759578705,0.1009673923254013,0.019970932975411415,0.04192150756716728,0.023478630930185318,0.013865449465811253,0.036482445895671844,0.021509593352675438,0.01584126614034176,0.055049218237400055,0.05191134288907051,0.05466006323695183,0.04351736977696419,0.02834985964000225,-0.022774936631321907,-0.013348772190511227,0.8476651310920715,-0.016304010525345802,-0.013945184648036957,-0.012984920293092728,-0.0027163932099938393,-0.012388846836984158,-0.005535949487239122,-0.0092982342466712,-0.02955726906657219,-0.04773883521556854,-0.02991931512951851,-0.01802026852965355,-0.02324429713189602,0.05831553786993027,-0.03732339292764664,0.24632878601551056,-0.05545299872756004,-0.017101304605603218,-0.1755329966545105,-0.011509559117257595,-0.021610479801893234,-0.07784798741340637,-0.07508524507284164,0.10352034121751785,0.7073677182197571,0.07185958325862885,-0.053868044167757034,0.27208173274993896,0.24398355185985565,-0.2285347580909729,0.2912147045135498,0.5949616432189941,-0.08625003695487976,-0.45112931728363037,0.5575506091117859,-0.3293210566043854,-0.7460311651229858,-0.293647825717926,0.6017194390296936,0.15850253403186798,0.47176310420036316,-0.6422329545021057,-0.5851084589958191,0.1897549033164978,0.04001835361123085,0.607490062713623,0.31152766942977905,-0.31967270374298096,0.014013879932463169,0.10590489953756332,-0.03616946563124657,-0.12072359025478363,0.07594557851552963,0.03325088322162628,-0.20154978334903717,-0.170691579580307,0.3313436210155487,-0.15262366831302643,0.18364350497722626,0.09518761932849884,-0.3837544620037079,0.38900402188301086,0.4690322279930115,-0.21908485889434814,0.06534942984580994,-0.09507252275943756,-0.10922722518444061,-0.05699625611305237,-0.031187966465950012,0.08619870245456696,0.1474815160036087,0.09033043682575226,-0.04655039682984352,-0.18047496676445007,-0.12387923151254654,0.26619669795036316,-0.16795149445533752,-0.12015614658594131,-0.37688353657722473,-0.28292834758758545,0.07180991023778915,0.9948890805244446,0.08901812136173248,0.16902439296245575,0.09275513887405396,0.0014691996620967984,0.005647479090839624,-0.3428989052772522,-0.31666532158851624,-0.4016331136226654,0.1796700656414032,-0.1153717190027237,-0.02604624256491661,0.21946120262145996,0.6540656089782715,0.1338268518447876,-0.1489035189151764,1.1311229467391968,0.9888778328895569,-0.9279384613037109,-0.6369128823280334,-0.2257556915283203,0.916847288608551,0.7586669921875,0.29432398080825806,-0.361706405878067,-0.006152325309813023,0.09592199325561523,0.19373579323291779,0.08641296625137329,0.10175519436597824,0.2515322268009186,0.014469844289124012,-0.17101824283599854,0.04422421008348465,-0.12461630254983902,-0.30275166034698486,0.5333916544914246,0.06643041968345642,0.03210492804646492,0.14926141500473022,0.07832732796669006,-0.03323187306523323,-0.5999218225479126,0.2416476309299469,0.06577883660793304,0.04328596591949463,0.11503281444311142,0.36899903416633606,0.35206255316734314,-0.14235715568065643,0.006640881299972534,-0.057446278631687164,0.015142158605158329,0.40110906958580017,-0.2773725092411041,-0.016524402424693108,0.047951750457286835,0.07167202234268188,-0.16193455457687378,0.03690257668495178,0.25288742780685425,0.04912714287638664,-0.10478393733501434,-0.08806739747524261,-0.026553120464086533,-0.19559796154499054,0.25458988547325134,-0.012580211274325848,-0.053777772933244705,-0.011997280642390251,0.0037699805106967688,-0.07730097323656082,0.9006826877593994,0.06179668381810188,-0.010403776541352272,-0.0886271744966507,0.03487853705883026,0.2578091621398926,-0.5855004191398621,0.3450313210487366,0.20694376528263092,-0.0310494527220726,0.018108515068888664,-0.07476796209812164,0.01882769726216793,0.008881991729140282,-0.33994725346565247,0.20755980908870697,0.36053040623664856,-0.4929382801055908,0.6407672166824341,-0.2640534043312073,0.0756574496626854,-0.09167305380105972,0.04820699989795685,0.12184742838144302,0.3310514986515045,0.01969449408352375,0.04393260180950165,-0.1141587421298027,0.04316375032067299,0.03724924847483635,0.4118242859840393,0.03441966697573662,0.06868327409029007,0.09885704517364502,-0.07200797647237778,-0.17254434525966644,0.46763256192207336,0.022165071219205856,0.1919018030166626,0.3366394639015198,0.3589342534542084,0.16031454503536224,-0.6433174014091492,0.04733126610517502,-0.09951262921094894,0.059655409306287766,0.004702924285084009,0.0761677622795105,-0.6042079329490662,-0.01885995641350746,-0.018688198179006577,-0.10474279522895813,0.06551960110664368,0.06703010946512222,-0.45956951379776,-0.005746491253376007,-0.17116868495941162,-0.3802606761455536,-0.2647726237773895,-0.14840957522392273,0.5841073393821716,0.03802467882633209,0.11474578827619553,0.04503820836544037,0.24843822419643402,0.016823846846818924,0.11341434717178345,-0.0008356450707651675,-0.06703321635723114,0.07848859578371048,0.024161599576473236,-0.0035771820694208145,0.22426073253154755,0.04011731222271919,-0.04321274161338806,0.14549490809440613,-0.025712233036756516,0.02366986870765686,-0.01782803423702717,0.05392251908779144,-0.08337150514125824,0.11046622693538666,-0.013644182123243809,-0.030412456020712852,0.0240046214312315,-0.07773593813180923,0.1039014607667923,-0.0958295688033104,0.01869843900203705,0.019236905500292778,0.21336813271045685,0.34422242641448975,-0.319243460893631,-0.39324823021888733,0.45094138383865356,0.02845090813934803,0.18155111372470856,-0.08651205152273178,-0.09162988513708115,-0.13530051708221436,-0.21564792096614838,-0.2328002154827118,-0.17725519835948944,0.03038480132818222,0.029090847820043564,0.06109246984124184,0.08712486177682877,0.11939854919910431,-0.13102349638938904,-0.026446377858519554,-0.02958752028644085,-0.05399693176150322,-0.01953728124499321,0.04659644141793251,-0.11446240544319153,-0.04875820502638817,0.025174353271722794,0.020836347714066505,-0.029433388262987137,0.050300247967243195,-0.8252735137939453,0.01601352170109749,-0.019899846985936165,-0.005916091613471508,-0.0032847232650965452,-0.02384144626557827,-0.47836223244667053,0.0007863653590902686,0.0023984701838344336,0.006836785469204187,0.014324359595775604,-0.09210367500782013,0.21945828199386597,-0.13566380739212036,-0.062271930277347565,-0.11334778368473053,-0.05663912370800972,-0.017131386324763298,0.06530536711215973,0.039832279086112976,0.03215375542640686,0.03897356241941452,0.03206166252493858,0.0035584254655987024,1.4705729484558105,-0.011424228549003601,-0.025117743760347366,0.017151866108179092,0.017307188361883163,-0.21890093386173248,0.0515814833343029,-0.17531372606754303,0.01111412514001131,-0.03920687735080719,0.018594130873680115,-0.07948680967092514,-0.02049228921532631,-0.0673556849360466,0.2809760570526123,0.10224907845258713,0.013941392302513123,0.13300032913684845,0.738284707069397,0.1895570307970047,-0.11416144669055939,0.20689840614795685,-0.23171168565750122,0.44691362977027893,-0.3195688724517822,-0.029310191050171852,-0.08314323425292969,-0.09653011709451675,-0.02628924697637558,-0.08270561695098877,0.10026688873767853,-0.1106480285525322,-0.006533506326377392,0.0026308593805879354,-0.02729727141559124,-0.01298778411000967,-0.23126523196697235,0.10406032204627991,0.24133187532424927,0.2280656099319458,0.20596632361412048,0.20327994227409363,-0.4009084105491638,-0.09376328438520432,0.06942594796419144,0.10149380564689636,0.13736122846603394,0.21569569408893585,0.154552161693573,0.09739657491445541,-0.0030859860125929117,-0.014125283807516098,-0.03227918967604637,-0.044040746986866,0.29667213559150696,-0.11798983812332153,-0.2397173047065735,-0.2310023307800293,-0.18406134843826294,-0.1826174557209015,0.44684505462646484,0.08694919943809509,-0.0046926033683121204,0.08368433266878128,0.08411066979169846,0.016939710825681686,0.3009517788887024,0.0032006201799958944,-0.0034124997910112143,-0.02417127601802349,-0.0254774559289217,0.01519435178488493,0.3979244530200958,-0.05322978273034096,-0.04194755479693413,-0.04148422181606293,-0.023416774347424507,-0.011169090867042542,-0.2717444896697998,-0.1014086976647377,0.08036136627197266,0.04182131960988045,0.029329003766179085,0.05583757534623146,-0.010043137706816196,-0.1464369297027588,0.11816923320293427,-0.012017850764095783,0.2589063048362732,-0.36908045411109924,-0.35938766598701477,0.8864460587501526,-0.8998825550079346,0.5013027191162109,-0.2257995903491974,0.22479185461997986,-0.0675414428114891,0.002782267052680254,-0.02724994532763958,-0.028992721810936928,-0.009121596813201904,-0.06178080290555954,0.021934743970632553,0.004436344839632511,0.026032842695713043,0.024056412279605865,0.02026424929499626,0.039798177778720856,-0.6955615878105164,0.07282725721597672,0.07608980685472488,0.12042932212352753,0.1753741204738617,0.18118061125278473,-0.5514914393424988,0.002555098617449403,0.03264492005109787,0.022012468427419662,0.07239676266908646,0.02546601928770542,-0.03030533716082573,-0.005074485670775175,-0.029462313279509544,-0.020047131925821304,-0.04671751707792282,-0.037415869534015656,0.7706627249717712,-0.07706645876169205,-0.08271367847919464,-0.13075940310955048,-0.18076038360595703,-0.2071143090724945,0.6055879592895508,0.1262311339378357,0.15871037542819977,0.07941827923059464,0.16202005743980408,0.08094903081655502,-0.04096280783414841,-0.06543227285146713,-0.09776101261377335,-0.05924657732248306,-0.09913641214370728,-0.04335082322359085,0.5658462643623352,-0.13599401712417603,-0.14157456159591675,-0.15046806633472443,-0.07188783586025238,-0.019120121374726295,0.03155511990189552,0.008467300795018673,-0.10507164150476456,-0.042841143906116486,0.0011448387522250414,-0.027831709012389183,0.009626454673707485,-0.014806988649070263,0.16584385931491852,0.28264516592025757,0.4486536681652069,-0.21796318888664246,-0.31900110840797424,0.6371962428092957,-0.6182395219802856,0.03133954480290413,0.09415530413389206,-0.045720357447862625,-0.042131002992391586,0.013573694042861462,0.02541964314877987,-0.010312024503946304,-0.03710467740893364,0.0678456723690033,0.2693142890930176,0.0008730444824323058,-0.01821768842637539,0.01374217588454485,0.04943842440843582,-0.03424232453107834,0.34124118089675903,0.021895170211791992,0.017238054424524307,0.010336000472307205,0.024893512949347496,0.04815058037638664,-0.3108771741390228,0.10412284731864929,0.0861377939581871,0.043321214616298676,0.05031534284353256,0.15001454949378967,0.14362213015556335,-0.1446680724620819,-0.11162202805280685,-0.08736463636159897,-0.09995895624160767,-0.21683676540851593,0.2091200202703476,-0.041534245014190674,-0.03841639682650566,-0.0582425557076931,-0.059896696358919144,-0.0833626240491867,0.33971959352493286,0.11230512708425522,0.0858721062541008,0.04409798979759216,0.0488688200712204,0.044027794152498245,0.31547704339027405,-0.2052435427904129,0.0705895647406578,0.07439230382442474,0.04826352745294571,0.04345687851309776,0.6741552352905273,-0.08405507355928421,0.03277426213026047,0.07917097955942154,0.03888147696852684,0.056686654686927795,-0.1785184144973755,0.07047679275274277,0.17008596658706665,0.09589804708957672,-0.0038658473640680313,0.007242206484079361,-0.00807781983166933,-7.194824032632412e-39,-1.1412282593443063e-38,1.2728917407150304e-38,7.008008526562485e-39,4.360937510572869e-39,-1.5801862644626732e-38,6.941910679298748e-39,-1.0416056074902157e-38,-1.9987167010874985e-38,-1.692427048468623e-38,-2.0657591636460372e-39,7.971978555753098e-39,-0.0007643940043635666,-7.517994527006522e-05,0.0011423630639910698,0.007460514549165964,0.013150885701179504,0.02047545649111271,0.030100321397185326,0.04066313058137894,0.05117891728878021,0.06268710643053055,0.06856943666934967,0.0640471950173378,0.050829995423555374,0.03251292183995247,0.014471824280917645,0.0029876301996409893,0.0006870522629469633,0.00227527623064816,-0.0012763356789946556,-0.004584061913192272,-0.012437601573765278,-0.019491909071803093,-0.02720937691628933,-0.03773266077041626,-0.0506129115819931,-0.06274384260177612,-0.07136803865432739,-0.07072915881872177,-0.05693242698907852,-0.036609478294849396,-0.017761869356036186,-0.00528827914968133,-0.0014733538264408708,0.0016680218977853656,-0.0017726797377690673,0.0025163134559988976,0.00026513051125220954,0.0019456174923107028,0.0020801671780645847,0.0008694561547599733,0.0009016917319968343,0.0036132519599050283,0.0037702464032918215,0.006309461314231157,0.01314935926347971,0.01287764497101307,0.0050207204185426235,-0.0003493284457363188,-0.01819089613854885,0.013119162060320377],"sample_rate":48000.0}},{"max_value":1.0,"model":{"version":"0.7.0","metadata":{"date":{"year":2026,"month":5,"day":17,"hour":9,"minute":1,"second":28},"loudness":-12.049386978149414,"gain":0.612789648007463},"architecture":"WaveNet","config":{"layers":[{"input_size":1,"condition_size":1,"head":{"out_channels":1,"kernel_size":16,"bias":true},"channels":8,"kernel_sizes":[6,6,6,6,6,6,6,6,6,6,6,6,6,6,15,15,6,6,6,6,6,6,6],"dilations":[1,3,7,17,41,101,239,1,3,7,17,41,101,239,1,13,1,3,7,17,41,101,239],"activation":[{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01},{"type":"LeakyReLU","negative_slope":0.01}],"bottleneck":8,"head1x1":{"active":false,"out_channels":1,"groups":1},"layer1x1":{"active":true,"groups":1},"groups_input":1,"groups_input_mixin":1,"conv_pre_film":{"active":false,"shift":true,"groups":1},"conv_post_film":{"active":false,"shift":true,"groups":1},"input_mixin_pre_film":{"active":false,"shift":true,"groups":1},"input_mixin_post_film":{"active":false,"shift":true,"groups":1},"activation_pre_film":{"active":false,"shift":true,"groups":1},"activation_post_film":{"active":false,"shift":true,"groups":1},"layer1x1_post_film":{"active":false,"shift":true,"groups":1},"head1x1_post_film":{"active":false,"shift":true,"groups":1},"gating_mode":["none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none","none"],"secondary_activation":[null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null],"slimmable":null}],"head":null,"head_scale":0.013119162069029721},"weights":[-2.708199977874756,1.518722653388977,2.0678908824920654,-1.4746893644332886,1.8213872909545898,1.583485722541809,0.43026724457740784,-2.332751750946045,0.010750990360975266,0.011915381997823715,0.011935512535274029,0.016323460265994072,0.01673297770321369,0.017248285934329033,-0.008671868592500687,-0.009857114404439926,-0.009906550869345665,-0.013710119761526585,-0.013825307600200176,-0.013919103890657425,-0.009980551898479462,-0.011135874316096306,-0.01124157290905714,-0.015299483202397823,-0.015604638494551182,-0.015871070325374603,0.008626227267086506,0.009722459129989147,0.009800530038774014,0.013623515143990517,0.013611781410872936,0.013682014308869839,-0.009396251291036606,-0.010530810803174973,-0.010661841370165348,-0.014683360233902931,-0.01494394987821579,-0.01524931937456131,-0.008933411911129951,-0.009977768175303936,-0.010045178234577179,-0.01389186829328537,-0.014010310173034668,-0.014149187132716179,-0.003665241878479719,-0.004495521076023579,-0.00490069342777133,-0.007277719676494598,-0.006902802269905806,-0.006552433595061302,0.010334426537156105,0.011462384834885597,0.01152486726641655,0.015768369659781456,0.016090799123048782,0.016407733783125877,-0.22126078605651855,-0.16176877915859222,-0.09408030658960342,-0.24135459959506989,-0.3601699769496918,-0.5376079678535461,0.10701704025268555,0.07671711593866348,0.07630858570337296,0.1591598093509674,0.24871820211410522,0.32435551285743713,0.14989745616912842,0.14074751734733582,0.1529988795518875,0.15210354328155518,0.350324809551239,0.4424012005329132,-0.12005482614040375,-0.10728505998849869,-0.08462131023406982,-0.17147760093212128,-0.2478979080915451,-0.30178287625312805,0.13311949372291565,0.0843011885881424,0.08390852808952332,0.1688326895236969,0.35139402747154236,0.4230392873287201,0.13811549544334412,0.09212638437747955,0.05540633574128151,0.16856394708156586,0.27142223715782166,0.3446059226989746,0.041794080287218094,0.022458622232079506,0.025374339893460274,0.06077304854989052,0.09303129464387894,0.11520745605230331,-0.14149031043052673,-0.10911538451910019,-0.09488236904144287,-0.1638813465833664,-0.36184218525886536,-0.4446091651916504,0.10268570482730865,0.05700470879673958,0.06608925014734268,0.08909949660301208,0.43510472774505615,0.6243542432785034,-0.06537137925624847,-0.023880504071712494,-0.035500142723321915,-0.07227897644042969,-0.2820356786251068,-0.4140956997871399,-0.07145122438669205,-0.01178249903023243,-0.03827093914151192,-0.09420035779476166,-0.380927175283432,-0.5251759886741638,0.06906753778457642,0.022641761228442192,0.032873645424842834,0.07498612254858017,0.2867022156715393,0.41280636191368103,-0.08002938330173492,-0.03263408690690994,-0.034250929951667786,-0.07832571119070053,-0.34592801332473755,-0.4952373802661896,-0.06393209099769592,-0.029876215383410454,-0.03724568709731102,-0.0830356553196907,-0.3046635687351227,-0.4529934525489807,-0.018433447927236557,-0.005087459925562143,-0.013559828512370586,-0.027506191283464432,-0.09682443737983704,-0.14162777364253998,0.10551492869853973,0.05306422337889671,0.046254027634859085,0.08746477961540222,0.3899362087249756,0.605506181716919,0.18329885601997375,0.1554454267024994,0.1004151776432991,0.15097621083259583,0.2953614890575409,0.25351423025131226,-0.12572002410888672,-0.08539800345897675,-0.058826662600040436,-0.09593193233013153,-0.14477548003196716,-0.14708206057548523,-0.19528046250343323,-0.13524703681468964,-0.15226945281028748,-0.17342600226402283,-0.24479831755161285,-0.15947869420051575,0.12808138132095337,0.10757962614297867,0.06675813347101212,0.1200144812464714,0.14151734113693237,0.17858661711215973,-0.1383955329656601,-0.1436891108751297,-0.13082921504974365,-0.13333868980407715,-0.154122456908226,-0.14799895882606506,-0.16486220061779022,-0.07851517200469971,-0.09308408200740814,-0.13599009811878204,-0.1624670922756195,-0.17618121206760406,-0.04448150098323822,-0.02641434594988823,-0.02333538420498371,-0.03613685443997383,-0.05507902055978775,-0.06228704750537872,0.20076912641525269,0.16247287392616272,0.1400408148765564,0.12912917137145996,0.19108611345291138,0.15839475393295288,-0.005738380365073681,-0.005796929821372032,-0.0065437196753919125,-0.011671068146824837,-0.018894534558057785,-0.03450039401650429,0.002968445885926485,0.0024787806905806065,0.003163605695590377,0.007610717322677374,0.014055093750357628,0.025926558300852776,0.004077491350471973,0.0035650143399834633,0.004303657915443182,0.009375077672302723,0.016614850610494614,0.030676115304231644,-0.0029278157744556665,-0.0024051149375736713,-0.003095983760431409,-0.0074216024950146675,-0.013718685135245323,-0.025535786524415016,0.003937237896025181,0.0035400015767663717,0.004202478565275669,0.008799494244158268,0.01551251020282507,0.02878757193684578,0.0032072158064693213,0.0027412709314376116,0.003467801259830594,0.007937275804579258,0.014387133531272411,0.026693154126405716,0.0004102429957129061,-0.00017850598669610918,0.0002553259255364537,0.0019968091510236263,0.005252325441688299,0.009958280250430107,-0.0052640000358223915,-0.004946265835314989,-0.00552966445684433,-0.01036661583930254,-0.017266152426600456,-0.0319945402443409,-0.3002333641052246,-0.3285265862941742,-0.30392152070999146,0.018921317532658577,0.7487377524375916,1.3850520849227905,0.20561358332633972,0.21890433132648468,0.1611560583114624,-0.005037939641624689,-0.44345253705978394,-0.8231340646743774,0.20645084977149963,0.33162033557891846,0.21083973348140717,0.03017219714820385,-0.6196483969688416,-1.1663920879364014,-0.1683957725763321,-0.24229121208190918,-0.16351088881492615,0.030219541862607002,0.47303763031959534,0.8594751358032227,0.20772896707057953,0.26761293411254883,0.16133293509483337,-0.0037443446926772594,-0.5594484806060791,-1.0485031604766846,0.21391627192497253,0.2463735193014145,0.16063621640205383,0.010698996484279633,-0.49031826853752136,-0.8967832922935486,0.058822423219680786,0.07960306107997894,0.04207496717572212,-0.010659916326403618,-0.164829820394516,-0.2901567220687866,-0.25468313694000244,-0.3543860912322998,-0.2399986833333969,0.03583882376551628,0.7005712985992432,1.2310742139816284,0.15079627931118011,0.19553515315055847,0.20545202493667603,-0.004535813815891743,-0.42152073979377747,-0.8715760707855225,-0.12315821647644043,-0.15299226343631744,-0.10295090079307556,0.0007206678856164217,0.28835031390190125,0.5656916499137878,-0.13561491668224335,-0.23435251414775848,-0.1383228898048401,-0.0016416548751294613,0.43607795238494873,0.7610684633255005,0.14337950944900513,0.17180562019348145,0.11756827682256699,-0.013632307760417461,-0.32243847846984863,-0.6353448033332825,-0.16035152971744537,-0.1768438071012497,-0.12162724137306213,-0.002356340177357197,0.34809258580207825,0.688629686832428,-0.14408911764621735,-0.16684317588806152,-0.09970123320817947,0.010389352217316628,0.3391743004322052,0.6389474272727966,-0.04807505011558533,-0.07534753531217575,-0.03267041593790054,0.009829462505877018,0.13398373126983643,0.25002187490463257,0.16355301439762115,0.2359524816274643,0.11024220287799835,-0.008425520732998848,-0.40126949548721313,-0.8084657788276672,-0.33923885226249695,-0.2034686803817749,-0.3384210765361786,-0.15468959510326385,0.25549525022506714,0.6251497268676758,0.19193491339683533,0.16749005019664764,0.15252962708473206,0.075452521443367,-0.21181200444698334,-0.3912357985973358,0.2553708255290985,0.205717995762825,0.28056079149246216,0.05236024409532547,-0.3288089632987976,-0.5270744562149048,-0.1702525019645691,-0.24794121086597443,-0.12256213277578354,-0.0860290378332138,0.19056956470012665,0.5152596831321716,0.23062066733837128,0.2999825179576874,0.2538178265094757,0.11813925206661224,-0.30323535203933716,-0.5561526417732239,0.15190763771533966,0.26818954944610596,0.17765510082244873,0.10672548413276672,-0.23406702280044556,-0.44726863503456116,0.10495960712432861,0.08403388410806656,0.07011962682008743,0.015321205370128155,-0.10330726951360703,-0.20474499464035034,-0.2342272400856018,-0.29432612657546997,-0.2178887277841568,0.008299601264297962,0.15945352613925934,0.6278958320617676,-0.0059206229634583,-0.15397733449935913,0.13532091677188873,0.11278944462537766,-0.018226174637675285,0.21533656120300293,-0.14123910665512085,0.5734551548957825,-0.01121464092284441,0.08397475630044937,-0.39534780383110046,-0.0940430760383606,0.020442143082618713,-0.8062251210212708,0.5968456268310547,-0.21952183544635773,-0.02625054307281971,-0.14417868852615356,0.2245415598154068,-0.15881232917308807,-0.031093716621398926,0.4970831274986267,-0.6442520022392273,0.5532757043838501,0.04994813725352287,0.41248130798339844,-0.6049526333808899,-0.253282368183136,0.06892772018909454,-0.9229233264923096,1.2375531196594238,-0.011512749828398228,0.031224649399518967,0.6906029582023621,-0.1676395684480667,-0.4021105468273163,0.0452188216149807,-0.3265276253223419,0.6795313358306885,-0.12018236517906189,-0.0010329406941309571,-0.372653067111969,0.21250146627426147,-0.059256456792354584,-0.01855330914258957,-0.07623443752527237,-0.20145247876644135,-0.2121296375989914,0.019872885197401047,0.006072149146348238,-0.23188620805740356,-0.10077086836099625,-0.004281274043023586,-0.17462392151355743,0.3456522524356842,-0.31364569067955017,0.016733473166823387,0.05721651762723923,-0.5640072822570801,-0.31921935081481934,0.015499633736908436,-0.8537059426307678,0.8418593406677246,-0.44851213693618774,-0.00355526152998209,0.4454911947250366,-0.0926581472158432,-0.3453078866004944,-0.015604039654135704,0.18643637001514435,-0.42529189586639404,0.4700900912284851,-0.0383007787168026,-0.3283631503582001,0.16173234581947327,0.09336846321821213,-0.03704991564154625,0.6543859839439392,-1.0920120477676392,-0.007253760006278753,-0.388923317193985,0.005992469377815723,0.28683552145957947,-0.27566227316856384,-0.28491199016571045,-0.2968733608722687,-0.24099107086658478,-0.48536819219589233,0.01671735756099224,0.06176357716321945,0.09257186949253082,-0.011216072365641594,0.1961139291524887,0.001531283720396459,0.07558295875787735,0.03767234459519386,0.01839909330010414,0.08602690696716309,-0.11614809185266495,-0.007759366650134325,0.03361544758081436,-0.04119519516825676,-0.04850197583436966,0.030912654474377632,-0.05480314418673515,0.059304408729076385,-0.012375143356621265,0.024884218350052834,0.02014109678566456,-0.0023581727873533964,-0.012789477594196796,-0.06560037285089493,0.004156098701059818,8.901669207261875e-05,0.0037612570449709892,0.03574274107813835,-0.045320410281419754,-0.0026014980394393206,0.11762697994709015,0.05922267958521843,0.02990509569644928,0.09805292636156082,-0.12659592926502228,-0.02666502073407173,0.0507286936044693,0.03397751972079277,0.030519386753439903,-0.023266779258847237,0.061572249978780746,0.004513496533036232,0.007689472287893295,0.05743740126490593,0.05200942978262901,-0.041574690490961075,0.10635819286108017,-0.05103112384676933,-0.09397557377815247,-0.12907321751117706,0.030066102743148804,0.05334176495671272,0.02363498881459236,-0.35540035367012024,-0.15253587067127228,-0.17835062742233276,-0.06366938352584839,-0.172315314412117,0.10495588928461075,0.4659712314605713,-0.12265897542238235,-0.13999652862548828,-0.018610069528222084,-0.01602994091808796,0.02821856178343296,0.256560355424881,-0.06123470515012741,-0.1071091741323471,-0.07033681124448776,0.0035160344559699297,0.04291835427284241,-0.22148467600345612,-0.10591140389442444,-0.10074524581432343,-0.041027747094631195,-0.2145809680223465,-0.19610756635665894,-0.013137800619006157,-0.06869187951087952,-0.13841612637043,-0.0540473647415638,-0.08517111092805862,0.0037892209365963936,0.1491401344537735,-0.43919864296913147,-0.11606945842504501,-0.09243464469909668,-0.15260161459445953,-0.13331317901611328,0.09969991445541382,0.028857208788394928,-0.02287720888853073,0.19746434688568115,0.08939436078071594,-0.05910190939903259,-0.42931002378463745,0.00859649945050478,0.06071055307984352,0.07014016807079315,0.008189585991203785,-0.07373469322919846,0.07922210544347763,-0.0160506758838892,-0.018148327246308327,-0.04000607505440712,0.05046712979674339,0.1170235201716423,-0.023961583152413368,-0.06878514587879181,-0.03708510100841522,-0.040572892874479294,0.0025610162410885096,0.06170102208852768,-0.02586347796022892,0.03269012272357941,-0.010249271988868713,0.005453358869999647,-0.0168931744992733,-0.012269830331206322,0.004465654958039522,0.012299200519919395,0.010183705016970634,-0.006961156148463488,0.00929503794759512,0.09150023758411407,0.005840563215315342,5.037813025410287e-05,-0.029195189476013184,-0.05711640417575836,4.8145593609660864e-05,0.148042231798172,-0.027741974219679832,-0.10108014196157455,0.004228118807077408,0.0046540736220777035,0.01769271120429039,-0.030020445585250854,0.05478322505950928,-0.0017457257490605116,0.004530992358922958,0.03564131632447243,-0.04752432554960251,-0.07964825630187988,0.053937751799821854,0.09443334490060806,-0.18829351663589478,-0.027323441579937935,-0.132404625415802,-0.03124210052192211,0.21216963231563568,0.1304091364145279,0.05783667787909508,-0.006009179167449474,0.06505470722913742,-0.016018208116292953,-0.3236023783683777,0.165941521525383,0.2163054496049881,0.20955879986286163,0.2248595952987671,0.08468297123908997,-0.2189478576183319,-0.20647811889648438,-0.13165590167045593,-0.17779521644115448,-0.1263318657875061,-0.01411525346338749,0.06016666442155838,0.009004662744700909,-0.066159687936306,0.03770207613706589,-0.09519249200820923,-0.11270695179700851,-0.1209762915968895,-0.09747524559497833,-0.0700671449303627,-0.017003551125526428,-0.08700615912675858,-0.002408405067399144,-0.3559064567089081,0.38206374645233154,0.12714852392673492,0.018895160406827927,0.10780483484268188,0.040944699198007584,-0.0642591193318367,-0.2157774120569229,-0.3020498752593994,-0.17793676257133484,-0.18784238398075104,-0.10400306433439255,0.4231954514980316,-0.000294152123387903,0.0006300618988461792,0.0004305029578972608,0.00026825579698197544,0.00016243947902694345,0.00010705285239964724,0.0006054156110621989,-0.0007268072222359478,-0.0006798469694331288,-0.0004699719720520079,-2.9640172215295024e-05,0.0002957381075248122,-0.0001498271303717047,-0.001181987696327269,-0.0013591874158009887,-0.0013809697702527046,-0.0010950122959911823,-0.0013865239452570677,0.0008473169291391969,0.0009849700145423412,0.0010898568434640765,0.0011160590220242739,0.0009460237342864275,0.0014293172862380743,0.0017643470782786608,0.0012443733867257833,0.0013475969899445772,0.0013693050714209676,0.0014409755822271109,0.0015834799269214272,0.00391932250931859,0.002634766511619091,0.002811016049236059,0.0031395009718835354,0.0036402472760528326,0.004343329928815365,-0.00032970382017083466,-5.795828838017769e-05,-0.0002086995664285496,-0.00037291907938197255,-0.00033627758966758847,-0.00018070489750243723,0.0006323399138636887,0.001864540739916265,0.0018266147235408425,0.0016301210271194577,0.0011358082992956042,0.0013438729802146554,-0.0051566907204687595,0.1404748558998108,0.15056847035884857,0.08275532722473145,0.022559957578778267,-0.37452664971351624,-0.1216825470328331,-0.18418146669864655,-0.08110414445400238,-0.10602259635925293,-0.14103639125823975,0.3238690495491028,-0.1052369624376297,0.0025019992608577013,0.00028890592511743307,0.020329535007476807,0.0718579888343811,0.284004271030426,0.012461197562515736,-0.08441285043954849,-0.038891952484846115,-0.0326850600540638,-0.1566188484430313,-0.2258622944355011,-0.09681341797113419,-0.07990109920501709,-0.1940959244966507,-0.11121398955583572,-0.08915156126022339,-0.008983917534351349,-0.23740597069263458,-0.10993089526891708,-0.3335903584957123,-0.19223156571388245,-0.22858445346355438,0.29265958070755005,-0.28826501965522766,0.02889983169734478,0.03882423788309097,-0.1409720480442047,0.12265006452798843,0.08071926981210709,-0.009660765528678894,0.009400520473718643,0.018417147919535637,-0.038190703839063644,0.11727260053157806,-0.342854768037796,-0.040822505950927734,0.0051960633136332035,0.033160220831632614,0.11073539406061172,0.15754173696041107,0.10382233560085297,-0.061702385544776917,0.04692671075463295,0.03159019723534584,0.04880473390221596,0.016711505129933357,-0.19149722158908844,0.08163660019636154,-0.026529759168624878,0.07010263204574585,0.09888721257448196,0.08485063910484314,-0.1157124936580658,-0.018462464213371277,-0.0673307478427887,-0.06015852838754654,-0.017700571566820145,-0.13750289380550385,-0.13570156693458557,0.045005448162555695,-0.07286074012517929,-0.055368874222040176,-0.04356183856725693,-0.05879174545407295,0.029787477105855942,-0.034241534769535065,-0.0002158792340196669,-0.048067159950733185,-0.037060368806123734,0.0028395934496074915,0.09157146513462067,0.029073376208543777,0.0918935239315033,0.21850304305553436,0.2570393979549408,0.27203553915023804,0.10973580181598663,0.04454120621085167,-0.019653426483273506,-0.021947238594293594,-0.010664386674761772,-0.009948051534593105,0.0987626239657402,-0.04307130351662636,-0.016418153420090675,0.006903953850269318,-0.014562714844942093,0.04002838581800461,0.07020139694213867,0.1258683204650879,0.07475505769252777,0.014304486103355885,0.028294727206230164,-0.0356714203953743,-0.09286336600780487,0.040604010224342346,0.02429606392979622,-0.01973036862909794,-0.010704790242016315,-0.047494206577539444,-0.07974839955568314,0.001500979415141046,0.0015129580860957503,0.03440569341182709,0.03365175426006317,0.04183359816670418,0.05164439231157303,0.08066762238740921,0.07123157382011414,0.05618549883365631,0.06580105423927307,0.042567722499370575,0.018636375665664673,0.12155824899673462,0.08731400221586227,0.05436716601252556,0.06718118488788605,0.01249144971370697,-0.03788569197058678,-0.03228972852230072,0.0073615750297904015,-0.016078969463706017,-0.029832862317562103,0.009290582500398159,0.020811470225453377,-0.09044414013624191,-0.03378421068191528,0.021715516224503517,0.003938682842999697,0.06444857269525528,0.11419308185577393,0.0017894390039145947,0.04462771862745285,-0.07245565205812454,0.08262426406145096,-0.0039152768440544605,0.19842135906219482,-0.030124995857477188,-0.1267433613538742,0.004348946735262871,0.3351999819278717,-0.005623926408588886,-0.3673456311225891,-6.410803325707093e-05,0.6640926599502563,-0.12481506913900375,-0.00444164639338851,-0.015928607434034348,-0.3064620792865753,-0.03344106301665306,0.2894522547721863,-0.0005709708202630281,-0.16813097894191742,0.24603839218616486,0.01061626709997654,0.09117615967988968,0.18128521740436554,0.11894391477108002,-0.09743382036685944,9.483896428719163e-05,0.24262049794197083,-0.27619364857673645,0.15535810589790344,-0.1869833916425705,0.3025248646736145,0.23178866505622864,-0.17950738966464996,-6.385172309819609e-05,-0.21588344871997833,-0.2814822793006897,0.12262426316738129,0.16349993646144867,0.076729416847229,0.0767659991979599,-0.3567904233932495,0.0009674248867668211,0.25308048725128174,0.06851716339588165,0.03089563176035881,0.14967910945415497,0.3211625814437866,-0.0005753624718636274,-0.27974995970726013,0.000887626432813704,0.28154709935188293,0.16805794835090637,-0.05646105483174324,0.2477494478225708,0.1891796588897705,0.0640111044049263,-0.221125066280365,0.00028423918411135674,0.2283799946308136,0.00725936284288764,0.019042009487748146,0.011297913268208504,-0.28863972425460815,0.10029296576976776,-0.007310154847800732,-0.0016831224784255028,-0.21823635697364807,0.1654759645462036,0.1089717298746109,-0.04176551103591919,0.022058235481381416,0.14484049379825592,0.2508445978164673,-0.0002037770755123347,-0.17966775596141815,-0.08358731120824814,-0.09356968849897385,-0.1700841635465622,0.01663350872695446,-0.21114733815193176,0.15972939133644104,-0.2165370136499405,-0.2434169203042984,-0.013812948018312454,-0.42748066782951355,0.0523671954870224,0.054377734661102295,0.037146031856536865,0.04002833366394043,0.0619649700820446,0.0385211743414402,-0.059768181294202805,-0.0583934411406517,-0.07037615031003952,-0.04489662125706673,-0.020285971462726593,0.275398313999176,0.03363176807761192,0.04261424019932747,-0.027923181653022766,-0.04611523449420929,-0.10699914395809174,-0.006728603038936853,0.024269118905067444,0.03853019326925278,0.010770815424621105,0.022118285298347473,0.032804373651742935,0.042554453015327454,0.09163503348827362,0.12531845271587372,0.06372274458408356,0.09367607533931732,0.1207931712269783,0.20004071295261383,0.040795378386974335,0.007941878400743008,0.016518469899892807,0.05376218631863594,0.062134694308042526,0.19554851949214935,0.019901540130376816,0.05834946036338806,0.008867520838975906,0.07958585023880005,-0.028471941128373146,0.012695408426225185,0.0843578577041626,0.09347549825906754,0.04109833762049675,0.012055070139467716,0.09953458607196808,0.09594827890396118,-0.07583725452423096,-0.018474778160452843,-0.007468302734196186,0.043171100318431854,0.06660676747560501,-0.41212624311447144,0.004809670615941286,-0.036360811442136765,-0.12711472809314728,0.0515279583632946,-0.12755164504051208,0.35075557231903076,-0.17895786464214325,-0.2336111068725586,-0.2110031694173813,-0.06676923483610153,-0.18691693246364594,0.2062385231256485,-0.07587884366512299,-0.030171187594532967,0.1495290845632553,0.08950269222259521,0.07708059251308441,-0.32924672961235046,-0.013658973388373852,-0.00931402388960123,-0.07580022513866425,-0.011942746117711067,0.05081188678741455,-0.06038709729909897,-0.0992460772395134,-0.07602903991937637,-0.12525172531604767,-0.13784553110599518,-0.06249382346868515,-0.07308287173509598,-0.12619973719120026,-0.1756342500448227,-0.270840585231781,-0.10027841478586197,-0.16374492645263672,0.008689376525580883,-0.1550237238407135,0.00026779400650411844,-0.07149972766637802,-0.09154235571622849,-0.16285130381584167,-0.06862877309322357,0.008635654114186764,0.010708088055253029,0.007718465756624937,0.015633409842848778,0.014170495793223381,-0.03412938490509987,0.0020433771423995495,-0.004499347880482674,0.034249626100063324,0.005255833268165588,0.007279494311660528,0.04123767465353012,0.01949581876397133,0.011652637273073196,0.021411290392279625,0.03580055013298988,0.02590886317193508,0.044920921325683594,-0.001118327840231359,0.014883507043123245,-0.015351954847574234,-0.007687628734856844,-0.004417939577251673,0.026402730494737625,0.020515503361821175,0.021560657769441605,0.04171721637248993,0.010585580952465534,0.02511487901210785,0.05368751287460327,0.037464965134859085,0.025885749608278275,0.04827757179737091,0.023892100900411606,0.020927701145410538,0.0752844288945198,0.010931667871773243,0.011449955403804779,-0.009349029511213303,0.02813384309411049,-0.003447741735726595,-0.010289013385772705,0.02542746439576149,0.03376775607466698,0.022295450791716576,0.0363890677690506,0.0480746291577816,-0.01896338164806366,0.034698788076639175,0.03850173577666283,0.02759825438261032,0.04962453246116638,-0.036568522453308105,0.08439334481954575,-0.0054175350815057755,-0.011165662668645382,-0.02090415358543396,0.011934877373278141,0.12687313556671143,0.05467475950717926,0.005408817902207375,-0.02448509819805622,-0.009562828578054905,-0.0095833083614707,0.08377265930175781,-0.06994747370481491,-0.027418918907642365,-0.010717041790485382,-0.004543830640614033,-0.0390525683760643,-0.04496122524142265,-0.002549197990447283,0.07022318243980408,0.06836364418268204,0.06759583950042725,0.08399451524019241,0.11191296577453613,0.08115581423044205,0.030886180698871613,0.03399934619665146,0.03396384045481682,-0.020664513111114502,0.11197930574417114,0.10990709066390991,-0.00099488056730479,-0.010907778516411781,-0.0175660140812397,-0.017512472346425056,-0.04087314382195473,-0.0034894465934485197,0.07816433161497116,0.06898213922977448,0.04981285706162453,0.05690738558769226,-0.024673186242580414,0.11953626573085785,0.00019903089560102671,-0.03217708319425583,0.025918971747159958,-0.08547508716583252,-0.006667228415608406,0.21781562268733978,-0.0030725242104381323,-0.0117790000513196,0.18505366146564484,0.013383954763412476,0.11626381427049637,-0.23511354625225067,0.013035294599831104,0.005897060967981815,0.0731019452214241,0.19465282559394836,0.121580570936203,-0.38036781549453735,0.012776757590472698,-0.1575668901205063,-0.10493676364421844,-0.09088645875453949,-0.10148728638887405,0.20445843040943146,-0.013937246054410934,-0.15578800439834595,-0.14129751920700073,-0.13613605499267578,-0.1961241513490677,0.05199126899242401,-0.09273052960634232,0.04113195091485977,0.0033197004813700914,-0.030299870297312737,0.044239409267902374,-0.15872301161289215,-0.12929369509220123,-0.10818168520927429,0.06027272716164589,-0.06214713305234909,0.016247570514678955,0.02781098708510399,-0.07098721712827682,-0.12982510030269623,-0.09571253508329391,-0.05650828778743744,-0.1777983009815216,0.03617636859416962,-0.0021940087899565697,-0.024523328989744186,0.046891018748283386,0.084752656519413,0.011830458417534828,-0.0013555999612435699,-0.010674274526536465,0.039334241300821304,0.04291641339659691,-0.031721558421850204,0.01590653508901596,-0.04558776319026947,0.04379231110215187,0.07732492685317993,0.04175667464733124,-0.06489399820566177,0.01977561041712761,0.0030831308104097843,0.011249246075749397,-0.027654001489281654,-0.05264374241232872,-0.04334403946995735,0.023776326328516006,0.09435819834470749,-0.02771688438951969,0.017914822325110435,0.02141532674431801,-0.03364897519350052,0.007538455072790384,0.024606481194496155,0.03024943359196186,0.05941232293844223,0.02387557551264763,0.016014374792575836,0.07532386481761932,0.036887120455503464,0.06899737566709518,0.036351051181554794,0.03441528230905533,0.00853211060166359,0.03595176711678505,0.020660683512687683,-0.007352238986641169,-0.0035285388585180044,0.052605364471673965,0.07819133251905441,0.034610141068696976,0.04194194823503494,0.028783636167645454,0.03529110550880432,0.011622321791946888,-0.045510195195674896,0.031188923865556717,0.012137295678257942,0.021600909531116486,0.0015005908207967877,0.032728660851716995,0.0325440876185894,0.010755767114460468,0.07453563064336777,-0.005346877500414848,-0.037023477256298065,-0.005629666615277529,0.04663420096039772,-0.03396358713507652,0.03839906305074692,-0.00023350465926341712,0.0201772041618824,0.014924861490726471,0.04754600301384926,-0.010788747109472752,0.012022429145872593,0.020939400419592857,0.024836692959070206,0.04655245691537857,0.0813441053032875,0.04568389058113098,0.07182585448026657,0.03617653623223305,0.034970302134752274,0.06671155244112015,0.059511058032512665,0.014645902439951897,0.06823325902223587,0.018221572041511536,-0.0048096696846187115,-0.014992598444223404,0.003104773350059986,-0.06296943128108978,0.011763323098421097,0.0211641825735569,0.0408158116042614,0.05370762199163437,-0.039014872163534164,0.05117574706673622,0.02803357131779194,-0.026971960440278053,0.026483049616217613,-0.05663062632083893,-0.1598261296749115,-0.23553557693958282,0.24874956905841827,0.11702996492385864,0.1648595929145813,0.23406477272510529,0.10364668816328049,0.1197642907500267,-0.38680827617645264,-0.10108324885368347,0.026386912912130356,0.16826708614826202,0.01184565108269453,0.11161427944898605,-0.4136316776275635,-0.03168490529060364,-0.08300041407346725,-0.20110705494880676,-0.24642351269721985,-0.20690152049064636,0.28356191515922546,0.008458222262561321,0.058623794466257095,-0.01688445918262005,-0.11053680628538132,-0.14598609507083893,0.06398764997720718,-0.16113512217998505,-0.13770325481891632,-0.11491525918245316,0.022440822795033455,-0.0694180503487587,-0.15644140541553497,-0.0572284460067749,-0.0031222286634147167,0.11310567706823349,-0.08115889877080917,-0.06035425886511803,-0.12927353382110596,-0.11887788772583008,-0.042523059993982315,-0.08742079883813858,-0.09448093920946121,-0.21814271807670593,0.06663951277732849,-0.1745867282152176,0.13855895400047302,-0.04950752109289169,-0.10279973596334457,0.16483499109745026,-0.017586560919880867,-0.03894122689962387,0.014245161786675453,-0.017486289143562317,0.3068403899669647,0.0028079391922801733,-0.0009640166535973549,-0.2948325276374817,-0.006986032240092754,-0.0029079276137053967,-0.20947615802288055,0.23589956760406494,-0.15965814888477325,-0.09113407880067825,-0.07632257044315338,0.08539314568042755,0.17495885491371155,-0.03405684605240822,-0.026288378983736038,-0.4687057137489319,-0.02838587760925293,-0.035240620374679565,-0.042636580765247345,0.04977845028042793,0.0186173003166914,-0.011378702707588673,-0.06569356471300125,0.0495651513338089,0.24896171689033508,-0.026817817240953445,0.28438088297843933,-0.05251812934875488,-0.10766613483428955,0.060513678938150406,-0.08213762193918228,-0.010235300287604332,0.1128019392490387,0.03766458109021187,-0.03605806082487106,-0.08645742386579514,-0.09445487707853317,0.058590780943632126,-0.17179390788078308,-0.28182822465896606,0.08183522522449493,-0.0497114323079586,-0.0868896022439003,-0.2655007541179657,-0.09879906475543976,0.10172862559556961,-0.059158805757761,0.033619362860918045,0.05481790378689766,-0.00027265032986178994,-0.05725462734699249,-0.17941810190677643,-0.08368373662233353,0.09153258800506592,-0.1175018772482872,0.05921661853790283,-0.14738360047340393,-0.08480354398488998,0.17752119898796082,-0.03588859736919403,0.0018017677357420325,0.06418490409851074,0.397685170173645,0.4285106658935547,-0.21650998294353485,0.09549561142921448,0.010109075345098972,0.3622681200504303,0.013464853167533875,0.11120018362998962,0.2618601620197296,-0.0542166493833065,-0.22457009553909302,0.10143184661865234,-0.03974224627017975,0.07576480507850647,0.2814709544181824,0.07409278303384781,0.054549891501665115,0.011962908320128918,0.01168088149279356,0.01116303913295269,0.012452411465346813,0.009372062981128693,0.0123392753303051,0.002154232934117317,0.001237757271155715,0.0010000787442550063,5.434970807982609e-05,0.0014069706667214632,0.0016423302004113793,-0.0063824765384197235,-0.006594018545001745,-0.00501681724563241,-0.005036284681409597,-0.007364107761532068,-0.003581519238650799,0.005765901878476143,0.00632316991686821,0.005200889892876148,0.00909434910863638,0.009770887903869152,0.005606087855994701,0.0033716419711709023,0.0018058273708447814,0.0005642978940159082,-0.0002425648708594963,0.00030371505999937654,-0.00045247009256854653,0.013209923170506954,0.011907127685844898,0.012416686862707138,0.012295459397137165,0.012457812204957008,0.013338334858417511,0.0035846815444529057,0.006266904063522816,0.008037656545639038,0.008724838495254517,0.008109892718493938,0.011082351207733154,-0.0038092853501439095,-0.004773668013513088,-0.004001562017947435,-0.003936573397368193,-0.002829217351973057,-0.00411025108769536,0.12909850478172302,0.08692913502454758,0.08730274438858032,0.10930146276950836,0.12089847773313522,0.2192147821187973,-0.09466466307640076,-0.018489588052034378,-0.0156226372346282,0.011110604740679264,0.048806313425302505,-0.04509643465280533,0.027094244956970215,-0.003216381184756756,0.02726573869585991,0.02597147226333618,0.041957881301641464,-0.08336161822080612,0.002706762170419097,-0.008092505857348442,0.005752407014369965,-0.03555629029870033,-0.08140803128480911,-0.0929933488368988,0.10655923187732697,0.08618416637182236,0.09524797648191452,0.06543367356061935,0.0761793926358223,-0.012619884684681892,0.055261217057704926,0.022620683535933495,0.021847182884812355,0.08967307209968567,0.10562926530838013,-0.15576371550559998,0.004450745414942503,-0.01353090163320303,0.029185261577367783,0.0413433276116848,0.1037459522485733,0.18652763962745667,-0.03658280521631241,-0.00286339083686471,9.53976486925967e-05,-0.015642287209630013,-0.03031601570546627,0.24547043442726135,-0.01887286640703678,-0.013113261200487614,-0.08529169857501984,-0.09342736750841141,-0.14357900619506836,-0.008059951476752758,-0.018121175467967987,-0.03229586035013199,-0.03122078813612461,-0.011007159948348999,0.1460372507572174,-0.23556064069271088,0.02236752212047577,-0.004565653391182423,0.03633173927664757,0.05974516645073891,0.1444656252861023,-0.19998832046985626,0.020405594259500504,0.03399614989757538,-0.008417044766247272,-0.1095084622502327,-0.26380112767219543,0.14081238210201263,-0.04842190444469452,-0.020641524344682693,0.020943380892276764,-0.057767558842897415,-0.1613515466451645,0.008876554667949677,-0.05611741542816162,-0.027223894372582436,-0.0643998309969902,-0.015420571900904179,-0.013155593536794186,-0.14625225961208344,-0.09538555890321732,-0.07082431018352509,-0.03909013420343399,0.014154018834233284,0.18224096298217773,-0.20614637434482574,-0.018617570400238037,0.011396427638828754,0.050688933581113815,-0.028880475088953972,-0.1924402415752411,-0.16665758192539215,-0.025672949850559235,-0.003201801562681794,-0.06319621950387955,-0.05989924818277359,-0.060889605432748795,-0.12633554637432098,0.01588679850101471,0.0054619391448795795,-0.003554966766387224,-0.0606420673429966,-0.1831756830215454,0.13330310583114624,-0.0040900022722780704,0.02658306248486042,0.0013190784957259893,-0.0032488792203366756,-0.10866834223270416,-0.03231745958328247,-0.05992860347032547,-0.026471000164747238,-0.06053772196173668,-0.0033075350802391768,0.13105496764183044,-0.18857936561107635,0.011347758583724499,-0.005493626929819584,-0.0021895102690905333,0.016965294256806374,0.029894491657614708,-0.09091351926326752,-0.043138109147548676,-0.004523246083408594,-0.06455691903829575,-0.06085521727800369,-0.16389049589633942,0.009557751007378101,-0.04652474448084831,-0.004110026638954878,-0.046538058668375015,-0.054851751774549484,-0.19124644994735718,0.07874557375907898,0.0017829649150371552,0.0060291048139333725,0.007322156801819801,0.007697688415646553,-0.042638830840587616,0.02902526594698429,-0.03606262430548668,0.03431568294763565,-0.015885721892118454,-0.033159349113702774,-0.03614380955696106,0.05400572344660759,-0.0009507177164778113,0.001412928570061922,-0.017898032441735268,-0.013301283121109009,0.1010018140077591,-0.15506507456302643,-0.006195329129695892,0.00212924275547266,-0.0029798487666994333,-0.005293677095323801,0.14727917313575745,-0.09143264591693878,-0.011227048933506012,0.003804838052019477,0.004613082855939865,-0.04922519251704216,-0.1665089726448059,0.1963547319173813,-0.043420277535915375,-0.03343382477760315,0.02465178444981575,0.006145465187728405,-0.06085861101746559,0.028548533096909523,-0.03394890949130058,0.0527004636824131,-0.03333134576678276,-0.03135065361857414,0.02701612561941147,0.07040238380432129,-0.04464814066886902,-0.011746708303689957,0.0039555663242936134,-0.00752687593922019,0.11276738345623016,-0.1380392163991928,-0.023153379559516907,0.0020129806362092495,-0.008358621038496494,-0.06511514633893967,-0.04391409829258919,-0.036915749311447144,0.015984540805220604,0.02215067483484745,0.009586573578417301,0.03776748478412628,0.004767243284732103,-0.022249698638916016,0.015427946113049984,-0.007760830223560333,0.010698826052248478,0.00027191516710445285,-0.023059485480189323,0.05828193575143814,0.00846070796251297,-0.007240499369800091,0.014202374964952469,0.0034337742254137993,-0.023986533284187317,-0.023356758058071136,-0.015991516411304474,0.016185138374567032,0.002332740928977728,0.0010137205244973302,0.0406915508210659,0.055922314524650574,0.007992754690349102,0.020940791815519333,0.027756597846746445,0.03479727357625961,0.04592948406934738,0.05810113623738289,0.028231794014573097,0.022439058870077133,-0.0033925457391887903,-0.016415003687143326,-0.008130514062941074,0.08553842455148697,0.040416035801172256,0.009134011343121529,0.021934164687991142,0.021438097581267357,-0.03634916990995407,-0.0688432902097702,-0.013043832033872604,0.025448894128203392,-0.007640753872692585,0.03268642723560333,0.047593750059604645,0.02370806410908699,-0.04399384185671806,-0.011948148719966412,0.00317075802013278,-0.06658932566642761,-0.08103692531585693,-0.04337267950177193,-0.023365356028079987,-0.052465178072452545,-0.030956458300352097,-0.05088644102215767,0.0703824907541275,-0.2588604688644409,0.023407630622386932,0.03392265737056732,0.06753463298082352,0.08164247125387192,0.15510277450084686,0.0011452015023678541,-0.03879097104072571,-0.009392376989126205,-0.012997829355299473,-0.16343724727630615,-0.2353842705488205,0.0916450023651123,0.037001900374889374,-0.014430666342377663,-0.0011643551988527179,0.005117789842188358,-0.027942446991801262,0.014490150846540928,-0.03277868404984474,0.011851785704493523,-0.05282112956047058,0.014084258116781712,-0.01007429976016283,-0.12721730768680573,-0.003789447946473956,-0.01894785277545452,-0.023262031376361847,-0.025929678231477737,0.1534019261598587,-0.05066618695855141,0.008582773618400097,-0.0014871950261294842,-0.022923005744814873,-0.02590959146618843,-0.0809846892952919,-0.17623405158519745,-5.296582821756601e-05,-0.010371465235948563,0.01046698447316885,0.018660306930541992,-0.027369758114218712,-0.06596720963716507,0.00394004350528121,-0.0003436569531913847,0.015244418755173683,0.007451585028320551,-0.20645754039287567,0.0739370658993721,0.003440768923610449,-0.007658598478883505,0.01665991172194481,0.04277757555246353,-0.07102920860052109,-0.45434120297431946,-0.00031951398705132306,-0.03338506072759628,-0.003327078651636839,-0.023215793073177338,0.06005462259054184,0.3133610486984253,0.005246760789304972,0.00020440314256120473,-0.0030961160082370043,0.0617624931037426,0.08038321882486343,0.019375387579202652,-0.007900119759142399,0.005186088383197784,-0.008308961987495422,-0.024317370727658272,-0.0492163710296154,-0.3024464249610901,-0.006565870717167854,-0.017027761787176132,-0.012569603510200977,0.012271598912775517,-0.12424815446138382,-0.21630685031414032,0.006355376914143562,0.0015507243806496263,0.0028890392277389765,0.016685662791132927,-0.0009072396787814796,0.278278648853302,-0.012761072255671024,-0.3042920231819153,0.028039922937750816,0.045091696083545685,-0.0031253162305802107,-0.02635340578854084,0.15057329833507538,-0.004785833414644003,-0.0004970547743141651,0.015646932646632195,-0.09306631982326508,0.07830124348402023,-0.02687942050397396,-0.002228447003290057,-0.0009436106774955988,0.03676794096827507,0.002524727489799261,-0.1506466418504715,-0.005782684776932001,-0.3680035173892975,0.12511423230171204,0.036775361746549606,0.2711549401283264,-0.2344704270362854,0.0006696264026686549,0.30279073119163513,0.051748231053352356,0.09257961809635162,-0.000458657304989174,-0.1612149178981781,0.0007750566001050174,0.2346341460943222,-0.001092005637474358,0.004583359230309725,-0.09405200928449631,0.19261953234672546,0.03180311620235443,0.07911784201860428,-0.13232725858688354,0.21669380366802216,0.00026902765966951847,0.25169527530670166,-0.1456364393234253,-0.012061388231813908,-0.15985502302646637,0.1210893839597702,-0.07896390557289124,0.03925540670752525,-0.00018099798762705177,0.23339582979679108,0.016357263550162315,0.09504535049200058,-0.18607313930988312,0.029673222452402115,-0.13572274148464203,0.08992228657007217,-0.0011269589886069298,0.2093667984008789,-0.10966247320175171,0.056027814745903015,-0.16726382076740265,0.03395035117864609,0.015443289652466774,0.39030760526657104,0.002137497067451477,-0.19155019521713257,0.2582554817199707,0.00529828853905201,0.00395243801176548,-0.03599749132990837,-0.14827829599380493,0.3168904483318329,0.00386819988489151,-0.1431242823600769,0.19060319662094116,0.1602030247449875,-0.17516060173511505,0.006891300901770592,0.006364353932440281,-0.12534025311470032,0.17453676462173462,-0.24781173467636108,-0.2274036705493927,-0.07716944068670273,0.1364249289035797,-0.22001849114894867,-0.195912167429924,0.05082407221198082,-0.06157756596803665,-0.049609988927841187,-0.09645874053239822,-0.09845763444900513,-0.058724477887153625,-0.2910315990447998,0.05992788448929787,0.09997627139091492,0.06988915801048279,0.05733877792954445,0.07662343233823776,0.05876659229397774,-0.027041543275117874,-0.035558149218559265,-0.04862343892455101,-0.02315247245132923,-0.0737437829375267,-0.10528222471475601,-0.0407523475587368,-0.022975783795118332,-0.09554355591535568,-0.09753696620464325,0.050971951335668564,0.10738711804151535,-0.062209583818912506,-0.10057979077100754,-0.08776405453681946,-0.07698513567447662,-0.05568546801805496,0.005430766846984625,-0.06702272593975067,-0.057420603930950165,-0.013089178130030632,-0.06717434525489807,-0.07609580457210541,0.021132364869117737,-0.05051890388131142,-0.04687965288758278,-0.03363131359219551,-0.05710434168577194,-0.04783117026090622,-0.18991978466510773,-0.015309907495975494,-0.04459810629487038,-0.03981209173798561,-0.041416093707084656,0.011443929746747017,-0.359861820936203,0.015126680955290794,0.037950221449136734,0.03945678472518921,-0.010739504359662533,-0.06329593807458878,-0.03659160062670708,-0.0009168778196908534,0.010264221578836441,-0.0064476472325623035,0.0009197673061862588,0.020644301548600197,0.21476587653160095,-0.004168917890638113,0.018253212794661522,0.005428450647741556,-0.008239667862653732,-0.007943765260279179,0.16841907799243927,0.004009283147752285,0.03880644217133522,0.024656597524881363,0.02345881424844265,-0.024551549926400185,-0.22187010943889618,-0.0038578230887651443,-0.0061666835099458694,-0.005587984342128038,0.0018453766824677587,-0.010929650627076626,-0.061180803924798965,0.011982284486293793,0.007930515334010124,0.028988823294639587,0.01675187051296234,-0.003072875551879406,-0.02577994018793106,-0.002892138436436653,0.009060136042535305,0.01320548728108406,0.04464981332421303,0.03488237038254738,0.1432895064353943,-0.005590073298662901,0.0034555981401354074,-0.00804834347218275,-0.006901430897414684,0.00558615755289793,-0.02731095626950264,0.06491319090127945,0.0460221990942955,0.06545333564281464,0.07600213587284088,0.07684418559074402,-0.14747315645217896,0.0040442124009132385,0.005059823859483004,0.008940947242081165,0.022495364770293236,-0.012908759526908398,-0.09595384448766708,-0.003185620065778494,-0.01381529401987791,-0.02833646535873413,-0.027411142364144325,-0.018610501661896706,0.4955068826675415,-0.03680838271975517,-0.048237383365631104,-0.06346838921308517,-0.05696268752217293,-0.026799650862812996,0.3616291880607605,0.055949870496988297,0.03790578246116638,0.08075135946273804,0.09250252693891525,0.18799641728401184,0.6970909833908081,0.04255687817931175,0.0603620782494545,0.11556648463010788,0.07363750040531158,-0.019113963469862938,0.3265516459941864,0.003810744732618332,0.0022992687299847603,0.057988204061985016,0.057865943759679794,0.047047778964042664,0.05014481022953987,0.007962768897414207,0.010712034069001675,0.0161159485578537,-0.006972467992454767,0.015311154536902905,-0.013085782527923584,0.009560755454003811,0.00468518678098917,0.008349267765879631,0.007802641950547695,0.0067701456137001514,0.007483162451535463,0.005442208610475063,0.00213241227902472,4.239217741996981e-05,-0.002190625760704279,-0.0017774321604520082,-0.0008781905053183436,0.0038308114744722843,0.0009181617060676217,0.0056389449164271355,0.0020690420642495155,-0.00010446264786878601,0.00025002757320180535,-0.002759535564109683,0.005665327422320843,0.003953750245273113,0.008026551455259323,-0.0006373234209604561,0.007547320332378149,-0.001816987874917686,0.001469874638132751,-0.0008039652020670474,0.0006603687652386725,-0.0026205938775092363,0.00027394157950766385,0.009669439867138863,0.008217450231313705,0.007990799844264984,0.007284959312528372,0.0019996294286102057,0.008623096160590649,0.003010833403095603,-0.0017207895871251822,0.003569627646356821,0.0018618355970829725,0.004213252104818821,0.002797188237309456,-0.009942363016307354,-0.005406934302300215,-0.008276848122477531,-0.0055063399486243725,-0.0014002694515511394,-0.009442046284675598,-0.06432446092367172,-0.08929097652435303,-0.05199465900659561,-0.03929929435253143,-0.06181908771395683,-0.29216739535331726,0.01854567974805832,0.040646716952323914,0.07613226771354675,0.11495848000049591,0.06739386171102524,0.13051921129226685,-0.04068278893828392,-0.002201022580265999,-0.030943455174565315,0.037253424525260925,-0.058939408510923386,-0.09200569242238998,-0.06125922128558159,-0.0508534200489521,-0.07764054089784622,-0.0795878916978836,-0.021311486139893532,-0.11570984870195389,-0.020296219736337662,-0.05565427616238594,-0.025586072355508804,0.012524345889687538,0.04679558426141739,0.026460275053977966,-0.006594987120479345,-0.015441959723830223,-0.0009926656493917108,-0.07880520075559616,-0.11565646529197693,-0.10942148417234421,-0.00231841835193336,0.03065745159983635,0.016909891739487648,0.03281160071492195,-0.0002066885499516502,0.014521203935146332,-0.0009934678673744202,-0.009500415064394474,0.005318612791597843,-0.004327478818595409,0.027035946026444435,-0.0118045499548316,-0.029595565050840378,-0.06648647785186768,-0.0906176045536995,-0.09536019712686539,-0.022987639531493187,0.16578364372253418,-0.05715147405862808,-0.04012991860508919,-0.12436384707689285,-0.04936036840081215,-0.016090702265501022,-0.06462869048118591,0.007728311698883772,0.023551754653453827,0.004764785524457693,0.0023942063562572002,0.06662081182003021,-0.08532979339361191,-0.055295322090387344,-0.007275500800460577,-0.058010444045066833,-0.03233575448393822,0.029073011130094528,-0.11544042080640793,0.04459136351943016,0.007343911565840244,0.05636969208717346,0.014941293746232986,-0.004851503297686577,0.03395029157400131,-0.008499113842844963,-0.10668300837278366,-0.034300949424505234,-0.06803740561008453,-0.0742773488163948,-0.10931037366390228,-0.017125487327575684,-0.04512796923518181,-0.021469242870807648,-0.01802324876189232,-0.023960033431649208,0.09640704840421677,0.0028235684148967266,-0.008472803048789501,0.008178508840501308,0.005763400811702013,0.017590491101145744,-0.11370882391929626,-0.02043485827744007,-0.04773068055510521,-0.052825845777988434,-0.00025244904099963605,-0.008831620216369629,0.015651965513825417,0.03392547369003296,0.03173569217324257,0.03287873417139053,0.03619180992245674,0.032130107283592224,-0.011306362226605415,0.007267269771546125,0.006958526559174061,0.010454564355313778,0.016934921965003014,0.028693586587905884,0.00602825777605176,0.014752041548490524,0.0009245105320587754,0.02478424273431301,0.022086435928940773,0.009907713159918785,0.06687363237142563,-0.0038486451376229525,-0.006175129674375057,-0.01975204050540924,-0.027650389820337296,0.010037977248430252,-0.020818820223212242,0.0007990593439899385,-0.0351545512676239,-0.024667734280228615,0.02004738338291645,-0.012462334707379341,0.05567775294184685,0.02229267731308937,0.012979385443031788,0.022206388413906097,0.011836318299174309,0.0204591304063797,-0.03172696754336357,0.029953857883810997,0.021859997883439064,0.02669902890920639,0.04169936850667,0.0045562065206468105,0.021584587171673775,-0.07602842152118683,-0.09600120782852173,-0.16818730533123016,-0.052323002368211746,-0.05999279022216797,0.1293080598115921,-0.01640959456562996,-0.04595176503062248,-0.11202888190746307,-0.035003386437892914,-0.07958421111106873,-0.07762105762958527,0.0301678404211998,0.020115135237574577,0.027767080813646317,0.013779059052467346,0.017346110194921494,0.010373283177614212,-0.018533237278461456,0.014206326566636562,0.01581493206322193,0.011893676593899727,-0.05068211257457733,-0.08697935938835144,0.0197775699198246,0.029216302558779716,0.004591223783791065,0.04837799817323685,-0.005554300732910633,-0.11287916451692581,-0.09125594049692154,-0.10015533119440079,-0.11096338927745819,-0.06633907556533813,0.04030323028564453,-0.25569871068000793,-0.019004149362444878,-0.0005054314387962222,-0.026937438175082207,-0.023281028494238853,-0.00044196497765369713,0.05007847771048546,0.006916156969964504,0.0030350303277373314,0.02438214235007763,0.041907597333192825,-0.006019611842930317,-0.05775178223848343,0.04578792676329613,-0.06354375928640366,-0.40788108110427856,-0.006600453518331051,0.03025164268910885,0.054115381091833115,0.048155877739191055,0.11999569088220596,0.2036009430885315,0.0010909271659329534,-0.07039333134889603,0.0007010801346041262,0.04303617402911186,-0.0684196874499321,0.001527521526440978,-0.05880119279026985,0.019845711067318916,0.360970139503479,0.4622195363044739,-0.002268172800540924,0.049339424818754196,-0.07644987106323242,0.027917737141251564,-0.10681350529193878,0.13277238607406616,-0.2341519296169281,-0.14114779233932495,-0.002139428397640586,-0.17912988364696503,-0.0442054383456707,-0.026652516797184944,0.16574572026729584,0.0723133385181427,-0.19068360328674316,-0.3517877161502838,0.005713907536119223,0.2745997905731201,0.074082151055336,0.06941728293895721,-0.24736015498638153,0.09426059573888779,0.28834694623947144,-0.05880694091320038,-0.004469916690140963,0.20784859359264374,0.021492060273885727,-0.050359055399894714,-0.14829622209072113,0.24467001855373383,0.011162861250340939,-0.536612331867218,0.0009479831205680966,0.10707296431064606,-0.3203868567943573,0.03897557407617569,0.14941252768039703,-0.006176822818815708,0.15050196647644043,-0.0841926857829094,0.001436928054317832,0.18215157091617584,0.005407342221587896,0.05680146813392639,0.12489937245845795,-0.06785723567008972,-0.14878861606121063,-0.05590175464749336,0.001017837435938418,-0.08520101010799408,-0.00877400767058134,0.015181723050773144,0.1729975789785385,-0.07366211712360382,0.08724728226661682,0.3374062180519104,-0.005689971148967743,-0.08821878582239151,0.2531837821006775,-0.11063425242900848,0.21724645793437958,0.1862861067056656,-0.05009208619594574,0.33754247426986694,0.10525709390640259,0.4296097755432129,0.33775594830513,-0.19488966464996338,-0.14823494851589203,-0.01860814169049263,-0.016833091154694557,-0.028372375294566154,-0.021025149151682854,-0.03978007659316063,0.024270012974739075,0.013410355895757675,0.003393375314772129,0.009981946088373661,0.015116669237613678,-0.0009628817206248641,0.16167432069778442,-0.019004985690116882,-0.010371167212724686,-0.027646029368042946,-0.01687701605260372,-0.015540276654064655,0.13487409055233002,0.00818693544715643,-0.004664368461817503,0.0008040961110964417,0.011237806640565395,0.007410658057779074,-0.10289369523525238,-0.03578348830342293,-0.04414636641740799,-0.04631277918815613,-0.03832201287150383,-0.0686180591583252,-0.11908920854330063,0.0010883691720664501,0.0002564047754276544,0.016151273623108864,0.006090502720326185,0.010026785545051098,0.06913840025663376,0.0025610995944589376,-0.014307288452982903,-0.0029882730450481176,-0.0008001321111805737,-0.008594125509262085,-0.08142521232366562,-0.022067183628678322,-0.020098859444260597,-0.016951214522123337,-0.011837287805974483,-0.027384409680962563,-0.019458532333374023,-0.024814343079924583,-0.002281044377014041,0.05090751498937607,0.114610455930233,0.10128908604383469,-0.33807727694511414,-0.019723698496818542,-0.01838008128106594,-0.034148916602134705,-0.03452584892511368,-0.02412194199860096,-0.005897671915590763,0.0037611944135278463,0.017345134168863297,0.013207273557782173,0.07707484066486359,0.022679325193166733,0.07186967134475708,-0.048714376986026764,-0.02634131722152233,-0.04273952171206474,0.0022766804322600365,-0.10568215698003769,0.050680361688137054,0.02047356404364109,0.02181834913790226,0.055084023624658585,0.06125600263476372,0.08884141594171524,0.20060254633426666,-0.00116652250289917,-0.006875098217278719,0.01935185119509697,0.034849897027015686,0.016133809462189674,0.0379820242524147,-0.022916704416275024,-0.01521907839924097,-0.007712727412581444,0.020070454105734825,-0.02545625902712345,-0.12106851488351822,0.0069971452467143536,0.008861472830176353,0.001038735150359571,0.022271765395998955,-0.011266089044511318,-0.07612865418195724,0.006242228206247091,0.009349490515887737,0.005628910381346941,0.019917214289307594,0.013332701288163662,-0.049985121935606,-0.018641231581568718,0.014543609693646431,0.028980642557144165,0.013337572105228901,0.022357139736413956,0.06114953011274338,-0.001501290244050324,-0.0017929082969203591,0.001005501369945705,-0.013888407498598099,0.02441292814910412,-0.00757893780246377,-0.013889549300074577,-0.004949802998453379,-0.003512877505272627,0.006219217088073492,0.009919412434101105,0.0772988572716713,-0.011649354360997677,0.003145968308672309,0.005632378160953522,-0.020197173580527306,-0.03189452737569809,0.09706363081932068,0.03283818066120148,-3.5748027585214004e-05,-0.010410639457404613,-0.008934971876442432,0.014069204218685627,0.08001704514026642,-0.017615588381886482,-0.006001038942486048,-0.012394118122756481,-0.012678657658398151,-0.02338436245918274,-0.06339394301176071,0.0036917794495821,0.003410519566386938,0.013670184649527073,-0.014025796204805374,0.027416937053203583,-0.07876793295145035,-0.001968535128980875,0.016224762424826622,-0.012243582867085934,0.008430488407611847,-0.04917353019118309,-0.11799439787864685,0.003456472884863615,0.004492688458412886,-0.005937594920396805,0.013277958147227764,-0.023399870842695236,0.041871629655361176,0.01836906373500824,0.03415081650018692,0.03708576783537865,0.02640111744403839,-0.02581215277314186,-0.05928773805499077,0.006917813327163458,0.006076488643884659,0.022463878616690636,0.01566438004374504,0.04042171686887741,-0.00459289038553834,0.005736653693020344,0.024251746013760567,0.007134370040148497,-0.005344155710190535,0.01943145878612995,0.0038345414213836193,-0.003699151799082756,-0.015537766739726067,0.0012270106235519052,0.002562633715569973,-0.021950500085949898,-0.0194077268242836,0.005813048221170902,0.006526461802423,0.018930042162537575,0.010938763618469238,-0.01050648931413889,0.0014732804847881198,0.017288854345679283,0.025347962975502014,0.030423447489738464,0.028324030339717865,0.0260428786277771,-0.005618341267108917,0.020866110920906067,0.009614896960556507,0.0300692617893219,0.0549757219851017,0.12584275007247925,-0.30236539244651794,-0.0007584209088236094,0.0009703192044980824,0.010710547678172588,-0.027266530320048332,0.006027622148394585,-0.020152343437075615,0.010367927141487598,0.0030418497044593096,0.003376569366082549,0.07200530916452408,0.025403611361980438,-0.04665382206439972,0.018444858491420746,-0.02646319940686226,-0.023916488513350487,-0.04475677013397217,-0.06333761662244797,-0.07974414527416229,-0.010024141520261765,0.012496370822191238,0.02184184081852436,0.05194254592061043,0.03250065818428993,0.03257770836353302,0.0034553646109998226,-0.016148876398801804,-0.0019218785455450416,-0.010992632247507572,0.01834697835147381,0.0026562416460365057,-0.000533913669642061,-0.02957891672849655,-0.016232838854193687,-0.03588607907295227,-0.011057966388761997,-0.012827735394239426,0.008884166367352009,0.004861308261752129,0.0033386102877557278,0.025271909311413765,-0.023394614458084106,-0.03817884996533394,0.00987447053194046,0.006286920979619026,-0.0017946426523849368,-0.004759322851896286,-0.007556240074336529,0.13376060128211975,-0.013765363022685051,0.001524074818007648,-0.007506739813834429,-0.011266205459833145,0.0106890257447958,-0.25975215435028076,0.008066905662417412,0.013336144387722015,0.021975336596369743,0.020909929648041725,0.009435840882360935,0.008699169382452965,-0.0012372609926387668,-0.0007593536865897477,0.0033387921284884214,0.004462151322513819,-0.004945357795804739,-0.0007783150067552924,0.01425726804882288,0.03466477245092392,0.020240556448698044,0.023127857595682144,0.04703188315033913,0.22742553055286407,0.017213348299264908,-0.010867971926927567,-9.121960465563461e-05,0.007925871759653091,-0.025636887177824974,0.03134413808584213,0.008487475104629993,0.006983190309256315,0.016194552183151245,0.022567003965377808,0.011933554895222187,0.03902706131339073,0.013737647794187069,0.02005794644355774,0.015847517177462578,0.020516594871878624,0.025355365127325058,0.02699325792491436,0.0023456146009266376,0.003534374525770545,0.002291397424414754,-0.0011132345534861088,-0.00021099338482599705,0.001767497742548585,-0.0017209635116159916,0.000732846325263381,-0.0020939107052981853,-0.0012969155795872211,-0.0021205549128353596,0.004622600041329861,-0.003177026053890586,-0.0029006456024944782,-0.00030229275580495596,-0.0002724419755395502,-0.00029946863651275635,-0.008223564364016056,0.003612476633861661,0.0017561843851581216,0.0023694918490946293,0.00202890089713037,0.0029844576492905617,0.004275116138160229,-0.0012416709214448929,-0.001964209834113717,-0.0014412420568987727,-0.002155579626560211,-0.0019110642606392503,-0.001435203943401575,-0.0009523597545921803,0.00019007574883289635,-0.001430241740308702,0.0011090460466220975,0.0005674409912899137,0.0033162587787956,0.003655532142147422,0.0032306017819792032,0.0037755800876766443,0.005220312159508467,0.005132325924932957,0.0033382605761289597,-0.006373151671141386,-0.0057617477141320705,-0.007524011190980673,-0.006503239739686251,-0.006254561711102724,-0.007061397656798363,-0.03484179824590683,-0.015555502846837044,0.012132275849580765,0.03928210958838463,0.053219374269247055,0.1304119974374771,0.00768635468557477,0.002594209974631667,-0.00011258899030508474,-0.019195888191461563,-0.013637380674481392,-0.2633104622364044,-0.024698911234736443,-0.0028483348432928324,-0.02149086818099022,0.02207978069782257,0.013553948141634464,0.1545083224773407,-0.010752424597740173,-0.004049758892506361,-0.023398766294121742,-0.012434441596269608,-0.022791724652051926,0.030861902981996536,-0.010173416696488857,-0.010394014418125153,0.00785239040851593,0.027430664747953415,0.039288636296987534,0.1214815005660057,0.02104133926331997,0.005636829882860184,0.00952793937176466,-0.0008214336121454835,-0.005480497609823942,-0.029805079102516174,-0.0002927859022747725,0.0008699150639586151,-0.012235058471560478,-0.007804282009601593,-0.02079441025853157,0.0021960821468383074,0.007348956074565649,-0.0021685902029275894,-0.012761130928993225,-0.001736209262162447,0.005398925859481096,-0.008801481686532497,0.12745051085948944,0.2573653757572174,0.002128337975591421,-0.014018023386597633,0.15344694256782532,0.0839482918381691,-0.012146753259003162,-0.036794811487197876,0.006460408214479685,0.1885974407196045,-0.007822506129741669,0.014901570975780487,0.08206186443567276,-0.03905417397618294,-4.485156296141213e-06,-0.02938128262758255,0.028484215959906578,0.11763985455036163,0.09797674417495728,0.07128920406103134,-0.06079784780740738,-0.16530215740203857,-0.00023092326591722667,-0.125187486410141,-0.10782282799482346,0.04357478395104408,-0.0647103488445282,0.0622636079788208,0.07598234713077545,0.041107162833213806,0.00013669108739122748,-0.09392711520195007,-0.13477972149848938,0.2819616198539734,-0.08120597898960114,0.052223529666662216,0.16693376004695892,-0.2543966472148895,0.0003776351804845035,-0.19189976155757904,0.157841295003891,0.15878017246723175,-0.02790757827460766,-0.1087944284081459,0.15197744965553284,-0.13074420392513275,-0.00017231293895747513,0.03511018678545952,0.2639314830303192,0.050312381237745285,-0.03320528566837311,-0.07784096896648407,0.13187964260578156,-0.2863723933696747,-0.00043099085451103747,-0.19271676242351532,0.11683584004640579,0.01234407164156437,-0.029550353065133095,0.09072517603635788,-0.0508899949491024,-0.09828942269086838,-0.00016288823098875582,-0.03398805484175682,-0.06071007251739502,-0.10453396290540695,0.042993754148483276,-0.024001289159059525,-0.04378911852836609,0.0034463333431631327,-9.803317516343668e-05,0.017136117443442345,0.041832949966192245,-0.1465303897857666,0.002895959187299013,-0.04892216995358467,-0.14609958231449127,-0.19523154199123383,-4.4501215597847477e-05,-0.2038000375032425,-0.020752938464283943,-0.21087107062339783,0.039748869836330414,0.05646795406937599,-0.0022784217726439238,-0.15420937538146973,-0.10519378632307053,-0.22723576426506042,-0.0276349950581789,-0.005707947071641684,-0.010060635395348072,-0.01985171064734459,-0.03435448184609413,0.019230715930461884,0.03339696675539017,0.0036327987909317017,0.0028262585401535034,0.008432355709373951,-0.03662541136145592,0.0851462334394455,-0.013741662725806236,-0.002693138550966978,-0.0005618739523924887,-0.002579422201961279,0.019178781658411026,0.09849664568901062,0.006649029906839132,0.0031891765538603067,-0.007852827198803425,-0.011993123218417168,0.029482750222086906,-0.08435375988483429,0.01599232666194439,0.0013240852858871222,0.0051012057811021805,0.0026656126137822866,0.0005593307432718575,-0.06557809561491013,-0.00030112938839010894,-0.004174623638391495,-0.00823303870856762,-0.01971404440701008,-0.036672867834568024,0.0031936524901539087,0.022208809852600098,0.007356775458902121,0.0003297670336905867,-0.013179633766412735,-0.005567733198404312,0.0528072863817215,0.01855912059545517,-0.002984552877023816,-0.001965399831533432,0.005824321415275335,0.013449864462018013,-0.009072331711649895,-0.021617598831653595,-0.011463413015007973,-0.013678234070539474,-0.002839035354554653,0.07338939607143402,-0.22112777829170227,0.01399198267608881,-0.0007335224072448909,0.00914817489683628,-8.263098607130814e-06,0.02262033335864544,-0.026024606078863144,-0.005614751949906349,-0.0061329444870352745,-0.003950922284275293,0.004804844036698341,0.011241178028285503,0.11324433982372284,0.002107416046783328,-0.0024436721578240395,-0.00039775579352863133,-0.009494149126112461,0.001479227445088327,-0.011537540704011917,0.033048711717128754,0.009687463752925396,0.01996762491762638,0.028356442227959633,0.0026988727040588856,0.008013546466827393,0.0015844205627217889,0.005752845201641321,-0.00112936703953892,0.002338132355362177,-0.013164211995899677,-0.16390608251094818,0.017448175698518753,0.00937837827950716,0.010841389186680317,0.005406048148870468,0.009193913079798222,-0.17982977628707886,0.023922890424728394,0.0007036873139441013,0.012597600929439068,0.01728670299053192,-0.02901981957256794,-0.09962426871061325,0.013841294683516026,-0.010471299290657043,0.01246753428131342,-0.002176208421587944,-0.07646478712558746,-0.06473451107740402,-0.016358163207769394,-0.006553176790475845,-0.000176996472873725,0.015659336000680923,-0.06897132843732834,-0.04295092821121216,0.0017238035798072815,0.0022406051866710186,-0.002785556251183152,-0.005818480160087347,0.003867518389597535,-0.09177690744400024,0.006540913134813309,-0.0075669437646865845,0.01552874781191349,0.007236254867166281,0.073137067258358,0.03691287338733673,-0.04507960379123688,-0.014654047787189484,-0.008937043137848377,-0.017029622569680214,-0.014252875931560993,0.06683694571256638,-0.00852535106241703,-0.018229767680168152,-0.012499737553298473,-0.017285332083702087,-0.014078473672270775,-0.052428364753723145,-0.020910505205392838,-0.014070807956159115,-0.005129932891577482,-0.01661696285009384,-0.004258215427398682,-0.008492706343531609,-0.023236963897943497,-0.011277991347014904,-0.004487165249884129,0.0012026292970404029,0.023578321561217308,-0.06294575333595276,-0.013689886778593063,-0.0019399247830733657,0.024106096476316452,0.016170715913176537,0.08193232864141464,0.04756452143192291,0.015620890073478222,0.003609523642808199,0.011595671996474266,-0.01971970871090889,0.07666169852018356,0.08960895240306854,0.009357626549899578,0.018498029559850693,0.010680959559977055,0.03076089546084404,0.011092477478086948,0.019592713564634323,-0.006145718973129988,0.004843448288738728,-0.002088532317429781,-0.0007243769359774888,-0.059513844549655914,-0.03960555046796799,0.0010091557633131742,-0.013269531540572643,-0.004621906206011772,0.0018958612345159054,-0.0037038233131170273,-0.047317735850811005,-0.008626087568700314,0.005171171389520168,4.089415233465843e-05,-0.0025219430681318045,0.020558277145028114,0.057100214064121246,0.008703548461198807,0.006266864947974682,-0.0037064284551888704,-0.006889404729008675,0.02906021475791931,0.028478039428591728,0.0068532251752913,0.012218944728374481,0.005162563640624285,0.011799001134932041,-0.025451237335801125,-0.015290148556232452,0.027924712747335434,0.00827592983841896,-0.005950778257101774,0.03295321762561798,0.045068614184856415,-0.05399885028600693,-0.027386240661144257,-0.016215885058045387,0.0016407863004133105,-0.025026878342032433,0.03815924748778343,-0.06403820216655731,0.022557329386472702,0.01061945129185915,0.0019901711493730545,0.016054805368185043,0.0076987105421721935,-0.05122416466474533,0.013346776366233826,0.0008172622183337808,0.017638111487030983,0.011312361806631088,-0.022400876507163048,0.06335050612688065,-0.015355397947132587,-0.011596799828112125,0.0013431974221020937,-0.00847726222127676,0.011643596924841404,0.05347365140914917,0.010434486903250217,0.016848109662532806,0.006732464302331209,0.007507700473070145,0.016942759975790977,-0.0400577150285244,-0.006121583748608828,-0.0012084540212526917,0.00806424394249916,0.006786024197936058,0.0116852642968297,0.02053436078131199,-0.004062514286488295,-0.0004796432622242719,0.009088347665965557,-0.022793913260102272,-0.011920366436243057,-0.02681005373597145,0.005027020815759897,0.006635239813476801,0.0028679175302386284,-0.03432772681117058,-0.042419079691171646,-0.02814493700861931,0.017306510359048843,0.02056104689836502,0.010843065567314625,0.01892578974366188,-0.0031632648315280676,-0.0051155914552509785,0.00023194600362330675,0.0016124268295243382,0.009694898501038551,-0.010393169708549976,0.027763254940509796,0.011393793858587742,0.01642509736120701,0.02046395093202591,0.007283410057425499,-0.0016978011699393392,0.02848086692392826,-0.008311523124575615,-0.002197565510869026,-0.004970910027623177,-0.007865147665143013,-0.013305183500051498,0.0062459479086101055,-0.012493157759308815,0.01742441952228546,0.019221708178520203,0.0020455289632081985,0.021074913442134857,0.014363517984747887,0.006986967287957668,-0.00014383815869223326,-0.00404507527127862,0.010867769829928875,0.007619248237460852,-0.005480930674821138,0.030710294842720032,0.018511245027184486,0.023967724293470383,0.00098259502556175,0.0059007927775382996,-0.024633079767227173,-0.011961197480559349,0.005301324650645256,0.007394830696284771,0.008066954091191292,0.024757038801908493,0.06350794434547424,0.1072184219956398,-0.0027621302288025618,-0.008039865642786026,-0.005100382957607508,-0.012556646950542927,-0.0041697523556649685,0.039408400654792786,-0.002721700817346573,0.008535629138350487,0.0014668325893580914,-0.006210633087903261,0.006366613321006298,0.08422387391328812,0.02687140181660652,0.013657591305673122,0.01796390488743782,0.019732333719730377,0.005334788933396339,0.08622299134731293,0.03341282159090042,0.02382759004831314,0.028430277481675148,0.04487500339746475,0.041289620101451874,-0.037620559334754944,0.03213480859994888,0.015072232112288475,0.025158647447824478,0.04436074569821358,0.023399876430630684,0.20986023545265198,0.04689149558544159,0.01951630786061287,0.03213588520884514,0.05530892312526703,0.024238221347332,-0.06355420500040054,0.020657632499933243,0.01928671821951866,0.016384610906243324,0.00811236072331667,0.00033051855280064046,0.17039310932159424,0.014065878465771675,0.00033805359271354973,0.01550716906785965,0.008245271630585194,0.009564082138240337,0.009281759150326252,-0.01778441295027733,-0.007434742990881205,-0.014447739347815514,-0.006424386985599995,-0.009391921572387218,-0.007784420158714056,-0.004802433308213949,0.003026670543476939,0.0005000265664421022,-0.005539176054298878,-0.002488994738087058,-0.15989136695861816,0.0003222262021154165,0.008221893571317196,-0.00031988549744710326,0.0024983505718410015,0.007786872796714306,0.0500975027680397,0.018239924684166908,0.010919462889432907,0.020467014983296394,0.027757197618484497,0.016386443749070168,0.09925473481416702,0.014471546746790409,0.010550771839916706,0.013103239238262177,0.02087954245507717,0.01321299560368061,-0.03482825309038162,0.005207110662013292,0.013856885954737663,0.010461138561367989,0.012154066003859043,0.007617691066116095,-0.039720017462968826,0.006988046690821648,0.012767890468239784,0.005962492898106575,0.01521282084286213,0.008092476055026054,0.05423661321401596,0.032705213874578476,0.12163292616605759,-0.01764015480875969,0.0047594355419278145,-0.07893404364585876,-0.012751010246574879,-0.26200518012046814,-0.12093070149421692,-0.007864435203373432,0.08000782132148743,0.01190511416643858,0.00574525585398078,-0.00629684841260314,0.00019134400645270944,-0.003958191256970167,-0.008939872495830059,0.08531860262155533,0.11952407658100128,-0.05283540487289429,-0.010301629081368446,-0.06387005001306534,0.008961035870015621,0.02074357122182846,-0.02760336920619011,-0.05430535227060318,0.17234568297863007,0.09940506517887115,-0.09800131618976593,0.029386812821030617,0.04340991750359535,-0.09489671140909195,0.08528367429971695,-0.07594242691993713,-0.03585381805896759,0.04800359904766083,0.01160782016813755,0.06790266185998917,-0.06021445244550705,-0.06595179438591003,0.07337028533220291,0.034561045467853546,0.2761957347393036,-0.16304902732372284,0.13854612410068512,0.01715177670121193,-0.04998588562011719,-0.08162588626146317,0.0158248171210289,0.046994950622320175,0.15381644666194916,-0.024124959483742714,0.02382778562605381,-0.036266013979911804,0.017743704840540886,-0.19139929115772247,-0.10393698513507843,0.024562818929553032,0.059274934232234955,0.10298912972211838,-0.02949344925582409,-0.0063715060241520405,0.05825284868478775,-0.06764225661754608,0.02300546132028103,-0.08011709153652191,0.11529957503080368,0.07107049226760864,-0.0862356647849083,-0.017914658412337303,0.01670934446156025,0.00230672606267035,-0.03823602944612503,0.1970224529504776,-0.1037282645702362,0.04319211468100548,0.025574523955583572,-0.05647466331720352,0.028341282159090042,0.21100516617298126,-0.06501714885234833,-0.031102726235985756,-0.1470935344696045,0.04607786983251572,-0.18597406148910522,0.3044760823249817,-0.11714522540569305,-0.052136924117803574,0.18703682720661163,0.016565876081585884,-0.016370849683880806,0.047223493456840515,0.12453562021255493,-0.0947396531701088,0.06804384291172028,-0.026404930278658867,-0.0421702116727829,-0.012893985025584698,0.05401160567998886,0.23266488313674927,-0.0659346729516983,0.004093197174370289,0.0607774555683136,-0.03179023787379265,0.058059822767972946,0.14884668588638306,-0.15836618840694427,0.03679199144244194,0.002086648717522621,0.08182375133037567,0.02500399574637413,0.17675788700580597,0.06376516073942184,-0.0517842099070549,-0.07338108867406845,-0.04606150835752487,-0.1660141497850418,-0.027166396379470825,-0.09924393147230148,-0.0020055249333381653,-0.06195729225873947,-0.004115215037018061,0.1159985139966011,0.3394463360309601,-0.056668322533369064,-0.029000181704759598,0.11150865256786346,0.04449405148625374,-0.09391636401414871,-0.20572680234909058,0.07828065752983093,-0.07854954153299332,-0.004012946039438248,-0.012795798480510712,0.046957094222307205,-0.08687762171030045,0.059326350688934326,-0.02878211811184883,-0.052242446690797806,-0.07843931764364243,-0.02016940526664257,0.04298168793320656,0.13471485674381256,0.01107969880104065,0.08371403068304062,0.12316697835922241,0.07888603210449219,-0.09692102670669556,-0.1717015951871872,-0.026660442352294922,0.03747722506523132,0.12835510075092316,0.051231883466243744,-0.03934201970696449,-0.15567812323570251,-0.018204014748334885,0.02116173505783081,0.05712083727121353,0.03894875943660736,0.015916569158434868,-0.06759120523929596,0.06058935448527336,0.05245833843946457,0.07026316970586777,0.11732839792966843,-0.03233882412314415,-0.036665480583906174,-0.10972896963357925,-0.024228397756814957,0.04709567874670029,0.09830443561077118,-0.013891682960093021,-0.06637153774499893,0.052362337708473206,0.06670809537172318,-0.09765931963920593,0.017117908224463463,-0.011241002008318901,0.0481097474694252,0.063627690076828,-0.01350055169314146,-0.08429491519927979,-0.13852541148662567,0.06397639214992523,0.0738721415400505,0.022301435470581055,0.006365153472870588,-0.005867138970643282,-0.17337970435619354,-0.1615705043077469,-0.18812350928783417,-0.0981840267777443,-0.11870665103197098,-0.036018259823322296,0.031326960772275925,0.02329772524535656,0.03969481587409973,0.037849102169275284,0.03642219677567482,0.07036338746547699,0.16825424134731293,0.14929091930389404,0.3016076683998108,0.042377982288599014,0.06161079555749893,0.01844056323170662,0.07223266363143921,-0.030703391879796982,0.04164333641529083,-0.11910045146942139,-0.10339245200157166,0.04063047841191292,0.16206562519073486,0.07242107391357422,-0.02926163747906685,-0.006946404930204153,-0.009953820146620274,-0.17867411673069,-0.029377298429608345,-0.15746167302131653,0.026880215853452682,-0.14799335598945618,-0.15003462135791779,0.004466906189918518,-0.12682577967643738,-0.05042384937405586,-0.1518028974533081,0.009518844075500965,-0.06042925640940666,0.020267345011234283,-0.05581238865852356,-0.02674066461622715,0.010896708816289902,0.02604801207780838,-0.015221809968352318,-0.021442007273435593,-0.03740431368350983,-0.11084140092134476,-0.07580946385860443,0.07893288880586624,0.048250969499349594,0.041717950254678726,0.09244038164615631,0.04611574858427048,0.05535323545336723,0.037123240530490875,0.10142437368631363,0.05420973524451256,0.07640345394611359,0.08660498261451721,0.07099631428718567,-0.007184851914644241,0.003605975303798914,-0.027540817856788635,0.010596146807074547,-0.007782220374792814,-0.013129434548318386,-0.04114324599504471,-0.06215786561369896,-0.08883733302354813,-0.09034579992294312,-0.07970480620861053,-0.15944577753543854,-0.00732512166723609,-0.009369108825922012,-0.07072345912456512,0.0060222032479941845,-0.03157682716846466,-0.031926821917295456,0.020624440163373947,-0.03285244479775429,-0.02115623652935028,0.017478682100772858,-0.04449387267231941,-0.04200407862663269,0.03750857710838318,-0.006977111101150513,-0.03323286026716232,-0.005893507972359657,0.019489530473947525,0.11742949485778809,-0.0035874254535883665,-0.015641245990991592,-0.024614037945866585,0.01127716526389122,0.01883167214691639,-0.035671744495630264,-0.010295109823346138,-0.05454229563474655,-0.08182564377784729,-0.05676639452576637,-0.022146839648485184,-0.0100363465026021,-0.02173537388443947,-0.05074220895767212,-0.01922975294291973,-0.09396667778491974,-0.0883619636297226,-0.1646628975868225,0.0725354477763176,0.011612357571721077,0.1323462724685669,0.043716948479413986,0.1459580510854721,0.12850263714790344,0.10523061454296112,0.010691460222005844,0.06895067542791367,0.08994009345769882,0.09210821241140366,-0.00013807268987875432,-0.036310359835624695,-0.09185317903757095,0.005754078738391399,-0.021339280530810356,-0.040516093373298645,-0.026512710377573967,0.019850745797157288,-0.0030932961963117123,0.05632689222693443,-0.02593015506863594,-0.0598544105887413,0.03366759791970253,0.015242421068251133,0.08184660971164703,0.013073578476905823,0.0038578121457248926,0.018105272203683853,-0.0044570076279342175,0.08459098637104034,0.07197441905736923,0.012117483653128147,-0.05428324639797211,-0.08251345157623291,-0.13434268534183502,-0.0904269590973854,-0.04948589205741882,-0.12732578814029694,-0.082152359187603,0.11857886612415314,0.11292039602994919,-0.03326556459069252,-0.018894916400313377,-0.052222225815057755,-0.1304585188627243,0.00012197605974506587,0.0629713237285614,0.04575014114379883,-0.0023179040290415287,0.002905580447986722,-0.09577832370996475,-0.08858300000429153,-0.06975395232439041,0.029463892802596092,0.06450328230857849,0.014465367421507835,-0.06924916058778763,0.17193615436553955,0.09862048178911209,0.006619633641093969,-0.030915211886167526,0.013372521847486496,-0.1953902691602707,-0.11100370436906815,-0.03246927633881569,-0.014774863608181477,-0.06742914766073227,-0.0767289400100708,0.15803872048854828,-0.012111824005842209,-0.06530696898698807,-0.13844017684459686,-0.10944163799285889,-0.14259426295757294,-0.028227059170603752,-0.24349357187747955,-0.07277221232652664,-0.014545953832566738,0.046439480036497116,0.04509919881820679,-0.02607554756104946,0.24718299508094788,0.14343753457069397,-0.01293589361011982,0.04425528272986412,-0.0018187202513217926,0.06177736818790436,-0.03413061425089836,0.05746258422732353,0.010386677458882332,-0.11005841940641403,0.05176281929016113,-0.04230957478284836,-0.3091678321361542,-0.091904416680336,0.006105856969952583,0.060408417135477066,-0.007152164820581675,0.06662539392709732,-0.08796500414609909,-0.1891527771949768,0.04786508157849312,0.0683855190873146,-0.004524781368672848,-0.07841050624847412,-0.20234599709510803,-0.11760199815034866,0.03630883991718292,0.04065876081585884,0.0824335366487503,0.1584082692861557,-0.060351885855197906,0.06298317015171051,0.17741453647613525,0.09526931494474411,-0.017522450536489487,-0.03298521786928177,0.2654248774051666,0.22483815252780914,-0.023118149489164352,-0.04235503822565079,-0.007747250609099865,0.036402907222509384,0.28160908818244934,0.1535172462463379,0.019083794206380844,0.06665435433387756,0.040343277156353,0.04973504692316055,0.0385211817920208,0.03574690967798233,0.037077274173498154,0.010428912937641144,-0.028999855741858482,0.02608902007341385,0.001533069764263928,0.07708725333213806,0.03375444561243057,-0.06366394460201263,-0.057942185550928116,0.0008511418709531426,-0.043970681726932526,0.048385486006736755,0.025659779086709023,-0.00018465217726770788,-0.08330879360437393,-0.036241017282009125,0.0019551648292690516,0.07591289281845093,-0.024166764691472054,-0.04572447016835213,-0.09159506857395172,-0.03161883354187012,0.029452495276927948,0.14519502222537994,0.10525941848754883,0.008061286993324757,-0.04789050295948982,-0.01814308762550354,-0.06210850924253464,-0.004547127988189459,-0.07699038088321686,0.016185840591788292,0.060125380754470825,0.015525201335549355,0.04860684275627136,-0.033716391772031784,-0.044586095958948135,0.036564022302627563,0.04881097376346588,0.05261571332812309,0.05600806698203087,0.03198758512735367,-0.10306987166404724,-0.0162307508289814,0.10476817190647125,0.13501223921775818,-0.061936259269714355,0.11620824784040451,-0.5822197198867798,-0.012114688754081726,-0.020808767527341843,-0.03102140501141548,0.044857244938611984,0.005674312822520733,-0.002016895217821002,0.10646520555019379,0.021710867062211037,-0.004384246189147234,0.03104994259774685,0.021824121475219727,0.08433349430561066,0.17708167433738708,-0.19110475480556488,0.12938910722732544,-0.2724190354347229,0.02035687118768692,-0.26837706565856934,0.1304720640182495,0.27453380823135376,-0.004979434423148632,-0.009574315510690212,0.186019629240036,0.053065959364175797,-0.15477898716926575,-0.191025048494339,-0.29615193605422974,-0.1802576631307602,-0.15706481039524078,0.2297021448612213,0.203477144241333,0.5755424499511719,-0.07075967639684677,-0.133414626121521,-0.21891163289546967,0.08952058106660843,-0.04804720729589462,0.06322683393955231,0.15871672332286835,0.29333850741386414,0.1367144137620926,-0.1670621633529663,-0.08795080333948135,0.2297237515449524,0.1580786108970642,-0.14952827990055084,0.05003310367465019,0.26088517904281616,-0.015945689752697945,-0.22346298396587372,-0.07588586956262589,0.07761869579553604,-0.06617185473442078,-0.03236749395728111,0.22020642459392548,0.14045746624469757,0.22092020511627197,0.3384370505809784,-0.19654974341392517,0.2595534026622772,0.019771600142121315,0.11029312759637833,-0.23279979825019836,-0.3136809468269348,0.03194567561149597,0.4958762526512146,0.04139024391770363,-0.22647294402122498,0.1408761441707611,-0.1256798952817917,0.12624698877334595,-0.3934728801250458,-0.11788436770439148,0.13648708164691925,-0.31955286860466003,-0.23539192974567413,-0.23436221480369568,0.14403371512889862,0.21727396547794342,0.2570420205593109,0.23384489119052887,0.01635698787868023,-0.0025974183809012175,0.03167610988020897,0.035571809858083725,0.02698635868728161,-0.0056953951716423035,0.010679716244339943,0.07290482521057129,0.023324143141508102,0.037086449563503265,0.04194570705294609,-0.06547054648399353,0.016433684155344963,-0.023703021928668022,-0.0001395229046465829,-0.030705930665135384,-0.009235741570591927,0.039637841284275055,-0.01279559638351202,0.04145308583974838,-0.053978461772203445,0.04369623586535454,0.02713591791689396,0.06648332625627518,0.022315429523587227,-0.028878377750515938,-0.011980853043496609,0.03890518471598625,-0.031040038913488388,0.06602945923805237,0.0024006711319088936,0.1071651503443718,-0.019040144979953766,0.046466901898384094,-0.03701910749077797,-0.04589824378490448,-0.03388935700058937,0.04514236003160477,0.045478031039237976,0.09381946921348572,0.019335299730300903,-0.023519335314631462,0.1212797462940216,0.08889606595039368,0.13818371295928955,-0.06331761926412582,0.015858357772231102,0.016057316213846207,0.04328090324997902,-0.03717772290110588,-0.009613716043531895,0.10983377695083618,0.057700105011463165,0.03460211306810379,0.017774995416402817,0.00313973193988204,0.013015893287956715,0.09455522894859314,-0.024295281618833542,-0.05358615517616272,-0.012402256019413471,-0.006650728639215231,-0.057935941964387894,-0.11412815004587173,0.046035077422857285,-0.04144647344946861,0.016347607597708702,0.009067660197615623,0.030911998823285103,-0.016301026567816734,0.000734531378839165,0.0632239356637001,0.017984867095947266,0.018126966431736946,0.05695090442895889,0.010911332443356514,-0.053048647940158844,0.03679541498422623,-0.01185695081949234,0.06000106781721115,0.07434463500976562,-0.0035795317962765694,0.021431615576148033,-0.008251368999481201,-0.04610137268900871,0.06485873460769653,0.113835908472538,0.07949687540531158,0.05262986943125725,0.03544631227850914,0.1330934762954712,0.06425518542528152,-0.053677935153245926,-0.022778458893299103,0.026299111545085907,-0.07316708564758301,0.06138893961906433,0.015599582344293594,-0.012886229902505875,0.033341601490974426,0.09958359599113464,-0.06636148691177368,-0.011303802952170372,-0.009378333576023579,-0.034305207431316376,-0.06330399215221405,-0.11175953596830368,-0.0097859101369977,0.06658871471881866,0.019131958484649658,0.08998008817434311,-0.04520273953676224,-0.03284149244427681,0.0410529300570488,-0.08134046941995621,-0.05556904524564743,-0.018540486693382263,-0.10849515348672867,-0.03987133502960205,0.02516959421336651,0.006210220046341419,-0.012053669430315495,0.12970946729183197,0.0014609681675210595,0.11781346052885056,0.2721075415611267,0.018888210877776146,-0.056401848793029785,-0.010075903497636318,-0.08272384107112885,-0.15599662065505981,0.016165267676115036,0.024800797924399376,0.030935173854231834,0.0024840557016432285,0.05644623190164566,0.003188618691638112,-0.1222434788942337,-0.03973982110619545,-0.035049255937337875,-0.023314623162150383,-0.049450378865003586,0.07702665030956268,-0.08855025470256805,-0.00879074539989233,0.01138379331678152,0.020588111132383347,-0.015993665903806686,-0.05253526195883751,-0.16426552832126617,0.014358839020133018,-0.0636199340224266,-0.12135197967290878,-0.0875491201877594,-0.10817506909370422,0.1466933637857437,-0.01959720067679882,-0.03574463352560997,0.07906337082386017,0.07306846976280212,-0.017434073612093925,0.07614043354988098,0.0008692551054991782,0.07061222195625305,0.014370927587151527,-0.06636673212051392,0.04090317711234093,-0.08275328576564789,-0.025606853887438774,-0.059604573994874954,0.011451167985796928,0.0048378524370491505,0.12835551798343658,0.14099612832069397,0.06892091035842896,-0.04102691635489464,0.0033372296020388603,-0.0497116856276989,-0.07755158096551895,0.1275917887687683,0.010640362277626991,-0.029014453291893005,-0.09619209170341492,-0.12056326866149902,-0.04659326747059822,-0.09252747893333435,-0.035723976790905,-0.07857254892587662,-0.023625170812010765,0.005505668465048075,-0.04308690130710602,-0.13846243917942047,-0.035469409078359604,-0.015006974339485168,-0.010398943908512592,-0.04876837879419327,-0.06607270985841751,-0.03579200431704521,-0.030060624703764915,0.0038663793820887804,0.00999426655471325,-0.039492640644311905,0.05378318578004837,-0.05247202143073082,0.008463628590106964,-0.011255880817770958,-0.012120524421334267,-0.01663309521973133,0.016921356320381165,-0.12124049663543701,-0.018573015928268433,-0.03416843339800835,-0.01500276941806078,-0.04917960986495018,0.09563113749027252,-0.1975991129875183,-0.031017262488603592,-0.0039003374986350536,-0.02432643435895443,-0.09158772230148315,0.12502172589302063,-0.1694665253162384,0.006126489490270615,-0.013243480585515499,0.012089317664504051,-0.006745121907442808,0.06617631763219833,-0.12464923411607742,-0.031989935785532,-0.004516884684562683,-0.06498641520738602,-0.056024741381406784,-0.05456527695059776,0.00511553417891264,-0.06994733214378357,-0.01771339774131775,-0.04479273408651352,0.022783732041716576,-0.11856793612241745,0.031801458448171616,0.017861975356936455,-0.012113025411963463,0.011184824630618095,0.025595344603061676,0.025904366746544838,-0.0396004356443882,-0.044502876698970795,-0.030356815084815025,-0.09122686088085175,0.02174633927643299,-0.011316980235278606,0.06193608045578003,-0.02418464794754982,0.0007789732189849019,-0.09195757657289505,0.10834653675556183,-0.06841973215341568,0.05340573564171791,-0.005233679432421923,0.019758768379688263,-0.03487256541848183,0.062042057514190674,0.01847301609814167,-0.018286652863025665,-0.011429582722485065,0.036111630499362946,-0.04246358200907707,0.006089791655540466,0.022329283878207207,0.03966307267546654,0.010424396023154259,0.047281209379434586,0.023510685190558434,0.034825317561626434,0.014461481012403965,-0.03957975655794144,0.03025084175169468,-0.09581615030765533,-0.05994673818349838,-0.12577813863754272,-0.027237599715590477,0.026675118133425713,-0.04914272576570511,-0.051162656396627426,-0.014525331556797028,-0.023626048117876053,0.031352605670690536,0.004425545688718557,-0.002262543886899948,0.03383719548583031,0.025236977264285088,-0.025934772565960884,0.004928139504045248,-0.0011571210343390703,0.019217394292354584,0.015725214034318924,-0.0016965538961812854,-0.024875862523913383,-0.062007687985897064,0.11756701022386551,-0.05405079573392868,0.031403400003910065,0.058699216693639755,0.04915747791528702,-0.039361126720905304,0.07513219118118286,0.02644360437989235,0.029633304104208946,0.024020414799451828,0.026841923594474792,-0.07030186057090759,0.011636711657047272,-0.04209152236580849,-0.06112553924322128,-0.0568150170147419,-0.06194240227341652,-0.06317328661680222,-0.04120243340730667,0.03876779228448868,0.04551095888018608,0.032194141298532486,-0.055402688682079315,-0.1358901560306549,0.10542215406894684,0.05595214292407036,-0.049286022782325745,-0.04144373908638954,-0.08364438265562057,0.0722450315952301,-0.0584244579076767,0.010108571499586105,0.02113063633441925,0.00094483548309654,0.09089481085538864,0.05331201106309891,-0.030287712812423706,-0.038798049092292786,-0.021869849413633347,-0.04220254719257355,-0.11586184799671173,-0.14618639647960663,-0.022671878337860107,0.010849189944565296,-0.017948457971215248,-0.03753725066781044,-0.03415240719914436,0.026000693440437317,0.009784734807908535,-0.0035707640927284956,0.02371242083609104,0.04935620725154877,0.05407010018825531,0.15872475504875183,0.030926205217838287,-0.006268047261983156,0.02726241759955883,0.01793217472732067,-0.018901173025369644,0.05884953588247299,-0.05521726235747337,-0.06073581427335739,-0.0075376261956989765,-0.05867849290370941,-0.12714427709579468,0.053261179476976395,-0.13584919273853302,0.02560943178832531,-0.014833295717835426,0.00691424822434783,-0.04790825396776199,-0.033081602305173874,-0.02081124484539032,-0.027574719861149788,-0.08801498264074326,-0.05679338797926903,-0.09174614399671555,-0.03314061090350151,-0.020046455785632133,0.0030851815827190876,-0.03256654739379883,0.03608047962188721,-0.003428220748901367,-0.09160961955785751,-0.1723024696111679,0.04429299011826515,-0.06381706148386002,0.012959246523678303,-0.011570130474865437,-0.027875671163201332,0.012400572188198566,-0.06903690099716187,0.004244004376232624,-0.013867867179214954,-0.009530328214168549,-0.0035843774676322937,0.07272769510746002,0.02013309672474861,0.026244251057505608,0.006924567744135857,0.01156124472618103,0.024541033431887627,0.01025326270610094,-0.06532396376132965,0.15887613594532013,0.10497421771287918,-0.026010919362306595,0.05833110958337784,0.1327156126499176,-0.0678795725107193,-0.073076993227005,-0.062164999544620514,0.2319113165140152,0.19395065307617188,0.21126878261566162,-0.00866692140698433,0.20268741250038147,-0.13317787647247314,-0.16517142951488495,0.025119798257946968,0.1850389540195465,-0.12383300065994263,0.010342257097363472,0.2042207419872284,0.21019452810287476,-0.10240092128515244,-0.10827936232089996,0.2754465937614441,0.2796039283275604,0.007641127333045006,-0.08989249169826508,0.13537056744098663,-0.017713090404868126,-0.09452234208583832,-0.11262716352939606,-0.058249715715646744,0.029217587783932686,0.14966443181037903,0.07140937447547913,0.06682658940553665,0.17270587384700775,-0.14333418011665344,-0.1917029470205307,0.1115349605679512,0.04363594204187393,0.16216501593589783,0.11639150232076645,-0.008430268615484238,-0.07501814514398575,0.1530643105506897,0.08368663489818573,0.27556392550468445,-0.07838069647550583,-0.2362992912530899,-0.09262967854738235,-0.23996108770370483,-0.18216632306575775,0.03407234326004982,0.15881414711475372,0.15090928971767426,0.1103942021727562,0.00850534439086914,-0.01727660559117794,-0.105401910841465,-0.025064535439014435,0.207062229514122,-0.25282949209213257,0.017604202032089233,0.05408588796854019,0.28620976209640503,-0.08336206525564194,-0.1413872092962265,0.00883965753018856,0.009708653204143047,0.10499986261129379,-0.05389411374926567,-0.02436693012714386,-0.18361903727054596,0.16830173134803772,-0.015814803540706635,-0.05392041429877281,-0.09487640857696533,-0.15580537915229797,-0.12919233739376068,0.2644427716732025,0.04496938735246658,0.016110273078083992,0.08714481443166733,0.04554268717765808,0.00984109379351139,0.2679480016231537,0.013255326077342033,0.04785946384072304,0.09719541668891907,0.16133105754852295,0.05017276108264923,0.04328865557909012,0.013973535969853401,0.027295876294374466,-0.03438001498579979,0.0023513678461313248,0.03756497800350189,0.4216269254684448,-0.03793877735733986,-0.05304295942187309,-0.06641528010368347,-0.006599195767194033,-0.08024319261312485,0.2993588447570801,0.02355785109102726,-0.015296454541385174,-0.03841632232069969,-0.014091984368860722,-0.021625645458698273,-0.1494770497083664,0.0028195292688906193,-0.015256653539836407,-0.02588263340294361,0.11173483729362488,0.010023842565715313,-0.14871585369110107,0.011446432210505009,-0.03281797841191292,-0.02064361236989498,-0.054927945137023926,-0.002191904466599226,-0.0036473588552325964,-0.010269270278513432,-0.011980528943240643,0.07381930202245712,0.04962092265486717,0.00705846818163991,-0.2137126326560974,0.020224960520863533,0.00903127621859312,0.014782779850065708,0.040872812271118164,0.029420031234622,-0.15649479627609253,-7.037684554234147e-05,-0.003931264393031597,-0.007746538147330284,-0.05158991739153862,-0.03845414146780968,-0.30996719002723694,-0.013664505444467068,-0.014716897159814835,0.03484766557812691,0.0456589013338089,-0.06132819503545761,-0.30154454708099365,-0.0033045518212020397,0.024005556479096413,-0.019125862047076225,0.07114080339670181,0.04721949249505997,-0.1366240680217743,-0.010713504627346992,0.019299117848277092,0.05899392068386078,0.0012391390046104789,-0.021581746637821198,0.11690468341112137,-0.011266382411122322,-0.0009851237991824746,0.06338829547166824,0.11475168913602829,0.03777994588017464,0.2583865225315094,-0.025615563616156578,-0.03145597502589226,-0.005006607621908188,0.02602469176054001,0.07071004062891006,-0.06972284615039825,-0.014027547091245651,0.0011021561222150922,0.01660172827541828,0.005495518445968628,0.07957131415605545,-0.03157396242022514,0.0023172965738922358,0.05583943426609039,0.0679989829659462,0.007397620007395744,0.04209393635392189,0.004593467339873314,0.04980049654841423,-0.04818281531333923,-0.06463588774204254,-0.003585473168641329,-0.055586304515600204,0.038724664598703384,0.004484930541366339,0.02912459522485733,0.024960311129689217,-0.012578100897371769,-0.009255505166947842,-0.0938715711236,-0.021205367520451546,-0.07112132757902145,-0.04620100557804108,-0.004160509444773197,-0.04203586280345917,0.05971437320113182,0.006160689517855644,-0.051190003752708435,0.018505403771996498,-0.03528041020035744,-0.07025990635156631,-0.09134504199028015,0.01320500299334526,0.024817388504743576,-0.023892071098089218,0.01345887128263712,0.002783101284876466,-0.02546784095466137,0.011387109756469727,-0.02840087190270424,-0.04923955351114273,-0.0372733511030674,-0.05800221860408783,0.0042753373272717,0.017283441498875618,0.008226130157709122,-0.0060755605809390545,-0.0006702296086587012,-0.009981612674891949,0.003262670710682869,0.03846284747123718,0.055883828550577164,-0.017179008573293686,-0.03110869601368904,-0.029582621529698372,-0.13265256583690643,-0.01574028842151165,0.007401286158710718,-0.012360083870589733,-0.0039245509542524815,0.008357971906661987,-0.10303306579589844,-0.02135605178773403,0.012395436875522137,-0.016031859442591667,-0.0018409674521535635,0.015952518209815025,-0.042818937450647354,0.023104950785636902,-0.036844104528427124,0.0029677567072212696,0.0012632940197363496,0.018349619582295418,0.008924543857574463,0.015763064846396446,0.04411347210407257,-0.0009238386410288513,-0.013067418709397316,-0.004641887731850147,0.005456381943076849,0.0036510752979665995,0.0026010230649262667,-0.030768902972340584,-0.006126063875854015,-0.046288736164569855,-0.010136779397726059,-0.004594739060848951,-0.019276339560747147,-0.04189266636967659,-0.02827327512204647,-0.036530714482069016,-0.008375723846256733,0.02307966537773609,-0.003910561557859182,-0.0026134802028536797,-0.00898828450590372,-0.021255122497677803,0.02764795534312725,0.0027016224339604378,0.014587233774363995,-0.025225237011909485,-0.054526425898075104,-0.020860645920038223,-0.007756691426038742,-0.018074968829751015,0.061471953988075256,0.037185657769441605,0.016017349436879158,0.02840600162744522,-0.04071745276451111,-0.020445290952920914,-0.011707429774105549,-0.014783419668674469,-0.03149424120783806,-0.051995791494846344,-0.03813716769218445,0.010104380548000336,-0.027328243479132652,-0.046011947095394135,-0.025085223838686943,0.02255261316895485,0.10300146043300629,0.011540054343640804,0.023092832416296005,0.04085602983832359,0.020388484001159668,-0.00390661833807826,-0.014421625062823296,-0.024694442749023438,0.017233919352293015,-0.012727584689855576,-0.02867792546749115,0.020049097016453743,0.05086226016283035,0.028582945466041565,0.045511238276958466,0.03082006610929966,0.017973702400922775,-0.07117453962564468,-0.06575281918048859,-0.0235992930829525,-0.06715473532676697,-0.07912005484104156,-0.04344988986849785,-0.10621320456266403,-0.031666334718465805,0.035982005298137665,-0.025668105110526085,-0.03272957727313042,-0.038759563118219376,-0.03173001855611801,-0.065443255007267,0.0007957794587127864,0.05362705886363983,0.08321051299571991,0.07215892523527145,0.12105834484100342,-0.008922199718654156,0.015066824853420258,-0.0008977591642178595,-0.016456272453069687,0.003211460541933775,-0.05661286413669586,-0.027298958972096443,-0.027855725958943367,-0.021997813135385513,-0.018729209899902344,-0.05721083655953407,-0.050199851393699646,-0.14120925962924957,0.001386547228321433,-0.016820108518004417,-0.05249881371855736,-0.03449896723031998,-0.07501670718193054,0.02681763842701912,-0.003522244282066822,-0.0211764108389616,-0.016892075538635254,0.031168881803750992,-0.01976669579744339,-0.04605674371123314,0.030342306941747665,0.0003911071107722819,-0.057895857840776443,0.010657798498868942,-0.012152360752224922,0.039510492235422134,0.018202930688858032,-0.007971891202032566,-0.027069540694355965,0.0368281714618206,0.0033808911684900522,-0.07257061451673508,0.04130469262599945,0.0035338029265403748,0.022290432825684547,-0.005506410729140043,0.017741847783327103,-0.10923159867525101,-0.01558088418096304,0.02536584809422493,0.02782558649778366,0.08936235308647156,0.09243134409189224,-0.16822735965251923,-0.04141257703304291,-0.030593283474445343,-0.0465930812060833,0.018819134682416916,0.024823570623993874,-0.14848920702934265,0.012064945884048939,0.026626646518707275,0.031227275729179382,0.0921349748969078,0.013600917533040047,-0.02619628608226776,0.026663968339562416,-0.007707123178988695,0.010027733631432056,-0.05085579305887222,-0.00025607561110518873,-0.0018018961418420076,0.016152935102581978,-0.0036466429010033607,-0.014069365337491035,0.011924530379474163,-0.023275135084986687,0.0526529885828495,0.02800046093761921,0.008248819969594479,0.07191342860460281,0.03255216032266617,0.10475650429725647,0.007159733679145575,0.003572821384295821,0.05127925053238869,0.09106400609016418,0.031080039218068123,0.033774398267269135,-0.031231163069605827,0.023863332346081734,0.07585986703634262,0.053149979561567307,0.05995570868253708,0.06757515668869019,0.02001543529331684,-0.002410219283774495,-0.03827187418937683,-0.09167781472206116,-0.09949976950883865,-0.05523030087351799,-0.05147944763302803,-0.020076341927051544,0.004441834520548582,0.00974587257951498,-0.01892356015741825,-0.01731269061565399,0.0020752723794430494,0.015825891867280006,-0.04178515449166298,-0.02300211228430271,0.024524301290512085,-0.00020285202481318265,0.010716147720813751,0.023692570626735687,-0.023461151868104935,0.022895732894539833,0.040310267359018326,0.012028972618281841,0.05486277490854263,0.010050774551928043,0.024718452244997025,0.05456969141960144,-0.033355824649333954,-0.002495300956070423,-0.0551062636077404,0.32384034991264343,0.05740165337920189,0.2288072556257248,-0.018799839541316032,-0.06008131429553032,0.09304626286029816,-0.034943610429763794,0.0021431229542940855,0.05812867730855942,-0.011449260637164116,-0.008923539891839027,-0.0003702272952068597,0.0017744494834914804,0.020470216870307922,0.002747128950431943,-0.0019427669467404485,-0.0447828508913517,0.10887808352708817,-0.01037090178579092,0.01047907117754221,-0.028654618188738823,-0.22154347598552704,-0.0589231513440609,0.034075573086738586,-0.06296106427907944,-0.16690383851528168,-0.10937616229057312,-0.059911079704761505,0.026183458045125008,0.29675260186195374,0.13925997912883759,-0.18089263141155243,0.28849953413009644,-0.4951404631137848,-0.07139864563941956,-0.1333490014076233,0.03282565250992775,0.18493101000785828,-0.17741818726062775,-0.06619701534509659,0.3115416169166565,0.1357364058494568,-0.02973918989300728,-0.03221359848976135,-0.046417370438575745,-0.19089193642139435,-0.02632455714046955,0.07870367169380188,-0.006930837873369455,-0.02814146876335144,0.09166011214256287,-0.02625725418329239,-0.03311838209629059,0.1577124446630478,0.06606908142566681,-0.07975238561630249,-0.09934893250465393,0.23962658643722534,-0.018103670328855515,0.06380940973758698,-0.0519668310880661,0.15514658391475677,0.19743973016738892,-0.006666420493274927,-0.1443309187889099,-0.056091759353876114,0.016904862597584724,-0.022156786173582077,-0.020759714767336845,0.012911469675600529,0.02168758027255535,0.15169799327850342,-0.2438567727804184,0.27096524834632874,0.11692735552787781,0.01668950542807579,-0.0025650160387158394,-0.04375584051012993,-0.10728627443313599,0.08916842192411423,0.1760166436433792,-0.46055474877357483,-0.44149264693260193,-0.4355415999889374,0.45008814334869385,0.12597042322158813,0.29080426692962646,0.12967431545257568,-0.0007177169900387526,-0.0015023433370515704,-0.0014871293678879738,-0.0011666457867249846,-0.001184009131975472,-0.001046700868755579,0.004357173107564449,0.003780151018872857,0.001407284289598465,0.0024313104804605246,0.003968934994190931,0.006011130288243294,-9.671866428107023e-05,0.0012445738539099693,-0.0001742750609992072,-0.0002588293282315135,-0.0014517673989757895,-0.00021682106307707727,-0.004804810043424368,-0.0026264030020684004,-0.00027317384956404567,0.002135993679985404,-0.001100421417504549,-0.0036930604837834835,-0.01283601950854063,-0.011893738992512226,-0.01298411376774311,-0.011282398365437984,-0.011287236586213112,-0.010997200384736061,-0.0008270450052805245,-0.0016836656723171473,-0.002867428818717599,-0.0020054862834513187,-0.0012785206781700253,-0.0007689651683904231,0.0008072691271081567,0.0002123953017871827,2.7885849704034626e-05,-0.0007423987262882292,0.0010557310888543725,0.0020169515628367662,-0.007053320296108723,-0.006052385084331036,-0.004679244477301836,-0.005232995375990868,-0.006047294940799475,-0.007322330493479967,0.036131225526332855,0.06754235178232193,0.03844776004552841,-0.0018937467830255628,-0.06530072540044785,-0.0009940976742655039,-0.03042731247842312,-0.026897303760051727,0.01937989704310894,0.02102578617632389,-0.11091316491365433,0.18038421869277954,0.016734614968299866,0.04742571339011192,-0.007891274057328701,-0.0025615792255848646,0.0019663621205836535,-0.011178247630596161,0.0010205438593402505,-0.02130780555307865,0.006220218259841204,0.04573565348982811,0.028288569301366806,0.024123936891555786,0.0496029406785965,-0.02272183634340763,0.08263137936592102,-0.034752506762742996,0.06491058319807053,0.21433031558990479,-0.009537843987345695,0.06174289062619209,-0.015455246903002262,-0.025339970365166664,-0.10193605720996857,0.03053002431988716,0.009406039491295815,0.019382771104574203,0.01680668257176876,0.021772217005491257,-0.029114343225955963,-0.13597165048122406,0.010820207186043262,-0.012096704915165901,0.03682258352637291,0.0008326457464136183,-0.011295560747385025,-0.07066529989242554,-0.009724373929202557,-0.007454246282577515,-0.002133263973519206,0.02207436040043831,-0.009454749524593353,0.07045317441225052,0.010986887849867344,0.009830099530518055,0.03808224946260452,0.01579977758228779,0.027650726959109306,0.10280333459377289,-0.005181522108614445,0.004327430855482817,-0.009334816597402096,0.019124675542116165,-0.003158650128170848,-0.04276302456855774,0.004914674907922745,0.002887718379497528,-0.005407662130892277,0.011022885330021381,0.029977954924106598,-0.003631043480709195,-0.03694247454404831,-0.034033965319395065,-0.030526485294103622,-0.03515869006514549,-0.03764266520738602,-0.031660836189985275,0.017256613820791245,-0.0140593983232975,0.01624342054128647,-0.031796373426914215,0.0167215708643198,0.0428328774869442,0.0038845243398100138,-0.01389898918569088,-0.008809769526124,-0.02478811703622341,-0.000280623760772869,-0.055443961173295975,-0.0020865562837570906,-0.0073470743373036385,4.5130807848181576e-05,0.030507463961839676,0.02599925361573696,0.048393648117780685,-0.024643000215291977,-0.031369078904390335,-0.021871542558073997,-0.02013094536960125,-0.024272536858916283,-0.03719685226678848,0.016427068039774895,0.032079968601465225,0.029536858201026917,0.044212706387043,0.05975734815001488,0.060693297535181046,-0.0022368845529854298,-0.010687095113098621,-0.0023534693755209446,-0.0008057219674810767,0.038103435188531876,-0.007154753431677818,0.015459882095456123,0.0052326335571706295,0.002485196804627776,-0.012349443510174751,-0.007943346165120602,0.04628206789493561,-0.049883242696523666,-0.043723221868276596,-0.05466083437204361,-0.06307879090309143,-0.07310879230499268,-0.09837859123945236,-0.007716701831668615,-0.012342182919383049,-0.00023893508478067815,0.011135012842714787,0.037814993411302567,0.09703768044710159,0.003248135559260845,-0.005667502526193857,-7.6119131335872225e-06,-0.02124187909066677,-0.0007359978626482189,-0.07926775515079498,-0.004689761437475681,-0.005603042896836996,-0.00021149574604351074,-0.0028025219216942787,0.025676634162664413,0.026296675205230713,0.013384179212152958,-0.03337012603878975,0.0003898422000929713,-0.0017859019571915269,0.002872214885428548,-0.05115750432014465,-0.00759153114631772,-0.014773867093026638,-0.006537782493978739,-0.016623197123408318,0.056965067982673645,-0.0711263045668602,-0.03278637304902077,-0.01656954362988472,0.00588449090719223,0.0019851813558489084,0.06629464775323868,-0.1473049372434616,0.0188283734023571,0.01025963295251131,-0.01355651579797268,-0.014349636621773243,-0.03048151545226574,-0.16898693144321442,-0.01960139349102974,-0.024465689435601234,-0.045816369354724884,0.017076916992664337,0.044562701135873795,-0.10304863005876541,-0.0183319840580225,0.011461992748081684,0.03344066068530083,0.025354385375976562,0.04862924665212631,-0.08014227449893951,-0.03011087141931057,-0.011816299520432949,-0.06530847400426865,-0.04441852122545242,-0.03359405696392059,0.04476216435432434,-0.02751282975077629,-0.009054088033735752,-0.014054201543331146,-0.013118612580001354,-0.036157041788101196,0.0705127939581871,0.03379319980740547,0.004779744427651167,0.018961116671562195,-0.011819004081189632,-0.00776713527739048,-0.03355734422802925,0.022431576624512672,0.02136053331196308,-0.03317442908883095,0.0167554821819067,-0.040335189551115036,-0.008922550827264786,0.006586425006389618,-0.00943142268806696,0.009597190655767918,-0.0820479616522789,-0.014260801486670971,0.016911836341023445,-0.03155089542269707,-0.046122681349515915,0.008328182622790337,-0.002950396854430437,-0.00331140891648829,-0.0025645324494689703,-0.012971608899533749,0.0033982382155954838,-0.035885922610759735,0.028008297085762024,-0.018198827281594276,0.018747340887784958,0.01353017520159483,-0.013156246393918991,-0.04771474003791809,-0.03846358135342598,-0.024182239547371864,0.010285215452313423,0.012799020856618881,0.007010072935372591,-0.017355622723698616,-0.008302450180053711,-0.010411073453724384,0.023745287209749222,-0.0031134190503507853,-0.00696741184219718,-0.008795435540378094,-0.01889779046177864,-0.017010394483804703,-0.047319814562797546,0.03929676488041878,0.09865379333496094,0.03079524263739586,-0.005985569208860397,-0.02597947046160698,0.006006944924592972,0.04322362691164017,0.05639735981822014,0.01304059848189354,0.0020685307681560516,-0.06522033363580704,0.01658301241695881,0.023779219016432762,0.0757085457444191,0.00854966789484024,-0.05748409032821655,-0.049294132739305496,0.023520538583397865,-0.01794280670583248,-0.043002814054489136,0.013365553691983223,0.09163524955511093,0.011998986825346947,-0.09225005656480789,-0.036135777831077576,-0.018003838136792183,-0.019605321809649467,-0.00517710717394948,-0.0020943735726177692,-0.01091750804334879,0.010227608494460583,0.015683168545365334,-0.06401423364877701,-0.05807584896683693,-0.05547381937503815,-3.041302079509478e-05,-0.00903942808508873,0.02499009482562542,-0.06851929426193237,-0.009981339797377586,-0.06303192675113678,0.05058113485574722,0.01002067606896162,-0.0025288863107562065,-0.009922904893755913,0.006438370794057846,-0.05701608210802078,-0.06350012123584747,-0.011756557039916515,-0.030582547187805176,-0.020709367468953133,-0.10613734275102615,-0.11775720119476318,-0.04509371519088745,0.05415428429841995,0.06494639813899994,0.07623860239982605,0.068124920129776,0.05138740316033363,0.21359124779701233,0.010959080420434475,0.004506295546889305,0.023977970704436302,0.008749152533710003,0.03223064914345741,0.07248643040657043,0.010802638716995716,0.023750370368361473,0.036196883767843246,0.054507799446582794,0.05563441663980484,0.16233976185321808,-0.09079817682504654,-0.11197284609079361,-0.12738293409347534,-0.15363238751888275,-0.1595509946346283,-0.043661344796419144,0.0027601458132267,-0.02477247826755047,-0.009025928564369678,-0.047184236347675323,-0.0485471747815609,0.15893088281154633,0.023323386907577515,0.015898600220680237,0.03902299329638481,0.015181075781583786,0.014060739427804947,-0.07002290338277817,-0.011577568016946316,-0.0164553914219141,-0.015601128339767456,-0.00936522614210844,0.001701227854937315,0.03567945584654808,-0.006332076620310545,0.05542360618710518,-0.034595951437950134,-0.0492388978600502,0.053190335631370544,0.017435956746339798,-0.018759507685899734,-0.22466257214546204,6.986634980421513e-05,0.033401262015104294,0.0010269680060446262,0.0006762804696336389,0.002849629847332835,0.012069636024534702,0.008682779967784882,-0.007065874990075827,-0.0004754163674078882,0.10349714010953903,-0.037319350987672806,-0.04857685789465904,0.061387915164232254,0.0032060437370091677,-0.03297891840338707,-0.08000912517309189,0.001330091035924852,-0.03731777146458626,0.054287832230329514,0.016867462545633316,0.08188928663730621,-0.01886831782758236,0.05386931449174881,-0.18328730762004852,0.0008138661505654454,0.17021824419498444,0.06802884489297867,0.0563177727162838,-0.07594571262598038,0.020835058763623238,0.06567062437534332,0.015323125757277012,0.001247580861672759,0.2124485820531845,0.0270035769790411,-0.00463011022657156,0.056474748998880386,0.05736837536096573,0.13539747893810272,-0.15721645951271057,-0.0010172014590352774,-0.14546255767345428,-0.06950214505195618,-0.07987423241138458,0.012380760163068771,0.009133999235928059,0.11570782214403152,-0.16118581593036652,2.0291014152462594e-05,0.10616502165794373,-0.0710366815328598,-0.14344899356365204,0.11781319230794907,0.08445493131875992,0.0614783950150013,-0.3548583388328552,-0.0005394205218181014,0.04311952739953995,-0.012677731923758984,-0.04862979054450989,0.06437526643276215,0.03581008315086365,-0.0400652177631855,-0.09869465976953506,-1.5382069250335917e-05,-0.06306391209363937,0.08985069394111633,0.09737201780080795,0.15738922357559204,-0.09249473363161087,-0.23127955198287964,0.26733583211898804,0.0360252782702446,-0.1093512699007988,-0.40473464131355286,-0.34347063302993774,0.5204105377197266,-0.02695140428841114,0.1077716276049614,-0.13752277195453644,-0.012027970515191555,-0.0009656342444941401,-0.007294847629964352,-0.005928021855652332,-0.02127714455127716,-0.01139768585562706,0.008933060802519321,0.024999519810080528,0.018202204257249832,0.013219671323895454,-0.0025374984834343195,-0.011544888839125633,0.000950838380958885,0.008451850153505802,0.006041769869625568,0.012599455192685127,0.000535289291292429,-0.008287016302347183,0.002433371264487505,-0.017209501937031746,-0.007355046924203634,-0.010319184511899948,0.007780869025737047,0.017187507823109627,-0.013380125164985657,-0.01673216186463833,-0.014641203917562962,-0.018708130344748497,-0.018299534916877747,-0.019254084676504135,-0.014168894849717617,0.0036340453661978245,0.0026603781152516603,0.005404083523899317,-0.005960721988230944,-0.03663785010576248,0.012585502117872238,0.008928160183131695,0.004413208924233913,0.014566916972398758,0.000640926300548017,0.01881290040910244,-0.0034847718197852373,0.0023004314862191677,-0.0030329639557749033,-0.0033241775818169117,-0.009665943682193756,0.01169495563954115,0.045006949454545975,0.044832970947027206,0.06865164637565613,0.08859699964523315,0.04272197559475899,-0.08359025418758392,-0.07341466844081879,-0.08136450499296188,-0.07250934839248657,-0.029633430764079094,-0.07515868544578552,0.10920011252164841,-0.009228693321347237,0.013355782255530357,0.0196871105581522,-0.00364358676597476,-0.014109908603131771,0.10017065703868866,-0.019905390217900276,-0.032605499029159546,-0.017650963738560677,-0.008306660689413548,-0.012748136185109615,0.1179981604218483,0.09879148006439209,0.0820140466094017,0.04080786556005478,0.023983566090464592,0.029468988999724388,0.23846709728240967,-0.009172619320452213,0.026927104219794273,0.02548927627503872,0.007279256824404001,-0.03432939574122429,-0.05702497065067291,-0.014710525050759315,-0.020854230970144272,0.012894706800580025,-0.011827139183878899,-0.046417489647865295,-0.13724079728126526,-0.013913856819272041,-0.008107266388833523,-0.019823087379336357,-0.025154100731015205,-0.047817736864089966,-0.4362754821777344,-0.0029319687746465206,-0.007316363975405693,-0.001535222982056439,0.002743668854236603,-0.004569237120449543,0.036284297704696655,0.014335150830447674,0.008211868815124035,0.013603364117443562,0.015782561153173447,0.01986389420926571,0.03185148537158966,0.0033366840798407793,0.019871054217219353,0.010030549019575119,0.019365951418876648,0.00024782904074527323,-0.026958193629980087,-0.0010177107760682702,-0.004168285522609949,-0.0015560690080747008,-0.008135901764035225,-0.004323793575167656,-0.015914740040898323,-0.028452429920434952,-0.03274759650230408,-0.031196309253573418,-0.0310064647346735,-0.02506825514137745,-0.03646254539489746,-0.00926891528069973,-0.0003607178805395961,-0.006501144263893366,-0.008697712793946266,0.0011150272330269217,-0.015453745611011982,0.004952720832079649,0.011132892221212387,0.0024368988815695047,0.00020679431327152997,-0.00027657512691803277,0.0158024150878191,-0.006564124021679163,0.0012549017556011677,0.0021123234182596207,0.00098882254678756,-0.001028920989483595,0.026913804933428764,-0.012420705519616604,-0.03993004187941551,-0.034949254244565964,-0.04911472648382187,-0.05129415914416313,0.07714542746543884,0.043638925999403,0.050678789615631104,0.050752196460962296,0.04578956961631775,0.05354948341846466,0.023321473971009254,0.009613347239792347,0.00749908247962594,0.006008389871567488,0.005525137763470411,0.011904650367796421,-0.04502938687801361,0.003286748891696334,0.006206225603818893,0.011143133975565434,0.025895945727825165,-0.002356864744797349,-0.032967690378427505,-0.07392466068267822,-0.06337420642375946,-0.06335271894931793,-0.05469824746251106,-0.06691723316907883,-0.1756727695465088,-0.0277230367064476,-0.0027444225270301104,-0.02110311947762966,-0.022616585716605186,-0.0177915021777153,-0.10282685607671738,0.0073113287799060345,0.015120670199394226,0.009179254062473774,0.024174826219677925,-0.0027747848071157932,0.0694555938243866,-0.013317924924194813,0.005363975185900927,-0.005443797912448645,0.004399384371936321,-0.004284627735614777,0.2826424837112427,-0.008879391476511955,-0.019379062578082085,-0.023846453055739403,0.005295868963003159,0.024855153635144234,-0.02059110254049301,-0.018324295058846474,-0.005811769515275955,-0.018583577126264572,-0.08788924664258957,0.0041537173092365265,-0.1966736763715744,-0.010861099697649479,-0.03872577100992203,-0.009548051282763481,-0.013626058585941792,0.026083985343575478,-0.08443372696638107,-0.015330503694713116,-0.004479409195482731,-0.011504552327096462,-0.019413897767663002,-0.054316867142915726,-0.18256531655788422,0.06806981563568115,0.051231469959020615,0.014543664641678333,0.0577910840511322,-0.014499184675514698,0.08665765821933746,0.02149626426398754,0.020070137456059456,0.027138831093907356,0.06965509057044983,0.014862318523228168,0.0332174226641655,-0.028740981593728065,-0.01024872437119484,-0.031202783808112144,-0.0032953682821244,-0.0441431887447834,0.055769722908735275,0.031481727957725525,0.006952918600291014,0.009926814585924149,-0.01748296245932579,-0.013675786554813385,0.09201657772064209,0.024998119100928307,-0.015799004584550858,-0.00909122359007597,-0.01298392191529274,0.010862947441637516,-0.012651165015995502,-0.0004886696697212756,-0.020956015214323997,0.0017585533205419779,-0.005487194284796715,0.025542056187987328,0.0065523176454007626,-0.009373046457767487,-0.008649931289255619,-0.010156149044632912,-0.015214857645332813,0.015813926234841347,-0.02686590701341629,0.007045924197882414,0.03938356041908264,0.032408278435468674,0.021287869662046432,0.010954983532428741,-0.08196472376585007,-0.014722101390361786,-0.02973288670182228,-0.011363928206264973,0.011591416783630848,0.0071730646304786205,-0.022715888917446136,-0.005037238355726004,-0.018489038571715355,-0.021995360031723976,-0.01992650516331196,-0.014724666252732277,0.013180569745600224,-0.007525261491537094,-0.020493600517511368,-0.024687448516488075,0.004081751685589552,0.010941972024738789,-0.015409472398459911,-0.003875367110595107,0.004947967361658812,0.010511724278330803,-0.001933566527441144,-0.011150209233164787,0.005757358856499195,-0.03244985267519951,-0.01536429114639759,-0.010610855184495449,-0.03429598733782768,0.028887391090393066,0.17890600860118866,0.04546188935637474,0.04548240080475807,0.02455967850983143,0.029625272378325462,0.032891444861888885,-0.025972837582230568,-0.004811686463654041,-0.015900909900665283,-0.0020362879149615765,0.004055106081068516,0.04353160038590431,-0.18396307528018951,0.03339555114507675,0.017821790650486946,0.02306828275322914,0.02920634113252163,-0.01965559646487236,-0.08696206659078598,-0.01791800744831562,-0.01840970665216446,-0.04350195825099945,-0.03306206688284874,-0.06263533234596252,-0.47869107127189636,-0.018432488664984703,-0.005862923804670572,-0.0050299265421926975,-0.00045519685954786837,0.05213655158877373,-0.10759004950523376,0.017039278522133827,0.01854037307202816,0.003984678536653519,0.011192177422344685,0.008680981583893299,0.2686369717121124,0.023699576035141945,0.009327944368124008,0.0022592602763324976,0.02411695383489132,0.0011177699780091643,0.42475396394729614,-0.019755983725190163,-0.015827201306819916,-0.025880590081214905,-0.03230320289731026,-0.10247258841991425,-0.22532197833061218,0.06812979280948639,0.07029152661561966,0.08247550576925278,0.08732536435127258,0.1244267001748085,0.4034559428691864,0.00303309946320951,0.008681987412273884,0.013035147450864315,0.009126292541623116,-0.014143147505819798,0.3006867468357086,0.027675345540046692,0.02509661391377449,0.022843671962618828,0.024481644853949547,0.05425669997930527,0.5114333033561707,-0.13945667445659637,-0.14238597452640533,-0.16858607530593872,-0.1738055944442749,-0.1967606097459793,-0.2161826491355896,-0.017418285831809044,-0.016791438683867455,-0.02565300278365612,-0.021597320213913918,-0.0466265007853508,0.06264626979827881,0.03444599360227585,0.033084459602832794,0.031164545565843582,0.03190530464053154,0.034543607383966446,-0.22097395360469818,-0.032145630568265915,-0.0329652763903141,-0.03655678406357765,-0.03675997257232666,-0.02131369151175022,-0.22273191809654236,-0.010670570656657219,-0.002251485362648964,-0.016729604452848434,-0.1299920529127121,0.05099371820688248,-0.0015923093305900693,-0.09839268773794174,-0.49201035499572754,-0.007650628685951233,0.02866385132074356,-0.0005278514581732452,0.00921799149364233,-0.028258105739951134,-0.0008993669762276113,0.014354047365486622,-0.03215295448899269,-0.004482197109609842,-0.04635161906480789,-0.0292667206376791,-0.10245048254728317,-0.09897272288799286,0.04831860959529877,-0.13185229897499084,0.10509029030799866,0.015846654772758484,0.2757180333137512,0.045333512127399445,0.21684114634990692,-0.11512438207864761,-0.031805846840143204,0.19252294301986694,-0.05307061970233917,-0.00033737948979251087,0.19266913831233978,0.0611385703086853,0.10612770915031433,0.013855021446943283,-0.07304496318101883,0.2759372591972351,-0.07874979078769684,0.021693363785743713,0.313888818025589,0.03541017323732376,0.25033095479011536,-0.09081139415502548,0.04633219167590141,0.4282185733318329,-0.2538493871688843,0.015894832089543343,-0.03511681407690048,-0.026888763532042503,0.00514141283929348,-0.2329917997121811,0.03389052674174309,0.13370902836322784,-0.2330711930990219,0.016321439296007156,-0.033705487847328186,0.02216581255197525,0.12234263122081757,-0.2427009493112564,0.04020953178405762,0.017350230365991592,-0.7607975006103516,-0.011648779734969139,-0.1153607964515686,-0.018579166382551193,-0.13795948028564453,0.07298921793699265,0.024401739239692688,-0.2646816372871399,0.08563793450593948,-0.019630707800388336,0.14424510300159454,-0.023996589705348015,-0.22164395451545715,0.08962574601173401,0.030287668108940125,-0.0014492584159597754,0.3086436688899994,0.08149601519107819,0.1026722639799118,-0.08894044160842896,0.18849089741706848,0.10177157819271088,0.022631747648119926,0.2977038323879242,-0.11729063838720322,-0.026545511558651924,-0.020205268636345863,-0.011338689364492893,0.0004439836193341762,0.0070957583375275135,-0.0007829495007172227,-0.009377414360642433,0.0013728301273658872,0.002318847458809614,-0.005788236390799284,0.0029422189109027386,-0.007775275502353907,0.006428872235119343,0.0031921544577926397,0.004110562149435282,0.0005393375759012997,0.0064140199683606625,-0.0006685293628834188,-0.0002129747881554067,-0.0033688670955598354,-0.018475988879799843,-0.017805781215429306,-0.012759219855070114,-0.013948505744338036,0.0008233948610723019,-0.00017973042849916965,-0.0015773362247273326,-0.0058532459661364555,-0.006847324315458536,-0.0071027204394340515,0.003438417799770832,0.008762193843722343,0.015477228909730911,0.009630244225263596,0.009796480648219585,0.007122197188436985,0.018318375572562218,0.011298125609755516,0.008382638916373253,0.011347067542374134,0.007453115191310644,0.011949761770665646,-0.016023939475417137,-0.008600503206253052,-0.008764756843447685,-0.005555711220949888,-0.005020061507821083,-0.010729702189564705,0.003082503331825137,0.01678338274359703,0.0239393413066864,-0.0004175352514721453,0.007295905612409115,0.03130064532160759,0.009366114623844624,-0.008174014277756214,-0.006067280657589436,0.005655215587466955,0.020751548931002617,-0.018977032974362373,-0.00790982786566019,-0.004751404747366905,-0.017592865973711014,-0.018237341195344925,0.0003384941373951733,0.11559604853391647,-0.008682508021593094,0.028567111119627953,0.023563958704471588,0.01996820606291294,0.010978175327181816,-0.11087924242019653,0.03842242807149887,0.002141364151611924,0.045718882232904434,0.02538074180483818,0.013722001574933529,0.13459357619285583,0.0012463204329833388,0.020150206983089447,0.002023976529017091,0.030550803989171982,-0.007263506297022104,0.07290609180927277,-0.01274342741817236,0.01748267561197281,0.02401459775865078,0.022549016401171684,0.030786940827965736,-0.05140236020088196,0.01815005950629711,-0.00044093444012105465,0.006989309098571539,0.014824247919023037,0.016302259638905525,-0.03393807262182236,-0.021953480318188667,-0.010870929807424545,0.011033615097403526,0.00027169176610186696,-0.001733611454255879,-0.013515914790332317,0.061514150351285934,-0.00441261800006032,0.011432921513915062,0.010666067712008953,-0.0052563962526619434,0.0014709393726661801,0.060549039393663406,-0.0014684913912788033,0.0011008535511791706,0.009044084697961807,0.014662775211036205,-0.02411401830613613,0.003962303511798382,-0.007147164084017277,-0.006356744095683098,-0.023011093959212303,-0.016394231468439102,-0.0366431288421154,-0.045287832617759705,-0.0186262596398592,-0.020534783601760864,-0.01772458478808403,-0.020463017746806145,-0.007220660801976919,-0.05451728031039238,0.0026900211814790964,-0.0052290381863713264,-0.013959884643554688,0.005679448135197163,0.016353538259863853,-0.06874381005764008,0.0018881511641666293,-0.012368062511086464,-0.010610383003950119,-0.005117003805935383,-0.02256612852215767,-0.007147676777094603,-0.007391055580228567,-0.0003587240935303271,-0.010723447427153587,-0.011171354912221432,-0.02168293483555317,-0.023696165531873703,-0.0353531651198864,-0.04137608781456947,-0.03455103933811188,-0.039840735495090485,-0.06979554891586304,0.041917137801647186,0.04220709949731827,0.055400945246219635,0.033107053488492966,0.0540478453040123,0.29550856351852417,-0.01826162077486515,-0.013502314686775208,-0.017340850085020065,-0.0052286963909864426,-0.010385899804532528,0.3395656943321228,0.011286926455795765,0.017177052795886993,0.022408556193113327,0.016161639243364334,0.016339005902409554,0.18795683979988098,-0.09077873080968857,-0.1247495636343956,-0.13312454521656036,-0.11443488299846649,-0.1621071994304657,-0.33122536540031433,0.0056897616013884544,0.018688494339585304,0.021786199882626534,0.03755834698677063,0.03413363918662071,-0.2619089186191559,0.0075889467261731625,0.012009995058178902,0.02198474481701851,0.01438799686729908,0.012775738723576069,-0.20163442194461823,-0.028571683913469315,-0.03571575507521629,-0.027075527235865593,-0.021629683673381805,-0.04080863296985626,0.0036792417522519827,0.02429022453725338,0.02520567551255226,0.03558589518070221,0.00381839438341558,0.012416169047355652,0.11329364776611328,-0.024684779345989227,-0.04764651507139206,-0.059654463082551956,-0.030782010406255722,-0.020594296976923943,-0.15515677630901337,-0.003253053640946746,0.016305187717080116,0.019418152049183846,0.019070357084274292,-0.002989896573126316,-0.18096058070659637,-0.01741752028465271,-0.02189628593623638,-0.015994790941476822,-0.014528991654515266,-0.005054633133113384,-0.21104280650615692,0.10518760234117508,0.10236150026321411,0.1332811862230301,0.09158821403980255,0.07002231478691101,0.07106218487024307,-0.026071328669786453,-0.02379642240703106,-0.029491353780031204,-0.014186069369316101,-0.02261546440422535,-0.07480907440185547,-0.003701298963278532,-0.02109985612332821,-0.0007796994759701192,-0.0024311980232596397,0.004564558155834675,0.3259615898132324,0.017332278192043304,0.015105688944458961,0.021366730332374573,0.024706419557332993,-0.0046970960684120655,-0.041616249829530716,0.0018916052067652345,0.004431080538779497,-0.007134702522307634,-0.017598647624254227,-0.008114156313240528,0.06789607554674149,0.004934767726808786,0.015018780715763569,0.020986342802643776,0.015878520905971527,0.016868675127625465,-0.11467868089675903,0.00418785959482193,-0.010310572572052479,-0.025179829448461533,0.011077314615249634,-0.03171389549970627,-0.15072868764400482,0.006566979922354221,0.007934977300465107,0.016651488840579987,-0.013659263961017132,-0.015020935796201229,-0.051258914172649384,0.008523311465978622,-0.02891879715025425,-0.003026098944246769,0.057212237268686295,0.10032673180103302,0.2117156982421875,0.013949035666882992,0.008905455470085144,-0.0031755694653838873,-0.018109427765011787,-0.03348146378993988,0.11250492185354233,0.024130869656801224,0.008029126562178135,0.005082415882498026,0.014627039432525635,-0.016922475770115852,0.18250590562820435,0.014445389620959759,-0.00509860273450613,0.013043373823165894,0.03556514158844948,0.03592145815491676,0.05319822207093239,-0.014749848283827305,-0.018045106902718544,-0.003533882787451148,-0.036141037940979004,-0.033029742538928986,0.04970943555235863,0.007825423963367939,0.00527762807905674,0.026150988414883614,0.007450041361153126,0.018844641745090485,-0.10057497024536133,0.0012480939039960504,0.005365450866520405,-0.02563183568418026,-0.01687581278383732,-0.042634572833776474,-0.085434190928936,0.0053018187172710896,-0.00420865835621953,-0.010201070457696915,-0.00835945550352335,0.003383565228432417,-0.06482721120119095,-0.00847671739757061,0.02404898591339588,0.040657833218574524,0.03422173485159874,-0.00613854406401515,0.09829875826835632,0.0031380264554172754,0.012728487141430378,-0.01681172475218773,0.014011838473379612,-0.014345948584377766,0.047280360013246536,0.002313393633812666,0.012525878846645355,-0.018862247467041016,-0.016388963907957077,-0.03670124337077141,0.10266240686178207,0.006851153448224068,0.020624808967113495,0.014458376914262772,0.029903870075941086,0.003212863579392433,-0.02130332589149475,-0.001632216852158308,-0.005078581627458334,-0.002252056961879134,-0.01051438506692648,0.004270539153367281,0.009181133471429348,0.0025647575967013836,-0.013229059055447578,-0.01623939722776413,0.004010828677564859,-0.0012463410384953022,-0.03013615310192108,0.024342099204659462,0.0009989055106416345,0.005801463034003973,0.0033691718708723783,-0.00862880703061819,0.020273033529520035,-0.016042031347751617,0.001380637171678245,0.008111122995615005,-0.008150926791131496,-4.836211883230135e-05,-0.031267717480659485,-0.02827337011694908,8.75612113304669e-06,-0.003987882286310196,-0.011696937493979931,0.009118126705288887,0.05452730134129524,-0.02648676186800003,0.006016386207193136,0.013025335036218166,-0.014785119332373142,-0.008545378223061562,0.0426894873380661,-0.03374844789505005,-0.016786420717835426,-0.010796577669680119,-0.03577887639403343,-0.01640980876982212,0.03571911156177521,-0.0029665224719792604,-0.0027208449319005013,-0.004239776637405157,-0.006307879462838173,0.0017785732634365559,-0.06357885897159576,-0.004398129880428314,0.08649712055921555,-0.006843851879239082,-0.40418511629104614,0.2193661630153656,0.02143564447760582,0.04007400572299957,0.009117241017520428,0.00039674845174886286,0.010600625537335873,0.0024327703285962343,0.004289061762392521,-0.03467223793268204,-0.015875760465860367,-0.00502247828990221,-0.014967819675803185,-0.0032376362942159176,0.07231656461954117,0.07791513949632645,-0.0171032901853323,-0.005609450861811638,-0.09499490261077881,-0.0904051810503006,0.012418505735695362,-0.01614142209291458,0.1069798618555069,0.004986390471458435,-0.20627719163894653,-0.01265384629368782,-0.07440222799777985,-0.14791545271873474,-0.014991261065006256,-0.006613737903535366,-0.13716739416122437,-0.10392069071531296,-0.3266824185848236,-0.07710051536560059,-0.24621011316776276,0.10539334267377853,-0.036993201822042465,-0.02672642469406128,0.2950358986854553,0.0072238389402627945,-0.3043290376663208,-0.2950894832611084,-0.12129011750221252,-0.097808338701725,-0.009252864867448807,0.016628451645374298,-0.008804102428257465,-0.0060527147725224495,-0.2312251776456833,-0.31871548295021057,-0.13646791875362396,-0.33482980728149414,0.00891477894037962,-0.016581224277615547,-0.057967495173215866,-0.007429846096783876,-0.2608758807182312,-0.3134474754333496,-0.04957820102572441,-0.08766614645719528,-0.01623961143195629,-0.002170978346839547,-0.05478057265281677,0.0373687706887722,0.23878084123134613,-0.30586785078048706,0.041466087102890015,-0.03043214976787567,0.0038139193784445524,0.01899062469601631,0.0615614578127861,-0.063657246530056,0.2118789553642273,0.03783612698316574,0.06571612507104874,-0.07368813455104828,0.0032133387867361307,0.15195757150650024,0.26315367221832275,-0.15021435916423798,-0.23178720474243164,0.3314002454280853,0.20011559128761292,0.20511016249656677,-0.011754634790122509,-0.02712460421025753,-0.028812522068619728,-0.021774670109152794,-0.013428599573671818,-0.009802744723856449,-0.18908104300498962,0.0017674177652224898,-0.007167277857661247,-0.0020905614364892244,-0.016774088144302368,-0.015188619494438171,0.16581647098064423,-0.01896098628640175,-0.01935132034122944,-0.021983902901411057,-0.007928888313472271,-0.007642799522727728,0.20991303026676178,0.012094804085791111,0.014259996823966503,0.0015117130242288113,0.0011873382609337568,0.010910194367170334,0.28119394183158875,-0.02773914858698845,0.0006176666938699782,0.012787437997758389,0.016479289159178734,0.003247660817578435,0.1994500458240509,0.01972213201224804,0.022902270779013634,0.02369515970349312,0.014304774813354015,0.03044845536351204,0.21903188526630402,-0.006891601253300905,0.011405831202864647,-0.009470164775848389,-0.004464953672140837,0.01117696799337864,-0.06863971799612045,-0.012828823179006577,0.0035318585578352213,0.008391150273382664,0.0008358244085684419,0.012343944050371647,-0.2810300290584564,0.015868758782744408,0.026619266718626022,0.009403449483215809,0.005343059543520212,0.0056792753748595715,0.008483323268592358,-0.011557596735656261,-0.018489088863134384,-0.002326695714145899,0.0008874799241311848,-0.016832849010825157,0.2097475230693817,-0.006169695872813463,0.004429447930306196,0.004026907030493021,0.0051796650514006615,0.01216746587306261,0.07925064116716385,-0.011045442894101143,-0.017470598220825195,-0.0016359760193154216,-0.0051810601726174355,-0.01272442378103733,0.20559509098529816,0.029847102239727974,0.03648819029331207,-0.00964722316712141,-0.00021859616390429437,0.028679631650447845,0.12505532801151276,0.023006154224276543,0.009344608522951603,0.00581847969442606,0.004247021861374378,-0.00661345012485981,0.2493021935224533,-0.016130926087498665,-0.003154967911541462,-0.007759385742247105,-0.006037765182554722,-0.013714863918721676,0.05571872740983963,0.009774582460522652,0.0018669009441509843,-0.005493384785950184,0.0011501424014568329,0.009508872404694557,-0.21854469180107117,-0.015909958630800247,-0.022559359669685364,-0.014203714206814766,-0.01975480280816555,-0.04161371290683746,-0.034520138055086136,0.009031765162944794,0.016205141320824623,0.006721631158143282,0.016246089711785316,0.013799998909235,-0.0370243676006794,0.0002908736059907824,-0.0015163321513682604,-0.002630413742735982,-0.0057387687265872955,0.00016646803123876452,0.10852034389972687,0.01412018109112978,0.009310293011367321,0.010060546919703484,0.016037415713071823,0.009027758613228798,-0.02587748132646084,-0.020045971497893333,-0.0341462641954422,-0.027751753106713295,-0.025602128356695175,-0.0031191406305879354,-0.3371497094631195,0.021000035107135773,0.012655777856707573,0.016702987253665924,0.022641245275735855,-0.0027121305465698242,-0.000925111526157707,0.01760009489953518,0.008007078431546688,0.008706634864211082,0.03106098249554634,0.02700568176805973,0.10771982371807098,0.012405438348650932,-0.003943834453821182,-0.003227428300306201,0.0077136000618338585,0.007753213867545128,0.17200404405593872,-0.001162484404630959,0.005536541808396578,-0.003831373294815421,-0.010984768159687519,-0.01164997462183237,-0.0625501498579979,0.016895735636353493,-0.0031558037735521793,0.018566789105534554,-0.0003228500427212566,0.008415203541517258,0.11723798513412476,-0.011091604828834534,0.00671829329803586,-0.015131352469325066,0.006399251986294985,-0.006945624481886625,0.10241631418466568,0.013210863806307316,0.004969676956534386,0.018634669482707977,0.0028749865014106035,0.01602715626358986,0.042615365236997604,-0.05793879181146622,-0.046614766120910645,-0.04390506073832512,-0.044991206377744675,-0.055667050182819366,-0.26896730065345764,0.018125474452972412,0.007031367626041174,0.002954442286863923,0.005321734584867954,0.010989156551659107,-0.07558699697256088,0.006365771871060133,0.003044563578441739,0.02728951722383499,-0.001977144740521908,0.004478458780795336,-0.12304619699716568,-0.023697834461927414,-0.02103637345135212,-0.019493967294692993,-0.015956595540046692,-0.02241840772330761,-0.009477013722062111,-0.005011569708585739,-0.0023715277202427387,-0.0014585006283596158,-0.010064738802611828,-0.00986920390278101,-0.02051685005426407,0.005138781853020191,0.010240563191473484,0.005455935839563608,0.010998550802469254,0.011339549906551838,0.0019034616416320205,0.006479279603809118,0.007766073569655418,0.0012409116607159376,-0.011701599694788456,0.0006066272617317736,-0.013624866493046284,0.010124250315129757,-0.014169740490615368,0.0022415239363908768,-0.019203422591090202,-0.009560960344970226,-0.015505583956837654,-0.01008171122521162,-0.022156640887260437,-0.014230460859835148,-0.015163511969149113,-0.007973117753863335,0.003611486405134201,-0.010560947470366955,-0.0006890548975206912,-0.009137864224612713,0.010710394009947777,-0.006270510144531727,0.014693278819322586,0.014756498858332634,-0.01897304691374302,-0.005935434252023697,-0.0017419991781935096,-0.0028464868664741516,0.0068581136874854565,0.0050703296437859535,0.0031739219557493925,-0.004787822254002094,-0.003244978841394186,-0.012686926871538162,-0.01965964399278164,0.020933043211698532,0.00567541504278779,0.012125859037041664,0.028207147493958473,-0.017141083255410194,-0.1744791716337204,-0.004629814997315407,0.002988577587530017,0.0017846679547801614,-0.02428993582725525,-0.003939291462302208,0.07348188757896423,0.026423748582601547,0.024881502613425255,-0.0008840751252137125,0.0013406961224973202,-0.012079261243343353,0.2032223641872406,0.008108788169920444,-0.002477293135598302,0.005997294094413519,-0.008516641333699226,-0.011907004751265049,0.08527792245149612,0.009548635222017765,0.0027526109479367733,-0.008250858634710312,0.04971688240766525,0.06959973275661469,0.19321444630622864,-0.034941669553518295,-0.04325171560049057,-0.00942303892225027,-0.013472477905452251,-0.05151617154479027,0.1851106584072113,0.013038337230682373,-0.03663887828588486,0.0020501618273556232,-0.011638915166258812,-0.018556702882051468,-0.03308527171611786,0.01969805918633938,0.016460347920656204,-0.008670840412378311,0.00517646549269557,-0.005023967009037733,-0.22728928923606873,0.0025120978243649006,-0.004916861653327942,-0.003276708535850048,-0.00045036798110231757,-0.0018983223708346486,-0.0106576606631279,0.00037586045800708234,-0.00036815786734223366,0.0014337713364511728,-0.0017369374400004745,0.0037869936786592007,-0.003508207853883505,0.0008699579630047083,-0.0007891279528848827,3.9895039662951604e-05,-0.002189223887398839,-0.0016127126291394234,-0.005674536805599928,-0.004505063407123089,-0.0004095376643817872,0.002598297316581011,-0.0024647084064781666,-0.005666812881827354,-0.00012533363769762218,-0.008478887379169464,-0.006575928535312414,-0.005766304209828377,-0.008005916140973568,-0.005704541690647602,-0.0049801417626440525,0.001319232047535479,0.0028044991195201874,0.0022306658793240786,0.002130963373929262,0.006082391366362572,0.0005748647381551564,0.002788594691082835,0.0012097223661839962,0.0004747257044073194,7.829568176020985e-07,0.0008170068613253534,0.0012021177681162953,-0.0067033967934548855,-0.004339857492595911,-0.0027932506054639816,-0.00040530681144446135,-0.005727042444050312,-0.0042037381790578365,-0.0325690321624279,-0.033025167882442474,-0.029095236212015152,-0.004359835758805275,0.02585461363196373,0.1558196097612381,0.0010718766134232283,0.0012304730480536819,-0.00017594416567590088,-0.00788195338100195,0.0002681251789908856,-0.006937244441360235,0.0037248735316097736,0.008109713904559612,0.0009705563425086439,0.005135017447173595,0.004966109059751034,-0.051635850220918655,-0.0070120664313435555,-0.00814203079789877,0.001815221505239606,0.0011182436719536781,0.00020766216039191931,-0.16653338074684143,-0.008197431452572346,-0.00458485959097743,-0.00043527010711841285,-0.005758611019700766,-0.004412022419273853,0.08292457461357117,-0.019680948927998543,-0.008029911667108536,-0.0037175267934799194,-0.006192241795361042,0.007771321106702089,-0.04915972799062729,-0.0031528864055871964,0.0031356739345937967,0.0022447877563536167,0.005897652357816696,0.019741566851735115,-0.007092892192304134,-0.019770152866840363,-0.0066357264295220375,0.0009242536034435034,-0.011237076483666897,-0.00031291061895899475,-0.036054544150829315,-0.009490390308201313,0.04454319179058075,-0.14344407618045807,-0.09262167662382126,-0.005439334083348513,0.10316701978445053,-0.005402575712651014,0.04045027494430542,-0.03417664393782616,-0.03589441999793053,0.006944905035197735,0.01600528694689274,-0.0005000384408049285,-0.009232064709067345,-0.0012836161768063903,-0.032565172761678696,0.31944796442985535,0.1147516593337059,-0.07885687053203583,-0.03683696687221527,0.0021758023649454117,0.21625493466854095,-0.0007823983323760331,0.00016299735580105335,-0.3866589665412903,-0.2701027989387512,0.16677051782608032,-0.07996061444282532,0.001010906184092164,-0.20125874876976013,-2.494525870133657e-05,-0.07684823125600815,0.1413268744945526,-0.0676613599061966,0.1448773741722107,-0.11269301921129227,-0.0010270284255966544,-0.036474477499723434,0.001020739902742207,-0.17534609138965607,0.046931322664022446,0.049699816852808,0.2522965371608734,-0.19420622289180756,9.36902069952339e-05,0.21413934230804443,-0.002592954318970442,-0.08578614890575409,-0.006470244377851486,0.25084051489830017,0.023197438567876816,-0.16868534684181213,-0.0004041173669975251,0.07769311219453812,-0.0005570069770328701,0.0672110840678215,-0.01035800389945507,0.2645187973976135,0.028372682631015778,-0.156899556517601,0.014177107252180576,0.18452279269695282,-0.0012914196122437716,-0.07305104285478592,-0.21380624175071716,0.006759717594832182,-0.09076949954032898,0.10561054199934006,-0.00971096009016037,-0.0037739642430096865,-4.345384513726458e-05,-0.06348466128110886,0.20926183462142944,0.3075309693813324,-0.19104304909706116,0.12277371436357498,0.009178010746836662,0.0408317930996418,0.0022792562376707792,0.03203177452087402,0.17176684737205505,-0.11259521543979645,-0.35137367248535156,0.3232383728027344,0.00489308126270771,-0.1087421104311943,-0.01968010514974594,-0.06790177524089813,0.009032142348587513,-0.0010736305266618729,-0.0044992826879024506,0.020921872928738594,0.03549931198358536,0.02440084144473076,-0.02335752360522747,0.008662139065563679,0.06109528988599777,0.04475764185190201,0.02334425412118435,0.00391411455348134,-0.016076333820819855,-0.017760412767529488,0.09866464883089066,0.019785793498158455,-0.0022279610857367516,-0.0070306663401424885,-0.0628795400261879,-0.050468478351831436,-0.014871526509523392,-0.04147215187549591,0.0015238169580698013,0.021420201286673546,-0.10442133992910385,-0.06293515115976334,0.04419853165745735,0.07801366597414017,0.1634724736213684,0.003523360239341855,-0.029901638627052307,0.015971986576914787,-0.04433447867631912,-0.07145191729068756,-0.06343861669301987,-0.0579061396420002,-0.03743641451001167,-0.024524236097931862,-0.04228759557008743,-0.09967479854822159,-0.08292784541845322,-0.004007382318377495,-0.06218444183468819,0.20631438493728638,0.04738672077655792,0.005453793797641993,0.05283493548631668,-0.04213472828269005,0.007118762005120516,0.04866643249988556,-0.0015765646239742637,0.05832501873373985,0.04906352609395981,0.031333401799201965,-0.02684597671031952,-0.07600664347410202,0.09984074532985687,-0.05275941267609596,0.1638685166835785,0.022765235975384712,0.05180586129426956,0.07494926452636719,0.01657692901790142,0.04393694922327995,0.08670371770858765,0.07473915815353394,0.02959679812192917,0.09771443903446198,0.06123519316315651,0.097243532538414,0.06319811195135117,0.1348593384027481,0.13716758787631989,0.17297229170799255,0.15238867700099945,-0.0552583746612072,-0.050015632063150406,-0.017484428361058235,-0.10974915325641632,-0.059621699154376984,-0.042973875999450684,-0.06142856553196907,-0.07937204092741013,-0.05644693225622177,-0.03106319159269333,-0.05056711286306381,0.06844779849052429,0.008712275885045528,0.2854726016521454,0.21227024495601654,0.06990769505500793,-0.024497751146554947,-0.021146951243281364,-0.011892599053680897,-0.04243822395801544,-0.007158999796956778,0.11200359463691711,0.10727641731500626,0.011202416382730007,0.005258957855403423,-0.008463370613753796,-0.06377919018268585,0.08844880014657974,-0.03498105704784393,-0.045316025614738464,-0.010135403834283352,-0.06366336345672607,0.024836590513586998,0.009311939589679241,-0.11339234560728073,-0.044270195066928864,-0.02035589888691902,-0.02228844352066517,-0.02046523615717888,0.04320363700389862,-0.04934864491224289,-0.0020315430592745543,-0.039478424936532974,-0.4484596848487854,-0.3573192358016968,0.02072061225771904,0.006241136230528355,-0.008643066510558128,-0.025653984397649765,-0.018307527527213097,-0.022895287722349167,-0.02992766909301281,0.0031919570174068213,0.013418409042060375,0.015336349606513977,0.0005665046628564596,-0.003987452480942011,-0.13899953663349152,-0.027835221961140633,-0.0677252933382988,-0.005235329270362854,0.03666262701153755,0.07221091538667679,0.03147352486848831,0.06416007876396179,0.052701227366924286,0.06690747290849686,0.049457814544439316,-0.006865810602903366,-0.014562240801751614,-0.09796477854251862,0.0978560745716095,0.02087939716875553,-0.007793975528329611,0.0409303680062294,-0.0708586648106575,-0.014540131203830242,0.053385376930236816,-0.015604247339069843,-0.005711701232939959,0.03823912888765335,0.031079387292265892,0.014079133979976177,-0.012754845432937145,-0.0061613707803189754,0.030126867815852165,0.23909008502960205,-0.129447802901268,-0.1386338323354721,-0.000952943111769855,0.047946881502866745,-0.012241201475262642,-0.02218584343791008,0.0059920260682702065,0.0020224680192768574,0.011887815780937672,0.004770399536937475,-0.005474959034472704,-0.00043587147956714034,0.06177964806556702,0.01637204922735691,0.2738669216632843,-0.3320842981338501,0.0031582233496010303,0.05219680443406105,-0.032030247151851654,-0.02462816797196865,-0.027628418058156967,-0.033883754163980484,-0.046397339552640915,-0.05268830060958862,-0.0546773262321949,-0.03871842846274376,-0.07464800029993057,-0.0745149701833725,0.04939286410808563,0.030866943299770355,-0.2965913414955139,-0.21161960065364838,-0.10504398494958878,0.040216974914073944,0.017849313095211983,0.004633635748177767,0.04347923398017883,0.050261471420526505,0.03196004405617714,-0.027668051421642303,-0.0031497592572122812,-0.0049618822522461414,-0.007638953626155853,0.19530925154685974,0.09323450177907944,-0.20537935197353363,-0.048278726637363434,-0.050208285450935364,-0.001740866107866168,0.018940821290016174,-0.0009784211870282888,0.017967935651540756,0.006626039277762175,0.023416684940457344,0.007757667917758226,-0.020815156400203705,-0.014285615645349026,-0.02369057387113571,0.058063484728336334,-0.06452993303537369,0.09408998489379883,0.006946200039237738,-0.023007625713944435,0.04033752903342247,-0.03714264929294586,-0.04481453076004982,0.009066510945558548,-0.00040030138916336,-0.019821930676698685,-0.026695851236581802,-0.03309103474020958,-0.0017334340373054147,0.025602271780371666,-0.03848261386156082,-0.03630199283361435,0.016636217013001442,0.13935358822345734,0.04890183359384537,0.0012612721184268594,-0.004774066619575024,-0.0005703559145331383,-0.009764838963747025,-0.017928414046764374,0.015366952866315842,0.05131206661462784,0.02827977016568184,-0.10013136267662048,0.014069626107811928,0.06587398052215576,-0.14478835463523865,0.14574749767780304,0.06318755447864532,-0.0669894814491272,0.04084852337837219,-0.02597675286233425,-0.049144063144922256,-0.01007337961345911,0.05210178717970848,0.0622822642326355,0.030390089377760887,0.0813068151473999,0.14136768877506256,0.013456535525619984,-0.2909480631351471,-0.19324445724487305,0.11431511491537094,0.01592925563454628,0.009437940083444118,0.0337473563849926,-0.012585349380970001,-0.0005188602954149246,-0.01649763248860836,0.018301691859960556,0.01680050790309906,-0.015308761037886143,-0.0010623064590618014,0.0974615216255188,0.04129894822835922,-0.3627839982509613,-0.4354844093322754,0.15604080259799957,0.17937228083610535,0.07686243951320648,0.007965468801558018,0.033357977867126465,-0.008714400231838226,-0.036039941012859344,0.022189460694789886,-0.020821934565901756,0.08385638892650604,0.10400684922933578,0.050793860107660294,0.23719479143619537,-0.28524214029312134,-0.4119008183479309,0.5429198145866394,-0.030542928725481033,-0.22438956797122955,0.033297453075647354,-0.011585349217057228,-0.04379564896225929,-0.05596126988530159,-0.057471297681331635,-0.00046743417624384165,0.030463404953479767,-0.028187084943056107,-0.11465198546648026,-0.022184589877724648,-0.11953360587358475,-0.016353251412510872,0.4190024733543396,-0.011828912422060966,-0.1333330273628235,-0.03817526251077652,5.9401485486887395e-05,0.0456552691757679,0.047559674829244614,0.013654332607984543,0.014564408920705318,0.08535926789045334,0.0463232658803463,-0.124600350856781,0.01939363405108452,-0.03799408674240112,0.04599132761359215,0.47953081130981445,-0.1458190530538559,-0.25140658020973206,0.06757404655218124,0.0002977115800604224,-0.0427696630358696,-0.07034855335950851,0.024735113605856895,0.10000304877758026,0.058994270861148834,0.07467655837535858,0.20012643933296204,0.039805080741643906,-0.04274270683526993,0.16805370151996613,-0.23343956470489502,-0.22763675451278687,-0.08497442305088043,0.02571697346866131,-0.009990670718252659,-0.04910166561603546,0.001011169864796102,0.05508054420351982,0.05911380425095558,0.04101235792040825,-0.03248968347907066,-0.1652161329984665,0.002780792536213994,0.15065132081508636,0.017432963475584984,-0.13848745822906494,-0.066679447889328,-0.10550049692392349,-0.018008798360824585,-0.0032373478170484304,-0.025009015575051308,0.07517451047897339,0.0662408247590065,0.031286634504795074,-0.024363284930586815,-0.04761592298746109,0.014824174344539642,0.044735562056303024,0.019028816372156143,0.023057846352458,-0.07302039116621017,-0.06771501898765564,-0.029278989881277084,-0.012796374969184399,0.0341913141310215,0.06591010093688965,-0.03583098575472832,-0.07230281829833984,0.02351371943950653,0.04465736448764801,0.01112980954349041,-0.10281161963939667,-0.07660385221242905,-0.030231960117816925,-0.006218943744897842,0.04852260276675224,0.063754603266716,0.08438859134912491,0.018505584448575974,0.015647923573851585,-0.05036013573408127,0.036249324679374695,0.07818640023469925,0.0730864405632019,0.10652021318674088,-0.11422695964574814,-0.19125017523765564,-0.20144563913345337,-0.02459464594721794,0.04983546957373619,-0.020247293636202812,-0.07012714445590973,0.003715149825438857,0.030206948518753052,0.028968634083867073,-0.05560540035367012,0.03325316682457924,0.03036307543516159,0.07673677057027817,0.021585695445537567,-0.18551118671894073,-0.2453170269727707,-0.13148623704910278,0.049141280353069305,0.06955021619796753,-0.017402783036231995,-0.04410203918814659,0.0674619972705841,-0.06303410232067108,-0.005741264205425978,-0.06784141063690186,-0.043253783136606216,-0.02450767531991005,-0.02995321713387966,0.020319299772381783,-0.14590339362621307,-0.11425599455833435,-0.11929789185523987,0.007796081248670816,0.015909399837255478,-0.07660529017448425,-0.0708458423614502,-0.001827541971579194,0.011993465945124626,0.03579680621623993,-0.035290710628032684,-0.03996037691831589,-0.04494910314679146,-0.0034410813823342323,0.011134175583720207,-0.0658465102314949,0.01949402317404747,0.009507790207862854,0.07534515857696533,0.022317523136734962,-0.046019863337278366,-0.026888662949204445,0.02148324064910412,-0.0014706864021718502,-0.01903780922293663,0.031454525887966156,-0.05312711372971535,-0.013759756460785866,0.019229883328080177,0.039509762078523636,0.0864214077591896,0.04582773894071579,-0.016579005867242813,-0.00839849840849638,0.03596969693899155,0.04963548853993416,0.024528425186872482,-0.01069438923150301,-0.00786446500569582,-0.03637250140309334,-0.02981105074286461,0.06362726539373398,0.06973766535520554,0.0011685066856443882,-0.1089792475104332,0.04811510071158409,0.04886971786618233,0.13326309621334076,-0.009313655085861683,-0.04743654280900955,-0.07157152146100998,-0.039685577154159546,-0.06384340673685074,-0.00024580300669185817,0.016786562278866768,-0.015596971847116947,0.018841631710529327,0.05203045904636383,0.051604900509119034,0.030530165880918503,0.007500890642404556,0.016652308404445648,-0.020670665428042412,0.005907850805670023,-0.02250882424414158,-0.03313060849905014,0.01867230050265789,0.04082389920949936,-0.013324730098247528,0.004006627481430769,0.015934959053993225,0.023635251447558403,-0.019274886697530746,-0.06736381351947784,-0.021377213299274445,0.01993658021092415,0.03192127123475075,0.06933468580245972,0.039277564734220505,-0.07556113600730896,-0.10821374505758286,-0.09461396187543869,-0.05615644529461861,-0.021018803119659424,0.03846155107021332,0.04051230847835541,-0.007221213076263666,0.006562618538737297,-0.011734415777027607,0.003233977360650897,0.020078057423233986,-0.002210065023973584,-0.03174963220953941,-0.04490583389997482,-0.10057878494262695,-0.1608307659626007,-0.11870212852954865,-0.11101749539375305,-0.04356009513139725,-0.02293519116938114,-0.0014875628985464573,0.018159329891204834,-0.014651189558207989,-0.060725193470716476,-0.06771345436573029,-0.0019134461181238294,0.01821543090045452,-0.03513655066490173,-0.014890899881720543,-0.08227479457855225,-0.2179698944091797,-0.014995794743299484,0.09565306454896927,-0.01781008206307888,0.005745338276028633,0.03293602541089058,0.03692299872636795,0.04352165013551712,0.024333856999874115,-0.0026488967705518007,0.03867173194885254,0.07888875901699066,0.08927752077579498,0.06338834017515182,0.03224058076739311,-0.05573195964097977,0.12459956109523773,0.21874654293060303,-0.028447655960917473,0.014270147308707237,0.011074233800172806,0.02254507690668106,-0.0076225497759878635,-0.08385786414146423,-0.08700849115848541,-0.01631256751716137,-0.023499870672822,-0.017961563542485237,-0.033163636922836304,-0.01873963698744774,-0.09129147231578827,0.0618085078895092,0.05292261391878128,0.007982275448739529,-0.028624355792999268,0.028765948489308357,0.020351851359009743,-0.0046743168495595455,-0.02633654698729515,0.000594956858549267,0.0004047682450618595,0.03745286166667938,0.08647754788398743,0.0827285647392273,0.11986818164587021,0.05994819104671478,0.16583578288555145,0.202241450548172,0.034128349274396896,0.012136561796069145,-0.05208297818899155,-0.02291422337293625,-1.2383115972625092e-05,0.005632825195789337,-0.011708274483680725,0.01269055437296629,-0.011183304712176323,-0.05243301019072533,-0.08782224357128143,-0.034690022468566895,0.08260893076658249,0.09464582055807114,-0.03661808371543884,-0.012633657082915306,-0.016706425696611404,0.005854385439306498,-0.0068300762213766575,-0.03366502746939659,-0.019697822630405426,-0.06350100785493851,-0.05017933249473572,-0.018156686797738075,0.0003148428222630173,0.012922445312142372,0.013956605456769466,0.012912306003272533,-0.053699515759944916,0.09405864775180817,0.039684001356363297,-0.015801001340150833,0.009704023599624634,0.0042159175500273705,0.014777122065424919,0.027026081457734108,0.04424402490258217,0.045260775834321976,0.06699145585298538,0.05103264003992081,0.014280534349381924,-0.009232155978679657,0.0010084682144224644,0.009837860241532326,0.02278927154839039,-0.005587826017290354,0.021435828879475594,-0.03320985659956932,-0.01755640096962452,0.031800806522369385,-0.005420750472694635,0.019485102966427803,0.008618392050266266,0.039455756545066833,0.027855364605784416,0.04003989323973656,0.013352463021874428,-0.06448543816804886,0.012387103401124477,0.025186294689774513,0.01887282356619835,0.0055850353091955185,0.0064216116443276405,0.01768563687801361,0.005419604945927858,0.033780355006456375,0.008476964198052883,0.006264130584895611,0.03602079302072525,0.02377491444349289,0.02969076670706272,-0.060399819165468216,-0.061967916786670685,-0.030044790357351303,0.12130268663167953,0.004048209637403488,0.0005669641541317105,-0.005859861616045237,0.022400377318263054,0.006137377116829157,-0.010772798210382462,-0.010255779139697552,-0.03633556142449379,-0.025082161650061607,-0.03046177141368389,-0.029147394001483917,-0.09032991528511047,-0.09360188245773315,0.07119237631559372,0.07445354014635086,0.01884239912033081,0.010007091797888279,-0.002526373602449894,-0.024889769032597542,-0.054089341312646866,0.000303660926874727,-0.004426295403391123,-0.016387417912483215,-0.034753259271383286,-0.05119795724749565,-0.037566620856523514,-0.09905236214399338,-0.07116705179214478,0.11598580330610275,0.11676919460296631,0.033301614224910736,0.008951899595558643,-0.00658645061776042,0.004004702437669039,0.009917089715600014,-0.01305923517793417,0.024062365293502808,0.0019422938348725438,-0.0357896164059639,-0.05199192836880684,-0.05427829548716545,-0.041898854076862335,-0.016568053513765335,0.09062937647104263,-0.027781851589679718,0.028614675626158714,-0.0025802634190768003,0.055289726704359055,0.021618248894810677,-0.03622308373451233,0.013063323684036732,-0.027594275772571564,-0.015261244028806686,-0.025161802768707275,-0.014438553713262081,0.002057356759905815,0.022751281037926674,0.04322964325547218,-0.12369180470705032,0.0004996996140107512,0.0679653137922287,0.04244536906480789,0.046459048986434937,0.02114293724298477,0.04213527590036392,0.04220610857009888,0.054431889206171036,0.06018141657114029,-0.036999646574258804,0.04904709756374359,0.10271641612052917,-0.021155616268515587,-0.04905085638165474,0.035493288189172745,0.026236414909362793,-0.0032747434452176094,0.018729032948613167,0.020846085622906685,-0.042867206037044525,-0.07425066828727722,-0.0814436748623848,-0.012460040859878063,0.0699155330657959,0.2008974403142929,0.12282922863960266,-0.07610638439655304,-0.05940327048301697,-0.02799377590417862,0.07609880715608597,0.024624023586511612,0.0390436127781868,0.06569073349237442,-0.06228114664554596,-0.03337765112519264,0.025706972926855087,-0.02793310210108757,0.000516651023644954,-0.01038855966180563,0.01579216867685318,-0.03184516355395317,-0.003883598605170846,-0.17484639585018158,-0.2129206359386444,-0.06466397643089294,-0.12815213203430176,-0.009006532840430737,-0.013226493261754513,0.032145772129297256,-0.030728012323379517,-0.024075858294963837,-0.04648258164525032,0.018244339153170586,0.007451681420207024,-0.015204407274723053,-0.012223713099956512,0.02526276931166649,-0.1999179720878601,-0.20385116338729858,0.1431330144405365,0.09569183737039566,0.015940265730023384,-0.00285102310590446,-0.018519267439842224,0.03052337095141411,0.016334807500243187,0.022306108847260475,0.09334888309240341,0.08500320464372635,0.07247290760278702,0.022227196022868156,0.06653016060590744,-0.053318463265895844,-0.04445794224739075,0.02111450396478176,0.12009390443563461,-0.008502312004566193,0.009139233268797398,-0.006169437896460295,-0.03152458369731903,0.0027251343708485365,-0.01781635358929634,-0.015766555443406105,0.0018789094174280763,-0.09404724836349487,-0.09405659139156342,-0.0699772834777832,-0.09640995413064957,-0.04486609622836113,0.023850636556744576,-0.01541417371481657,0.016061559319496155,-0.02034919336438179,0.006783000193536282,-0.011188683100044727,-0.05702543631196022,0.046288833022117615,0.04722260683774948,0.04800772666931152,0.0343734435737133,-0.0038690457586199045,-0.00759310694411397,0.0636598989367485,0.0765727236866951,-0.05137530714273453,-0.024569885805249214,-0.012633733451366425,-0.012409688904881477,0.09112878888845444,-0.012641570530831814,0.014558717608451843,0.04239673167467117,0.05225488170981407,-0.005417924374341965,-0.13778358697891235,-0.08684936165809631,0.0032129769679158926,0.07715631276369095,0.01697571948170662,0.1605330854654312,0.026043904945254326,-0.13702772557735443,-0.04258185252547264,-0.04151517525315285,-0.10592132806777954,-0.04305613413453102,-0.011044962331652641,-0.04383455216884613,-0.10017465054988861,-0.13250820338726044,-0.08641991764307022,0.0617302767932415,0.059902217239141464,0.02439969964325428,-0.014207038097083569,-0.05324753373861313,0.047247789800167084,-0.06331691890954971,-0.05171172693371773,0.028109483420848846,0.04918011277914047,-0.013023610226809978,0.0016022383933886886,0.08567860722541809,0.06047052517533302,0.15614987909793854,0.11588876694440842,-0.04927179589867592,0.04212775453925133,-0.04933364689350128,0.03351037576794624,0.02549860253930092,-0.03446432575583458,0.014097166247665882,0.04593803733587265,0.014710024930536747,-0.01584213599562645,0.05309710651636124,0.07702634483575821,-0.015780985355377197,-0.01050171721726656,0.08830628544092178,0.21947738528251648,0.3123164474964142,-0.11436212062835693,0.05577157437801361,0.043960824608802795,0.0276560727506876,0.05580577626824379,-0.005114040803164244,0.029845068231225014,0.029703408479690552,0.03437308594584465,0.000780405243858695,-0.0838000625371933,0.05060361698269844,0.15637202560901642,0.1858912855386734,0.26653367280960083,-0.12905724346637726,-0.09841093420982361,-0.08133187144994736,-0.041622862219810486,-0.06524843722581863,-0.0908282920718193,-0.044962383806705475,0.003943218383938074,-0.021545754745602608,-0.1285324990749359,-0.15172791481018066,-0.033817827701568604,0.09565208852291107,0.0643923431634903,0.12148914486169815,-0.12518411874771118,-0.2028277963399887,-0.02647521160542965,0.10882695019245148,0.054854393005371094,-0.012065844610333443,0.03310234472155571,-0.00814872607588768,-0.022957991808652878,-0.04424378648400307,-0.09600616991519928,-0.018952419981360435,0.1111828088760376,0.2265302687883377,0.24209192395210266,-0.07326038926839828,-0.12453185021877289,-0.02009468898177147,-0.03720021992921829,-0.06758967787027359,0.016413716599345207,0.06116345524787903,0.04543657228350639,-0.025114785879850388,-0.03491205722093582,0.04209785908460617,0.005828904453665018,-0.07594149559736252,-0.13668090105056763,-0.11088578402996063,0.03374417871236801,-0.17912907898426056,-0.036145299673080444,-0.00362368649803102,0.0389019139111042,-0.05541828274726868,0.013143360614776611,0.0773455947637558,-0.048265304416418076,-0.034097377210855484,-0.07553543150424957,-0.24215096235275269,-0.07372858375310898,-0.047844935208559036,-0.2474737912416458,0.03295346349477768,0.02314911037683487,0.15025638043880463,-0.07346706092357635,0.018693849444389343,-0.020299552008509636,0.03951149061322212,0.012035111896693707,0.015556097030639648,-0.08958655595779419,0.03867821767926216,-0.006065695080906153,-0.011436272412538528,-0.0016457809833809733,-0.005105283576995134,0.001913450425490737,0.01898857206106186,0.006782500538975,-0.015379663556814194,-0.1911546289920807,0.4989303648471832,-0.2583763599395752,-0.17385053634643555,-0.16686387360095978,-0.16974705457687378,-0.007702530361711979,0.3958096206188202,0.33749911189079285,-0.050441037863492966,0.10230699181556702,-0.15661895275115967,0.12893156707286835,0.059168703854084015,-0.16845910251140594,0.18968869745731354,0.527280330657959,-0.27073952555656433,-0.10973590612411499,0.1198183223605156,0.16820381581783295,0.34342682361602783,-0.5733219981193542,0.1922038495540619,0.0660361796617508,0.06643257290124893,-0.12720750272274017,-0.23949991166591644,0.019153030589222908,-0.20022505521774292,-0.05184537544846535,0.0036745339166373014,0.15908488631248474,0.14605511724948883,0.009559208527207375,-0.24306952953338623,-0.10886012017726898,-0.375943660736084,-0.13521762192249298,0.07959312945604324,0.14437739551067352,-0.22665832936763763,0.16893106698989868,0.06274231523275375,0.14294983446598053,0.25122877955436707,0.1299838125705719,0.05796927958726883,-0.042061273008584976,0.6377348303794861,-0.32525357604026794,-0.2281484454870224,-0.21313656866550446,0.08129087090492249,-0.13185204565525055,0.11930844187736511,0.26630428433418274,-0.5213972330093384,0.1533105969429016,-0.05467516556382179,0.008790643885731697,-0.15712371468544006,-0.5434730052947998,0.13307052850723267,-0.2698586881160736,-0.167836531996727,0.03053177148103714,-0.11220739781856537,0.10381901264190674,0.17233292758464813,0.22368469834327698,-0.02731272764503956,0.0003847597981803119,0.0002025673456955701,-0.0005968130426481366,-0.005686185322701931,-0.015172899700701237,-0.03256845101714134,-0.028957830742001534,-0.028850452974438667,-0.005380199756473303,-0.014252711087465286,-0.04855092987418175,-0.03707082197070122,-0.06950711458921432,0.03610139340162277,-0.052632562816143036,-0.03819306567311287,-0.04373105242848396,-0.0576605424284935,-0.032655179500579834,-0.04069929197430611,-0.035724908113479614,-0.03608335554599762,-0.029499046504497528,-0.030341703444719315,-0.021372584626078606,-0.00931282714009285,-0.0419493094086647,-0.04062116518616676,0.06385928392410278,-0.035421986132860184,0.0011597559787333012,-0.0016503602964803576,-0.0087974164634943,-0.027519021183252335,-0.019804049283266068,-0.017613694071769714,-0.012969267554581165,-0.03149194270372391,-0.043276961892843246,-0.041496120393276215,-0.030537910759449005,-0.013657603412866592,-0.0527033731341362,0.07405754178762436,-0.04531211405992508,-0.06597504019737244,-0.04976290836930275,-0.05495257303118706,-0.03965674340724945,-0.04251749441027641,-0.04614423215389252,-0.058051347732543945,-0.03588768094778061,-0.047998737543821335,-0.051947589963674545,-0.03170478716492653,-0.08568184077739716,-0.08915887773036957,0.16121618449687958,-0.027529286220669746,-0.022070836275815964,-0.015872035175561905,-0.015007054433226585,-0.035404372960329056,-0.023025711998343468,-0.012977361679077148,-0.027432965114712715,-0.027789216488599777,-0.026996392756700516,-0.053266339004039764,-0.03632933273911476,-0.048963166773319244,-0.014102356508374214,0.09886056929826736,-0.015669139102101326,-0.026960112154483795,-0.030136022716760635,-0.01363524328917265,-0.03529401496052742,-0.03934936970472336,-0.02377624437212944,-0.0433259978890419,-0.02631218731403351,-0.02498074620962143,-0.013204936869442463,-0.021256649866700172,-0.027059873566031456,-0.03442781791090965,0.14240235090255737,-0.020315831527113914,-0.013099526055157185,0.0020409259013831615,0.00341456918977201,-0.008121552877128124,0.011321792379021645,0.005491833668202162,-0.007060042582452297,-0.006039062049239874,-0.016253432258963585,-0.016273727640509605,0.01885836198925972,0.06692717969417572,0.004178709350526333,-0.038882289081811905,-0.026844242587685585,-0.01642383635044098,-0.008139500394463539,-0.02561834640800953,-0.021818330511450768,-0.0046693915501236916,-0.01769920624792576,-0.02436772547662258,0.023495234549045563,0.0022939718328416348,-0.01364198699593544,0.007850591093301773,-0.0032618239056319,0.0036786627024412155,-0.018461918458342552,-0.02675444632768631,-0.01623074896633625,0.01631895639002323,0.0018762757536023855,-0.008690256625413895,-0.017751086503267288,-0.03924376890063286,-0.034709203988313675,-0.024643464013934135,-0.023173600435256958,-0.020854774862527847,-0.037344127893447876,-0.03339289873838425,-0.02421504072844982,0.005039446987211704,-0.07206223160028458,-0.07489041239023209,-0.06903980672359467,-0.07199300825595856,-0.06057683378458023,-0.043299827724695206,-0.03790188953280449,-0.033530283719301224,-0.06127605214715004,-0.050769656896591187,-0.011942666955292225,-0.030923768877983093,-0.05754953250288963,-0.03166356310248375,0.13157591223716736,-0.011857160367071629,-0.000439525640103966,-0.00550020532682538,-0.0010504539823159575,0.0013633273774757981,-0.038157835602760315,-0.0046639516949653625,-0.01323426328599453,-0.027473757043480873,-0.021949904039502144,-0.026838546618819237,-0.020833957940340042,-0.04354383423924446,-0.018903544172644615,0.02343989722430706,-0.04210372641682625,-0.04370225593447685,-0.04066857323050499,-0.05767348036170006,-0.04601634293794632,-0.022545550018548965,-0.043853286653757095,-0.08569125831127167,-0.05402249097824097,-0.045231111347675323,-0.07879488170146942,-0.05780917778611183,-0.05943315476179123,-0.03879617899656296,0.10452710092067719,-0.02859879471361637,-0.007192287594079971,-0.010390875861048698,0.0022069441620260477,-0.022593822330236435,-0.028780095279216766,-0.01096052210777998,-0.013212976045906544,0.0057943714782595634,-0.027974456548690796,-0.032183002680540085,-0.009649860672652721,-0.013386651873588562,-0.045400381088256836,0.11289513111114502,-0.02718566544353962,-0.03610913082957268,-0.05012928694486618,-0.03507176786661148,-0.03715195879340172,-0.05835316330194473,-0.028984570875763893,-0.027351178228855133,-0.014961129985749722,-0.0032146438024938107,-0.022233257070183754,0.004894159268587828,-0.06694118678569794,-0.02058415487408638,0.13342593610286713,-0.008008631877601147,0.034543078392744064,-0.0026641099248081446,0.01592221111059189,0.03213882073760033,-0.006013339385390282,0.015453863888978958,0.013343056663870811,0.014837611466646194,-0.00729095796123147,-0.017350956797599792,0.002532800193876028,0.056589748710393906,0.027629278600215912,-0.05129623785614967,-0.021835485473275185,0.0030281522776931524,-0.019259091466665268,-0.018240947276353836,-0.010227302089333534,0.01782318763434887,0.0004870293487329036,-0.00697367126122117,0.024058492854237556,-0.014804973267018795,0.025725599378347397,0.025434650480747223,0.013501688838005066,0.022357085719704628,0.035747744143009186,-0.0187082402408123,-0.007090229541063309,-0.015489708632230759,-0.005930323153734207,-0.009893577545881271,-0.016656989231705666,-0.013684076257050037,-0.01799461990594864,-0.025248082354664803,-0.013497349806129932,0.002766847610473633,0.010732556693255901,0.10283198207616806,0.23426543176174164,0.12085016071796417,-0.023600907996296883,-0.021887587383389473,-0.021187670528888702,0.004582571797072887,-0.013224197551608086,-0.013537846505641937,-0.029975665733218193,-0.02562306448817253,-0.005572075489908457,-0.0216959398239851,-0.007683716248720884,-0.013692203909158707,-0.0602504201233387,-0.06055460870265961,-0.15699099004268646,0.01621411368250847,0.0010148726869374514,0.010232468135654926,0.009275061078369617,0.006710771471261978,0.005752404220402241,0.01694796048104763,-0.002099730772897601,0.010323612950742245,0.02363624982535839,-0.002303563989698887,0.01608552783727646,0.0034289455506950617,-0.1651889532804489,0.029756808653473854,0.011290284804999828,0.011514165438711643,0.0010427591623738408,-0.005329023580998182,-0.0014707937370985746,0.0025837128050625324,0.013085602782666683,0.006951478309929371,0.008580491878092289,0.006736882962286472,0.001162096974439919,-0.0051905312575399876,-0.008348710834980011,0.053918465971946716,0.008262988179922104,0.013472695834934711,0.0032401399221271276,0.01821018196642399,0.004564602859318256,0.021090151742100716,0.01567355915904045,0.0009987782686948776,0.017092376947402954,0.004795118235051632,0.011495530605316162,0.008582878857851028,0.01306413859128952,-0.0018734729383140802,0.00017919241508934647,-0.10702482610940933,-0.02381746470928192,-0.0009900458389893174,-0.01153947226703167,0.005601809825748205,-0.023384416475892067,-0.0037469060625880957,-0.007688973098993301,-0.0029587873723357916,-0.024404898285865784,-0.03113248012959957,0.004413330927491188,-0.023771092295646667,0.012886451557278633,-0.07442435622215271,0.05949133262038231,-0.003248096676543355,-0.007962902076542377,0.002947211964055896,0.00017728078819345683,-0.033616479486227036,-0.007869131863117218,-0.0015245111426338553,0.0010132294846698642,-0.014664769172668457,0.006215312052518129,-0.00957330409437418,-0.012930012308061123,0.06783817708492279,0.13573575019836426,0.016209017485380173,-0.008563359268009663,0.0023956706281751394,-0.0013433793792501092,0.008027750067412853,0.004846462979912758,-0.0038076755590736866,-0.005459052510559559,0.001565398182719946,-0.00798715278506279,0.007037699688225985,0.027249416336417198,-0.009840189479291439,-0.11476525664329529,-0.0385754369199276,-0.033146489411592484,-0.021479230374097824,-0.010632001794874668,-0.032680753618478775,-0.00649596331641078,-0.009979445487260818,-0.02590847760438919,0.009996488690376282,0.018927203491330147,0.01747755892574787,-0.043302346020936966,-0.047527920454740524,-0.021573452278971672,-0.011092747561633587,0.06522026658058167,-0.07668203860521317,-0.01982824318110943,-0.042141687124967575,-0.03318380191922188,-0.009215221740305424,0.009058685973286629,-0.007061122916638851,-0.03050808049738407,0.003331966931000352,-0.0249672532081604,0.0018638280453160405,-0.003491292940452695,-0.030272575095295906,-0.06943131238222122,0.00990203395485878,-0.03665294870734215,0.005463126581162214,-0.02151774987578392,0.0049711368046700954,-0.017091065645217896,-0.0009261462255381048,0.003348472062498331,-0.0038761093746870756,-0.0036901128478348255,-0.0027396974619477987,0.0006772379856556654,-0.017539165914058685,-0.010630329139530659,-0.08630333840847015,-0.020765110850334167,-0.0568663589656353,-0.006309622898697853,0.006583129521459341,0.0024862890131771564,-0.0033704452216625214,-0.004135778173804283,-0.012062369845807552,-0.0031130488496273756,-0.016269436106085777,-0.020478732883930206,0.004236252047121525,0.002425196347758174,-0.01891494356095791,-0.031249480322003365,-0.00960059929639101,0.03899283707141876,0.011315407231450081,0.020529676228761673,0.00525312777608633,0.017735300585627556,0.011845353990793228,0.02839638851583004,0.00889117456972599,-0.006202235817909241,0.03460259735584259,0.030002973973751068,0.031243739649653435,0.033286478370428085,0.13416188955307007,-0.020779283717274666,0.05182620882987976,0.01230640709400177,0.010728895664215088,0.008124131709337234,-0.006554721388965845,-0.020749621093273163,0.017154105007648468,0.014091124758124352,0.014297105371952057,-0.002931946190074086,0.0037846574559807777,0.007457979954779148,0.06242837756872177,0.030248790979385376,0.06988177448511124,0.0909806489944458,-0.011507382616400719,-0.005351864267140627,0.020795611664652824,-0.007800856605172157,-0.006110112648457289,0.007186198607087135,0.0014472565380856395,0.010239195078611374,-0.002988530322909355,0.022676972672343254,0.038249142467975616,0.01856035739183426,-0.07838299125432968,0.10080520808696747,0.09526824951171875,-0.014563439413905144,-0.007195707876235247,-0.017990393564105034,0.005208006594330072,0.01850605197250843,-0.000771355873439461,-0.025064056739211082,-0.006196053232997656,0.0018822563579306006,0.0010325758485123515,0.007867231033742428,0.011473516933619976,-0.014755173586308956,0.035857539623975754,0.006009452976286411,0.08627159893512726,0.06745606660842896,0.07253469526767731,0.07222799211740494,0.07431495934724808,0.055014804005622864,0.09598129242658615,0.09747850894927979,0.07112286239862442,0.0718500167131424,0.05899898335337639,0.003182103391736746,0.03584073483943939,0.023813525214791298,-0.06296523660421371,0.07003229111433029,0.07055546343326569,0.05517553165555,0.06383330374956131,0.045088030397892,0.039829760789871216,0.06191256642341614,0.06332709640264511,0.07595151662826538,0.053753919899463654,0.06357062608003616,0.058142125606536865,0.06497615575790405,0.06259091198444366,-0.14117318391799927,0.044648535549640656,-0.003615713445469737,0.01843852549791336,0.009551444090902805,0.02479306049644947,0.06129780039191246,0.034249063581228256,0.03858552873134613,0.03096405789256096,0.02092999592423439,0.028630739077925682,0.01368743646889925,0.042285364121198654,0.09027989953756332,-0.12364166229963303,0.09074427932500839,0.10535120964050293,0.09725087881088257,0.10152319073677063,0.11007987707853317,0.09024103730916977,0.07714708894491196,0.11812634766101837,0.07563850283622742,0.09586028754711151,0.0830521211028099,0.10033673793077469,0.17766207456588745,0.1420292854309082,-0.2981865704059601,0.02303401567041874,0.04299771413207054,0.0265569556504488,0.045949552208185196,0.03836693987250328,0.027077892795205116,0.022025909274816513,0.013712608255445957,0.043567027896642685,0.034888364374637604,0.05677158385515213,0.04940364137291908,0.07527627050876617,0.054112572222948074,-0.19416770339012146,0.09508663415908813,0.05846070498228073,0.03534848242998123,0.018779629841446877,0.025040047243237495,0.06710489094257355,0.04315168038010597,0.045862309634685516,0.055998917669057846,0.04966014251112938,0.07062020152807236,0.016607660800218582,0.04443657398223877,0.058354735374450684,-0.10460290312767029,0.0187988318502903,0.014010211452841759,0.013714514672756195,0.011816812679171562,0.018158018589019775,-0.012439040467143059,-0.04165252298116684,-0.00984395295381546,0.0046908180229365826,0.03983664885163307,0.0011761768255382776,-0.022419480606913567,-0.019595975056290627,-0.03286297619342804,0.011234648525714874,0.026810627430677414,0.009541536681354046,-0.014442714862525463,0.003552571637555957,-0.005390006583184004,-0.025542540475726128,-0.02603515051305294,-0.010267399251461029,-0.04581262916326523,-0.050101447850465775,-0.03177414834499359,-0.04205179959535599,-0.05169513076543808,-0.05356000363826752,0.04400889202952385,-0.0035512966569513083,-0.011839860118925571,0.0056259045377373695,-0.0025706165470182896,-0.041096869856119156,-0.011778882704675198,0.002750223269686103,0.004920111503452063,-0.011527449823915958,-0.001656086533330381,-0.009909897111356258,0.008105712942779064,-0.04982231557369232,0.0005202456377446651,-0.002691322937607765,0.010809521190822124,0.010715518146753311,0.011588584631681442,0.010222634300589561,-0.005378421396017075,0.010059910826385021,0.023528702557086945,0.026470964774489403,0.015007609501481056,0.017520349472761154,0.013911415822803974,0.012365186586976051,0.05055542290210724,-0.015517384745180607,0.07088581472635269,-0.007653749547898769,-0.01284310407936573,-0.018865449354052544,-0.0042510563507676125,0.004097046796232462,-0.017164109274744987,-0.009486259892582893,-0.01010268833488226,-0.001803897088393569,0.002870628610253334,-0.01703825779259205,-0.04423189163208008,-0.0064401463605463505,-0.0057853031903505325,-0.004858046304434538,0.015385570004582405,0.013641302473843098,0.01762508414685726,0.015251281671226025,0.021358788013458252,0.010911543853580952,-0.00699745723977685,0.003260399214923382,0.014075626619160175,0.013286224566400051,0.02024507336318493,0.014568010345101357,0.011525824666023254,0.04872698709368706,0.129710853099823,-0.03189850598573685,-0.023216869682073593,-0.028819700703024864,-0.029774215072393417,-0.025467069819569588,-0.019970519468188286,-0.011906404048204422,-0.02992674708366394,-0.029268227517604828,-0.0365402027964592,-0.03215337544679642,-0.018015528097748756,-0.015576152130961418,-0.02626838907599449,0.07831315696239471,-0.0007418134482577443,0.018958892673254013,-0.00558440899476409,0.0037025369238108397,0.021537352353334427,0.010980640538036823,0.009567439556121826,0.01329229399561882,0.01152786798775196,0.0053076837211847305,0.017027802765369415,0.0030939707066863775,-0.05318405479192734,-0.07017862051725388,0.1566140055656433,0.0008659044397063553,0.007232097443193197,0.007578469812870026,0.008040889166295528,0.02478254772722721,-0.002619935665279627,0.0038225383032113314,0.009506349451839924,0.00933376420289278,-0.009667580015957355,0.0041231317445635796,0.021887756884098053,0.03990713506937027,0.08575630187988281,0.0551738440990448,-0.023393718525767326,-0.003544947598129511,-0.02756371907889843,-0.011876633390784264,0.005013743415474892,-0.028805488720536232,-0.02328234352171421,0.0025009645614773035,-0.0283858310431242,-0.010016931220889091,-0.006802803836762905,-0.017142891883850098,-0.05545145645737648,-0.082811638712883,-0.05218878760933876,-0.019175153225660324,-0.0013957791961729527,-0.015384470112621784,-0.011126967146992683,-0.01842436008155346,0.000350983755197376,0.00655706599354744,-0.008625992573797703,0.008657550439238548,-0.01092017162591219,-0.0020047477446496487,-0.005397455301135778,-0.0037358447443693876,-0.001214560936205089,0.0010000297334045172,2.368380137340864e-06,-0.0018671222496777773,0.0017684487393125892,0.01915445737540722,0.004877548664808273,0.013918246142566204,0.011342065408825874,0.013560655526816845,-0.00045665056677535176,-0.0011384588433429599,0.006744893733412027,0.006981884595006704,0.005668338388204575,0.002586015500128269,0.012560553848743439,0.0007320769364014268,-0.019191492348909378,0.01663522608578205,0.0007043377845548093,0.008737033233046532,-0.020282894372940063,-0.011916604824364185,-0.017358867451548576,0.004353290889412165,-0.012527208775281906,-0.008082372136414051,-0.019682221114635468,-0.024260949343442917,-0.020332103595137596,-0.026288239285349846,-0.013297206722199917,-0.007617452181875706,-0.004996770527213812,0.0004218959074933082,0.0018992128316313028,0.006687580142170191,-0.0024659710470587015,0.007441244088113308,0.013196246698498726,0.010393060743808746,-0.0009641062351875007,0.002343326574191451,0.015011911280453205,0.008740976452827454,0.0028699873946607113,-0.0012276602210476995,-0.0076031810604035854,-0.010455182753503323,-0.009554600343108177,-0.02186726965010166,-0.0035022441297769547,-0.01329139992594719,-0.0092370780184865,-0.008279227651655674,-0.010439637117087841,-0.004966130014508963,-0.012263912707567215,-0.018501410260796547,-0.02294134348630905,-0.028565283864736557,-0.013187538832426071,-0.025707479566335678,-0.011947031132876873,0.0015117865987122059,0.013263773173093796,-0.015496068634092808,-0.021185334771871567,-0.003382994793355465,-0.02081403136253357,0.001980019034817815,-0.014786259271204472,-0.0193796269595623,-0.03509021922945976,-0.018628794699907303,-0.028818506747484207,-0.0013151022139936686,0.024175364524126053,-0.005194886587560177,-0.00756129901856184,-0.01084953173995018,-0.0054831127636134624,0.02083835005760193,-0.013524601235985756,0.03633461892604828,-0.005265649873763323,0.014511690475046635,-0.004140202887356281,0.03370695561170578,0.024020995944738388,-0.02437051571905613,-0.02829836867749691,-0.030423421412706375,-0.02185841277241707,-0.008348610252141953,-0.027320703491568565,-0.011096795089542866,-0.01720258966088295,0.006105839740484953,-0.03094041347503662,-0.020877692848443985,-0.017847901210188866,-0.010820724070072174,0.016367396339774132,0.005545654334127903,-0.021762052550911903,-0.014747104607522488,-0.022099949419498444,-0.004252780694514513,-0.002329729963093996,-0.005199038423597813,-0.032265789806842804,-0.017397256568074226,-0.029825538396835327,-0.021392790600657463,-0.016278037801384926,0.004944225773215294,0.010056893341243267,0.005894689820706844,0.05128176882863045,0.09718065708875656,0.008210553787648678,0.015229745768010616,0.026019461452960968,0.02175156958401203,0.020754562690854073,-0.003812138456851244,0.01730261743068695,0.011226369999349117,0.0017469091108068824,0.040378473699092865,0.03457770496606827,0.030286187306046486,0.03981789946556091,0.040071215480566025,-0.0024496293626725674,-0.014528642408549786,-0.0016853369306772947,-0.020762184634804726,-0.005422373302280903,-0.007278650067746639,-0.016478698700666428,-0.012816423550248146,-0.010039297863841057,-0.018400562927126884,0.003624122589826584,-0.007866711355745792,-0.034341923892498016,-0.00517020421102643,-0.030180536210536957,0.013246046379208565,0.02251942828297615,0.00959511287510395,0.01931171491742134,0.005323116667568684,0.005446222145110369,0.024870283901691437,0.025215845555067062,0.020104849711060524,0.03289607912302017,0.005684259347617626,0.0010129540460184216,0.006057118531316519,0.003377840854227543,-0.0032008171547204256,0.15327122807502747,-0.02846387028694153,-0.02011715993285179,-0.028656238690018654,-0.02847893536090851,-0.01737876608967781,-0.0123314019292593,-0.0411161445081234,-0.02542373724281788,-0.03599000349640846,-0.036322519183158875,-0.036599691957235336,-0.01891503669321537,1.3289813978190068e-05,-0.03275509923696518,-0.017082421109080315,0.0012681351508945227,0.0002881360414903611,-0.00036323058884590864,0.014920596964657307,0.009305908344686031,-0.005726468283683062,0.005862466059625149,0.011510231532156467,0.004412891808897257,0.0029051578603684902,0.01863343082368374,0.014824766665697098,-0.03942729905247688,-0.017938263714313507,-0.08620977401733398,0.017780637368559837,0.015606525354087353,0.011502441018819809,0.01468151155859232,0.009498391300439835,0.004913792014122009,0.015796752646565437,0.03645903989672661,0.030574653297662735,0.0002637552679516375,0.010073996149003506,0.013499573804438114,0.027150457724928856,0.07049630582332611,-0.11439332365989685,-0.02444012276828289,-0.0063120038248598576,-0.009455211460590363,-0.013528442941606045,-0.004379219841212034,-0.03255321830511093,-0.019362103193998337,3.395947351236828e-05,-0.007625954691320658,-0.016914134845137596,-0.002263459609821439,-0.00422428036108613,-0.015243341214954853,-0.07964702695608139,-0.07896038889884949,0.061642687767744064,0.024442942813038826,0.14105361700057983,0.023637155070900917,-0.03337330371141434,-0.10956141352653503,-0.031005946919322014,-0.1831607073545456,-0.0007502210210077465,-0.004264688119292259,0.012955387122929096,0.008327063173055649,0.009250442497432232,-0.0035307754296809435,0.002313669305294752,0.006468155886977911,0.25671640038490295,0.237216979265213,0.06882791966199875,0.12190469354391098,-0.28149527311325073,-0.006519104819744825,0.007304952014237642,0.18087682127952576,-0.08410964161157608,-0.055838171392679214,-0.08557451516389847,-0.053994353860616684,0.08776145428419113,-0.12935473024845123,-0.015488713048398495,-0.08749568462371826,0.04846764728426933,0.1413854956626892,-0.08934034407138824,0.1397477686405182,-0.10239721089601517,0.180202916264534,-0.0033381925895810127,-0.10081317275762558,-0.2633430063724518,-0.1487269550561905,0.1365818828344345,-0.0743863508105278,0.19604019820690155,0.04290677607059479,-0.054780591279268265,0.07700330018997192,-0.033838193863630295,-0.21244403719902039,-0.09219452738761902,-0.05042899027466774,0.14213277399539948,-0.16267870366573334,-0.052660778164863586,0.03816885128617287,0.0014109337935224175,-0.05781257525086403,-0.05514617636799812,-0.033848900347948074,0.04184583202004433,0.04424460604786873,-0.018208811059594154,-0.2133096605539322,-0.043474696576595306,-0.08155197650194168,0.23998355865478516,0.02152159810066223,0.06806764751672745,-0.07373303174972534,0.010673865675926208,0.23070329427719116,-0.03557707741856575,0.056290142238140106,-0.08162044733762741,-0.07129549235105515,-0.017193136736750603,-0.01809229515492916,0.03389119356870651,-0.05930395796895027,-0.006247926969081163,-0.16257193684577942,-0.3435472846031189,0.17376840114593506,0.019070785492658615,0.045757368206977844,0.004400050733238459,0.34262773394584656,-0.0016028628451749682,0.011093361303210258,-0.008423924446105957,0.019545625895261765,0.11421450972557068,-0.008549237623810768,-0.015857413411140442,-0.033273737877607346,0.02962474152445793,-0.058064572513103485,-0.0019788402132689953,-0.007519792299717665,0.06581265479326248,0.07630176097154617,0.08867353945970535,-0.048511747270822525,-0.05227435380220413,0.019695965573191643,-6.951155228307471e-05,0.010658757761120796,0.025942876935005188,0.010171902365982533,0.025642458349466324,0.000639565521851182,0.028321733698248863,0.07753625512123108,0.0760558694601059,-0.0437285341322422,0.09406787902116776,0.0333808995783329,0.03427211567759514,0.0605788454413414,0.0044797626323997974,0.0028861970640718937,0.01941077969968319,-0.022091569378972054,-0.034456245601177216,-0.018863068893551826,-0.05649777874350548,0.003423749003559351,0.043792515993118286,0.09170754253864288,-0.002428269712254405,-0.008047527633607388,0.05650623142719269,-0.0744512602686882,-0.16558632254600525,-0.005757224280387163,0.0071420748718082905,-0.024433551356196404,0.03744414821267128,0.06845684349536896,0.06415406614542007,0.013302006758749485,-0.12072259187698364,-0.03512592986226082,-0.09196485579013824,-0.0032059785444289446,-0.18498069047927856,-0.1504044085741043,0.03877934440970421,0.10693338513374329,0.03395048901438713,0.1174534261226654,-0.05525593459606171,-0.07920310646295547,0.05279180780053139,-0.10462719202041626,0.07334540039300919,0.21392251551151276,-0.04884985834360123,-0.032504137605428696,0.07407324016094208,-0.0021985098719596863,0.14624811708927155,0.24771028757095337,0.08478032052516937,-0.03411414101719856,0.04566739872097969,0.13312318921089172,0.10122982412576675,0.08269346505403519,0.08945026993751526,0.005522702354937792,-0.07562074810266495,0.03100632317364216,-0.07502780854701996,-0.10622989386320114,0.14936567842960358,-0.003951127640902996,-0.05916174128651619,-0.05657517910003662,-0.03601321950554848,0.10450855642557144,-0.07494261860847473,-0.10162454843521118,0.12840856611728668,0.10196354240179062,-0.10971882939338684,-0.0735720843076706,-0.01724441535770893,0.17116422951221466,0.020717475563287735,-0.11827795207500458,-0.11795785278081894,-0.08940500766038895,0.10092516243457794,0.11678051203489304,0.09691791981458664,-0.03678121045231819,-0.20323562622070312,-0.19397272169589996,0.12802660465240479,-0.11540794372558594,0.04495599865913391,-0.00771827157586813,-0.07501312345266342,0.01284127589315176,-0.02885255590081215,0.12720423936843872,0.20014746487140656,0.09949366748332977,-0.11612594127655029,-0.11115755885839462,-0.008100227452814579,-0.04573100432753563,-0.06948578357696533,-0.03616661578416824,0.19502761960029602,0.24984857439994812,0.06757368892431259,-0.021231167018413544,0.11739377677440643,0.1359819918870926,-0.14263516664505005,-0.3096913993358612,-0.046340085566043854,0.15737268328666687,0.28272947669029236,0.06390459835529327,-0.30296266078948975,-0.0937785878777504,0.232670858502388,-0.006055028643459082,-0.024172062054276466,0.030204907059669495,0.04482339322566986,0.0831451416015625,0.12618803977966309,0.07706788927316666,-0.022595105692744255,-0.12286429107189178,-0.0731736272573471,-0.04421606287360191,0.06095113977789879,0.07221612334251404,-0.04474908858537674,-0.09579788148403168,-0.14643605053424835,-0.23735757172107697,-0.010164075531065464,-0.03157355263829231,-0.09314022213220596,-0.07378066331148148,-0.04537404701113701,-0.06708940118551254,-0.008365224115550518,0.06038881838321686,0.12679162621498108,0.14093467593193054,0.1016363799571991,-0.05389147251844406,0.1713719218969345,0.13566406071186066,0.03561747446656227,-0.03623921051621437,0.05785944685339928,-0.20803555846214294,-0.057656191289424896,0.04246002808213234,-0.005732519086450338,0.11117588728666306,-0.03153832629323006,0.048831433057785034,0.2754836976528168,0.013667874969542027,0.07715897262096405,0.17460884153842926,0.08835051953792572,0.04305007681250572,-0.07165481150150299,0.015535549260675907,0.0710982233285904,0.018689988180994987,0.03434889391064644,0.029833214357495308,0.06801806390285492,0.015164712443947792,-0.03364141657948494,-0.07830510288476944,-0.01941516436636448,-0.13917383551597595,0.061811260879039764,0.06674932688474655,-0.08989470452070236,-0.03681601956486702,-0.13788537681102753,-0.23077936470508575,0.06616293638944626,0.02797566168010235,-0.0004603115376085043,0.129752978682518,-0.11337825655937195,-0.09291861951351166,-0.017379838973283768,-0.03213529288768768,0.01140605378895998,0.09519656002521515,0.036732982844114304,0.10684508085250854,0.052078720182180405,0.05383756384253502,-0.06482657790184021,-0.009518987499177456,0.04071537405252457,-0.0733882337808609,0.06650599092245102,0.027972808107733727,-0.022834379225969315,0.031204497441649437,0.10107965022325516,0.08919941633939743,-0.015793975442647934,0.07079973816871643,0.10922934114933014,0.10138913989067078,0.011725726537406445,0.0867946445941925,0.11358314007520676,-0.07174186408519745,-0.0028755266685038805,-0.046640198677778244,0.007236640900373459,-0.02588953822851181,-0.023995984345674515,0.11057834327220917,0.024844830855727196,0.09610400348901749,0.02478271909058094,0.031433820724487305,0.007496674545109272,-0.025089627131819725,-0.10807503759860992,-0.04003845155239105,-0.07228554785251617,-0.04106764867901802,0.1419648975133896,0.030156945809721947,-0.054986607283353806,-0.10162670910358429,-0.04966449365019798,-0.023719049990177155,0.08983146399259567,0.21262697875499725,-0.057143330574035645,-0.05826783552765846,-0.048631276935338974,-0.04370821267366409,0.011232774704694748,-0.003681453410536051,0.014525237493216991,0.00449365796521306,0.02142420969903469,0.006539961788803339,0.0024417478125542402,0.04070639982819557,0.0389762818813324,0.027716422453522682,0.1414526104927063,-0.005688903387635946,-0.0669151246547699,-0.06689692288637161,0.005643829237669706,0.01589290425181389,-0.0731162428855896,0.12495144456624985,0.11376602202653885,0.07517790049314499,-0.024185894057154655,0.022717434912919998,-0.0002334756136406213,0.032467108219861984,0.10817878693342209,0.10047568380832672,-0.07592149078845978,-0.04699881374835968,0.017104173079133034,-0.17198660969734192,-0.2104315161705017,0.08769699931144714,0.02874135598540306,-0.05537935718894005,-0.05302293598651886,-0.22367745637893677,-0.2434222549200058,0.02666609175503254,0.031505633145570755,-0.06991641223430634,-0.04858087748289108,-0.10867191106081009,-0.07724127918481827,0.010599788278341293,-0.07418258488178253,0.1140265092253685,-0.001295661204494536,0.029288439080119133,0.20203903317451477,0.14271295070648193,-0.006059643812477589,0.09718652069568634,0.04940718412399292,0.08594366163015366,-0.05547688156366348,0.1845308095216751,-0.0035977482330054045,0.03894748538732529,0.07296036183834076,0.08965934067964554,0.11886852234601974,0.04135220870375633,-0.03933762013912201,0.15815624594688416,0.009497220627963543,0.11700505018234253,0.22597269713878632,0.06130187585949898,0.04917217791080475,0.01695866882801056,-0.08484011888504028,0.033624131232500076,-0.19918470084667206,0.2028811275959015,0.05903993546962738,0.24882207810878754,0.19817982614040375,0.13919468224048615,-0.12428746372461319,-0.25427761673927307,-0.12410831451416016,0.011134062893688679,0.1673841029405594,0.193581223487854,-0.20412036776542664,-0.08325064182281494,-0.08198650926351547,0.07390214502811432,-0.11290635913610458,0.0986378937959671,-0.3305661678314209,0.28990060091018677,-0.04004834592342377,0.05144284665584564,-0.031272225081920624,0.012628492899239063,-0.14499394595623016,0.14335140585899353,-0.03343139961361885,-0.0341191366314888,0.027043744921684265,-0.022252779453992844,-0.08049309253692627,-0.20043091475963593,0.03734199330210686,-0.0481884628534317,0.1488109678030014,-0.17891235649585724,0.07088196277618408,0.0650949478149414,0.03267790749669075,0.02410992793738842,-0.04046798497438431,0.07323727011680603,0.002364127663895488,-0.06967154145240784,-0.031460072845220566,0.32943397760391235,-0.11949893087148666,0.2451779693365097,0.18545101583003998,-0.05840824544429779,0.3241754174232483,-0.1968894600868225,-0.010468798689544201,0.030045874416828156,-0.003292028559371829,0.023835593834519386,0.002025305526331067,-0.0012293874751776457,-0.018943455070257187,-0.013621630147099495,-0.12367576360702515,-0.15976248681545258,0.34613174200057983,0.028717422857880592,-0.08387225866317749,0.19834411144256592,-0.22780291736125946,0.2835622727870941,-0.13188476860523224,0.18185333907604218,-0.03904455527663231,0.07278301566839218,-0.03842273727059364,-0.09206788241863251,-0.11046791821718216,-0.027072550728917122,0.11095515638589859,-0.3987448513507843,0.15507447719573975,0.3865607976913452,0.22766493260860443,-0.12223134934902191,0.15370376408100128,-0.5919889211654663,-0.14825887978076935,0.1328139752149582,-0.027630742639303207,0.022213466465473175,0.17568963766098022,-0.005445271730422974,-0.19978217780590057,0.0392245277762413,0.20598484575748444,0.22150732576847076,0.166180819272995,-0.36119401454925537,-0.3944496810436249,0.2546522915363312,-0.3888353705406189,0.07672042399644852,-0.11503329873085022,-0.118575319647789,-0.23246850073337555,0.14507734775543213,0.05791623517870903,0.0923364907503128,-0.1737484633922577,-0.06168600916862488,-0.34944674372673035,-0.15037356317043304,0.5217400789260864,0.01487204059958458,-0.07032483071088791,0.21572494506835938,0.07017668336629868,0.2669895887374878,0.0956084132194519,-0.12362423539161682,-0.33914509415626526,-0.07973150908946991,-0.1189228892326355,-0.08785392343997955,0.02474178932607174,-0.30968761444091797,0.17182539403438568,0.17243948578834534,-0.15098656713962555,-0.026244670152664185,0.36132919788360596,0.08702090382575989,0.04548963904380798,-0.12039526551961899,-0.05606228858232498,-0.007979126647114754,0.012713217176496983,0.0031267241574823856,-0.0026457190979272127,-0.058966562151908875,-0.10209119319915771,-0.04254074767231941,-0.0040117958560585976,0.0379180833697319,-0.014285169541835785,0.13414131104946136,0.047455795109272,-0.01739862561225891,0.026552122086286545,0.0804593563079834,0.034723617136478424,0.08055513352155685,-0.1212739422917366,-0.10695593059062958,0.020106175914406776,-0.0011782676447182894,0.12369012832641602,0.18164759874343872,-0.08933009207248688,-0.035388294607400894,-0.12729217112064362,0.06406820565462112,-0.026017075404524803,0.03791746124625206,0.005303739104419947,-0.00039789179572835565,0.020975202322006226,0.03262675553560257,0.04855440929532051,0.10654101520776749,0.03025694750249386,0.06284435838460922,0.029755650088191032,0.006844032555818558,0.020832141861319542,-0.05040333792567253,0.008666476234793663,0.03375934064388275,-0.04452652111649513,0.035567544400691986,0.005683983210474253,-0.04892709478735924,-0.06030179187655449,-0.0552535243332386,-0.06752559542655945,-0.051797423511743546,0.16472725570201874,0.101371631026268,-0.09755919128656387,-0.11105722188949585,-0.07805934548377991,0.004376168362796307,0.04982968419790268,0.2163010984659195,-0.06661344319581985,-0.04445938393473625,-0.0029737488366663456,0.10464824736118317,-0.03713272884488106,-0.0827505961060524,-0.14892661571502686,-0.10362518578767776,-0.08561635762453079,-0.017475072294473648,0.1826981157064438,0.167080357670784,-0.17536617815494537,-0.1156359538435936,-0.09060465544462204,-0.0018521909369155765,0.20511379837989807,0.19689317047595978,-0.05274170637130737,-0.03475916385650635,-0.03703708574175835,-0.001512897782959044,0.0980939194560051,0.08424720168113708,0.07629912346601486,0.08368703722953796,-0.003239694982767105,-0.018627803772687912,3.34090473188553e-05,-0.052615776658058167,-0.017653921619057655,-0.0314563512802124,0.04646718502044678,0.057060420513153076,-0.023545093834400177,-0.08286931365728378,0.0047780293971300125,-0.051126908510923386,-0.044978756457567215,0.08893504738807678,-0.13652212917804718,0.20680472254753113,-0.06353926658630371,0.05390481278300285,-0.07821480929851532,-0.017301112413406372,-0.02904447168111801,-0.04647429659962654,-0.04613146558403969,0.004722855519503355,0.057775601744651794,-0.05861474573612213,0.10034767538309097,-0.19285449385643005,-0.015792984515428543,-0.044442422688007355,0.04666414484381676,0.015094825997948647,0.062233686447143555,0.10212287306785583,0.008181021548807621,0.008758053183555603,-0.06543956696987152,0.14732427895069122,-0.15273672342300415,0.042230308055877686,0.07445540279150009,0.058347705751657486,0.12672275304794312,0.035143665969371796,0.10861314088106155,0.09070361405611038,-0.09555061906576157,-0.07828608900308609,-0.09993616491556168,-0.04842441901564598,-0.17568881809711456,0.09599117934703827,0.03101121447980404,0.09975674748420715,-0.08983203768730164,0.203499898314476,0.04119249805808067,-0.028016066178679466,-0.05758758634328842,-0.0521724671125412,0.06243205815553665,-0.0844510942697525,0.012949001044034958,0.20903092622756958,0.02005906216800213,0.10279221087694168,0.011387674137949944,-0.01064244657754898,-0.13483506441116333,-0.0031662448309361935,0.0227355919778347,0.07453830540180206,0.008584178052842617,-0.10455437004566193,0.043171048164367676,-0.17168907821178436,-0.05129942297935486,0.015091033652424812,0.07814451307058334,-0.04056928679347038,-0.12066596746444702,0.11315778642892838,0.030175574123859406,-0.03817393630743027,0.017036072909832,-0.08568614721298218,0.005587523803114891,0.1490015685558319,0.04026900976896286,-0.010463850572705269,-0.01983306184411049,0.01502299215644598,-0.01579088531434536,-0.040246766060590744,-0.030818728730082512,-0.05570031329989433,-0.0008489086758345366,-0.061662424355745316,0.1478387713432312,0.11007408797740936,-0.032918233424425125,0.02826067805290222,0.0895703062415123,-0.06801752001047134,0.13535556197166443,-0.06255000829696655,0.0851919874548912,0.062380559742450714,0.0077525777742266655,-0.008205164223909378,-0.01755145750939846,-0.06862236559391022,0.08912468701601028,0.07842981815338135,0.03289465233683586,0.0010514275636523962,-0.08593849092721939,-0.15481042861938477,-0.002998416544869542,-0.012016315944492817,0.0010723861632868648,-0.09981343895196915,-0.08797671645879745,-0.03537830337882042,0.17051243782043457,0.09615624696016312,0.026748914271593094,-0.007283469662070274,-0.23220299184322357,-0.239592045545578,0.27822017669677734,0.11223391443490982,0.1419556885957718,0.023525672033429146,-0.14524376392364502,-0.1622362732887268,0.07915276288986206,0.017682380974292755,0.010553369298577309,-0.05802208557724953,-0.06886496394872665,-0.022291945293545723,-0.027534954249858856,-0.01641860604286194,-0.05868702381849289,0.028889672830700874,-0.03165736049413681,-0.048243358731269836,-0.12216760218143463,-0.008684114553034306,0.007617112249135971,0.024905851110816002,0.11017915606498718,0.04344264790415764,0.0016361016314476728,-0.0242876335978508,0.040024351328611374,0.015224405564367771,-0.05029769986867905,0.022229455411434174,-0.049662016332149506,0.044671107083559036,0.03631770238280296,0.18197381496429443,-0.15542832016944885,0.15985924005508423,0.005293930880725384,0.0029218445997685194,0.04166838899254799,-0.10314658284187317,-0.3231969177722931,0.4147873520851135,-0.001485871965996921,0.016761554405093193,0.07865100353956223,0.06756877899169922,-0.18166780471801758,0.2485102117061615,-0.04675065353512764,-0.026888806372880936,0.06957825273275375,-0.02591560408473015,-0.2447858452796936,-0.15797705948352814,0.039340708404779434,0.10518161952495575,0.05139920115470886,-0.01918615587055683,-0.12420152872800827,0.15132559835910797,-0.02478879503905773,-0.07418747246265411,0.02929801307618618,-0.07681243121623993,-0.051418475806713104,0.05507054552435875,0.10267405956983566,-0.027292560786008835,0.0970320776104927,-0.03408001735806465,0.010726104490458965,0.07837668806314468,0.06280417740345001,0.027830176055431366,0.08946218341588974,-0.03385962173342705,-0.06562492251396179,0.031694378703832626,0.007638639770448208,0.059484273195266724,0.05502070486545563,0.007213515695184469,-0.05984830856323242,-0.102488674223423,0.026393016800284386,-0.013143016025424004,0.04209347441792488,-0.021337131038308144,-0.13966216146945953,0.020331664010882378,0.14508725702762604,0.06867209821939468,0.05016394332051277,-0.0043486375361680984,-0.1293262243270874,-0.13969159126281738,0.06926373392343521,0.0452880896627903,0.11376591771841049,0.09774547070264816,-0.19855374097824097,-0.09407376497983932,0.0916314423084259,-0.006393276154994965,0.03762590140104294,-0.08063963800668716,-0.08231417089700699,-0.04680606350302696,-0.10157208889722824,-0.039582617580890656,0.011772923171520233,-0.00692460872232914,0.02283279411494732,0.09905073046684265,0.035204604268074036,0.03838939219713211,-0.0066110603511333466,-0.01709665171802044,-0.021509991958737373,-0.029531879350543022,-0.015357360243797302,0.019038280472159386,0.007642277982085943,-0.021428070962429047,0.14374279975891113,0.02645229361951351,0.034648679196834564,-0.08023668825626373,-0.015437930822372437,-0.13455498218536377,0.11886652559041977,-0.019947776570916176,-0.03569324314594269,0.02980290725827217,-0.042202968150377274,0.03680342808365822,0.18115736544132233,-0.5409073233604431,0.055754415690898895,-0.03930310532450676,-0.006760180462151766,0.016833439469337463,0.054319027811288834,-0.12737247347831726,0.03349810838699341,-0.008279925212264061,-0.08460435271263123,-0.04384570196270943,0.14009475708007812,0.06524797528982162,0.022202594205737114,0.019729699939489365,-0.027603251859545708,-0.018577711656689644,0.17687180638313293,-0.10630589723587036,0.093966543674469,-0.014269810169935226,-0.005620808806270361,0.08150898665189743,-0.08383667469024658,-0.021102573722600937,-0.07841968536376953,-0.0391065888106823,0.0058632660657167435,0.02389136701822281,-0.0771651640534401,-0.034529611468315125,0.02573847956955433,-0.004577124025672674,0.15390647947788239,0.0943274274468422,-0.06189951300621033,-0.010760082863271236,-0.016185881569981575,0.03450249508023262,0.0028496438171714544,-0.010763058438897133,7.12506371201016e-05,0.007624304853379726,0.0033060763962566853,0.019510382786393166,-0.013703156262636185,0.006139456294476986,-0.07434424757957458,0.04311070963740349,0.185553178191185,0.20929443836212158,-0.028949428349733353,-0.038959961384534836,0.03230825439095497,0.0897824689745903,0.06684699654579163,0.05316077545285225,-0.047029171139001846,0.02996465191245079,-0.127297043800354,-0.1999741792678833,-0.10137974470853806,0.009561349637806416,0.02227046713232994,0.04662364348769188,-0.07887226343154907,-0.2063315361738205,-0.023852743208408356,0.4077996611595154,0.03205499053001404,-0.45312997698783875,0.028283091261982918,-0.1100432425737381,-0.02679917961359024,-0.08038093149662018,0.11837651580572128,-0.11231421679258347,-0.06164029613137245,-0.0743376687169075,-0.07223541289567947,0.1938290297985077,0.06594832241535187,0.14357125759124756,0.010005077347159386,-0.010458637960255146,-0.13992741703987122,0.10051469504833221,0.08192860335111618,0.07770876586437225,-0.0585847832262516,-0.19544848799705505,-0.07849343121051788,0.3659873604774475,0.087118960916996,-0.12970353662967682,-0.008199057541787624,-0.04287317022681236,0.16630294919013977,0.2401617020368576,-0.06380768120288849,0.35533565282821655,0.08581890165805817,-0.047010887414216995,-0.027305692434310913,0.011342447251081467,-0.3152892291545868,-0.20595747232437134,0.06487634778022766,-0.2704029977321625,-0.08766630291938782,-0.11248455941677094,0.09936373680830002,-0.03507976979017258,-0.05357476323843002,0.04404434561729431,0.1402427703142166,-0.08731293678283691,0.06583549827337265,-0.01053420640528202,0.054868631064891815,0.04871583357453346,-0.014973737299442291,0.004980355966836214,-0.06046511232852936,-0.030901141464710236,-0.03663152828812599,-0.019728533923625946,-0.042514968663454056,-0.006133725866675377,-0.1247425228357315,0.035443250089883804,-0.009045708924531937,0.0027369530871510506,-0.02509661763906479,-0.12479390949010849,-0.039567455649375916,0.01125265285372734,-0.02198559045791626,-0.04619959741830826,0.023727726191282272,-0.007487806491553783,-0.0848863273859024,0.11542718857526779,-0.020661739632487297,0.028082210570573807,-0.012888040393590927,0.005270063877105713,-0.08423205465078354,0.25703859329223633,0.01414104737341404,0.06764886528253555,0.08069465309381485,0.00901078712195158,0.020107436925172806,0.10676918923854828,-0.0028520135674625635,0.05997929349541664,-0.039567481726408005,0.04888833686709404,-0.062485404312610626,0.009979275986552238,-0.0004270410572644323,-0.0680277869105339,-0.023117145523428917,-0.0500926598906517,0.12058588862419128,0.014507047832012177,-0.030251512303948402,-0.10675348341464996,-0.06295597553253174,-0.15951313078403473,0.0019785533659160137,-0.0029638756532222033,-0.07184936106204987,-0.11943349242210388,-0.16669423878192902,-0.06505939364433289,-0.004297972656786442,0.16879047453403473,0.013561761938035488,-0.035956159234046936,0.024434352293610573,0.03591843321919441,0.08065618574619293,-0.015010563656687737,-0.04160074144601822,-0.0846044197678566,-0.14540670812129974,0.013004307635128498,0.18267977237701416,0.2204279601573944,-0.0420110858976841,-0.11980009078979492,-0.15874840319156647,-0.14287178218364716,0.01380270067602396,0.23839569091796875,-0.02184469811618328,-0.04239538684487343,-0.06138725206255913,0.03728101775050163,0.03412193804979324,0.04288776218891144,0.043113041669130325,0.067380391061306,0.06887921690940857,0.08525992184877396,-0.009514594450592995,-0.07626283913850784,0.008779174648225307,0.025769099593162537,0.13025280833244324,0.022622788324952126,0.009390070103108883,-0.007590544410049915,0.02174696885049343,0.0043756538070738316,-0.02880469523370266,-0.1369248330593109,0.008072265423834324,0.06261072307825089,-0.023110659793019295,0.03247147426009178,0.008827995508909225,0.043270472437143326,0.17423222959041595,-0.07995716482400894,-0.01908266916871071,0.018978139385581017,0.0028037854935973883,-0.029589327052235603,0.14491130411624908,0.1669759899377823,0.02661280892789364,0.014861827716231346,-0.0052615804597735405,0.004824329167604446,0.06631354242563248,0.15947072207927704,-0.02521195448935032,-0.042613521218299866,-0.012716575525701046,0.021148940548300743,-0.10614725202322006,-0.2834508717060089,0.019881242886185646,-0.011367138475179672,0.04828272759914398,-0.09195908159017563,0.043327897787094116,-0.013096516951918602,0.007081272546201944,-0.003558774944394827,-0.02383282221853733,0.08542532473802567,-0.10121548920869827,0.12273678928613663,-0.0021346532739698887,0.024031411856412888,-0.006243601907044649,0.000901074381545186,-0.09210999310016632,-0.10836631804704666,-0.020443076267838478,-0.024555684998631477,-0.012668173760175705,0.03969403728842735,0.006925093475729227,-0.1742669939994812,-0.061569370329380035,0.03871878236532211,-0.0467146560549736,-0.09783535450696945,-0.06882651895284653,-0.054332029074430466,-0.016940850764513016,0.05938238650560379,0.10997752100229263,-0.030079053714871407,0.039999328553676605,0.2103765606880188,0.002028533723205328,-0.04210924357175827,-0.04054379090666771,0.01595815271139145,0.0384182445704937,-0.0187357310205698,0.030864909291267395,0.048834607005119324,0.06758115440607071,0.03360501304268837,0.033945098519325256,0.0402987040579319,0.004038442391902208,-0.08154717087745667,-0.07675798237323761,0.026530086994171143,-0.11242823302745819,0.21715384721755981,-0.013299345038831234,0.0504768006503582,0.0838959664106369,-0.03951500356197357,-0.06938599050045013,-0.04901747778058052,0.006731668021529913,-0.03470015898346901,0.02839958854019642,-0.0010275576496496797,0.04403059184551239,0.04939568042755127,0.007119723130017519,0.15680551528930664,0.13192760944366455,0.14661051332950592,0.014411723241209984,-0.060479022562503815,0.06079957261681557,0.08311419188976288,0.10443662852048874,0.12428507953882217,-0.01274164766073227,-0.18555288016796112,-0.019674139097332954,0.032487500458955765,0.011080494150519371,-0.05439477786421776,-0.14006710052490234,0.03297272324562073,0.0626242607831955,0.09232043474912643,0.08087893575429916,-0.004937467165291309,-0.12341073155403137,-0.27471378445625305,0.06198152154684067,0.13332206010818481,0.22248800098896027,0.12170597165822983,-0.0460214838385582,-0.2024478316307068,0.040700145065784454,0.050506431609392166,0.0904531478881836,0.007009605877101421,-0.02573687769472599,-0.043811604380607605,-0.03603794425725937,-0.07020803540945053,-0.07545386254787445,-0.09273596853017807,0.01945112831890583,0.057140178978443146,0.013325628824532032,-0.028051944449543953,-0.06929661333560944,-0.06363111734390259,-0.013113203458487988,0.064604751765728,-0.03874428570270538,-0.058261677622795105,-0.06503787636756897,-0.015761476010084152,0.010058019310235977,0.018974177539348602,0.03251243382692337,0.08544233441352844,-0.022025378420948982,0.01287812553346157,0.005239854566752911,-0.026657357811927795,0.0046572997234761715,0.027221618220210075,0.07129790633916855,0.03789498656988144,-0.06224605441093445,-0.03941495344042778,0.0233834870159626,-0.06061480939388275,-0.05690031126141548,0.008458316326141357,-0.04113250970840454,0.014187066815793514,0.006451786495745182,-0.05069576948881149,-0.15694402158260345,0.041200488805770874,-0.0010649287141859531,0.05159318447113037,0.005299580283463001,-0.0837540552020073,-0.0030578316655009985,-0.039986301213502884,-0.03349955752491951,-0.027156466618180275,-0.012141726911067963,0.07451675087213516,0.08638167381286621,-0.014388586394488811,0.05424110218882561,0.015373571775853634,0.013598550111055374,0.07273586839437485,-0.0618169866502285,0.08170843124389648,-0.018795032054185867,-0.018273010849952698,0.03988388925790787,0.07801112532615662,0.06356708705425262,-0.06222284585237503,0.04028943181037903,0.04219895601272583,-0.05847320333123207,-0.044219911098480225,0.05165202170610428,0.0743463933467865,-0.04792221263051033,0.009192095138132572,-0.0395587682723999,-0.012644791044294834,0.04573189839720726,0.03239474445581436,0.010095563717186451,0.08279852569103241,0.01774727925658226,-0.006519337184727192,0.13912731409072876,0.007592401932924986,0.027171270921826363,-0.07481078803539276,0.03183187171816826,0.04808533191680908,0.22231411933898926,0.042532309889793396,0.03921863064169884,-0.03294070437550545,-0.0062544746324419975,-0.022715063765645027,0.07486535608768463,-0.007439624518156052,-0.06655765324831009,-0.02950242906808853,0.07276470214128494,-0.011249789036810398,-0.06850259751081467,-0.12680889666080475,0.01757253147661686,-0.04131767898797989,-0.05708610638976097,0.02097473479807377,0.03483998030424118,0.07278371602296829,0.04734567552804947,-0.0097691984847188,-0.012072619050741196,0.006907458417117596,-0.022629790008068085,0.002691514091566205,-0.01860055699944496,0.2716912031173706,0.0062679327093064785,-4.169622479821555e-05,-0.0018842853605747223,0.04799901321530342,-0.12218884378671646,0.10903505980968475,0.02430250309407711,0.003049340797588229,-0.012353911995887756,0.05647456645965576,-0.18211281299591064,-0.5057637095451355,0.007179556880146265,0.007009692024439573,0.005029099993407726,-0.047397419810295105,-0.15887881815433502,-0.04491753131151199,0.006497408729046583,-0.004374926444143057,-0.01836118847131729,0.04189715161919594,0.0006400799029506743,0.25291526317596436,-0.03331969305872917,0.007976270280778408,0.04912320896983147,0.04961012676358223,-0.050558533519506454,-0.3286236822605133,0.011615517549216747,0.00016078428598120809,-0.016987789422273636,-0.015393263660371304,0.023448973894119263,0.2321317046880722,-0.013539700768887997,-0.0022569941356778145,-0.058690473437309265,-0.07296396791934967,0.04597051069140434,-0.25575730204582214,-0.06730262190103531,-0.0044295103289186954,-0.16025161743164062,0.10245054960250854,-0.029265563935041428,-0.052344221621751785,0.11998771876096725,-0.10720325261354446,0.003828606102615595,-0.007445781957358122,-0.0006448819767683744,0.009867398999631405,-0.0033150336239486933,0.0030222234781831503,0.0022359092254191637,0.012772978283464909,-0.19658422470092773,0.17675437033176422,-0.057242076843976974,0.00394738232716918,-0.12070812284946442,0.00766207417473197,0.050903692841529846,0.20398929715156555,-0.15500256419181824,0.14604540169239044,-0.07899981737136841,0.0973801538348198,-0.15014143288135529,0.03905915096402168,-0.04529019072651863,0.03438127785921097,0.29482394456863403,-0.09929828345775604,0.02011536806821823,-0.002140755532309413,0.1213839203119278,0.13478095829486847,-0.0780622661113739,-0.28032493591308594,-0.20087239146232605,-0.23372776806354523,-0.03635315224528313,-0.011806159280240536,0.13705356419086456,0.08738706260919571,0.025984641164541245,-0.038641951978206635,0.022223087027668953,0.09742450714111328,-0.0766834244132042,-0.06068303808569908,0.004482374060899019,0.0004267839540261775,0.11020374298095703,0.16255418956279755,-0.10613268613815308,0.12717124819755554,-0.006707143969833851,0.042863838374614716,-0.017114222049713135,0.10222504287958145,-0.07377202063798904,-0.04973180964589119,0.06747984141111374,-0.07204543054103851,0.2930537462234497,-0.17970429360866547,0.05702183023095131,-0.014621410518884659,-0.09888558834791183,0.3008023500442505,-0.08110709488391876,0.019689194858074188,-0.18531380593776703,0.13626134395599365,0.03465369716286659,0.0178112480789423,0.09185317158699036,-0.3000105917453766,0.1181536316871643,0.1096944808959961,0.059753596782684326,0.05563477799296379,0.3112551271915436,-0.07338059693574905,0.02662121132016182,0.006192430388182402,-0.004951909184455872,0.021254392340779305,-0.014670108444988728,0.014451988972723484,-0.04528065770864487,0.2006101757287979,-0.0034944931976497173,-0.0026904612313956022,-0.027397019788622856,-0.0342656709253788,-0.04339589551091194,-0.013380476273596287,0.01489167008548975,0.014835563488304615,0.002884790301322937,-0.01298222690820694,0.012546141631901264,-0.01864139921963215,-0.010470140725374222,-0.03843013569712639,-0.00022819543664809316,-0.01777123287320137,-0.03435760736465454,0.06366635113954544,0.028508776798844337,0.015298877842724323,0.021875018253922462,0.013986802659928799,0.03267659246921539,0.2303493618965149,-0.030970420688390732,-0.048083361238241196,-0.036105088889598846,-0.04462100937962532,-0.03826150298118591,0.03875735029578209,0.019939258694648743,-0.003361293114721775,-0.004006677772849798,0.015730932354927063,0.03576775640249252,0.06355700641870499,0.01792951673269272,-0.0037184590473771095,0.019694028422236443,0.03263738751411438,0.09075096994638443,-0.05632621422410011,-0.004373012110590935,-0.013637233525514603,-0.03061104379594326,-0.047413650900125504,-0.0643952265381813,0.14497493207454681,-0.0031952194403856993,-0.01603119820356369,-0.042802851647138596,-0.009140620008111,-0.06015237420797348,-0.03174988180398941,0.02384427934885025,0.032058656215667725,0.014097800478339195,0.03485243767499924,0.006955413147807121,-0.14512024819850922,-0.04748820140957832,-0.023381799459457397,-0.020661281421780586,-0.02086193859577179,0.027048248797655106,0.09718991070985794,0.019921613857150078,0.013295389711856842,0.025921784341335297,-0.009321870282292366,-0.08140112459659576,0.22188091278076172,-0.012445572763681412,-0.016774926334619522,-0.03391590341925621,-0.03463952988386154,-0.028523627668619156,0.01881370320916176,0.0073534962721168995,-0.00989818386733532,0.019065631553530693,0.009472152218222618,0.007591093424707651,0.06572773307561874,-0.004635029938071966,0.00989461038261652,0.028920015320181847,-0.0035308534279465675,0.026242192834615707,-0.036353468894958496,-0.0066913338378071785,-0.01773710735142231,-0.019660023972392082,0.006395101081579924,-0.008443810045719147,0.24089990556240082,-0.02947668917477131,-0.035742390900850296,-0.028803110122680664,-0.03674484044313431,-0.09725234657526016,-0.13119657337665558,0.0049036466516554356,0.0027225830126553774,-0.003751816228032112,-0.013929316774010658,-0.1432885378599167,-0.006678557954728603,-0.007010793313384056,0.016569895669817924,0.012524553574621677,-0.013286309316754341,-0.017126036807894707,0.11225549131631851,0.00876504834741354,0.014402809552848339,0.023786459118127823,0.013712325133383274,0.032571032643318176,-0.022883949801325798,-0.004468244034796953,0.008498693816363811,-0.023985430598258972,-0.007052906788885593,0.05568218231201172,0.02187148481607437,-0.010606181807816029,0.00023102923296391964,-0.011061369441449642,-0.016053995117545128,0.033170703798532486,0.5556071996688843,-0.03858882188796997,-0.013710994273424149,-0.05429702252149582,-0.05588201433420181,-0.15941789746284485,-0.262715607881546,0.0018391433404758573,-0.0004557025677058846,-0.0013988602440804243,-0.029834270477294922,-0.010248411446809769,-0.0018634528387337923,0.009950068779289722,0.010858642868697643,0.013632682152092457,0.022120516747236252,-0.07336550951004028,0.004594400059431791,0.0005505456356331706,0.002873888937756419,0.009861002676188946,-0.005273778922855854,-0.10360968112945557,-0.014090449549257755,0.011145791970193386,0.009095738641917706,-0.007088849786669016,-0.0069394600577652454,-0.03879232332110405,0.00023300913744606078,-0.006525329779833555,-0.0021058693528175354,-0.005118425004184246,0.001051443163305521,-0.17451243102550507,0.0032799725886434317,-0.004703314043581486,0.0060063316486775875,0.0020635738037526608,0.00201305840164423,0.07939934730529785,0.03195629641413689,0.00834212638437748,0.015490113757550716,-0.00897516030818224,-0.013845957815647125,0.03846129775047302,0.010732090100646019,0.004788190126419067,0.003843985963612795,-0.0069630988873541355,-0.0015118583105504513,0.05849394574761391,0.03211989998817444,-0.041848476976156235,0.0210672989487648,0.044638264924287796,0.046263813972473145,0.0870155468583107,0.0233796127140522,0.026598649099469185,-0.03553019464015961,-0.037956658750772476,0.03234612196683884,0.0741683766245842,-0.1439150869846344,0.00719872023910284,-0.013220153748989105,0.05210452154278755,-0.013264729641377926,-0.04941054806113243,0.013517258688807487,0.03789355233311653,0.0046334597282111645,-0.009770304895937443,0.08380117267370224,-0.030418626964092255,-0.1896195411682129,0.06954430788755417,0.14645953476428986,0.19095340371131897,0.1380070447921753,0.02479124628007412,-0.1509941816329956,0.012940899468958378,-0.006024377420544624,0.016626792028546333,-0.04035947844386101,-0.03489656373858452,-0.021994005888700485,-0.022073877975344658,-0.035922251641750336,-0.04090884327888489,0.014338538981974125,-0.01282384805381298,-0.021899400278925896,-0.026975169777870178,0.034462373703718185,0.006732690613716841,-0.01844770275056362,-0.09613842517137527,0.030168849974870682,0.04999779909849167,0.041961390525102615,0.0026571403723210096,0.027728689834475517,0.005800939165055752,0.009009601548314095,-0.07331099361181259,-0.04827601835131645,-0.018862886354327202,-0.03799434378743172,0.013817772269248962,0.062271952629089355,0.030338821932673454,0.055107828229665756,0.001703150337561965,0.018739236518740654,0.02954413741827011,0.039876945316791534,-0.04127480834722519,-0.07040026038885117,-0.07347475737333298,-0.09938744455575943,-0.02503090538084507,0.0666290894150734,-0.06089073419570923,-0.07631700485944748,-0.09065548330545425,-0.06937438994646072,-0.038877103477716446,0.13381992280483246,-0.03390873596072197,-0.016095826402306557,-0.041282352060079575,-0.008546175435185432,0.04857373610138893,0.10911121219396591,-0.026454772800207138,-0.011454449035227299,-0.0023329749237746,0.06409244239330292,0.002564018592238426,-0.0033579159062355757,0.01770266704261303,-0.01097017526626587,0.0224591176956892,-0.052034277468919754,-0.027944032102823257,-0.02635445073246956,0.019052887335419655,0.017842071130871773,0.017414921894669533,-0.024629289284348488,-0.01846500299870968,0.01579936034977436,0.00923833902925253,0.01767665520310402,0.015263131819665432,0.016860825940966606,0.05995962768793106,0.02274188958108425,-0.0016736635006964207,0.0016355984844267368,0.0045699444599449635,0.01999226212501526,0.04489950090646744,-0.09095194935798645,-0.01537228748202324,0.01078183762729168,0.010832514613866806,0.022846776992082596,-0.023743439465761185,0.014697926118969917,-0.011814904399216175,-0.03336676210165024,-0.024325426667928696,-0.027463706210255623,-0.03928457200527191,-0.07156910747289658,-0.01296036783605814,-0.008615691214799881,-0.018550390377640724,-0.0189194455742836,-0.029360191896557808,-0.05552315711975098,0.002544247778132558,0.00746040977537632,0.004100501537322998,-0.009867154993116856,-0.0331307128071785,-0.1245141476392746,-0.02809918485581875,-0.013163984753191471,-0.013026374392211437,-0.03621744364500046,0.02834121137857437,-0.020626835525035858,0.000864903733599931,0.011351537890732288,-0.001190836657769978,-0.03369950130581856,0.01702287420630455,0.2091432362794876,-0.012023059651255608,-0.004398711025714874,-0.00884163100272417,0.005726051516830921,0.0011452906765043736,-0.040782541036605835,-0.002147400751709938,-0.008656165562570095,-0.0008460938115604222,0.010372436605393887,-0.01847548969089985,-0.5059886574745178,-0.0027394972275942564,0.011615410447120667,-0.0027547208592295647,0.01685914769768715,0.056284479796886444,-0.0809868797659874,0.00019839515152852982,-0.012411970645189285,-0.000300252519082278,0.02534269168972969,-0.013935910537838936,-0.00669061066582799,-0.0049889106303453445,-0.023287659510970116,-0.01860496960580349,0.0017816569888964295,0.05599157512187958,-0.14942502975463867,0.0018869576742872596,0.002638286678120494,0.003841526573523879,0.00032635321258567274,-0.019791336730122566,0.3353439271450043,-0.03078017756342888,-0.030047541484236717,-0.03746933862566948,-0.023553993552923203,-0.042418546974658966,-0.4062248170375824,0.23050102591514587,0.10009860247373581,-0.044005777686834335,0.00909569300711155,0.07492736726999283,-0.04409612715244293,-0.08325572311878204,-0.10509004443883896,0.0007333567482419312,0.005051909945905209,-0.0014124242588877678,0.004342180211097002,0.0038413526490330696,-0.002112558577209711,0.0010413479758426547,0.0073057543486356735,-0.275392085313797,-0.024535654112696648,0.04839122295379639,0.03631570562720299,-0.15082576870918274,0.173878014087677,0.010466182604432106,0.02304045669734478,-0.11567411571741104,-0.14419569075107574,-0.01993751712143421,0.021624885499477386,0.038444384932518005,0.17193427681922913,-0.009944669902324677,0.011007755063474178,0.09889166802167892,0.0414707250893116,0.1311590075492859,0.05223849043250084,-0.02985435537993908,-0.001301532262004912,0.002763835247606039,0.011024664156138897,-0.06601396948099136,0.07517395913600922,-0.003911333158612251,-0.01684488169848919,0.2450936734676361,-0.23876139521598816,-0.038080088794231415,-0.09163588285446167,-0.15474097430706024,0.03794793039560318,0.015365003608167171,0.12660899758338928,0.3292219638824463,0.14394880831241608,-0.04785642772912979,0.16177357733249664,-0.09408446401357651,-0.11093313992023468,-0.049285437911748886,0.08938370645046234,0.19826216995716095,-0.11570803821086884,-0.04607071727514267,-0.0227857306599617,0.22327940165996552,0.018957551568746567,0.18933866918087006,0.058310385793447495,-0.005957947112619877,-0.18845395743846893,-0.020145051181316376,0.23323549330234528,-0.08117760717868805,-0.1369916796684265,-0.1940271407365799,-0.04127868637442589,-0.009719450026750565,0.05875490978360176,0.009299562312662601,-0.18223083019256592,-0.05808820202946663,0.12146685272455215,-0.04341721907258034,0.08897563815116882,0.0479716882109642,-0.07948017120361328,-0.19427672028541565,-0.01015518419444561,0.01724299043416977,0.042263876646757126,0.015159904025495052,-0.03569834306836128,-0.04693872854113579,0.11534754931926727,0.027963465079665184,-0.006211601663380861,0.008407356217503548,0.02890140190720558,0.017622945830225945,0.044218458235263824,-0.014650625176727772,-0.015581237152218819,-0.021398425102233887,-0.027017908170819283,-0.035533614456653595,-0.04051538556814194,-0.017999622970819473,0.036440975964069366,0.04105057567358017,0.03286414593458176,-0.008876404725015163,0.13037584722042084,-0.0385039784014225,-0.020965704694390297,-0.02848326414823532,-0.008283278904855251,-0.03483182191848755,0.19584591686725616,0.007886474952101707,0.006140772718936205,0.012904857285320759,0.038567036390304565,0.02947608008980751,0.059351351112127304,-0.012625642120838165,0.00909190159291029,0.02687462419271469,0.018851637840270996,0.036484260112047195,0.009368100203573704,-0.006803339347243309,0.006508287508040667,0.007849220186471939,-0.003555749310180545,0.006191655062139034,-0.1677422672510147,0.017682507634162903,0.004314655903726816,-0.0029630998615175486,-0.0030395444482564926,0.01040071900933981,0.010005087591707706,-0.01058922614902258,-0.027782121673226357,-0.014993177726864815,0.0019441194599494338,-0.006646219175308943,0.10894609242677689,0.0231398306787014,0.011339278891682625,0.04713118076324463,0.0011034581111744046,0.05146549269556999,-0.07199906557798386,-0.012064899317920208,-0.030631570145487785,-0.0729966089129448,-0.03977666795253754,-0.11793777346611023,0.1320350617170334,0.022271744906902313,0.02701277658343315,0.039566271007061005,-0.01018631923943758,-0.0026437928900122643,0.3431195318698883,-0.015195832587778568,-0.016806645318865776,-0.03550601378083229,-0.009579414501786232,-0.00926705077290535,0.001853586407378316,0.004663275089114904,-0.014561458490788937,-0.009767596609890461,-0.009015575051307678,-0.007968675345182419,0.027319051325321198,0.020714756101369858,0.019444728270173073,0.027983766049146652,0.01186680793762207,0.016486922279000282,-0.09707916527986526,0.011085133999586105,-0.0044005378149449825,-0.026134537532925606,-0.004434176255017519,0.020794633775949478,0.010693946853280067,-0.020345283672213554,-0.01474146731197834,-0.010924456641077995,-0.019152594730257988,-0.04427171126008034,-0.11121316999197006,0.009642243385314941,-0.007111645303666592,0.0053225005976855755,0.0036616430152207613,-0.007300041150301695,0.13126026093959808,-0.009303873404860497,0.011158652603626251,-0.004905465990304947,0.002908842172473669,-0.017313869670033455,-0.05114299803972244,-0.006417714990675449,0.005327598657459021,0.02574833296239376,0.020999906584620476,0.0251547172665596,0.034895747900009155,-0.005435539875179529,0.010535869747400284,0.0009073077235370874,0.004543366841971874,0.013456917367875576,0.09119337797164917,-0.00477601308375597,-0.0024310906883329153,-0.016300657764077187,-0.003786597866564989,-0.025640428066253662,0.19620008766651154,-0.02994534559547901,-0.005274175200611353,-0.018597010523080826,-0.00983359944075346,-0.011168025434017181,-0.18385452032089233,-0.0034060871694236994,0.0170842744410038,-0.004943564534187317,0.000519079971127212,-0.013651848770678043,-0.06300520896911621,-0.01427283976227045,-0.005832212045788765,-0.0029291100800037384,-0.0060793026350438595,-0.02714797668159008,-0.0705958902835846,0.013643954880535603,-0.017904940992593765,0.010790546424686909,0.004560125060379505,-0.0001039734052028507,0.08781176805496216,-0.008193905465304852,0.0150002995505929,0.002440237905830145,-0.0035718369763344526,-0.018833275884389877,-0.10840674489736557,0.005495637655258179,-0.0019194368505850434,0.00841866061091423,0.012449887581169605,0.046700749546289444,-0.06309395283460617,-0.0249144546687603,0.014968800358474255,-0.024087633937597275,-0.02625073865056038,-0.026033101603388786,0.034669868648052216,-0.021042607724666595,0.01630007103085518,-0.00692060636356473,0.0008879808592610061,0.001285730628296733,0.1256706565618515,-0.016706472262740135,0.016328351572155952,0.0027431563939899206,0.018690360710024834,0.01568589173257351,-0.0426686592400074,-0.11013001203536987,-0.11718959361314774,-0.13932691514492035,-0.05557222664356232,-0.0867435559630394,0.11705099791288376,0.046261128038167953,-0.03678567335009575,-0.06094730645418167,-0.05533411726355553,-0.059664659202098846,-0.11398161202669144,-0.03545654937624931,0.07224413007497787,-0.01276514120399952,0.028232159093022346,-0.008347864262759686,0.1547425538301468,0.06055193766951561,0.06919635087251663,0.0895194560289383,0.06917379051446915,0.04736671596765518,-0.12217619270086288,0.0909496620297432,0.1129753366112709,0.18094348907470703,0.10514158010482788,0.18189659714698792,-0.09986882656812668,-0.02476632036268711,0.023835262283682823,-0.0030356321949511766,-0.050924018025398254,-0.07884053140878677,-0.2820051610469818,-0.05973457917571068,6.913243851158768e-05,0.025470538064837456,0.004406035877764225,-0.047152742743492126,-0.17243850231170654,-0.016205819323658943,-0.022742919623851776,0.03271816670894623,0.013815398328006268,-0.01083410531282425,-0.0739288479089737,-0.011886403895914555,0.0029942188411951065,-9.167320058622863e-06,0.017927374690771103,-0.011696030385792255,-0.0506887286901474,0.012593647465109825,0.008583742193877697,0.014982873573899269,-0.0008682282641530037,0.01261997688561678,-0.0016271329950541258,0.0017698407173156738,0.010563709773123264,-0.01277267374098301,-0.013953592628240585,0.0018247927073389292,0.09535188227891922,0.006349275819957256,0.00763245252892375,0.009770205244421959,0.01272447220981121,0.012976045720279217,0.03614890202879906,-0.015372815541923046,-0.014021149836480618,-0.013163008727133274,-0.022240910679101944,-0.016251442953944206,-0.06764762103557587,0.011176686733961105,0.012415510602295399,0.0013008639216423035,-0.010886664502322674,0.008275353349745274,0.03825487568974495,-0.0038903416134417057,0.005439141299575567,-0.0006672628223896027,0.010169538669288158,-0.007907489314675331,-0.0029931182507425547,0.0017740243347361684,0.0100462157279253,0.010067782364785671,-0.004805375821888447,-0.01013159565627575,0.005408281926065683,-0.011014205403625965,-0.03941473737359047,-0.02210618369281292,0.011793435551226139,0.043130986392498016,-0.07345002144575119,-0.025973381474614143,0.028636394068598747,0.024754170328378677,-0.013895412907004356,0.004359425511211157,-0.14542293548583984,-0.007002905942499638,0.013520041480660439,-0.010052036494016647,0.03168641775846481,0.006690339185297489,0.06734947860240936,0.01842753030359745,-0.0402647890150547,-0.023494895547628403,-0.01337770652025938,0.030024845153093338,-0.18135443329811096,0.029754742980003357,0.008585725910961628,0.002453901804983616,0.004200720228254795,0.04097062349319458,-0.19909894466400146,0.004376091528683901,-0.008473061956465244,0.009900901466608047,-0.028421808034181595,-0.032426659017801285,0.013683685101568699,0.0008977777324616909,-0.007169677410274744,-0.02208080142736435,-0.0085725924000144,-0.03760578855872154,0.0038352764677256346,-0.00039519101846963167,0.005718492902815342,-0.004002552013844252,0.012713485397398472,0.01781505160033703,0.07685776054859161,-0.0019522493239492178,-0.013388345018029213,-0.00980647373944521,-0.008310369215905666,-0.011612096801400185,-0.020650452002882957,0.012549135833978653,0.013126389123499393,0.007081991992890835,0.004834112245589495,0.008338079787790775,0.04215119034051895,0.008927673101425171,0.008195534348487854,0.008983736857771873,0.004960441030561924,0.011182352900505066,0.07077065110206604,-0.013723624870181084,0.000565190042834729,-0.0018409285694360733,-0.006119679659605026,-0.0031458097510039806,0.03661063313484192,0.0021464810706675053,0.0021615310106426477,0.002807852579280734,0.008933259174227715,0.003637404413893819,-0.0708063468337059,-0.012055350467562675,-0.010391667485237122,-0.008633742108941078,-0.00876812543720007,-0.008354545570909977,0.02799099311232567,-0.012715989723801613,-0.01483831275254488,-0.019320065155625343,-0.018411310389637947,-0.022007398307323456,-0.0690794512629509,0.020997339859604836,0.03338547796010971,0.0267240758985281,0.019338075071573257,0.028635531663894653,0.03165632486343384,9.792916534934193e-05,0.12545299530029297,0.0377853661775589,0.09074932336807251,0.13942459225654602,-0.013528630137443542,0.0157326590269804,0.02544596418738365,-0.0022191640455275774,-0.001601475989446044,0.006202720571309328,0.0041688974015414715,0.010178646072745323,0.0025294534862041473,-0.0035215243697166443,-0.032528311014175415,-0.0942002609372139,-0.04238176345825195,0.033168110996484756,0.07309529185295105,-0.13011743128299713,0.015291470102965832,0.073981873691082,0.013548502698540688,-0.11160364001989365,-0.08628488332033157,-0.06509514153003693,-0.02106185257434845,0.13635753095149994,0.019639533013105392,0.010698235593736172,-0.0025883398484438658,-0.05462116003036499,0.07468706369400024,0.048207081854343414,0.06718871742486954,-0.2451208084821701,-0.0009594136499799788,0.07413079589605331,0.012392736971378326,0.1209949254989624,-0.08628728240728378,-0.04848284274339676,-0.002237459411844611,0.12068767845630646,0.023400520905852318,-0.051988258957862854,-0.0062837982550263405,-0.03499460965394974,-0.08345954120159149,0.015148344449698925,0.05958547815680504,0.25748592615127563,0.03330545499920845,0.10161624103784561,0.0029108570888638496,-0.0007281217258423567,0.07392366230487823,0.011205444112420082,0.015176712535321712,0.46066519618034363,0.038724999874830246,-0.0797390267252922,-0.0046591744758188725,0.30032679438591003,0.025655191391706467,0.05383702367544174,0.054368384182453156,0.22733436524868011,0.03315075859427452,-0.1482374668121338,0.009623775258660316,0.03719234839081764,-0.04660721868276596,-0.07960981130599976,-0.02078857272863388,-0.010492061264812946,-0.015367396175861359,0.016949228942394257,0.005338029004633427,0.11179859191179276,0.050914764404296875,0.054633282124996185,-0.12890930473804474,-0.03050544671714306,-0.08904657512903214,-0.24523650109767914,0.010333511047065258,0.11448045074939728,0.12425045669078827,0.12160679697990417,0.09364241361618042,0.04268277436494827,-0.05782640725374222,0.0003247462445870042,-0.006744183599948883,-0.027757830917835236,0.004633112344890833,-0.006186287384480238,0.1460389792919159,0.06868583709001541,0.04746004566550255,0.06535546481609344,0.011057439260184765,0.029907336458563805,-0.05536673963069916,-0.09185267984867096,-0.11266034841537476,-0.08599455654621124,-0.06912866234779358,-0.06904056668281555,0.11338517814874649,-0.06466536223888397,-0.05438216030597687,-0.0321606807410717,-0.03835387900471687,0.0006681310478597879,0.1676410287618637,-0.09132909774780273,-0.1039084792137146,-0.12346851080656052,-0.06503429263830185,-0.07617665082216263,0.24360615015029907,-0.06178978458046913,-0.05921035632491112,-0.05982942134141922,-0.06863819807767868,-0.04319486767053604,0.07135019451379776,0.0177970789372921,-0.009828266687691212,-0.019116122275590897,-0.019724421203136444,-0.016682233661413193,-0.008967449888586998,0.010328744538128376,-0.015505097806453705,0.030859265476465225,0.0611283965408802,0.054512251168489456,0.05441056936979294,-0.04027821496129036,0.0019416416762396693,-0.026843534782528877,-0.02136397734284401,-0.026433123275637627,0.1346115618944168,0.015965139493346214,0.001097888802178204,0.036647405475378036,0.023849602788686752,0.04586226865649223,-0.19679982960224152,-0.024925492703914642,-0.057726532220840454,-0.07792245596647263,-0.10822901874780655,-0.06688018143177032,0.22370684146881104,0.042763449251651764,0.04252224788069725,0.017373977228999138,0.002639883430674672,0.018446438014507294,0.23975352942943573,-0.05270219221711159,-0.0630129799246788,-0.09367179870605469,-0.030099568888545036,-0.08669130504131317,0.15294969081878662,0.009849626570940018,-0.023792780935764313,-0.03904944658279419,-0.019963338971138,-0.016613097861409187,0.034087326377630234,0.007186091039329767,0.014379080384969711,0.030559832230210304,0.013410639949142933,0.016382433474063873,-0.0011626173509284854,-0.004025137517601252,0.008222192525863647,-0.022648947313427925,-0.0055024889297783375,-0.019363775849342346,0.1313306838274002,-0.011952945962548256,-0.0063632093369960785,-0.009746734984219074,-0.030895689502358437,-0.03526097536087036,-0.25344809889793396,-0.003575482638552785,-0.005897478200495243,0.001964441267773509,-0.005356703884899616,0.006487804464995861,0.12416578084230423,-0.0031918580643832684,0.014155126176774502,0.011516734026372433,-0.005875203292816877,0.03442119061946869,0.008126339875161648,0.0026107740122824907,-0.01892644353210926,0.006364447996020317,0.00019012064149137586,-0.0064878617413342,0.05756644904613495,0.020335864275693893,0.027734391391277313,0.02140646241605282,0.0007464279187843204,0.02554033324122429,-0.06700784713029861,-0.0017386064864695072,0.007982168346643448,0.004461427219212055,-0.016738668084144592,-0.0008030344033613801,0.4914581775665283,-0.019557863473892212,-0.016960302367806435,-0.017586302012205124,-0.04936058819293976,-0.036390338093042374,-0.4075899124145508,-0.023662541061639786,0.0028370146173983812,-0.023271964862942696,-0.029639508575201035,0.011128012090921402,-0.0869663655757904,-0.0006923466571606696,0.00889733899384737,-0.006394500378519297,-0.0198917668312788,0.011178779415786266,-0.02860669605433941,0.0073844389989972115,-0.00895628985017538,-0.005004040896892548,0.01723422482609749,0.01170434895902872,0.1879182755947113,0.006393710151314735,0.0059425244107842445,0.026800842955708504,-0.0052437931299209595,-0.020735645666718483,-0.0019130909349769354,0.020468711853027344,-7.458893378498033e-06,0.026824137195944786,0.030678218230605125,0.009541845880448818,-0.006116844248026609,0.013326662592589855,0.02552269957959652,0.028669502586126328,0.010966461151838303,0.02914271131157875,0.21747271716594696,-0.0007507265545427799,0.01649029739201069,0.00740209361538291,-0.004641481209546328,0.011169585399329662,-0.0777994766831398,0.017017725855112076,0.015664150938391685,0.028619250282645226,0.01078046765178442,0.039544906467199326,0.04870909824967384,-0.003485326422378421,0.010308867320418358,-0.029851987957954407,-0.07150302082300186,-0.04844297468662262,-0.023005083203315735,0.028458410874009132,-0.00015102769248187542,0.03804453834891319,0.03867438808083534,0.013918516226112843,-0.154847651720047,-0.026058634743094444,0.003907315898686647,-0.020016860216856003,-0.018813135102391243,-0.012658033519983292,0.02487216517329216,0.025076543912291527,0.041734080761671066,0.06809235364198685,0.10148991644382477,0.03978944569826126,-0.22921648621559143,-0.039258334785699844,-0.022150376811623573,-0.021312205120921135,-0.014171822927892208,0.0009829486953094602,-0.20310580730438232,0.051731374114751816,0.05383731797337532,0.060763534158468246,0.034381985664367676,0.04232524707913399,-0.08187706768512726,-0.02529793232679367,0.027310045436024666,0.0340648777782917,0.03396442532539368,0.004772127140313387,0.029454626142978668,-0.000803868577349931,0.009846664033830166,-0.024375731125473976,-0.014721965417265892,-0.014966744929552078,0.016035171225667,0.011300701647996902,-0.011527211405336857,0.0012701291125267744,-0.010072462260723114,-0.011735544539988041,-0.046225834637880325,-0.020304283127188683,0.003796136472374201,-0.01203887164592743,0.014595548622310162,0.01737922430038452,-0.0013355989940464497,-0.009925000369548798,0.00710147712379694,-0.01899125427007675,0.01516297459602356,-0.019011588767170906,0.11313976347446442,0.00039441982517018914,0.0029847656842321157,0.00020179289276711643,-0.007716394029557705,0.008294861763715744,0.037582434713840485,0.007129215635359287,-0.004180521704256535,0.007692186161875725,-0.0020526214502751827,-0.004952394403517246,-0.07296004891395569,-0.01437404379248619,-0.0034208556171506643,-0.01792566291987896,-0.017220791429281235,-0.002654721960425377,0.04250367730855942,0.05825848877429962,-0.012537665665149689,0.002834965707734227,0.01600404642522335,0.005755062215030193,-0.055185724049806595,-0.06006703898310661,-0.012503367848694324,-0.031975045800209045,0.009225625544786453,-0.024982409551739693,-0.012010078877210617,-0.10782475769519806,-0.11852604895830154,-0.14168183505535126,-0.08233456313610077,-0.10249018669128418,0.013714071363210678,-0.010652037337422371,0.023197485134005547,0.03073493018746376,-0.018167106434702873,0.035124268382787704,-0.2727327346801758,-0.05867573246359825,-0.0663982629776001,-0.09811539947986603,-0.05107228830456734,-0.030411621555685997,0.14302967488765717,0.0927843376994133,0.12232039868831635,0.11072792857885361,0.11899751424789429,0.10100807249546051,-0.1523439884185791,0.0997471958398819,0.041176337748765945,0.03685615956783295,0.029126785695552826,0.017879320308566093,-0.1897706836462021,0.11063782125711441,0.1678255945444107,0.2118827998638153,0.16349606215953827,0.10618198662996292,-0.3277449607849121,0.15463051199913025,0.08202524483203888,0.0709569975733757,0.10596568137407303,0.08182942122220993,-0.17228735983371735,-0.0032996765803545713,0.019773194566369057,0.036960095167160034,0.04275963827967644,0.013740533962845802,-0.06960142403841019,0.0023783401120454073,-1.7576161553733982e-05,-0.0007651306223124266,-0.004746968857944012,0.0030756837222725153,-0.0021406440064311028,0.01425481028854847,0.009646417573094368,0.010907648131251335,0.005429358687251806,-0.0013157789362594485,-0.0010092910379171371,0.01703874208033085,0.010033070109784603,0.014507213607430458,0.007728327065706253,0.019186463207006454,0.059578318148851395,-0.011272978037595749,-0.011535115540027618,-0.013469075784087181,-0.004261546768248081,-0.013857733458280563,0.10193337500095367,0.00025765824830159545,0.0062568592838943005,0.004198969341814518,0.006065502297133207,0.005933748092502356,-0.13601060211658478,-0.022967956960201263,-0.005845923908054829,-0.017358142882585526,-0.011271278373897076,-0.018378641456365585,0.042040981352329254,0.0027185254730284214,0.002053867094218731,0.0030683360528200865,-0.0027860510163009167,-0.008125360123813152,0.05257219448685646,0.006448062136769295,0.006156533490866423,-0.004340403247624636,0.00236941734328866,0.0019137135241180658,0.0031253660563379526,-0.06218269094824791,0.041191767901182175,-0.05338490754365921,0.07199648767709732,0.011205380782485008,0.06515459716320038,0.05349208787083626,-0.02578979730606079,0.006080546882003546,-0.00700960960239172,0.0015848121838644147,0.0069084507413208485,-0.003563456004485488,0.0026795503217726946,0.006155161652714014,-0.010146548971533775,0.029626281931996346,0.028663694858551025,0.11745745688676834,-0.003066814038902521,-0.017439208924770355,0.08046802133321762,0.08971207588911057,0.005303369369357824,-0.21921251714229584,-0.14221709966659546,-0.061608318239450455,0.038519248366355896,0.08189882338047028,0.05959154665470123,0.29974234104156494,-0.0013076213654130697,0.20427945256233215,0.07673569023609161,-0.007614254951477051,0.016920147463679314,-0.07006558775901794,-0.0368339903652668,-0.1483737677335739,-0.018699556589126587,-0.2426563799381256,-0.1364288628101349,-0.003359416965395212,-0.010709855705499649,0.08923576772212982,0.01235034130513668,0.15950210392475128,0.00040370551869273186,-0.23388700187206268,-0.13660027086734772,0.06707119196653366,0.04306285083293915,0.1597978174686432,-0.00958574190735817,0.3723754286766052,0.013985920697450638,-0.3663635551929474,-0.21961575746536255,0.010979163460433483,0.04053511470556259,0.14264464378356934,-0.0034465172793716192,0.3525424599647522,-0.006113084033131599,-0.18489545583724976,-0.03509024158120155,0.10686521232128143,-0.12555387616157532,0.04537711665034294,0.08385082334280014,0.2103237807750702,0.005713424179702997,-0.231953963637352,-0.0723176822066307,-0.04219165816903114,0.060241684317588806,0.06123334914445877,0.0304730162024498,0.32447293400764465,-0.010102994740009308,-0.022992756217718124,0.012807494029402733,0.010546107776463032,-0.014394705183804035,-0.03718918561935425,-0.018680864945054054,0.04187014698982239,0.027599090710282326,-0.00268796319141984,0.017776217311620712,0.04090813919901848,-0.005364298354834318,0.04947458952665329,-0.0435117669403553,-0.003140015061944723,0.007458202075213194,-0.014580128714442253,-0.018526526167988777,-0.03440629690885544,0.17879796028137207,0.036154236644506454,0.03282518684864044,0.05309383198618889,0.04055217653512955,0.03083103336393833,-0.11129281669855118,-0.03664407506585121,-0.016583049669861794,-0.03298044949769974,-0.009288783185184002,-0.06988170742988586,0.19744177162647247,-0.11299306154251099,-0.07852417230606079,-0.09976125508546829,-0.08324194699525833,-0.05236712843179703,0.17944534122943878,-0.0901859849691391,-0.06356828659772873,-0.08646915853023529,-0.07168365269899368,-0.05384772643446922,0.28777825832366943,-0.0637948289513588,-0.03568407520651817,-0.04617100581526756,-0.04284103214740753,-0.04665651544928551,0.011630275286734104,-0.08474361151456833,-0.031095042824745178,-0.03687189519405365,-0.07007712125778198,-0.013424440287053585,0.03329426050186157,-0.019392216578125954,-0.01915808580815792,-0.02461007609963417,0.011779531836509705,0.049158208072185516,-0.0937436893582344,-0.06681003421545029,-0.05457579717040062,-0.04437149316072464,-0.05250117927789688,-0.04939209297299385,0.1448129266500473,0.031421829015016556,0.03163162246346474,0.03377465903759003,0.04529856890439987,0.048649441450834274,-0.10359629988670349,-0.03788068890571594,-0.04155135899782181,-0.04519006609916687,-0.06850611418485641,-0.07284943014383316,0.12104155123233795,0.052874043583869934,0.07108011096715927,0.05715523287653923,0.04519796743988991,-0.024671634659171104,0.30100417137145996,-0.03224984556436539,-0.03180563077330589,-0.027995530515909195,-0.038809943944215775,-0.07244554907083511,0.2234567552804947,-0.018064530566334724,0.0018378415843471885,-0.016708044335246086,-0.027551904320716858,-0.03837365284562111,-0.05775393918156624,-0.020284250378608704,0.01230517216026783,-0.0002422887773718685,0.018687142059206963,-0.013151231221854687,0.17799830436706543,0.028615009039640427,-0.005177528131753206,0.0019185804994776845,0.002243173774331808,-0.02142484299838543,0.25282764434814453,0.021161528304219246,-0.004439355805516243,0.013507930561900139,0.014012801460921764,0.013345506973564625,-0.23197774589061737,-0.044103577733039856,-0.004091175738722086,-0.009212651289999485,-0.0088458601385355,-0.013163934461772442,-0.0224470142275095,0.02718370594084263,0.005059719085693359,0.013153038918972015,0.030817069113254547,0.033478207886219025,0.05224055051803589,0.018229778856039047,-0.020828833803534508,-0.01730961911380291,-0.030988123267889023,-0.016684046015143394,0.034094344824552536,0.04013039171695709,-0.014460685662925243,-0.009738052263855934,-0.003923170734196901,0.01914525032043457,-0.09994383901357651,0.06455761939287186,0.02226618491113186,0.021331243216991425,0.013412284664809704,0.0252443365752697,0.650275468826294,0.027769815176725388,-0.049601662904024124,-0.016668379306793213,-0.017407502979040146,-0.016928087919950485,-0.3887535035610199,-0.02497466467320919,-0.02298174798488617,-0.01940763182938099,-0.011202111840248108,-0.023101994767785072,-0.019845493137836456,-0.0008675153949297965,0.0007249414338730276,0.0001379223249386996,-0.004124574363231659,-0.006267320830374956,-0.003943646792322397,0.020823510363698006,0.010490359738469124,0.007365534082055092,0.012333299964666367,-0.0013451679842546582,0.29167675971984863,-0.014208385720849037,-0.008551781065762043,-0.005099347326904535,-0.01107007171958685,0.010967493988573551,-0.03818399831652641,0.010381338186562061,0.02315337583422661,0.02219008281826973,0.02024601213634014,0.014508624561131,-0.04982247203588486,-0.011351985856890678,0.0007650617626495659,0.003634457243606448,-0.005387656390666962,0.015027535147964954,0.11331234872341156,-0.017208417877554893,-0.006872826721519232,-0.001532002817839384,-0.013867226429283619,-0.000458324997453019,0.04920386150479317,-0.013509901240468025,0.005840553902089596,0.005864591337740421,-0.004296083003282547,0.0037982144858688116,0.057409144937992096,-0.009502925910055637,-0.0084905456751585,-0.011469290591776371,0.00013988451974000782,-0.040056295692920685,-0.004871339071542025,-0.001586836646310985,-0.0039975508116185665,-0.0010785851627588272,0.01231304556131363,0.012568820267915726,0.001853279653005302,0.029399771243333817,0.023136718198657036,0.003499030601233244,-0.00374264782294631,-0.0012380329426378012,0.006518077105283737,-0.028183365240693092,-0.01766122691333294,-0.00679159676656127,-0.0010886085219681263,-0.014969906769692898,-0.0693328008055687,-0.01130959577858448,-0.017198488116264343,-0.013031206093728542,-0.011451133526861668,0.036406274884939194,-0.013588925823569298,-0.046877309679985046,-0.04810356721282005,-0.04015170782804489,-0.019236981868743896,1.040046936395811e-05,-0.04444758966565132,-0.0003033973334822804,-0.020912863314151764,-0.013585967943072319,0.0009578438475728035,0.004389723297208548,0.02001989632844925,0.0044698226265609264,-0.0045863124541938305,-0.0025854306295514107,0.013531316071748734,0.007958807051181793,-0.003631585044786334,0.0013225217116996646,-0.005828208290040493,-0.007292077876627445,-0.0027333195321261883,0.00889604538679123,-0.005092503968626261,-0.006708827335387468,-0.004461392294615507,-0.0017096595838665962,-0.0028510787524282932,-0.023244859650731087,0.009590051136910915,-0.0018788465531542897,0.006183380261063576,-0.001674738246947527,-0.005257157608866692,-0.0034137971233576536,0.13734930753707886,-0.001287175458855927,-0.008780678734183311,-0.00726293446496129,-0.0009037030395120382,-0.016552193090319633,0.04935471713542938,0.01024599839001894,0.013996490277349949,0.014689105562865734,0.0031431487295776606,0.009564066305756569,-0.1248442530632019,0.004291372373700142,-0.010165131650865078,-0.0018595920410007238,-0.00018473766976967454,-0.014919230714440346,0.0650278776884079,0.022121747955679893,0.024386735633015633,0.015694720670580864,0.012430085800588131,0.009554327465593815,-0.003830188186839223,-0.00627843989059329,-0.009250467643141747,-0.008496110327541828,-0.0106721306219697,-0.030782105401158333,-0.004007773008197546,0.0052697849459946156,-0.01678869128227234,-0.04048793762922287,0.0068121254444122314,-0.06833252310752869,0.022042235359549522,0.005216095596551895,-0.009248810820281506,0.019922195002436638,0.015171191655099392,0.03969822824001312,-0.23031845688819885,-0.05681830644607544,-0.038170721381902695,-0.05162934213876724,-0.05146745964884758,-0.05280754715204239,0.15611065924167633,0.04507214203476906,0.020026054233312607,0.03474081680178642,0.021402794867753983,0.10327979922294617,-0.242219939827919,0.11729465425014496,0.09550047665834427,0.1027306541800499,0.10605807602405548,0.057209312915802,-0.19955278933048248,0.09520876407623291,0.07036363333463669,0.09256355464458466,0.10352896898984909,0.09294313192367554,-0.3544035851955414,0.08021260052919388,0.046312540769577026,0.051001034677028656,0.05788564682006836,0.060675933957099915,-0.007136733736842871,0.07942794263362885,0.04167438670992851,0.04052775725722313,0.0807342529296875,-0.0029306812211871147,-0.07263459265232086,0.05608174204826355,0.015759622678160667,0.004749262239784002,0.01040139514952898,0.013737790286540985,0.12561920285224915,0.10969765484333038,0.01379009336233139,0.04429234191775322,0.05205632373690605,0.07666847109794617,-0.04819454252719879,-0.07074148207902908,0.007557679433375597,-0.017074845731258392,-0.012819772586226463,0.01363682933151722,-0.3081345856189728,0.0669664517045021,0.01812131702899933,0.03532964363694191,0.035604264587163925,0.009379168972373009,-0.03437117487192154,-0.0014802617952227592,-0.0798124223947525,-0.07646110653877258,-0.06655829399824142,-0.0706103965640068,-0.13985086977481842,0.07387411594390869,-0.07837601006031036,-0.024035325273871422,-0.01601932756602764,-0.039086636155843735,-0.2149616926908493,0.12320676445960999,0.017768843099474907,0.023789698258042336,0.032352957874536514,0.02798447385430336,0.3077956736087799,0.08096340298652649,-0.04702970013022423,-0.03741580992937088,-0.013677145354449749,-0.022937078028917313,-0.26344144344329834,-0.06279647350311279,0.07123956829309464,-0.0857352465391159,0.061965297907590866,0.035018354654312134,0.03431125730276108,0.09075736254453659,0.08306217938661575,0.0019967108964920044,-0.012413154356181622,-0.008779071271419525,0.010217228904366493,-0.011091870255768299,0.00729399174451828,0.0015728480648249388,-0.00474591227248311,1.0186072168340239e-38,1.0746695250156526e-38,3.9963406696325166e-39,2.0052447901134021e-38,1.6378453522444e-38,-1.648565005236792e-38,-1.0874842593420566e-38,-1.4295978269417032e-38,-1.4096024188945595e-38,7.714728183672348e-39,-1.8433121408744906e-38,1.9817957418710835e-38,3.9770644079572644e-39,-1.0004056109510624e-38,1.501600325544566e-38,1.678350585004941e-38,-7.890808342207083e-39,1.70995126667393e-38,-1.5935745502505253e-38,-1.241019767663348e-38,-1.5899821815073822e-38,4.914165940392914e-39,-1.1304039293566293e-38,2.7186465389503987e-39,7.468263605871507e-39,-6.399347532090679e-39,4.736963341788255e-40,1.5665568152091107e-38,-1.4570295055494794e-39,-1.1151098776572954e-38,1.9565230438072926e-38,-8.433224353076398e-40,1.1287906144346521e-38,-1.07216988881499e-39,7.694201963766919e-39,3.4659590099748236e-39,5.2535646323477776e-39,-7.87002848727961e-39,3.3715269077624385e-39,-5.048253387847227e-39,3.029008925425988e-39,1.1600639527823748e-38,2.6553484860183824e-39,7.311554997307598e-40,4.193132019469972e-39,1.527973743552084e-38,-3.5078312093873135e-39,-1.4847927313739148e-38,2.3181049911016503e-39,-1.2615506118537103e-38,-1.1998744217643035e-39,2.5405681288055366e-40,1.5232908843440034e-38,-1.0054690628220537e-38,3.625030207749584e-39,1.1239475868120991e-38,5.516507078790615e-39,9.939100320495312e-39,-1.8163770824424711e-40,-3.322140945984239e-39,7.212784475049663e-39,-2.1211188607776534e-39,-2.423367729144802e-39,1.5250405456065593e-38,1.7394344462334776e-38,-8.018582940079613e-39,-1.6592822760217946e-38,-1.7013016117730386e-38,-3.278023866431901e-39,1.66615438382069e-38,-1.9026712839531363e-38,3.75875892279703e-39,0.002105402760207653,0.0021524899639189243,0.0066046882420778275,0.015485798008739948,0.02048439346253872,0.026210831478238106,0.035964831709861755,0.04133031144738197,0.042900849133729935,0.04532887786626816,0.04158208146691322,0.0289973895996809,0.019111942499876022,0.01040467619895935,0.0017236554995179176,-0.001677026622928679,0.00133381353225559,0.002751134568825364,0.004642687272280455,0.009968235157430172,0.017373664304614067,0.023292331025004387,0.029760872945189476,0.03677051141858101,0.039935242384672165,0.040587227791547775,0.0409630686044693,0.03615052253007889,0.027798913419246674,0.019182933494448662,0.011166532523930073,0.0033989297226071358,0.004674665629863739,0.002508667530491948,0.0027805003337562084,0.009089983068406582,0.005192168056964874,0.007154455874115229,0.010722311213612556,0.010130348615348339,0.01292505580931902,0.0033223216887563467,0.006331641227006912,-0.005408005323261023,-0.023166000843048096,-0.035156331956386566,-0.01459423266351223,-7.866859959904104e-05,0.005719808395951986,0.008986661210656166,0.014288077130913734,0.01909140683710575,0.017126472666859627,0.01810835301876068,0.020928019657731056,0.012994794175028801,0.0015449938364326954,-0.011299903504550457,-0.031168563291430473,-0.04373184219002724,-0.03867826983332634,-0.03130693361163139,-0.022020764648914337,-0.008578051812946796,-0.0013187170261517167,-0.00410851463675499,-0.007373117841780186,-0.013011344708502293,-0.020997747778892517,-0.027860520407557487,-0.03435580059885979,-0.04086102917790413,-0.043405186384916306,-0.0424937941133976,-0.04021992161870003,-0.03365069627761841,-0.02410084567964077,-0.015562263317406178,-0.008522171527147293,-0.002084582345560193,-0.00025239394744858146,0.001326124300248921,4.0544804505771026e-05,0.0005001064273528755,0.004275680519640446,0.005808766465634108,0.007585893385112286,0.011135957203805447,0.007796776480972767,-0.0012274776818230748,-0.005804082378745079,-0.008575319312512875,-0.01106658112257719,-0.006893262267112732,-0.0007629985921084881,0.0004665535525418818,-0.002140045864507556,-0.006125465501099825,-0.007649112958461046,-0.013374852016568184,-0.022036204114556313,-0.024686619639396667,-0.03255839645862579,-0.04130873084068298,-0.04154517501592636,-0.04029945656657219,-0.039622511714696884,-0.030358489602804184,-0.018562018871307373,-0.011499913409352303,-0.00608939491212368,0.00029405858367681503,-0.001983680995181203,-0.007027945015579462,-0.008903254754841328,-0.014589381404221058,-0.02141682058572769,-0.019293010234832764,-0.025776918977499008,-0.032348040491342545,-0.010892681777477264,-0.009521812200546265,0.01932375505566597,0.036592770367860794,0.052840705960989,0.02423705905675888,0.015388608910143375,0.0015029978239908814,0.08536387234926224,0.013119162060320377],"sample_rate":48000.0}}]},"weights":[],"sample_rate":48000.0,"metadata":{"date":{"year":2026,"month":5,"day":17,"hour":9,"minute":1,"second":28},"loudness":-12.049386978149414,"gain":0.612789648007463,"gear_make":"BOSS SD-1 Distortion","gear_model":"BOSS SD-1 Distortion","tone_type":"distortion","gear_type":"pedal"}} \ No newline at end of file diff --git a/Utils/Models/README.md b/Utils/Models/README.md index c00565d..9246273 100644 --- a/Utils/Models/README.md +++ b/Utils/Models/README.md @@ -1,3 +1,5 @@ -Licensing note: +# Licensing Note -The "tw40_blues_deluxe_deerinkstudios.json" model was created by Deer Ink Studios and is under the CC BY-NC-ND 4.0 license. +These models are provided under the [CC BY-NC-ND 4.0 license](https://creativecommons.org/licenses/by-nc-nd/4.0/deed.en). + +The "tw40_blues_deluxe_deerinkstudios.json" model was created by Deer Ink Studios. diff --git a/deps/NeuralAmpModelerCore b/deps/NeuralAmpModelerCore index e181f61..896c5f1 160000 --- a/deps/NeuralAmpModelerCore +++ b/deps/NeuralAmpModelerCore @@ -1 +1 @@ -Subproject commit e181f61efb8d05d34add45b5eecb3893ff21177c +Subproject commit 896c5f17f0fd00c0af056e18a4fe66f19cfbeb52 diff --git a/deps/RTNeural-NAM/LICENSE b/deps/RTNeural-NAM/LICENSE deleted file mode 100644 index c761450..0000000 --- a/deps/RTNeural-NAM/LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2024, jatinchowdhury18 -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/deps/RTNeural-NAM/wavenet/arena.hpp b/deps/RTNeural-NAM/wavenet/arena.hpp deleted file mode 100644 index 1aa03ca..0000000 --- a/deps/RTNeural-NAM/wavenet/arena.hpp +++ /dev/null @@ -1,146 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace wavenet -{ -/** - * Returns the next pointer with a given byte alignment, - * or the base pointer if it is already aligned. - */ -template -Type* snap_pointer_to_alignment (Type* base_pointer, - Integer_Type alignment_bytes) noexcept -{ - return (Type*) ((((size_t) base_pointer) + (alignment_bytes - 1)) & ~(alignment_bytes - 1)); -} - -static constexpr int default_byte_alignment = 16; - -/** - * A simple memory arena. By default the arena will be - * backed with a vector of bytes, but the underlying - * memory resource can be changed via the template argument. - */ -template > -class Memory_Arena -{ -public: - Memory_Arena() = default; - - /** Constructs the arena with an initial allocated size. */ - explicit Memory_Arena (size_t size_in_bytes) { resize (size_in_bytes); } - - Memory_Arena (const Memory_Arena&) = delete; - Memory_Arena& operator= (const Memory_Arena&) = delete; - - Memory_Arena (Memory_Arena&&) noexcept = default; - Memory_Arena& operator= (Memory_Arena&&) noexcept = default; - - /** Re-allocates the internal buffer with a given number of bytes */ - void resize (size_t new_size_bytes) - { - clear(); - raw_data.resize (new_size_bytes, std::byte {}); - } - - /** - * Moves the allocator "stack pointer" back to zero, - * effectively "reclaiming" all allocated memory. - */ - void clear() noexcept - { -#if DEBUG - std::fill (raw_data.begin(), raw_data.begin() + bytes_used, std::byte { 0xDD }); -#endif - bytes_used = 0; - } - - /** Returns the number of bytes currently being used */ - [[nodiscard]] size_t get_bytes_used() const noexcept { return bytes_used; } - - /** - * Allocates a given number of bytes. - * The returned memory will be un-initialized, so be sure to clear it manually - * if needed. - */ - void* allocate_bytes (size_t num_bytes, size_t alignment = 1) - { - auto* pointer = snap_pointer_to_alignment (raw_data.data() + bytes_used, alignment); - const auto bytes_increment = static_cast (std::distance (raw_data.data() + bytes_used, pointer + num_bytes)); - - if (bytes_used + bytes_increment > raw_data.size()) - { - assert (false); - return nullptr; - } - - bytes_used += bytes_increment; - return pointer; - } - - /** - * Allocates space for some number of objects of type T - * The returned memory will be un-initialized, so be sure to clear it manually - * if needed. - */ - template - T* allocate (IntType num_Ts, size_t alignment = alignof (T)) - { - return static_cast (allocate_bytes ((size_t) num_Ts * sizeof (T), alignment)); - } - - /** - * Returns a span of type T, and size count. - * The returned memory will be un-initialized, so be sure to clear it manually - * if needed. - */ - template - auto make_span (IntType count, size_t alignment = default_byte_alignment) - { - return std::span { allocate (count, alignment), - static_cast (count) }; - } - - /** Returns a pointer to the internal buffer with a given offset in bytes */ - template - T* data (IntType offset_bytes) noexcept - { - return reinterpret_cast (raw_data.data() + offset_bytes); - } - - /** - * Creates a "frame" for the allocator. - * Once the frame goes out of scope, the allocator will be reset - * to whatever it's state was at the beginning of the frame. - */ - struct Frame - { - Frame() = default; - explicit Frame (Memory_Arena& allocator) - : alloc (&allocator), bytes_used_at_start (alloc->bytes_used) {} - - ~Frame() { alloc->bytes_used = bytes_used_at_start; } - - Memory_Arena* alloc = nullptr; - size_t bytes_used_at_start = 0; - }; - - /** Creates a frame for this allocator */ - auto create_frame() { return Frame { *this }; } - - void reset_to_frame (const Frame& frame) - { - assert (frame.alloc == this); - bytes_used = frame.bytes_used_at_start; - } - -private: - MemoryResourceType raw_data {}; - size_t bytes_used = 0; -}; -} // namespace wavenet diff --git a/deps/RTNeural-NAM/wavenet/wavenet_layer.hpp b/deps/RTNeural-NAM/wavenet/wavenet_layer.hpp deleted file mode 100644 index 3d81fe9..0000000 --- a/deps/RTNeural-NAM/wavenet/wavenet_layer.hpp +++ /dev/null @@ -1,178 +0,0 @@ -#pragma once - -#include - -namespace wavenet -{ -template > -// TODO: gated? -struct Wavenet_Layer -{ - RTNeural::Conv1DT conv; - RTNeural::DenseT input_mixin; - RTNeural::DenseT _1x1; - Activation activation; - -#if RTNEURAL_USE_EIGEN - Eigen::Matrix outs; -#elif RTNEURAL_USE_XSIMD - xsimd::batch outs[RTNeural::ceil_div (channels, (int) xsimd::batch::size)]; -#endif - - void reset() - { - conv.reset(); - } - - void load_weights (std::vector::iterator& weights) - { - conv.reset(); - - std::vector>> conv_weights (channels, std::vector> (channels, std::vector (kernel_size))); - for (int i = 0; i < channels; ++i) - for (int j = 0; j < channels; ++j) - for (int k = 0; k < kernel_size; k++) - conv_weights[i][j][k] = *(weights++); - RTNeural::torch_helpers::detail::reverseKernels (conv_weights); - conv.setWeights (conv_weights); - - std::vector conv_bias (channels); - for (int i = 0; i < channels; ++i) - conv_bias[i] = *(weights++); - conv.setBias (conv_bias); - - std::vector> input_mixin_weights (channels, std::vector (condition_size)); - for (int i = 0; i < channels; i++) - for (int j = 0; j < condition_size; j++) - input_mixin_weights[i][j] = *(weights++); - input_mixin.setWeights (input_mixin_weights); - - std::vector> _1x1_weights (channels, std::vector (channels)); - for (int i = 0; i < channels; i++) - for (int j = 0; j < channels; j++) - _1x1_weights[i][j] = *(weights++); - _1x1.setWeights (_1x1_weights); - - std::vector _1x1_bias (channels); - for (int i = 0; i < channels; i++) - _1x1_bias[i] = *(weights++); - _1x1.setBias (_1x1_bias.data()); - } - -#if RTNEURAL_USE_EIGEN - void forward (const Eigen::Matrix& ins, - const Eigen::Matrix& condition, - Eigen::Matrix& head_io) -#elif RTNEURAL_USE_XSIMD - void forward (const xsimd::batch (&ins)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)], - const xsimd::batch (&condition)[RTNeural::ceil_div (condition_size, (int) xsimd::batch::size)], - xsimd::batch (&head_io)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)]) -#endif - { - conv.forward (ins); - input_mixin.forward (condition); - -#if RTNEURAL_USE_EIGEN - outs = conv.outs + input_mixin.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (outs); ++i) - outs[i] = conv.outs[i] + input_mixin.outs[i]; -#endif - - activation.forward (outs); - -#if RTNEURAL_USE_EIGEN - head_io.noalias() += activation.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (head_io); ++i) - head_io[i] += activation.outs[i]; -#endif - - _1x1.forward (activation.outs); - -#if RTNEURAL_USE_EIGEN - outs = ins + _1x1.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (outs); ++i) - outs[i] = ins[i] + _1x1.outs[i]; -#endif - } - -#if RTNEURAL_USE_EIGEN - void forward (const Eigen::Matrix* ins, - const Eigen::Matrix* condition, - Eigen::Matrix* head_io, - Eigen::Matrix* layer_outputs, - int N, - Memory_Arena<>& arena) -#elif RTNEURAL_USE_XSIMD - void forward (const xsimd::batch (*ins)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)], - const xsimd::batch (*condition)[RTNeural::ceil_div (condition_size, (int) xsimd::batch::size)], - xsimd::batch (*head_io)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)], - xsimd::batch (*layer_outputs)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)], - int N, - Memory_Arena<>& arena) -#endif - { - const auto _ = arena.create_frame(); -#if RTNEURAL_USE_EIGEN - auto* temp_outs = arena.allocate> (N, RTNEURAL_DEFAULT_ALIGNMENT); -#elif RTNEURAL_USE_XSIMD - auto* temp_outs = arena.allocate[RTNeural::ceil_div (channels, (int) xsimd::batch::size)]> (N, RTNEURAL_DEFAULT_ALIGNMENT); -#endif - - for (int n = 0; n < N; ++n) - { - conv.forward (ins[n]); -#if RTNEURAL_USE_EIGEN - temp_outs[n].noalias() = conv.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (conv.outs); ++i) - temp_outs[n][i] = conv.outs[i]; -#endif - } - - for (int n = 0; n < N; ++n) - { - input_mixin.forward (condition[n]); -#if RTNEURAL_USE_EIGEN - temp_outs[n].noalias() += input_mixin.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (input_mixin.outs); ++i) - temp_outs[n][i] += input_mixin.outs[i]; -#endif - } - - for (int n = 0; n < N; ++n) - { - activation.forward (temp_outs[n]); -#if RTNEURAL_USE_EIGEN - temp_outs[n].noalias() = activation.outs; - head_io[n].noalias() += activation.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (activation.outs); ++i) - temp_outs[n][i] = activation.outs[i]; - for (int i = 0; i < std::size (activation.outs); ++i) - head_io[n][i] += activation.outs[i]; -#endif - } - - for (int n = 0; n < N; ++n) - { - _1x1.forward (temp_outs[n]); -#if RTNEURAL_USE_EIGEN - layer_outputs[n].noalias() = ins[n] + _1x1.outs; -#elif RTNEURAL_USE_XSIMD - for (int i = 0; i < std::size (_1x1.outs); ++i) - layer_outputs[n][i] = ins[n][i] + _1x1.outs[i]; -#endif - } - } -}; -} // namespace wavenet diff --git a/deps/RTNeural-NAM/wavenet/wavenet_layer_array.hpp b/deps/RTNeural-NAM/wavenet/wavenet_layer_array.hpp deleted file mode 100644 index 37ced02..0000000 --- a/deps/RTNeural-NAM/wavenet/wavenet_layer_array.hpp +++ /dev/null @@ -1,179 +0,0 @@ -#pragma once - -#include "wavenet_layer.hpp" - -namespace wavenet -{ -template -using Dilations = std::integer_sequence; - -template > -struct Layer_Array -{ - template - struct Layers_Helper - { - }; - - template - struct Layers_Helper> - { - using type = std::tuple...>; - }; - - using Layers = typename Layers_Helper::type; - - static constexpr auto n_channels = channels; - - RTNeural::DenseT rechannel; // no bias! - Layers layers; - static constexpr auto num_layers = std::tuple_size_v; - RTNeural::DenseT head_rechannel; - - using Last_Layer_Type = std::remove_reference_t - 1> (layers))>; - decltype (Last_Layer_Type::outs)& layer_outputs { std::get - 1> (layers).outs }; - -#if RTNEURAL_USE_EIGEN - Eigen::Matrix head_outputs {}; - Eigen::Matrix* layer_outputs_arr; - Eigen::Matrix* head_outputs_arr; -#elif RTNEURAL_USE_XSIMD - decltype (RTNeural::DenseT::outs)& head_outputs { head_rechannel.outs }; - using Layer_Out = xsimd::batch[RTNeural::ceil_div (channels, (int) xsimd::batch::size)]; - Layer_Out* layer_outputs_arr; - using Head_Out = xsimd::batch[RTNeural::ceil_div (head_size, (int) xsimd::batch::size)]; - Head_Out* head_outputs_arr; -#endif - - void reset() - { - RTNeural::modelt_detail::forEachInTuple ([] (auto& layer, size_t) - { layer.reset(); }, - layers); - } - - static size_t get_arena_bytes_needed (int N) - { -#if RTNEURAL_USE_EIGEN - return 2 * sizeof (Eigen::Matrix) * N + sizeof (Eigen::Matrix) * N; -#elif RTNEURAL_USE_XSIMD - return 2 * sizeof (Layer_Out) * N + sizeof (Head_Out) * N; -#endif - } - - void load_weights (std::vector::iterator& weights) - { - std::vector> rechannel_weights (channels, std::vector (in_size)); - for (int i = 0; i < channels; i++) - for (int j = 0; j < in_size; j++) - rechannel_weights[i][j] = *(weights++); - rechannel.setWeights (rechannel_weights); - - RTNeural::modelt_detail::forEachInTuple ([&weights] (auto& layer, size_t) - { layer.load_weights (weights); }, - layers); - - std::vector> head_rechannel_weights (head_size, std::vector (channels)); - for (int i = 0; i < head_size; i++) - for (int j = 0; j < channels; j++) - head_rechannel_weights[i][j] = *(weights++); - head_rechannel.setWeights (head_rechannel_weights); - - if constexpr (has_head_bias) - { - std::vector head_rechannel_bias (head_size); - for (int i = 0; i < head_size; i++) - head_rechannel_bias[i] = *(weights++); - head_rechannel.setBias (head_rechannel_bias.data()); - } - } - -#if RTNEURAL_USE_EIGEN - void forward (const Eigen::Matrix& ins, - const Eigen::Matrix& condition, - Eigen::Matrix& head_io) -#elif RTNEURAL_USE_XSIMD - void forward (const xsimd::batch (&ins)[RTNeural::ceil_div (in_size, (int) xsimd::batch::size)], - const xsimd::batch (&condition)[RTNeural::ceil_div (condition_size, (int) xsimd::batch::size)], - xsimd::batch (&head_io)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)]) -#endif - { - rechannel.forward (ins); - - RTNeural::modelt_detail::forEachInTuple ( - [&] (auto& layer, auto index_t) - { - static constexpr size_t index = index_t; - if constexpr (index == 0) - layer.forward (rechannel.outs, condition, head_io); - else - layer.forward (std::get (layers).outs, condition, head_io); - }, - layers); - - head_rechannel.forward (head_io); -#if RTNEURAL_USE_EIGEN - head_outputs = head_rechannel.outs; -#endif - } - -#if RTNEURAL_USE_EIGEN - void forward (const Eigen::Matrix* ins, - const Eigen::Matrix* condition, - Eigen::Matrix* head_io, - int N, - Memory_Arena<>& arena) -#elif RTNEURAL_USE_XSIMD - void forward (const xsimd::batch (*ins)[RTNeural::ceil_div (in_size, (int) xsimd::batch::size)], - const xsimd::batch (*condition)[RTNeural::ceil_div (condition_size, (int) xsimd::batch::size)], - xsimd::batch (*head_io)[RTNeural::ceil_div (channels, (int) xsimd::batch::size)], - int N, - Memory_Arena<>& arena) -#endif - { -#if RTNEURAL_USE_EIGEN - layer_outputs_arr = arena.allocate> (N, RTNEURAL_DEFAULT_ALIGNMENT); - head_outputs_arr = arena.allocate> (N, RTNEURAL_DEFAULT_ALIGNMENT); -#elif RTNEURAL_USE_XSIMD - layer_outputs_arr = arena.allocate (N, RTNEURAL_DEFAULT_ALIGNMENT); - head_outputs_arr = arena.allocate (N, RTNEURAL_DEFAULT_ALIGNMENT); -#endif - - for (int n = 0; n < N; ++n) - { - rechannel.forward (ins[n]); -#if RTNEURAL_USE_EIGEN - layer_outputs_arr[n] = rechannel.outs; -#elif RTNEURAL_USE_XSIMD - std::copy (std::begin (rechannel.outs), std::end (rechannel.outs), std::begin (layer_outputs_arr[n])); -#endif - } - - RTNeural::modelt_detail::forEachInTuple ( - [&] (auto& layer, auto) - { - layer.forward (layer_outputs_arr, condition, head_io, layer_outputs_arr, N, arena); - }, - layers); - - for (int n = 0; n < N; ++n) - { - head_rechannel.forward (head_io[n]); -#if RTNEURAL_USE_EIGEN - head_outputs_arr[n] = head_rechannel.outs; -#elif RTNEURAL_USE_XSIMD - std::copy (std::begin (head_rechannel.outs), std::end (head_rechannel.outs), std::begin (head_outputs_arr[n])); -#endif - } - } -}; -} // namespace wavenet diff --git a/deps/RTNeural-NAM/wavenet/wavenet_model.hpp b/deps/RTNeural-NAM/wavenet/wavenet_model.hpp deleted file mode 100644 index cecefce..0000000 --- a/deps/RTNeural-NAM/wavenet/wavenet_model.hpp +++ /dev/null @@ -1,162 +0,0 @@ -#pragma once - -#include "arena.hpp" -#include "wavenet_layer_array.hpp" - -namespace wavenet -{ -template -struct Wavenet_Model -{ - std::tuple layer_arrays; - - static constexpr auto head_layer_n_channels = std::tuple_element_t<0, std::tuple>::n_channels; - -#if RTNEURAL_USE_EIGEN - Eigen::Matrix head_input {}; -#elif RTNEURAL_USE_XSIMD - xsimd::batch head_input[RTNeural::ceil_div (head_layer_n_channels, (int) xsimd::batch::size)]; -#endif - T head_scale = (T) 0; - - Memory_Arena<> arena {}; - - Wavenet_Model() = default; - - void prepare (int block_size) - { -#if RTNEURAL_USE_EIGEN - size_t arena_bytes_needed = sizeof (Eigen::Matrix) * block_size; -#elif RTNEURAL_USE_XSIMD - size_t arena_bytes_needed = sizeof (xsimd::batch[RTNeural::ceil_div (head_layer_n_channels, (int) xsimd::batch::size)]) * block_size - + sizeof (xsimd::batch) * block_size; -#endif - RTNeural::modelt_detail::forEachInTuple ( - [&arena_bytes_needed, block_size] (auto& layer_array, auto) - { - arena_bytes_needed += layer_array.get_arena_bytes_needed (block_size); - }, - layer_arrays); - - arena.resize (arena_bytes_needed + 256); - // prewarm(); - } - - void prewarm() - { - RTNeural::modelt_detail::forEachInTuple ( - [] (auto& layer, size_t) - { - layer.reset(); - }, - layer_arrays); - for (int i = 0; i < 1 << 14; ++i) - forward (0.0f); - } - - void load_weights (const nlohmann::json& model_json) - { - std::vector model_weights = model_json.at ("weights"); - auto weights_iterator = model_weights.begin(); - RTNeural::modelt_detail::forEachInTuple ( - [&weights_iterator] (auto& layer, size_t) - { - layer.load_weights (weights_iterator); - }, - layer_arrays); - - head_scale = *weights_iterator++; - - // Make sure we use the all of the weights exactly - assert (std::distance (model_weights.begin(), weights_iterator) == model_weights.size()); - } - - T forward (T input) noexcept - { -#if RTNEURAL_USE_EIGEN - const auto v_ins = Eigen::Matrix::Constant (input); -#elif RTNEURAL_USE_XSIMD - xsimd::batch v_ins[1]; - v_ins[0] = RTNeural::set_value (v_ins[0], 0, input); -#endif - - RTNeural::modelt_detail::forEachInTuple ( - [this, v_ins] (auto& layer_array, auto index_t) - { - static constexpr size_t index = index_t; - if constexpr (index == 0) - { -#if RTNEURAL_USE_EIGEN - head_input.setZero(); - std::get<0> (layer_arrays).forward (v_ins, v_ins, head_input); -#elif RTNEURAL_USE_XSIMD - std::fill (std::begin (head_input), std::end (head_input), xsimd::batch {}); - std::get<0> (layer_arrays).forward (v_ins, v_ins, head_input); -#endif - } - else - { - std::get (layer_arrays).forward (std::get (layer_arrays).layer_outputs, v_ins, std::get (layer_arrays).head_outputs); - } - }, - layer_arrays); - -#if RTNEURAL_USE_EIGEN - return std::get - 1> (layer_arrays).head_outputs[0] * head_scale; -#elif RTNEURAL_USE_XSIMD - return std::get - 1> (layer_arrays).head_outputs[0].get (0) * head_scale; -#endif - } - - void forward (const T* input, T* output, int N) noexcept - { -#if RTNEURAL_USE_EIGEN - const auto* v_ins = reinterpret_cast*> (input); -#elif RTNEURAL_USE_XSIMD - auto* v_ins = arena.allocate[1]> (N, RTNEURAL_DEFAULT_ALIGNMENT); - for (int n = 0; n < N; ++n) - v_ins[n][0] = RTNeural::set_value (v_ins[n][0], 0, input[n]); -#endif - - RTNeural::modelt_detail::forEachInTuple ( - [this, v_ins, N, output] (auto& layer_array, auto index_t) - { - static constexpr size_t index = index_t; - if constexpr (index == 0) - { -#if RTNEURAL_USE_EIGEN - auto* head_inputs = arena.allocate> (N, RTNEURAL_DEFAULT_ALIGNMENT); - for (int n = 0; n < N; ++n) - head_inputs[n].setZero(); - std::get<0> (layer_arrays).forward (v_ins, v_ins, head_inputs, N, arena); -#elif RTNEURAL_USE_XSIMD - auto* head_inputs = arena.allocate[RTNeural::ceil_div (head_layer_n_channels, (int) xsimd::batch::size)]> (N, RTNEURAL_DEFAULT_ALIGNMENT); - for (int n = 0; n < N; ++n) - std::fill (std::begin (head_inputs[n]), std::end (head_inputs[n]), xsimd::batch {}); - std::get<0> (layer_arrays).forward (v_ins, v_ins, head_inputs, N, arena); -#endif - } - else - { - auto& prev_layer_array = std::get (layer_arrays); - std::get (layer_arrays).forward (prev_layer_array.layer_outputs_arr, v_ins, prev_layer_array.head_outputs_arr, N, arena); - } - }, - layer_arrays); - - auto& last_layer_array = std::get - 1> (layer_arrays); - for (int n = 0; n < N; ++n) - { -#if RTNEURAL_USE_EIGEN - output[n] = last_layer_array.head_outputs_arr[n][0] * head_scale; -#elif RTNEURAL_USE_XSIMD - output[n] = last_layer_array.head_outputs_arr[n][0].get (0) * head_scale; -#endif - } - - arena.clear(); - } -}; -} // namespace wavenet