VctrBase.h

/*
  ==============================================================================
    DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 
    Copyright 2022- by sonible GmbH.
 
    This file is part of VCTR - Versatile Container Templates Reconceptualized.
 
    VCTR is free software: you can redistribute it and/or modify
    it under the terms of the GNU Lesser General Public License version 3
    only, as published by the Free Software Foundation.
 
    VCTR is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License version 3 for more details.
 
    You should have received a copy of the GNU Lesser General Public License
    version 3 along with VCTR.  If not, see <https://www.gnu.org/licenses/>.
  ==============================================================================
*/
 
// clang-format off
// Disable MSVC warning C4324 "structure was padded due to alignment specifier"
#if VCTR_MSVC
__pragma (warning (push))
__pragma (warning (disable : 4324))
#endif
// clang-format on
 
namespace vctr
{
 
template <class ElementType, class StorageType, size_t extent, class StorageInfoType = StorageInfo<StorageType>>
class VctrBase : private Config,
                 private StorageInfoType
{
public:
    //==============================================================================
    using value_type = std::remove_cv_t<ElementType>;
 
    //==============================================================================
    // Retrieving size related information.
    //==============================================================================
 
    // clang-format off
 
    static consteval size_t getExtent (size_t amountToShrink = 0)
    {
        return extent == std::dynamic_extent ? std::dynamic_extent : (extent - amountToShrink);
    }
 
    [[nodiscard]] constexpr size_t size() const noexcept requires (extent == std::dynamic_extent) { return storage.size(); }
 
    static constexpr size_t size() noexcept requires (extent != std::dynamic_extent) { return extent; }
 
    [[nodiscard]] constexpr bool empty() const noexcept { return storage.empty(); }
 
    [[nodiscard]] constexpr size_t sizeInBytes() const noexcept requires (extent == std::dynamic_extent) { return size() * sizeof (value_type); }
 
    [[nodiscard]] static constexpr size_t sizeInBytes() noexcept requires (extent != std::dynamic_extent) { return size() * sizeof (value_type); }
 
    [[nodiscard]] constexpr size_t backIdx() const noexcept requires (extent == std::dynamic_extent) { return size() - 1; }
 
    static constexpr size_t backIdx() noexcept requires (extent != std::dynamic_extent) { return size() - 1; }
 
    //==============================================================================
    // Accessing elements.
    //==============================================================================
    constexpr auto& operator[] (size_t i)       { VCTR_ASSERT (i < size()); return storage[i]; }
 
    constexpr auto& operator[] (size_t i) const { VCTR_ASSERT (i < size()); return storage[i]; }
 
    constexpr auto& at (size_t i)       { throwIfOutOfRange (i); return storage[i]; }
 
    constexpr auto& at (size_t i) const { throwIfOutOfRange (i); return storage[i]; }
 
    // clang-format on
 
    constexpr auto&& front() { return storage.front(); }
 
    constexpr auto&& front() const { return storage.front(); }
 
    constexpr auto&& back() { return storage[backIdx()]; }
 
    constexpr auto&& back() const { return storage[backIdx()]; }
 
    VCTR_FORCEDINLINE constexpr auto* data()
    {
        if constexpr (requires { typename RequireConstexpr<StorageInfoType::dataIsSIMDAligned>; } && StorageInfoType::dataIsSIMDAligned)
            return assumeAlignedToMaxSIMDRegisterSize (storage.data());
 
        return storage.data();
    }
 
    VCTR_FORCEDINLINE constexpr auto* data() const
    {
        if constexpr (requires { typename RequireConstexpr<StorageInfoType::dataIsSIMDAligned>; } && StorageInfoType::dataIsSIMDAligned)
            return assumeAlignedToMaxSIMDRegisterSize (storage.data());
 
        return storage.data();
    }
 
    constexpr auto begin() { return storage.begin(); }
 
    constexpr auto begin() const { return storage.begin(); }
 
    constexpr auto end() { return is::stdArray<StorageType> ? storage.begin() + extent : storage.end(); }
 
    constexpr auto end() const { return is::stdArray<StorageType> ? storage.begin() + extent : storage.end(); }
 
    constexpr auto rbegin() { return std::reverse_iterator (end()); }
 
    constexpr auto rbegin() const { return std::reverse_iterator (end()); }
 
    constexpr auto rend() { return std::reverse_iterator (begin()); }
 
    constexpr auto rend() const { return std::reverse_iterator (begin()); }
 
    //==============================================================================
    // Accessing sub spans.
    //==============================================================================
    template <size_t startIdx>
    constexpr auto subSpan()
    {
        assertIsInRange<startIdx>();
        return constCorrectSpan<getExtent (startIdx)> (data() + startIdx, size() - startIdx);
    }
 
    template <size_t startIdx>
    constexpr auto subSpan() const
    {
        assertIsInRange<startIdx>();
        return constCorrectSpan<getExtent (startIdx)> (data() + startIdx, size() - startIdx);
    }
 
    constexpr auto subSpan (size_t startIdx)
    {
        VCTR_ASSERT (startIdx < size());
        return constCorrectSpan<std::dynamic_extent> (data() + startIdx, size() - startIdx);
    }
 
    constexpr auto subSpan (size_t startIdx) const
    {
        VCTR_ASSERT (startIdx < size());
        return constCorrectSpan<std::dynamic_extent> (data() + startIdx, size() - startIdx);
    }
 
    template <size_t startIdx, size_t numElements>
    constexpr auto subSpan()
    {
        VCTR_ASSERT (int64_t (size()) - int64_t (startIdx) >= int64_t (numElements));
        return constCorrectSpan<numElements> (data() + startIdx, numElements);
    }
 
    template <size_t startIdx, size_t numElements>
    constexpr auto subSpan() const
    {
        VCTR_ASSERT (int64_t (size()) - int64_t (startIdx) >= int64_t (numElements));
        return constCorrectSpan<numElements> (data() + startIdx, numElements);
    }
 
    constexpr auto subSpan (size_t startIdx, size_t numElements)
    {
        VCTR_ASSERT (int64_t (size()) - int64_t (startIdx) >= int64_t (numElements));
        return constCorrectSpan<std::dynamic_extent> (data() + startIdx, numElements);
    }
 
    constexpr auto subSpan (size_t startIdx, size_t numElements) const
    {
        VCTR_ASSERT (int64_t (size()) - int64_t (startIdx) >= int64_t (numElements));
        return constCorrectSpan<std::dynamic_extent> (data() + startIdx, numElements);
    }
 
    //==============================================================================
    // Assign data.
    //==============================================================================
 
    constexpr void assign (std::initializer_list<ElementType> elements)
    {
        if constexpr (has::resize<StorageType>)
        {
            storage.resize (elements.size());
        }
        else
        {
            VCTR_ASSERT (elements.size() == size());
        }
 
        std::copy (elements.begin(), elements.end(), begin());
    }
 
    constexpr void copyFrom (const ElementType* otherData, size_t otherSize)
    requires is::nonConst<ElementType>
    {
        resizeOrAssertSizeMatches (otherSize);
 
        if constexpr (is::triviallyCopyable<ElementType>)
        {
            if (! std::is_constant_evaluated())
            {
                std::memcpy (data(), otherData, sizeof (ElementType) * otherSize);
                return;
            }
        }
 
        std::copy_n (otherData, otherSize, begin());
    }
 
    constexpr void fill (const value_type& value) { std::fill (begin(), end(), value); }
 
    //==============================================================================
    // Generators
    //==============================================================================
    constexpr void fillLinspace (ElementType start, ElementType stop, bool includeEnd = true)
    requires is::realNumber<ElementType>
    {
        const auto num = size();
 
        VCTR_ASSERT (num > 0);
        VCTR_ASSERT (! includeEnd || num != 1); // a num-one range cannot include the end
 
        const auto increment = (stop - start) / double (num - size_t (includeEnd));
 
#if VCTR_USE_GCEM
        [[maybe_unused]] const auto isIntegerIncrement = increment - gcem::floor (increment) == ElementType (0);
#else
        [[maybe_unused]] const auto isIntegerIncrement = increment - std::floor (increment) == ElementType (0);
#endif
        VCTR_ASSERT (std::is_floating_point_v<ElementType> || isIntegerIncrement);
 
        auto value = double (start);
 
        for (int i = 0; i < num; ++i)
        {
            storage[i] = ElementType (value);
            value += increment;
        };
    }
 
    //==============================================================================
    // For each functionality.
    //==============================================================================
    template <is::functionWithSignatureOrImplicitlyConvertible<void (value_type&)> Fn>
    constexpr void forEach (Fn&& fn)
    {
        for (auto& e : *this)
            fn (e);
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<void (const value_type&)> Fn>
    constexpr void forEach (Fn&& fn) const
    {
        for (const auto& e : *this)
            fn (e);
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<value_type (const value_type&)> Fn>
    constexpr void forEach (Fn&& fn)
    {
        for (auto& e : *this)
            e = fn (e);
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<void (value_type&, size_t)> Fn>
    constexpr void forEach (Fn&& fn)
    {
        const auto s = size();
        for (size_t i = 0; i < s; ++i)
            fn (storage[i], i);
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<void (const value_type&, size_t)> Fn>
    constexpr void forEach (Fn&& fn) const
    {
        const auto s = size();
        for (size_t i = 0; i < s; ++i)
            fn (storage[i], i);
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<value_type (const value_type&, size_t)> Fn>
    constexpr void forEach (Fn&& fn)
    {
        const auto s = size();
        for (size_t i = 0; i < s; ++i)
            storage[i] = fn (storage[i], i);
    }
 
    template <class... Args, is::functionWithSignatureOrImplicitlyConvertible<void (value_type&, Args&&...)> Fn>
    constexpr void forEach (Fn&& fn, Args&&... fnArgs)
    {
        for (auto& e : *this)
            fn (e, std::forward<Args> (fnArgs)...);
    }
 
    template <class... Args, is::functionWithSignatureOrImplicitlyConvertible<void (const value_type&, Args&&...)> Fn>
    constexpr void forEach (Fn&& fn, Args&&... fnArgs) const
    {
        for (const auto& e : *this)
            fn (e, std::forward<Args> (fnArgs)...);
    }
 
    template <class... Args, is::functionWithSignatureOrImplicitlyConvertible<value_type (const value_type&, Args&&...)> Fn>
    constexpr void forEach (Fn&& fn, Args&&... fnArgs)
    {
        for (auto& e : *this)
            e = fn (e, std::forward<Args> (fnArgs)...);
    }
 
    //==============================================================================
    // Finding elements and manipulating them.
    //==============================================================================
    template <std::equality_comparable_with<ElementType> T>
    constexpr auto find (const T& valueToLookFor)
    {
        return std::find (begin(), end(), valueToLookFor);
    }
 
    template <std::equality_comparable_with<ElementType> T>
    constexpr auto find (const T& valueToLookFor) const
    {
        return std::find (begin(), end(), valueToLookFor);
    }
 
    template <std::equality_comparable_with<ElementType> T>
    constexpr auto findReverse (const T& valueToLookFor)
    {
        return std::find (rbegin(), rend(), valueToLookFor);
    }
 
    template <std::equality_comparable_with<ElementType> T>
    constexpr auto findReverse (const T& valueToLookFor) const
    {
        return std::find (rbegin(), rend(), valueToLookFor);
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<bool (const ElementType&)> Fn>
    constexpr auto findIf (Fn&& predicate)
    {
        return std::find_if (begin(), end(), std::forward<Fn> (predicate));
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<bool (const ElementType&)> Fn>
    constexpr auto findIf (Fn&& predicate) const
    {
        return std::find_if (begin(), end(), std::forward<Fn> (predicate));
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<bool (const ElementType&)> Fn>
    constexpr auto findIfReverse (Fn&& predicate)
    {
        return std::find_if (rbegin(), rend(), std::forward<Fn> (predicate));
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<bool (const ElementType&)> Fn>
    constexpr auto findIfReverse (Fn&& predicate) const
    {
        return std::find_if (rbegin(), rend(), std::forward<Fn> (predicate));
    }
 
    constexpr size_t count (const ElementType& valueToLookFor) const
    {
        return std::count (begin(), end(), valueToLookFor);
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<bool (const ElementType&)> Fn>
    constexpr size_t countIf (Fn&& predicate) const
    {
        return std::count_if (begin(), end(), std::forward<Fn> (predicate));
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<bool (const ElementType&)> Fn>
    constexpr bool all (Fn&& predicate) const
    {
        return std::all_of (begin(), end(), std::forward<Fn> (predicate));
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<bool (const ElementType&)> Fn>
    constexpr bool any (Fn&& predicate) const
    {
        return std::any_of (begin(), end(), std::forward<Fn> (predicate));
    }
 
    template <std::equality_comparable_with<ElementType> T>
    constexpr bool allElementsEqual (const T& value) const
    {
        return all ([&] (const auto& e) { return e == value; });
    }
 
    [[nodiscard]] constexpr bool allElementsEqual() const
    requires std::equality_comparable<ElementType>
    {
        if (size() < 2)
            return true;
 
        return std::all_of (begin() + 1, end(), [&v = storage[0]] (const auto& e) { return v == e; });
    }
 
    template <std::equality_comparable_with<ElementType> T>
    constexpr bool contains (const T& value) const
    {
        return find (value) != end();
    }
 
    template <is::contiguousIteratorWithValueTypeSameAs<ElementType> It>
    constexpr bool contains (It it) const
    {
        const auto* address = std::to_address (it);
        const auto* first = std::to_address (begin());
        const auto* last = std::to_address (end() - 1);
 
        return address >= first && address <= last;
    }
 
    template <std::equality_comparable_with<ElementType> T>
    constexpr std::optional<size_t> indexOf (const T& value) const
    {
        auto it = find (value);
        return it == end() ? std::nullopt : std::optional<size_t> (std::distance (begin(), it));
    }
 
    template <std::equality_comparable_with<ElementType> T>
    constexpr std::optional<size_t> indexOfReverse (const T& value) const
    {
        auto it = findReverse (value);
        return it == rend() ? std::nullopt : std::optional<size_t> (std::distance (it, rend()) - 1);
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<bool (const ElementType&)> Fn>
    constexpr std::optional<size_t> indexIf (Fn&& predicate) const
    {
        auto it = findIf (predicate);
        return it == end() ? std::nullopt : std::optional<size_t> (std::distance (begin(), it));
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<bool (const ElementType&)> Fn>
    constexpr std::optional<size_t> indexIfReverse (Fn&& predicate) const
    {
        auto it = findIfReverse (predicate);
        return it == rend() ? std::nullopt : std::optional<size_t> (std::distance (it, rend()) - 1);
    }
 
    constexpr std::optional<value_type> firstValueGreaterThanOrEqualTo (const value_type& valueToLookFor) const
    requires std::totally_ordered<value_type>
    {
        VCTR_ASSERT (elementsAreSorted());
        auto it = std::lower_bound (begin(), end(), valueToLookFor);
        return it == end() ? std::nullopt : std::optional<value_type> (*it);
    }
 
    constexpr std::optional<value_type> firstValueGreaterThan (const value_type& valueToLookFor) const
    requires std::totally_ordered<value_type>
    {
        VCTR_ASSERT (elementsAreSorted());
        auto it = std::upper_bound (begin(), end(), valueToLookFor);
        return it == end() ? std::nullopt : std::optional<value_type> (*it);
    }
 
    //==============================================================================
    // Shuffling and sorting elements
    //==============================================================================
    constexpr void reverse()
    {
        std::reverse (begin(), end());
    }
 
    constexpr void rotate (size_t newFirstElementIdx)
    {
        VCTR_ASSERT (newFirstElementIdx < size());
        std::rotate (begin(), begin() + newFirstElementIdx, end());
    }
 
    void shiftLeft (size_t n, bool clearFreeSpaceAfterShiftedRegion)
    requires std::is_trivially_copyable_v<ElementType>
    {
        if (n == size())
        {
            if (clearFreeSpaceAfterShiftedRegion)
                clear (data(), size());
 
            return;
        }
 
        VCTR_ASSERT (n < size());
 
        auto numElementsToMove = size() - n;
        memMove (data() + n, data(), numElementsToMove);
 
        if (clearFreeSpaceAfterShiftedRegion)
            clear (data() + numElementsToMove, n);
    }
 
    void shiftRight (size_t n, bool clearFreeSpaceBeforeShiftedRegion)
    requires std::is_trivially_copyable_v<ElementType>
    {
        if (n == size())
        {
            if (clearFreeSpaceBeforeShiftedRegion)
                clear (data(), size());
 
            return;
        }
 
        VCTR_ASSERT (n < size());
 
        const auto numElementsToMove = size() - n;
        memMove (data(), data() + n, numElementsToMove);
 
        if (clearFreeSpaceBeforeShiftedRegion)
            clear (data(), n);
    }
 
    constexpr void sort()
    requires std::totally_ordered<value_type>
    {
        std::sort (begin(), end());
    }
 
    template <is::functionWithSignatureOrImplicitlyConvertible<bool (const value_type&, const value_type&)> ComparatorFn>
    constexpr void sort (ComparatorFn&& compare)
    {
        std::sort (begin(), end(), compare);
    }
 
    [[nodiscard]] constexpr bool elementsAreSorted() const
    requires std::totally_ordered<value_type>
    {
        return std::is_sorted (begin(), end());
    }
 
    //==============================================================================
    // SIMD Register Access
    //==============================================================================
    NeonRegister<std::remove_const_t<ElementType>> getNeon (size_t i) const
    requires archARM && is::realNumber<ElementType>
    {
        VCTR_ASSERT (i % NeonRegister<std::remove_const_t<ElementType>>::numElements == 0);
        return NeonRegister<std::remove_const_t<ElementType>>::load (data() + i);
    }
 
    VCTR_TARGET ("avx")
    AVXRegister<std::remove_const_t<ElementType>> getAVX (size_t i) const
    requires archX64 && is::realNumber<ElementType>
    {
        VCTR_ASSERT (i % AVXRegister<std::remove_const_t<ElementType>>::numElements == 0);
        if (StorageInfoType::dataIsSIMDAligned)
            return AVXRegister<std::remove_const_t<ElementType>>::loadAligned (data() + i);
        else
            return AVXRegister<std::remove_const_t<ElementType>>::loadUnaligned (data() + i);
    }
 
    VCTR_TARGET ("sse4.1")
    SSERegister<std::remove_const_t<ElementType>> getSSE (size_t i) const
    requires archX64 && is::realNumber<ElementType>
    {
        VCTR_ASSERT (i % SSERegister<std::remove_const_t<ElementType>>::numElements == 0);
        if (StorageInfoType::dataIsSIMDAligned)
            return SSERegister<std::remove_const_t<ElementType>>::loadAligned (data() + i);
        else
            return SSERegister<std::remove_const_t<ElementType>>::loadUnaligned (data() + i);
    }
 
    //==============================================================================
    // Expression chain related functions
    //==============================================================================
    template <is::expressionChainBuilder ExpressionChain>
    void evalInPlace (const ExpressionChain& expression)
    {
        assignExpressionTemplate (expression << *this);
    }
 
    constexpr bool isNotAliased (const void*) const { return true; }
 
    VCTR_FORCEDINLINE const ElementType* evalNextVectorOpInExpressionChain (void*) const { return data(); }
 
    constexpr const StorageInfoType& getStorageInfo() const { return *this; }
 
    //==============================================================================
    // Math operators.
    //==============================================================================
    template <is::anyVctrOrExpression V>
    constexpr void operator*= (const V& v);
 
    constexpr void operator*= (value_type c);
 
    template <is::anyVctrOrExpression V>
    constexpr void operator/= (const V& v);
 
    constexpr void operator/= (value_type c);
 
    template <is::anyVctrOrExpression V>
    constexpr void operator+= (const V& v);
 
    constexpr void operator+= (value_type c);
 
    template <is::anyVctrOrExpression V>
    constexpr void operator-= (const V& v);
 
    constexpr void operator-= (value_type c);
 
    //==============================================================================
    // Math reduction operations.
    //==============================================================================
    constexpr ElementType min() const requires std::totally_ordered<ElementType>;
 
    constexpr ElementType minAbs() const requires is::number<ElementType>;
 
    constexpr ElementType max() const requires std::totally_ordered<ElementType>;
 
    constexpr ElementType maxAbs() const requires is::number<ElementType>;
 
    constexpr ElementType mean() const requires is::number<ElementType>;
 
    constexpr ElementType meanSquare() const requires is::number<ElementType>;
 
    constexpr ElementType rms() const requires is::number<ElementType>;
 
    constexpr ElementType sum() const requires has::operatorPlusEquals<std::remove_cv_t<ElementType>>;
 
    //==============================================================================
    // Math sanity checks.
    //==============================================================================
    constexpr bool allElementsAreFinite() requires std::floating_point<ElementType>;
 
    constexpr bool allElementsAreFinite() requires is::complexFloatNumber<ElementType>;
 
    constexpr bool anyElementIsNaN() requires std::floating_point<ElementType>;
 
    constexpr bool anyElementIsNaN() requires is::complexFloatNumber<ElementType>;
 
protected:
    //==============================================================================
    constexpr VctrBase()
    {
        if constexpr (has::init<StorageInfoType>)
            StorageInfoType::init (storage.data(), storage.size());
    }
 
    constexpr VctrBase (StorageType&& s)
        : storage (std::move (s))
    {
        if constexpr (has::init<StorageInfoType>)
            StorageInfoType::init (storage.data(), storage.size());
    }
 
    template <is::storageInfo OtherStorageInfoType>
    constexpr VctrBase (StorageType&& s, const OtherStorageInfoType& otherInfo)
        : StorageInfoType (otherInfo),
          storage (std::move (s))
    {}
 
    //==============================================================================
    constexpr void resizeOrAssertSizeMatches (size_t desiredSize)
    requires has::resize<StorageType>
    {
        storage.resize (desiredSize);
    }
 
    constexpr void resizeOrAssertSizeMatches ([[maybe_unused]] size_t desiredSize) const
    {
        VCTR_ASSERT (size() == desiredSize);
    }
 
    template <size_t i>
    constexpr void assertIsInRange() const
    requires (extent == std::dynamic_extent)
    {
        VCTR_ASSERT (i < size());
    }
 
    template <size_t i>
    constexpr void assertIsInRange() const
    requires (extent != std::dynamic_extent)
    {
        static_assert (i < extent);
    }
 
    void throwIfOutOfRange (size_t i) const
    {
        if (i >= size())
        {
            VCTR_ASSERT (false);
            throw std::out_of_range ("Vctr of size " + std::to_string (size()) + ": Element " + std::to_string (i) + " is out of range.");
        }
    }
 
    //==============================================================================
    template <is::expression Expression>
    VCTR_FORCEDINLINE constexpr void assignExpressionTemplate (const Expression& e)
    {
        if (! std::is_constant_evaluated())
        {
            if constexpr (has::evalNextVectorOpInExpressionChain<Expression, ElementType>)
            {
                if (e.isNotAliased (data()))
                {
                    e.evalNextVectorOpInExpressionChain (data());
                    return;
                }
            }
 
            if constexpr (has::getNeon<Expression>)
            {
                assignExpressionTemplateNeon (e);
                return;
            }
 
            if constexpr (has::getAVX<Expression>)
            {
                if constexpr (is::realFloatNumber<ElementType>)
                {
                    if (supportsAVX)
                    {
                        assignExpressionTemplateAVX (e);
                        return;
                    }
                }
                else
                {
                    if (supportsAVX2)
                    {
                        assignExpressionTemplateAVX2 (e);
                        return;
                    }
                }
            }
 
            if constexpr (has::getSSE<Expression>)
            {
                if (highestSupportedCPUInstructionSet != CPUInstructionSet::fallback)
                {
                    assignExpressionTemplateSSE4_1 (e);
                    return;
                }
            }
        }
 
        const auto n = size();
 
        for (size_t i = 0; i < n; ++i)
            storage[i] = e[i];
    }
 
    template <class OtherContainer>
    static constexpr bool shouldMoveFromOtherContainer = has::begin<OtherContainer> &&
                                                         has::end<OtherContainer> &&
                                                         (! is::view<OtherContainer>) &&
                                                         (! std::is_reference_v<OtherContainer>) &&
                                                         (! std::is_trivially_copyable_v<ElementType>);
 
    //==============================================================================
    alignas (StorageInfoType::memberAlignment) StorageType storage = StorageType {};
 
private:
    //==============================================================================
    template <class Expression>
    void assignExpressionTemplateNeon (const Expression& e)
    requires archARM
    {
        constexpr auto inc = NeonRegister<ElementType>::numElements;
        const bool hasExtendedSIMDStorage = e.getStorageInfo().hasSIMDExtendedStorage && StorageInfoType::hasSIMDExtendedStorage;
        const auto n = storage.size();
        const auto nSIMD = hasExtendedSIMDStorage ? detail::nextMultipleOf<inc> (n) : detail::previousMultipleOf<inc> (n);
 
        auto* d = data();
 
        size_t i = 0;
        for (; i < nSIMD; i += inc, d += inc)
            e.getNeon (i).store (d);
 
        for (; i < n; ++i, ++d)
            storage[i] = e[i];
    }
 
    template <class Expression>
    VCTR_TARGET ("avx2")
    void assignExpressionTemplateAVX2 (const Expression& e)
    requires archX64
    {
        constexpr auto inc = AVXRegister<ElementType>::numElements;
        const bool hasExtendedSIMDStorage = e.getStorageInfo().hasSIMDExtendedStorage && StorageInfoType::hasSIMDExtendedStorage;
        const auto n = storage.size();
        const auto nSIMD = hasExtendedSIMDStorage ? detail::nextMultipleOf<inc> (n) : detail::previousMultipleOf<inc> (n);
 
        auto* d = data();
 
        if (StorageInfoType::dataIsSIMDAligned)
        {
            size_t i = 0;
            for (; i < nSIMD; i += inc, d += inc)
                e.getAVX (i).storeAligned (d);
 
            for (; i < n; ++i, ++d)
                storage[i] = e[i];
        }
        else
        {
            size_t i = 0;
            for (; i < nSIMD; i += inc, d += inc)
                e.getAVX (i).storeUnaligned (d);
 
            for (; i < n; ++i, ++d)
                storage[i] = e[i];
        }
    }
 
    template <class Expression>
    VCTR_TARGET ("avx")
    void assignExpressionTemplateAVX (const Expression& e)
    requires archX64
    {
        constexpr auto inc = AVXRegister<ElementType>::numElements;
        const bool hasExtendedSIMDStorage = e.getStorageInfo().hasSIMDExtendedStorage && StorageInfoType::hasSIMDExtendedStorage;
        const auto n = storage.size();
        const auto nSIMD = hasExtendedSIMDStorage ? detail::nextMultipleOf<inc> (n) : detail::previousMultipleOf<inc> (n);
 
        auto* d = data();
 
        if (StorageInfoType::dataIsSIMDAligned)
        {
            size_t i = 0;
            for (; i < nSIMD; i += inc, d += inc)
                e.getAVX (i).storeAligned (d);
 
            for (; i < n; ++i, ++d)
                storage[i] = e[i];
        }
        else
        {
            size_t i = 0;
            for (; i < nSIMD; i += inc, d += inc)
                e.getAVX (i).storeUnaligned (d);
 
            for (; i < n; ++i, ++d)
                storage[i] = e[i];
        }
    }
 
    template <class Expression>
    VCTR_TARGET ("sse4.1")
    void assignExpressionTemplateSSE4_1 (const Expression& e)
    requires archX64
    {
        constexpr auto inc = SSERegister<ElementType>::numElements;
        const bool hasExtendedSIMDStorage = e.getStorageInfo().hasSIMDExtendedStorage && StorageInfoType::hasSIMDExtendedStorage;
        const auto n = storage.size();
        const auto nSIMD = hasExtendedSIMDStorage ? detail::nextMultipleOf<inc> (n) : detail::previousMultipleOf<inc> (n);
 
        auto* d = data();
 
        if (StorageInfoType::dataIsSIMDAligned)
        {
            size_t i = 0;
            for (; i < nSIMD; i += inc, d += inc)
                e.getSSE (i).storeAligned (d);
 
            for (; i < n; ++i, ++d)
                storage[i] = e[i];
        }
        else
        {
            size_t i = 0;
            for (; i < nSIMD; i += inc, d += inc)
                e.getSSE (i).storeUnaligned (d);
 
            for (; i < n; ++i, ++d)
                storage[i] = e[i];
        }
    }
 
    //==============================================================================
    template <size_t spanExtent>
    constexpr auto constCorrectSpan (ElementType* data, size_t spanSize);
 
    template <size_t spanExtent>
    constexpr auto constCorrectSpan (const ElementType* data, size_t spanSize) const
    requires (! is::stdSpan<StorageType>);
 
    template <size_t spanExtent>
    constexpr auto constCorrectSpan (ElementType* data, size_t spanSize) const
    requires is::stdSpan<StorageType>;
 
    //==============================================================================
    static void clear (ElementType* ptr, size_t numElements)
    {
        std::memset (ptr, 0, numElements * sizeof (ElementType));
    }
 
    static void memMove (const ElementType* src, ElementType* dst, size_t numElements)
    requires std::is_trivially_copyable_v<ElementType>
    {
        std::memmove (dst, src, numElements * sizeof (ElementType));
    }
 
    template <class T>
    VCTR_FORCEDINLINE constexpr static T* assumeAlignedToMaxSIMDRegisterSize (T* ptr)
    {
        if (std::is_constant_evaluated())
            return ptr;
 
    #if __cpp_lib_assume_aligned
        return std::assume_aligned<maxSIMDRegisterSize> (ptr);
    #elif VCTR_CLANG
        return static_cast<T*> (__builtin_assume_aligned (ptr, maxSIMDRegisterSize));
    #else
        return ptr;
    #endif
    }
};
 
template <is::anyVctr Lhs, is::anyVctr Rhs>
requires std::same_as<ValueType<Lhs>, ValueType<Rhs>>
constexpr bool operator== (const Lhs& lhs, const Rhs& rhs)
{
    return std::equal (lhs.begin(), lhs.end(), rhs.begin(), rhs.end());
}
 
} // namespace vctr
 
#if VCTR_MSVC
__pragma (warning (pop))
#endif