Skip to content

Commit

Permalink
Make cuda::std::tuple trivially copyable (#2127)
Browse files Browse the repository at this point in the history 
* Make `cuda::std::tuple` trivially copyable

This is similar to the situation with `cuda::std::pair`
We have a lot of users that rely on types being trivially copyable, so that they can utilize memcpy and friends.

Previously, `cuda::std::tuple` did not satisfy this because it needs to handle reference types.

Given that we already specialize `__tuple_leaf` depending on whether the class is empty or not, we can simply
add a third specialization that handles the trivially copyable types and one that synthesizes assignment.

Co-authored-by: Bernhard Manfred Gruber <[email protected]>
  • Loading branch information
@miscco @bernhardmgruber
miscco and bernhardmgruber committed Aug 1, 2024
1 parent fadb135 commit 4634d81
Show file tree
Hide file tree
Showing 4 changed files with 196 additions and 94 deletions.
22 changes: 19 additions & 3 deletions libcudacxx/include/cuda/std/__tuple_dir/sfinae_helpers.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
#include <cuda/std/__type_traits/is_assignable.h>
#include <cuda/std/__type_traits/is_constructible.h>
#include <cuda/std/__type_traits/is_convertible.h>
#include <cuda/std/__type_traits/is_copy_assignable.h>
#include <cuda/std/__type_traits/is_move_assignable.h>
#include <cuda/std/__type_traits/is_same.h>
#include <cuda/std/__type_traits/remove_cvref.h>
#include <cuda/std/__type_traits/remove_reference.h>
Expand Down Expand Up @@ -145,8 +147,6 @@ struct _LIBCUDACXX_TYPE_VIS __check_tuple_constructor_fail
using __enable_assign = false_type;
};

#if _CCCL_STD_VER > 2011

enum class __smf_availability
{
__trivial,
Expand Down Expand Up @@ -209,7 +209,23 @@ struct __sfinae_move_assign_base<_CanCopy, _CanMove, _CanCopyAssign, false>

template <bool _CanCopy, bool _CanMove, bool _CanCopyAssign, bool _CanMoveAssign>
using __sfinae_base = __sfinae_move_assign_base<_CanCopy, _CanMove, _CanCopyAssign, _CanMoveAssign>;
#endif // _CCCL_STD_VER > 2011

// We need to synthesize the copy / move assignment if it would be implicitly deleted as a member of a class
// In that case _Tp would be copy assignable but _TestSynthesizeAssignment<_Tp> would not
// This happens e.g for reference types
template <class _Tp>
struct _TestSynthesizeAssignment
{
_Tp __dummy;
};

template <class _Tp>
struct __must_synthesize_assignment
: integral_constant<
bool,
(_CCCL_TRAIT(is_copy_assignable, _Tp) && !_CCCL_TRAIT(is_copy_assignable, _TestSynthesizeAssignment<_Tp>))
|| (_CCCL_TRAIT(is_move_assignable, _Tp) && !_CCCL_TRAIT(is_move_assignable, _TestSynthesizeAssignment<_Tp>))>
{};

_LIBCUDACXX_END_NAMESPACE_STD

Expand Down
24 changes: 3 additions & 21 deletions libcudacxx/include/cuda/std/__utility/pair.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,28 +115,10 @@ struct __pair_constraints
};
};

// We need to synthesize the copy / move assignment if it would be implicitly deleted as a member of a class
// In that case _T1 would be copy assignable but _TestSynthesizeAssignment<_T1> would not
// This happens e.g for reference types
template <class _T1>
struct _TestSynthesizeAssignment
{
_T1 __dummy;
};

template <class _T1, class _T2>
struct __must_synthesize_assignment
: integral_constant<bool,
(_CCCL_TRAIT(is_copy_assignable, _T1) && _CCCL_TRAIT(is_copy_assignable, _T2)
&& !(_CCCL_TRAIT(is_copy_assignable, _TestSynthesizeAssignment<_T1>)
&& _CCCL_TRAIT(is_copy_assignable, _TestSynthesizeAssignment<_T2>)))
|| (_CCCL_TRAIT(is_move_assignable, _T1) && _CCCL_TRAIT(is_move_assignable, _T2)
&& !(_CCCL_TRAIT(is_move_assignable, _TestSynthesizeAssignment<_T1>)
&& _CCCL_TRAIT(is_move_assignable, _TestSynthesizeAssignment<_T2>)))>
{};

// base class to ensure `is_trivially_copyable` when possible
template <class _T1, class _T2, bool = __must_synthesize_assignment<_T1, _T2>::value>
template <class _T1,
class _T2,
bool = __must_synthesize_assignment<_T1>::value || __must_synthesize_assignment<_T2>::value>
struct __pair_base
{
_T1 first;
Expand Down
191 changes: 121 additions & 70 deletions libcudacxx/include/cuda/std/detail/libcxx/include/tuple
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,18 +198,34 @@ _LIBCUDACXX_BEGIN_NAMESPACE_STD
struct __tuple_leaf_default_constructor_tag
{};

template <size_t _Ip, class _Hp, bool = _CCCL_TRAIT(is_empty, _Hp) && !__libcpp_is_final<_Hp>::value>
enum class __tuple_leaf_specialization
{
__default,
__synthesize_assignment,
__empty_non_final,
};

template <class _Tp>
_LIBCUDACXX_INLINE_VISIBILITY constexpr __tuple_leaf_specialization __tuple_leaf_choose()
{
return _CCCL_TRAIT(is_empty, _Tp) && !__libcpp_is_final<_Tp>::value ? __tuple_leaf_specialization::__empty_non_final
: __must_synthesize_assignment<_Tp>::value
? __tuple_leaf_specialization::__synthesize_assignment
: __tuple_leaf_specialization::__default;
}

template <size_t _Ip, class _Hp, __tuple_leaf_specialization = __tuple_leaf_choose<_Hp>()>
class __tuple_leaf;

template <size_t _Ip, class _Hp, bool _Ep>
template <size_t _Ip, class _Hp, __tuple_leaf_specialization _Ep>
inline _LIBCUDACXX_INLINE_VISIBILITY void
swap(__tuple_leaf<_Ip, _Hp, _Ep>& __x, __tuple_leaf<_Ip, _Hp, _Ep>& __y) noexcept(__is_nothrow_swappable<_Hp>::value)
{
swap(__x.get(), __y.get());
}

template <size_t _Ip, class _Hp, bool>
class __tuple_leaf
template <size_t _Ip, class _Hp>
class __tuple_leaf<_Ip, _Hp, __tuple_leaf_specialization::__default>
{
_Hp __value_;

Expand All @@ -223,39 +239,97 @@ class __tuple_leaf
#endif
}

_LIBCUDACXX_INLINE_VISIBILITY __tuple_leaf& operator=(const __tuple_leaf&);

public:
_LIBCUDACXX_INLINE_VISIBILITY constexpr __tuple_leaf() noexcept(_CCCL_TRAIT(is_nothrow_default_constructible, _Hp))
: __value_()
{
static_assert(!_CCCL_TRAIT(is_reference, _Hp), "Attempted to default construct a reference element in a tuple");
}
{}

_LIBCUDACXX_INLINE_VISIBILITY constexpr __tuple_leaf(__tuple_leaf_default_constructor_tag) noexcept(
_CCCL_TRAIT(is_nothrow_default_constructible, _Hp))
: __value_()
{
static_assert(!_CCCL_TRAIT(is_reference, _Hp), "Attempted to default construct a reference element in a tuple");
}
{}

template <class _Alloc>
_LIBCUDACXX_INLINE_VISIBILITY __tuple_leaf(integral_constant<int, 0>, const _Alloc&)
: __value_()
{
static_assert(!_CCCL_TRAIT(is_reference, _Hp), "Attempted to default construct a reference element in a tuple");
}
{}

template <class _Alloc>
_LIBCUDACXX_INLINE_VISIBILITY __tuple_leaf(integral_constant<int, 1>, const _Alloc& __a)
: __value_(allocator_arg_t(), __a)
{
static_assert(!_CCCL_TRAIT(is_reference, _Hp), "Attempted to default construct a reference element in a tuple");
}
{}

template <class _Alloc>
_LIBCUDACXX_INLINE_VISIBILITY __tuple_leaf(integral_constant<int, 2>, const _Alloc& __a)
: __value_(__a)
{}

template <class _Tp>
using __can_forward = _And<_IsNotSame<__remove_cvref_t<_Tp>, __tuple_leaf>, is_constructible<_Hp, _Tp>>;

template <class _Tp, __enable_if_t<__can_forward<_Tp>::value, int> = 0>
_LIBCUDACXX_INLINE_VISIBILITY
_CCCL_CONSTEXPR_CXX14 explicit __tuple_leaf(_Tp&& __t) noexcept(_CCCL_TRAIT(is_nothrow_constructible, _Hp, _Tp))
: __value_(_CUDA_VSTD::forward<_Tp>(__t))
{}

template <class _Tp, class _Alloc>
_LIBCUDACXX_INLINE_VISIBILITY explicit __tuple_leaf(integral_constant<int, 0>, const _Alloc&, _Tp&& __t)
: __value_(_CUDA_VSTD::forward<_Tp>(__t))
{}

template <class _Tp, class _Alloc>
_LIBCUDACXX_INLINE_VISIBILITY explicit __tuple_leaf(integral_constant<int, 1>, const _Alloc& __a, _Tp&& __t)
: __value_(allocator_arg_t(), __a, _CUDA_VSTD::forward<_Tp>(__t))
{}

template <class _Tp, class _Alloc>
_LIBCUDACXX_INLINE_VISIBILITY explicit __tuple_leaf(integral_constant<int, 2>, const _Alloc& __a, _Tp&& __t)
: __value_(_CUDA_VSTD::forward<_Tp>(__t), __a)
{}

template <class _Tp>
_LIBCUDACXX_INLINE_VISIBILITY __tuple_leaf&
operator=(_Tp&& __t) noexcept(_CCCL_TRAIT(is_nothrow_assignable, _Hp&, _Tp))
{
__value_ = _CUDA_VSTD::forward<_Tp>(__t);
return *this;
}

_LIBCUDACXX_INLINE_VISIBILITY int swap(__tuple_leaf& __t) noexcept(__is_nothrow_swappable<__tuple_leaf>::value)
{
_CUDA_VSTD::swap(*this, __t);
return 0;
}

_LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX14 _Hp& get() noexcept
{
return __value_;
}
_LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX14 const _Hp& get() const noexcept
{
return __value_;
}
};

template <size_t _Ip, class _Hp>
class __tuple_leaf<_Ip, _Hp, __tuple_leaf_specialization::__synthesize_assignment>
{
_Hp __value_;

template <class _Tp>
_LIBCUDACXX_INLINE_VISIBILITY static constexpr bool __can_bind_reference()
{
#if __has_keyword(__reference_binds_to_temporary)
return !__reference_binds_to_temporary(_Hp, _Tp);
#else
return true;
#endif
}

public:
_LIBCUDACXX_INLINE_VISIBILITY constexpr __tuple_leaf() noexcept(_CCCL_TRAIT(is_nothrow_default_constructible, _Hp))
: __value_()
{
static_assert(!_CCCL_TRAIT(is_reference, _Hp), "Attempted to default construct a reference element in a tuple");
}
Expand All @@ -282,25 +356,20 @@ public:
"temporary whose lifetime has ended");
}

template <class _Tp, class _Alloc>
_LIBCUDACXX_INLINE_VISIBILITY explicit __tuple_leaf(integral_constant<int, 1>, const _Alloc& __a, _Tp&& __t)
: __value_(allocator_arg_t(), __a, _CUDA_VSTD::forward<_Tp>(__t))
__tuple_leaf(const __tuple_leaf& __t) = default;
__tuple_leaf(__tuple_leaf&& __t) = default;

_LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX14 __tuple_leaf& operator=(const __tuple_leaf& __t) noexcept
{
static_assert(!_CCCL_TRAIT(is_reference, _Hp),
"Attempted to uses-allocator construct a reference element in a tuple");
__value_ = __t.__value_;
return *this;
}

template <class _Tp, class _Alloc>
_LIBCUDACXX_INLINE_VISIBILITY explicit __tuple_leaf(integral_constant<int, 2>, const _Alloc& __a, _Tp&& __t)
: __value_(_CUDA_VSTD::forward<_Tp>(__t), __a)
_LIBCUDACXX_INLINE_VISIBILITY _CCCL_CONSTEXPR_CXX14 __tuple_leaf& operator=(__tuple_leaf&& __t) noexcept
{
static_assert(!_CCCL_TRAIT(is_reference, _Hp),
"Attempted to uses-allocator construct a reference element in a tuple");
__value_ = _CUDA_VSTD::move(__t.__value_);
return *this;
}

__tuple_leaf(const __tuple_leaf& __t) = default;
__tuple_leaf(__tuple_leaf&& __t) = default;

template <class _Tp>
_LIBCUDACXX_INLINE_VISIBILITY __tuple_leaf&
operator=(_Tp&& __t) noexcept(_CCCL_TRAIT(is_nothrow_assignable, _Hp&, _Tp))
Expand All @@ -326,10 +395,8 @@ public:
};

template <size_t _Ip, class _Hp>
class __tuple_leaf<_Ip, _Hp, true> : private _Hp
class __tuple_leaf<_Ip, _Hp, __tuple_leaf_specialization::__empty_non_final> : private _Hp
{
_LIBCUDACXX_INLINE_VISIBILITY __tuple_leaf& operator=(const __tuple_leaf&);

public:
_LIBCUDACXX_INLINE_VISIBILITY constexpr __tuple_leaf() noexcept(is_nothrow_default_constructible<_Hp>::value) {}

Expand Down Expand Up @@ -376,10 +443,15 @@ public:
: _Hp(_CUDA_VSTD::forward<_Tp>(__t), __a)
{}

__tuple_leaf(__tuple_leaf const&) = default;
__tuple_leaf(__tuple_leaf&&) = default;
template <class _Tp, __enable_if_t<_CCCL_TRAIT(is_assignable, _Hp&, const _Tp&), int> = 0>
_LIBCUDACXX_INLINE_VISIBILITY __tuple_leaf&
operator=(const _Tp& __t) noexcept(_CCCL_TRAIT(is_nothrow_assignable, _Hp&, const _Tp&))
{
_Hp::operator=(__t);
return *this;
}

template <class _Tp>
template <class _Tp, __enable_if_t<_CCCL_TRAIT(is_assignable, _Hp&, _Tp), int> = 0>
_LIBCUDACXX_INLINE_VISIBILITY __tuple_leaf&
operator=(_Tp&& __t) noexcept(_CCCL_TRAIT(is_nothrow_assignable, _Hp&, _Tp))
{
Expand Down Expand Up @@ -425,9 +497,14 @@ struct __tuple_impl;
template <size_t... _Indx, class... _Tp>
struct _LIBCUDACXX_DECLSPEC_EMPTY_BASES __tuple_impl<__tuple_indices<_Indx...>, _Tp...>
: public __tuple_leaf<_Indx, _Tp>...
, public __sfinae_base<true,
true,
__all<_CCCL_TRAIT(is_copy_assignable, _Tp)...>::value,
__all<_CCCL_TRAIT(is_move_assignable, _Tp)...>::value>
{
_LIBCUDACXX_INLINE_VISIBILITY constexpr __tuple_impl() noexcept(
__all<_CCCL_TRAIT(is_nothrow_default_constructible, _Tp)...>::value)
: __tuple_leaf<_Indx, _Tp>()...
{}

// Handle non-allocator, full initialization
Expand Down Expand Up @@ -487,24 +564,10 @@ struct _LIBCUDACXX_DECLSPEC_EMPTY_BASES __tuple_impl<__tuple_indices<_Indx...>,
return *this;
}

__tuple_impl(const __tuple_impl&) = default;
__tuple_impl(__tuple_impl&&) = default;

_LIBCUDACXX_INLINE_VISIBILITY __tuple_impl&
operator=(const __tuple_impl& __t) noexcept((__all<_CCCL_TRAIT(is_nothrow_copy_assignable, _Tp)...>::value))
{
_CUDA_VSTD::__swallow(
__tuple_leaf<_Indx, _Tp>::operator=(static_cast<const __tuple_leaf<_Indx, _Tp>&>(__t).get())...);
return *this;
}

_LIBCUDACXX_INLINE_VISIBILITY __tuple_impl&
operator=(__tuple_impl&& __t) noexcept((__all<_CCCL_TRAIT(is_nothrow_move_assignable, _Tp)...>::value))
{
_CUDA_VSTD::__swallow(__tuple_leaf<_Indx, _Tp>::operator=(
_CUDA_VSTD::forward<_Tp>(static_cast<__tuple_leaf<_Indx, _Tp>&>(__t).get()))...);
return *this;
}
__tuple_impl(const __tuple_impl&) = default;
__tuple_impl(__tuple_impl&&) = default;
__tuple_impl& operator=(const __tuple_impl&) = default;
__tuple_impl& operator=(__tuple_impl&&) = default;

_LIBCUDACXX_INLINE_VISIBILITY void
swap(__tuple_impl& __t) noexcept(__all<__is_nothrow_swappable<_Tp>::value...>::value)
Expand Down Expand Up @@ -853,20 +916,8 @@ public:
using _CanCopyAssign = __all<_CCCL_TRAIT(is_copy_assignable, _Tp)...>;
using _CanMoveAssign = __all<_CCCL_TRAIT(is_move_assignable, _Tp)...>;

_LIBCUDACXX_INLINE_VISIBILITY tuple&
operator=(__conditional_t<_CanCopyAssign::value, tuple, __nat> const& __t) noexcept(
(__all<_CCCL_TRAIT(is_nothrow_copy_assignable, _Tp)...>::value))
{
__base_.operator=(__t.__base_);
return *this;
}

_LIBCUDACXX_INLINE_VISIBILITY tuple& operator=(__conditional_t<_CanMoveAssign::value, tuple, __nat>&& __t) noexcept(
(__all<_CCCL_TRAIT(is_nothrow_move_assignable, _Tp)...>::value))
{
__base_.operator=(static_cast<_BaseT&&>(__t.__base_));
return *this;
}
tuple& operator=(const tuple& __t) = default;
tuple& operator=(tuple&& __t) = default;

template <class _Tuple, __enable_if_t<__tuple_assignable<_Tuple, tuple>::value, bool> = false>
_LIBCUDACXX_INLINE_VISIBILITY tuple&
Expand Down
Loading

0 comments on commit 4634d81

Please sign in to comment.