Skip to content

Commit

Permalink
unseq dispatch psuhed to tag_invoke stage
Browse files Browse the repository at this point in the history 
Signed-off-by: Hari Hara Naveen S <[email protected]>
  • Loading branch information
@Johan511
Johan511 committed Jul 28, 2023
1 parent 15e8964 commit bb560af
Show file tree
Hide file tree
Showing 2 changed files with 125 additions and 69 deletions.
157 changes: 108 additions & 49 deletions libs/core/algorithms/include/hpx/parallel/util/loop.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -747,16 +747,15 @@ namespace hpx::parallel::util {
}

namespace detail {

// Helper class to repeatedly call a function a given number of times
// starting from a given iterator position.
template <typename IterCat>
struct loop_with_cleanup_n
struct loop_with_cleanup_n final
: hpx::functional::detail::tag_fallback<loop_with_cleanup_n>
{
///////////////////////////////////////////////////////////////////
template <typename ExPolicy, typename Iter, typename FwdIter,
typename F, typename Cleanup,
HPX_CONCEPT_REQUIRES_(
!hpx::is_unsequenced_execution_policy_v<ExPolicy>)>
typename F, typename Cleanup>
static FwdIter call(ExPolicy, Iter it, std::size_t num,
FwdIter dest, F&& f, Cleanup&& cleanup)
{
Expand Down Expand Up @@ -788,84 +787,79 @@ namespace hpx::parallel::util {
}
}

template <typename ExPolicy, typename Iter, typename FwdIter,
typename F, typename Cleanup,
HPX_CONCEPT_REQUIRES_(
hpx::is_unsequenced_execution_policy_v<ExPolicy>)>
static FwdIter call(ExPolicy, Iter it, std::size_t num,
FwdIter dest, F&& f, Cleanup&& cleanup)
template <typename ExPolicy, typename FwdIter, typename F,
typename Cleanup>
static FwdIter call(
ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup)
{
FwdIter base = dest;
FwdIter base = it;
try
{
std::size_t count(num & std::size_t(-4)); // -V112

// clang-format off
HPX_IVDEP HPX_UNROLL HPX_VECTORIZE
for (std::size_t i = 0; i < count;
(void) ++it, ++dest, ++i) // -V112
(void) ++it, i += 4) // -V112
{
HPX_INVOKE(f, it, dest);
HPX_INVOKE(f, it);
HPX_INVOKE(f, ++it);
HPX_INVOKE(f, ++it);
HPX_INVOKE(f, ++it);
}

HPX_IVDEP HPX_UNROLL HPX_VECTORIZE
for (/**/; count < num; (void) ++count, ++it, ++dest)
for (/**/; count < num; (void) ++count, ++it)
{
HPX_INVOKE(f, it, dest);
HPX_INVOKE(f, it);
}
// clang-format on

return dest;
return it;
}
catch (...)
{
for (/**/; base != dest; ++base)
for (/**/; base != it; ++base)
{
HPX_INVOKE(cleanup, base);
}
throw;
}
}

template <typename ExPolicy, typename FwdIter, typename F,
typename Cleanup,
HPX_CONCEPT_REQUIRES_(
!hpx::is_unsequenced_execution_policy_v<ExPolicy>)>
static FwdIter call(
ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup)
template <typename ExPolicy, typename Iter, typename FwdIter,
typename F, typename Cleanup>
FwdIter unseq_call(Iter it, std::size_t num, FwdIter dest, F&& f,
Cleanup&& cleanup)
{
FwdIter base = it;
FwdIter base = dest;
try
{
std::size_t count(num & std::size_t(-4)); // -V112

// clang-format off
HPX_IVDEP HPX_UNROLL HPX_VECTORIZE
for (std::size_t i = 0; i < count;
(void) ++it, i += 4) // -V112
(void) ++it, ++dest, ++i) // -V112
{
HPX_INVOKE(f, it);
HPX_INVOKE(f, ++it);
HPX_INVOKE(f, ++it);
HPX_INVOKE(f, ++it);
HPX_INVOKE(f, it, dest);
}
for (/**/; count < num; (void) ++count, ++it)

HPX_IVDEP HPX_UNROLL HPX_VECTORIZE
for (/**/; count < num; (void) ++count, ++it, ++dest)
{
HPX_INVOKE(f, it);
HPX_INVOKE(f, it, dest);
}
return it;
// clang-format on

return dest;
}
catch (...)
{
for (/**/; base != it; ++base)
for (/**/; base != dest; ++base)
{
HPX_INVOKE(cleanup, base);
}
throw;
}
}

template <typename ExPolicy, typename FwdIter, typename F,
typename Cleanup,
HPX_CONCEPT_REQUIRES_(
hpx::is_unsequenced_execution_policy_v<ExPolicy>)>
static FwdIter call(
typename Cleanup>
static FwdIter unseq_call(
ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup)
{
FwdIter base = it;
Expand Down Expand Up @@ -930,13 +924,78 @@ namespace hpx::parallel::util {
};
} // namespace detail

#if !defined(HPX_COMPUTE_DEVICE_CODE)
inline constexpr detail::loop_with_cleanup_n loop_with_cleanup_n =
detail::loop_with_cleanup_n{};
#else
template <typename ExPolicy, typename Begin, typename F, typename Cleanup>
HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup_n(
ExPolicy&& policy, Begin begin, std::size_t n, F&& f, Cleanup&& cleanup)
{
return hpx::parallel::util::detail::loop_with_cleanup_n{}(
HPX_FORWARD(ExPolicy, policy), begin, n, HPX_FORWARD(F, f),
HPX_FORWARD(Cleanup, cleanup));
}

template <typename ExPolicy, typename Begin, typename Begin2, typename F,
typename Cleanup>
HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup_n(
ExPolicy&& policy, Begin begin, std::size_t n, Begin2 dest, F&& f,
Cleanup&& cleanup)
{
return hpx::parallel::util::detail::loop_with_cleanup_n{}(
HPX_FORWARD(ExPolicy, policy), begin, n, dest, HPX_FORWARD(F, f),
HPX_FORWARD(Cleanup, cleanup));
}
#endif

template <typename Begin, typename F, typename Cleanup>
HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke(
hpx::parallel::util::detail::loop_with_cleanup_n,
hpx::execution::unsequenced_policy, Begin HPX_RESTRICT begin,
std::size_t n, F&& f, Cleanup&& cleanup)
{
return hpx::parallel::util::detail::loop_with_cleanup_n::unseq_call(
begin, n, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup));
}

template <typename Begin, typename F, typename Cleanup>
HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke(
hpx::parallel::util::detail::loop_with_cleanup_n,
hpx::execution::unsequenced_task_policy, Begin HPX_RESTRICT begin,
std::size_t n, F&& f, Cleanup&& cleanup)
{
return hpx::parallel::util::detail::loop_with_cleanup_n::unseq_call(
begin, n, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup));
}

template <typename Begin, typename Begin2, typename F, typename Cleanup>
HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke(
hpx::parallel::util::detail::loop_with_cleanup_n,
hpx::execution::unsequenced_policy, Begin HPX_RESTRICT begin,
std::size_t n, Begin2 dest, F&& f, Cleanup&& cleanup)
{
return hpx::parallel::util::detail::loop_with_cleanup_n::unseq_call(
begin, n, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup));
}

template <typename Begin, typename Begin2, typename F, typename Cleanup>
HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke(
hpx::parallel::util::detail::loop_with_cleanup_n,
hpx::execution::unsequenced_task_policy, Begin HPX_RESTRICT begin,
std::size_t n, Begin2 dest, F&& f, Cleanup&& cleanup)
{
return hpx::parallel::util::detail::loop_with_cleanup_n::unseq_call(
begin, n, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup));
}

template <typename Iter, typename CancelToken, typename F, typename Cleanup,
HPX_CONCEPT_REQUIRES_(hpx::traits::is_iterator_v<Iter>)>
HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n_with_token(
Iter it, std::size_t count, CancelToken& tok, F&& f, Cleanup&& cleanup)
{
using cat = typename std::iterator_traits<Iter>::iterator_category;
return detail::loop_with_cleanup_n<cat>::call_with_token(
return detail::loop_with_cleanup_n::call_with_token(
::hpx::execution::seq, it, count, tok, HPX_FORWARD(F, f),
HPX_FORWARD(Cleanup, cleanup));
}
Expand All @@ -950,7 +1009,7 @@ namespace hpx::parallel::util {
Cleanup&& cleanup)
{
using cat = typename std::iterator_traits<Iter>::iterator_category;
return detail::loop_with_cleanup_n<cat>::call_with_token(
return detail::loop_with_cleanup_n::call_with_token(
::hpx::execution::seq, it, count, dest, tok, HPX_FORWARD(F, f),
HPX_FORWARD(Cleanup, cleanup));
}
Expand All @@ -964,7 +1023,7 @@ namespace hpx::parallel::util {
Cleanup&& cleanup)
{
using cat = typename std::iterator_traits<Iter>::iterator_category;
return detail::loop_with_cleanup_n<cat>::call_with_token(
return detail::loop_with_cleanup_n::call_with_token(
HPX_FORWARD(ExPolicy, policy), it, count, tok, HPX_FORWARD(F, f),
HPX_FORWARD(Cleanup, cleanup));
}
Expand All @@ -979,7 +1038,7 @@ namespace hpx::parallel::util {
CancelToken& tok, F&& f, Cleanup&& cleanup)
{
using cat = typename std::iterator_traits<Iter>::iterator_category;
return detail::loop_with_cleanup_n<cat>::call_with_token(
return detail::loop_with_cleanup_n::call_with_token(
HPX_FORWARD(ExPolicy, policy), it, count, dest, tok,
HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup));
}
Expand Down
37 changes: 17 additions & 20 deletions libs/core/algorithms/include/hpx/parallel/util/transfer.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -370,16 +370,14 @@ namespace hpx::parallel::util {
ExPolicy&& policy, InIter first, std::size_t num, OutIter dest)
{
return in_out_result<InIter, OutIter>{std::next(first, num),
::hpx::parallel::util::detail::loop_with_cleanup_n<InIter>::
call(
HPX_FORWARD(ExPolicy, policy), first, num, dest,
[](InIter it, OutIter current) -> void {
hpx::construct_at(
std::addressof(*current), *it);
},
[](OutIter it) -> void {
std::destroy_at(std::addressof(*it));
})};
::hpx::parallel::util::detail::loop_with_cleanup_n::call(
HPX_FORWARD(ExPolicy, policy), first, num, dest,
[](InIter it, OutIter current) -> void {
hpx::construct_at(std::addressof(*current), *it);
},
[](OutIter it) -> void {
std::destroy_at(std::addressof(*it));
})};
}
};

Expand Down Expand Up @@ -429,16 +427,15 @@ namespace hpx::parallel::util {
ExPolicy&& policy, InIter first, std::size_t num, OutIter dest)
{
return in_out_result<InIter, OutIter>{std::next(first, num),
::hpx::parallel::util::detail::loop_with_cleanup_n<InIter>::
call(
HPX_FORWARD(ExPolicy, policy), first, num, dest,
[](InIter it, OutIter current) -> void {
hpx::construct_at(
std::addressof(*current), HPX_MOVE(*it));
},
[](OutIter it) -> void {
std::destroy_at(std::addressof(*it));
})};
::hpx::parallel::util::detail::loop_with_cleanup_n::call(
HPX_FORWARD(ExPolicy, policy), first, num, dest,
[](InIter it, OutIter current) -> void {
hpx::construct_at(
std::addressof(*current), HPX_MOVE(*it));
},
[](OutIter it) -> void {
std::destroy_at(std::addressof(*it));
})};
}
};

Expand Down

0 comments on commit bb560af

Please sign in to comment.