diff --git a/stan/math/opencl/copy.hpp b/stan/math/opencl/copy.hpp index fe2d88b3b3c..e74cf790fcd 100644 --- a/stan/math/opencl/copy.hpp +++ b/stan/math/opencl/copy.hpp @@ -27,9 +27,15 @@ namespace stan { namespace math { /** \ingroup opencl - * Copies the source Eigen matrix to - * the destination matrix that is stored - * on the OpenCL device. + * Copies the source Eigen matrix to the destination matrix that is stored on + * the OpenCL device. If a lvalue matrix is passed to this function the caller + * must make sure that the matrix does not go out of scope before copying is + * complete. + * + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * returned matrix or any matrix that is calculated from that one. * * @param src source Eigen matrix * @return matrix_cl with a copy of the data in the source matrix @@ -41,9 +47,14 @@ inline matrix_cl to_matrix_cl(Mat&& src) { } /** \ingroup opencl - * Copies the source std::vector to - * the destination matrix that is stored - * on the OpenCL device. + * Copies the source `std::vector` to the destination matrix that is stored on + * the OpenCL device. If a lvalue is passed to this constructor the caller must + * make sure that it does not go out of scope before copying is complete. + * + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * returned matrix or any matrix that is calculated from that one. * * @param src source std::vector * @return matrix_cl with a copy of the data in the source matrix @@ -55,9 +66,8 @@ inline matrix_cl to_matrix_cl(Vec&& src) { } /** \ingroup opencl - * Copies the source matrix that is stored - * on the OpenCL device to the destination Eigen - * matrix. + * Copies the source matrix that is stored on the OpenCL device to the + * destination Eigen matrix. * * @param src source matrix on the OpenCL device * @return Eigen matrix with a copy of the data in the source matrix @@ -93,7 +103,7 @@ inline Eigen::Matrix from_matrix_cl(const matrix_cl& src) { } /** \ingroup opencl - * Packs the flat triagnular matrix on the OpenCL device and + * Packs the flat triangular matrix on the OpenCL device and * copies it to the std::vector. * * @param src the flat triangular source matrix on the OpenCL device @@ -129,9 +139,15 @@ inline std::vector packed_copy(const matrix_cl& src) { } /** \ingroup opencl - * Copies the packed triangular matrix from - * the source std::vector to an OpenCL buffer and - * unpacks it to a flat matrix on the OpenCL device. + * Copies the packed triangular matrix from the source std::vector to an OpenCL + * buffer and unpacks it to a flat matrix on the OpenCL device. If a lvalue is + * passed to this constructor the caller must make sure that it does not go out + * of scope before copying is complete. + * + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * returned matrix or any matrix that is calculated from that one. * * @tparam matrix_view the triangularity of the source matrix * @param src the packed source std::vector @@ -212,7 +228,15 @@ inline T from_matrix_cl_error_code(const matrix_cl& src) { } /** \ingroup opencl - * Copy an arithmetic type to the device. + * Copy an arithmetic type to the device. If a lvalue is passed to this + * constructor the caller must make sure that it does not go out of scope before + * copying is complete. + * + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * returned matrix or any matrix that is calculated from that one. + * * @tparam T An arithmetic type to pass the value from the OpenCL matrix to. * @param src Arithmetic to receive the matrix_cl value. * @return A 1x1 matrix on the device. diff --git a/stan/math/opencl/matrix_cl.hpp b/stan/math/opencl/matrix_cl.hpp index 4b680f2f271..d1cff51c7b5 100644 --- a/stan/math/opencl/matrix_cl.hpp +++ b/stan/math/opencl/matrix_cl.hpp @@ -218,10 +218,17 @@ class matrix_cl> { write_events_(std::move(A.write_events_)), read_events_(std::move(A.read_events_)) {} - /** \ingroup opencl + /** * Constructor for the matrix_cl that creates a copy of a std::vector of Eigen * matrices on the OpenCL device. Each matrix is flattened into one column - * of the resulting matrix_cl. + * of the resulting matrix_cl. If a lvalue is passed to this constructor the + * caller must make sure that the vector does not go out of scope before + * copying is complete. + * + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * this matrix or any matrix that is calculated from this one. * * @param A the vector of Eigen matrices * @@ -284,29 +291,88 @@ class matrix_cl> { } /** \ingroup opencl - * Constructor for the matrix_cl that - * creates a copy of the Eigen matrix on the OpenCL device. - * Regardless of `partial_view`, whole matrix is stored. + * Constructor for the matrix_cl that creates a copy of the Eigen matrix or + * Eigen expression on the OpenCL device. Regardless of `partial_view`, whole + * matrix is stored. If a lvalue matrix is passed to this constructor the + * caller must make sure that the matrix does not go out of scope before + * copying is complete. * - * @tparam T type of data in the \c Eigen \c Matrix - * @param A the \c Eigen \c Matrix + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * this matrix or any matrix that is calculated from this one. + * + * @tparam Mat type of \c Eigen \c Matrix or expression + * @param A the \c Eigen \c Matrix or expression * @param partial_view which part of the matrix is used * * @throw std::system_error if the memory on the device could not * be allocated */ - template ..., require_same_vt...> + template ..., require_same_vt..., + require_not_t>...> explicit matrix_cl(Mat&& A, matrix_cl_view partial_view = matrix_cl_view::Entire) : rows_(A.rows()), cols_(A.cols()), view_(partial_view) { - initialize_buffer::value - || !is_eigen_matrix::value>(A.eval().data()); + using Mat_type = std::decay_t; + if (size() == 0) { + return; + } + if (is_eigen_matrix_or_array::value + && std::is_lvalue_reference::value) { + initialize_buffer(A.eval().data()); + } else { + auto* A_heap = new Mat_type(std::move(A.eval())); + try { + cl::Event e = initialize_buffer(A_heap->data()); + // We set a callback that will delete the memory once copying is + // complete. This event object does not hold the information about + // callback. OpenCL implementation does. So nothing is lost as event + // goes out of scope. + e.setCallback(CL_COMPLETE, &delete_it, A_heap); + } catch (...) { + delete A_heap; + throw; + } + } } /** \ingroup opencl - * Constructor for the matrix_cl that - * creates a copy of a scalar on the OpenCL device. - * Regardless of `partial_view`, whole matrix is stored. + * Constructor for the matrix_cl that creates a copy of the Eigen Map on the + * OpenCL device. Regardless of `partial_view`, whole matrix is stored. The + * caller must make sure that the memory referenced by map is not deleted + * before copying is complete. + * + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * this matrix or any matrix that is calculated from this one. + * + * @tparam T type of data in the \c Eigen \c Matrix + * @param A the \c Eigen \c Map + * @param partial_view which part of the matrix is used + * + * @throw std::system_error if the memory on the device could not + * be allocated + */ + template >..., + require_same_vt...> + explicit matrix_cl(Map&& A, + matrix_cl_view partial_view = matrix_cl_view::Entire) + : rows_(A.rows()), cols_(A.cols()), view_(partial_view) { + initialize_buffer(A.data()); + } + + /** \ingroup opencl + * Constructor for the matrix_cl that creates a copy of a scalar on the OpenCL + * device. Regardless of `partial_view`, whole matrix is stored. If a lvalue + * is passed to this constructor the caller must make sure that it does not go + * out of scope before copying is complete. + * + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * this matrix or any matrix that is calculated from this one. * * @param A the scalar * @param partial_view which part of the matrix is used @@ -323,7 +389,14 @@ class matrix_cl> { } /** \ingroup opencl - * Construct a matrix_cl of size Nx1 from \c std::vector + * Construct a matrix_cl of size Nx1 from \c std::vector. If a lvalue is + * passed to this constructor the caller must make sure that it does not go + * out of scope before copying is complete. + * + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * this matrix or any matrix that is calculated from this one. * * @param A Standard vector * @param partial_view which part of the matrix is used @@ -337,8 +410,15 @@ class matrix_cl> { matrix_cl_view partial_view = matrix_cl_view::Entire) : matrix_cl(std::forward(A), A.size(), 1) {} - /** - * Construct from \c std::vector with given rows and columns + /** \ingroup opencl + * Construct from \c std::vector with given rows and columns. If a lvalue + * is passed to this constructor the caller must make sure that it does not + * go out of scope before copying is complete. + * + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * this matrix or any matrix that is calculated from this one. * * @param A Standard vector * @param R Number of rows the matrix should have. @@ -353,11 +433,17 @@ class matrix_cl> { explicit matrix_cl(Vec&& A, const int& R, const int& C, matrix_cl_view partial_view = matrix_cl_view::Entire) : rows_(R), cols_(C), view_(partial_view) { - initialize_buffer::value>(A.data()); + initialize_buffer_optionally_from_heap(std::forward(A)); } /** \ingroup opencl - * Construct from \c array of doubles with given rows and columns + * Construct from \c array with given rows and columns. The caller + * must make sure that data is not deleted before copying is complete. + * + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * this matrix or any matrix that is calculated from this one. * * @param A array of doubles * @param R Number of rows the matrix should have. @@ -392,35 +478,43 @@ class matrix_cl> { * Copy assignment operator. */ matrix_cl& operator=(const matrix_cl& a) { - if (a.size() == 0) { - return *this; - } this->view_ = a.view(); this->rows_ = a.rows(); this->cols_ = a.cols(); + if (a.size() == 0) { + return *this; + } this->wait_for_read_write_events(); initialize_buffer(a); return *this; } private: - /** + /** \ingroup opencl * Initializes the OpencL buffer of this matrix by copying the data from given * buffer. Assumes that size of \c this is already set and matches the - * buffer size. + * buffer size. If \c in_order is false the caller must make sure that data + * is not deleted before copying is complete. + * + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * this matrix or any matrix that is calculated from this one. + * * @tparam in_order whether copying must be done in order * @param A pointer to buffer + * @return event for the copy */ template - void initialize_buffer(const T* A) { + cl::Event initialize_buffer(const T* A) { + cl::Event transfer_event; if (size() == 0) { - return; + return transfer_event; } cl::Context& ctx = opencl_context.context(); cl::CommandQueue& queue = opencl_context.queue(); try { buffer_cl_ = cl::Buffer(ctx, CL_MEM_READ_WRITE, sizeof(T) * size()); - cl::Event transfer_event; queue.enqueueWriteBuffer(buffer_cl_, opencl_context.in_order() || in_order, 0, sizeof(T) * size(), A, nullptr, &transfer_event); @@ -428,9 +522,47 @@ class matrix_cl> { } catch (const cl::Error& e) { check_opencl_error("initialize_buffer", e); } + return transfer_event; } - /** + /** \ingroup opencl + * Initializes the OpencL buffer of this matrix by copying the data from given + * object. Assumes that size of \c this is already set and matches the + * buffer size. If the object is rvalue (temporary) it is first moved to heap + * and callback is set to delete it after copying to OpenCL device is + * complete. If a lvalue is passed to this function the caller must make + * sure that input object does not go out of scope before copying is complete. + * + * That means `.wait()` must be called on the event associated on copying or + * any other event that requires completion of this event. This can be done by + * calling `.wait_for_write_events()` or `.wait_for_read_write_events()` on + * this matrix or any matrix that is calculated from this one. + * + * @tparam U type of object + * @param obj object + * @return event for the copy + */ + template + void initialize_buffer_optionally_from_heap(U&& obj) { + using U_val = std::decay_t; + if (size() == 0) { + return; + } + if (std::is_lvalue_reference::value) { + initialize_buffer(obj.data()); + } else { + auto* obj_heap = new U_val(std::move(obj)); + try { + cl::Event e = initialize_buffer(obj_heap->data()); + e.setCallback(CL_COMPLETE, &delete_it, obj_heap); + } catch (...) { + delete obj_heap; + throw; + } + } + } + + /** \ingroup opencl * Initializes the OpencL buffer of this matrix by copying the data from given * matrix_cl. Assumes that size of \c this is already set and matches the * size of given matrix. @@ -451,6 +583,18 @@ class matrix_cl> { check_opencl_error("copy (OpenCL)->(OpenCL)", e); } } + + /** \ingroup opencl + * Deletes the container. Used as a callback for OpenCL event. + * @tparam U type of container + * @param e cl_event handle + * @param status status of event + * @param container container to delete + */ + template + static void delete_it(cl_event e, cl_int status, void* container) { + delete static_cast(container); + } }; template diff --git a/stan/math/prim/mat/meta/is_eigen.hpp b/stan/math/prim/mat/meta/is_eigen.hpp index fff50e83d94..b9c4fa7df13 100644 --- a/stan/math/prim/mat/meta/is_eigen.hpp +++ b/stan/math/prim/mat/meta/is_eigen.hpp @@ -3,6 +3,7 @@ #include #include +#include #include namespace stan { @@ -43,11 +44,37 @@ template struct is_eigen_matrix_impl> : std::true_type {}; template struct is_eigen_matrix_impl> : std::true_type {}; - } // namespace internal template struct is_eigen_matrix : internal::is_eigen_matrix_impl> {}; +namespace internal { +template +struct is_eigen_array_impl : std::false_type {}; +template +struct is_eigen_array_impl> : std::true_type {}; +} // namespace internal + +template +struct is_eigen_array : internal::is_eigen_array_impl> {}; + +template +using is_eigen_matrix_or_array + = math::disjunction, is_eigen_array>; + +namespace internal { +template +struct is_eigen_contiguous_map_impl : std::false_type {}; +template +struct is_eigen_contiguous_map_impl>> + : std::true_type {}; + +} // namespace internal + +template +struct is_eigen_contiguous_map + : internal::is_eigen_contiguous_map_impl> {}; + } // namespace stan #endif