Commit 522e26e3 authored by Joseph Noir's avatar Joseph Noir

Enable kernel executions with multiple stages

This commit introduces memory references to memory buffers on OpenCL
devices and thus allows passing buffers between OpenCL actors on the
same host without the need to transfer memory between GPU and CPU
context. For this purpose, the existing wrappers that tag arguments
as input, output, or input as well as output now accept template
parameters to specify whether the argument is expected and / or
returned as a value (i.e., std::vector) or a reference.

On device computations that only return references reply with a message
containing the references directly and do not wait for the computation
to finish. OpenCL actors that receive such references enqueue their
kernels using OpenCL events to express the dependencies.

The function that maps incoming arguments of OpenCL actors now accepts
an optional additional argument, to allow adaption of the index space
for the execution depending on the arguments.
parent a31c0fc8
......@@ -3,8 +3,7 @@ project(caf_opencl C CXX)
# get header files; only needed by CMake generators,
# e.g., for creating proper Xcode projects
file(GLOB LIBCAF_OPENCL_HDRS "caf/opencl/*.hpp")
file(GLOB LIBCAF_OPENCL_HDRS "caf/opencl/detail/*.hpp")
file(GLOB_RECURSE LIBCAF_OPENCL_HDRS "caf/*.hpp")
add_custom_target(libcaf_opencl)
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -20,7 +20,7 @@
#ifndef CAF_OPENCL_DETAIL_SPAWN_HELPER_HPP
#define CAF_OPENCL_DETAIL_SPAWN_HELPER_HPP
#include "caf/opencl/actor_facade.hpp"
#include "caf/opencl/opencl_actor.hpp"
namespace caf {
namespace opencl {
......@@ -28,13 +28,13 @@ namespace detail {
struct tuple_construct { };
template <class... Ts>
template <bool PassConfig, class... Ts>
struct cl_spawn_helper {
using impl = opencl::actor_facade<Ts...>;
using map_in_fun = std::function<optional<message> (message&)>;
using impl = opencl::opencl_actor<PassConfig, Ts...>;
using map_in_fun = typename impl::input_mapping;
using map_out_fun = typename impl::output_mapping;
actor operator()(actor_config actor_cfg, const opencl::program& p,
actor operator()(actor_config actor_cfg, const opencl::program_ptr p,
const char* fn, const opencl::spawn_config& spawn_cfg,
Ts&&... xs) const {
return actor_cast<actor>(impl::create(std::move(actor_cfg),
......@@ -42,7 +42,16 @@ struct cl_spawn_helper {
map_in_fun{}, map_out_fun{},
std::move(xs)...));
}
actor operator()(actor_config actor_cfg, const opencl::program& p,
actor operator()(actor_config actor_cfg, const opencl::program_ptr p,
const char* fn, const opencl::spawn_config& spawn_cfg,
map_in_fun map_input, Ts&&... xs) const {
return actor_cast<actor>(impl::create(std::move(actor_cfg),
p, fn, spawn_cfg,
std::move(map_input),
map_out_fun{},
std::move(xs)...));
}
actor operator()(actor_config actor_cfg, const opencl::program_ptr p,
const char* fn, const opencl::spawn_config& spawn_cfg,
map_in_fun map_input, map_out_fun map_output,
Ts&&... xs) const {
......@@ -58,4 +67,4 @@ struct cl_spawn_helper {
} // namespace opencl
} // namespace caf
#endif // CAF_OPENCL_DETAIL_SPAWN_HELPER_HPP
#endif // CAF_OPENCL_DETAIL_SPAWN_HELPER_HPP
......@@ -22,23 +22,100 @@
#include <vector>
#include "caf/sec.hpp"
#include "caf/opencl/global.hpp"
#include "caf/opencl/smart_ptr.hpp"
#include "caf/opencl/opencl_err.hpp"
namespace caf {
namespace opencl {
class program;
class manager;
template <class T> class mem_ref;
class device;
using device_ptr = intrusive_ptr<device>;
class device {
class device : public ref_counted {
public:
friend class program;
friend class manager;
template <class T> friend class mem_ref;
template <class T, class... Ts>
friend intrusive_ptr<T> caf::make_counted(Ts&&...);
~device();
/// Create an argument for an OpenCL kernel with data placed in global memory.
template <class T>
mem_ref<T> global_argument(const std::vector<T>& data,
cl_mem_flags flags = buffer_type::input_output,
optional<size_t> size = none,
cl_bool blocking = CL_FALSE) {
size_t num_elements = size ? *size : data.size();
size_t buffer_size = sizeof(T) * num_elements;
auto buffer = v2get(CAF_CLF(clCreateBuffer), context_.get(), flags,
buffer_size, nullptr);
cl_event_ptr event{v1get<cl_event>(CAF_CLF(clEnqueueWriteBuffer),
queue_.get(), buffer, blocking,
cl_uint{0}, buffer_size, data.data()),
false};
return mem_ref<T>{num_elements, queue_, std::move(buffer), flags,
std::move(event)};
}
/// Create an argument for an OpenCL kernel in global memory without data.
template <class T>
mem_ref<T> scratch_argument(size_t size,
cl_mem_flags flags = buffer_type::scratch_space) {
auto buffer = v2get(CAF_CLF(clCreateBuffer), context_.get(), flags,
sizeof(T) * size, nullptr);
return mem_ref<T>{size, queue_, std::move(buffer), flags, nullptr};
}
/// Intialize a new device in a context using a sepcific device_id
static device create(const context_ptr& context, const device_ptr& device_id,
template <class T>
expected<mem_ref<T>> copy(mem_ref<T>& mem) {
if (!mem.get())
return make_error(sec::runtime_error, "No memory assigned.");
auto buffer_size = sizeof(T) * mem.size();
cl_event event;
auto buffer = v2get(CAF_CLF(clCreateBuffer), context_.get(), mem.access(),
buffer_size, nullptr);
std::vector<cl_event> prev_events;
cl_event e = mem.take_event();
if (e)
prev_events.push_back(e);
auto err = clEnqueueCopyBuffer(queue_.get(), mem.get().get(), buffer,
0, 0, // no offset for now
buffer_size, prev_events.size(),
prev_events.data(), &event);
if (err != CL_SUCCESS)
return make_error(sec::runtime_error, get_opencl_error(err));
// callback to release the previous event
if (e) {
err = clSetEventCallback(event, CL_COMPLETE,
[](cl_event, cl_int, void* data) {
auto tmp = reinterpret_cast<cl_event>(data);
if (tmp)
clReleaseEvent(tmp);
},
e);
if (err != CL_SUCCESS)
return make_error(sec::runtime_error, get_opencl_error(err));
}
// decrements the previous event we used for waiting above
return mem_ref<T>(mem.size(), queue_, std::move(buffer),
mem.access(), {event, false});
}
/// Initialize a new device in a context using a specific device_id
static device_ptr create(const cl_context_ptr& context,
const cl_device_ptr& device_id,
unsigned id);
/// Synchronizes all commands in its queue, waiting for them to finish.
void synchronize();
/// Get the id assigned by caf
inline unsigned get_id() const;
/// Returns device info on CL_DEVICE_ADDRESS_BITS
......@@ -93,21 +170,21 @@ public:
inline const std::string& get_name() const;
private:
device(device_ptr device_id, command_queue_ptr queue, context_ptr context,
unsigned id);
device(cl_device_ptr device_id, cl_command_queue_ptr queue,
cl_context_ptr context, unsigned id);
template <class T>
static T info(const device_ptr& device_id, unsigned info_flag) {
static T info(const cl_device_ptr& device_id, unsigned info_flag) {
T value;
clGetDeviceInfo(device_id.get(), info_flag, sizeof(T), &value, nullptr);
return value;
}
static std::string info_string(const device_ptr& device_id,
static std::string info_string(const cl_device_ptr& device_id,
unsigned info_flag);
device_ptr device_id_;
command_queue_ptr command_queue_;
context_ptr context_;
cl_device_ptr device_id_;
cl_command_queue_ptr queue_;
cl_context_ptr context_;
unsigned id_;
bool profiling_enabled_; // CL_DEVICE_QUEUE_PROPERTIES
......
......@@ -50,6 +50,14 @@ enum device_type {
all = CL_DEVICE_TYPE_ALL
};
/// Default values to create OpenCL buffers
enum buffer_type : cl_mem_flags {
input = CL_MEM_READ_WRITE | CL_MEM_HOST_WRITE_ONLY,
input_output = CL_MEM_READ_WRITE,
output = CL_MEM_READ_WRITE | CL_MEM_HOST_READ_ONLY,
scratch_space = CL_MEM_READ_WRITE | CL_MEM_HOST_NO_ACCESS
};
std::ostream& operator<<(std::ostream& os, device_type dev);
device_type device_type_from_ulong(cl_ulong dev);
......@@ -58,6 +66,9 @@ using dim_vec = detail::limited_vector<size_t, 3>;
std::string get_opencl_error(cl_int err);
std::string event_status(cl_event event);
} // namespace opencl
} // namespace caf
......
This diff is collapsed.
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2016 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#ifndef CAF_OPENCL_MEM_REF_HPP
#define CAF_OPENCL_MEM_REF_HPP
#include <ios>
#include <vector>
#include "caf/sec.hpp"
#include "caf/optional.hpp"
#include "caf/ref_counted.hpp"
#include "caf/opencl/smart_ptr.hpp"
namespace caf {
namespace opencl {
struct msg_adding_event {
msg_adding_event(cl_event_ptr event) : event_(event) {
// nop
}
template <class T, class... Ts>
message operator()(T& x, Ts&... xs) {
return make_message(add_event(std::move(x)), add_event(std::move(xs))...);
}
template <class... Ts>
message operator()(std::tuple<Ts...>& values) {
return apply_args(*this, detail::get_indices(values), values);
}
template <class T>
mem_ref<T> add_event(mem_ref<T> ref) {
ref.set_event(event_);
return std::move(ref);
}
cl_event_ptr event_;
};
class device;
/// A reference type for buffers on a OpenCL devive. Access is not thread safe.
/// Hence, a mem_ref should only be passed to actors sequentially.
template <class T>
class mem_ref {
public:
using value_type = T;
friend struct msg_adding_event;
template <bool PassConfig, class... Ts>
friend class opencl_actor;
friend class device;
expected<std::vector<T>> data(optional<size_t> result_size = none) {
if (!memory_)
return make_error(sec::runtime_error, "No memory assigned.");
if (0 != (access_ & CL_MEM_HOST_NO_ACCESS))
return make_error(sec::runtime_error, "No memory access.");
if (result_size && *result_size > num_elements_)
return make_error(sec::runtime_error, "Buffer has less elements.");
auto num_elements = (result_size ? *result_size : num_elements_);
auto buffer_size = sizeof(T) * num_elements;
std::vector<T> buffer(num_elements);
std::vector<cl_event> prev_events;
if (event_)
prev_events.push_back(event_.get());
cl_event event;
auto err = clEnqueueReadBuffer(queue_.get(), memory_.get(), CL_TRUE,
0, buffer_size, buffer.data(),
static_cast<cl_uint>(prev_events.size()),
prev_events.data(), &event);
if (err != CL_SUCCESS)
return make_error(sec::runtime_error, get_opencl_error(err));
// decrements the previous event we used for waiting above
event_.reset(event, false);
return buffer;
}
void reset() {
num_elements_ = 0;
access_ = CL_MEM_HOST_NO_ACCESS;
memory_.reset();
access_ = 0;
event_.reset();
}
inline const cl_mem_ptr& get() const {
return memory_;
}
inline cl_mem_ptr& get() {
return memory_;
}
inline size_t size() const {
return num_elements_;
}
inline cl_mem_flags access() const {
return access_;
}
mem_ref()
: num_elements_{0},
access_{CL_MEM_HOST_NO_ACCESS},
queue_{nullptr},
event_{nullptr},
memory_{nullptr} {
// nop
}
mem_ref(size_t num_elements, cl_command_queue_ptr queue,
cl_mem_ptr memory, cl_mem_flags access, cl_event_ptr event)
: num_elements_{num_elements},
access_{access},
queue_{queue},
event_{event},
memory_{memory} {
// nop
}
mem_ref(size_t num_elements, cl_command_queue_ptr queue,
cl_mem memory, cl_mem_flags access, cl_event_ptr event)
: num_elements_{num_elements},
access_{access},
queue_{queue},
event_{event},
memory_{memory} {
// nop
}
mem_ref(mem_ref&& other) = default;
mem_ref& operator=(mem_ref<T>&& other) = default;
mem_ref(const mem_ref& other) = default;
mem_ref& operator=(const mem_ref& other) = default;
~mem_ref() {
// nop
}
private:
inline void set_event(cl_event e, bool increment_reference = true) {
event_.reset(e, increment_reference);
}
inline void set_event(cl_event_ptr e) {
event_ = std::move(e);
}
inline cl_event_ptr event() {
return event_;
}
inline cl_event take_event() {
return event_.release();
}
size_t num_elements_;
cl_mem_flags access_;
cl_command_queue_ptr queue_;
cl_event_ptr event_;
cl_mem_ptr memory_;
};
} // namespace opencl
} // namespace caf
#endif // CAF_OPENCL_MEM_REF_HPP
This diff is collapsed.
......@@ -20,40 +20,50 @@
#ifndef CAF_OPENCL_PLATFORM_HPP
#define CAF_OPENCL_PLATFORM_HPP
#include <caf/opencl/device.hpp>
#include "caf/ref_counted.hpp"
#include "caf/opencl/device.hpp"
namespace caf {
namespace opencl {
class platform {
class platform;
using platform_ptr = intrusive_ptr<platform>;
class platform : public ref_counted {
public:
friend class program;
template <class T, class... Ts>
friend intrusive_ptr<T> caf::make_counted(Ts&&...);
inline const std::vector<device>& get_devices() const;
inline const std::vector<device_ptr>& get_devices() const;
inline const std::string& get_name() const;
inline const std::string& get_vendor() const;
inline const std::string& get_version() const;
static platform create(cl_platform_id platform_id, unsigned start_id);
static platform_ptr create(cl_platform_id platform_id, unsigned start_id);
private:
platform(cl_platform_id platform_id, context_ptr context,
platform(cl_platform_id platform_id, cl_context_ptr context,
std::string name, std::string vendor, std::string version,
std::vector<device> devices);
std::vector<device_ptr> devices);
~platform();
static std::string platform_info(cl_platform_id platform_id,
unsigned info_flag);
cl_platform_id platform_id_;
context_ptr context_;
cl_context_ptr context_;
std::string name_;
std::string vendor_;
std::string version_;
std::vector<device> devices_;
std::vector<device_ptr> devices_;
};
/******************************************************************************\
* implementation of inline member functions *
\******************************************************************************/
inline const std::vector<device>& platform::get_devices() const {
inline const std::vector<device_ptr>& platform::get_devices() const {
return devices_;
}
......@@ -73,4 +83,4 @@ inline const std::string& platform::get_version() const {
} // namespace opencl
} // namespace caf
#endif // CAF_OPENCL_PLTFORM_HPP
#endif // CAF_OPENCL_PLATFORM_HPP
......@@ -23,6 +23,8 @@
#include <map>
#include <memory>
#include "caf/ref_counted.hpp"
#include "caf/opencl/device.hpp"
#include "caf/opencl/global.hpp"
#include "caf/opencl/smart_ptr.hpp"
......@@ -33,21 +35,31 @@ namespace opencl {
template <class... Ts>
class actor_facade;
class program;
using program_ptr = intrusive_ptr<program>;
/// @brief A wrapper for OpenCL's cl_program.
class program {
class program : public ref_counted {
public:
friend class manager;
template <class... Ts>
friend class actor_facade;
template <bool PassConfig, class... Ts>
friend class opencl_actor;
template <class T, class... Ts>
friend intrusive_ptr<T> caf::make_counted(Ts&&...);
private:
program(context_ptr context, command_queue_ptr queue, program_ptr prog,
std::map<std::string,kernel_ptr> available_kernels);
program(cl_context_ptr context, cl_command_queue_ptr queue,
cl_program_ptr prog,
std::map<std::string,cl_kernel_ptr> available_kernels);
~program();
context_ptr context_;
program_ptr program_;
command_queue_ptr queue_;
std::map<std::string,kernel_ptr> available_kernels_;
cl_context_ptr context_;
cl_program_ptr program_;
cl_command_queue_ptr queue_;
std::map<std::string,cl_kernel_ptr> available_kernels_;
};
} // namespace opencl
......
......@@ -37,20 +37,20 @@
} /* namespace opencl */ \
} // namespace caf
CAF_OPENCL_PTR_ALIAS(mem_ptr, cl_mem, clRetainMemObject, clReleaseMemObject)
CAF_OPENCL_PTR_ALIAS(cl_mem_ptr, cl_mem, clRetainMemObject, clReleaseMemObject)
CAF_OPENCL_PTR_ALIAS(event_ptr, cl_event, clRetainEvent, clReleaseEvent)
CAF_OPENCL_PTR_ALIAS(cl_event_ptr, cl_event, clRetainEvent, clReleaseEvent)
CAF_OPENCL_PTR_ALIAS(kernel_ptr, cl_kernel, clRetainKernel, clReleaseKernel)
CAF_OPENCL_PTR_ALIAS(cl_kernel_ptr, cl_kernel, clRetainKernel, clReleaseKernel)
CAF_OPENCL_PTR_ALIAS(context_ptr, cl_context, clRetainContext, clReleaseContext)
CAF_OPENCL_PTR_ALIAS(cl_context_ptr, cl_context, clRetainContext, clReleaseContext)
CAF_OPENCL_PTR_ALIAS(program_ptr, cl_program, clRetainProgram, clReleaseProgram)
CAF_OPENCL_PTR_ALIAS(cl_program_ptr, cl_program, clRetainProgram, clReleaseProgram)
CAF_OPENCL_PTR_ALIAS(device_ptr, cl_device_id,
CAF_OPENCL_PTR_ALIAS(cl_device_ptr, cl_device_id,
clRetainDeviceDummy, clReleaseDeviceDummy)
CAF_OPENCL_PTR_ALIAS(command_queue_ptr, cl_command_queue,
CAF_OPENCL_PTR_ALIAS(cl_command_queue_ptr, cl_command_queue,
clRetainCommandQueue, clReleaseCommandQueue)
#endif // CAF_OPENCL_SMART_PTR_HPP
......@@ -45,6 +45,12 @@ public:
// nop
}
spawn_config(const spawn_config&) = default;
spawn_config(spawn_config&&) = default;
spawn_config& operator=(const spawn_config&) = default;
spawn_config& operator=(spawn_config&&) = default;
const opencl::dim_vec& dimensions() const {
return dims_;
}
......@@ -58,10 +64,9 @@ public:
}
private:
const opencl::dim_vec dims_;
const opencl::dim_vec offset_;
const opencl::dim_vec local_dims_;
opencl::dim_vec dims_;
opencl::dim_vec offset_;
opencl::dim_vec local_dims_;
};
} // namespace opencl
......
cmake_minimum_required(VERSION 2.8)
project(caf_examples_opencl CXX)
if(OpenCL_LIBRARIES)
if(OpenCL_LIBRARIES AND NOT CAF_NO_EXAMPLES)
add_custom_target(opencl_examples)
include_directories(${LIBCAF_INCLUDE_DIRS})
if(${CMAKE_SYSTEM_NAME} MATCHES "Window")
......@@ -21,4 +21,5 @@ if(OpenCL_LIBRARIES)
endmacro()
add(proper_matrix .)
add(simple_matrix .)
add(scan .)
endif()
......@@ -44,17 +44,16 @@ constexpr const char* kernel_name = "matrix_mult";
// opencl kernel, multiplies matrix1 and matrix2
// last parameter is, by convention, the output parameter
constexpr const char* kernel_source = R"__(
__kernel void matrix_mult(__global float* matrix1,
__global float* matrix2,
__global float* output) {
kernel void matrix_mult(global const float* matrix1,
global const float* matrix2,
global float* output) {
// we only use square matrices, hence: width == height
size_t size = get_global_size(0); // == get_global_size_(1);
size_t x = get_global_id(0);
size_t y = get_global_id(1);
float result = 0;
for (size_t idx = 0; idx < size; ++idx) {
for (size_t idx = 0; idx < size; ++idx)
result += matrix1[idx + y * size] * matrix2[x + idx * size];
}
output[x+y*size] = result;
}
)__";
......@@ -83,7 +82,6 @@ public:
explicit square_matrix(fvec d) : data_(move(d)) {
assert(data_.size() == num_elements);
}
inline float& operator()(size_t column, size_t row) {
......@@ -152,11 +150,9 @@ void multiplier(event_based_actor* self) {
<< to_string(m1) << endl;
auto unbox_args = [](message& msg) -> optional<message> {
return msg.apply(
[](matrix_type& lhs, matrix_type& rhs) {
return msg.apply([](matrix_type& lhs, matrix_type& rhs) {
return make_message(std::move(lhs.data()), std::move(rhs.data()));
}
);
});
};
auto box_res = [] (fvec& result) -> message {
......@@ -176,11 +172,13 @@ void multiplier(event_based_actor* self) {
// 5th arg: converts the ouptut vector back to a matrix that is then
// used as response message
// from 6 : a description of the kernel signature using in/out/in_out classes
// with the argument type packed in vectors
// with the argument type. Since the actor always expects input
// arguments for global memory to be contained in vectors,
// the vector is omitted here.
auto worker = mngr.spawn(kernel_source, kernel_name,
spawn_config{dim_vec{matrix_size, matrix_size}},
unbox_args, box_res,
in<fvec>{}, in<fvec>{}, out<fvec>{});
in<float>{}, in<float>{}, out<float>{});
// send both matrices to the actor and
// wait for results in form of a matrix_type
......
This diff is collapsed.
......@@ -41,17 +41,16 @@ constexpr const char* kernel_name = "matrix_mult";
// opencl kernel, multiplies matrix1 and matrix2
// last parameter is, by convention, the output parameter
constexpr const char* kernel_source = R"__(
__kernel void matrix_mult(__global float* matrix1,
__global float* matrix2,
__global float* output) {
kernel void matrix_mult(global const float* matrix1,
global const float* matrix2,
global float* output) {
// we only use square matrices, hence: width == height
size_t size = get_global_size(0); // == get_global_size_(1);
size_t x = get_global_id(0);
size_t y = get_global_id(1);
float result = 0;
for (size_t idx = 0; idx < size; ++idx) {
for (size_t idx = 0; idx < size; ++idx)
result += matrix1[idx + y * size] * matrix2[x + idx * size];
}
output[x+y*size] = result;
}
)__";
......@@ -92,11 +91,13 @@ void multiplier(event_based_actor* self) {
// - offsets for global dimensions (optional)
// - local dimensions (optional)
// 4th to Nth arg: the kernel signature described by in/out/in_out classes
// that contain the argument type in their template, requires vectors
// that contain the argument type in their template. Since the actor
// expects its arguments for global memory to be passed in vectors,
// the vector type is omitted for brevity.
auto worker = self->system().opencl_manager().spawn(
kernel_source, kernel_name,
spawn_config{dim_vec{matrix_size, matrix_size}},
in<fvec>{}, in<fvec>{}, out<fvec>{}
in<float>{}, in<float>{}, out<float>{}
);
// send both matrices to the actor and wait for a result
self->request(worker, chrono::seconds(5), move(m1), move(m2)).then(
......
......@@ -21,6 +21,7 @@
#include <utility>
#include "caf/logger.hpp"
#include "caf/ref_counted.hpp"
#include "caf/string_algorithms.hpp"
#include "caf/opencl/global.hpp"
......@@ -32,71 +33,77 @@ using namespace std;
namespace caf {
namespace opencl {
device device::create(const context_ptr& context, const device_ptr& device_id,
device_ptr device::create(const cl_context_ptr& context,
const cl_device_ptr& device_id,
unsigned id) {
CAF_LOG_DEBUG("creating device for opencl device with id:" << CAF_ARG(id));
// look up properties we need to create the command queue
auto supported = info<cl_ulong>(device_id, CL_DEVICE_QUEUE_PROPERTIES);
bool profiling = (supported & CL_QUEUE_PROFILING_ENABLE) != 0u;
bool profiling = false; // (supported & CL_QUEUE_PROFILING_ENABLE) != 0u;
bool out_of_order = (supported & CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE) != 0u;
unsigned properties = profiling ? CL_QUEUE_PROFILING_ENABLE : 0;
properties |= out_of_order ? CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE : 0;
// create the command queue
command_queue_ptr command_queue;
command_queue.reset(v2get(CAF_CLF(clCreateCommandQueue), context.get(),
device_id.get(), properties),
false);
cl_command_queue_ptr command_queue{v2get(CAF_CLF(clCreateCommandQueue),
context.get(), device_id.get(),
properties),
false};
// create the device
device dev{device_id, command_queue, context, id};
auto dev = make_counted<device>(device_id, std::move(command_queue), context, id);
//device dev{device_id, std::move(command_queue), context, id};
// look up device properties
dev.address_bits_ = info<cl_uint>(device_id, CL_DEVICE_ADDRESS_BITS);
dev.little_endian_ = info<cl_bool>(device_id, CL_DEVICE_ENDIAN_LITTLE);
dev.global_mem_cache_size_ =
dev->address_bits_ = info<cl_uint>(device_id, CL_DEVICE_ADDRESS_BITS);
dev->little_endian_ = info<cl_bool>(device_id, CL_DEVICE_ENDIAN_LITTLE);
dev->global_mem_cache_size_ =
info<cl_ulong>(device_id, CL_DEVICE_GLOBAL_MEM_CACHE_SIZE);
dev.global_mem_cacheline_size_ =
dev->global_mem_cacheline_size_ =
info<cl_uint>(device_id, CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE);
dev.global_mem_size_ =
dev->global_mem_size_ =
info<cl_ulong >(device_id,CL_DEVICE_GLOBAL_MEM_SIZE);
dev.host_unified_memory_ =
dev->host_unified_memory_ =
info<cl_bool>(device_id, CL_DEVICE_HOST_UNIFIED_MEMORY);
dev.local_mem_size_ =
dev->local_mem_size_ =
info<cl_ulong>(device_id, CL_DEVICE_LOCAL_MEM_SIZE);
dev.local_mem_type_ =
dev->local_mem_type_ =
info<cl_uint>(device_id, CL_DEVICE_LOCAL_MEM_TYPE);
dev.max_clock_frequency_ =
dev->max_clock_frequency_ =
info<cl_uint>(device_id, CL_DEVICE_MAX_CLOCK_FREQUENCY);
dev.max_compute_units_ =
dev->max_compute_units_ =
info<cl_uint>(device_id, CL_DEVICE_MAX_COMPUTE_UNITS);
dev.max_constant_args_ =
dev->max_constant_args_ =
info<cl_uint>(device_id, CL_DEVICE_MAX_CONSTANT_ARGS);
dev.max_constant_buffer_size_ =
dev->max_constant_buffer_size_ =
info<cl_ulong>(device_id, CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE);
dev.max_mem_alloc_size_ =
dev->max_mem_alloc_size_ =
info<cl_ulong>(device_id, CL_DEVICE_MAX_MEM_ALLOC_SIZE);
dev.max_parameter_size_ =
dev->max_parameter_size_ =
info<size_t>(device_id, CL_DEVICE_MAX_PARAMETER_SIZE);
dev.max_work_group_size_ =
dev->max_work_group_size_ =
info<size_t>(device_id, CL_DEVICE_MAX_WORK_GROUP_SIZE);
dev.max_work_item_dimensions_ =
dev->max_work_item_dimensions_ =
info<cl_uint>(device_id, CL_DEVICE_MAX_WORK_ITEM_DIMENSIONS);
dev.profiling_timer_resolution_ =
dev->profiling_timer_resolution_ =
info<size_t>(device_id, CL_DEVICE_PROFILING_TIMER_RESOLUTION);
dev.max_work_item_sizes_.resize(dev.max_work_item_dimensions_);
dev->max_work_item_sizes_.resize(dev->max_work_item_dimensions_);
clGetDeviceInfo(device_id.get(), CL_DEVICE_MAX_WORK_ITEM_SIZES,
sizeof(size_t) * dev.max_work_item_dimensions_,
dev.max_work_item_sizes_.data(), nullptr);
dev.device_type_ =
sizeof(size_t) * dev->max_work_item_dimensions_,
dev->max_work_item_sizes_.data(), nullptr);
dev->device_type_ =
device_type_from_ulong(info<cl_ulong>(device_id, CL_DEVICE_TYPE));
string extensions = info_string(device_id, CL_DEVICE_EXTENSIONS);
split(dev.extensions_, extensions, " ", false);
dev.opencl_c_version_ = info_string(device_id, CL_DEVICE_EXTENSIONS);
dev.device_vendor_ = info_string(device_id, CL_DEVICE_VENDOR);
dev.device_version_ = info_string(device_id, CL_DEVICE_VERSION);
dev.name_ = info_string(device_id, CL_DEVICE_NAME);
split(dev->extensions_, extensions, " ", false);
dev->opencl_c_version_ = info_string(device_id, CL_DEVICE_EXTENSIONS);
dev->device_vendor_ = info_string(device_id, CL_DEVICE_VENDOR);
dev->device_version_ = info_string(device_id, CL_DEVICE_VERSION);
dev->name_ = info_string(device_id, CL_DEVICE_NAME);
return dev;
}
string device::info_string(const device_ptr& device_id, unsigned info_flag) {
void device::synchronize() {
clFinish(queue_.get());
}
string device::info_string(const cl_device_ptr& device_id, unsigned info_flag) {
size_t size;
clGetDeviceInfo(device_id.get(), info_flag, 0, nullptr, &size);
vector<char> buffer(size);
......@@ -105,12 +112,18 @@ string device::info_string(const device_ptr& device_id, unsigned info_flag) {
return string(buffer.data());
}
device::device(device_ptr device_id, command_queue_ptr queue,
context_ptr context, unsigned id)
device::device(cl_device_ptr device_id, cl_command_queue_ptr queue,
cl_context_ptr context, unsigned id)
: device_id_(std::move(device_id)),
command_queue_(std::move(queue)),
queue_(std::move(queue)),
context_(std::move(context)),
id_(id) { }
id_(id) {
// nop
}
device::~device() {
// nop
}
} // namespace opencl
} // namespace caf
......@@ -18,6 +18,7 @@
******************************************************************************/
#include <string>
#include <sstream>
#include "caf/opencl/global.hpp"
......@@ -171,5 +172,122 @@ std::string get_opencl_error(cl_int err) {
}
}
std::string event_status(cl_event e) {
std::stringstream ss;
cl_int s;
auto err = clGetEventInfo(e, CL_EVENT_COMMAND_EXECUTION_STATUS,
sizeof(cl_int), &s, nullptr);
if (err != CL_SUCCESS) {
ss << std::string("ERROR ") + std::to_string(s);
return ss.str();
}
cl_command_type t;
err = clGetEventInfo(e, CL_EVENT_COMMAND_TYPE, sizeof(cl_command_type),
&t, nullptr);
if (err != CL_SUCCESS) {
ss << std::string("ERROR ") + std::to_string(s);
return ss.str();
}
switch (s) {
case(CL_QUEUED):
ss << std::string("CL_QUEUED");
break;
case(CL_SUBMITTED):
ss << std::string("CL_SUBMITTED");
break;
case(CL_RUNNING):
ss << std::string("CL_RUNNING");
break;
case(CL_COMPLETE):
ss << std::string("CL_COMPLETE");
break;
default:
ss << std::string("DEFAULT ") + std::to_string(s);
return ss.str();
}
ss << " / ";
switch (t) {
case(CL_COMMAND_NDRANGE_KERNEL):
ss << std::string("CL_COMMAND_NDRANGE_KERNEL");
break;
case(CL_COMMAND_TASK):
ss << std::string("CL_COMMAND_TASK");
break;
case(CL_COMMAND_NATIVE_KERNEL):
ss << std::string("CL_COMMAND_NATIVE_KERNEL");
break;
case(CL_COMMAND_READ_BUFFER):
ss << std::string("CL_COMMAND_READ_BUFFER");
break;
case(CL_COMMAND_WRITE_BUFFER):
ss << std::string("CL_COMMAND_WRITE_BUFFER");
break;
case(CL_COMMAND_COPY_BUFFER):
ss << std::string("CL_COMMAND_COPY_BUFFER");
break;
case(CL_COMMAND_READ_IMAGE):
ss << std::string("CL_COMMAND_READ_IMAGE");
break;
case(CL_COMMAND_WRITE_IMAGE):
ss << std::string("CL_COMMAND_WRITE_IMAGE");
break;
case(CL_COMMAND_COPY_IMAGE):
ss << std::string("CL_COMMAND_COPY_IMAGE");
break;
case(CL_COMMAND_COPY_BUFFER_TO_IMAGE):
ss << std::string("CL_COMMAND_COPY_BUFFER_TO_IMAGE");
break;
case(CL_COMMAND_COPY_IMAGE_TO_BUFFER):
ss << std::string("CL_COMMAND_COPY_IMAGE_TO_BUFFER");
break;
case(CL_COMMAND_MAP_BUFFER):
ss << std::string("CL_COMMAND_MAP_BUFFER");
break;
case(CL_COMMAND_MAP_IMAGE):
ss << std::string("CL_COMMAND_MAP_IMAGE");
break;
case(CL_COMMAND_UNMAP_MEM_OBJECT):
ss << std::string("CL_COMMAND_UNMAP_MEM_OBJECT");
break;
case(CL_COMMAND_MARKER):
ss << std::string("CL_COMMAND_MARKER");
break;
case(CL_COMMAND_ACQUIRE_GL_OBJECTS):
ss << std::string("CL_COMMAND_ACQUIRE_GL_OBJECTS");
break;
case(CL_COMMAND_RELEASE_GL_OBJECTS):
ss << std::string("CL_COMMAND_RELEASE_GL_OBJECTS");
break;
case(CL_COMMAND_READ_BUFFER_RECT):
ss << std::string("CL_COMMAND_READ_BUFFER_RECT");
break;
case(CL_COMMAND_WRITE_BUFFER_RECT):
ss << std::string("CL_COMMAND_WRITE_BUFFER_RECT");
break;
case(CL_COMMAND_COPY_BUFFER_RECT):
ss << std::string("CL_COMMAND_COPY_BUFFER_RECT");
break;
case(CL_COMMAND_USER):
ss << std::string("CL_COMMAND_USER");
break;
case(CL_COMMAND_BARRIER):
ss << std::string("CL_COMMAND_BARRIER");
break;
case(CL_COMMAND_MIGRATE_MEM_OBJECTS):
ss << std::string("CL_COMMAND_MIGRATE_MEM_OBJECTS");
break;
case(CL_COMMAND_FILL_BUFFER):
ss << std::string("CL_COMMAND_FILL_BUFFER");
break;
case(CL_COMMAND_FILL_IMAGE):
ss << std::string("CL_COMMAND_FILL_IMAGE");
break;
default:
ss << std::string("DEFAULT ") + std::to_string(s);
return ss.str();
}
return ss.str();
}
} // namespace opencl
} // namespace caf
......@@ -18,11 +18,14 @@
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#include <fstream>
#include "caf/detail/type_list.hpp"
#include "caf/opencl/device.hpp"
#include "caf/opencl/manager.hpp"
#include "caf/opencl/platform.hpp"
#include "caf/opencl/smart_ptr.hpp"
#include "caf/opencl/opencl_err.hpp"
using namespace std;
......@@ -30,15 +33,15 @@ using namespace std;
namespace caf {
namespace opencl {
const optional<const device&> manager::get_device(size_t dev_id) const {
optional<device_ptr> manager::get_device(size_t dev_id) const {
if (platforms_.empty())
return none;
size_t to = 0;
for (auto& pl : platforms_) {
auto from = to;
to += pl.get_devices().size();
to += pl->get_devices().size();
if (dev_id >= from && dev_id < to)
return pl.get_devices()[dev_id - from];
return pl->get_devices()[dev_id - from];
}
return none;
}
......@@ -56,7 +59,7 @@ void manager::init(actor_system_config&) {
for (auto& pl_id : platform_ids) {
platforms_.push_back(platform::create(pl_id, current_device_id));
current_device_id +=
static_cast<unsigned>(platforms_.back().get_devices().size());
static_cast<unsigned>(platforms_.back()->get_devices().size());
}
}
......@@ -81,7 +84,29 @@ actor_system::module* manager::make(actor_system& sys,
return new manager{sys};
}
program manager::create_program(const char* kernel_source, const char* options,
program_ptr manager::create_program_from_file(const char* path,
const char* options,
uint32_t device_id) {
std::ifstream read_source{std::string(path), std::ios::in};
string kernel_source;
if (read_source) {
read_source.seekg(0, std::ios::end);
kernel_source.resize(static_cast<size_t>(read_source.tellg()));
read_source.seekg(0, std::ios::beg);
read_source.read(&kernel_source[0],
static_cast<streamsize>(kernel_source.size()));
read_source.close();
} else {
ostringstream oss;
oss << "No file at '" << path << "' found.";
CAF_LOG_ERROR(CAF_ARG(oss.str()));
throw runtime_error(oss.str());
}
return create_program(kernel_source.c_str(), options, device_id);
}
program_ptr manager::create_program(const char* kernel_source,
const char* options,
uint32_t device_id) {
auto dev = get_device(device_id);
if (!dev) {
......@@ -93,61 +118,83 @@ program manager::create_program(const char* kernel_source, const char* options,
return create_program(kernel_source, options, *dev);
}
program manager::create_program(const char* kernel_source, const char* options,
const device& dev) {
program_ptr manager::create_program_from_file(const char* path,
const char* options,
const device_ptr dev) {
std::ifstream read_source{std::string(path), std::ios::in};
string kernel_source;
if (read_source) {
read_source.seekg(0, std::ios::end);
kernel_source.resize(static_cast<size_t>(read_source.tellg()));
read_source.seekg(0, std::ios::beg);
read_source.read(&kernel_source[0],
static_cast<streamsize>(kernel_source.size()));
read_source.close();
} else {
ostringstream oss;
oss << "No file at '" << path << "' found.";
CAF_LOG_ERROR(CAF_ARG(oss.str()));
throw runtime_error(oss.str());
}
return create_program(kernel_source.c_str(), options, dev);
}
program_ptr manager::create_program(const char* kernel_source,
const char* options,
const device_ptr dev) {
// create program object from kernel source
size_t kernel_source_length = strlen(kernel_source);
program_ptr pptr;
auto rawptr = v2get(CAF_CLF(clCreateProgramWithSource), dev.context_.get(),
cl_uint{1}, &kernel_source, &kernel_source_length);
pptr.reset(rawptr, false);
cl_program_ptr pptr;
pptr.reset(v2get(CAF_CLF(clCreateProgramWithSource), dev->context_.get(),
1u, &kernel_source, &kernel_source_length),
false);
// build programm from program object
auto dev_tmp = dev.device_id_.get();
cl_int err = clBuildProgram(pptr.get(), 1, &dev_tmp,
options, nullptr, nullptr);
auto dev_tmp = dev->device_id_.get();
auto err = clBuildProgram(pptr.get(), 1, &dev_tmp, options, nullptr, nullptr);
if (err != CL_SUCCESS) {
ostringstream oss;
oss << "clBuildProgram: " << get_opencl_error(err);
// the build log will be printed by the pfn_notify (see opencl/manger.cpp)
#ifndef __APPLE__
// seems that just apple implemented the
// pfn_notify callback, but we can get
// the build log
if (err == CL_BUILD_PROGRAM_FAILURE) {
size_t buildlog_buffer_size = 0;
// get the log length
clGetProgramBuildInfo(pptr.get(), dev_tmp, CL_PROGRAM_BUILD_LOG,
sizeof(buildlog_buffer_size), nullptr,
&buildlog_buffer_size);
0, nullptr, &buildlog_buffer_size);
vector<char> buffer(buildlog_buffer_size);
// fill the buffer with buildlog informations
clGetProgramBuildInfo(pptr.get(), dev_tmp, CL_PROGRAM_BUILD_LOG,
sizeof(buffer[0]) * buildlog_buffer_size,
sizeof(char) * buildlog_buffer_size,
buffer.data(), nullptr);
ostringstream ss;
ss << "Build log:\n" << string(buffer.data())
<< "\n#######################################";
ss << "############## Build log ##############"
<< endl << string(buffer.data()) << endl
<< "#######################################";
// seems that just apple implemented the
// pfn_notify callback, but we can get
// the build log
#ifndef __APPLE__
CAF_LOG_ERROR(CAF_ARG(ss.str()));
}
#endif
oss << endl << ss.str();
}
throw runtime_error(oss.str());
}
map<string, kernel_ptr> available_kernels;
cl_uint number_of_kernels = 0;
clCreateKernelsInProgram(pptr.get(), 0, nullptr, &number_of_kernels);
clCreateKernelsInProgram(pptr.get(), 0u, nullptr, &number_of_kernels);
map<string, cl_kernel_ptr> available_kernels;
if (number_of_kernels > 0) {
vector<cl_kernel> kernels(number_of_kernels);
err = clCreateKernelsInProgram(pptr.get(), number_of_kernels, kernels.data(),
nullptr);
err = clCreateKernelsInProgram(pptr.get(), number_of_kernels,
kernels.data(), nullptr);
if (err != CL_SUCCESS) {
ostringstream oss;
oss << "clCreateKernelsInProgram: " << get_opencl_error(err);
throw runtime_error(oss.str());
} else {
}
for (cl_uint i = 0; i < number_of_kernels; ++i) {
size_t ret_size;
clGetKernelInfo(kernels[i], CL_KERNEL_FUNCTION_NAME, 0, nullptr, &ret_size);
vector<char> name(ret_size);
err = clGetKernelInfo(kernels[i], CL_KERNEL_FUNCTION_NAME, ret_size,
size_t len;
clGetKernelInfo(kernels[i], CL_KERNEL_FUNCTION_NAME, 0, nullptr, &len);
vector<char> name(len);
err = clGetKernelInfo(kernels[i], CL_KERNEL_FUNCTION_NAME, len,
reinterpret_cast<void*>(name.data()), nullptr);
if (err != CL_SUCCESS) {
ostringstream oss;
......@@ -155,15 +202,20 @@ program manager::create_program(const char* kernel_source, const char* options,
<< get_opencl_error(err);
throw runtime_error(oss.str());
}
kernel_ptr kernel;
cl_kernel_ptr kernel;
kernel.reset(move(kernels[i]));
available_kernels.emplace(string(name.data()), move(kernel));
}
} else {
CAF_LOG_WARNING("Could not built all kernels in program. Since this happens"
" on some platforms, we'll ignore this and try to build"
" each kernel individually by name.");
}
return {dev.context_, dev.command_queue_, pptr, move(available_kernels)};
return make_counted<program>(dev->context_, dev->queue_, pptr,
move(available_kernels));
}
manager::manager(actor_system& sys) : system_(sys){
manager::manager(actor_system& sys) : system_(sys) {
// nop
}
......
......@@ -29,7 +29,7 @@ using namespace std;
namespace caf {
namespace opencl {
platform platform::create(cl_platform_id platform_id,
platform_ptr platform::create(cl_platform_id platform_id,
unsigned start_id) {
vector<unsigned> device_types = {CL_DEVICE_TYPE_GPU,
CL_DEVICE_TYPE_ACCELERATOR,
......@@ -48,16 +48,16 @@ platform platform::create(cl_platform_id platform_id,
v2callcl(CAF_CLF(clGetDeviceIDs), platform_id, device_type,
discoverd, (ids.data() + known));
}
vector<device_ptr> devices;
vector<cl_device_ptr> devices;
devices.resize(ids.size());
auto lift = [](cl_device_id ptr) { return device_ptr{ptr, false}; };
auto lift = [](cl_device_id ptr) { return cl_device_ptr{ptr, false}; };
transform(ids.begin(), ids.end(), devices.begin(), lift);
context_ptr context;
cl_context_ptr context;
context.reset(v2get(CAF_CLF(clCreateContext), nullptr,
static_cast<unsigned>(ids.size()),
ids.data(), pfn_notify, nullptr),
false);
vector<device> device_information;
vector<device_ptr> device_information;
for (auto& device_id : devices) {
device_information.push_back(device::create(context, device_id,
start_id++));
......@@ -70,9 +70,9 @@ platform platform::create(cl_platform_id platform_id,
auto name = platform_info(platform_id, CL_PLATFORM_NAME);
auto vendor = platform_info(platform_id, CL_PLATFORM_VENDOR);
auto version = platform_info(platform_id, CL_PLATFORM_VERSION);
return platform{platform_id, move(context), move(name),
return make_counted<platform>(platform_id, move(context), move(name),
move(vendor), move(version),
move(device_information)};
move(device_information));
}
string platform::platform_info(cl_platform_id platform_id,
......@@ -87,15 +87,21 @@ string platform::platform_info(cl_platform_id platform_id,
return string(buffer.data());
}
platform::platform(cl_platform_id platform_id, context_ptr context,
platform::platform(cl_platform_id platform_id, cl_context_ptr context,
string name, string vendor, string version,
vector<device> devices)
vector<device_ptr> devices)
: platform_id_(platform_id),
context_(std::move(context)),
name_(move(name)),
vendor_(move(vendor)),
version_(move(version)),
devices_(move(devices)) { }
devices_(move(devices)) {
// nop
}
platform::~platform() {
// nop
}
} // namespace opencl
} // namespace caf
......@@ -32,8 +32,9 @@ using namespace std;
namespace caf {
namespace opencl {
program::program(context_ptr context, command_queue_ptr queue, program_ptr prog,
map<string, kernel_ptr> available_kernels)
program::program(cl_context_ptr context, cl_command_queue_ptr queue,
cl_program_ptr prog,
map<string, cl_kernel_ptr> available_kernels)
: context_(move(context)),
program_(move(prog)),
queue_(move(queue)),
......@@ -41,5 +42,9 @@ program::program(context_ptr context, command_queue_ptr queue, program_ptr prog,
// nop
}
program::~program() {
// nop
}
} // namespace opencl
} // namespace caf
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment