inplace task now uses small buffer optimization for bigger captures

lib/include/srslte/common/task.h

@@ -28,9 +28,9 @@
 
 namespace srslte {
 
-constexpr size_t default_buffer_size = 256;
+constexpr size_t default_buffer_size = 32;
 
-template <class Signature, size_t Capacity = default_buffer_size, size_t Alignment = alignof(std::max_align_t)>
+template <class Signature, size_t Capacity = default_buffer_size>
 class inplace_task;
 
 namespace task_details {
@@ -39,30 +39,42 @@ template <typename R, typename... Args>
 struct oper_table_t {
   using call_oper_t = R (*)(void* src, Args&&... args);
   using move_oper_t = void (*)(void* src, void* dest);
-  //  using copy_oper_t = void (*)(void* src, void* dest);
   using dtor_oper_t = void (*)(void* src);
 
-  static oper_table_t* get_empty() noexcept
+  const static oper_table_t* get_empty() noexcept
   {
-    static oper_table_t t;
+    const static oper_table_t t{true,
-    t.call = [](void* src, Args&&... args) -> R { throw std::bad_function_call(); };
+                                [](void* src, Args&&... args) -> R { throw std::bad_function_call(); },
-    t.move = [](void*, void*) {};
+                                [](void*, void*) {},
-    //    t.copy = [](void*, void*) {};
+                                [](void*) {}};
-    t.dtor = [](void*) {};
     return &t;
   }
 
   template <typename Func>
-  static oper_table_t* get() noexcept
+  const static oper_table_t* get_small() noexcept
   {
-    static oper_table_t t{};
+    const static oper_table_t t{
-    t.call = [](void* src, Args&&... args) -> R { return (*static_cast<Func*>(src))(std::forward<Args>(args)...); };
+        true,
-    t.move = [](void* src, void* dest) -> void {
+        [](void* src, Args&&... args) -> R { return (*static_cast<Func*>(src))(std::forward<Args>(args)...); },
-      ::new (dest) Func{std::move(*static_cast<Func*>(src))};
+        [](void* src, void* dest) -> void {
-      static_cast<Func*>(src)->~Func();
+          ::new (dest) Func{std::move(*static_cast<Func*>(src))};
-    };
+          static_cast<Func*>(src)->~Func();
-    //    t.copy = [](void* src, void* dest) -> void { ::new (dest) Func{*static_cast<Func*>(src)}; };
+        },
-    t.dtor = [](void* src) -> void { static_cast<Func*>(src)->~Func(); };
+        [](void* src) -> void { static_cast<Func*>(src)->~Func(); }};
+    return &t;
+  }
+
+  template <typename Func>
+  const static oper_table_t* get_big() noexcept
+  {
+    const static oper_table_t t{
+        false,
+        [](void* src, Args&&... args) -> R { return (*static_cast<Func*>(src))(std::forward<Args>(args)...); },
+        [](void* src, void* dest) -> void {
+          *static_cast<Func**>(dest) = *static_cast<Func**>(src);
+          *static_cast<Func**>(src)  = nullptr;
+        },
+        [](void* src) -> void { static_cast<Func*>(src)->~Func(); }};
     return &t;
   }
 
@@ -72,101 +84,123 @@ struct oper_table_t {
   oper_table_t& operator=(oper_table_t&&) = delete;
   ~oper_table_t()                         = default;
 
+  bool        is_in_buffer;
   call_oper_t call;
   move_oper_t move;
-  //  copy_oper_t copy;
   dtor_oper_t dtor;
 
-  static oper_table_t<R, Args...>* empty_oper;
-
 private:
   oper_table_t() = default;
+  oper_table_t(bool is_in_buffer_, call_oper_t call_, move_oper_t move_, dtor_oper_t dtor_) :
+    is_in_buffer(is_in_buffer_),
+    call(call_),
+    move(move_),
+    dtor(dtor_)
+  {}
 };
 
 template <class>
 struct is_inplace_task : std::false_type {};
-template <class Sig, size_t Cap, size_t Align>
+template <class Sig, size_t Capacity>
-struct is_inplace_task<inplace_task<Sig, Cap, Align> > : std::true_type {};
+struct is_inplace_task<inplace_task<Sig, Capacity> > : std::true_type {};
-
-template <typename R, typename... Args>
-oper_table_t<R, Args...>* oper_table_t<R, Args...>::empty_oper = oper_table_t<R, Args...>::get_empty();
 
 } // namespace task_details
 
-template <class R, class... Args, size_t Capacity, size_t Alignment>
+template <class R, class... Args, size_t Capacity>
-class inplace_task<R(Args...), Capacity, Alignment>
+class inplace_task<R(Args...), Capacity>
 {
-  using storage_t    = typename std::aligned_storage<Capacity, Alignment>::type;
+  static constexpr size_t capacity = Capacity >= sizeof(void*) ? Capacity : sizeof(void*);
-  using oper_table_t = task_details::oper_table_t<R, Args...>;
+  using storage_t                  = typename std::aligned_storage<capacity, alignof(std::max_align_t)>::type;
+  using oper_table_t               = task_details::oper_table_t<R, Args...>;
 
 public:
-  inplace_task() noexcept { oper_ptr = oper_table_t::empty_oper; }
+  inplace_task() noexcept { oper_ptr = oper_table_t::get_empty(); }
 
   template <typename T,
             typename FunT = typename std::decay<T>::type,
+            typename      = typename std::enable_if<sizeof(FunT) <= capacity>::type,
             typename      = typename std::enable_if<not task_details::is_inplace_task<FunT>::value>::type>
   inplace_task(T&& function)
   {
-    static_assert(sizeof(FunT) <= sizeof(buffer), "inplace_task cannot store object with given size.\n");
+    oper_ptr = oper_table_t::template get_small<FunT>();
-    static_assert(Alignment % alignof(FunT) == 0, "inplace_task cannot store object with given alignment.\n");
-
     ::new (&buffer) FunT{std::forward<T>(function)};
-    oper_ptr = oper_table_t::template get<T>();
+  }
+
+  template <typename T,
+            typename FunT = typename std::decay<T>::type,
+            typename      = typename std::enable_if<not task_details::is_inplace_task<FunT>::value and
+                                               (sizeof(FunT) > capacity)>::type>
+  inplace_task(T&& function)
+  {
+    oper_ptr = oper_table_t::template get_big<FunT>();
+    ptr      = static_cast<void*>(new FunT{std::forward<T>(function)});
   }
 
   inplace_task(inplace_task&& other) noexcept
   {
     oper_ptr       = other.oper_ptr;
-    other.oper_ptr = oper_table_t::empty_oper;
+    other.oper_ptr = oper_table_t::get_empty();
-    oper_ptr->move(&other.buffer, &buffer);
+    if (oper_ptr->is_in_buffer) {
+      oper_ptr->move(&other.buffer, &buffer);
+    } else {
+      oper_ptr->move(&other.ptr, &ptr);
+    }
   }
 
-  //  inplace_task(const inplace_task& other) noexcept
+  ~inplace_task() { oper_ptr->dtor(get_buffer()); }
-  //  {
-  //    oper_ptr = other.oper_ptr;
-  //    oper_ptr->copy(&other.buffer, &buffer);
-  //  }
-
-  ~inplace_task() { oper_ptr->dtor(&buffer); }
 
   inplace_task& operator=(inplace_task&& other) noexcept
   {
-    oper_ptr->dtor(&buffer);
+    oper_ptr->dtor(get_buffer());
     oper_ptr       = other.oper_ptr;
-    other.oper_ptr = oper_table_t::empty_oper;
+    other.oper_ptr = oper_table_t::get_empty();
-    oper_ptr->move(&other.buffer, &buffer);
+    if (oper_ptr->is_in_buffer) {
+      oper_ptr->move(&other.buffer, &buffer);
+    } else {
+      oper_ptr->move(&other.ptr, &ptr);
+    }
     return *this;
   }
 
-  //  inplace_task& operator=(const inplace_task& other) noexcept
+  R operator()(Args&&... args) { return oper_ptr->call(get_buffer(), std::forward<Args>(args)...); }
-  //  {
-  //    if (this != &other) {
-  //      oper_ptr->dtor(&buffer);
-  //      oper_ptr = other.oper_ptr;
-  //      oper_ptr->copy(&other.buffer, &buffer);
-  //    }
-  //    return *this;
-  //  }
 
-  R operator()(Args&&... args) { return oper_ptr->call(&buffer, std::forward<Args>(args)...); }
+  bool is_empty() const { return oper_ptr == oper_table_t::get_empty(); }
-
+  bool is_in_small_buffer() const { return oper_ptr->is_in_buffer; }
-  bool is_empty() const { return oper_ptr == oper_table_t::empty_oper; }
 
   void swap(inplace_task& other) noexcept
   {
     if (this == &other)
       return;
 
-    storage_t tmp;
+    if (oper_ptr->is_in_buffer and other.oper_ptr->is_in_buffer) {
-    oper_ptr->move(&buffer, &tmp);
+      storage_t tmp;
-    other.oper_ptr->move(&other.buffer, &buffer);
+      oper_ptr->move(&buffer, &tmp);
-    oper_ptr->move(&tmp, &other.buffer);
+      other.oper_ptr->move(&other.buffer, &buffer);
+      oper_ptr->move(&tmp, &other.buffer);
+    } else if (oper_ptr->is_in_buffer and not other.oper_ptr->is_in_buffer) {
+      void* tmpptr = other.ptr;
+      oper_ptr->move(&buffer, &other.buffer);
+      ptr = tmpptr;
+    } else if (not oper_ptr->is_in_buffer and other.oper_ptr->is_in_buffer) {
+      void* tmpptr = ptr;
+      other.oper_ptr->move(&other.buffer, &buffer);
+      oper_ptr->move(&tmpptr, &other.ptr);
+    } else {
+      std::swap(ptr, other.ptr);
+    }
     std::swap(oper_ptr, other.oper_ptr);
   }
 
+  friend void swap(inplace_task& lhs, inplace_task& rhs) noexcept { lhs.swap(rhs); }
+
 private:
-  storage_t     buffer;
+  union {
-  oper_table_t* oper_ptr;
+    storage_t buffer;
+    void*     ptr;
+  };
+  const oper_table_t* oper_ptr;
+
+  void* get_buffer() { return oper_ptr->is_in_buffer ? &buffer : ptr; }
 };
 
 } // namespace srslte

lib/test/common/queue_test.cc

@@ -324,6 +324,10 @@ int test_task_thread_pool3()
 struct C {
   std::unique_ptr<int> val{new int{5}};
 };
+struct D {
+  std::array<int, 64> big_val;
+  D() { big_val[0] = 6; }
+};
 
 int test_inplace_task()
 {
@@ -350,6 +354,36 @@ int test_inplace_task()
     TESTASSERT(v == 5);
   }
 
+  D                            d;
+  srslte::inplace_task<void()> t6 = [&v, d]() { v = d.big_val[0]; };
+  {
+    srslte::inplace_task<void()> t7;
+    t6();
+    TESTASSERT(v == 6);
+    v  = 0;
+    t7 = std::move(t6);
+    t7();
+    TESTASSERT(v == 6);
+  }
+
+  auto l1 = std::bind([&v](C& c) { v = *c.val; }, C{});
+  auto l2 = [&v, d]() { v = d.big_val[0]; };
+  t       = std::move(l1);
+  t2      = l2;
+  v       = 0;
+  t();
+  TESTASSERT(v == 5);
+  t2();
+  TESTASSERT(v == 6);
+  TESTASSERT(t.is_in_small_buffer() and not t2.is_in_small_buffer());
+  swap(t, t2);
+  TESTASSERT(t2.is_in_small_buffer() and not t.is_in_small_buffer());
+  v = 0;
+  t();
+  TESTASSERT(v == 6);
+  t2();
+  TESTASSERT(v == 5);
+
   std::cout << "outcome: Success\n";
   std::cout << "========================================\n";
   return 0;