CnC
 All Classes Namespaces Functions Variables Typedefs Enumerator Friends Groups Pages
serializer.h
1 //********************************************************************************
2 // Copyright (c) 2007-2013 Intel Corporation. All Rights Reserved. **
3 // **
4 // The source code contained or described herein and all documents related to **
5 // the source code ("Material") are owned by Intel Corporation or its suppliers **
6 // or licensors. Title to the Material remains with Intel Corporation or its **
7 // suppliers and licensors. The Material contains trade secrets and proprietary **
8 // and confidential information of Intel or its suppliers and licensors. The **
9 // Material is protected by worldwide copyright and trade secret laws and **
10 // treaty provisions. No part of the Material may be used, copied, reproduced, **
11 // modified, published, uploaded, posted, transmitted, distributed, or **
12 // disclosed in any way without Intel's prior express written permission. **
13 // **
14 // No license under any patent, copyright, trade secret or other intellectual **
15 // property right is granted to or conferred upon you by disclosure or delivery **
16 // of the Materials, either expressly, by implication, inducement, estoppel or **
17 // otherwise. Any license under such intellectual property rights must be **
18 // express and approved by Intel in writing. **
19 //********************************************************************************
20 
21 
22 #ifndef _CNC_SERIALIZER_H_
23 #define _CNC_SERIALIZER_H_
24 
25 #include <cnc/internal/cnc_api.h>
26 #include <cnc/internal/scalable_object.h>
27 #include <complex>
28 #include <vector>
29 
30 #include <cnc/internal/cnc_stddef.h> // size_type
31 
32 #include <memory> // std::allocator
33 
34 namespace CnC
35 {
36  class CNC_API serializer;
37  class no_alloc{};
38 
39  //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
40 
41  /// \defgroup serialization Serialization
42  /// @{
43  ///
44  /// There are three possible ways of setting up the serialization
45  /// of your struct resp. class:
46  ///
47  /// \defgroup ser_bit Serialization of simple structs/classes without pointers or virtual functions.
48  /// @{
49  /// If your struct does not contain pointers (nor virtual
50  /// functions), instances of it may be serialized by simply
51  /// copying them bytewise. In this case, it suffices to
52  /// add your struct to the category bitwise_serializable which is done
53  /// by providing the following global function:
54  /// \code
55  /// inline bitwise_serializable serializer_category( const Your_Class * ) {
56  /// return bitwise_serializable();
57  /// }
58  /// \endcode
59  /// As a short-cut, you can just use #CNC_BITWISE_SERIALIZABLE( T ) to declare your class as bitwise serializable:
60  /// \code
61  /// CNC_BITWISE_SERIALIZABLE( my_class );
62  /// \endcode
63  /// @}
64  /// \defgroup ser_exp Non bitwise serializable objects
65  /// @{
66  /// You'll have to provide a serializer function for your class.
67  /// One way of doing this is to derive your class from
68  /// serializable and to provide a method "serialize":
69  /// \code
70  /// class Your_class : public serializable {
71  /// ...
72  /// public:
73  /// void serialize( CnC::serializer & );
74  /// ...
75  /// };
76  /// \endcode
77  /// In the implementation of this serialize method you should
78  /// use the operator& of the serializer class. Here is an example:
79  /// \code
80  /// class Your_class : public serializable {
81  /// private:
82  /// double x;
83  /// float y;
84  /// int len;
85  /// char* arr; // array of lenth len
86  /// ...
87  /// public:
88  /// void serialize( CnC::serializer & buf ) {
89  /// buf & x
90  /// & y
91  /// & len;
92  /// buf & CnC::chunk< char >( arr, len_arr ); // chunk declared below
93  /// ...
94  /// }
95  /// }
96  /// \endcode
97  /// This method will called for both packing and unpacking
98  /// (what will actually be done, depends on the internal mode
99  /// of the serializer.)
100  /// \n
101  /// Alternatively, you can provide a global function \n
102  /// \code
103  /// void serialize( serializer& buf, Your_class& obj );
104  /// \endcode
105  /// which should use the operator& of the serializer class
106  /// (same as for method 2. above). This is useful in cases
107  /// where you want to leave your class unchanged.
108  /// @}
109  /// \defgroup ser_ptr Marshalling pointer types (e.g. items which are pointers)
110  /// @{
111  /// A common optimization used by CnC programs targeting the
112  /// shared-memory runtime is to store pointers to large objects
113  /// inside of item collections instead of copies of the objects.
114  /// This is done to minimize the overhead of copying into and out
115  /// of the item collection. Programs written in this style can be
116  /// sued with distCnC, but special care needs to be taken.
117  /// \n
118  /// Distributed CnC requires that data stored in item collections be
119  /// serializable in order to communicate data between different
120  /// processes. Consequently, the programmer must modify such
121  /// pointer-using CnC programs in two ways in order to use the
122  /// distributed-memory runtime. First, the programmer must provide a
123  /// specialization of CnC::serialize for the pointer type pointing to T:
124  /// \code
125  /// void CnC::serialize( CnC::serializer & ser, T *& ptr ) {
126  /// ser & CnC::chunk< Your_class[, allocator] >( ptr, 1 );
127  /// }
128  /// \endcode
129  /// Please refer to CnC::chunk to learn about appropriate memory
130  /// management.
131  /// \n
132  /// If the pointer represents a single object and not a C-style array,
133  /// then the programmer may the use convenience macro:
134  /// \code CNC_POINTER_SERIALIZABLE( T ); \endcode
135  /// This macro implements the above method for the case when the
136  /// pointer refers to only a single object (length == 1). Note that
137  /// if T is a type that contains template parameters, the programmer
138  /// should create a typedef of the specific type being serialized and
139  /// use this type as the argument for the macro.
140  /// \n
141  /// Next, programmers must ensure that type T itself implements a
142  /// default constructor and is serializable by itself. The latter
143  /// as achieved as described above.
144  /// It is the same method that programmers must provide if storing
145  /// copies of objects instead of pointers to objects.
146  /// \n
147  /// Reading from and writing to item collections is not different in
148  /// this style of writing CnC programs. However, there are a few
149  /// things to note:
150  /// \n
151  /// When an object is communicated from one process to another, an
152  /// item must be allocated on the receiving end. By default, CnC::chunk
153  /// uses the default allocator for this and so requires the default constructor
154  /// of T. After object creation, any fields are updated by the provided
155  /// serialize methods. Thios default behavior simplifies the object
156  /// allocation/deallocation: most of it is done automatically behind the scenes.
157  /// The programmer should just pass an unitialized pointer to
158  /// get(). The pointer will then point to the allocated object/array after
159  /// the get completes.
160  /// \n
161  /// In advanced and potentially dangerous setups, the programmer might wish
162  /// to store the object into memory that has been previously allocated.
163  /// This can be done by an explicit copy and explicit object lifetime control.
164  /// Please note that such a scenario can easily lead to uses of CnC which
165  /// do not comply with CnC's methodology, e.g. no side effects in steps and
166  /// dynamic single assigments.
167  ///
168  /// \defgroup seralloc Serialization of std::shared_ptr
169  /// @{
170  /// std::shared_ptr are normally supported out-of-the-box (given the underlying type
171  /// is serializable (similar to \ref ser_ptr).
172  ///
173  /// Only shared_ptrs to more complex pointer-types require special treatment
174  /// to guarantee correct garbage collection. If
175  /// - using std::shared:ptr
176  /// - AND the pointer-class holds data which is dynamically allocated
177  /// - AND this dynamic memory is NOT allocated in the default constructor but during marshalling with CnC::chunk
178  ///
179  /// then (and only then) you might want to use construct_array and destruct_array.
180  /// It is probably most convenient to
181  /// - use the standard CnC::chunk mechanism during serialization
182  /// - allocate dynamic memory in the constructors with construct_array
183  /// - and deallocate it in the destructor using destruct_array
184  ///
185  /// This mechanism is used in the cholesky example which comes with this distribution.
186  ///
187  /// Alternatively you can use matching
188  /// allocation/deallocation explicitly in your con/destructors and during serialization.
189  /// @}
190  /// @}
191 
192  class serializable {
193  public:
194  // must provide member function
195  // void serialize( CnC::serializer& buf ) {
196  // /* use serializer::operator& for accessing buf */
197  // }
198  };
199 
200  template< class T > inline
201  void serialize( serializer & buf, T & obj ) {
202  obj.serialize( buf );
203  }
204 
205 
206  /// Specifies serialization category: explicit serialization via a "serialize" function
208 
209  /// General case: a type belongs to the category explicitly_serializable,
210  /// unless there is a specialization of the function template
211  /// "serializer_category" (see below).
212  template< class T > inline
214  {
215  return explicitly_serializable();
216  }
217 
218  /// simple structs/classes are bitwise serializable.
219 
220  /// Specifies serialization category: byte-wise copy
222 
223  /// \def CNC_BITWISE_SERIALIZABLE( T )
224  /// Convenience macro for defining that a type should be treated
225  /// as bitwise serializable:
226 # define CNC_BITWISE_SERIALIZABLE( T ) \
227  inline CnC::bitwise_serializable serializer_category( const T * ) { \
228  return CnC::bitwise_serializable(); \
229  }
230 
231  //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
232 
233  template< class A, class B > inline
234  void serialize( CnC::serializer & buf, std::pair< A, B > & obj ) {
235  buf & obj.first & obj.second;
236  }
237 
238  template< class T >
239  inline CnC::bitwise_serializable serializer_category( const std::complex< T > * ) {
240  return CnC::bitwise_serializable();
241  }
242 
243  //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
244 
245  /// \class chunk
246  /// \brief Serialization of arrays with and without automatic memory handling.
247  ///
248  /// Specified (to serializer::operator&) via
249  /// \code
250  /// CnC::chunk< type[, allocator] >( your_ptr, len )
251  /// \endcode
252  /// where your_ptr is a pointer variable to your array of <type>s.
253  ///
254  /// If the allocator is the special type CnC::no_alloc, the runtime assumes that
255  /// the programer appropriately allocates (if ser.is_unpacking())
256  /// and deallocates (if ser:is_cleaning_up()) the array. This implies that
257  /// the using serialize method/function needs to make sure it passes valid and correct
258  /// pointers to chunk.
259  ///
260  /// If not CnC::no_alloc , the Allocator class must meet "allocator" requirements of ISO C++ Standard, Section 20.1.5
261  ///
262  /// When unpacking, your_ptr must be NULL, and it is allocated automatically using
263  /// the given allocator and when cleaning up, your_ptr is deallocated accordingly.
264  ///
265  /// If using an allocator (in particular hre default std::allocator) object/array
266  /// allocation and deallocation is greatly simplified for the programmer.
267  /// There is no need to allocate and/or deallocate objects/arrays manually.
268  template< class T, class Allocator = std::allocator< T > >
269  struct chunk
270  {
271  chunk( T *& arr, size_type len ) : m_arrVar( arr ), m_len( len ) {}
272  template< template< class I > class Range, class J >
273  chunk( T *& arr, const Range< J > & range )
274  : m_arrVar( arr ), m_len( range.size() )
275  {
276  CNC_ASSERT( range.begin() == 0 );
277  }
278 
279  T *& m_arrVar;
280  size_type m_len;
281  };
282 
283  //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
284 
285  /// Convenience macro for declaring serializable pointers.
286  /// The object being pointed to should be serializable.
287 
288 # define CNC_POINTER_SERIALIZABLE( T ) \
289  namespace CnC { \
290  void serialize( serializer & ser, T *& t ) { \
291  ser & chunk< T >(t, 1); \
292  } \
293  }
294 
295  //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
296  //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
297 
298  /// \defgroup serspec Automatic serialization of built-in types.
299  /// They can be copied byte-wise.
300  ///
301  /// In order to add your own class to the category bitwise_serializable,
302  /// add a specialization of "serializer_category" returning bitwise_serializable for your class.
303  /// (e.g. by using CnC::CNC_BITWISE_SERIALIZABLE).
304  /// @{
305  CNC_BITWISE_SERIALIZABLE( bool ); ///< bool is bitwise serializable
306  CNC_BITWISE_SERIALIZABLE( char ); ///< char is bitwise serializable
307  CNC_BITWISE_SERIALIZABLE( signed char ); ///< signed char is bitwise serializable
308  CNC_BITWISE_SERIALIZABLE( unsigned char ); ///< unsigend char is bitwise serializable
309  CNC_BITWISE_SERIALIZABLE( short ); ///< short is bitwise serializable
310  CNC_BITWISE_SERIALIZABLE( unsigned short ); ///< unsigend short is bitwise serializable
311  CNC_BITWISE_SERIALIZABLE( int ); ///< int is bitwise serializable
312  CNC_BITWISE_SERIALIZABLE( unsigned int ); ///< unsigned int is bitwise serializable
313  CNC_BITWISE_SERIALIZABLE( long ); ///< long is bitwise serializable
314  CNC_BITWISE_SERIALIZABLE( unsigned long ); ///< unsigned long is bitwise serializable
315  CNC_BITWISE_SERIALIZABLE( long long ); ///< long long is bitwise serializable
316  CNC_BITWISE_SERIALIZABLE( unsigned long long ); ///< unsigned long long is bitwise serializable
317  CNC_BITWISE_SERIALIZABLE( float ); ///< float is bitwise serializable
318  CNC_BITWISE_SERIALIZABLE( double ); ///< double is bitwise serializable
319  CNC_BITWISE_SERIALIZABLE( long double ); ///< long double is bitwise serializable
320  /// @}
321 
322  //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
323  //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
324 
325  namespace Internal {
326  class Buffer;
327  class BufferAccess;
328  }
329 
330  /// \brief Handles serilialization of data-objects.
331  ///
332  /// objects of this class are passed to serialization methods/functions.
333  /// Try to use operator & only; using anything else is potentially dangerous.
334  class CNC_API serializer : public Internal::scalable_object
335  {
336  public:
337  enum Mode {
338  MODE_PACKED_SIZE,
339  MODE_PACK,
340  MODE_UNPACK,
341  MODE_CLEANUP
342  };
343 
344  inline serializer( bool addCRC = false, bool addSize = false );
345  inline ~serializer();
346 
347  /// (Re)allocates a buffer of the given size.
348  inline void resize( size_type len );
349 
350  /// Swap internal representations of the given two serializers.
351  friend void swap( serializer & s1, serializer & s2 );
352 
353  inline void set_mode_packed_size();
354  inline void set_mode_pack();
355  inline void set_mode_unpack();
356  inline void set_mode_cleanup();
357  inline void set_mode_pack( bool addCRC, bool addSize );
358  inline void set_mode_unpack( bool addCRC, bool addSize );
359  bool is_packing() const { return m_mode == MODE_PACK; }
360  bool is_unpacking() const { return m_mode == MODE_UNPACK; }
361  bool is_cleaning_up() const { return m_mode == MODE_CLEANUP; }
362  size_type get_size() const { return m_packedSize; }
363  /// @return size of header
364  inline size_type get_header_size() const;
365  /// @return the number of bytes already packed into the buffer
366  inline size_type get_body_size() const;
367  /// @return total number of bytes in buffer (header + data)
368  inline size_type get_total_size() const;
369  /// @return pointer to message header
370  inline void * get_header() const;
371  /// @return pointer to the message body (without header)
372  inline void * get_body() const;
373  /// @return body size from header, -1 if something goes wrong
374  inline size_type unpack_header() const;
375 
376  /// Top-level packing interface, to be used in the "serialize"
377  /// function of your classes.
378  /// Applies the serializer to one object resp. an array of objects
379  /// (e.g. packing them into the serializer or unpacking them from
380  /// the serializer).
381  /// Dispatches w.r.t. the packing category of the object's type
382  /// (i.e. byte-wise copying or object serialization).
383  template< class T > serializer & operator&( T & var );
384  template< class T > serializer & operator&( const T & var );
385  //template< class T > serializer & operator&( Internal::array_no_alloc_type< T > a ); //< used via array_no_alloc( yourArr, len )
386  template< class T, class Allocator >
387  serializer & operator&( chunk< T, Allocator > a ); //< used via auto_array( yourArrVar, len )
388 
389  class reserved;
390  /// reserve current position in buffer to be filled later with a call to "complete"
391  /// rserve/complete supported for bitwise-serializable types only
392  template< class T > reserved reserve( const T & _obj );
393 
394  /// fill in data at position that has been reserved before
395  /// rserve/complete supported for bitwise-serializable types only
396  template< class T > void complete( const reserved & r, const T & _obj );
397 
398  // Low-level packing interface:
399  template< class T > inline size_type packed_size( const T * arr, size_type len ) const {
400  return packed_size( arr, len, serializer_category( arr ) );
401  }
402  template< class T > inline void pack( const T * arr, size_type len ) {
403  pack( arr, len, serializer_category( arr ) );
404  }
405  template< class T > inline void unpack( T * arr, size_type len ) {
406  unpack( arr, len, serializer_category( arr ) );
407  }
408  template< class T > inline void cleanup( T * arr, size_type len ) {
409  cleanup( arr, len, serializer_category( arr ) );
410  }
411  template< class T > inline size_type packed_size( const T & var ) const { return packed_size( &var, 1 ); }
412  template< class T > inline void pack( const T & var ) { return pack( &var, 1 ); }
413  template< class T > inline void unpack( T & var ) { return unpack( &var, 1 ); }
414  template< class T > inline void cleanup( T & var ) { return cleanup( &var, 1 ); }
415 
416  inline serializer( const serializer& );
417 
418  /// Allocates an array of type T and size num in pointer variable arrVar
419  template< class T, class Allocator = std::allocator< T > >
421  {
422  construct_array( T *& arrVar, size_type num );
423  };
424  /// destructs the array of type T and isze num at arrVar and resets arrVar to NULL.
425  template< class T, class Allocator = std::allocator< T > >
427  {
428  destruct_array( T *& arrVar, size_type num );
429  };
430  /// @}
431  template< class T >
432  struct construct_array< T, no_alloc >
433  {
434  construct_array( T *&, size_type ){}
435  };
436  template< class T >
437  struct destruct_array< T, no_alloc >
438  {
439  destruct_array( T *&, size_type ){}
440  };
441 
442  private:
443  template< class T > inline size_type packed_size( const T * arr, size_type len, bitwise_serializable ) const;
444  template< class T > inline void pack( const T * arr, size_type len, bitwise_serializable );
445  template< class T > inline void unpack( T * arr, size_type len, bitwise_serializable );
446  template< class T > inline void cleanup( T * arr, size_type len, bitwise_serializable );
447  template< class T > inline size_type packed_size( const T * arr, size_type len, explicitly_serializable ) const;
448  template< class T > inline void pack( const T * arr, size_type len, explicitly_serializable );
449  template< class T > inline void unpack( T * arr, size_type len, explicitly_serializable );
450  template< class T > inline void cleanup( T * arr, size_type len, explicitly_serializable );
451 
452 
453  /// Don't allow copying
454  void operator=( const serializer& );
455 
456  /// misc:
457  size_type remaining_capacity() const;
458 
459  Internal::Buffer * m_buf;
460  size_type m_packedSize; /// < for MODE_PACKED_SIZE only
461  Mode m_mode;
462 
463  friend class Internal::BufferAccess;
464  };
465 
466  /// @}
467 
468 } // namespace CnC
469 
470 #include <cnc/internal/dist/Serializer.impl.h>
471 
472 #endif // _CNC_SERIALIZER_H_