My Useful Code Segment

本文记录我的一些很有用的代码片段

压缩和保存模型参数

对于模型参数的保存,我们首先要对参数进行序列化,然后对与序列化获得数据我们再进行压缩保存。

  1. 进行序列化使用的header-only的库是cereal
  2. 压缩所使用的header-only的库为Gzip 示例代码如下:
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
#include <gzip/compress.hpp>
#include <gzip/decompress.hpp>
#include <cereal/archives/binary.hpp>
#include <fstream>
#include <sstream>
#include "Include/Math/MathBase.h"
#include <iostream>
#include <string>

using namespace std;
using cereal::binary_data;
using gzip::compress;
using gzip::decompress;
class MyModel{
public:
  MyModel() {
    weights_ = VectorD::Random(10);
    bias_ = 0;
  }
  void SaveModel() {
    ofstream output("Temp/MyModel.model", ios::out | ios::binary);
    stringstream ss;
    cereal::BinaryOutputArchive oarchive(ss); // Create an output archive
    oarchive(*this);
    string data = ss.str();
    int size = data.size();
    string compressed_data =  compress(data.data(), size);
    output << compressed_data;
    output.close();
    cout << "save: " << weights_ << endl;
    cout << bias_ << endl;
  }
  void LoadModel() {
    ifstream input("Temp/MyModel.model", ios::in | ios::binary);
    string compressed_data = "";
    char c;
    while (input.get(c))
      compressed_data += c;
    string decompressed_data = decompress(compressed_data.data(), compressed_data.size());
    stringstream ss(decompressed_data);
    cereal::BinaryInputArchive iarchive(ss); // Create an input archive
    iarchive(*this); // Read the data from the archive
    cout << "loading: " << endl;
    cout << weights_ << endl;
    cout << bias_ << endl;
    input.close();
  }

private:
  friend class cereal::access;
  template<class Archive>
  void save(Archive & archive) const
  {
    int len = weights_.size();
    archive(len);
    archive(binary_data(weights_.data(), weights_.size() * sizeof(double)));
    archive(bias_);
  }

  template<class Archive>
  void load(Archive & archive)
  {
    int len;
    archive(len);
    weights_.resize(len);
    archive(binary_data(weights_.data(), weights_.size() * sizeof(double)));
    archive(bias_);
  }

  VectorD weights_;
  double bias_;
  string name;
};

int main()
{
  MyModel model;
  model.SaveModel();
  MyModel load_model;
  load_model.LoadModel();
  return 0;
}

使用cereal序列化Eigen库的matrix

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#include <cereal/cereal.hpp>
#include <cereal/archives/binary.hpp>
#include <eigen/Eigen/Dense>
#include <fstream>

using Eigen::Dynamic;
using Eigen::RowMajor;
typedef Eigen::Matrix<double, Dynamic, Dynamic> MatrixD;
typedef Eigen::Matrix<int, Dynamic, Dynamic> MatrixI;
typedef Eigen::Matrix<double, 1, Dynamic> VectorD;
typedef Eigen::Matrix<int, 1, Dynamic> VectorI;
using Eigen::MatrixBase;
using namespace Eigen;
namespace cereal
{
  template <class Archive, class _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols> inline
    typename std::enable_if<traits::is_output_serializable<BinaryData<_Scalar>, Archive>::value, void>::type
    save(Archive & ar, Eigen::Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> const & m)
    {
      int32_t rows = m.rows();
      int32_t cols = m.cols();
      ar(rows);
      ar(cols);
      ar(binary_data(m.data(), rows * cols * sizeof(_Scalar)));
    }

  template <class Archive, class _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols> inline
    typename std::enable_if<traits::is_input_serializable<BinaryData<_Scalar>, Archive>::value, void>::type
    load(Archive & ar, Eigen::Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> & m)
    {
      int32_t rows;
      int32_t cols;
      ar(rows);
      ar(cols);

      m.resize(rows, cols);

      ar(binary_data(m.data(), static_cast<std::size_t>(rows * cols * sizeof(_Scalar))));
    }
}

int main() {
  // Eigen::MatrixXd test = Eigen::MatrixXd::Random(10, 3);
  MatrixD test = MatrixD::Random(5, 5);

  {
    std::ofstream out("eigen.cereal", std::ios::binary);
    cereal::BinaryOutputArchive archive_o(out);
    archive_o(test);
  }

  std::cout << "test:" << std::endl << test << std::endl;

  MatrixD test_loaded;

  {
    std::ifstream in("eigen.cereal", std::ios::binary);
    cereal::BinaryInputArchive archive_i(in);
    archive_i(test_loaded);
  }

  std::cout << "test loaded:" << std::endl << test_loaded << std::endl;
}

不定参数

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#include <iostream>
using namespace   std;
class print
{
public:
  template<typename FIRST, typename ...PACK>
  static void cppprintf(FIRST first, PACK... params)
  {
    std::cout << first;
    cppprintf(params...);
  }

  //重载函数版本,递归结束条件
  template<typename T>
  static  void cppprintf(T end)
  {
    std::cout << end << std::endl;
  }
};
int main()
{
  print::cppprintf("sdf", 123, 'c', 456, 123.123);
  return 0;
}

updatedupdated2021-11-062021-11-06