1. add int8/int16 feature

commit.sh

@@ -1,4 +1,4 @@
-BRANCH=BREAKDOWN
+BRANCH=INT8
 git init
 git checkout -b $BRANCH
 git add .

commit_info

@@ -1 +1 @@
-1. add the function of exporting algorithm breakdown
+1. add int8/int16 feature

include/CPPAlgos/INT8CPPAlgo.h

@@ -20,8 +20,42 @@ namespace AMMBench {
 /**
  * @class INT8CPPAlgo CPPAlgos/INT8CPPAlgo.h
  * @brief The INT8 MM class of c++ algos
+ * @warning This function disables all additional optimization by libtorch, as it has different, and not fair SIMD/cache optimization
+ * over FP32/INT16/INT8 on cpu, which is hard to compare
+ * @note additionally parameters
+ * - fpMode, String, default FP32, can also use INT8 or INT16
  */
 class INT8CPPAlgo : public AMMBench::AbstractCPPAlgo {
+ protected:
+  /**
+  * @brief the inline amm under nested loop fp32
+  * @param A the A matrix
+  * @param B the B matrix
+  * @return the output c matrix
+  */
+  torch::Tensor fp32amm(torch::Tensor A, torch::Tensor B);
+  /**
+  * @brief the inline amm under nested loop int8
+  * @param A the A matrix
+  * @param B the B matrix
+  * @return the output c matrix
+  */
+  torch::Tensor int8amm(torch::Tensor A, torch::Tensor B);
+  /**
+  * @brief the inline amm under nested loop int4
+  * @param A the A matrix
+  * @param B the B matrix
+  * @return the output c matrix
+  */
+  torch::Tensor int4amm(torch::Tensor A, torch::Tensor B);
+  /**
+ * @brief the inline amm under nested loop int16
+ * @param A the A matrix
+ * @param B the B matrix
+ * @return the output c matrix
+ */
+  torch::Tensor int16amm(torch::Tensor A, torch::Tensor B);
+  std::string fpMode="FP32";
  public:
   INT8CPPAlgo() {
 
@@ -37,7 +71,10 @@ class INT8CPPAlgo : public AMMBench::AbstractCPPAlgo {
    * @return the output c matrix
    */
   virtual torch::Tensor amm(torch::Tensor A, torch::Tensor B, uint64_t sketchSize);
-
+  /**
+ * @brief set the alo-specfic config related to one algorithm
+ */
+  virtual void setConfig(INTELLI::ConfigMapPtr cfg);
 };
 /**
  * @ingroup AMMBENCH_CppAlgos

src/CPPAlgos/INT8CPPAlgo.cpp

@@ -3,43 +3,296 @@
 //
 
 #include <CPPAlgos/INT8CPPAlgo.h>
-torch::Tensor AMMBench::INT8CPPAlgo::amm(torch::Tensor A, torch::Tensor B, uint64_t sketchSize) {
-  std::cout << sketchSize;
-  float scalingA, scalingB;
-  // int16_t mulab;
-
-  /*auto AINT8=(A*scalingA).to(torch::kInt8);
-  auto BTINT8=(Bt*scalingB).to(torch::kInt8);*/
-  // return C;
-  /*
-  for(int64_t i=0;i<rows;i++)
-  {
-    for(int64_t j=0;j<cols;j++)
-    {
-       ta=AINT8[i];
-       tb=BTINT8[j];
+#include <Utils/UtilityFunctions.h>
+torch::Tensor AMMBench::INT8CPPAlgo::fp32amm(torch::Tensor tensor1, torch::Tensor tensor2)
+{
+  auto A_size = tensor1.sizes();
+  auto B_size =tensor2.sizes();
+  struct timeval tstart;
+  //INTELLI_INFO("I am mm");
+  gettimeofday(&tstart,NULL);
+  int64_t rows1 = A_size[0];
+  int64_t cols1 = A_size[1];
+  int64_t cols2 = B_size[1];
+  /**
+   * @brief build A into std vector
+   */
+  std::vector<float> matrix1(tensor1.data_ptr<float>(), tensor1.data_ptr<float>() + rows1 * cols1);
+  buildATime=INTELLI::UtilityFunctions::timeLastUs(tstart);
+  std::vector<float> matrix2(tensor2.data_ptr<float>(), tensor2.data_ptr<float>() + cols1 * cols2);
+  buildBTime=INTELLI::UtilityFunctions::timeLastUs(tstart)-buildATime;
+  // Create the output matrix
+  std::vector<float> result(rows1 * cols2, 0.0);
+  for (int64_t i = 0; i < rows1; ++i) {
+    for (int64_t j = 0; j < cols2; ++j) {
+      for (int64_t k = 0; k < cols1; ++k) {
+        result[i * cols2 + j] += matrix1[i * cols1 + k] * matrix2[k * cols2 + j];
+      }
+    }
+  }
+ // exit(-1);
+  fABTime = INTELLI::UtilityFunctions::timeLastUs(tstart)-buildATime-buildBTime;
+  torch::Tensor resultTensor = torch::from_blob(result.data(), {rows1,cols2});
+  //torch::Tensor resultTensor = torch::zeros({rows1,cols2});
+  postProcessTime=INTELLI::UtilityFunctions::timeLastUs(tstart)-buildATime-buildBTime-fABTime;
+
+  return resultTensor.clone();
+}
+torch::Tensor AMMBench::INT8CPPAlgo::int4amm(torch::Tensor tensor1, torch::Tensor tensor2)
+{
+  auto A_size = tensor1.sizes();
+  auto B_size =tensor2.sizes();
+  struct timeval tstart;
+  //INTELLI_INFO("I am mm");
+  gettimeofday(&tstart,NULL);
+  int64_t rows1 = A_size[0];
+  int64_t cols1 = A_size[1];
+  int64_t cols2 = B_size[1];
+ /**
+  * @brief build A
+  */
+  std::vector<float> matrix1Float(tensor1.data_ptr<float>(), tensor1.data_ptr<float>() + rows1 * cols1);
+  // Convert the input matrices to int8
+  std::vector<int8_t> matrix1(rows1 * cols1);
+  float scale1 = 7.0 / *std::max_element(matrix1Float.begin(), matrix1Float.end());
+  for (int i = 0; i < rows1 * cols1; ++i) {
+    matrix1[i] = static_cast<int8_t>(matrix1Float[i] * scale1);
+  }
+  buildATime=INTELLI::UtilityFunctions::timeLastUs(tstart);
+
+  /**
+   * @brief build B
+   */
+  std::vector<float> matrix2Float(tensor2.data_ptr<float>(), tensor2.data_ptr<float>() + cols1 * cols2);
+  std::vector<int8_t> matrix2(cols1 * cols2);
+  float scale2 = 7.0 / *std::max_element(matrix2Float.begin(), matrix2Float.end());
+  for (int i = 0; i < cols1 * cols2; ++i) {
+    matrix2[i] = static_cast<int8_t>(matrix2Float[i] * scale2);
+  }
+  buildBTime=INTELLI::UtilityFunctions::timeLastUs(tstart)-buildATime;
+  /**
+   * @brief run fAB
+   */
+  std::vector<int16_t> result(rows1 * cols2, 0);
+
+  // Perform matrix multiplication using nested loops
+  for (int64_t i = 0; i < rows1; ++i) {
+    for (int64_t j = 0; j < cols2; ++j) {
+      int16_t sRu=0;
+      int64_t k=0;
+      /**
+       * @brief 32/4=8, so we simulate a 8-way SHARED-NOTHING speed up in one loop
+       */
+      while ( k < cols1-8) {
+        int8_t tru1=matrix1[i * cols1 + k]*matrix2[k * cols2 + j];
+        int8_t tru2=matrix1[i * cols1 + k+1]*matrix2[(k+1) * cols2 + j];
+        int8_t tru3=matrix1[i * cols1 + (k+2)]*matrix2[(k+2) * cols2 + j];
+        int8_t tru4=matrix1[i * cols1 + (k+3)]*matrix2[(k+3) * cols2 + j];
+        //
+        int8_t tru5=matrix1[i * cols1 + k+4]*matrix2[(k+4) * cols2 + j];
+        int8_t tru6=matrix1[i * cols1 + k+5]*matrix2[(k+5) * cols2 + j];
+        int8_t tru7=matrix1[i * cols1 + (k+6)]*matrix2[(k+6) * cols2 + j];
+        int8_t tru8=matrix1[i * cols1 + (k+7)]*matrix2[(k+7) * cols2 + j];
+        sRu+= tru1+tru2+tru3+tru4+tru5+tru6+tru7+tru8;
+        k+=8;
+      }
+      for(int64_t k2=k;k2<cols1;++k2)
+      {
+        int16_t tru=matrix1[i * cols1 + k2]*matrix2[k2 * cols2 + j];
+        sRu+=tru;
+      }
+      result[i * cols2 + j]=sRu;
+    }
+  }
+  fABTime = INTELLI::UtilityFunctions::timeLastUs(tstart)-buildATime-buildBTime;
+
+  // Scale the result back to int8
+  /**
+ * @brief post process
+ */
+  std::vector<float> resultFP32(rows1 * cols2);
+  float scaleResult = 1.0 / (scale1 * scale2);
+  for (int i = 0; i < rows1 * cols2; ++i) {
+    resultFP32[i] = static_cast<float>(result[i] * scaleResult);
+  }
+  torch::Tensor resultTensor = torch::from_blob(resultFP32.data(), {rows1,cols2});
+  //torch::Tensor resultTensor = torch::zeros({rows1,cols2});
+  postProcessTime=INTELLI::UtilityFunctions::timeLastUs(tstart)-buildATime-buildBTime-fABTime;
+
+  return resultTensor.clone();
+}
+
+torch::Tensor AMMBench::INT8CPPAlgo::int8amm(torch::Tensor tensor1, torch::Tensor tensor2)
+{
+  auto A_size = tensor1.sizes();
+  auto B_size =tensor2.sizes();
+  struct timeval tstart;
+  //INTELLI_INFO("I am mm");
+  gettimeofday(&tstart,NULL);
+  int64_t rows1 = A_size[0];
+  int64_t cols1 = A_size[1];
+  int64_t cols2 = B_size[1];
+  /**
+   * @brief build A
+   */
+  std::vector<float> matrix1Float(tensor1.data_ptr<float>(), tensor1.data_ptr<float>() + rows1 * cols1);
+  // Convert the input matrices to int8
+  std::vector<int8_t> matrix1(rows1 * cols1);
+  float scale1 = 127.0 / *std::max_element(matrix1Float.begin(), matrix1Float.end());
+  for (int i = 0; i < rows1 * cols1; ++i) {
+    matrix1[i] = static_cast<int8_t>(matrix1Float[i] * scale1);
+  }
+  buildATime=INTELLI::UtilityFunctions::timeLastUs(tstart);
 
-      int32_t ru=0;
-      float ruf=0.0;
+  /**
+   * @brief build B
+   */
+  std::vector<float> matrix2Float(tensor2.data_ptr<float>(), tensor2.data_ptr<float>() + cols1 * cols2);
+  std::vector<int8_t> matrix2(cols1 * cols2);
+  float scale2 = 127.0 / *std::max_element(matrix2Float.begin(), matrix2Float.end());
+  for (int i = 0; i < cols1 * cols2; ++i) {
+    matrix2[i] = static_cast<int8_t>(matrix2Float[i] * scale2);
+  }
+  buildBTime=INTELLI::UtilityFunctions::timeLastUs(tstart)-buildATime;
+  /**
+   * @brief run fAB
+   */
+  std::vector<int32_t> result(rows1 * cols2, 0);
 
-      for(int64_t k=0;k<sumS;k++)
+  // Perform matrix multiplication using nested loops
+  for (int64_t i = 0; i < rows1; ++i) {
+    for (int64_t j = 0; j < cols2; ++j) {
+      int32_t sRu=0;
+      int64_t k=0;
+      /**
+       * @brief 32/8=4, so we simulate a 4-way SHARED-NOTHING speed up in one loop
+       */
+      while ( k < cols1-4) {
+        int16_t tru1=matrix1[i * cols1 + k]*matrix2[k * cols2 + j];
+        int16_t tru2=matrix1[i * cols1 + k+1]*matrix2[(k+1) * cols2 + j];
+        int16_t tru3=matrix1[i * cols1 + (k+2)]*matrix2[(k+2) * cols2 + j];
+        int16_t tru4=matrix1[i * cols1 + (k+3)]*matrix2[(k+3) * cols2 + j];
+        sRu+= tru1+tru2+tru3+tru4;
+        k+=4;
+      }
+      for(int64_t k2=k;k2<cols1;++k2)
       {
+        int16_t tru=matrix1[i * cols1 + k2]*matrix2[k2 * cols2 + j];
+        sRu+=tru;
+      }
+      result[i * cols2 + j]=sRu;
+    }
+  }
+  fABTime = INTELLI::UtilityFunctions::timeLastUs(tstart)-buildATime-buildBTime;
+
+  // Scale the result back to int8
+  /**
+ * @brief post process
+ */
+  std::vector<float> resultFP32(rows1 * cols2);
+  float scaleResult = 1.0 / (scale1 * scale2);
+  for (int i = 0; i < rows1 * cols2; ++i) {
+    resultFP32[i] = static_cast<float>(result[i] * scaleResult);
+  }
+  torch::Tensor resultTensor = torch::from_blob(resultFP32.data(), {rows1,cols2});
+  //torch::Tensor resultTensor = torch::zeros({rows1,cols2});
+  postProcessTime=INTELLI::UtilityFunctions::timeLastUs(tstart)-buildATime-buildBTime-fABTime;
 
-       int8_t tak=ta[k].item<int8_t>();
-        int8_t tbk=tb[k].item<int8_t>();
-         mulab=tak*tbk;
-        ru+=mulab;
+  return resultTensor.clone();
+}
+torch::Tensor AMMBench::INT8CPPAlgo::int16amm(torch::Tensor tensor1, torch::Tensor tensor2)
+{
+  auto A_size = tensor1.sizes();
+  auto B_size =tensor2.sizes();
+  struct timeval tstart;
+  //INTELLI_INFO("I am mm");
+  gettimeofday(&tstart,NULL);
+  int64_t rows1 = A_size[0];
+  int64_t cols1 = A_size[1];
+  int64_t cols2 = B_size[1];
+  /**
+   * @brief build A
+   */
+  std::vector<float> matrix1Float(tensor1.data_ptr<float>(), tensor1.data_ptr<float>() + rows1 * cols1);
+  // Convert the input matrices to int8
+  std::vector<int16_t> matrix1(rows1 * cols1);
+  float scale1 = 32767.0 / *std::max_element(matrix1Float.begin(), matrix1Float.end());
+  for (int i = 0; i < rows1 * cols1; ++i) {
+    matrix1[i] = static_cast<int16_t>(matrix1Float[i] * scale1);
+  }
+  buildATime=INTELLI::UtilityFunctions::timeLastUs(tstart);
+
+  /**
+   * @brief build B
+   */
+  std::vector<float> matrix2Float(tensor2.data_ptr<float>(), tensor2.data_ptr<float>() + cols1 * cols2);
+  std::vector<int16_t> matrix2(cols1 * cols2);
+  float scale2 = 32767.0/ *std::max_element(matrix2Float.begin(), matrix2Float.end());
+  for (int i = 0; i < cols1 * cols2; ++i) {
+    matrix2[i] = static_cast<int16_t>(matrix2Float[i] * scale2);
+  }
+  buildBTime=INTELLI::UtilityFunctions::timeLastUs(tstart)-buildATime;
+  /**
+   * @brief run fAB
+   */
+  std::vector<int64_t> result(rows1 * cols2, 0);
+
+  // Perform matrix multiplication using nested loops
+  for (int64_t i = 0; i < rows1; ++i) {
+    for (int64_t j = 0; j < cols2; ++j) {
+      int64_t sRu=0;
+      int64_t k=0;
+      /**
+       * @brief 32/16=2, so we simulate a 2-way SHARED-NOTHING speed up in one loop
+       */
+      while ( k < cols1-2) {
+        int32_t tru1=matrix1[i * cols1 + k]*matrix2[k * cols2 + j];
+        int32_t tru2=matrix1[i * cols1 + k+1]*matrix2[(k+1) * cols2 + j];
+        sRu+= tru1+tru2;
+        k+=2;
+      }
+      for(int64_t k2=k;k2<cols1;++k2)
+      {
+        int32_t tru=matrix1[i * cols1 + k2]*matrix2[k2 * cols2 + j];
+        sRu+=tru;
       }
-      ruf=ru;
-      C[i][j]=ruf;
+      result[i * cols2 + j]=sRu;
     }
-  }*/
-  scalingA = torch::abs(A).max().item<float>() / 127.0;
-  scalingB = torch::abs(B).max().item<float>() / 127.0;
-  auto ta = (A / scalingA).to(torch::kInt8);
-  auto tb = (B / scalingB).to(torch::kInt8);
-  torch::matmul(ta, tb);
-  return torch::zeros({A.size(0), B.size(1)});
-  return torch::matmul(ta, tb).to(torch::kFloat) * scalingA * scalingB;
-  //return C*scalingA*scalingB/127.0/127.0;
+  }
+  fABTime = INTELLI::UtilityFunctions::timeLastUs(tstart)-buildATime-buildBTime;
+
+  // Scale the result back to int8
+  /**
+ * @brief post process
+ */
+  std::vector<float> resultFP32(rows1 * cols2);
+  float scaleResult = 1.0 / (scale1 * scale2);
+  for (int i = 0; i < rows1 * cols2; ++i) {
+    resultFP32[i] = static_cast<float>(result[i] * scaleResult);
+  }
+  torch::Tensor resultTensor = torch::from_blob(resultFP32.data(), {rows1,cols2});
+  //torch::Tensor resultTensor = torch::zeros({rows1,cols2});
+  postProcessTime=INTELLI::UtilityFunctions::timeLastUs(tstart)-buildATime-buildBTime-fABTime;
+
+  return resultTensor.clone();
+}
+void AMMBench::INT8CPPAlgo::setConfig(INTELLI::ConfigMapPtr cfg) {
+  AbstractCPPAlgo::setConfig(cfg);
+  fpMode=cfg->tryString("fpMode","INT8",true);
+}
+torch::Tensor AMMBench::INT8CPPAlgo::amm(torch::Tensor A, torch::Tensor B, uint64_t sketchSize) {
+  assert(sketchSize);
+  if(fpMode=="INT4")
+  {
+    return int4amm(A,B);
+  }
+  else if(fpMode=="INT8")
+  {
+    return int8amm(A,B);
+  }
+  else if(fpMode=="INT16")
+  {
+    return int16amm(A,B);
+  }
+  return fp32amm(A,B);
 }