1. Update the example scripts in scripts/scanARowCPP and scripts/sca…

README.md

@@ -216,4 +216,6 @@ You will find the figures then.
 3. Clion may fail to render and highlight the torch apis. In this case, kindly type a random line of "555"
    to validate the highlight when you need it, and comment it during a compile. :)
 4. When setting up torch for cpu under python version > 3.10, torch == 1.13.0 would conflict with torchaudio according
-   to https://pytorch.org/audio/stable/installation.html. Use Python version <= 3.10 for smooth installation.
+   to https://pytorch.org/audio/stable/installation.html. Use Python version <= 3.10 for smooth installation.
+5. Some heavy-weight algos like co-occurring FD may be treated as a zombie thread by some OS like ubuntu if running in default benchmark program, in this case
+, please force them to be executed as a running child thread by setting forceMP(U64)=1 and threads(U64)=1 in config

benchmark/scripts/scanAColCPP/config_CPPCOFD.csv

@@ -0,0 +1,10 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/CRS.pt,String
+useCPP,1,U64
+cppAlgoTag,cooFD,String
+threads,1,U64
+forceMP,1,U64

benchmark/scripts/scanAColCPP/config_CPPCOUNTERSKETCH.csv

@@ -0,0 +1,10 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/CRS.pt,String
+useCPP,1,U64
+cppAlgoTag,countSketch,String
+threads,1,U64
+forceMP,1,U64

benchmark/scripts/scanAColCPP/config_CPPCRS.csv

@@ -5,4 +5,6 @@ bCol,500,U64
 sketchDimension,25,U64
 ptFile,torchscripts/CRS.pt,String
 useCPP,1,U64
-cppAlgoTag,crs,String
+cppAlgoTag,crs,String
+threads,1,U64
+forceMP,1,U64

benchmark/scripts/scanAColCPP/config_CPPEWS.csv

@@ -0,0 +1,10 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/RAWMM.pt,String
+useCPP,1,U64
+cppAlgoTag,ews,String
+threads,1,U64
+forceMP,1,U64

benchmark/scripts/scanAColCPP/config_CPPMM.csv

@@ -5,4 +5,6 @@ bCol,500,U64
 sketchDimension,25,U64
 ptFile,torchscripts/RAWMM.pt,String
 useCPP,1,U64
-cppAlgoTag,mm,String
+cppAlgoTag,mm,String
+threads,1,U64
+forceMP,1,U64

benchmark/scripts/scanAColCPP/config_CPPTOW.csv

@@ -0,0 +1,10 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/RAWMM.pt,String
+useCPP,1,U64
+cppAlgoTag,tugOfWar,String
+threads,1,U64
+forceMP,1,U64

benchmark/scripts/scanAColCPP/drawTogether.py

@@ -79,7 +79,9 @@ def readResultSingle(singleValue, resultPath):
     elapsedTime = readConfig(resultFname, "perfElapsedTime")
     cacheMiss = readConfig(resultFname, "cacheMiss")
     cacheRefs = readConfig(resultFname, "cacheRefs")
-    return elapsedTime, cacheMiss, cacheRefs
+    froError = readConfig(resultFname, "froError")
+    errorBoundRatio = readConfig(resultFname, "errorBoundRatio")
+    return elapsedTime, cacheMiss, cacheRefs, froError, errorBoundRatio
 
 
 def cleanPath(path):
@@ -91,33 +93,42 @@ def readResultVector(singleValueVec, resultPath):
     elapseTimeVec = []
     cacheMissVec = []
     cacheRefVec = []
+    froErrorVec = []
+    errorBoundRatioVec = []
     for i in singleValueVec:
-        elapsedTime, cacheMiss, cacheRefs = readResultSingle(i, resultPath)
+        elapsedTime, cacheMiss, cacheRefs, froError, errorBoundRatio = readResultSingle(i, resultPath)
         elapseTimeVec.append(float(elapsedTime) / 1000.0)
         cacheMissVec.append(float(cacheMiss))
         cacheRefVec.append(float(cacheRefs))
-    return np.array(elapseTimeVec), np.array(cacheMissVec), np.array(cacheRefVec)
+        froErrorVec.append(float(froError))
+        errorBoundRatioVec.append(float(errorBoundRatio))
+    return np.array(elapseTimeVec), np.array(cacheMissVec), np.array(cacheRefVec), np.array(froErrorVec), np.array(
+        errorBoundRatioVec)
 
 
 def compareMethod(exeSpace, commonPathBase, resultPaths, csvTemplates, periodVec, reRun=1):
     elapsedTimeAll = []
     cacheMissAll = []
     cacheRefAll = []
     periodAll = []
+    froAll = []
+    errorBoundRatioAll = []
     for i in range(len(csvTemplates)):
         resultPath = commonPathBase + resultPaths[i]
         if (reRun == 1):
             os.system("sudo rm -rf " + resultPath)
             os.system("sudo mkdir " + resultPath)
             runScanVector(exeSpace, periodVec, resultPath, csvTemplates[i])
-        elapsedTime, cacheMiss, cacheRef = readResultVector(periodVec, resultPath)
+        elapsedTime, cacheMiss, cacheRef, fro, eb = readResultVector(periodVec, resultPath)
         elapsedTimeAll.append(elapsedTime)
         cacheMissAll.append(cacheMiss)
         cacheRefAll.append(cacheRef)
         periodAll.append(periodVec)
         cacheMissRateAll = np.array(cacheMissAll) / np.array(cacheRefAll) * 100.0
+        froAll.append(fro)
+        errorBoundRatioAll.append(eb)
         # periodAll.append(periodVec)
-    return np.array(elapsedTimeAll), cacheMissRateAll, periodAll
+    return np.array(elapsedTimeAll), cacheMissRateAll, periodAll, np.array(froAll), np.array(errorBoundRatioAll)
 
 
 def main():
@@ -141,21 +152,36 @@ def main():
         reRun = 1
 
     # sampling
-    resultPaths = ["crs-cpp", "mm-cpp"]
-    csvTemplates = ["config_CPPCRS.csv", "config_CPPMM.csv"]
-    methodTags = ["crs-cpp", "mm-cpp"]
-    elapsedTimeAll, cacheMissAll, periodAll = compareMethod(exeSpace, commonBase, resultPaths, csvTemplates, valueVec,
+   # sampling
+    # resultPaths = ["crs", "bcrs", "ews", "mm","mm-cpp","crs-cpp"]
+    resultPaths = ["mm", "crs","ews","counterSketch","cofd","tow"]
+    csvTemplates = ["config_CPPMM.csv", "config_CPPCRS.csv","config_CPPEWS.csv","config_CPPCOUNTERSKETCH.csv","config_CPPCOFD.csv","config_CPPTOW.csv"]
+    methodTags = ["mm", "crs","ews","counterSketch","cofd","tow"]
+    # csvTemplates = ["config_CRS.csv", "config_BerCRS.csv", "config_EWS.csv", "config_RAWMM.csv","config_CPPMM.csv","config_CPPCRS.csv"]
+    # methodTags = ["CRS", "Ber-CRS", "EWS",, "MM","MM_CPP","CRS_CPP"]
+    elapsedTimeAll, cacheMissAll, periodAll,fro,eb = compareMethod(exeSpace, commonBase, resultPaths, csvTemplates, valueVec,
                                                             reRun)
     groupLine.DrawFigure(periodAll, elapsedTimeAll,
                          methodTags,
-                         "# A's col", "elapsed time (ms)", 0, 1,
-                         figPath + "/" + scanTag + "sampling_elapsedTime",
+                         "#A's col", "elapsed time (ms)", 0, 1,
+                         figPath + "/" + scanTag + "cpp_elapsedTime",
                          True)
     groupLine.DrawFigureYnormal(periodAll, cacheMissAll,
                                 methodTags,
-                                "# A's col", "cacheMiss (%)", 0, 1,
-                                figPath + "/" + scanTag + "sampling_cacheMiss",
+                                "#A's col", "cacheMiss (%)", 0, 1,
+                                figPath + "/" + scanTag + "cpp_cacheMiss",
                                 True)
+    groupLine.DrawFigureYnormal(periodAll,
+                                 fro * 100.0,
+                                 methodTags,
+                                 "#A's col", "normalized error %", 0, 1, figPath + "/" +scanTag + "_froError",
+                                 True)
+    groupLine.DrawFigureYnormal(periodAll,
+                                 eb * 100.0,
+                                 methodTags,
+                                 "#A's col", "error bound ratio %", 0, 1, figPath + "/" + scanTag + "_ebRatio",
+                                 True)
+    # draw2yLine("watermark time (ms)",singleValueVecDisp,lat95Vec,errVec,"95% Latency (ms)","Error","ms","",figPa
     # draw2yLine("watermark time (ms)",singleValueVecDisp,lat95Vec,errVec,"95% Latency (ms)","Error","ms","",figPath+"wm_lat")
     # draw2yLine("watermark time (ms)",singleValueVecDisp,thrVec,errVec,"Throughput (KTp/s)","Error","KTp/s","",figPath+"wm_thr")
     # draw2yLine("watermark time (ms)",singleValueVecDisp,lat95Vec,compVec,"95% Latency (ms)","Completeness","ms","",figPath+"wm_omp")

benchmark/scripts/scanAColCPP/groupLine.py

@@ -208,6 +208,7 @@ def DrawFigureXYnormal(xvalues, yvalues, legend_labels, x_label, y_label, y_min,
     # plt.show()
     plt.savefig(filename + ".pdf", bbox_inches='tight')
 
+import matplotlib.ticker as mtick
 
 # draw a line chart
 def DrawFigureYnormal(xvalues, yvalues, legend_labels, x_label, y_label, y_min, y_max, filename, allow_legend):
@@ -243,12 +244,14 @@ def DrawFigureYnormal(xvalues, yvalues, legend_labels, x_label, y_label, y_min,
     # plt.yscale('log')
 
     # you may control the limits on your own.
-
-    # plt.ylim(y_min, y_max)
+    yMax=np.max(y_values)
+    plt.ylim(y_min, yMax*1.2)
 
     plt.grid(axis='y', color='gray')
     plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号
-    # figure.yaxis.set_major_locator(LogLocator(base=10))
+    figure.yaxis.set_major_formatter(mtick.FormatStrFormatter('%.1f'))
+
+    figure.yaxis.set_major_locator(LinearLocator(numticks=5))
     # figure.xaxis.set_major_locator(LogLocator(base=10))
     plt.xticks(rotation=0, fontsize=TICK_FONT_SIZE)
     figure.get_xaxis().set_tick_params(direction='in', pad=10)
@@ -264,6 +267,7 @@ def DrawFigureYnormal(xvalues, yvalues, legend_labels, x_label, y_label, y_min,
     plt.savefig(filename + ".pdf", bbox_inches='tight')
 
 
+
 # example for reading csv file
 def ReadFile():
     y = []

benchmark/scripts/scanARowCPP/config_CPPCOFD.csv

@@ -0,0 +1,10 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/CRS.pt,String
+useCPP,1,U64
+cppAlgoTag,cooFD,String
+threads,1,U64
+forceMP,1,U64

benchmark/scripts/scanARowCPP/config_CPPCOUNTERSKETCH.csv

@@ -0,0 +1,10 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/CRS.pt,String
+useCPP,1,U64
+cppAlgoTag,countSketch,String
+threads,1,U64
+forceMP,1,U64

benchmark/scripts/scanARowCPP/config_CPPCRS.csv

@@ -5,4 +5,6 @@ bCol,500,U64
 sketchDimension,25,U64
 ptFile,torchscripts/CRS.pt,String
 useCPP,1,U64
-cppAlgoTag,crs,String
+cppAlgoTag,crs,String
+threads,1,U64
+forceMP,1,U64

benchmark/scripts/scanARowCPP/config_CPPEWS.csv

@@ -0,0 +1,10 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/RAWMM.pt,String
+useCPP,1,U64
+cppAlgoTag,ews,String
+threads,1,U64
+forceMP,1,U64

benchmark/scripts/scanARowCPP/config_CPPMM.csv

@@ -5,4 +5,6 @@ bCol,500,U64
 sketchDimension,25,U64
 ptFile,torchscripts/RAWMM.pt,String
 useCPP,1,U64
-cppAlgoTag,mm,String
+cppAlgoTag,mm,String
+threads,1,U64
+forceMP,1,U64

benchmark/scripts/scanARowCPP/config_CPPTOW.csv

@@ -0,0 +1,10 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/RAWMM.pt,String
+useCPP,1,U64
+cppAlgoTag,tugOfWar,String
+threads,1,U64
+forceMP,1,U64

benchmark/scripts/scanARowCPP/drawTogether.py

@@ -79,7 +79,9 @@ def readResultSingle(singleValue, resultPath):
     elapsedTime = readConfig(resultFname, "perfElapsedTime")
     cacheMiss = readConfig(resultFname, "cacheMiss")
     cacheRefs = readConfig(resultFname, "cacheRefs")
-    return elapsedTime, cacheMiss, cacheRefs
+    froError = readConfig(resultFname, "froError")
+    errorBoundRatio = readConfig(resultFname, "errorBoundRatio")
+    return elapsedTime, cacheMiss, cacheRefs, froError, errorBoundRatio
 
 
 def cleanPath(path):
@@ -91,33 +93,42 @@ def readResultVector(singleValueVec, resultPath):
     elapseTimeVec = []
     cacheMissVec = []
     cacheRefVec = []
+    froErrorVec = []
+    errorBoundRatioVec = []
     for i in singleValueVec:
-        elapsedTime, cacheMiss, cacheRefs = readResultSingle(i, resultPath)
+        elapsedTime, cacheMiss, cacheRefs, froError, errorBoundRatio = readResultSingle(i, resultPath)
         elapseTimeVec.append(float(elapsedTime) / 1000.0)
         cacheMissVec.append(float(cacheMiss))
         cacheRefVec.append(float(cacheRefs))
-    return np.array(elapseTimeVec), np.array(cacheMissVec), np.array(cacheRefVec)
+        froErrorVec.append(float(froError))
+        errorBoundRatioVec.append(float(errorBoundRatio))
+    return np.array(elapseTimeVec), np.array(cacheMissVec), np.array(cacheRefVec), np.array(froErrorVec), np.array(
+        errorBoundRatioVec)
 
 
 def compareMethod(exeSpace, commonPathBase, resultPaths, csvTemplates, periodVec, reRun=1):
     elapsedTimeAll = []
     cacheMissAll = []
     cacheRefAll = []
     periodAll = []
+    froAll = []
+    errorBoundRatioAll = []
     for i in range(len(csvTemplates)):
         resultPath = commonPathBase + resultPaths[i]
         if (reRun == 1):
             os.system("sudo rm -rf " + resultPath)
             os.system("sudo mkdir " + resultPath)
             runScanVector(exeSpace, periodVec, resultPath, csvTemplates[i])
-        elapsedTime, cacheMiss, cacheRef = readResultVector(periodVec, resultPath)
+        elapsedTime, cacheMiss, cacheRef, fro, eb = readResultVector(periodVec, resultPath)
         elapsedTimeAll.append(elapsedTime)
         cacheMissAll.append(cacheMiss)
         cacheRefAll.append(cacheRef)
         periodAll.append(periodVec)
         cacheMissRateAll = np.array(cacheMissAll) / np.array(cacheRefAll) * 100.0
+        froAll.append(fro)
+        errorBoundRatioAll.append(eb)
         # periodAll.append(periodVec)
-    return np.array(elapsedTimeAll), cacheMissRateAll, periodAll
+    return np.array(elapsedTimeAll), cacheMissRateAll, periodAll, np.array(froAll), np.array(errorBoundRatioAll)
 
 
 def main():
@@ -143,23 +154,33 @@ def main():
 
     # sampling
     # resultPaths = ["crs", "bcrs", "ews", "mm","mm-cpp","crs-cpp"]
-    resultPaths = ["mm-cpp", "crs-cpp"]
-    csvTemplates = ["config_CPPMM.csv", "config_CPPCRS.csv"]
-    methodTags = ["MM_CPP", "CRS_CPP"]
+    resultPaths = ["mm", "crs","ews","counterSketch","cofd","tow"]
+    csvTemplates = ["config_CPPMM.csv", "config_CPPCRS.csv","config_CPPEWS.csv","config_CPPCOUNTERSKETCH.csv","config_CPPCOFD.csv","config_CPPTOW.csv"]
+    methodTags = ["mm", "crs","ews","counterSketch","cofd","tow"]
     # csvTemplates = ["config_CRS.csv", "config_BerCRS.csv", "config_EWS.csv", "config_RAWMM.csv","config_CPPMM.csv","config_CPPCRS.csv"]
-    # methodTags = ["CRS", "Ber-CRS", "EWS", "MM","MM_CPP","CRS_CPP"]
-    elapsedTimeAll, cacheMissAll, periodAll = compareMethod(exeSpace, commonBase, resultPaths, csvTemplates, valueVec,
+    # methodTags = ["CRS", "Ber-CRS", "EWS",, "MM","MM_CPP","CRS_CPP"]
+    elapsedTimeAll, cacheMissAll, periodAll,fro,eb = compareMethod(exeSpace, commonBase, resultPaths, csvTemplates, valueVec,
                                                             reRun)
     groupLine.DrawFigure(periodAll, elapsedTimeAll,
                          methodTags,
-                         "#elements in A's row", "elapsed time (ms)", 0, 1,
-                         figPath + "/" + scanTag + "sampling_elapsedTime",
+                         "#A's row", "elapsed time (ms)", 0, 1,
+                         figPath + "/" + scanTag + "cpp_elapsedTime",
                          True)
     groupLine.DrawFigureYnormal(periodAll, cacheMissAll,
                                 methodTags,
-                                "#elements in A's row", "cacheMiss (%)", 0, 1,
-                                figPath + "/" + scanTag + "sampling_cacheMiss",
+                                "#A's row", "cacheMiss (%)", 0, 100,
+                                figPath + "/" + scanTag + "cpp_cacheMiss",
                                 True)
+    groupLine.DrawFigureYnormal(periodAll,
+                                 fro * 100.0,
+                                 methodTags,
+                                 "#A's row", "normalized error %", 0, 100, figPath + "/" +scanTag + "_froError",
+                                 True)
+    groupLine.DrawFigureYnormal(periodAll,
+                                 eb * 100.0,
+                                 methodTags,
+                                 "#A's row", "error bound ratio %", 0, 100, figPath + "/" + scanTag + "_ebRatio",
+                                 True)
     # draw2yLine("watermark time (ms)",singleValueVecDisp,lat95Vec,errVec,"95% Latency (ms)","Error","ms","",figPath+"wm_lat")
     # draw2yLine("watermark time (ms)",singleValueVecDisp,thrVec,errVec,"Throughput (KTp/s)","Error","KTp/s","",figPath+"wm_thr")
     # draw2yLine("watermark time (ms)",singleValueVecDisp,lat95Vec,compVec,"95% Latency (ms)","Completeness","ms","",figPath+"wm_omp")