Streaming

.github/workflows/cmake.yml

@@ -48,4 +48,5 @@ jobs:
           ./sketch_test "--success"
           ./crs_test "--success"
           ./weighted_cr_test "--success"
-          ./block_partition_test "--success"
+          ./block_partition_test "--success"
+          ./streaming_test "--success"

README.md

@@ -217,5 +217,6 @@ You will find the figures then.
    to validate the highlight when you need it, and comment it during a compile. :)
 4. When setting up torch for cpu under python version > 3.10, torch == 1.13.0 would conflict with torchaudio according
    to https://pytorch.org/audio/stable/installation.html. Use Python version <= 3.10 for smooth installation.
-5. Some heavy-weight algos like co-occurring FD may be treated as a zombie thread by some OS like ubuntu if running in default benchmark program, in this case
-, please force them to be executed as a running child thread by setting forceMP(U64)=1 and threads(U64)=1 in config
+5. Some heavy-weight algos like co-occurring FD may be treated as a zombie thread by some OS like ubuntu if running in
+   default benchmark program, in this case
+   , please force them to be executed as a running child thread by setting forceMP(U64)=1 and threads(U64)=1 in config

benchmark/scripts/scanACol/config_CPPCRS.csv

@@ -0,0 +1,8 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/CRS.pt,String
+useCPP,1,U64
+cppAlgoTag,crs,String

benchmark/scripts/scanACol/config_FDAMM.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/FDAMM.pt,String

benchmark/scripts/scanAColCPP/config_BCoAMM.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/BetaCoOccurringFD.pt,String

benchmark/scripts/scanAColCPP/config_BerCRS.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/BernoulliCRS.pt,String

benchmark/scripts/scanAColCPP/config_CRS.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/CRS.pt,String

benchmark/scripts/scanAColCPP/config_CoAMM.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/CoOccurringFD.pt,String

benchmark/scripts/scanAColCPP/config_CounterSketch.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/CountSketch.pt,String

benchmark/scripts/scanAColCPP/config_EWS.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/EWS.pt,String

benchmark/scripts/scanAColCPP/config_FDAMM.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/FDAMM.pt,String

benchmark/scripts/scanAColCPP/config_RAWMM.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/RAWMM.pt,String

benchmark/scripts/scanAColCPP/config_WCR.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,1000,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/WeightedCR.pt,String

benchmark/scripts/scanAColCPP/drawTogether.py

@@ -151,8 +151,9 @@ def main():
         os.system("sudo mkdir " + commonBase)
         reRun = 1
     # skech
-    elapsedTimeAll, cacheMissAll, periodAll, fro, eb = compareMethod(exeSpace, commonBase, resultPaths, csvTemplates, valueVec,
-                                                            reRun)
+    elapsedTimeAll, cacheMissAll, periodAll, fro, eb = compareMethod(exeSpace, commonBase, resultPaths, csvTemplates,
+                                                                     valueVec,
+                                                                     reRun)
     groupLine.DrawFigure(periodAll, elapsedTimeAll,
                          methodTags,
                          "#elements in A's col", "elapsed time (ms)", 0, 1,
@@ -164,17 +165,17 @@ def main():
                                 figPath + "/" + scanTag + "sketch_cpp_cacheMiss",
                                 True)
     groupLine.DrawFigureYnormal(periodAll,
-                                 fro * 100.0,
-                                 methodTags,
-                                 "#A's row", "normalized error %", 0, 100, figPath + "/" +scanTag +
+                                fro * 100.0,
+                                methodTags,
+                                "#A's row", "normalized error %", 0, 100, figPath + "/" + scanTag +
                                 "sketch_cpp_froError",
-                                 True)
+                                True)
     groupLine.DrawFigureYnormal(periodAll,
-                                 eb * 100.0,
-                                 methodTags,
-                                 "#A's row", "error bound ratio %", 0, 100, figPath + "/" + scanTag
+                                eb * 100.0,
+                                methodTags,
+                                "#A's row", "error bound ratio %", 0, 100, figPath + "/" + scanTag
                                 + "sketch_cpp_ebRatio",
-                                 True)
+                                True)
     # sampling
     # resultPaths = ["crs", "bcrs", "ews", "mm","mm-cpp","crs-cpp"]
     resultPaths = ["crs", "crsv2", "bcrs", "wcr", "ews", "int8", "mm"]
@@ -183,8 +184,9 @@ def main():
     methodTags = ["CRS", "CRSV2", "Ber-CRS", "Weighted-CR", "EWS", "INT8", "MM"]
     # csvTemplates = ["config_CRS.csv", "config_BerCRS.csv", "config_EWS.csv", "config_RAWMM.csv","config_CPPMM.csv","config_CPPCRS.csv"]
     # methodTags = ["CRS", "Ber-CRS", "EWS",, "MM","MM_CPP","CRS_CPP"]
-    elapsedTimeAll, cacheMissAll, periodAll,fro,eb = compareMethod(exeSpace, commonBase, resultPaths, csvTemplates, valueVec,
-                                                            reRun)
+    elapsedTimeAll, cacheMissAll, periodAll, fro, eb = compareMethod(exeSpace, commonBase, resultPaths, csvTemplates,
+                                                                     valueVec,
+                                                                     reRun)
     groupLine.DrawFigure(periodAll, elapsedTimeAll,
                          methodTags,
                          "#A's col", "elapsed time (ms)", 0, 1,
@@ -196,15 +198,15 @@ def main():
                                 figPath + "/" + scanTag + "sampling_cpp_cacheMiss",
                                 True)
     groupLine.DrawFigureYnormal(periodAll,
-                                 fro * 100.0,
-                                 methodTags,
-                                 "#A's col", "normalized error %", 0, 1, figPath + "/" +scanTag + "_froError",
-                                 True)
+                                fro * 100.0,
+                                methodTags,
+                                "#A's col", "normalized error %", 0, 1, figPath + "/" + scanTag + "_froError",
+                                True)
     groupLine.DrawFigureYnormal(periodAll,
-                                 eb * 100.0,
-                                 methodTags,
-                                 "#A's col", "error bound ratio %", 0, 1, figPath + "/" + scanTag + "_ebRatio",
-                                 True)
+                                eb * 100.0,
+                                methodTags,
+                                "#A's col", "error bound ratio %", 0, 1, figPath + "/" + scanTag + "_ebRatio",
+                                True)
     # draw2yLine("watermark time (ms)",singleValueVecDisp,lat95Vec,errVec,"95% Latency (ms)","Error","ms","",figPa
     # draw2yLine("watermark time (ms)",singleValueVecDisp,lat95Vec,errVec,"95% Latency (ms)","Error","ms","",figPath+"wm_lat")
     # draw2yLine("watermark time (ms)",singleValueVecDisp,thrVec,errVec,"Throughput (KTp/s)","Error","KTp/s","",figPath+"wm_thr")

benchmark/scripts/scanAColCPP/groupLine.py

@@ -208,8 +208,10 @@ def DrawFigureXYnormal(xvalues, yvalues, legend_labels, x_label, y_label, y_min,
     # plt.show()
     plt.savefig(filename + ".pdf", bbox_inches='tight')
 
+
 import matplotlib.ticker as mtick
 
+
 # draw a line chart
 def DrawFigureYnormal(xvalues, yvalues, legend_labels, x_label, y_label, y_min, y_max, filename, allow_legend):
     # you may change the figure size on your own.
@@ -244,8 +246,8 @@ def DrawFigureYnormal(xvalues, yvalues, legend_labels, x_label, y_label, y_min,
     # plt.yscale('log')
 
     # you may control the limits on your own.
-    yMax=np.max(y_values)
-    plt.ylim(y_min, yMax*1.2)
+    yMax = np.max(y_values)
+    plt.ylim(y_min, yMax * 1.2)
 
     plt.grid(axis='y', color='gray')
     plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号
@@ -267,7 +269,6 @@ def DrawFigureYnormal(xvalues, yvalues, legend_labels, x_label, y_label, y_min,
     plt.savefig(filename + ".pdf", bbox_inches='tight')
 
 
-
 # example for reading csv file
 def ReadFile():
     y = []

benchmark/scripts/scanARow/config_FDAMM.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/FDAMM.pt,String

benchmark/scripts/scanARowCPP/config_BCoAMM.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/BetaCoOccurringFD.pt,String

benchmark/scripts/scanARowCPP/config_BerCRS.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/BernoulliCRS.pt,String

benchmark/scripts/scanARowCPP/config_CRS.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/CRS.pt,String

benchmark/scripts/scanARowCPP/config_CoAMM.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/CoOccurringFD.pt,String

benchmark/scripts/scanARowCPP/config_CounterSketch.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/CountSketch.pt,String

benchmark/scripts/scanARowCPP/config_EWS.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/EWS.pt,String

benchmark/scripts/scanARowCPP/config_FDAMM.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/FDAMM.pt,String

benchmark/scripts/scanARowCPP/config_RAWMM.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/RAWMM.pt,String

benchmark/scripts/scanARowCPP/config_WCR.csv

@@ -0,0 +1,6 @@
+key,value,type
+aRow,100,U64
+aCol,1000,U64
+bCol,500,U64
+sketchDimension,25,U64
+ptFile,torchscripts/WeightedCR.pt,String

benchmark/scripts/scanARowCPP/drawTogether.py

@@ -151,8 +151,9 @@ def main():
         os.system("sudo mkdir " + commonBase)
         reRun = 1
     # skech
-    elapsedTimeAll, cacheMissAll, periodAll, fro, eb = compareMethod(exeSpace, commonBase, resultPaths, csvTemplates, valueVec,
-                                                            reRun)
+    elapsedTimeAll, cacheMissAll, periodAll, fro, eb = compareMethod(exeSpace, commonBase, resultPaths, csvTemplates,
+                                                                     valueVec,
+                                                                     reRun)
     groupLine.DrawFigure(periodAll, elapsedTimeAll,
                          methodTags,
                          "#elements in A's row", "elapsed time (ms)", 0, 1,
@@ -164,17 +165,17 @@ def main():
                                 figPath + "/" + scanTag + "sketch_cpp_cacheMiss",
                                 True)
     groupLine.DrawFigureYnormal(periodAll,
-                                 fro * 100.0,
-                                 methodTags,
-                                 "#A's row", "normalized error %", 0, 100, figPath + "/" +scanTag +
+                                fro * 100.0,
+                                methodTags,
+                                "#A's row", "normalized error %", 0, 100, figPath + "/" + scanTag +
                                 "sketch_cpp_froError",
-                                 True)
+                                True)
     groupLine.DrawFigureYnormal(periodAll,
-                                 eb * 100.0,
-                                 methodTags,
-                                 "#A's row", "error bound ratio %", 0, 100, figPath + "/" + scanTag
+                                eb * 100.0,
+                                methodTags,
+                                "#A's row", "error bound ratio %", 0, 100, figPath + "/" + scanTag
                                 + "sketch_cpp_ebRatio",
-                                 True)
+                                True)
 
     # sampling
     # resultPaths = ["crs", "bcrs", "ews", "mm","mm-cpp","crs-cpp"]
@@ -184,8 +185,9 @@ def main():
     methodTags = ["CRS", "CRSV2", "Ber-CRS", "Weighted-CR", "EWS", "INT8", "MM"]
     # csvTemplates = ["config_CRS.csv", "config_BerCRS.csv", "config_EWS.csv", "config_RAWMM.csv","config_CPPMM.csv","config_CPPCRS.csv"]
     # methodTags = ["CRS", "Ber-CRS", "EWS",, "MM","MM_CPP","CRS_CPP"]
-    elapsedTimeAll, cacheMissAll, periodAll,fro,eb = compareMethod(exeSpace, commonBase, resultPaths, csvTemplates, valueVec,
-                                                            reRun)
+    elapsedTimeAll, cacheMissAll, periodAll, fro, eb = compareMethod(exeSpace, commonBase, resultPaths, csvTemplates,
+                                                                     valueVec,
+                                                                     reRun)
     groupLine.DrawFigure(periodAll, elapsedTimeAll,
                          methodTags,
                          "#A's row", "elapsed time (ms)", 0, 1,
@@ -197,17 +199,17 @@ def main():
                                 figPath + "/" + scanTag + "sampling_cpp_cacheMiss",
                                 True)
     groupLine.DrawFigureYnormal(periodAll,
-                                 fro * 100.0,
-                                 methodTags,
-                                 "#A's row", "normalized error %", 0, 100, figPath + "/" +scanTag +
+                                fro * 100.0,
+                                methodTags,
+                                "#A's row", "normalized error %", 0, 100, figPath + "/" + scanTag +
                                 "sampling_cpp_froError",
-                                 True)
+                                True)
     groupLine.DrawFigureYnormal(periodAll,
-                                 eb * 100.0,
-                                 methodTags,
-                                 "#A's row", "error bound ratio %", 0, 100, figPath + "/" + scanTag
+                                eb * 100.0,
+                                methodTags,
+                                "#A's row", "error bound ratio %", 0, 100, figPath + "/" + scanTag
                                 + "sampling_cpp_ebRatio",
-                                 True)
+                                True)
     # draw2yLine("watermark time (ms)",singleValueVecDisp,lat95Vec,errVec,"95% Latency (ms)","Error","ms","",figPath+"wm_lat")
     # draw2yLine("watermark time (ms)",singleValueVecDisp,thrVec,errVec,"Throughput (KTp/s)","Error","KTp/s","",figPath+"wm_thr")
     # draw2yLine("watermark time (ms)",singleValueVecDisp,lat95Vec,compVec,"95% Latency (ms)","Completeness","ms","",figPath+"wm_omp")

benchmark/scripts/scanARowCPP/groupLine.py

@@ -11,6 +11,7 @@
 from matplotlib.ticker import LinearLocator, LogLocator, MaxNLocator, ScalarFormatter
 from numpy import double
 import matplotlib.ticker as mtick
+
 # 获取系统中可用的字体路径
 # font_paths = fm.findSystemFonts()
 # OPT_FONT_NAME = 'Helvetica'
@@ -243,8 +244,8 @@ def DrawFigureYnormal(xvalues, yvalues, legend_labels, x_label, y_label, y_min,
     # plt.yscale('log')
 
     # you may control the limits on your own.
-    yMax=np.max(y_values)
-    plt.ylim(y_min, yMax*1.2)
+    yMax = np.max(y_values)
+    plt.ylim(y_min, yMax * 1.2)
 
     plt.grid(axis='y', color='gray')
     plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号