Intel experiments code

.gitmodules

@@ -38,13 +38,9 @@
 [submodule "third-party/ParResKernels"]
 	path = third-party/ParResKernels
 	url = https://github.com/Shillaker/Kernels.git
-[submodule "third-party/LULESH"]
-	path = third-party/LULESH
-	url = https://github.com/mfournial/LULESH.git
 [submodule "third-party/wasi-libc"]
-    path = third-party/wasi-libc
-    url = https://github.com/Shillaker/wasi-libc
-
+	path = third-party/wasi-libc
+	url = https://github.com/Shillaker/wasi-libc
 [submodule "third-party/wamr"]
 	path = third-party/wamr
 	url = https://github.com/bytecodealliance/wasm-micro-runtime.git

CMakeLists.txt

@@ -31,7 +31,7 @@ if (FAASM_WAMR_SUPPORT)
     set(WAMR_BUILD_AOT 0)
     set(WAMR_BUILD_LIBC_WASI 1)
     set(WAMR_BUILD_LIBC_BUILTIN 0)
-endif()
+endif ()
 
 # Library type (for shared libraries)
 if (FAASM_STATIC_LIBS)
@@ -51,6 +51,12 @@ else ()
     endfunction()
 endif ()
 
+option(FAASM_OMP_PTS "PREALLOCATE THREAD STACK: Only available for local WasmMP and non recursive levels")
+if (${FAASM_OMP_PTS})
+    message("-- Activated OMP_PTS: OpenMP Thread stack preallocation feature")
+    add_compile_definitions(OMP_PTS)
+endif ()
+
 # Switch on WAVM stack traces in debug (potential performance gain?)
 set(WAVM_ENABLE_UNWIND ON CACHE BOOL "WAVM unwind")
 #if(CMAKE_BUILD_TYPE MATCHES Debug)
@@ -64,11 +70,14 @@ add_definitions(-DDLL_EXPORT=)
 add_definitions(-DDLL_IMPORT=)
 
 # Faasm profiling
-add_definitions(-DFAASM_PROFILE_ON=0)
+option(FAASM_PERF_PROFILING "Turn on profiling features as described in debugging.md")
+option(FAASM_SELF_PROFILING "Turn on system profiling logged at tracing level")
+
+if (${FAASM_SELF_PROFILING})
+    add_compile_definitions(PROFILE_ALL)
+endif ()
 
-# Custom LLVM build (also for profiling)
-set(FAASM_CUSTOM_LLVM 0)
-if (${FAASM_CUSTOM_LLVM})
+if (${FAASM_PERF_PROFILING})
     # In accordance with bin/build_llvm_perf.sh and LLVM version for WAVM
     set(LLVM_DIR /usr/local/code/llvm-perf/build/lib/cmake/llvm)
     message(STATUS "Using custom LLVM at ${LLVM_DIR} for profiling")
@@ -143,10 +152,10 @@ else ()
     # WAVM
     add_subdirectory(third-party/WAVM)
 
-    if(FAASM_WAMR_SUPPORT)
-    include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)
-    add_library(libwamr ${WAMR_RUNTIME_LIB_SOURCE})
-    endif()
+    if (FAASM_WAMR_SUPPORT)
+        include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)
+        add_library(libwamr ${WAMR_RUNTIME_LIB_SOURCE})
+    endif ()
 
     # Faasm functions
     add_subdirectory(func)

docs/debugging.md

@@ -5,8 +5,10 @@
 As Faasm functions are compiled to WebAssembly and executed using [WAVM](https://github.com/WAVM/WAVM/),
 any general tips that apply in WAVM also apply to Faasm.
 
-The instructions below assume that you're building Faasm locally as per the [local dev](local_dev.md) 
-instructions and that Faasm's built executables are on your `PATH`.
+The instructions below assume that 
+- you're building Faasm locally as per the [local dev](local_dev.md) instructions,
+- that you built the `func_sym` target,
+- that Faasm's built executables are on your `PATH`.
 
 ### Symbols
 
@@ -53,8 +55,8 @@ To set things up you need to do the following:
 
 - Run the `perf.yml` Ansible playbook to set up `perf`
 - Create a build of LLVM with perf support by running `./bin/build/llvm_perf` (this takes ages)
-- Modify the top-level `CMakeLists.txt` to set `FAASM_CUSTOM_LLVM` to `1`
-- Rebuild the target you want to profile
+- Turn on the `FAASM_PERF_PROFILING` option in you CMake build configuration. (`ccmake <build_dir>` makes this easy).
+- Rebuild `codegen_func`, and your runner to build and run the target you want to profile, 
 
 Once this is done you can use perf as described [here](https://lwn.net/Articles/633846/), i.e.:
 

faasmcli/faasmcli/tasks/libs.py

@@ -70,10 +70,11 @@ def _build_faasm_lib(dir_name, clean, verbose):
 
     clean_dir(build_dir, clean)
 
-    verbose_str = "VERBOSE=1" if verbose else ""
+    verbose_str = "-s" if verbose else ""
     build_cmd = [
         verbose_str,
         "cmake",
+        "-G Ninja"
         "-DFAASM_BUILD_TYPE=wasm",
         "-DCMAKE_BUILD_TYPE=Release",
         "-DCMAKE_TOOLCHAIN_FILE={}".format(FAASM_TOOLCHAIN_FILE),
@@ -87,15 +88,10 @@ def _build_faasm_lib(dir_name, clean, verbose):
     if res != 0:
         exit(1)
 
-    res = call("{} make".format(verbose_str), shell=True, cwd=build_dir)
+    res = call("ninja {} install".format(verbose_str), shell=True, cwd=build_dir)
     if res != 0:
         exit(1)
 
-    res = call("make install", shell=True, cwd=build_dir)
-    if res != 0:
-        exit(1)
-
-
 @task
 def faasm(ctx, clean=False, lib=None, verbose=False):
     """
@@ -134,6 +130,30 @@ def rust(ctx, clean=False, verbose=False):
     """
     _build_faasm_lib("rust", clean, verbose)
 
+@task
+def malloc(ctx, clean=False):
+    """
+    Compile and install dlmalloc
+    """
+    work_dir = join(PROJ_ROOT, "third-party", "malloc")
+    build_dir = join(PROJ_ROOT, "build", "malloc")
+
+    clean_dir(build_dir, clean)
+
+    build_cmd = [
+        "cmake",
+        "-G Ninja",
+        "-DCMAKE_BUILD_TYPE=Release",
+        "-DCMAKE_TOOLCHAIN_FILE={}".format(FAASM_TOOLCHAIN_FILE),
+        work_dir,
+    ]
+
+    build_cmd_str = " ".join(build_cmd)
+    print(build_cmd_str)
+
+    call(build_cmd_str, shell=True, cwd=build_dir)
+    call("ninja install", shell=True, cwd=build_dir)
+
 
 @task
 def fake(ctx, clean=False):
@@ -181,11 +201,11 @@ def fake(ctx, clean=False):
 
 
 @task
-def lulesh(ctx, mpi=False, omp=False, clean=True, debug=False, cp=True):
+def lulesh(ctx, lulesh_dir, mpi=False, omp=False, clean=True, debug=False, cp=True):
     """
     Compile and install the LULESH code
     """
-    work_dir = join(THIRD_PARTY_DIR, "LULESH")
+    work_dir = lulesh_dir
 
     if omp and mpi:
         build_dir = "ompi"

func/omp/intel_nstreams.cpp

@@ -152,8 +152,8 @@ FAASM_MAIN_FUNC()
     printf("OpenMP stream triad: A = B + scalar*C\n");
 
     // Faasm: does not support arguments:
-    nthread_input = 4;
-    iterations    = 1000;
+    nthread_input = 2;
+    iterations    = 1'000;
     length        = 10000000;
     offset        = 0;
     /* FAASM, use defaults specified above

func/omp/intel_synch_p2p.cpp

@@ -105,8 +105,8 @@ FAASM_MAIN_FUNC() {
     printf("Parallel Research Kernels version %s\n", PRKVERSION);
     printf("OpenMP pipeline execution on 2D grid\n");
 
-    nthread_input = 4;
-    iterations  = 100;
+    nthread_input = 20;
+    iterations  = 1000;
     m  = 10000;
     n  = 1000;
 

func/omp/intel_transpose.cpp

@@ -101,7 +101,7 @@ FAASM_MAIN_FUNC() {
     printf("Parallel Research Kernels version %s\n", PRKVERSION);
     printf("OpenMP Matrix transpose: B = A^T\n");
 
-    nthread_input = 4;
+    nthread_input = 20;
     iterations    = 300;
     order         = 2000;
     /* Faasm - Hardcode the default

include/util/timing.h

@@ -3,7 +3,7 @@
 #include <util/clock.h>
 #include <string>
 
-#ifdef FAASM_PROFILE_ON
+#ifdef PROFILE_ALL
 #define PROF_START(name) const util::TimePoint name = util::startTimer();
 #define PROF_END(name) util::logEndTimer(#name, name);
 #else

include/wasm/WasmModule.h

@@ -115,9 +115,7 @@ namespace wasm {
         virtual void doRestore(std::istream &inStream) = 0;
 
         void prepareArgcArgv(const message::Message &msg);
-
-        void prepareOpenMPContext(const message::Message &msg);
-};
+    };
 
     // ----- Global functions -----
     message::Message *getExecutingCall();

include/wavm/PlatformThreadPool.h

@@ -0,0 +1,51 @@
+#pragma once
+
+#include <condition_variable>
+#include <future>
+#include <mutex>
+#include <queue>
+#include <vector>
+
+#include <WAVM/Runtime/Runtime.h>
+#include <WAVM/Inline/BasicTypes.h>
+#include <WAVM/Platform/Thread.h>
+
+#include <util/locks.h>
+#include <util/locks.h>
+#include <wavm/PlatformThreadPool.h>
+
+namespace wasm {
+
+    using namespace WAVM;
+    class WAVMWasmModule;
+
+    namespace openmp {
+        struct LocalThreadArgs;
+    }
+
+    class PlatformThreadPool {
+    public:
+        PlatformThreadPool(size_t numThreads, WAVMWasmModule *module);
+
+        friend I64 workerEntryFunc(void* _args);
+
+        std::future<I64> runThread(openmp::LocalThreadArgs &&threadArgs);
+
+        ~PlatformThreadPool();
+
+    private:
+        std::queue<std::pair<std::promise<I64>, openmp::LocalThreadArgs>> tasks;
+        std::vector<WAVM::Platform::Thread *> workers;
+
+        std::mutex mutexQueue;
+        std::condition_variable condition;
+        bool stop = false;
+    };
+
+    struct WorkerArgs {
+        U32 stackTop;
+        PlatformThreadPool *pool;
+    };
+
+}
+

include/wavm/WAVMWasmModule.h

@@ -1,11 +1,11 @@
 #pragma once
 
-#include <wasm/WasmModule.h>
-
 #include <WAVM/Runtime/Intrinsics.h>
 #include <WAVM/Runtime/Linker.h>
 #include <WAVM/Runtime/Runtime.h>
 
+#include <wasm/WasmModule.h>
+
 using namespace WAVM;
 
 namespace wasm {
@@ -17,6 +17,8 @@ namespace wasm {
 
     struct WasmThreadSpec;
 
+    class PlatformThreadPool;
+
     class WAVMWasmModule : public WasmModule, Runtime::Resolver {
     public:
         WAVMWasmModule();
@@ -112,6 +114,10 @@ namespace wasm {
 
         int getDataOffsetFromGOT(const std::string &name);
 
+        U32 allocateThreadStack();
+
+        std::unique_ptr<PlatformThreadPool> &getPool();
+
     protected:
         void doSnapshot(std::ostream &outStream) override;
 
@@ -172,6 +178,10 @@ namespace wasm {
         void syncPythonFunctionFile(const message::Message &msg);
 
         void executeRemoteOMP(message::Message &msg);
+
+        void prepareOpenMPContext(const message::Message &msg);
+
+        std::unique_ptr<PlatformThreadPool> ompPool;
     };
 
     WAVMWasmModule *getExecutingModule();
@@ -182,5 +192,6 @@ namespace wasm {
         Runtime::ContextRuntimeData *contextRuntimeData;
         Runtime::Function *func;
         IR::UntaggedValue *funcArgs;
+        U32 stackTop;
     };
 }

include/wavm/openmp/Level.h

@@ -1,10 +1,11 @@
 #pragma once
 
-#include <wavm/openmp/ClangTypes.h>
 #include <mutex>
+
 #include <proto/faasm.pb.h>
 #include <util/barrier.h>
 #include <util/environment.h>
+#include <wavm/openmp/ClangTypes.h>
 
 namespace wasm {
     namespace openmp {
@@ -20,6 +21,7 @@ namespace wasm {
         // Global variables controlled by level master
         class Level {
         public:
+            // Defaults set to mimic Clang 9.0.1 behaviour
             const int depth = 0; // Number of nested OpenMP constructs, 0 for serial code
             const int effectiveDepth = 0; // Number of parallel regions (> 1 thread) above this level
             int maxActiveLevel = 1; // Max number of effective parallel regions allowed from the top
@@ -29,8 +31,6 @@ namespace wasm {
             // TODO - This implementation limits to one lock for all critical sections at a level.
             // Mention in report (maybe fix looking at the lck address and doing a lookup on it though?)
             std::mutex criticalSection; // Mutex used in critical sections.
-
-            // Defaults set to mimic Clang 9.0.1 behaviour
             Level() = default;
 
             // Local constructor
@@ -54,9 +54,11 @@ namespace wasm {
 
         class SingleHostLevel : public Level {
         public:
+
             SingleHostLevel() = default;
 
-            SingleHostLevel(const std::shared_ptr<Level>& parent, int numThreads);
+            SingleHostLevel(const std::shared_ptr<Level> &parent, int numThreads);
+
             ReduceTypes reductionMethod() override;
 
             ~SingleHostLevel() = default;

include/wavm/openmp/openmp.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <wavm/openmp/Level.h>
+#include <wavm/WAVMWasmModule.h>
+
+namespace wasm {
+
+    namespace openmp {
+        struct LocalThreadArgs {
+            int tid = 0;
+            std::shared_ptr<Level> level = nullptr;
+            WAVMWasmModule *parentModule;
+            message::Message *parentCall;
+            WasmThreadSpec spec;
+        };
+    }
+
+}

src/runner/CMakeLists.txt

@@ -15,3 +15,6 @@ target_link_libraries(simple_runner ${RUNNER_LIBS})
 
 add_executable(func_sym func_sym.cpp)
 target_link_libraries(func_sym ${RUNNER_LIBS})
+
+add_executable(bench_omp bench_omp.cpp)
+target_link_libraries(bench_omp ${RUNNER_LIBS})