Interpreter work

analyze

@@ -1,17 +1,47 @@
 #!/usr/bin/env bash
+set -euo pipefail
 
+# Separate --jobs from passthrough koga args, and capture any positional
+# extension names (./analyze cando seqan-clasp).
 NINJA_OPTIONS=""
-
-for i in "$@"; do
-  if [[ $i == --jobs=* ]]; then
-    NINJA_OPTIONS="-j ${i#*=}"
+KOGA_ARGS=()
+SELECTED_EXTS=()
+for arg in "$@"; do
+  if [[ $arg == --jobs=* ]]; then
+    NINJA_OPTIONS="-j ${arg#*=}"
+  elif [[ $arg == -* ]]; then
+    KOGA_ARGS+=("$arg")
+  else
+    SELECTED_EXTS+=("$arg")
   fi
 done
 
-./koga --build-mode=bytecode --build-path=build-analysis --clean --no-extensions "$@"
-ninja -C build-analysis $NINJA_OPTIONS analyze
+# Discover extensions: every extensions/*/ that contains a src/.
+discover_extensions() {
+  local d
+  for d in extensions/*/; do
+    if [[ -d "${d}src" ]]; then
+      basename "$d"
+    fi
+  done
+}
+
+if [[ ${#SELECTED_EXTS[@]} -eq 0 ]]; then
+  EXTS=( $(discover_extensions) )
+  RUN_BASE=1
+else
+  EXTS=( "${SELECTED_EXTS[@]}" )
+  RUN_BASE=0    # selective re-run: skip base, diff against committed base
+fi
 
-./koga --build-mode=bytecode --build-path=build-analysis --clean --extensions=cando "$@"
-ninja -C build-analysis $NINJA_OPTIONS analyze
+if [[ $RUN_BASE -eq 1 ]]; then
+  ./koga --build-mode=bytecode --build-path=build-analysis --clean --no-extensions "${KOGA_ARGS[@]}"
+  ninja -C build-analysis $NINJA_OPTIONS analyze
+fi
+
+for ext in "${EXTS[@]}"; do
+  ./koga --build-mode=bytecode --build-path=build-analysis --clean --extensions="$ext" "${KOGA_ARGS[@]}"
+  ninja -C build-analysis $NINJA_OPTIONS analyze
+done
 
 rm -rf build-analysis

include/clasp/core/bytecode.h

@@ -9,6 +9,7 @@
 
 namespace core {
 class Bytecode_O;
+struct VMDynRecord;   // defined in clasp/gctools/threadlocal.h
 };
 
 template <> struct gctools::GCInfo<core::Bytecode_O> {
@@ -326,19 +327,25 @@ class VMFrameDynEnv_O : public DynEnv_O {
   LISP_CLASS(core, CorePkg, VMFrameDynEnv_O, "VMFrameDynEnv", DynEnv_O);
 
 public:
-  VMFrameDynEnv_O(T_O** a_old_sp, T_O** a_old_fp) : old_sp(a_old_sp), old_fp(a_old_fp) {}
+  VMFrameDynEnv_O(T_O** a_old_sp, T_O** a_old_fp, VMDynRecord* a_old_dyn_top)
+    : old_sp(a_old_sp), old_fp(a_old_fp), old_dyn_top(a_old_dyn_top) {}
   // Slightly sketchy: We use the destructor to reset the stack pointer,
   // so that C++ unwinds are also affected by this dynenv.
   // This means VMFrames must be stack allocated.
+  // old_dyn_top is the _dynRecordTop mark saved when this bytecode_call was
+  // entered. If an SJLJ longjmp bypasses this frame, proceed() restores it so
+  // the VM dynenv-record stack does not keep stale records across activations.
   ~VMFrameDynEnv_O() {
     VirtualMachine& vm = my_thread->_VM;
     vm._stackPointer = this->old_sp;
     vm._framePointer = this->old_fp;
+    vm._dynRecordTop = this->old_dyn_top;
   }
 
 public:
   T_O** old_sp;
   T_O** old_fp;
+  VMDynRecord* old_dyn_top;
 
 public:
   virtual SearchStatus search() const { return Continue; }

include/clasp/core/clasp_gmpxx.h

@@ -28,6 +28,9 @@ THE SOFTWARE.
 
 /* Define a C++ GMP wrapper */
 
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-literal-operator"
 #include <gmpxx.h>
+#pragma clang diagnostic pop
 
 typedef mpz_class Bignum;

include/clasp/core/commandLineOptions.h

@@ -77,6 +77,7 @@ struct CommandLineOptions {
   bool _NoRc;
   bool _PauseForDebugger;
   bool _GenerateTrampolines;
+  std::vector<std::string> _ExtensionArguments;
 
   bool validStartupTypeOption(const std::string& arg);
   void printVersion();

include/clasp/core/function.h

@@ -198,7 +198,7 @@ namespace core {
 // check out bytecode simple funs).
 FORWARD(SimpleFun);
 class SimpleFun_O : public Function_O {
-  LISP_CLASS(core, CorePkg, SimpleFun_O, "SimpleFun", Function_O);
+  LISP_ABSTRACT_CLASS(core, CorePkg, SimpleFun_O, "SimpleFun", Function_O);
 
 public:
   FunctionDescription_sp _FunctionDescription;
@@ -243,6 +243,19 @@ class SimpleFun_O : public Function_O {
   }
 };
 
+// add to function.h, near the other SimpleFun_O subclasses
+class XepSimpleFun_O : public SimpleFun_O {
+  LISP_CLASS(core, CorePkg, XepSimpleFun_O, "XepSimpleFun", SimpleFun_O);
+
+public:
+  XepSimpleFun_O(FunctionDescription_sp fdesc, T_sp code, const ClaspXepTemplate& xep)
+      : SimpleFun_O(fdesc, code, xep) {}
+
+  virtual size_t templatedSizeof() const override { return sizeof(*this); }
+};
+
+
+
 // Now that SimpleFun exists we can define these.
 template <typename... Ts>
 LCC_RETURN Function_O::funcall(Ts... args) {

include/clasp/core/iterator.h

@@ -36,9 +36,17 @@ namespace core {
 
 SMART(Iterator);
 class Iterator_O : public General_O {
-  LISP_CLASS(core, CorePkg, Iterator_O, "Iterator", General_O);
+  LISP_ABSTRACT_CLASS(core, CorePkg, Iterator_O, "Iterator", General_O);
 
 public:
+  // Untagged C++ subclasses (e.g. clbind::Iterator<X>) don't have their own
+  // LISP_CLASS-generated __class() override.  Without this, virtual dispatch
+  // walks past us to General_O::__class() and instances report General_O as
+  // their class — breaking single-dispatch on iterator= and similar.
+  virtual core::Instance_sp __class() const override {
+    return core::lisp_getStaticClass(Iterator_O::static_ValueStampWtagMtag);
+  }
+
   void initialize() override;
 
 private:

include/clasp/core/lisp.h

@@ -279,6 +279,10 @@ class Lisp {
     std::atomic<T_sp> _AllObjectFiles;
     std::atomic<T_sp> _AllCodeBlocks;
     std::atomic<T_sp> _AllBytecodeModules;
+    // Every GFBytecodeSimpleFun ever made (atomic-pushed list of cons cells).
+    // Walked by arena_post_load_regenerate_trampolines after a snapshot load
+    // so the dispatch trampoline for each generic function gets re-attached.
+    std::atomic<T_sp> _AllGFBytecodeFuns;
     SimpleFun_sp _UnboundCellFunctionEntryPoint;
     T_sp _TerminalIO;
     List_sp _ActiveThreads;

include/clasp/core/sampling_profiler.h

@@ -0,0 +1,91 @@
+/*
+ * sampling_profiler.h — CPU-time sampling profiler.
+ *
+ * At rate `N` Hz, an ITIMER_PROF timer delivers SIGPROF to an arbitrary
+ * running thread. The handler walks the frame-pointer chain via the
+ * ucontext registers and appends a sample (timestamp, thread id, depth,
+ * optional bytecode-VM pc, variable-length PC array) to a per-process
+ * bump-allocated ring.
+ *
+ * Separate from src/core/profiler.cc's RangePush/RangePop instrumentation.
+ * That profiler measures user-annotated regions; this one periodically
+ * snapshots whatever code is running.
+ *
+ * See Phase 4 / Phase 5 for post-mortem symbolication and flame-graph
+ * output — this header covers the recording side only.
+ */
+#pragma once
+
+#include <cstdint>
+#include <cstddef>
+#include <string>
+#include <vector>
+
+namespace core {
+
+// Per-sample header (variable-length record). A SampleHeader is followed
+// immediately in the ring buffer by `depth` × uint64_t native PCs.
+struct SampleHeader {
+  uint64_t timestamp_ns;   // CLOCK_MONOTONIC at signal delivery
+  uint64_t vm_pc;          // bytecode VM's _pc at sample time, or 0
+  uint32_t thread_id;      // Linux tid / macOS port id (truncated)
+  uint32_t depth;          // number of trailing PCs (0 if walk failed)
+};
+
+// Aggregated symbolicated sample: one entry per unique (thread_id, frames)
+// group. `frames` is outermost-first (index 0 is the root, last is the
+// leaf). `sample_count` is the number of raw samples that collapsed into
+// this entry.
+struct SymbolicatedSample {
+  uint32_t thread_id;
+  size_t   sample_count;
+  std::vector<std::string> frames;
+  core::T_sp encode();
+};
+
+// Start the profiler.
+//   rate_hz          : sampling rate in Hz (e.g. 97). Clamped to [1, 10000].
+//   max_depth        : per-sample stack-depth cap. Clamped to [1, 8192].
+//   buffer_bytes     : ring buffer size (0 = default 256 MiB).
+// Returns true on success. Fails if the profiler is already running or the
+// OS timer/signal setup fails.
+bool sampling_profiler_start(unsigned rate_hz,
+                             unsigned max_depth,
+                             size_t buffer_bytes);
+
+// Stop sampling. The buffer is preserved; call
+// sampling_profiler_save / sampling_profiler_reset to drain / clear.
+void sampling_profiler_stop();
+
+// True while a profile session is active.
+bool sampling_profiler_running();
+
+// Discard all captured samples and reset the bump pointer.
+void sampling_profiler_reset();
+
+// Drop the ring buffer contents to `path` as collapsed-stacks format
+// (one stack per line, semicolon-separated, trailing ' <count>'), ready
+// to feed Brendan Gregg's flamegraph.pl. Symbolicates on the fly using
+// the arena side table, ObjectFile lookup, bytecode-module scan, and
+// dladdr. Returns true on success, false on I/O error.
+bool sampling_profiler_save(const char* path);
+
+// Return one entry per recorded sample. Each inner vector holds the
+// symbolicated frame names for that sample, outermost-first (index 0
+// is the root, last index is the leaf). Prints a warning and returns
+// an empty vector if the profiler is still running.
+std::vector<SymbolicatedSample> sampling_profiler_symbolicated_samples();
+
+// Populate the calling thread's stack bounds for later frame-walking.
+// Must be called from a non-signal context. sampling_profiler_start
+// calls this automatically for the calling thread; other threads that
+// should be fully profiled need to call ext:profile-register-thread
+// (or this function) themselves once before being sampled.
+void sampling_profiler_register_current_thread();
+
+// Diagnostics.
+size_t sampling_profiler_samples_recorded();
+size_t sampling_profiler_samples_dropped();
+size_t sampling_profiler_bytes_used();
+
+} // namespace core

include/clasp/core/unwind.h

@@ -46,6 +46,7 @@ class DynEnv_O : public General_O {
  * C++ code with unknown dynamic environment, e.g. because nontrivial
  * destructors need to be run, there are catch blocks, or we simply
  * don't know. */
+FORWARD(UnknownDynEnv);
 class UnknownDynEnv_O : public DynEnv_O {
   LISP_CLASS(core, CorePkg, UnknownDynEnv_O, "UnknownDynEnv", DynEnv_O);
 

include/clasp/core/wrappedPointer.h

@@ -35,9 +35,20 @@ namespace core {
 SMART(WrappedPointer);
 class WrappedPointer_O : public core::General_O {
   FRIEND_GC_SCANNER(core::WrappedPointer_O);
-  LISP_CLASS(core, CorePkg, WrappedPointer_O, "WrappedPointer", core::General_O);
+  LISP_ABSTRACT_CLASS(core, CorePkg, WrappedPointer_O, "WrappedPointer", core::General_O);
 
 public:
+  // Untagged C++ subclasses (e.g. clbind::Wrapper<OT, HolderType>) don't
+  // have their own LISP_CLASS-generated __class() override.  Without this,
+  // virtual dispatch walks past us to General_O::__class() and instances
+  // report General_O as their class — breaking single-dispatch on methods
+  // that specialize on WrappedPointer.  _instanceClass() below returns
+  // Class_ for tagged subclasses; this provides the fallback for untagged
+  // templated subclasses that haven't set Class_.
+  virtual core::Instance_sp __class() const override {
+    return core::lisp_getStaticClass(WrappedPointer_O::static_ValueStampWtagMtag);
+  }
+
   gctools::ShiftedStamp ShiftedStamp_;
   core::Instance_sp Class_;
 

include/clasp/gctools/threadlocal.h

@@ -78,10 +78,39 @@ extern int global_debug_virtual_machine;
 #define VM_RESET_COUNTERS(vm)
 #endif
 
+// ---------- Dynamic-environment records for the bytecode interpreter ----------
+// The bytecode VM establishes dynamic environments (tagbody, catch,
+// special-bind, progv, unwind-protect) by pushing records onto a side stack
+// instead of recursing into bytecode_vm. The entering opcodes push; the
+// matching exit opcodes pop; an outer try/catch(Unwind&) in bytecode_vm walks
+// the stack to run cleanups / resume at a saved pc on non-local exits.
+//
+// Currently only the type and the stack exist — no opcodes are migrated yet.
+enum class VMDynKind : uint8_t {
+  Tagbody = 1,       // from `entry` opcode (not yet migrated)
+  Catch,             // from `catch_8/16` (not yet migrated)
+  SpecialBind,       // from `special_bind` (one per bound cell)
+  Progv,             // from `progv` (one record covers N bindings)
+  UnwindProtect,     // from `protect`
+};
+
+struct VMDynRecord {
+  VMDynKind kind;
+  uint8_t _pad[7];
+  void* frame;                      // __builtin_frame_address at establishment
+  core::T_O* slot0;                 // kind-specific GC-managed: tag / cell / cleanup closure
+  core::T_O* slot1;                 // kind-specific GC-managed: old binding value
+  core::T_O** sp_mark;              // stack pointer at establishment
+  core::T_O** fp_mark;              // frame pointer at establishment
+  unsigned char* target_pc;         // resume pc (Tagbody/Catch)
+  core::T_O* dynenv_mark;           // saved head of my_thread->dynEnvStackGet()
+};
+
 struct VirtualMachine {
   // Stack size is kind of arbitrary, and really we should make it
   // grow and etc.
   static constexpr size_t MaxStackWords = 65536;
+  static constexpr size_t MaxDynRecords = 4096;
   bool _Running;
   core::T_O** _stackBottom = nullptr;
   size_t _stackBytes;
@@ -101,6 +130,13 @@ struct VirtualMachine {
   core::T_O** _literals;
   unsigned char* _pc;
 
+  // Dynamic-environment record stack. Root-allocated so GC scans the
+  // T_O*/T_O** slots conservatively. _dynRecordTop points one past the last
+  // live record, so an empty stack has _dynRecordTop == _dynRecordBottom.
+  VMDynRecord* _dynRecordBottom = nullptr;
+  VMDynRecord* _dynRecordLimit = nullptr;
+  VMDynRecord* _dynRecordTop = nullptr;
+
   void error();
 
   void enable_guards();

include/clasp/llvmo/code.h

@@ -84,6 +84,13 @@ class ObjectFile_O : public LibraryBase_O {
   size_t _Size;
   size_t _ObjectId;
   JITDylib_sp _TheJITDylib;
+  // If true, this ObjectFile is transient arena-init scaffolding (shared
+  // trampoline / stub template) that must not be serialized into snapshots.
+  // The ObjectFile is still registered in _AllObjectFiles normally — LLVM's
+  // link layer plugin looks it up by name during materialization, so it must
+  // stay findable at runtime. The snapshot save walker checks this flag and
+  // skips any ObjectFile with it set.
+  bool _TransientSkipSnapshot = false;
   //
   // Code data
   void* _TextSectionStart;

include/clasp/llvmo/jit.h

@@ -89,6 +89,7 @@ THE SOFTWARE.
 #include <clasp/llvmo/debugInfoExpose.fwd.h>
 #include <clasp/llvmo/translators.h>
 #include <clasp/llvmo/insertPoint.fwd.h>
+#include <clasp/llvmo/llvmoExpose.h>
 #include <clasp/llvmo/debugLoc.fwd.h>
 #include <clasp/llvmo/llvmoPackage.h>
 

include/clasp/llvmo/llvmoPackage.h

@@ -68,6 +68,4 @@ ClaspJIT_sp llvm_sys__clasp_jit();
 void initialize_llvm();
 void initialize_ClaspJIT();
 
-core::Pointer_mv cmp__compile_trampoline(core::T_sp name);
-
 }; // namespace llvmo