Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,7 @@ MUSCLE3_PKG=libmuscle
{% endif %}

# main IMAS libs to be used
{% if build_info.al_version.startswith('4.') %}
IMAS_LIB_NAME=imas-cpp
{% else %}
IMAS_LIB_NAME=al-cpp
{% endif %}
CORE_LIBS_FLAGS= $(shell pkg-config --cflags $(IMAS_LIB_NAME) ymmsl $(MUSCLE3_PKG))
CORE_LIBS=$(shell pkg-config --libs $(IMAS_LIB_NAME) ymmsl $(MUSCLE3_PKG))

Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,5 @@
{% macro imports(al_version) -%}

{% if al_version.startswith('4.') %}
#include "UALClasses.h"
{% else %}
#include "ALClasses.h"
{% endif %}
{%- endmacro -%}

{% macro declare(ids_name, ids_var_name) -%}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,7 @@ MUSCLE3_PKG=libmuscle_fortran
{% endif %}

# main IMAS libs to be used
{% if build_info.al_version.startswith('4.') %}
IMAS_LIB_NAME=imas-$(FC)
{% else %}
IMAS_LIB_NAME=al-fortran
{% endif %}
CORE_LIBS_FLAGS:= $(shell pkg-config --cflags $(IMAS_LIB_NAME) ymmsl $(MUSCLE3_PKG))
CORE_LIBS:=$(shell pkg-config --libs $(IMAS_LIB_NAME) ymmsl $(MUSCLE3_PKG))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import sys
from . import actor_info
from .code_parameters import CodeParameters
from .definitions import Argument
from .runtime_settings import RuntimeSettings, DebugMode, RunMode
from .runtime_settings import RuntimeSettings, DebugMode, RunMode, SandboxLifeTime
from .definitions import Actor
from .sandbox import Sandbox
from .runners import Runner
Expand Down Expand Up @@ -39,6 +39,12 @@ class ActorBaseClass(Actor):
self.__runtime_settings = runtime_settings

self.is_standalone = self.is_standalone_run()
# Standalone actors use persistent IDS storage in the sandbox between calls.
if self.is_standalone and self.__runtime_settings.sandbox.life_time == SandboxLifeTime.ACTOR_RUN:
self.__logger.debug(
"Promoting sandbox lifetime from ACTOR_RUN to WORKFLOW_RUN for standalone actor storage."
)
self.__runtime_settings.sandbox.life_time = SandboxLifeTime.WORKFLOW_RUN
Comment thread
SimonPinches marked this conversation as resolved.
self.sandbox.initialize()
self.__initialized = True
value = func(self, *args, **kwargs )
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,20 +69,19 @@ class IdsStorageSettings:
"""Settings of temporary storage being used while passing IDSes between the actor and the code.

Attributes:
db_name (str, default='tmp'): name of the database to be used
backend (str, default=imas.ids_defs.MEMORY_BACKEND): backend to be used
persistent_backend (str, default=imas.ids_defs.MDSPLUS_BACKEND): backend to be used when temporary data
cannot be stored in memory (e.g. while running actor in a standalone mode,
when the code is run as separate process, so it doesn’t share memory with other actors.
backend (int, default=imas.ids_defs.MEMORY_BACKEND): backend to be used
persistent_backend (int, default=imas.ids_defs.HDF5_BACKEND): backend to be used when
temporary data cannot be stored in memory (e.g. while running actor in a standalone
mode, when the code is run as separate process, so it doesn't share memory with other
actors). The sandbox directory is used as the data path via an IMAS URI.
"""

# Class logger
__logger = logging.getLogger(__name__ + "." + __qualname__)

def __init__(self):
self.db_name = "tmp"
self.backend = imas.ids_defs.MEMORY_BACKEND
self.persistent_backend = imas.ids_defs.MDSPLUS_BACKEND
self.persistent_backend = imas.ids_defs.HDF5_BACKEND


class RuntimeSettings:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,48 +1,24 @@
class IDSDescription:
@property
def uri(self):
uri = (
f"imas:{self.backend_id}"
f"?user={self.user};database={self.database};version={self.version};"
f"shot={self.shot};run={self.run};idx={self.idx}"
f"#{self.ids_type}:{self.occurrence}"
)
return uri
"""Identifies a single IDS inside an IMAS data entry using URI."""

def __init__(self, db_entry, ids_name, occurrence):
self.backend_id = db_entry.backend_id
self.user = db_entry.user_name
self.database = db_entry.db_name
self.version = db_entry.dd_version[0] if db_entry.dd_version else ""
try:
self.pulse = db_entry.pulse
except AttributeError:
self.pulse = db_entry.shot
self.run = db_entry.run
self.idx = db_entry._dbe_impl._db_ctx.ctx
def __init__(self, base_uri: str, ids_name: str, occurrence: int):
self.base_uri = base_uri
self.ids_type = ids_name
self.occurrence = occurrence
self.intent = None

@property
def uri(self) -> str:
"""IMAS URI including IDS type and occurrence fragment."""
return f"{self.base_uri}#{self.ids_type}:{self.occurrence}"

def save(self, stream):
stream.write("------- IDS -------\n")
stream.write(self.ids_type)
stream.write("\n")
stream.write(str(self.pulse))
stream.write("\n")
stream.write(str(self.run))
stream.write("\n")
stream.write(str(self.occurrence))
stream.write("\n")
stream.write(str(self.backend_id))
stream.write("\n")
stream.write(str(self.idx))
stream.write("\n")
stream.write(self.database)
stream.write("\n")
stream.write(self.user)
stream.write("\n")
stream.write(self.version)
stream.write(self.base_uri)
stream.write("\n")

def convert_to_native_type(self):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,11 @@ def initialize(
storage_settings: IdsStorageSettings,
) -> None:

db_name = storage_settings.db_name
if is_standalone:
backend_id = storage_settings.persistent_backend
else:
backend_id = storage_settings.backend
self.__data_storage.initialize(sandbox_dir, db_name, backend_id)
self.__data_storage.initialize(sandbox_dir, backend_id)

def finalize(self) -> None:
self.__data_storage.finalize()
Expand Down Expand Up @@ -71,5 +70,8 @@ def convert_to_actor_type(self, ids_description: IDSDescription):
ids = self.__data_storage.read_data(ids_description)
return ids

def sync_for_external_access(self):
self.__data_storage.sync_for_external_access()

def release(self, ids_description: IDSDescription):
self.__data_storage.release_data(ids_description.ids_type)
Original file line number Diff line number Diff line change
@@ -1,20 +1,22 @@
from pathlib import Path

import imas

from .data_descriptions import IDSDescription
from .generic_storage import GenericIDSStorage

_BACKEND_NAMES = {
imas.ids_defs.MEMORY_BACKEND: "memory",
imas.ids_defs.HDF5_BACKEND: "hdf5",
imas.ids_defs.MDSPLUS_BACKEND: "mdsplus",
imas.ids_defs.ASCII_BACKEND: "ascii",
}


class LegacyIDSStorage(GenericIDSStorage):
def __init__(self):
self.__occ_dict = {}
self.__db_entry = None
self.__backend_id: int = -1
self.__db_name: str = ""
self.__pulse: int = -1
self.__run: int = -1
self.__sandbox_dir: str = ""
self.__uri: str = ""
self.__backend_name: str = ""

def __get_occurrence(self, ids_name):
occ = 1 + self.__occ_dict.get(ids_name, -1)
Expand All @@ -25,86 +27,48 @@ def __release_occurrence(self, ids_name):
occ = self.__occ_dict.get(ids_name, 1)
self.__occ_dict[ids_name] = occ - 1

def __open_db(self):
if self.__backend_id == imas.ids_defs.MEMORY_BACKEND:
return

try:
self.__db_entry.open()
except Exception as e:
raise RuntimeError(
f"Error opening the temporary DB:\n"
f" backend = {self.__backend_id}\n"
f" name = {self.__db_name}\n"
f" pulse = {self.__pulse}\n"
f" run = {self.__run}\n"
f" dir = {self.__sandbox_dir}\n"
f"Original exception: {e}"
) from e

def __close_db(self):
if self.__backend_id == imas.ids_defs.MEMORY_BACKEND:
return
self.__db_entry.close()

def initialize(self, sandbox_dir: str, db_name: str, backend_id):
def initialize(self, sandbox_dir: str, backend_id: int):
backend_name = _BACKEND_NAMES.get(backend_id)
if backend_name is None:
raise ValueError(
f"Unsupported backend_id {backend_id}. "
f"Supported backends: {list(_BACKEND_NAMES.keys())}"
)

self.__pulse = 1
self.__run = 1
self.__backend_id = backend_id
self.__db_name = db_name
self.__sandbox_dir = sandbox_dir
self.__uri = f"imas:{backend_name}?path={sandbox_dir}"
self.__backend_name = backend_name

Path(sandbox_dir, "tmp", "3", "0").mkdir(parents=True, exist_ok=True)

self.__db_entry = imas.DBEntry( # pylint: disable=no-member
backend_id=self.__backend_id, # backend_id
db_name=self.__db_name, # db_name
pulse=self.__pulse, # shot / pulse
run=self.__run, # run
user_name=self.__sandbox_dir, # AL hack to use sandbox dir
)
try:
self.__db_entry.create()
self.__db_entry = imas.DBEntry(self.__uri, "w")
except Exception as e:
raise RuntimeError(
f"Error creating the temporary DB:\n"
f" backend = {self.__backend_id}\n"
f" name = {self.__db_name}\n"
f" pulse = {self.__pulse}\n"
f" run = {self.__run}\n"
f" dir = {self.__sandbox_dir}\n"
f" uri = {self.__uri}\n"
f"Original exception: {e}"
) from e

self.__close_db()

def prepare_data(self, ids_name):
occurrence = self.__get_occurrence(ids_name)
self.__open_db()
ids_description = IDSDescription(self.__db_entry, ids_name, occurrence)
self.__close_db()
return ids_description
return IDSDescription(self.__uri, ids_name, occurrence)

def save_data(self, ids_description: IDSDescription, legacy_ids):
self.__open_db()
self.__db_entry.put(legacy_ids, ids_description.occurrence)
self.__close_db()

def read_data(self, ids_description: IDSDescription):
self.__open_db()
legacy_ids = self.__db_entry.get(
ids_description.ids_type, ids_description.occurrence
)
self.__close_db()
def sync_for_external_access(self):
# External standalone processes must own persistent backends while they run.
if self.__backend_name != "memory" and self.__db_entry is not None:
self.__db_entry.close()
self.__db_entry = None

return legacy_ids
def read_data(self, ids_description: IDSDescription):
if self.__db_entry is None:
self.__db_entry = imas.DBEntry(self.__uri, "a")
return self.__db_entry.get(ids_description.ids_type, ids_description.occurrence)

def release_data(self, ids_name):
self.__release_occurrence(ids_name)

def finalize(self):
try:
self.__db_entry.close(erase=True)
except Exception:
if self.__db_entry is not None:
self.__db_entry.close()
Comment thread
SimonPinches marked this conversation as resolved.
self.__db_entry = None
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ def run_standalone(

# prepares input files
Binder.save_input(method_name, ids_ctypes_list, param_ctype, sandbox_dir)
self.ids_converter.sync_for_external_access()
self.__logger.debug("EXECUTING command: " + str(exec_command))
exec_system_cmd(exec_command, output_stream=output_stream)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,62 +8,38 @@
class IDSCType(ctypes.Structure, IDSDescription):
_fields_ = (
("ids_type_", ctypes.c_byte * 132),
("pulse", ctypes.c_int),
("run", ctypes.c_int),
("occurrence", ctypes.c_int),
("backend_id", ctypes.c_int),
("idx", ctypes.c_int),
("database_", ctypes.c_byte * 132),
("user_", ctypes.c_byte * 132),
("version_", ctypes.c_byte * 132),
("uri_", ctypes.c_byte * 4096),
)

@property
def ids_type(self):
return "".join((chr(x) for x in self.ids_type_[:])).strip()
return bytes(self.ids_type_).split(b"\0", 1)[0].decode("ascii")

@ids_type.setter
def ids_type(self, ids_type_):
self.ids_type_[:] = len(self.ids_type_) * [ord(" ")]
self.ids_type_[: len(ids_type_)] = [ord(x) for x in ids_type_]
encoded = ids_type_.encode("ascii")
if len(encoded) >= ctypes.sizeof(self.ids_type_):
raise ValueError(f"IDS type is too long for C buffer: {ids_type_}")
ctypes.memset(self.ids_type_, 0, ctypes.sizeof(self.ids_type_))
ctypes.memmove(self.ids_type_, encoded, len(encoded))

@property
def database(self):
return "".join((chr(x) for x in self.database_[:])).strip()
def base_uri(self):
return bytes(self.uri_).split(b"\0", 1)[0].decode("ascii")

@database.setter
def database(self, database_):
self.database_[:] = len(self.database_) * [ord(" ")]
self.database_[: len(database_)] = [ord(x) for x in database_]

@property
def user(self):
return "".join((chr(x) for x in self.user_[:])).strip()

@user.setter
def user(self, user_):
self.user_[:] = len(self.user_) * [ord(" ")]
self.user_[: len(user_)] = [ord(x) for x in user_]

@property
def version(self):
return "".join((chr(x) for x in self.version_[:])).strip()

@version.setter
def version(self, version_):
self.version_[:] = len(self.version_) * [ord(" ")]
self.version_[: len(version_)] = [ord(x) for x in version_]
@base_uri.setter
def base_uri(self, uri_):
encoded = uri_.encode("ascii")
if len(encoded) >= ctypes.sizeof(self.uri_):
raise ValueError(f"URI is too long for C buffer: {uri_}")
ctypes.memset(self.uri_, 0, ctypes.sizeof(self.uri_))
ctypes.memmove(self.uri_, encoded, len(encoded))

def __init__(self, ids_description: IDSDescription):
self.ids_type = ids_description.ids_type
self.pulse = ids_description.pulse
self.run = ids_description.run
self.occurrence = ids_description.occurrence
self.backend_id = ids_description.backend_id
self.idx = ids_description.idx
self.database = ids_description.database
self.user = ids_description.user
self.version = ids_description.version
self.base_uri = ids_description.base_uri

def convert_to_native_type(self):
return ctypes.byref(self)
Expand Down
Loading