diff --git a/.gitignore b/.gitignore index 259148f..85a23ad 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,9 @@ *.exe *.out *.app + +.vscode/ +install/ +src/build*/ +build*/ + diff --git a/README.md b/docs/README_1.0.0.md similarity index 99% rename from README.md rename to docs/README_1.0.0.md index 69d8276..e4052d9 100644 --- a/README.md +++ b/docs/README_1.0.0.md @@ -154,7 +154,6 @@ $ ./install/bin/hb-list-sensors --config=SensorTable/SensorTable_ATDS.ktf { "number": 268435472, "name": "K.RTD2.Acc.AS.ATDS", "default_calibration": "degC:degC+273.15" }, ..... (continues) ``` - The sensor list can be filtered by providing (partial) name matches: ``` ### all sensors in the "Diss.AS" section ### diff --git a/docs/RELEASE_NOTES_1.1.0.md b/docs/RELEASE_NOTES_1.1.0.md new file mode 100644 index 0000000..0b08de1 --- /dev/null +++ b/docs/RELEASE_NOTES_1.1.0.md @@ -0,0 +1,383 @@ +Honeybee Upgraded Calibration Framework Release +================================================== + +## 1. Overview + +### 2.1 Architecture Overview + +![Architecture Diagram](./images/classDiagram.svg) + +![Sequence Diagram](./images/sequenceDiagram.svg) + + +- Key components and relationships +The current design is language agnostic so that in the future, other calibration engines from a Rust/python/... source would easily be able to be integrated into the system + +- abstraction point: + sensor_config is abstract so that we can define sensor_config variants that tailor to a specific calibration engine. + + Ex: derived classes specific to: + sensor_config_by_ktf + sensor_config_by_rust + sensor_config_by_py + . + . + . + . + +#### Class responsibilities: + +honeybee: main orchestrator of application, lifetime of running + + holds: Registry of all sensors, connection to data backend, configuration loader, user-provided runtime variable + + Does: Resolves sensor names to IDs, delegates to data_source for raw data fetch, returns packaged results. + + +sensor_config_by_ktf: Parse KTF files and populate sensor table + + Holds: + The Kebap parser instance that compiles all UDFs and global variables, the KTF file path being processed, user-supplied configuration variables + + Does: + Reads KTF files, extracts #% script blocks, parses Kebap code, creates sensor objects with calibration objects, registers them in sensor table + +sensor: signifies a single measurement point + + Holds: (also some metadata about a sensor) + - Sensor's unique ID, full hierarchical name path (e.g., "ATDS.Gas.Inj.Alicat.sccm") and display label + - raw calibration formula text, reference to the calibration computer + +sensor_table: Manage collection of all sensors + + Holds: Complete registry of all sensors (indexed by unique ID and searchable by hierarchical name) + Does: Store/retrieve sensors by ID or name chain. Provides lookup helpers for finding sensors by partial name match. Read-only after configuration + +data_source: calibration applied here, fetch data and apply calibration + + Holds: connection to DB, query and etc. + Does: binds to sensor table, use each sensor's calibration object to apply and return the transformed series + +calibration: Define calibration interface(Abstract) + + Holds: what the calibration does and sensor input is coming from + +kebap_calibration: Evaluate Kebap calibration expression during runtime + + Holds: Expression + it's evaluator, ktf source and the line for error sending + Does: parse expression and create evaluator when constructing, then use evaluator to transform values during runtime + + +evaluator: runtime calculator for calibration expression + + Holds: Compiled expression tree, reference to the global symbol table with all defined functions and constants + Does: evaluate expression using the referenced value and return result + + +**Main Kebap engines (external)** + +KPParser: Compile Kebap scripts into executable + + Holds: Global symbol table with all functions and constants, parsed module with variable definitions + Does: compiles script block and gives parser for expressions + +KPSymbolTable: Central namespace for all Kebap definitions + + Holds all functions and variables, and maps names to value + + + + +### 2.2 Core Extension Capabilities + +**The following are offered in the new release version 1.1.0** +- **User-defined functions and Global Variables**: + Write calibration logic in Kebap(light embedded script) without recompiling + Including: + - Functions of any-type + - global variables + +- **Modular calibration design**: + Import calibration scripts across configs +- **Dual data streams**: + Access both raw and calibrated data simultaneously. + Endpoints can be fetched as calibrated or raw in the same request. + +## 3. Usage Guide + +### 3.1 KTF Configuration Syntax + +#### 3.1.1 Unimported script + +All lines you wish to be recognized and extracted as calibration script **must start with #% and must come before the channel definitions**. + + Here is a simple example: + + #% float times5(float x) { return 5 * x; } + + later a channel endpoint could call it + +**Practical Example**: + + Assumptions: + - you are using a digital Pirani gauge model giving original indicated pressure (not accounting for specific type of gas in system) + - We have a gas system that has helium, but because our pirani may under or over report pressure without that awareness, + we will calibrate readout out before any further analytical steps + +Use this conversion graph for guidance on why our functions are the way they are for this example. + +![Calibration Reference](./images/refImage.png) + +This is what our function script section would look like above the channel definitions + +``` +**Script section** +#% float torr_He (float torr) { +#% double torr_He_val = 0.0; +#% if (torr < 1){ +#% torr_He_val = torr * 1.1; +#% } else if (torr < 10) { +#% torr_He_val = torr **0.7; +#% } else { +#% torr_He_val = torr **0.33 + 5; +#% } +#% return torr_He_val; +#% } +#% float conversion_f = 1013.25 / 760; /* Global constant: used in mbar_He function below */ +#% +#% float mbar_He (float mbar) { return torr_He(mbar / conversion_f) * conversion_f; } /* Function using global constant conversion_f */ + +{This is a simplified channel structure starting at channel definition. Please reference README.md to see an example of how a full channel structure looks.} + +**Channel definition section** +# Module: +# id: { name: prg, label: Pirani gauge } +# channel: +# id: { name: torr, label: Vacuum pressure in Torr } +# x-dripline_endpoint: +# tag: pirani +# channel: +# id: { name: mbar, label: pressure in Mbar } +# default_calibration: torr: torr * conversion_f +# channel: +# id: { name: mbarHe, label: Helium corrected pressure in Mbar } +# default_calibration: mbar: mbar_He(mbar) + +``` + +Channels and what they represent: + + 1. torr: Initial gauge pressure held in our database (Torr) + 2. mbar: Initial pressure converted(Mbar) + 3. mbarHe, Helium corrected pressure (Mbar). **Notice** it is dependent on two previous chained calibrations. + +This shows: + + 1. Utilizing User Function: + + Channels can reference and call user-defined functions without compiling the function script. The function script is extracted and parsed into Kebap before any channel calls. So each function is mapped and prepared first and can then be used for calibration, similar to a function call in programming. + Notice the input parameter must be defined and pointed out before being used. + + ```ex: + @channel 'mbarHe' + Notice ---> { **mbar**: mbar_He(**mbar**) } + ``` + + + The channel id, which has a name, identifies the output (calibrated or not) of the channel it is dependent on. + + - part of resolving and referencing dependencies in chain calling + + 2. Chaining: + Using the result/output of a channel as calibration input for another + honeybee + Kebap: resolve the dependency of channels before runtime to make this happen + + The example channels: **mbarHe** --(uses)-> **mbar** --(uses)-> **torr** (change) + + +#### 3.1.2 Global constants/variables + +Global constants and variables are defined in the script section (all starting with #%) alongside function definitions. They can be used: +- **Within function definitions** to reduce code repetition +- **In channel calibrations** to reference common conversion factors or parameters +- **Across multiple functions** for shared constants + +**Reference the example from 3.1.1 above:** + +Notice the `conversion_f` global constant defined as `#% float conversion_f = 1013.25 / 760;`. This value is then used in the `mbar_He()` function definition to convert between torr and mbar units. Later, in the channel definitions, the `mbar` channel uses this same constant: + +``` +# channel: +# id: { name: mbar, label: pressure in Mbar } +# default_calibration: torr: torr * conversion_f +``` + +#### 3.1.3 Comments + +You can comment on your calibration scripts using three comment styles: +- `//` - single line, comment until end of line +- `#!` - single line, comment until end of line +- `/* multi-line */` - comment spanning multiple lines + + +**In Unimported Scripts (with #%):** + +Comments in the script section must be prefixed with `#%`: + +``` +#% // Conversion function for helium-corrected pressure +#% float torr_He (float torr) { +#% /* Piecewise calibration factors */ +#% ... +#% } +#% +#% float conversion_f = 1013.25 / 760; #! Torr to mbar conversion factor +``` + +**In Imported Scripts (without #%):** + +When using separate imported files (see 3.1.4), comments are written normally: + +``` +// Conversion function for helium-corrected pressure +float torr_He (float torr) { + /* Piecewise calibration factors */ + ... +} + +float conversion_f = 1013.25 / 760; #! Torr to mbar conversion factor +``` + +The difference: unimported scripts require `#%` prefix for all comments that are on its own lines, while imported scripts use plain comments without the prefix. + + + + +#### 3.1.4 Importing calibration scripts + +Instead of keeping all function definitions and global constants in the same file as channel definitions, you can organize them into separate calibration files and import them. + +**NOTES** +- Imported files should have the suffix **".ktfs"**. + +- Includes are processed at parse time (during config load), not at runtime. Included functions will be immediately available to all subsequent calibration definitions. +- The symbol used for function script lines (`#%`) **IS NOT** needed in an external file, but the import statement in the main file must start with `#%`. + +**(RECOMMENDED)** +**Example - Extracting from 3.1.1:** + +**File 1: Functions.ktfs** (extracted calibration logic) +``` +// No #% markers in imported files +float torr_He (float torr) { + double torr_He_val = 0.0; + if (torr < 1){ + torr_He_val = torr * 1.1; + } else if (torr < 10) { + torr_He_val = torr **0.7; + } else { + torr_He_val = torr **0.33 + 5; + } + return torr_He_val; +} + +float conversion_f = 1013.25 / 760; + +float mbar_He (float mbar) { return torr_He(mbar / conversion_f) * conversion_f; } +``` + +**File 2: SensorTable.ktf** (main configuration with import) +``` +#% import "/path/to/Functions.ktfs"; +``` +**Everything the same as usual** +``` +# Module: +# id: { name: prg, label: Pirani gauge } +# channel: + . + . + . + +``` + +**Dual data streams**: + +Honeybee allows users to fetch multiple types of data from sensor endpoints in a single request. In our current database design, we are able to pull raw and calibrated values from the database, simultaneously fetching from the `value_raw` and `value_cal` columns. + +**Honeybee has a system-level default for what column value to extract, `value_raw`, which users are able to override using the CLI.** + +Syntax: add `--value-column=...` when running in the CLI. + +ex: ./install/bin/hb-get-data gass sccm **--value-column=value_cal** --config=/Users/nobeltsegai/Documents/CENPA/project-8/first-Mesh-Honeybee/SensorTable.ktf --from="2025-08-21T08:04:00Z" --to="2025-08-21T08:08:00Z" + +**Users are able to individually define the column level for each specified endpoint in the ktf file.** + +Examples: +Currently the system default is `value_raw`, which is raw values from the database. We will continue with this assumption for the examples below. + +In the example below, this endpoint is mapped to the system default since `field` is omitted. +``` +# channel: +# id: { name: torr, label: Vacuum pressure in Torr } +# x-dripline_endpoint: +# tag: pirani +``` + +Example of pulling raw value for pirani readout: +Note: in this instance, since the system default is `value_raw`, this is not necessary. +``` +# channel: +# id: { name: torr, label: Vacuum pressure in Torr } +# x-dripline_endpoint: +# tag: pirani +# field: value_raw +``` + +Example of retrieving calibrated values for the pirani readout: +``` +# channel: +# id: { name: torr, label: Vacuum pressure in Torr } +# x-dripline_endpoint: +# tag: pirani +# field: value_cal +``` + +You could also have multiple channels from the same endpoint (e.g., pirani) but with different field values. + +**Honeybee's Column selection hierarchy**: +The priority order is: +Sensor-level explicit field (dripline_endpoint_field in KTF config) + Requested/overridden default column (--value-column=... from CLI) + Application fallback (current default behavior when no override is provided) + + +### 3.2 Additional Notes: + +- User defined function and global variables can be used within the function script just like a regular programming language + + --> function calling functions + --> Variables being called within function + +- Current Issues: + + - (Kebap) Lack of (true) line number propagation when encountering a bug from imported code + In the case of an error from code in the imported file, users get only the relative file line number. This is where the buggy function or variable is being called in the main KTF file. + - Currently Kebap lacks the functionality to carry sourcefile metadata for imports + - Temporary solution: user can still use their search command to find function/variable + +Resolving bugs: + +Things to check when running into issue: + +- Please make sure your Docker compose file is set up properly + matching port, access and etc. + +- Getting nan value readout: + - check the error output and see if its a post data extraction error(calibration stage or ...) + + - if there is no error output in command line, please make sure your time window for db query data is correct + To check, open up a session into your database and check + SELECT MIN(timestamp), MAX(timestamp) FROM {table_name}; + +- Ensure your binary is up-to-date and not lagging behind an older version, and follow README instructions for proper build instructions. \ No newline at end of file diff --git a/docs/images/classDiagram.svg b/docs/images/classDiagram.svg new file mode 100644 index 0000000..dbda9c9 --- /dev/null +++ b/docs/images/classDiagram.svg @@ -0,0 +1,102 @@ +

owns

owns

owns

inherits

owns

fills

creates

creates

populates f_calibration_table

calls ExecuteBareStatements()

owns

owns

inherits

owns

owns

borrows (uses)

owns

owns

creates via ExpressionParser

uses %%query sensors

aggregates in f_calibration_table

1
1
1
1
1
1
1
1
1
1
1
1
*
1
*
1
1
*
0..1
1
1
1
1
*

honeybee_app

- shared_ptr<sensor_table> f_sensor_table

- shared_ptr<data_source> f_data_source

- map~string, shared_ptr<sensor_config~> f_loaders

- variables f_variables

+ add_config_file(filepath)

+ get_sensor_table()

+ get_data_source()

«abstract»

sensor_config

+ load(sensor_table&, filepath) : void

+ create_calibration(sensor&, sensor_table&) : shared_ptr<calibration>

sensor_config_by_ktf

- shared_ptr f_parser

- string f_ktf_path

- variables f_variables

+ load(sensor_table&, filepath) : void

+ create_calibration(sensor&, sensor_table&) : shared_ptr<calibration>

- extract_scripts() : string

- compile_expression(expr_text) : KPExpression

- load_layer(...)

- add_sensor(...)

+ set_variables(...)

sensor_table

- map<int, sensor> f_table

- unordered_map<string, int> f_reverse_table

+ operator[](int) : sensor&

+ find_like(name_chain) : vector<int>

+ add(sensor) : void

sensor

- int f_number

- name_chain f_name

- name_chain f_label

- string f_calibration %% exp txt

- shared_ptr<calibration> f_calibration_obj %% new ownership

+ get_calibration_object() : shared_ptr<calibration>

+ get_number() : int

+ get_name() : name_chain

+ get_calibration() : string

+ set_calibration_object(cal) : void %% needed for cal ownership

+ apply_calibration(x) : double %% needed for ...

«abstract»

calibration

# string f_description

# int f_input %% sensor id

# bool f_is_identity

+ operator()(double x) : double %% changed to virtual

+ get_input_sensor() : int

+ get_description() : string

+ get_error_context() : string %% for error, virtual

kebap_calibration

- string f_ktf_path

- int f_line_offset

- string f_expression_text

- shared_ptr<evaluator> f_evaluator

+ kebap_calibration(sensor&, sensor_table&, KPStandardParser*, string, int)

+ operator()(double x) : double %%overrides

+ get_error_context() : string %%overrides

evaluator

- KPExpression* f_expression

- KPSymbolTable* f_symbol_table %% burrow from parser

+ evaluator(KPExpression*, KPSymbolTable*) : %% new constructor

+ ~evaluator() : %% can delete expression, sym_table not owned

+ operator()(double x) : double %% changes

«abstract»

data_source

- map~int, shared_ptr<calibration~> f_calibration_table

+ bind(sensor_table&) : void %% changing, not making calibration

+ read(sensors, from, to, ...) : vector<series> %% changing

+ apply_calibration(int sensor_id, series&) : void

+ find_input(int sensor_id) : int

# bind_inputs(sensor_table&) : void

# fetch(...) : vector<series>

KPStandardParser

- KPSymbolTable* fSymbolTable

- KPExpressionParser* fExpressionParser

- KPCxxModule* fModule

+ Parse(istream&) : void

+ GetSymbolTable() : KPSymbolTable

+ GetExpressionParser() : KPExpressionParser

KPExpression

+ Evaluate(KPSymbolTable*) : KPValue

KPSymbolTable

+ SetVariable(name, value) : void

+ GetVariable(name) : KPValue

KPCxxModule

- fBareStatementList

- fEntryTable

+ ExecuteBareStatements(KPSymbolTable*) : void

+ Execute(...) : KPValue

+ GetEntry(name) : KPModuleEntry

\ No newline at end of file diff --git a/docs/images/refImage.png b/docs/images/refImage.png new file mode 100644 index 0000000..fab1dad Binary files /dev/null and b/docs/images/refImage.png differ diff --git a/docs/images/sequenceDiagram.svg b/docs/images/sequenceDiagram.svg new file mode 100644 index 0000000..e9947bc --- /dev/null +++ b/docs/images/sequenceDiagram.svg @@ -0,0 +1,102 @@ +KPCxxModuleKPExpressionevaluatorkebap_calibrationdata_sourcesensorsensor_tableKPStandardParser (per KTF)sensor_config_by_ktf (per KTF)honeybee_appKPCxxModuleKPExpressionevaluatorkebap_calibrationdata_sourcesensorsensor_tableKPStandardParser (per KTF)sensor_config_by_ktf (per KTF)honeybee_appLoad phase (per KTF) and builidng thE PARSERcreating sensor-table: pt 1BUILDING PARSERbuilding calibrationBind data source (no calibration creation)Calibration creation (per sensor)Runtime readapplying recursivelyadd_config_file(ktf_path)1create loader, store in f_loaders[ktf_path]2extract <script> blocks3new KPStandardParser()4Parse(<script>)5build symbol table (UDF)6ExecuteBareStatements(SymbolTable), register global var7add(sensor) with calib text + provenance8sensor refs (id, name_chain)9new kebap_calibration(sensor, TABLE, PARSER, ktf_path, line_offset) %% update creation10GetExpressionParser()->Parse(expr_text)11KPExpression* (EXPR)12new evaluator(EXPR, PARSER.GetSymbolTable())13evaluator*14set_calibration_object(CAL)15bind(TABLE)16discover sensors(get all of them)17extract calbration object(**get_cal_obj()** )18populate f_calibration_table19bind_input (endpoint binding)20ready with calibration table populated21read(sensor_id, from, to)22find_input(), recursive dependecy resolver23fetch raw series24apply_calibration(sensor_id, series)25recursive: apply_calibration(input_sensor, series) first26get calibration for sensor_id27operator()(value) [for each value in series]28operator()(raw_value)29Evaluate(symbol_table)30KPValue (or throw)31on KPException, add context "Error in expression at line <n>"32add context "<ktf_path>:<line> expr: <text>"33propagate enriched error34propagate to user35 \ No newline at end of file diff --git a/src/Honeybee/Applications/hb-get-data.cxx b/src/Honeybee/Applications/hb-get-data.cxx index fe4f8bf..82a5b9f 100644 --- a/src/Honeybee/Applications/hb-get-data.cxx +++ b/src/Honeybee/Applications/hb-get-data.cxx @@ -7,6 +7,7 @@ #include #include #include "honeybee.hh" +#include "error_logger.hh" namespace hb = honeybee; @@ -31,7 +32,11 @@ int main(int argc, char** argv) std::cerr << " --delimiter=VALUE set channel name delimiter"<< std::endl; std::cerr << " --delimiter-input=VALUE set channel name delimiter in the data store"<< std::endl; std::cerr << " --delimiter-output=VALUE set channel name delimiter for output"<< std::endl; + std::cerr << " --value-column=NAME default data column (value_raw/value_cal/etc)"<< std::endl; std::cerr << " --verbose make it verbose"<< std::endl; + std::cerr << " --log-mode=MODE all|first|counted|summary"<< std::endl; + std::cerr << " --log-metadata enable metadata/stage logger output"<< std::endl; + std::cerr << " --no-log-metadata disable metadata/stage logger output"<< std::endl; return -1; } @@ -45,6 +50,7 @@ int main(int argc, char** argv) std::string t_delimiter = args["--delimiter"].Or(""); std::string t_delimiter_input = args["--delimiter-input"].Or(t_delimiter); std::string t_delimiter_output = args["--delimiter-output"].Or(t_delimiter.substr(0,1)); + std::string t_value_column = args["--value-column"].Or(""); double t_to_ts = args["--to-ts"].Or(long(hb::datetime::now())); std::string t_to = args["--to"].Or(hb::datetime(t_to_ts).as_string()); @@ -86,6 +92,28 @@ int main(int argc, char** argv) hb::g_log_level = hb::e_log_level_debug; } + auto& t_logger = hb::error_logger::instance(); + std::string t_log_mode = args["--log-mode"].Or(""); + if (t_log_mode == "all") { + t_logger.set_message_mode(hb::error_logger::e_message_all); + } + else if (t_log_mode == "first") { + t_logger.set_message_mode(hb::error_logger::e_message_first_only); + } + else if (t_log_mode == "counted") { + t_logger.set_message_mode(hb::error_logger::e_message_first_with_count); + } + else if (t_log_mode == "summary") { + t_logger.set_message_mode(hb::error_logger::e_message_summary); + } + + if (! args["--log-metadata"].IsVoid()) { + t_logger.set_metadata_enabled(true); + } + if (! args["--no-log-metadata"].IsVoid()) { + t_logger.set_metadata_enabled(false); + } + //// Fetch //// @@ -93,6 +121,9 @@ int main(int argc, char** argv) t_honeybee_app.add_config_file(t_config_file); t_honeybee_app.add_dripline_db(t_dripline_db); t_honeybee_app.set_delimiter(t_delimiter_input, t_delimiter_output); + if (! t_value_column.empty()) { + t_honeybee_app.set_value_column_default(t_value_column); + } for (auto& variable: t_variables) { t_honeybee_app.add_variable(variable.first, variable.second); } @@ -141,6 +172,8 @@ int main(int argc, char** argv) } std::cout << std::endl << "}" << std::endl; + t_logger.create_summary(); + return 0; } @@ -201,6 +234,8 @@ int main(int argc, char** argv) else if (t_series_bundle.size() == 1) { std::cout << t_series_bundle[0].to_csv(t_series_bundle.keys()[0]); } + + t_logger.create_summary(); return 0; } diff --git a/src/Honeybee/Applications/hb-list-sensors.cxx b/src/Honeybee/Applications/hb-list-sensors.cxx index 7a051f8..663b71f 100644 --- a/src/Honeybee/Applications/hb-list-sensors.cxx +++ b/src/Honeybee/Applications/hb-list-sensors.cxx @@ -3,9 +3,11 @@ #include #include +#include #include #include #include "honeybee.hh" +#include "error_logger.hh" namespace hb = honeybee; @@ -28,6 +30,9 @@ int main(int argc, char** argv) std::cerr << " --delimiter-input=VALUE set channel name delimiter in the data store"<< std::endl; std::cerr << " --delimiter-output=VALUE set channel name delimiter for output"<< std::endl; std::cerr << " --verbose make it verbose"<< std::endl; + std::cerr << " --log-mode=MODE all|first|counted|summary"<< std::endl; + std::cerr << " --log-metadata enable metadata/stage logger output"<< std::endl; + std::cerr << " --no-log-metadata disable metadata/stage logger output"<< std::endl; return -1; } @@ -68,6 +73,28 @@ int main(int argc, char** argv) if (! args["--verbose"].IsVoid()) { hb::g_log_level = hb::e_log_level_info; } + + auto& t_logger = hb::error_logger::instance(); + std::string t_log_mode = args["--log-mode"].Or(""); + if (t_log_mode == "all") { + t_logger.set_message_mode(hb::error_logger::e_message_all); + } + else if (t_log_mode == "first") { + t_logger.set_message_mode(hb::error_logger::e_message_first_only); + } + else if (t_log_mode == "counted") { + t_logger.set_message_mode(hb::error_logger::e_message_first_with_count); + } + else if (t_log_mode == "summary") { + t_logger.set_message_mode(hb::error_logger::e_message_summary); + } + + if (! args["--log-metadata"].IsVoid()) { + t_logger.set_metadata_enabled(true); + } + if (! args["--no-log-metadata"].IsVoid()) { + t_logger.set_metadata_enabled(false); + } //// Construction //// @@ -104,5 +131,7 @@ int main(int argc, char** argv) } std::cout << std::endl << "]" << std::endl; + t_logger.create_summary(); + return 0; } diff --git a/src/Honeybee/Source/CMakeLists.txt b/src/Honeybee/Source/CMakeLists.txt index e58d124..0595cf9 100644 --- a/src/Honeybee/Source/CMakeLists.txt +++ b/src/Honeybee/Source/CMakeLists.txt @@ -4,9 +4,13 @@ add_library(HoneybeeLib STATIC data_source.cc pgsql.cc sensor_table.cc + sensor_config.cc + sensor_config_by_ktf.cc + kebap_calibration.cc series.cc utils.cc evaluator.cc + error_logger.cc ) set(MyPublicHeaders @@ -15,9 +19,13 @@ set(MyPublicHeaders data_source.hh pgsql.hh sensor_table.hh + sensor_config.hh + sensor_config_by_ktf.hh + kebap_calibration.hh series.hh utils.hh evaluator.hh + error_logger.hh ) target_compile_features(HoneybeeLib PRIVATE cxx_std_14) diff --git a/src/Honeybee/Source/calibration.cc b/src/Honeybee/Source/calibration.cc index 6220311..69f32ea 100644 --- a/src/Honeybee/Source/calibration.cc +++ b/src/Honeybee/Source/calibration.cc @@ -7,91 +7,10 @@ #include #include -#include -#include "sensor_table.hh" -#include "evaluator.hh" #include "calibration.hh" - using namespace std; using namespace honeybee; +// Abstract base class - implementations in subclass -calibration::calibration(const sensor& a_sensor, const sensor_table& a_sensor_table) -{ - auto strip = [](const string& a_text)->string { - string::size_type t_begin = 0, t_length = a_text.size(); - while (t_begin < t_length) { - if (a_text[t_begin] != ' ') { - break; - } - t_begin++; - } - while (t_length > t_begin) { - if (a_text[t_length-1] != ' ') { - break; - } - t_length--; - } - return a_text.substr(t_begin, t_length); - }; - - f_description = strip(a_sensor.get_calibration()); - f_is_identity = false; - f_input = sensor{}.get_number(); - f_evaluator = 0; - if (f_description.empty()) { - return; - } - - auto colon = f_description.find_first_of(':'); - f_variable_name = strip(f_description.substr(0, colon)); - string t_exp_text; - if (colon != string::npos) { - t_exp_text = strip(f_description.substr(colon+1)); - } - - name_chain t_input_name_chain = a_sensor.get_name(); - name_chain t_variable_name_chain = name_chain(f_variable_name, "./-_"); - if (t_input_name_chain.size() < t_variable_name_chain.size()) { - t_input_name_chain = t_variable_name_chain; - } - else { - for (unsigned i = 0; i < t_variable_name_chain.size(); i++) { - t_input_name_chain[i] = t_variable_name_chain[i]; - } - } - auto t_candidates = a_sensor_table.find_like(t_input_name_chain); - if (t_candidates.size() == 1) { - f_input = t_candidates.front(); - } - else if (t_candidates.size() > 0) { - cerr << "ERROR: ambiguous calibration input: " << f_variable_name << endl; - return; - } - else { - f_input = a_sensor_table[t_variable_name_chain]; - } - if (! f_input) { - cerr << "ERROR: unable to find calibration input: " << f_variable_name << endl; - return; - } - - if ((f_variable_name == t_exp_text) || t_exp_text.empty()) { - f_is_identity = true; - return; - } - - // replace the variable in the expression with "x" - string t_pattern = regex_replace(f_variable_name, regex("\\."), "\\."); - t_exp_text = regex_replace(t_exp_text, regex("(^|[^a-zA-Z_])(" + t_pattern + ")($|[^a-zA-Z0-9_])"), "$1x$3"); - - f_evaluator = make_shared(t_exp_text); - try { - f_evaluator->operator()(0); - } - catch (std::exception &e) { - cerr << "ERROR: bad calibration expression: " << e.what() << endl; - f_evaluator = 0; - } -} diff --git a/src/Honeybee/Source/calibration.hh b/src/Honeybee/Source/calibration.hh index 7eb6ce4..7ab6cf2 100644 --- a/src/Honeybee/Source/calibration.hh +++ b/src/Honeybee/Source/calibration.hh @@ -11,33 +11,30 @@ #include #include #include "series.hh" -#include "sensor_table.hh" -#include "evaluator.hh" namespace honeybee { using namespace std; + + class sensor; + class sensor_table; class calibration { public: - calibration(): f_is_identity(false) {} - calibration(const sensor& a_sensor, const sensor_table& a_sensor_table); + calibration() : f_input(0), f_is_identity(false) {} + virtual ~calibration() = default; + int get_input_sensor() const { return f_input; } string get_description() const { return f_description; } - double operator()(double x) const { - if (f_is_identity) { - return x; - } - if (! f_evaluator) { - return std::numeric_limits::quiet_NaN(); - } - return (*f_evaluator)(x); - } + + virtual double operator()(double x) = 0; + virtual string get_error_context() const = 0; + protected: - string f_description, f_variable_name; + string f_description; + string f_variable_name; int f_input; bool f_is_identity; - shared_ptr f_evaluator; }; } diff --git a/src/Honeybee/Source/data_source.cc b/src/Honeybee/Source/data_source.cc index 49531c2..7986c01 100644 --- a/src/Honeybee/Source/data_source.cc +++ b/src/Honeybee/Source/data_source.cc @@ -9,9 +9,12 @@ #include #include #include +#include #include #include "sensor_table.hh" +#include "sensor_config.hh" #include "pgsql.hh" +#include "error_logger.hh" #include "data_source.hh" using namespace std; @@ -36,34 +39,46 @@ static string sanitize(const string& text, const string& pattern=R"([a-zA-Z0-9_] + void data_source::bind(sensor_table& a_sensor_table) { - hINFO(cerr << "Calibration Chain:" << endl); + // NOTE: Calibration objects are now created and attached directly when loading KTF files + // via sensor_config_by_ktf and kebap_calibration. + hINFO(cerr << "Calibration Chain (from sensor attached objects):" << endl); for (int t_sensor_number: a_sensor_table.find_like({{}})) { auto& t_sensor = a_sensor_table[t_sensor_number]; if (t_sensor.get_calibration().empty()) { continue; } - f_calibration_table[t_sensor_number] = calibration(t_sensor, a_sensor_table); hINFO(cerr - << " " << t_sensor.get_name().join(".") << " <= " - << a_sensor_table[f_calibration_table[t_sensor_number].get_input_sensor()].get_name().join(".") << " : " - << f_calibration_table[t_sensor_number].get_description() << endl + << " " << t_sensor.get_name().join(".") << " : " + << t_sensor.get_calibration() << endl ); + + // Store calibration object in f_calibration_table for orchestration + auto t_calib_obj = t_sensor.get_calibration_object(); + if (t_calib_obj) { + f_calibration_table[t_sensor_number] = t_calib_obj; + } } this->bind_inputs(a_sensor_table); } -vector data_source::read(const vector& a_sensor_list, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) +vector data_source::read(const vector& a_sensor_list, const std::string& value_column, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) { + + + // Find the base (input) sensors for each requested sensor, resolve dependencies vector t_input_sensor_list; for (unsigned i = 0; i < a_sensor_list.size(); i++) { t_input_sensor_list.emplace_back(find_input(a_sensor_list[i])); } - vector t_series_list = this->fetch(t_input_sensor_list, a_from, a_to, a_resampling_interval, a_reducer); + // Fetch data from base sensors + vector t_series_list = this->fetch(t_input_sensor_list, a_from, a_to, a_resampling_interval, a_reducer, value_column); + // Apply calibrations for each requested sensor for (unsigned i = 0; i < a_sensor_list.size(); i++) { apply_calibration(a_sensor_list[i], t_series_list[i]); } @@ -71,6 +86,7 @@ vector data_source::read(const vector& a_sensor_list, double a_from return t_series_list; } + int data_source::find_input(int a_sensor) { auto iter = f_calibration_table.find(a_sensor); @@ -79,34 +95,44 @@ int data_source::find_input(int a_sensor) } const auto& t_calib = iter->second; - return this->find_input(t_calib.get_input_sensor()); + return this->find_input(t_calib->get_input_sensor()); } + void data_source::apply_calibration(int a_sensor, series& a_series) { + // Recursive orchestration: applies calibrations from input sensor up through the dependency chain auto iter = f_calibration_table.find(a_sensor); if (iter == f_calibration_table.end()) { - return; + return; // Base sensor - no calibration } + const auto& t_calib = iter->second; - this->apply_calibration(t_calib.get_input_sensor(), a_series); + // Recursively apply calibration to input sensor first + this->apply_calibration(t_calib->get_input_sensor(), a_series); - for (auto& xk: a_series.x()) { - xk = t_calib(xk); + // Then apply this sensor's calibration to all values + try { + for (auto& xk: a_series.x()) { + xk = t_calib->operator()(xk); + } + } + catch (exception& e) { + throw runtime_error(string("Sensor ID ") + to_string(a_sensor) + ": " + e.what()); } - hINFO(cerr << "Calibration: " << endl); - hINFO(cerr << " " << t_calib.get_description() << endl); + hINFO(cerr << "Calibration: " << t_calib->get_description() << endl); } -vector data_source::fetch(const vector& a_sensor_list, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) + +vector data_source::fetch(const vector& a_sensor_list, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer, const std::string& value_column) { // default implemantation, might be overriden as needed // vector t_series_list; for (auto& t_sensor: a_sensor_list) { t_series_list.emplace_back(a_from, a_to); - fetch_single(t_series_list.back(), t_sensor, a_from, a_to, a_resampling_interval, a_reducer); + fetch_single(t_series_list.back(), t_sensor, a_from, a_to, a_resampling_interval, a_reducer, value_column); } return t_series_list; @@ -182,20 +208,22 @@ void dripline_pgsql::bind_inputs(sensor_table& a_sensor_table) // 3: make a Dripline endpoint table hINFO(cerr << "Dripline Endpoint Binding: " << endl); set t_endpoint_list(t_dripline_names.begin(), t_dripline_names.end()); - for (int t_number: a_sensor_table.find_like({{}})) { + for (int t_number: a_sensor_table.find_like({{}})) { // --> getting all sensors const sensor& t_sensor = a_sensor_table[t_number]; - string t_endpoint = t_sensor.get_option("dripline_endpoint", ""); + string t_endpoint = t_sensor.get_option("dripline_endpoint", ""); + string t_field = t_sensor.get_option("dripline_endpoint_field", ""); // otherwise keep empty so the app default can apply later + if (t_endpoint_list.count(t_endpoint) > 0) { - f_endpoint_table[t_number] = t_endpoint; + f_endpoint_n_field_table[t_number] = {t_endpoint, t_field}; //---> HERE, STORES THE ENDPOINT MAPPING, like 131 --> {name, field pref(calibrated or raw)} hINFO(cerr << " " << t_endpoint << " => " << t_sensor.get_name().join(f_output_delimiter) << endl); } } } -void dripline_pgsql::fetch_single(series& a_series, int a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) +void dripline_pgsql::fetch_single(series& a_series, int a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer, const std::string& value_column) { try { - auto t_series_list = this->fetch({{a_sensor}}, a_from, a_to, a_resampling_interval, a_reducer); + auto t_series_list = this->fetch({a_sensor}, a_from, a_to, a_resampling_interval, a_reducer, value_column); //added for calibration if (t_series_list.size() == 1) { a_series = std::move(t_series_list[0]); } @@ -205,33 +233,87 @@ void dripline_pgsql::fetch_single(series& a_series, int a_sensor, double a_from, } } -vector dripline_pgsql::fetch(const vector& a_sensor_list, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) +vector dripline_pgsql::fetch(const vector& a_sensor_list, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer, const std::string& value_column) { + //seperating endpoints name based on their data type pref using the f_endpoint_n_field_table + vector t_series_list; + auto& t_logger = error_logger::instance(); + + map>> t_column_index_table; + map> t_column_target_sets; + vector t_valid_columns = f_pgsql.get_column_list("numeric_data"); + // --verbose, in that case you can return, invalid default data column + // better to have a nan output rather than a crash + + // so for running, right now, all the metedata that user gets before the actual data + // should also be option within the logger class + + // for a repetitive error message, you can have a cache of error messages and when an error comes + // based on tracked printed error: + + // These can be options within verbose and error + //printing all errors, + //first lelvel: do not repeat the error mess + // second: have a count of haw many times times an error message appear, message -> count mapping + // last: summary + + // having a class to do all this pre-processing error logging, and use that in these situation , a global instance. + bool valid_col = find(t_valid_columns.begin(), t_valid_columns.end(), value_column) != t_valid_columns.end(); + if (!valid_col) { + t_logger.warn( + "data_source", + "invalid_default_column", + string("invalid default data column '") + value_column + "'; returning NaN series" + ); + + for (auto t_sensor: a_sensor_list) { + t_series_list.emplace_back(a_from, a_to); + t_series_list.back().emplace_back(a_from, numeric_limits::quiet_NaN()); + } + return t_series_list; + } - map> t_series_index_table; - string t_targets; for (auto t_sensor: a_sensor_list) { - auto iter = f_endpoint_table.find(t_sensor); - if (iter != f_endpoint_table.end()) { - if (t_series_index_table.count(iter->second) == 0) { - t_targets += (t_targets.empty() ? "'" : ",'") + iter->second + "'"; - } - t_series_index_table[iter->second].push_back(t_series_list.size()); - } t_series_list.emplace_back(a_from, a_to); + auto iter = f_endpoint_n_field_table.find(t_sensor); + if (iter != f_endpoint_n_field_table.end()) { + const string& endpoint = iter->second.first; // endpoint_name + + // indentify column of endpoint, use default if not specified + string field = iter->second.second; // field pref + string t_column = field.empty() ? value_column : field; + + t_column_target_sets[t_column].insert(endpoint); + t_column_index_table[t_column][endpoint].push_back(t_series_list.size() - 1); + } + } + + for (const auto& t_group: t_column_index_table) { + string t_targets; + for (const auto& t_endpoint: t_column_target_sets[t_group.first]) { + t_targets += (t_targets.empty() ? "'" : ",'") + t_endpoint + "'"; + } + fetch_column(t_series_list, t_group.second, t_targets, t_group.first, a_from, a_to, a_resampling_interval, a_reducer); } - if (t_targets.empty()) { - return t_series_list; + return t_series_list; +} + +void dripline_pgsql::fetch_column(vector& a_series_list, const map>& a_endpoint_index_table, const string& a_targets, const string& a_column, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) +{ + auto& t_logger = error_logger::instance(); + + if (a_targets.empty()) { + return; } - + string t_sql; { string date_from = datetime(a_from).as_string() + "Z"; string date_to = datetime(a_to).as_string() + "Z"; string tag = f_sensorname_column; - string tag_values = t_targets; - string field = "value_raw"; + string tag_values = a_targets; + string field = a_column; string bucket = std::to_string(a_resampling_interval); string to = std::to_string(a_to); @@ -343,7 +425,7 @@ vector dripline_pgsql::fetch(const vector& a_sensor_list, double a_ + " timestamp asc" ); } -#else +#else // Nobel: current direct query path t_sql = (string("") + "SELECT" + " extract(epoch from timestamp), " + tag + ", " + field + " " @@ -355,41 +437,43 @@ vector dripline_pgsql::fetch(const vector& a_sensor_list, double a_ + "ORDER BY" + " timestamp asc" ); -#endif - } - - hINFO(cerr << "SQL: " << endl); - hINFO(cerr << " " << t_sql << endl); - double time; - map>::iterator t_channel_iter; - auto t_handler = [&](int a_row, int a_col, const char* a_value) { - if (a_col == 0) { - time = stod(a_value); - } - else if (a_col == 1) { - t_channel_iter = t_series_index_table.find(a_value); - } - else { - for (unsigned index: t_channel_iter->second) { - t_series_list[index].emplace_back(time, stod(a_value)); + hINFO(cerr << "SQL: " << endl); + hINFO(cerr << " " << t_sql << endl); + + double time; + map>::const_iterator t_channel_iter; + auto t_handler = [&](int a_row, int a_col, const char* a_value) { + if (a_col == 0) { + time = stod(a_value); } + else if (a_col == 1) { + t_channel_iter = a_endpoint_index_table.find(a_value); + } + else { + for (unsigned index: t_channel_iter->second) { + a_series_list[index].emplace_back(time, stod(a_value)); + } + } + }; + if (f_pgsql.query(t_sql, t_handler) < 0) { + t_logger.error( + "db", + "db_query_error", + string("DB Query Error: SQL: ") + t_sql + ); + throw std::runtime_error("DB Query Error: SQL: " + t_sql); } - }; - if (f_pgsql.query(t_sql, t_handler) < 0) { - throw std::runtime_error("DB Query Error: SQL: " + t_sql); +#endif } - - return t_series_list; } - vector csv_file::get_data_names() { return vector(); } -void csv_file::fetch_single(series& a_series, int a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) +void csv_file::fetch_single(series& a_series, int a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer, const std::string& value_column) { -} +} \ No newline at end of file diff --git a/src/Honeybee/Source/data_source.hh b/src/Honeybee/Source/data_source.hh index 44d6b59..b1d5e5e 100644 --- a/src/Honeybee/Source/data_source.hh +++ b/src/Honeybee/Source/data_source.hh @@ -10,12 +10,20 @@ #include #include +#include +#include #include "utils.hh" #include "series.hh" #include "sensor_table.hh" #include "calibration.hh" #include "pgsql.hh" +// Forward declarations for Kebap integration +namespace kebap { + class KPStatement; + class KPFunction; + class KPStandardParser; +} namespace honeybee { using namespace std; @@ -26,16 +34,16 @@ namespace honeybee { virtual ~data_source() {} virtual vector get_data_names() = 0; virtual void bind(sensor_table& a_sensor_table); - virtual vector read(const vector& a_sensor_list, double a_from, double a_to, double a_resampling_interval=-1, const std::string& a_reducer=""); + virtual vector read(const vector& a_sensor_list, const std::string& value_column, double a_from, double a_to, double a_resampling_interval=-1, const std::string& a_reducer=""); protected: virtual void bind_inputs(sensor_table& sensor_table) = 0; - virtual vector fetch(const vector& a_sensor_list, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer); - virtual void fetch_single(series& a_series, int a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) = 0; + virtual vector fetch(const vector& a_sensor_list, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer, const std::string& value_column); + virtual void fetch_single(series& a_series, int a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer, const std::string& value_column) = 0; protected: int find_input(int); void apply_calibration(int a_sensor, series& a_series); protected: - map f_calibration_table; + map> f_calibration_table; }; @@ -44,7 +52,7 @@ namespace honeybee { vector get_data_names() override { return vector(); } protected: void bind_inputs(sensor_table& sensor_table) override {} - void fetch_single(series& a_series, int a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) override {} + void fetch_single(series& a_series, int a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer, const std::string& value_column) override {} }; @@ -54,15 +62,16 @@ namespace honeybee { vector get_data_names() override; protected: void bind_inputs(sensor_table& a_sensor_table) override; - vector fetch(const vector& a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) override; - void fetch_single(series& a_series, int a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) override; + vector fetch(const vector& a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer, const std::string& value_column) override; // support selecting raw or calibrated values + void fetch_single(series& a_series, int a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer, const std::string& value_column) override; protected: + void fetch_column(vector& a_series_list, const map>& a_endpoint_index_table, const string& a_targets, const string& a_column, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer); string f_db_uri; vector f_basename; string f_input_delimiters, f_output_delimiter; protected: - pgsql f_pgsql; - map f_endpoint_table; + pgsql f_pgsql; + map> f_endpoint_n_field_table; // mapping of sensor_id to (endpoint, field) vector f_data_names; protected: bool f_has_idmap; @@ -74,7 +83,7 @@ namespace honeybee { public: vector get_data_names() override; protected: - void fetch_single(series& a_series, int a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) override; + void fetch_single(series& a_series, int a_sensor, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer, const std::string& value_column) override; protected: map f_column_map; }; diff --git a/src/Honeybee/Source/error_logger.cc b/src/Honeybee/Source/error_logger.cc new file mode 100644 index 0000000..a62fd5d --- /dev/null +++ b/src/Honeybee/Source/error_logger.cc @@ -0,0 +1,198 @@ +/* +* error_logger.cc +*/ + +#include +#include "error_logger.hh" + +using namespace honeybee; +using namespace std; + +error_logger& error_logger::instance() { + static error_logger t_logger; + return t_logger; +} + +error_logger::error_logger() : f_message_mode(e_message_first_with_count), f_metadata_enabled(false) {} + +bool error_logger::should_print(log_level_t a_level) const { + return a_level <= g_log_level; +} + +bool error_logger::is_cache_target(log_level_t a_level) const { + // caching warn or worst + return a_level <= e_log_level_warn; +} + +void error_logger::clear() { // clean up + f_records.clear(); + f_order.clear(); +} + +string error_logger::make_key(log_level_t a_level, const string& a_category, const string& a_error_id) const { + return to_string(a_level) + "|" + a_category + "|" + a_error_id; +} + +void error_logger::error(const string& a_category, const string& a_error_id, const string& a_message) { + log(e_log_level_error, a_category, a_error_id, a_message); +} + +void error_logger::warn(const string& a_category, const string& a_error_id, const string& a_message) { + log(e_log_level_warn, a_category, a_error_id, a_message); +} + +void error_logger::info(const string& a_category, const string& a_error_id, const string& a_message) { + log(e_log_level_info, a_category, a_error_id, a_message); +} + +void error_logger::debug(const string& a_category, const string& a_error_id, const string& a_message) { + log(e_log_level_debug, a_category, a_error_id, a_message); +} + + +// check the visiblity, and then do dedup lookup +void error_logger::log(log_level_t a_level, const string& a_category, const string& a_error_id, const string& a_message) +{ + if (! should_print(a_level)) { + return; + } + + if (! is_cache_target(a_level) || (f_message_mode == e_message_all)) { + emit_line(cerr, a_level, a_category, a_error_id, a_message); + return; + } + + // key = level + category + error_id, and check for repetition + string t_key = make_key(a_level, a_category, a_error_id); + auto t_iter = f_records.find(t_key); + if (t_iter == f_records.end()) { + message_record t_record; + t_record.level = a_level; + t_record.category = a_category; + t_record.error_id = a_error_id; + t_record.last_message = a_message; + t_record.count = 1; + f_records[t_key] = t_record; + f_order.emplace_back(t_key); + + // if its first occurence for instance + if (f_message_mode != e_message_summary) { + emit_line(cerr, a_level, a_category, a_error_id, a_message); + } + return; + } + + t_iter->second.count++; + t_iter->second.last_message = a_message; // track latest version of message + + if (f_message_mode == e_message_all) { + emit_line(cerr, a_level, a_category, a_error_id, a_message); + } + // first-only / first-with-count / summary: suppress repeated runtime prints +} + + +void error_logger::stage(const string& a_stage, const string& a_message) +{ + if(!f_metadata_enabled) { + return; + } + info("stage", a_stage, a_message); +} + +void error_logger::create_summary(ostream& a_os) const +{ + if (! should_print(e_log_level_warn)) { + return; + } + + if ((f_message_mode != e_message_first_with_count) && (f_message_mode != e_message_summary)) { + return; + } + + bool t_header_printed = false; + for (const auto& t_key: f_order) { + auto t_iter = f_records.find(t_key); + if (t_iter == f_records.end()) { + continue; + } + + const auto& t_record = t_iter->second; + if (! should_print(t_record.level)) { + continue; + } + + if (! t_header_printed) { + a_os << "##INFO: [error-summary] repeated message counts" << endl; + t_header_printed = true; + } + + a_os << level_prefix(t_record.level); + if (! t_record.category.empty()) { + a_os << "[" << t_record.category << "] "; + } + if (! t_record.error_id.empty()) { + a_os << "(" << t_record.error_id << ") "; + } + a_os << t_record.last_message << " (count=" << t_record.count << ")" << endl; + } +} + + +void error_logger::metadata(const string& a_title, const vector& a_items) +{ + if (! f_metadata_enabled || ! should_print(e_log_level_info)) { + return; + } + + emit_line(cerr, e_log_level_info, "metadata", a_title, ""); + for (const auto& t_item: a_items) { + emit_line(cerr, e_log_level_info, "metadata", "", string(" ") + t_item); + } +} + +void error_logger::metadata_kv(const string& a_key, const string& a_value) +{ + if (! f_metadata_enabled) { + return; + } + info("metadata", a_key, a_value); +} + +void error_logger::emit_line(ostream& a_os, log_level_t a_level, const string& a_category, const string& a_error_id, const string& a_message) const +{ + a_os << level_prefix(a_level); + if (! a_category.empty()) { + a_os << "[" << a_category << "] "; + } + if (! a_error_id.empty()) { + a_os << "(" << a_error_id << ") "; + } + a_os << a_message << endl; +} + + +const char* error_logger::level_prefix(log_level_t a_level) const // need to update later +{ + switch (a_level) { + case e_log_level_panic: + return "##PANIC: "; + case e_log_level_error: + return "##ERROR: "; + case e_log_level_warn: + return "##WARN: "; + case e_log_level_info: + return "##INFO: "; + case e_log_level_debug: + return "##DEBUG: "; + default: + return "##LOG: "; + } +} + + + + + + + diff --git a/src/Honeybee/Source/error_logger.hh b/src/Honeybee/Source/error_logger.hh new file mode 100644 index 0000000..6ae1ce3 --- /dev/null +++ b/src/Honeybee/Source/error_logger.hh @@ -0,0 +1,91 @@ +/* + * error_logger.hh + */ + +#ifndef HONEYBEE_ERROR_LOGGER_HH_ +#define HONEYBEE_ERROR_LOGGER_HH_ 1 + +#include +#include +#include +#include +#include "utils.hh" + +namespace honeybee { + using namespace std; + + class error_logger { + public: + enum message_mode_t { + e_message_all = 0, // print every message + e_message_first_only = 1, // print first only, suppress repeats + e_message_first_with_count = 2, // print first, summarize counts at end + e_message_summary = 3 // print only final summary + }; + + static error_logger& instance(); + + // way of configuring, based on the user choice of mode via CLI + void set_message_mode(message_mode_t a_mode) { f_message_mode = a_mode; } + // message_mode_t get_message_mode() const { return f_message_mode; } + + // this is for the metadata before showing the actual data + void set_metadata_enabled(bool a_enabled) { f_metadata_enabled = a_enabled; } + bool is_metadata_enabled() const { return f_metadata_enabled; } + + void clear(); + + void log(log_level_t a_level, const string& a_category, const string& a_error_id, const string& a_message); + // wrappers for log at various levels + void error(const string& a_category, const string& a_error_id, const string& a_message); + void warn(const string& a_category, const string& a_error_id, const string& a_message); + void info(const string& a_category, const string& a_error_id, const string& a_message); + void debug(const string& a_category, const string& a_error_id, const string& a_message); + + // Stage and metadata helpers (shown at info level if enabled) + + // Stage: for showing progress through stages of the program, e.g. "Loading data", "Applying calibrations", "Fetching from database", etc. + // Metadata: for showing key-value pairs or lists of items that are relevant to the user, e.g. "Sensors found: 10", or the names" + // so more run configuration context + void stage(const string& a_stage, const string& a_message); + void metadata(const string& a_title, const vector& a_items); + void metadata_kv(const string& a_key, const string& a_value); // one key-val pair, instead of a list of blocks + + // Call once near program shutdown for count/summary modes + void create_summary(ostream& a_os=std::cerr) const; + + protected: + error_logger(); + // one stored entry of a unique error, keyed by level + category + error_id + struct message_record { // ex: {l: e_log_level_warn, c: data_source, + log_level_t level; // id: invalid_default_column, last_msg: ..., count: 4} + string category; + string error_id; + string last_message; // most recent message text for that error_id + unsigned count; + }; + + // check against the g_log_level // verbose + bool should_print(log_level_t a_level) const; + // selecting which level should be deduplicated based on the mode + // idealy want to model depulication to warn, error but not necesarily info, debug, but also want to give user the option to choose which level to deduplicate based on their needs + bool is_cache_target(log_level_t a_level) const; + + // making cache key based on level, category and stable error_id + // identical triple (level, category, error_id) identifies the same error type + string make_key(log_level_t a_level, const string& a_category, const string& a_error_id) const; + // unified and consistent output formatting writer for all messages + // so avoid repeating formatting structure multiple times in log, create_summary, and metadata + void emit_line(ostream& a_os, log_level_t a_level, const string& a_category, const string& a_error_id, const string& a_message) const; + //mapping level to prefix text + const char* level_prefix(log_level_t a_level) const; + + protected: + message_mode_t f_message_mode; + bool f_metadata_enabled; + map f_records; // fast looking, updating count, make_key helps + vector f_order; // keys in FCFS order for summary + }; +} + +#endif \ No newline at end of file diff --git a/src/Honeybee/Source/evaluator.cc b/src/Honeybee/Source/evaluator.cc index ffb1613..ef68fb1 100644 --- a/src/Honeybee/Source/evaluator.cc +++ b/src/Honeybee/Source/evaluator.cc @@ -7,9 +7,13 @@ #include #include +#include #include #include "evaluator.hh" +using namespace std; +using namespace honeybee; + template static inline T sqr(const T& x) { return x*x; }; template static inline T cub(const T& x) { return x*x*x; }; @@ -35,3 +39,42 @@ int kebap::KPHoneybeeObject::pt100(std::vector& ArgumentList, kebap::K ReturnValue = kebap::KPValue(y); return 1; } + + +evaluator::evaluator(kebap::KPExpression* a_expression, kebap::KPSymbolTable* a_symbol_table) + : f_expression(a_expression), f_symbol_table(a_symbol_table) +{ +} + +evaluator::~evaluator() +{ + delete f_expression; +} + +double evaluator::operator()(double x) +{ + if (!f_expression || !f_symbol_table) { + throw runtime_error("evaluator: expression or symbol table not initialized"); + } + try { + // Register or update the input variable 'x' in the symbol table for expression evaluation + long x_var_id = f_symbol_table->NameToId("x"); + kebap::KPValue* x_var = f_symbol_table->GetVariable(x_var_id); + if (!x_var) { + // If x doesn't exist, register it as a new variable + x_var_id = f_symbol_table->RegisterVariable("x", kebap::KPValue(x)); + x_var = f_symbol_table->GetVariable(x_var_id); + } + + x_var->AssignDouble(x); + + // Evaluate the calibration expression with updated x value and any global variables/constants + // going from times5(x) -> times5(10) + kebap::KPValue& result = f_expression->Evaluate(f_symbol_table); + return result.AsDouble(); + } + catch (kebap::KPException& e) { + throw runtime_error(string("evaluator: ") + e.what()); + } +} + diff --git a/src/Honeybee/Source/evaluator.hh b/src/Honeybee/Source/evaluator.hh index 7424acb..3f92f5d 100644 --- a/src/Honeybee/Source/evaluator.hh +++ b/src/Honeybee/Source/evaluator.hh @@ -49,13 +49,18 @@ namespace kebap { namespace honeybee { - class evaluator: public kebap::KPEvaluator { - public: - evaluator(const std::string& Expression): kebap::KPEvaluator(Expression) { - fBuiltinFunctionTable->RegisterStaticObject(new kebap::KPHoneybeeObject()); - } + class evaluator { + public: + evaluator(kebap::KPExpression* a_expression, kebap::KPSymbolTable* a_symbol_table); + ~evaluator(); + + double operator()(double x); + private: + kebap::KPExpression* f_expression; + kebap::KPSymbolTable* f_symbol_table; + }; } -#endif +#endif \ No newline at end of file diff --git a/src/Honeybee/Source/honeybee.cc b/src/Honeybee/Source/honeybee.cc index be4249a..fd8e1c2 100644 --- a/src/Honeybee/Source/honeybee.cc +++ b/src/Honeybee/Source/honeybee.cc @@ -6,8 +6,11 @@ */ +#include +#include #include #include "honeybee.hh" +#include "sensor_config_by_ktf.hh" using namespace std; using namespace honeybee; @@ -21,6 +24,7 @@ honeybee_app::honeybee_app() f_default_delimiters = "./:;-_"; f_input_delimiters = ""; f_output_delimiter = ""; + f_value_column_default = ""; f_sensor_table = make_shared(); f_data_source = make_shared(); @@ -43,7 +47,7 @@ void honeybee_app::add_variable(const string& key, const tabree::KVariant& value f_variables.emplace_back(key, value); } -void honeybee_app::set_delimiter(const std::string& input_delimiters, const std::string& output_delimiter) +void honeybee_app::set_delimiter(const string& input_delimiters, const string& output_delimiter) { if (! input_delimiters.empty()) { f_input_delimiters = input_delimiters; @@ -53,6 +57,11 @@ void honeybee_app::set_delimiter(const std::string& input_delimiters, const std: } } +void honeybee_app::set_value_column_default(const string& value_column) +{ + f_value_column_default = value_column; +} + shared_ptr honeybee_app::get_sensor_table() { if (! f_is_constructed) { @@ -87,7 +96,7 @@ void honeybee_app::construct() try { tabree::KTreeFile(f_config_file_path).Read(t_config); } - catch (std::exception &e) { + catch (exception &e) { hERROR(cerr << e.what()); return; } @@ -98,9 +107,10 @@ void honeybee_app::construct() if (! f_config_file_path.empty()) { hINFO(cerr << "loading " << f_config_file_path << endl); - sensor_config_by_file t_sensor_config; - t_sensor_config.set_variables(f_variables); - t_sensor_config.load(*f_sensor_table, f_config_file_path); + auto t_loader = make_shared(); + t_loader->set_variables(f_variables); + t_loader->load(*f_sensor_table, f_config_file_path); + f_loaders[f_config_file_path] = t_loader; hINFO(cerr << f_sensor_table->find_like({{}}).size() << " sensors defined" << endl); } @@ -149,7 +159,7 @@ void honeybee_app::construct() f_data_source->bind(*f_sensor_table); } -std::vector honeybee_app::find_like(const std::string a_name) +vector honeybee_app::find_like(const string a_name) { vector t_name_list; @@ -168,7 +178,8 @@ std::vector honeybee_app::find_like(const std::string a_name) return t_name_list; } -series_bundle honeybee_app::read(const vector& a_sensor_list, double a_from, double a_to, double a_resampling_interval, const std::string& a_reducer) + +series_bundle honeybee_app::read(const vector& a_sensor_list, double a_from, double a_to, double a_resampling_interval, const string& a_reducer) { if (! f_is_constructed) { construct(); @@ -200,19 +211,33 @@ series_bundle honeybee_app::read(const vector& a_sensor_list, doubl } hINFO(cerr << "getting data "); - hINFO(cerr << "(" << datetime(a_from).as_string() << " to " << datetime(a_to).as_string() << ", "); - hINFO(cerr << t_sensor_number_list.size() << " sensors)..." << flush); + hINFO(cerr << "(" << datetime(a_from).as_string() << " to " << datetime(a_to).as_string() << ")..." << flush); datetime start = datetime::now(); - vector t_series_list = f_data_source->read( - t_sensor_number_list, datetime(a_from), datetime(a_to), - a_resampling_interval, a_reducer - ); + + vector t_series_list; + try { + string t_default_column = f_value_column_default; + if (t_default_column.empty()) { + t_default_column = "value_raw"; + } + t_series_list = f_data_source->read( + t_sensor_number_list, t_default_column, a_from, a_to, + a_resampling_interval, a_reducer + ); + } + catch (exception& e) { + hERROR(cerr << "Error reading data: " << e.what() << endl); + return series_bundle(); + } + datetime stop = datetime::now(); hINFO(cerr << "done. (" << (stop-start) << " s)" << endl); + // Resampling might have be done on the server-side, might not. // We will perform resampling on the returned result here; server-side resampling is to reduce the data size. - return hb::zip(std::move(t_sensor_name_list), std::move(t_series_list)); + // combine sensor name w/ its data + return hb::zip(move(t_sensor_name_list), move(t_series_list)); } diff --git a/src/Honeybee/Source/honeybee.hh b/src/Honeybee/Source/honeybee.hh index fc4e537..b4b1e3b 100644 --- a/src/Honeybee/Source/honeybee.hh +++ b/src/Honeybee/Source/honeybee.hh @@ -10,6 +10,7 @@ #include #include +#include #include #include "utils.hh" #include "series.hh" @@ -19,6 +20,11 @@ namespace honeybee { + using namespace std; + + // Forward declaration + class sensor_config; + class sensor_config_by_ktf; class honeybee_app { public: @@ -28,6 +34,7 @@ namespace honeybee { void add_dripline_db(const std::string& db_uri); void add_variable(const std::string& key, const tabree::KVariant& value); void set_delimiter(const std::string& input_delimiters, const std::string& output_delimiter=""); + void set_value_column_default(const std::string& value_column); std::shared_ptr get_sensor_table(); std::shared_ptr get_data_source(); std::vector find_like(const std::string a_name); @@ -40,11 +47,13 @@ namespace honeybee { std::string f_config_file_path; std::string f_dripline_db_uri; std::string f_default_delimiters, f_input_delimiters, f_output_delimiter; + std::string f_value_column_default; protected: bool f_is_constructed; std::shared_ptr f_sensor_table; std::shared_ptr f_data_source; sensor_config_by_file::variables f_variables; + std::map> f_loaders; }; } diff --git a/src/Honeybee/Source/kebap_calibration.cc b/src/Honeybee/Source/kebap_calibration.cc new file mode 100644 index 0000000..85191d3 --- /dev/null +++ b/src/Honeybee/Source/kebap_calibration.cc @@ -0,0 +1,167 @@ +/* + * kebap_calibration.cc + */ + +#include +#include +#include +#include +#include +#include +#include +#include "sensor_table.hh" +#include "evaluator.hh" +#include "kebap_calibration.hh" + +using namespace std; +using namespace honeybee; + +// Helper function to fix Kebap 0-indexed line numbers +static string normalize_line_numbers(const string& error_msg) +{ + string result = error_msg; + size_t pos = 0; + while ((pos = result.find("line ", pos)) != string::npos) { + pos += 5; // skip past "line " + if (pos < result.length() && isdigit(result[pos])) { + int line_num = stoi(result.substr(pos)); + string old_num = to_string(line_num); + string new_num = to_string(line_num + 1); + result.replace(pos, old_num.length(), new_num); + pos += new_num.length(); + } + } + return result; +} + + +kebap_calibration::kebap_calibration(const sensor& a_sensor, const sensor_table& a_sensor_table, + kebap::KPParser* a_parser, const string& a_ktf_path, int a_line_number) + : f_ktf_path(a_ktf_path), f_line_number(a_line_number) +{ + auto strip = [](const string& a_text)->string { + string::size_type t_begin = 0, t_length = a_text.size(); + while (t_begin < t_length) { + if (a_text[t_begin] != ' ') { + break; + } + t_begin++; + } + while (t_length > t_begin) { + if (a_text[t_length-1] != ' ') { + break; + } + t_length--; + } + return a_text.substr(t_begin, t_length); + }; + + f_description = strip(a_sensor.get_calibration()); + f_input = sensor{}.get_number(); + f_evaluator = 0; + + if (f_description.empty()) { + return; + } + + auto colon = f_description.find_first_of(':'); + f_variable_name = strip(f_description.substr(0, colon)); + string t_exp_text; + if (colon != string::npos) { + t_exp_text = strip(f_description.substr(colon+1)); + } + + f_expression_text = t_exp_text; + + name_chain t_input_name_chain = a_sensor.get_name(); + name_chain t_variable_name_chain = name_chain(f_variable_name, "./-_"); + if (t_input_name_chain.size() < t_variable_name_chain.size()) { + t_input_name_chain = t_variable_name_chain; + } + else { + for (unsigned i = 0; i < t_variable_name_chain.size(); i++) { + t_input_name_chain[i] = t_variable_name_chain[i]; + } + } + + auto t_candidates = a_sensor_table.find_like(t_input_name_chain); + if (t_candidates.size() == 1) { + f_input = t_candidates.front(); + } + else if (t_candidates.size() > 0) { + cerr << "ERROR: ambiguous calibration input: " << f_variable_name << endl; + return; + } + else { + f_input = a_sensor_table[t_variable_name_chain]; + } + + if (! f_input) { + cerr << "ERROR: unable to find calibration input: " << f_variable_name << endl; + return; + } + + if ((f_variable_name == t_exp_text) || t_exp_text.empty()) { + f_is_identity = true; + return; + } + + // replace the variable in the expression with "x" + string t_pattern = regex_replace(f_variable_name, regex("\\."), "\\."); + t_exp_text = regex_replace(t_exp_text, regex("(^|[^a-zA-Z_])(" + t_pattern + ")($|[^a-zA-Z0-9_])"), "$1x$3"); + + // Compile expression using provided parser + try { + kebap::KPExpressionParser* t_expr_parser = a_parser->GetExpressionParser(); + istringstream expr_stream(t_exp_text); + + kebap::KPTokenizer t_tokenizer(expr_stream, a_parser->GetTokenTable()); + kebap::KPExpression* t_expression = t_expr_parser->Parse(&t_tokenizer, a_parser->GetSymbolTable()); + kebap::KPSymbolTable* t_symbol_table = a_parser->GetSymbolTable(); + + f_evaluator = make_shared(t_expression, t_symbol_table); + + // Test evaluate to catch early errors + (*f_evaluator)(0); + } + catch (exception &e) { + string error_msg = normalize_line_numbers(e.what()); + // Extract just the error part after "evaluator: " + size_t eval_pos = error_msg.find("evaluator: "); + if (eval_pos != string::npos) { + error_msg = error_msg.substr(eval_pos + 11); // Skip "evaluator: " + } + cerr << "ERROR: " << f_ktf_path << " - calibration: '" << f_expression_text << "'" << endl; + cerr << endl; + cerr << "(ISSUE) " << error_msg << endl; + f_evaluator = 0; + } +} + +double kebap_calibration::operator()(double x) +{ + if (f_is_identity) { + return x; + } + if (!f_evaluator) { + return numeric_limits::quiet_NaN(); + } + + try { + return (*f_evaluator)(x); + } + catch (exception& e) { + throw runtime_error(get_error_context() + " - " + normalize_line_numbers(e.what())); + } +} + +string kebap_calibration::get_error_context() const +{ + ostringstream oss; + oss << f_ktf_path; + if (f_line_number > 0) { + oss << ":" << (f_line_number + 1); // Convert to 1-indexed for user display + } + oss << " in expression: '" << f_expression_text << "'"; + return oss.str(); +} diff --git a/src/Honeybee/Source/kebap_calibration.hh b/src/Honeybee/Source/kebap_calibration.hh new file mode 100644 index 0000000..7bccfe2 --- /dev/null +++ b/src/Honeybee/Source/kebap_calibration.hh @@ -0,0 +1,38 @@ +/* + * kebap_calibration.hh + */ + +#ifndef HONEYBEE_KEBAP_CALIBRATION_HH_ +#define HONEYBEE_KEBAP_CALIBRATION_HH_ 1 + +#include +#include +#include +#include "calibration.hh" + + +namespace honeybee { + using namespace std; + + class sensor; + class sensor_table; + class evaluator; + + class kebap_calibration : public calibration { + public: + kebap_calibration(const sensor& a_sensor, const sensor_table& a_sensor_table, + kebap::KPParser* a_parser, const string& a_ktf_path, int a_line_number = 0); + ~kebap_calibration() override = default; + + double operator()(double x) override; + string get_error_context() const override; + + private: + string f_ktf_path; + int f_line_number; + string f_expression_text; + shared_ptr f_evaluator; + }; + +} +#endif diff --git a/src/Honeybee/Source/sensor_config.cc b/src/Honeybee/Source/sensor_config.cc new file mode 100644 index 0000000..ef311b7 --- /dev/null +++ b/src/Honeybee/Source/sensor_config.cc @@ -0,0 +1,76 @@ +/* + * sensor_config.cc + */ + +#include "sensor_config.hh" +#include "sensor_table.hh" +#include "utils.hh" +namespace honeybee { + +void sensor_config_by_names::set_delimiters(const string& a_input_delimiters, const string& a_output_delimiter) +{ + f_input_delimiters = a_input_delimiters; + f_output_delimiter = a_output_delimiter; +} + +void sensor_config_by_names::load(sensor_table& a_table, const vector& a_name_list, name_chain a_basename) +{ + hINFO(cerr << "Sensor ID matching or creation" << endl); + if (! f_name_space.empty()) { + hINFO(cerr << " Namespace: " << f_name_space << endl); + hINFO(cerr << " Basename: " << a_basename.join() << endl); + } + + map t_binding; + if (! f_name_space.empty()) { + for (int t_number: a_table.find_like({{}})) { + const sensor& t_sensor = a_table[t_number]; + string t_endpoint = t_sensor.get_option(f_name_space, ""); + if (! t_endpoint.empty()) { + t_binding[t_endpoint] = t_sensor.get_name().join(f_output_delimiter); + } + } + } + + for (const string& t_name: a_name_list) { + // explicit matching + auto t_explicit_iter = t_binding.find(t_name); + if (t_explicit_iter != t_binding.end()) { + hINFO(cerr << " Explicit: " << t_name << " => " << t_explicit_iter->second << endl); + continue; + } + + // inference by loose matching + vector t_chain = name_chain{t_name, f_input_delimiters}.get_chain(); + t_chain.insert(t_chain.end(), a_basename.get_chain().begin(), a_basename.get_chain().end()); + + sensor t_sensor; + auto t_sensor_matches = a_table.find_like(t_chain); + if (t_sensor_matches.size() == 1) { + t_sensor = a_table[t_sensor_matches.front()]; + hINFO(cerr << " Inferred: " << t_name << " => " << t_sensor.get_name().join(f_output_delimiter) << endl); + } + + // non-unique matching, error, skipped + else if (t_sensor_matches.size() > 1) { + hERROR(cerr << " Mutiple possibilities on binding: " << t_name << ": " << endl); + for (auto& s: t_sensor_matches) { + hERROR(cerr << " " << a_table[s].get_name().join(f_output_delimiter) << endl); + } + hERROR(cerr << " hint: use explicit binding to resolve ambiguity" << endl); + continue; + } + + // create a new sensor entry + if (! t_sensor) { + auto t_number = a_table.create_unique_number(); + t_sensor = sensor{t_number, t_chain, t_chain}; + hINFO(cerr << " Created: " << t_name << " => " << t_sensor.get_name().join(f_output_delimiter) << endl); + } + if (! f_name_space.empty()) { + t_sensor.set_option(f_name_space, t_name); + } + a_table.add(t_sensor); + } +} +} diff --git a/src/Honeybee/Source/sensor_config.hh b/src/Honeybee/Source/sensor_config.hh new file mode 100644 index 0000000..1df03c4 --- /dev/null +++ b/src/Honeybee/Source/sensor_config.hh @@ -0,0 +1,40 @@ +/* + * sensor_config.hh + */ + +#ifndef HONEYBEE_SENSOR_CONFIG_HH_ +#define HONEYBEE_SENSOR_CONFIG_HH_ 1 + +#include +#include +#include "sensor_table.hh" + +namespace honeybee { + using namespace std; + + class sensor_table; + + class sensor_config { + public: + virtual ~sensor_config() {} + + // Load sensors from file, populate sensor_table + virtual void load(sensor_table& a_table, const string& a_filename) = 0; + + protected: + sensor_config() {} + }; + + class sensor_config_by_names { + public: + sensor_config_by_names(const string& a_name_space=""): f_name_space(a_name_space), f_input_delimiters("/.-_"), f_output_delimiter(".") {} + void set_delimiters(const string& a_delimiters, const string& f_output_delimiter); + void load(sensor_table& a_table, const vector& a_name_list, name_chain a_basename=name_chain()); + protected: + string f_name_space; + vector f_basenames; + string f_input_delimiters, f_output_delimiter; + }; +} + +#endif diff --git a/src/Honeybee/Source/sensor_config_by_ktf.cc b/src/Honeybee/Source/sensor_config_by_ktf.cc new file mode 100644 index 0000000..0a26469 --- /dev/null +++ b/src/Honeybee/Source/sensor_config_by_ktf.cc @@ -0,0 +1,272 @@ +/* + * sensor_config_by_ktf.cc + */ + +#include +#include +#include +#include +#include +#include +#include +#include "sensor_config_by_ktf.hh" +#include "sensor_table.hh" +#include "kebap_calibration.hh" + +using namespace std; +using namespace honeybee; + +sensor_config_by_ktf::sensor_config_by_ktf() + : f_standard_parser(nullptr) +{ +} + +sensor_config_by_ktf::~sensor_config_by_ktf() +{ +} + +void sensor_config_by_ktf::set_variables(const sensor_config_by_ktf::variables& a_variables) +{ + f_variables.insert(f_variables.end(), a_variables.begin(), a_variables.end()); +} + +void sensor_config_by_ktf::load(sensor_table& a_table, const string& a_filename) +{ + f_ktf_path = a_filename; + cout << "Loading KTF file: " << a_filename << endl; + + // Read KTF file + tabree::KTree t_tree; + try { + tabree::KTreeFile(a_filename).Read(t_tree); + } + catch (tabree::KException &e) { + cerr << "ERROR: " << e.what() << endl; + return; + } + + // Extract and compile scripts + string t_scripts = extract_scripts(); + if (!t_scripts.empty()) { + cout << "Extracted Kebap scripts (" << t_scripts.length() << " bytes)" << endl; + try { + f_standard_parser = make_shared(); + std::istringstream script_stream(t_scripts); + f_standard_parser->Parse(script_stream); + + // Ensure global variables from ktf script are registered in symbol table + // Executing bare statements immediately after parsing + f_standard_parser->GetModule()->ExecuteBareStatements(f_standard_parser->GetSymbolTable()); + + cout << "Successfully compiled Kebap parser" << endl; + } + catch (kebap::KPException &e) { + cerr << "ERROR: Failed to parse Kebap scripts: " << e.what() << endl; + f_standard_parser = nullptr; + return; + } + } else { + cout << "No Kebap scripts found in ktf header" << endl; + } + + // Recursively load and configure sensors from hierarchical KTF structure + load_context t_context; + load_layer(a_table, t_tree["sensor_table"], t_context); + cout << "Completed loading ktf file" << endl; +} + +string sensor_config_by_ktf::extract_scripts() +{ + ifstream t_file(f_ktf_path); + if (!t_file.is_open()) { + cout << "Could not open ktf file for script extraction" << endl; + return ""; // File read error, cont without scripts + } + + string t_scripts; + string t_line; + + while (getline(t_file, t_line)) { + // Trim leading whitespace + size_t t_start = t_line.find_first_not_of(" \t"); + if (t_start == string::npos) { + t_line = ""; // All whitespace + } else { + t_line = t_line.substr(t_start); + } + + // Check for script line, begin #% + if (t_line.size() >= 2 && t_line.substr(0, 2) == "#%") { + // Extract content after "#%" + if (t_line.size() > 3 && t_line[2] == ' ') { + t_scripts += t_line.substr(3) + "\n"; + } else if (t_line.size() > 2) { + t_scripts += t_line.substr(2) + "\n"; + } + } + else if (!t_line.empty()) { + // end of script section + break; + } + // no worries about empty lines before sensor definition lines + } + + return t_scripts; +} + +void sensor_config_by_ktf::load_layer(sensor_table& a_table, const tabree::KTree& a_node, load_context& a_context) +{ + // Helper for array index formatting + auto append_index = [](const string& text, int length, unsigned index)->string { + if (length < 0) { + return text; + } + int width = (length==0) ? 1 : int(log10(length-0.5)+1); + ostringstream os; + os << text << setw(width) << setfill('0') << index; + return os.str(); + }; + + // Check if this is a channel node + if (a_node.NodeName() == "channel") { + add_sensor(a_table, a_node, a_context); + return; + } + + static const char* t_subnode_types[] = { + "experiment", "setup", "teststand", "system", + "section", "subsection", "division", "segment", "crate", + "module", "device", "card", "board", + "channel", "endpoint", "metric" + }; + + for (const char* t_subnode_type: t_subnode_types) { + for (unsigned i = 0; i < a_node[t_subnode_type].Length(); i++) { + const auto& t_node = a_node[t_subnode_type][i]; + string t_name = t_node["id"]["name"].As(); + string t_label = t_node["id"]["label"].Or(t_name); + int t_array_length = t_node["array_length"].Or(-1); + string t_condition = t_node["valid_if"].Or(""); + + // Check guard conditions + if (! t_condition.empty()) { + kebap::KPEvaluator f(t_condition); + for (const auto& var: get_variables()) { + f[var.first] = var.second; + } + try { + if (! f(0)) { + continue; + } + } + catch (kebap::KPException &e) { + cerr << "ERROR: " << e.what() << ": " << t_node.NodePath() << endl; + } + } + + // Handle array expansion + for (int j = 0; j < std::max(1, t_array_length); j++) { + auto t_context = a_context; + t_context.f_name.push_front(append_index(t_name, t_array_length, j)); + t_context.f_label.push_front(append_index(t_label, t_array_length, j)); + + // Extract options (x- prefixed keys) + for (const auto& t_key: t_node.KeyList()) { + if ((t_key.substr(0, 2) == "x_") || (t_key.substr(0, 2) == "x-")) { + string t_opt_name = t_key.substr(2); + + if (t_node[t_key].IsLeaf()) { + // Simple string format + string t_opt_value = t_node[t_key].As(); + if (! t_opt_name.empty()) { + t_context.f_opts.emplace_back(t_opt_name, t_opt_value); + } + } else { + // Object format: x-dripline_endpoint + if (t_opt_name == "dripline_endpoint") { + string tag = t_node[t_key]["tag"].As(); + string field = t_node[t_key]["field"].Or(""); + t_context.f_opts.emplace_back("dripline_endpoint", tag); + t_context.f_opts.emplace_back("dripline_endpoint_field", field); + } + } + } + } + + // Recurse on the child node + load_layer(a_table, t_node, t_context); + } + } + } +} + +void sensor_config_by_ktf::add_sensor(sensor_table& a_table, const tabree::KTree& a_node, + const load_context& a_context) +{ + int t_number = sensor_table::create_unique_number(); + vector t_name_chain(a_context.f_name.begin(), a_context.f_name.end()); + vector t_label_chain(a_context.f_label.begin(), a_context.f_label.end()); + + sensor t_sensor(t_number, t_name_chain, t_label_chain); + + // Set calibration string + string t_calibration = a_node["default_calibration"].Or(""); + t_sensor.set_calibration(t_calibration); + + // Create kebap_calibration if calibration string exists and parser is valid + if (!t_calibration.empty() && f_standard_parser) { + cout << "Attaching calibration to " << t_name_chain.front() + << ": \"" << t_calibration << "\"" << endl; + try { + auto t_calib = make_shared(t_sensor, a_table, f_standard_parser.get(), f_ktf_path); + t_sensor.set_calibration_object(t_calib); + cout << "Successfully compiled calibration expression" << endl; + } + catch (exception &e) { + cerr << "WARNING: Could not create calibration for " + << t_name_chain.front() << ": " << e.what() << endl; + } + } else if (!t_calibration.empty() && !f_standard_parser) { + cout << "Calibration string exists but no Kebap parser available: " + << t_calibration << endl; + } else { + cout << "No calibration for sensor: " << t_name_chain.front() << endl; + } + + // Extract and set options + map t_options; + for (const auto& t_opt: a_context.f_opts) { + t_options[t_opt.first] = t_opt.second; + } + + if(!f_ktf_path.empty()) { + t_options["ktf_source"] = f_ktf_path; + } + + for (const auto& t_key: a_node.KeyList()) { + if ((t_key.substr(0, 2) == "x_") || (t_key.substr(0, 2) == "x-")) { + string t_opt_name = t_key.substr(2); + + if (a_node[t_key].IsLeaf()) { + string t_opt_value = a_node[t_key].As(); + if (!t_opt_name.empty()) { + t_options[t_opt_name] = t_opt_value; + } + } else { + // Handle dripline_endpoint format + if (t_opt_name == "dripline_endpoint") { + string tag = a_node[t_key]["tag"].As(); + string field = a_node[t_key]["field"].Or(""); + t_options["dripline_endpoint"] = tag; + t_options["dripline_endpoint_field"] = field; + } + } + } + } + + for (auto& t_opt: t_options) { + t_sensor.set_option(t_opt.first, t_opt.second); + } + + a_table.add(t_sensor); +} diff --git a/src/Honeybee/Source/sensor_config_by_ktf.hh b/src/Honeybee/Source/sensor_config_by_ktf.hh new file mode 100644 index 0000000..792d8ea --- /dev/null +++ b/src/Honeybee/Source/sensor_config_by_ktf.hh @@ -0,0 +1,52 @@ +/* + * sensor_config_by_ktf.hh + */ + +#ifndef HONEYBEE_SENSOR_CONFIG_BY_KTF_HH_ +#define HONEYBEE_SENSOR_CONFIG_BY_KTF_HH_ 1 + +#include +#include +#include +#include +#include +#include "sensor_config.hh" + +namespace kebap { + class KPStandardParser; +} + +namespace honeybee { + using namespace std; + + // Context for recursive load_layer traversal + struct load_context { + deque f_name, f_label; + deque> f_opts; + }; + + class sensor_config_by_ktf : public sensor_config { + public: + using variables = vector>; + + sensor_config_by_ktf(); + virtual ~sensor_config_by_ktf(); + + void set_variables(const variables& a_variables); + const variables& get_variables() const { return f_variables; } + void load(sensor_table& a_table, const string& a_filename) override; + + private: + shared_ptr f_standard_parser; + string f_ktf_path; + // Runtime variables passed from caller for guard condition evaluation (e.g., device state checks) + variables f_variables; + + string extract_scripts(); + void load_layer(sensor_table& a_table, const tabree::KTree& a_node, load_context& a_context); + void add_sensor(sensor_table& a_table, const tabree::KTree& a_node, + const load_context& a_context); + }; +} + +#endif diff --git a/src/Honeybee/Source/sensor_table.cc b/src/Honeybee/Source/sensor_table.cc index fe0234c..8257777 100644 --- a/src/Honeybee/Source/sensor_table.cc +++ b/src/Honeybee/Source/sensor_table.cc @@ -16,12 +16,21 @@ #include #include "utils.hh" #include "sensor_table.hh" +#include "calibration.hh" using namespace std; using namespace honeybee; int sensor_table::f_unique_sequence = 0; +double sensor::apply_calibration(double raw_value) const +{ + if (!f_calibration_obj) { // either raw or apply calibration + return raw_value; + } + return (*f_calibration_obj)(raw_value); +} + string sensor::to_json(vector a_field_list, const std::string& a_delimiter) const { if (a_field_list.empty()) { @@ -174,6 +183,7 @@ void sensor_config_by_file::load_layer(sensor_table& a_table, const tabree::KTre } } + //Nobel: checks if the x-'s retrieved is object of a string for (int j = 0; j < std::max(1, t_array_length); j++) { auto t_context = a_context; t_context.f_name.push_front(append_index(t_name, t_array_length, j)); @@ -181,9 +191,21 @@ void sensor_config_by_file::load_layer(sensor_table& a_table, const tabree::KTre for (const auto& t_key: t_node.KeyList()) { if ((t_key.substr(0, 2) == "x_") || (t_key.substr(0, 2) == "x-")) { string t_opt_name = t_key.substr(2); - string t_opt_value = t_node[t_key].As(); - if (! t_opt_name.empty()) { - t_context.f_opts.emplace_back(t_opt_name, t_opt_value); + + if (t_node[t_key].IsLeaf()) { + // Simple string format + string t_opt_value = t_node[t_key].As(); + if (! t_opt_name.empty()) { + t_context.f_opts.emplace_back(t_opt_name, t_opt_value); + } + } else { + // Object format: x-dripline_endpoint: { tag: ..., field: ... } + if (t_opt_name == "dripline_endpoint") { + string tag = t_node[t_key]["tag"].As(); + string field = t_node[t_key]["field"].Or(""); + t_context.f_opts.emplace_back("dripline_endpoint", tag); + t_context.f_opts.emplace_back("dripline_endpoint_field", field); + } } } } @@ -203,7 +225,7 @@ void sensor_config_by_file::add_sensor(sensor_table& a_table, const tabree::KTre t_sensor.set_calibration(a_node["default_calibration"]); map t_options; - // this step is to allow overriding // + // this step is to allow overriding for (auto& t_opt: a_context.f_opts) { t_options[t_opt.first] = t_opt.second; } @@ -213,72 +235,3 @@ void sensor_config_by_file::add_sensor(sensor_table& a_table, const tabree::KTre a_table.add(t_sensor); } - - - -void sensor_config_by_names::set_delimiters(const string& a_input_delimiters, const string& a_output_delimiter) -{ - f_input_delimiters = a_input_delimiters; - f_output_delimiter = a_output_delimiter; -} - -void sensor_config_by_names::load(sensor_table& a_table, const vector& a_name_list, name_chain a_basename) -{ - hINFO(cerr << "Sensor ID matching or creation" << endl); - if (! f_name_space.empty()) { - hINFO(cerr << " Namespace: " << f_name_space << endl); - hINFO(cerr << " Basename: " << a_basename.join() << endl); - } - - map t_binding; - if (! f_name_space.empty()) { - for (int t_number: a_table.find_like({{}})) { - const sensor& t_sensor = a_table[t_number]; - string t_endpoint = t_sensor.get_option(f_name_space, ""); - if (! t_endpoint.empty()) { - t_binding[t_endpoint] = t_sensor.get_name().join(f_output_delimiter); - } - } - } - - for (const string& t_name: a_name_list) { - // explicit matching - auto t_explicit_iter = t_binding.find(t_name); - if (t_explicit_iter != t_binding.end()) { - hINFO(cerr << " Explicit: " << t_name << " => " << t_explicit_iter->second << endl); - continue; - } - - // inference by loose matching - vector t_chain = name_chain{t_name, f_input_delimiters}.get_chain(); - t_chain.insert(t_chain.end(), a_basename.get_chain().begin(), a_basename.get_chain().end()); - - sensor t_sensor; - auto t_sensor_matches = a_table.find_like(t_chain); - if (t_sensor_matches.size() == 1) { - t_sensor = a_table[t_sensor_matches.front()]; - hINFO(cerr << " Inferred: " << t_name << " => " << t_sensor.get_name().join(f_output_delimiter) << endl); - } - - // non-unique matching, error, skipped - else if (t_sensor_matches.size() > 1) { - hERROR(cerr << " Mutiple possibilities on binding: " << t_name << ": " << endl); - for (auto& s: t_sensor_matches) { - hERROR(cerr << " " << a_table[s].get_name().join(f_output_delimiter) << endl); - } - hERROR(cerr << " hint: use explicit binding to resolve ambiguity" << endl); - continue; - } - - // create a new sensor entry - if (! t_sensor) { - auto t_number = a_table.create_unique_number(); - t_sensor = sensor{t_number, t_chain, t_chain}; - hINFO(cerr << " Created: " << t_name << " => " << t_sensor.get_name().join(f_output_delimiter) << endl); - } - if (! f_name_space.empty()) { - t_sensor.set_option(f_name_space, t_name); - } - a_table.add(t_sensor); - } -} diff --git a/src/Honeybee/Source/sensor_table.hh b/src/Honeybee/Source/sensor_table.hh index 0beb6dc..d3263f0 100644 --- a/src/Honeybee/Source/sensor_table.hh +++ b/src/Honeybee/Source/sensor_table.hh @@ -13,12 +13,15 @@ #include #include #include +#include #include namespace honeybee { using namespace std; + class calibration; // forward declaration + class name_chain { public: name_chain() {} @@ -85,17 +88,24 @@ namespace honeybee { auto iter = f_options.find(name); return (iter == f_options.end()) ? default_value : iter->second; } + inline shared_ptr get_calibration_object() const { return f_calibration_obj; } string to_json(vector a_field_list = {{}}, const std::string& a_delimiter=".") const; public: // used by sensor_config void set_calibration(const string& calibration) { f_calibration = calibration; } void set_option(const string& name, const string& value) { f_options[name] = value; } + void set_calibration_object(shared_ptr a_calibration) { f_calibration_obj = a_calibration; } + double apply_calibration(double raw_value) const; + protected: int f_number; name_chain f_name; name_chain f_label; string f_calibration; map f_options; + shared_ptr f_calibration_obj; + // provenance: which KTF (file) this sensor was created from + // }; @@ -156,16 +166,7 @@ namespace honeybee { - class sensor_config_by_names { - public: - sensor_config_by_names(const string& a_name_space=""): f_name_space(a_name_space), f_input_delimiters("/.-_"), f_output_delimiter(".") {} - void set_delimiters(const string& a_delimiters, const string& f_output_delimiter); - void load(sensor_table& a_table, const vector& a_name_list, name_chain a_basename=name_chain()); - protected: - string f_name_space; // "dripline_endpoint" etc - vector f_basenames; - string f_input_delimiters, f_output_delimiter; - }; + } #endif