Skip to content

perf(serialise): stream JSON directly and compile format strings #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
benmandrew merged 1 commit into from
Jun 13, 2026
Merged

perf(serialise): stream JSON directly and compile format strings #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
219 changes: 143 additions & 76 deletions src/serialise.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
#include "serialise.hpp"

#include <fmt/compile.h>
#include <fmt/format.h>
#include <sys/types.h>

#include <cstdint>
#include <cstdio>
#include <iterator>
#include <string>
#include <string_view>
#include <unordered_map>
#include <vector>

Expand Down Expand Up @@ -42,24 +44,6 @@ auto lerp(uint8_t start, uint8_t end, float t) -> uint8_t {
return static_cast<uint8_t>((start_f * (1.0F - t)) + (end_f * t));
}

auto value_to_colour(size_t value, size_t max_depth, bool winning)
-> std::string {
uint8_t r, g, b;
if (winning) {
r = WINNING_R;
g = WINNING_G;
b = WINNING_B;
} else {
float t = static_cast<float>(value) / static_cast<float>(max_depth);
r = lerp(START_R, END_R, t);
g = lerp(START_G, END_G, t);
b = lerp(START_B, END_B, t);
}
std::array<char, 10> buffer;
std::snprintf(buffer.data(), buffer.size(), "#%02X%02X%02XFF", r, g, b);
return {buffer.data()};
}

using NodeList = std::vector<const Node*>;

#define NODE_MIN_SIZE 1.0F
Expand All @@ -72,31 +56,85 @@ constexpr float get_node_size(size_t max_depth, size_t node_depth) {
(NODE_MAX_SIZE - NODE_MIN_SIZE) / max_depth_f);
}

auto serialise_nodes(const NodeList& nodes, size_t max_depth)
-> nlohmann::json {
nlohmann::json out = nlohmann::json::array();
for (size_t id = 0; id < nodes.size(); ++id) {
const auto& node_ptr = nodes[id];
nlohmann::json node_json;
node_json["id"] = id;
bool winning = node_ptr->m_table.is_complete();
node_json["color"] =
value_to_colour(node_ptr->m_depth, max_depth, winning);
std::array<char, 10> buffer;
std::snprintf(buffer.data(), buffer.size(), "%f",
get_node_size(max_depth, node_ptr->m_depth));
node_json["size"] = {buffer.data()};
if (id == 0) {
node_json["label"] = "Start";
node_json["forceLabel"] = true;
} else if (winning) {
node_json["label"] = "Winning";
node_json["forceLabel"] = true;
// Appends `s` to `out` as the contents of a JSON string (no surrounding
// quotes). Non-ASCII bytes (e.g. UTF-8 suit symbols) are passed through
// verbatim, matching nlohmann's default ensure_ascii=false behaviour.
auto append_json_escaped(std::string& out, std::string_view s) -> void {
// Bulk-append runs of characters that need no escaping (the common case:
// card glyphs, spaces, UTF-8 suit bytes), flushing only at the rare char
// that must be escaped (newlines in the board, defensively quotes etc.).
size_t run_start = 0;
for (size_t i = 0; i < s.size(); ++i) {
auto c = static_cast<unsigned char>(s[i]);
std::string_view esc;
switch (c) {
case '"':
esc = "\\\"";
break;
case '\\':
esc = "\\\\";
break;
case '\n':
esc = "\\n";
break;
case '\r':
esc = "\\r";
break;
case '\t':
esc = "\\t";
break;
default:
if (c >= 0x20) {
continue; // printable / UTF-8 byte: extend the safe run
}
}
out.append(s.data() + run_start, i - run_start);
if (!esc.empty()) {
out += esc;
} else {
fmt::format_to(std::back_inserter(out), FMT_COMPILE("\\u{:04x}"),
static_cast<unsigned int>(c));
}
node_json["table"] = node_ptr->m_table.to_string();
out.push_back(node_json);
run_start = i + 1;
}
return out;
out.append(s.data() + run_start, s.size() - run_start);
}

auto append_colour(std::string& out, size_t value, size_t max_depth,
bool winning) -> void {
uint8_t r, g, b;
if (winning) {
r = WINNING_R;
g = WINNING_G;
b = WINNING_B;
} else {
float t = static_cast<float>(value) / static_cast<float>(max_depth);
r = lerp(START_R, END_R, t);
g = lerp(START_G, END_G, t);
b = lerp(START_B, END_B, t);
}
fmt::format_to(std::back_inserter(out), FMT_COMPILE("#{:02X}{:02X}{:02X}FF"),
r, g, b);
}

auto append_node(std::string& out, size_t id, const Node* node,
size_t max_depth) -> void {
bool winning = node->m_table.is_complete();
fmt::format_to(std::back_inserter(out), FMT_COMPILE(R"({{"id":{},"color":")"),
id);
append_colour(out, node->m_depth, max_depth, winning);
// "size" is a single-element array of a formatted string, preserving the
// shape the previous nlohmann braced-init produced.
fmt::format_to(std::back_inserter(out), FMT_COMPILE(R"(","size":["{:.6f}"])"),
get_node_size(max_depth, node->m_depth));
if (id == 0) {
out += R"(,"label":"Start","forceLabel":true)";
} else if (winning) {
out += R"(,"label":"Winning","forceLabel":true)";
}
out += R"(,"table":")";
append_json_escaped(out, node->m_table.to_string());
out += "\"}";
}

auto build_ptr_to_id_map(const NodeList& nodes)
Expand All @@ -109,54 +147,83 @@ auto build_ptr_to_id_map(const NodeList& nodes)
return ptr_to_id;
}

auto serialise_edges(const NodeList& nodes) -> nlohmann::json {
nlohmann::json edges = nlohmann::json::array();
std::unordered_map<const Node*, size_t> ptr_to_id =
build_ptr_to_id_map(nodes);
for (size_t id = 0; id < nodes.size(); ++id) {
const auto& node_ptr = nodes[id];
for (const auto& edge : node_ptr->m_edges) {
nlohmann::json edge_json;
edge_json["source"] = id;
const Node* to_ptr = edge.m_to;
auto it = ptr_to_id.find(to_ptr);
if (it == ptr_to_id.end()) {
// target node wasn't in the traversal (maybe beyond max depth)
continue;
}
edge_json["target"] = it->second;
edge_json["type"] = "arrow";
edge_json["label"] = move_type_to_string(edge.m_move.m_type);
edge_json["size"] = 1;
edges.push_back(edge_json);
auto append_edges(std::string& out, size_t source_id, const Node* node,
const std::unordered_map<const Node*, size_t>& ptr_to_id,
bool& first) -> void {
for (const auto& edge : node->m_edges) {
auto it = ptr_to_id.find(edge.m_to);
if (it == ptr_to_id.end()) {
// target node wasn't in the traversal (maybe beyond max depth)
continue;
}
if (!first) {
out += ',';
}
first = false;
fmt::format_to(
std::back_inserter(out),
FMT_COMPILE(
R"({{"source":{},"target":{},"type":"arrow","label":"{}","size":1}})"),
source_id, it->second, move_type_to_string(edge.m_move.m_type));
}
return edges;
}

auto graph_to_json(const Graph& graph, size_t max_depth) -> nlohmann::json {
auto collect_nodes(const Graph& graph) -> NodeList {
NodeList nodes;
nodes.reserve(256);
auto it = graph.begin();
auto end = graph.end();
for (; it != end; ++it) {
nodes.reserve(4096);
for (auto it = graph.begin(); it != graph.end(); ++it) {
nodes.push_back(*it);
}
nlohmann::json j;
j["nodes"] = serialise_nodes(nodes, max_depth);
j["edges"] = serialise_edges(nodes);
return j;
return nodes;
}

// Streams the whole graph as compact JSON into `buf`, invoking `flush` after
// each record so callers can drain `buf` to a file and bound memory use.
template <typename Flush>
auto serialise_graph(const Graph& graph, size_t max_depth, std::string& buf,
Flush flush) -> void {
NodeList nodes = collect_nodes(graph);
auto ptr_to_id = build_ptr_to_id_map(nodes);
buf += R"({"nodes":[)";
for (size_t id = 0; id < nodes.size(); ++id) {
if (id != 0) {
buf += ',';
}
append_node(buf, id, nodes[id], max_depth);
flush();
}
buf += R"(],"edges":[)";
bool first = true;
for (size_t id = 0; id < nodes.size(); ++id) {
append_edges(buf, id, nodes[id], ptr_to_id, first);
flush();
}
buf += "]}";
}

auto graph_to_string(const Graph& graph, size_t max_depth) -> std::string {
return graph_to_json(graph, max_depth).dump(2);
std::string buf;
serialise_graph(graph, max_depth, buf, [] {});
return buf;
}

auto graph_to_json(const Graph& graph, size_t max_depth) -> nlohmann::json {
return nlohmann::json::parse(graph_to_string(graph, max_depth));
}

auto write_graph_to_file(const Graph& graph,
const std::filesystem::path& outpath, size_t max_depth)
-> void {
std::ofstream file;
file.open(outpath);
file << graph_to_string(graph, max_depth);
std::ofstream file(outpath, std::ios::binary);
std::string buf;
constexpr size_t flush_threshold = 1U << 20; // 1 MiB
buf.reserve(flush_threshold + (1U << 16));
serialise_graph(graph, max_depth, buf, [&] {
if (buf.size() >= flush_threshold) {
file.write(buf.data(), static_cast<std::streamsize>(buf.size()));
buf.clear();
}
});
file.write(buf.data(), static_cast<std::streamsize>(buf.size()));
file.close();
}
13 changes: 7 additions & 6 deletions src/table.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@

#include "table.hpp"

#include <fmt/compile.h>
#include <fmt/format.h>

#include <algorithm>
Expand Down Expand Up @@ -119,7 +120,7 @@ auto Table::tableau_to_string() const -> std::string {
for (const auto& row : table) {
for (uint8_t card : row) {
if (card != c_null_index) {
fmt::format_to(std::back_inserter(result), "{:^5}",
fmt::format_to(std::back_inserter(result), FMT_COMPILE("{:^5}"),
card_to_string(card));
} else {
result += " ";
Expand All @@ -133,17 +134,17 @@ auto Table::tableau_to_string() const -> std::string {
auto Table::header_to_string() const -> std::string {
std::string result;
if (m_stock_index == c_null_index) {
fmt::format_to(std::back_inserter(result), "Stock: {:<3}",
fmt::format_to(std::back_inserter(result), FMT_COMPILE("Stock: {:<3}"),
c_no_card_string);
} else {
fmt::format_to(std::back_inserter(result), "Stock: {:<3}",
fmt::format_to(std::back_inserter(result), FMT_COMPILE("Stock: {:<3}"),
c_hidden_card_string);
}
fmt::format_to(std::back_inserter(result), "Waste: {:<3}",
fmt::format_to(std::back_inserter(result), FMT_COMPILE("Waste: {:<3}"),
card_to_string(m_waste_index));
fmt::format_to(std::back_inserter(result), "Foundations: ");
result += "Foundations: ";
for (size_t suit = 0; suit < c_num_suits; suit++) {
fmt::format_to(std::back_inserter(result), "{:<4}",
fmt::format_to(std::back_inserter(result), FMT_COMPILE("{:<4}"),
card_to_string(m_foundation_indices[suit]));
}
result += "\n";
Expand Down
Loading