Add runtime regex notification checks

README.md

@@ -39,3 +39,33 @@ make
 ## Usage
 
 For comprehensive help, use `dooked --help`
+
+### Runtime regex checks
+
+Use `--checks <file.json>` or `--check-config <file.json>` to load custom
+notifications without changing the output JSON format.
+
+The config can be a JSON array, or an object with a `checks` array:
+
+```json
+{
+  "checks": [
+    {
+      "field": "domain",
+      "regex": "(dev|test)",
+      "alert": "development-looking domain"
+    },
+    {
+      "field": "body",
+      "regex": "copyright 2020",
+      "alert": "outdated copyright banner",
+      "ignore_case": true
+    }
+  ]
+}
+```
+
+Supported fields are `domain`, DNS fields (`type`, `info`, `rdata`, `ttl`),
+HTTP metadata (`http_code`, `code_string`, `content_length`), and response
+`body`. Body checks are evaluated in memory while probing and are not written
+to the JSON output.

dooked/CMakeLists.txt

@@ -69,6 +69,7 @@ set(SRC_FILES
     ./source/http/requests_handler.cpp
     ./source/utils/constants.cpp
     ./source/utils/io_utils.cpp
+    ./source/utils/regex_checks.cpp
     ./source/utils/string_utils.cpp
     ./source/utils/random_utils.cpp
     ./source/utils/ucstring.cpp
@@ -91,6 +92,7 @@ set(HEADERS_FILES
     ./include/utils/io_utils.hpp
     ./include/utils/probe_result.hpp
     ./include/utils/random_utils.hpp
+    ./include/utils/regex_checks.hpp
     ./include/utils/string_utils.hpp
     ./include/utils/ucstring.hpp
     ./include/cli_preprocessor.hpp

dooked/include/cli_preprocessor.hpp

@@ -2,6 +2,7 @@
 
 #include "dns/dns_resolver.hpp"
 #include "utils/io_utils.hpp"
+#include "utils/regex_checks.hpp"
 #include <thread>
 
 // maximum sockets to open regardless of the number of threads
@@ -19,6 +20,7 @@ struct cli_args_t {
   std::string resolver_filename{};
   std::string output_filename{};
   std::string input_filename{};
+  std::string checks_filename{};
 
   int file_type{};
   int post_http_request{};
@@ -36,6 +38,7 @@ struct runtime_args_t {
   http_process_e http_request_time_{};
   int thread_count{};
   int content_length{-1};
+  regex_check_list_t regex_checks{};
 };
 
 void run_program(cli_args_t const &cli_args);

dooked/include/dns/dns_resolver.hpp

@@ -3,6 +3,7 @@
 #include "dns.hpp"
 #include "http/requests_handler.hpp"
 #include "utils/containers.hpp"
+#include "utils/regex_checks.hpp"
 #include <boost/asio/ip/udp.hpp>
 #include <boost/asio/steady_timer.hpp>
 #include <optional>
@@ -33,6 +34,7 @@ class custom_resolver_socket_t {
   domain_list_t &names_;
   resolver_list_t &resolvers_;
   map_container_t<probe_result_t> &result_map_;
+  regex_check_list_t const *regex_checks_{nullptr};
   net::ssl::context *ssl_context_{nullptr};
   std::optional<udp_stream_t> udp_stream_{};
   std::optional<net::ip::udp::endpoint> default_ep_{};
@@ -67,7 +69,8 @@ class custom_resolver_socket_t {
 
   // http related "handlers"
   void perform_http_request();
-  void http_result_obtained(response_type_e, int, std::string const &);
+  void http_result_obtained(response_type_e, int, std::string const &,
+                            std::string const &);
   void on_http_resolve_error();
   void send_https_request(std::string const &address);
   void send_http_request(std::string const &address);
@@ -77,7 +80,8 @@ class custom_resolver_socket_t {
 public:
   custom_resolver_socket_t(net::io_context &, net::ssl::context *,
                            domain_list_t &, resolver_list_t &,
-                           map_container_t<probe_result_t> &);
+                           map_container_t<probe_result_t> &,
+                           regex_check_list_t const * = nullptr);
   void defer_http_request(bool const defer);
   void start();
 };

dooked/include/http/requests_handler.hpp

@@ -32,7 +32,8 @@ enum class ssl_method_e {
   undefined
 };
 
-using completion_cb_t = std::function<void(response_type_e, int, std::string)>;
+using completion_cb_t =
+    std::function<void(response_type_e, int, std::string, std::string)>;
 
 class http_request_handler_t {
   net::io_context &io_;

dooked/include/http/resolver.hpp

@@ -3,6 +3,7 @@
 #include "http/requests_handler.hpp"
 #include "utils/containers.hpp"
 #include "utils/probe_result.hpp"
+#include "utils/regex_checks.hpp"
 
 namespace dooked {
 
@@ -14,6 +15,7 @@ class http_resolver_t {
   std::optional<request_t> http_request_handler_{};
   std::optional<temporary_ssl_holder_t> tls_holder_{};
   std::string name_{};
+  regex_check_list_t const *regex_checks_{nullptr};
   int http_redirects_count_{};
   int http_retries_count_{};
   // this should have been a boolean but it's an int to keep the alignment
@@ -23,14 +25,16 @@ class http_resolver_t {
   void perform_http_request();
   void switch_ssl_method(std::string const &);
   void send_next_request();
-  void tcp_request_result(response_type_e, int, std::string const &);
+  void tcp_request_result(response_type_e, int, std::string const &,
+                          std::string const &);
   void send_http_request(std::string const &address);
   void send_https_request(std::string const &address);
   void on_resolve_error();
 
 public:
   http_resolver_t(net::io_context &, ssl::context *, domain_list_t &,
-                  map_container_t<probe_result_t> &);
+                  map_container_t<probe_result_t> &,
+                  regex_check_list_t const * = nullptr);
 
   void start() { send_next_request(); }
 };

dooked/include/utils/exceptions.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <stdexcept>
+#include <string>
 
 namespace dooked {
 

dooked/include/utils/regex_checks.hpp

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "utils/containers.hpp"
+#include "utils/probe_result.hpp"
+#include <optional>
+#include <regex>
+#include <string>
+#include <vector>
+
+namespace dooked {
+
+struct regex_check_t {
+  std::string field{};
+  std::string pattern{};
+  std::string alert{};
+  std::regex expression{};
+};
+
+using regex_check_list_t = std::vector<regex_check_t>;
+
+std::optional<regex_check_list_t>
+load_regex_checks(std::string const &filename);
+
+void run_result_checks(regex_check_list_t const &checks,
+                       map_container_t<probe_result_t> const &result_map);
+
+void run_body_checks(regex_check_list_t const &checks,
+                     std::string const &domain_name, int const http_code,
+                     int const content_length, std::string const &body);
+
+} // namespace dooked

dooked/source/cli_preprocessor.cpp

@@ -179,7 +179,7 @@ void dns_functor(net::io_context &io_context, net::ssl::context *ssl_context,
   for (std::size_t i = 0; i < socket_count; ++i) {
     sockets[i] = std::make_unique<custom_resolver_socket_t>(
         io_context, ssl_context, *rt_args.names, *rt_args.resolvers,
-        result_map);
+        result_map, &rt_args.regex_checks);
     sockets[i]->defer_http_request(deferring);
     sockets[i]->start();
   }
@@ -194,7 +194,8 @@ void http_functor(net::io_context &io_context, net::ssl::context *ssl_context,
   sockets.resize(socket_count);
   for (std::size_t i = 0; i < socket_count; ++i) {
     sockets[i] = std::make_unique<http_resolver_t>(io_context, ssl_context,
-                                                   *rt_args.names, result_map);
+                                                   *rt_args.names, result_map,
+                                                   &rt_args.regex_checks);
     sockets[i]->start();
   }
   io_context.run();
@@ -354,6 +355,7 @@ void start_name_checking(runtime_args_t &&rt_args) {
     spdlog::info("Writing JSON output");
   }
   write_json_result(result_map, rt_args);
+  run_result_checks(rt_args.regex_checks, result_map);
 
   // compare old with new result -- only if we had previous record
   if (rt_args.previous_data) {
@@ -404,6 +406,13 @@ void run_program(cli_args_t const &cli_args) {
   if (!read_input_file(cli_args, rt_args)) {
     return;
   }
+  if (!cli_args.checks_filename.empty()) {
+    auto checks = load_regex_checks(cli_args.checks_filename);
+    if (!checks) {
+      return;
+    }
+    rt_args.regex_checks = std::move(*checks);
+  }
   // try opening an output file
   {
     std::string filename{};

dooked/source/dns/dns_resolver.cpp

@@ -9,6 +9,17 @@ extern bool silent;
 
 namespace dooked {
 
+namespace {
+void run_final_body_checks(regex_check_list_t const *checks,
+                           std::string const &name, int const http_code,
+                           int const content_length,
+                           std::string const &body) {
+  if (checks != nullptr) {
+    run_body_checks(*checks, name, http_code, content_length, body);
+  }
+}
+} // namespace
+
 dns_rec_list_t const dns_supported_record_type_t::supported_types{
     dns_record_type_e::DNS_REC_A,     dns_record_type_e::DNS_REC_AAAA,
     dns_record_type_e::DNS_REC_CNAME, dns_record_type_e::DNS_REC_MX,
@@ -111,9 +122,11 @@ void dns_create_query(std::string const &name, std::uint16_t record_type,
 custom_resolver_socket_t::custom_resolver_socket_t(
     net::io_context &io_context, net::ssl::context *ssl_context,
     domain_list_t &domain_list, resolver_list_t &resolvers,
-    map_container_t<probe_result_t> &result_map)
+    map_container_t<probe_result_t> &result_map,
+    regex_check_list_t const *regex_checks)
     : io_{io_context}, names_{domain_list}, resolvers_{resolvers},
-      result_map_{result_map}, ssl_context_{ssl_context},
+      result_map_{result_map}, regex_checks_{regex_checks},
+      ssl_context_{ssl_context},
       supported_dns_record_size_(
           dns_supported_record_type_t::supported_types.size()) {}
 
@@ -382,8 +395,9 @@ void custom_resolver_socket_t::send_http_request(std::string const &address) {
       http_request_handler_->request_.emplace<http_request_handler_t>(
           io_, uri_t{address}.host());
   http_request.start([this](response_type_e const rt, int const content_length,
-                            std::string const &response) {
-    http_result_obtained(rt, content_length, response);
+                            std::string const &response,
+                            std::string const &body) {
+    http_result_obtained(rt, content_length, response, body);
   });
 }
 
@@ -392,8 +406,9 @@ void custom_resolver_socket_t::send_https_request(std::string const &address) {
       http_request_handler_->request_.emplace<https_request_handler_t>(
           io_, *ssl_context_, uri_t{address}.host());
   return https_request.start(
-      [this](auto const rt, auto const len, auto const &rstr) {
-        http_result_obtained(rt, len, rstr);
+      [this](auto const rt, auto const len, auto const &rstr,
+             auto const &body) {
+        http_result_obtained(rt, len, rstr, body);
       });
 }
 
@@ -413,14 +428,16 @@ void custom_resolver_socket_t::on_http_resolve_error() {
 
 void custom_resolver_socket_t::http_result_obtained(
     response_type_e const rt, int const content_length,
-    std::string const &response_string) {
+    std::string const &response_string, std::string const &body) {
 
   switch (rt) {
   case response_type_e::bad_request: {
+    run_final_body_checks(regex_checks_, name_, 400, content_length, body);
     result_map_.insert(name_, content_length, 400);
     return dns_continue_probe();
   }
   case response_type_e::forbidden: {
+    run_final_body_checks(regex_checks_, name_, 403, content_length, body);
     result_map_.insert(name_, content_length, 403);
     return dns_continue_probe();
   }
@@ -447,10 +464,12 @@ void custom_resolver_socket_t::http_result_obtained(
     return send_https_request(response_string);
   }
   case response_type_e::not_found: { // HTTP(S) 404
+    run_final_body_checks(regex_checks_, name_, 404, content_length, body);
     result_map_.insert(name_, content_length, 404);
     return dns_continue_probe();
   }
   case response_type_e::ok: {
+    run_final_body_checks(regex_checks_, name_, 200, content_length, body);
     result_map_.insert(name_, content_length, 200);
     return dns_continue_probe();
   }
@@ -477,6 +496,7 @@ void custom_resolver_socket_t::http_result_obtained(
     return send_https_request(response_string);
   }
   case response_type_e::server_error: {
+    run_final_body_checks(regex_checks_, name_, 503, content_length, body);
     result_map_.insert(name_, content_length, 503);
     return dns_continue_probe();
   }