From 52d2b7e7660ed11e8dabfa024fd6fc816466a066 Mon Sep 17 00:00:00 2001 From: David Scarpetti Date: Thu, 15 Feb 2024 17:40:43 -0700 Subject: [PATCH 1/6] allow unsorted maps and sets to be used as keys in paths --- .gitattributes | 1 + CHANGELOG.md | 4 ++++ README.md | 4 ++++ src/codax/pathwise.clj | Bin 7969 -> 8093 bytes test/codax/pathwise_test.clj | 34 +++++++++++++++++++++++++++------- 5 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6ffaf6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +pathwise.clj diff \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d058c6..05c71a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change Log +## 1.5.0 (2024-02-15 + +* Allow unsorted maps and sets to be used as keys in paths + ## 1.4.0 (2023-07-20) * implement upgradable transactions (using `with-upgradable-transaction` macro) diff --git a/README.md b/README.md index 8f4a15f..ef76c8e 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ Codax is an idiomatic transactional embedded database for clojure. A codax datab [![Clojars Project](http://clojars.org/codax/latest-version.svg)](http://clojars.org/codax) +Version 1.5.0 allows unsorted sets and maps to be used as [path](#paths) keys. + Version 1.4.0 implements [upgradable transactions](doc/upgradable-transactions.md). (using `with-upgradable-transaction` macro) and **fixes an RCE vulnerability**. See the [changelog](CHANGELOG.md) for details on upgrading from earlier codax versions. @@ -103,6 +105,8 @@ A `path` is a vector of keys similar to the `[k & ks]` used in function like `as - nil - java.time.Instant - org.joda.time.DateTime + - unsorted maps + - unsorted sets - the path can only target nested maps, and **cannot be used to descend into other data structures (e.g. vectors)**. - you can get the empty path (e.g. `(get-at db [])` returns the full database) but you cannot modify it (e.g. `(assoc-at [] :foo)` throws an error) diff --git a/src/codax/pathwise.clj b/src/codax/pathwise.clj index be7123ffbdf17ebe641ba8782c3f9dbfac1779b2..13621dc4315f34632e96b2bc99b9df33c0a95a1c 100644 GIT binary patch delta 1520 zcmb7E&2G~`5T;aWQBDo562obTDXf(`ZF)q7hyxNA#DNRQ1ZC@N+NA!|?xw=3>bw94 zDvKkBJ_6!B5O2ek8QV=9ng;M8v3GvHnfboi%V%FZpLylO6E_T$=7GNG)xKS{s}qZ~1w*fcVUt;*Xm#)uC~^u#AUY(ik!q0gyEFVP|u2r0`8p<3k&hm|egY zXmmEZU>c+z7#guK4Q-36UbU=ly0U4cyY)3c3nsTwR>U-qh5Qw@=YPpeDNA8dPm5s* z^|E$9ZPnLD|ACq-Zz&|Y+zeSXilW0Q{ad4%Oq)RbUuOn@2)oIaWJKfWJ0;E1B(1q& zqR{e=PE0tRaYjrNHCr^K3Bfb2ROsf^WWLvU!Mr^tg*%qSZMnlj2$XdCWV_~iTnb#Z zsZ5GjqON$wk&0q5=l8J`b#|U#6bl_Fm(yM1K%|*x=jnru&Gbk0pa!lNj>b~7JRTgi zUh(Kfpkn(PWB6Vu;7_6cO2K(J7Hcb`kiWBgc%XdlLqukuo8 kDvK?F{#AdwvXj0x?yU|&nFfY&M+&ZR!*{-aGQO<+2H@|;CjbBd delta 1112 zcmb7@O>Wab6vtPDHbg*3NbODauu_;&6q;^WpaydUmPnQtjy+XO{iT`bmZC}(H$WP( zL=`81igje(eq9)^1l`_d0j8IB@}KeX4A&XH!6SGr(QvX^;&+n>|sP0)lSE=Snc>3#6pxzeA}`= GwtfL=H&O%u diff --git a/test/codax/pathwise_test.clj b/test/codax/pathwise_test.clj index 3827dc5..fe461e7 100644 --- a/test/codax/pathwise_test.clj +++ b/test/codax/pathwise_test.clj @@ -148,7 +148,7 @@ ;;;; encoding decoding errors -(deftest no-encoder-for-element +#_(deftest no-encoder-for-element (try (do (encode #{}) @@ -159,7 +159,7 @@ (is (= type clojure.lang.PersistentHashSet)) (is (= element #{})))))) -(deftest no-decoder-for-element +#_(deftest no-decoder-for-element (try (do (decode (str (char 0x2) #{:unrecognized} (char 0x00))) @@ -216,12 +216,14 @@ (reduce str "" (take length (iterate (fn [_] (char (+ min-char (int (rand max-char))))) nil))))) -(declare gen-random-vector gen-random-element) +(declare gen-random-vector gen-random-element gen-random-set gen-random-map) -(defn gen-random-element [& {:keys [max-vector-length]}] - (let [num (rand 12)] +(defn gen-random-element [& {:keys [max-length]}] + (let [num (rand 16)] (cond - (> num 10) (gen-random-vector :max-length max-vector-length) + (> num 14) (gen-random-map :max-length max-length) + (> num 12) (gen-random-set :max-length max-length) + (> num 10) (gen-random-vector :max-length max-length) (> num 9) nil (> num 8) true (> num 7) false @@ -238,9 +240,27 @@ (let [max-length (if max-length (dec max-length) 15) length (int (Math/floor (rand max-length)))] (if (> length 0) - (vec (take (inc length) (rest (iterate (fn [_] (gen-random-element :max-vector-length max-length)) nil)))) + (vec (take (inc length) (rest (iterate (fn [_] (gen-random-element :max-length max-length)) nil)))) []))) +(defn gen-random-set [& {:keys [max-length]}] + (let [max-length (if max-length (dec max-length) 15) + length (int (Math/floor (rand max-length)))] + (if (> length 0) + (set (take (inc length) (rest (iterate (fn [_] (gen-random-element :max-length max-length)) nil)))) + #{}))) + +(defn gen-random-map [& {:keys [max-length]}] + (let [max-length (if max-length (dec max-length) 15) + length (int (Math/floor (rand max-length)))] + (if (> length 0) + (->> (iterate (fn [_] (gen-random-element :max-length max-length)) nil) + rest + (take (inc length)) + (partition 2) + (map vec) + (into {})) + {}))) (defn test-random [& {:keys [hide-success]}] (test-encoding (gen-random-vector) :hide-success hide-success)) From 4f33b2656b9e1f38fd34710e3cf5a7b124a427cd Mon Sep 17 00:00:00 2001 From: David Scarpetti Date: Fri, 6 Jun 2025 14:32:40 -0600 Subject: [PATCH 2/6] reserve 0xa1-0xaf --- CHANGELOG.md | 7 ++++++- README.md | 2 +- doc/types.md | 3 ++- src/codax/pathwise.clj | Bin 8093 -> 8471 bytes 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c71a1..5be728c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,12 @@ ## 1.5.0 (2024-02-15 -* Allow unsorted maps and sets to be used as keys in paths +* Allow unsorted maps and sets to be used as keys in paths using codes `0xa0` `0xa1`. +* Reserve all codes `0xa2` - `0xaf` for future use. + +Sorted maps and sets remain unsupported in paths. This is for several reasons including that the sorting criteria cannot be reliably encoded. + +*Note: if you have user-defined encoding types using codes between `0xa0` and `0xaf` a warning will be printed but as long as you do not use the new supported types (e.g. maps & sets) in paths your code should continue to work.* ## 1.4.0 (2023-07-20) diff --git a/README.md b/README.md index ef76c8e..84c6096 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Codax is an idiomatic transactional embedded database for clojure. A codax datab [![Clojars Project](http://clojars.org/codax/latest-version.svg)](http://clojars.org/codax) -Version 1.5.0 allows unsorted sets and maps to be used as [path](#paths) keys. +Version 1.5.0 allows unsorted sets and maps to be used as [path](#paths) keys. It also reserves pathwise encoding codes `0xa0` - `0xaf`. Version 1.4.0 implements [upgradable transactions](doc/upgradable-transactions.md). (using `with-upgradable-transaction` macro) and **fixes an RCE vulnerability**. diff --git a/doc/types.md b/doc/types.md index f08bd20..b53b458 100644 --- a/doc/types.md +++ b/doc/types.md @@ -7,7 +7,8 @@ It takes the following parameters: - `hex-code` - a number used for type identification in the database - `0x0` is used internally and off limits - - built-in codes which should **not be overwritten** are: `0x10` , `0x20`, `0x21`, `0x24`, `0x25`, `0x30`, `0x31`, `0x32`, `0x68`, `0x69`, `0x70`, `0xa0`. Attempts to overwrite them (or generally change encodings) should print a warning. + - built-in codes which should **not be overwritten** are: `0x10` , `0x20`, `0x21`, `0x24`, `0x25`, `0x30`, `0x31`, `0x32`, `0x68`, `0x69`, `0x70`, `0xa0`. Attempts to overwrite them (or generally change encodings) prints a warning. + - `0xa0` - `0xaf` are reserved for future use as of v1.5.0 - `types` - the types to be encoded (generally there should only be one) - `encoder` - a function which takes elements of the types specified in `types` and **returns a string representation** - `decoder` - a function which takes the string reprensentations generated by the encoder and reconstructs a value of the appropriate type diff --git a/src/codax/pathwise.clj b/src/codax/pathwise.clj index 13621dc4315f34632e96b2bc99b9df33c0a95a1c..cb80f286d2b8a6ea9e4357baee7c9ddb6b19373d 100644 GIT binary patch delta 321 zcmY+9y9&ZU5JkZ Date: Sun, 8 Jun 2025 11:05:55 -0600 Subject: [PATCH 3/6] fix map ordering in pathwise encoding see: https://github.com/dscarpetti/codax/pull/36#issuecomment-2954115653 --- src/codax/pathwise.clj | Bin 8471 -> 8462 bytes test/codax/pathwise_test.clj | 17 +++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/codax/pathwise.clj b/src/codax/pathwise.clj index cb80f286d2b8a6ea9e4357baee7c9ddb6b19373d..4ab418fc096525f7c16b4a04bf93caf32fdbbe6e 100644 GIT binary patch delta 16 XcmbR4)aSH8S7LITxZLItiNhQKH1Gx2 delta 21 ccmeBkn(nkgSAs(~sZt>=v#5Bpj>IDl08v>7WB>pF diff --git a/test/codax/pathwise_test.clj b/test/codax/pathwise_test.clj index 76ec7b2..ed458f4 100644 --- a/test/codax/pathwise_test.clj +++ b/test/codax/pathwise_test.clj @@ -280,3 +280,20 @@ (deftest random-path-test (is (test-many-random 100 :hide-success true))) + +(deftest map-ordering-1 + (is (= (encode {:a 1, {:k :b} 2}) + (encode {{:k :b} 2 :a 1})))) + +(deftest map-ordering-2 + (let [f1 {:a 1, {:k :b} 2} + e1 (encode f1) + d1 (decode e1) + f2 {{:k :b} 2 :a 1} + e2 (encode f2) + d2 (decode e2)] + (is (= f1 d1 f2 d2)))) + +(deftest set-ordering-1 + (is (= (encode #{#{:a :b} :c {:a 1}}) + (encode #{{:a 1} :c #{:b :a}})))) From f2636cc87f56c3eda108be17f266910765ef2ebf Mon Sep 17 00:00:00 2001 From: David Scarpetti Date: Sun, 8 Jun 2025 11:20:04 -0600 Subject: [PATCH 4/6] 1.5.0 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 2f6835a..8ff9f42 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject codax "1.4.3" +(defproject codax "1.5.0" :description "Codax is an idiomatic transactional embedded database for clojure" :url "https://github.com/dscarpetti/codax" :license {:name "Eclipse Public License" From a47afbc3bbad5aeb97dce3cd596b51b9d92643b9 Mon Sep 17 00:00:00 2001 From: David Scarpetti Date: Sun, 8 Jun 2025 11:24:12 -0600 Subject: [PATCH 5/6] remove references to clj-time --- src/codax/pathwise.clj | Bin 8462 -> 8269 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src/codax/pathwise.clj b/src/codax/pathwise.clj index 4ab418fc096525f7c16b4a04bf93caf32fdbbe6e..33619ce9871de8923345cd0585acac134802070a 100644 GIT binary patch delta 15 WcmeBkI_ogOYGcH9fz6u)WB36#bp}}g delta 203 zcmX@>(C0M4N|sAOK_NOhCrh^^GdEQ)Ex#x?u|&Ztu~;E1KP7RaW3|#G;bS#2m0z{n9)jw>VWn1Eg3VVsJ@n iQL%zmQetstvTh1cC)iR=O$@!rI%V-{-TX=*nI8aLLPui& From e9c5bc334fda97c3c70701832edd3b7e39f5a12a Mon Sep 17 00:00:00 2001 From: David Scarpetti Date: Sun, 8 Jun 2025 11:40:15 -0600 Subject: [PATCH 6/6] randomized map ordering tests --- test/codax/pathwise_test.clj | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/codax/pathwise_test.clj b/test/codax/pathwise_test.clj index ed458f4..87af715 100644 --- a/test/codax/pathwise_test.clj +++ b/test/codax/pathwise_test.clj @@ -292,8 +292,30 @@ f2 {{:k :b} 2 :a 1} e2 (encode f2) d2 (decode e2)] + (is (= e1 e2)) (is (= f1 d1 f2 d2)))) +(defn randomize-map-ordering [x] + (cond + (map? x) (into {} (shuffle (map (fn [[k v]] [k (randomize-map-ordering v)]) x))) + (set? x) (into #{} (map randomize-map-ordering x)) + (vector? x) (mapv randomize-map-ordering x) + :else x)) + +(deftest random-map-ordering + (loop [n 1000] + (when (pos? n) + (let [m1 (gen-random-map :max-length 12) + e1 (encode m1) + d1 (decode e1) + + m2 (randomize-map-ordering m1) + e2 (encode m2) + d2 (decode e2)] + (is (= e1 e2)) + (is (= m1 m2 d1 d2)) + (recur (dec n)))))) + (deftest set-ordering-1 (is (= (encode #{#{:a :b} :c {:a 1}}) (encode #{{:a 1} :c #{:b :a}}))))