From 26e75a8e1b40855f8fd03de211d03e47658ff404 Mon Sep 17 00:00:00 2001 From: dmind44 Date: Mon, 19 Oct 2020 22:23:19 -0700 Subject: [PATCH 01/11] added new test cases for urls --- basis/urls/urls-tests.factor | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/basis/urls/urls-tests.factor b/basis/urls/urls-tests.factor index fb14e1ecba3..641336fce29 100644 --- a/basis/urls/urls-tests.factor +++ b/basis/urls/urls-tests.factor @@ -115,6 +115,31 @@ CONSTANT: urls { } "t1000://www.google.com/" } + { + T{ url + { protocol "no-auth" } + { path "/some/random/path" } + } + "no-auth:/some/random/path" + } + { + T{ url + { protocol "https" } + { host "www.google.com" } + { path "/" } + } + "https://www.google.com:/" + } + { + T{ url + { protocol "http" } + { host "example.org" } + { path "/" } + { username "user" } + { password "" } + } + "http://user:@example.org/" + } } urls [ From 54a9a72baf3bcd8b4b68625c8bf88734b6d223f9 Mon Sep 17 00:00:00 2001 From: David Flores Date: Tue, 20 Oct 2020 21:35:34 -0700 Subject: [PATCH 02/11] Parsing URLs that do not have a authority component --- basis/urls/urls.factor | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/basis/urls/urls.factor b/basis/urls/urls.factor index 7439b5d3595..7f41d2cf2c0 100644 --- a/basis/urls/urls.factor +++ b/basis/urls/urls.factor @@ -58,7 +58,8 @@ auth = (username (":" password => [[ second ]])? "@" => [[ first2 2array ]])? url = (((protocol "://") => [[ first ]] auth hostname) - | (("//") => [[ f ]] auth hostname))? + | (("//") => [[ f ]] auth hostname) + | ((protocol ":") => [[ first V{ V{ f f } } swap prefix ]]))? (pathname)? ("?" query => [[ second ]])? ("#" anchor => [[ second ]])? From 7762ac18552bcb3d4829c08e5cd4ff4eacfffdbc Mon Sep 17 00:00:00 2001 From: David Flores Date: Tue, 20 Oct 2020 22:04:16 -0700 Subject: [PATCH 03/11] Increasing the robustness of the URL parser for passwords --- basis/urls/urls.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/basis/urls/urls.factor b/basis/urls/urls.factor index 7f41d2cf2c0..d8d56ade454 100644 --- a/basis/urls/urls.factor +++ b/basis/urls/urls.factor @@ -45,7 +45,7 @@ EBNF: parse-url [=[ protocol = [a-zA-Z0-9.+-]+ => [[ url-decode ]] username = [^/:@#?]+ => [[ url-decode ]] -password = [^/:@#?]+ => [[ url-decode ]] +password = [^/:@#?]* => [[ url-decode ]] pathname = [^#?]+ => [[ url-decode ]] query = [^#]+ => [[ query>assoc ]] anchor = .+ => [[ url-decode ]] From 1453d991304ba15a228cc79eeb30ab62c5f31a29 Mon Sep 17 00:00:00 2001 From: David Flores Date: Tue, 20 Oct 2020 23:13:18 -0700 Subject: [PATCH 04/11] Rewrote the no authority case for urls and updated prettyprinting to reflect the change. --- basis/urls/urls.factor | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/basis/urls/urls.factor b/basis/urls/urls.factor index d8d56ade454..54ee7c48ec8 100644 --- a/basis/urls/urls.factor +++ b/basis/urls/urls.factor @@ -59,7 +59,7 @@ auth = (username (":" password => [[ second ]])? "@" url = (((protocol "://") => [[ first ]] auth hostname) | (("//") => [[ f ]] auth hostname) - | ((protocol ":") => [[ first V{ V{ f f } } swap prefix ]]))? + | ((protocol ":") => [[ first V{ f f } V{ } 2sequence ]]))? (pathname)? ("?" query => [[ second ]])? ("#" anchor => [[ second ]])? @@ -118,20 +118,22 @@ M: pathname >url string>> >url ; ! URL" //foo.com" takes on the protocol of the url it's derived from : unparse-protocol ( url -- ) - dup protocol>> [ - % "://" % unparse-host-part + protocol>> [ + % ":" % + ] when* ; + +: unparse-authority ( url -- ) + dup host>> [ + "//" % unparse-host-part ] [ - dup host>> [ - "//" % unparse-host-part - ] [ - drop - ] if - ] if* ; + drop + ] if ; M: url present [ { [ unparse-protocol ] + [ unparse-authority ] [ path>> url-encode % ] [ query>> dup assoc-empty? [ drop ] [ "?" % assoc>query % ] if ] [ anchor>> [ "#" % present url-encode % ] when* ] From 0e8bace0cd14548b2f87500611e0cd577fbbf43a Mon Sep 17 00:00:00 2001 From: dmind44 Date: Wed, 21 Oct 2020 15:35:34 -0700 Subject: [PATCH 05/11] fixed matching empty port --- basis/urls/urls.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/basis/urls/urls.factor b/basis/urls/urls.factor index 54ee7c48ec8..dabf0898b84 100644 --- a/basis/urls/urls.factor +++ b/basis/urls/urls.factor @@ -30,7 +30,7 @@ ERROR: malformed-port ; : parse-host ( string -- host/f port/f ) [ ":" split1-last [ url-decode ] - [ dup [ string>number [ malformed-port ] unless* ] when ] bi* + [ dup [ dup empty? [ drop f ] [ string>number [ malformed-port ] unless* ] if ] when ] bi* ] [ f f ] if* ; GENERIC: >url ( obj -- url ) From 43433f8db7ae91cf7aaf1db34831e58a54c3d61b Mon Sep 17 00:00:00 2001 From: dmind44 Date: Thu, 22 Oct 2020 14:47:14 -0700 Subject: [PATCH 06/11] added upper bound for port number --- basis/urls/urls.factor | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/basis/urls/urls.factor b/basis/urls/urls.factor index dabf0898b84..91c736cb04a 100644 --- a/basis/urls/urls.factor +++ b/basis/urls/urls.factor @@ -30,7 +30,10 @@ ERROR: malformed-port ; : parse-host ( string -- host/f port/f ) [ ":" split1-last [ url-decode ] - [ dup [ dup empty? [ drop f ] [ string>number [ malformed-port ] unless* ] if ] when ] bi* + [ dup [ dup empty? [ drop f ] [ string>number [ + malformed-port ] unless* dup 65535 > over 0 < or [ + malformed-port ] + when ] if ] when ] bi* ] [ f f ] if* ; GENERIC: >url ( obj -- url ) From 997562e47a3a7b67ed3981cad8ef75a7e687912c Mon Sep 17 00:00:00 2001 From: David Flores Date: Tue, 27 Oct 2020 20:25:35 -0700 Subject: [PATCH 07/11] Updated the url parsing to accept an empty username. --- basis/urls/urls.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/basis/urls/urls.factor b/basis/urls/urls.factor index 91c736cb04a..6c6b0755a15 100644 --- a/basis/urls/urls.factor +++ b/basis/urls/urls.factor @@ -47,7 +47,7 @@ M: url >url ; EBNF: parse-url [=[ protocol = [a-zA-Z0-9.+-]+ => [[ url-decode ]] -username = [^/:@#?]+ => [[ url-decode ]] +username = [^/:@#?]* => [[ url-decode ]] password = [^/:@#?]* => [[ url-decode ]] pathname = [^#?]+ => [[ url-decode ]] query = [^#]+ => [[ query>assoc ]] From 2f6da6d81e8714663ecd12ed4de1595c3ebbdc38 Mon Sep 17 00:00:00 2001 From: dmind44 Date: Tue, 27 Oct 2020 20:30:50 -0700 Subject: [PATCH 08/11] check if port is valid --- basis/urls/urls.factor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/basis/urls/urls.factor b/basis/urls/urls.factor index 91c736cb04a..f7cfbcc019f 100644 --- a/basis/urls/urls.factor +++ b/basis/urls/urls.factor @@ -5,7 +5,7 @@ USING: accessors arrays ascii assocs combinators fry io.pathnames io.sockets io.sockets.secure kernel lexer linked-assocs make math.parser multiline namespaces peg.ebnf present sequences splitting strings strings.parser urls.encoding -vocabs.loader ; +vocabs.loader math ; IN: urls From 95b9866025fffa9bda8a4570cab7afe8d3bcc8bb Mon Sep 17 00:00:00 2001 From: David Flores Date: Tue, 27 Oct 2020 21:19:46 -0700 Subject: [PATCH 09/11] Updated the url tests to test new extensions to the url library. --- basis/urls/urls-tests.factor | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/basis/urls/urls-tests.factor b/basis/urls/urls-tests.factor index 641336fce29..0447be47e65 100644 --- a/basis/urls/urls-tests.factor +++ b/basis/urls/urls-tests.factor @@ -124,21 +124,23 @@ CONSTANT: urls { } { T{ url - { protocol "https" } - { host "www.google.com" } + { protocol "http" } + { host "example.org" } { path "/" } + { username "user" } + { password "" } } - "https://www.google.com:/" + "http://user:@example.org/" } { T{ url { protocol "http" } { host "example.org" } { path "/" } - { username "user" } - { password "" } + { username "" } + { password "pass" } } - "http://user:@example.org/" + "http://:pass@example.org/" } } @@ -150,6 +152,20 @@ urls [ swap [ 1array ] [ [ present ] curry ] bi* unit-test ] assoc-each +{ T{ url + { protocol "https" } + { host "www.google.com" } + { path "/" } + } } +[ "https://www.google.com:/" >url ] unit-test + +{ "https://www.google.com/" } +[ T{ url + { protocol "https" } + { host "www.google.com" } + { path "/" } +} present ] unit-test + { "b" } [ "a" "b" url-append-path ] unit-test { "a/b" } [ "a/c" "b" url-append-path ] unit-test From 710c44bc504b11e3013b3bd6df91a21f4e851344 Mon Sep 17 00:00:00 2001 From: David Flores Date: Wed, 28 Oct 2020 11:27:17 -0700 Subject: [PATCH 10/11] Updating the style for recently added extensions to urls.factor. --- basis/urls/urls.factor | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/basis/urls/urls.factor b/basis/urls/urls.factor index 10a768fa160..073e8ab2f6e 100644 --- a/basis/urls/urls.factor +++ b/basis/urls/urls.factor @@ -5,7 +5,7 @@ USING: accessors arrays ascii assocs combinators fry io.pathnames io.sockets io.sockets.secure kernel lexer linked-assocs make math.parser multiline namespaces peg.ebnf present sequences splitting strings strings.parser urls.encoding -vocabs.loader math ; +vocabs.loader math math.order ; IN: urls @@ -30,10 +30,10 @@ ERROR: malformed-port ; : parse-host ( string -- host/f port/f ) [ ":" split1-last [ url-decode ] - [ dup [ dup empty? [ drop f ] [ string>number [ - malformed-port ] unless* dup 65535 > over 0 < or [ - malformed-port ] - when ] if ] when ] bi* + [ [ f ] + [ string>number [ malformed-port ] unless* + dup 0 65535 between? [ malformed-port ] unless ] + if-empty ] bi* ] [ f f ] if* ; GENERIC: >url ( obj -- url ) @@ -121,16 +121,10 @@ M: pathname >url string>> >url ; ! URL" //foo.com" takes on the protocol of the url it's derived from : unparse-protocol ( url -- ) - protocol>> [ - % ":" % - ] when* ; + protocol>> [ % ":" % ] when* ; : unparse-authority ( url -- ) - dup host>> [ - "//" % unparse-host-part - ] [ - drop - ] if ; + dup host>> [ "//" % unparse-host-part ] [ drop ] if ; M: url present [ From 5c729cf87641bae31a7fe5b9893705179280301e Mon Sep 17 00:00:00 2001 From: dmind44 Date: Wed, 28 Oct 2020 21:48:40 -0700 Subject: [PATCH 11/11] removed sanity check for port, want to allow for IPvX support --- basis/urls/urls.factor | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/basis/urls/urls.factor b/basis/urls/urls.factor index 073e8ab2f6e..0a1deade940 100644 --- a/basis/urls/urls.factor +++ b/basis/urls/urls.factor @@ -30,10 +30,10 @@ ERROR: malformed-port ; : parse-host ( string -- host/f port/f ) [ ":" split1-last [ url-decode ] - [ [ f ] - [ string>number [ malformed-port ] unless* - dup 0 65535 between? [ malformed-port ] unless ] - if-empty ] bi* + [ [ f ] + [ string>number [ malformed-port ] unless* ] + if-empty + ] bi* ] [ f f ] if* ; GENERIC: >url ( obj -- url )