fetch_tests.patch

# HG changeset patch
# User Dmitry Volyntsev <xeioex@nginx.com>
# Date 1608753654 0
#      Wed Dec 23 20:00:54 2020 +0000
# Node ID a6792afeeac26e8747dd257cfc7ba96a05b22d28
# Parent  54f867e69cb536a749e7d9f59c94ffdd8b83f72a
Tests: adapted js stream tests to js_preread changes.

diff --git a/stream_js.t b/stream_js.t
--- a/stream_js.t
+++ b/stream_js.t
@@ -219,9 +219,7 @@ EOF
     function js_preread_step(s) {
         s.on('upload', function (data) {
             res += '2';
-            if (res.length >= 3) {
-                s.done();
-            }
+            s.done();
         });
     }
 
@@ -369,7 +367,7 @@ is(stream('127.0.0.1:' . port(8082))->re
 is(stream('127.0.0.1:' . port(8083))->read(), '', 'stream js unknown function');
 is(stream('127.0.0.1:' . port(8084))->read(), 'sess_unk=undefined', 's.unk');
 
-is(stream('127.0.0.1:' . port(8086))->io('0'), '0122345',
+is(stream('127.0.0.1:' . port(8086))->io('0'), '012345',
 	'async handlers order');
 is(stream('127.0.0.1:' . port(8087))->io('#'), 'OK', 'js_access_undecided');
 is(stream('127.0.0.1:' . port(8088))->io('#'), 'OK', 'js_access_allow');
diff --git a/stream_js_import.t b/stream_js_import.t
--- a/stream_js_import.t
+++ b/stream_js_import.t
@@ -73,9 +73,7 @@ EOF
     function preread(s) {
         s.on('upload', function (data) {
             res += '2';
-            if (res.length >= 3) {
-                s.done();
-            }
+            s.done();
         });
     }
 
@@ -112,6 +110,6 @@ EOF
 ###############################################################################
 
 is(stream('127.0.0.1:' . port(8081))->read(), 'P-TEST', 'foo.bar.p');
-is(stream('127.0.0.1:' . port(8082))->io('0'), 'x122345', 'lib.access');
+is(stream('127.0.0.1:' . port(8082))->io('0'), 'x12345', 'lib.access');
 
 ###############################################################################
# HG changeset patch
# User Dmitry Volyntsev <xeioex@nginx.com>
# Date 1609271069 0
#      Tue Dec 29 19:44:29 2020 +0000
# Node ID ffd768dc2e796fde5a0f505e59716650243b6cc5
# Parent  a6792afeeac26e8747dd257cfc7ba96a05b22d28
Tests: added js tests for ngx.fetch() method.

diff --git a/js_fetch.t b/js_fetch.t
new file mode 100644
--- /dev/null
+++ b/js_fetch.t
@@ -0,0 +1,527 @@
+#!/usr/bin/perl
+
+# (C) Dmitry Volyntsev
+# (C) Nginx, Inc.
+
+# Tests for http njs module, fetch method.
+
+###############################################################################
+
+use warnings;
+use strict;
+
+use Test::More;
+
+use Socket qw/ CRLF /;
+
+BEGIN { use FindBin; chdir($FindBin::Bin); }
+
+use lib 'lib';
+use Test::Nginx;
+
+###############################################################################
+
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+eval { require JSON::PP; };
+plan(skip_all => "JSON::PP not installed") if $@;
+
+my $t = Test::Nginx->new()->has(qw/http rewrite/)
+	->write_file_expand('nginx.conf', <<'EOF');
+
+%%TEST_GLOBALS%%
+
+daemon off;
+
+events {
+}
+
+http {
+    %%TEST_GLOBALS_HTTP%%
+
+    js_import test.js;
+
+    server {
+        listen       127.0.0.1:8080;
+        server_name  localhost;
+
+        location /njs {
+            js_content test.njs;
+        }
+
+        location /fetch {
+            js_content test.fetch;
+        }
+
+        location /broken {
+            js_content test.broken;
+        }
+
+        location /broken_response {
+            js_content test.broken_response;
+        }
+
+        location /body {
+            js_content test.body;
+        }
+
+        location /body_used {
+            js_content test.body_used;
+        }
+
+        location /chunked {
+            js_content test.chunked;
+        }
+
+        location /header {
+            js_content test.header;
+        }
+
+        location /multi {
+            js_content test.multi;
+        }
+
+        location /chain {
+            js_content test.chain;
+        }
+    }
+
+    server {
+        listen       127.0.0.1:8080;
+        server_name  aaa;
+
+        location /loc {
+            js_content test.loc;
+        }
+
+        location /json {
+            return 200 '{"a":[1,2], "b":{"c":"FIELD"}}';
+        }
+    }
+
+    server {
+        listen       127.0.0.1:8080;
+        server_name  bbb;
+
+        location /loc {
+            js_content test.loc;
+        }
+    }
+
+    server {
+        listen       127.0.0.2:8081;
+        server_name  ccc;
+
+        location /loc {
+            js_content test.loc;
+        }
+    }
+}
+
+EOF
+
+$t->write_file('test.js', <<EOF);
+    function test_njs(r) {
+        r.return(200, njs.version);
+    }
+
+    function str(v) { return v ? v : ''};
+
+    function body(r) {
+        var loc = r.args.loc;
+        var getter = r.args.getter;
+
+        function query(obj) {
+            var path = r.args.path;
+            var retval = (getter == 'arrayBuffer') ? Buffer.from(obj).toString()
+                                                   : obj;
+
+            if (path) {
+                retval = path.split('.').reduce((a, v) => a[v], obj);
+            }
+
+            return JSON.stringify(retval);
+        }
+
+        ngx.fetch(`http://127.0.0.1:8080/\${loc}`, {headers: {Host: 'AAA'}})
+        .then(reply => reply.body[getter]())
+        .then(data => r.return(200, query(data)))
+        .catch(e => r.return(501, e.message))
+    }
+
+    function body_used(r) {
+        ngx.fetch(`http://127.0.0.1:8080/loc`, {headers: {Host: 'AAA'}})
+        .then(reply => r.return(200, reply.bodyUsed))
+        .catch(e => r.return(501, e.message))
+    }
+
+    function process_errors(r, tests) {
+        var results = [];
+
+        tests.forEach(args => {
+            ngx.fetch.apply(r, args)
+            .then(reply => {
+                r.return(400, '["unexpected then"]');
+            })
+            .catch(e => {
+                results.push(e.message);
+
+                if (results.length == tests.length) {
+                    results.sort();
+                    r.return(200, JSON.stringify(results));
+                }
+            })
+        })
+    }
+
+    function broken(r) {
+        var tests = [
+            ['http://127.0.0.1:1/loc'],
+            ['http://127.0.0.1:80800/loc'],
+            ['http://localhost:8080/loc'],
+            [Symbol.toStringTag],
+            ['https://localhost:8080/loc'],
+            ['http://127.0.0.0:8080/loc']
+        ];
+
+        return process_errors(r, tests);
+    }
+
+    function broken_response(r) {
+        var tests = [
+            ['http://127.0.0.1:8082/status_line'],
+            ['http://127.0.0.1:8082/length'],
+            ['http://127.0.0.1:8082/header'],
+            ['http://127.0.0.1:8082/headers'],
+            ['http://127.0.0.1:8082/content_length'],
+        ];
+
+        return process_errors(r, tests);
+    }
+
+    function chain(r) {
+        var results = [];
+        var reqs = [
+             [ 'http://127.0.0.1:8080/loc', { headers: {Host: 'AAA'}} ],
+             [ 'http://127.0.0.2:8081/loc', { headers: {Host: 'CCC'}} ],
+             [ 'http://127.0.0.1:8080/loc', { headers: {Host: 'BBB'}} ],
+           ];
+
+           function next(reply) {
+              if (reqs.length == 0) {
+                 r.return(200, "SUCCESS");
+                 return;
+              }
+
+              ngx.fetch.apply(r, reqs.pop())
+              .then(next)
+              .catch(e => r.return(400, e.message))
+           }
+
+           next()
+    }
+
+    function chunked(r) {
+        var results = [];
+        var tests = [
+            ['http://127.0.0.1:8082/big', {max_response_body_size:128000}],
+            ['http://127.0.0.1:8082/big/ok', {max_response_body_size:128000}],
+            ['http://127.0.0.1:8082/chunked'],
+            ['http://127.0.0.1:8082/chunked/ok'],
+            ['http://127.0.0.1:8082/chunked/big', {max_response_body_size:128}],
+            ['http://127.0.0.1:8082/chunked/big'],
+        ];
+
+        function collect(v) {
+            results.push(v);
+
+            if (results.length == tests.length) {
+                results.sort();
+                r.return(200, JSON.stringify(results));
+            }
+        }
+
+        tests.forEach(args => {
+            ngx.fetch.apply(r, args)
+            .then(reply => reply.body.text())
+            .then(body => collect(body.length))
+            .catch(e => collect(e.message))
+        })
+    }
+
+    function header(r) {
+        var url = `http://127.0.0.1:8082/\${r.args.loc}`;
+		var method = r.args.method ? r.args.method : 'get';
+
+        ngx.fetch(url, {headers: {Host: 'AAA'}})
+        .then(reply => r.return(200, reply.headers[method](r.args.h)))
+        .catch(e => r.return(501, e.message))
+    }
+
+    function multi(r) {
+        var results = [];
+        var tests = [
+             [
+              'http://127.0.0.1:8080/loc',
+               { headers: {Host: 'AAA', Code: 201}},
+             ],
+             [
+              'http://127.0.0.1:8080/loc',
+               { method:'POST', headers: {Host: 'BBB', Code: 401}, body: 'OK'},
+             ],
+             [
+              'http://127.0.0.2:8081/loc',
+               { method:'PATCH', headers: {Host: 'CCC', foo:undefined,
+                                           bar:'xxx'}},
+             ],
+           ];
+
+        function cmp(a,b) {
+            if (a.body > b.body) {return 1;}
+            if (a.body < b.body) {return -1;}
+            return 0
+        }
+
+        tests.forEach(args => {
+            ngx.fetch.apply(r, args)
+            .then(reply =>
+                 reply.body.text().then(body =>
+                                        {reply.text = body; return reply;}))
+            .then(reply => {
+                results.push({body:reply.text,
+                              used:reply.bodyUsed,
+                              code:reply.status,
+                              ok:reply.ok,
+                              type:reply.type,
+                              url:reply.url});
+
+                if (results.length == tests.length) {
+                    results.sort(cmp);
+                    r.return(200, JSON.stringify(results));
+                }
+            })
+            .catch(e => {
+                r.return(400, `["\${e.message}"]`);
+                throw e;
+            })
+        })
+
+        if (r.args.throw) {
+            throw 'Oops';
+        }
+    }
+
+    function loc(r) {
+        var v = r.variables;
+        var body = str(r.requestText);
+        var foo = str(r.headersIn.foo);
+        var bar = str(r.headersIn.bar);
+        var code = r.headersIn.code ? Number(r.headersIn.code) : 200;
+        r.return(code, `\${v.host}:\${v.request_method}:\${foo}:\${bar}:\${body}`);
+    }
+
+     export default {njs: test_njs, body, body_used, broken, broken_response,
+                     chain, chunked, header, multi, loc}
+EOF
+
+$t->try_run('no njs.fetch')->plan(15);
+$t->run_daemon(\&http_daemon, port(8082));
+$t->waitforsocket('127.0.0.1:' . port(8082));
+
+###############################################################################
+
+TODO: {
+local $TODO = 'not yet'
+	unless http_get('/njs') =~ /^([.0-9]+)$/m && $1 ge '0.5.1';
+
+like(http_get('/body?getter=arrayBuffer&loc=loc'), qr/200 OK.*"aaa:GET:::"$/s,
+	'fetch body arrayBuffer');
+like(http_get('/body?getter=text&loc=loc'), qr/200 OK.*"aaa:GET:::"$/s,
+	'fetch body text');
+like(http_get('/body?getter=json&loc=json&path=b.c'),
+	qr/200 OK.*"FIELD"$/s, 'fetch body json');
+like(http_get('/body?getter=json&loc=loc'), qr/501/s,
+	'fetch body json invalid');
+like(http_get('/body_used'), qr/false$/s,
+	'fetch bodyUsed');
+like(http_get('/header?loc=duplicate_header&h=BAR'), qr/200 OK.*c$/s,
+	'fetch header');
+like(http_get('/header?loc=duplicate_header&h=foo'), qr/200 OK.*a,b$/s,
+	'fetch header duplicate');
+like(http_get('/header?loc=duplicate_header&h=bar&method=has'),
+	qr/200 OK.*true$/s, 'fetch header has');
+like(http_get('/header?loc=duplicate_header&h=buz&method=has'),
+	qr/200 OK.*false$/s, 'fetch header does not have');
+is(get_json('/multi'),
+	'[{"body":"aaa:GET:::","code":201,"ok":true,"type":"basic",' .
+	'"url":"http://127.0.0.1:8080/loc","used":true},' .
+	'{"body":"bbb:POST:::OK","code":401,"ok":false,"type":"basic",' .
+	'"url":"http://127.0.0.1:8080/loc","used":true},' .
+	'{"body":"ccc:PATCH::xxx:","code":200,"ok":true,"type":"basic",' .
+	'"url":"http://127.0.0.2:8081/loc","used":true}]', 'fetch multi');
+like(http_get('/multi?throw=1'), qr/500/s,
+	'fetch destructor');
+is(get_json('/broken'),
+	'["DNS addresses are not supported yet",' .
+	'"connect failed",' .
+	'"failed to convert url arg",' .
+	'"invalid url",' .
+	'"prematurely closed connection",' .
+	'"unsupported URL prefix"]', 'fetch broken');
+is(get_json('/broken_response'),
+	'["invalid fetch content length",' .
+	'"invalid fetch header",' .
+	'"invalid fetch status line",' .
+	'"prematurely closed connection",' .
+	'"prematurely closed connection"]', 'fetch broken response');
+is(get_json('/chunked'),
+	'[10,100010,25500,' .
+	'"invalid fetch chunked response",' .
+	'"prematurely closed connection",' .
+	'"very large fetch chunked response"]', 'fetch chunked');
+like(http_get('/chain'), qr/200 OK.*SUCCESS$/s,
+	'fetch chain');
+
+}
+
+###############################################################################
+
+sub recode {
+	my $json;
+	eval { $json = JSON::PP::decode_json(shift) };
+
+	if ($@) {
+		return "<failed to parse JSON>";
+	}
+
+	JSON::PP->new()->canonical()->encode($json);
+}
+
+sub get_json {
+	http_get(shift) =~ /\x0d\x0a?\x0d\x0a?(.*)/ms;
+	recode($1);
+}
+
+###############################################################################
+
+sub http_daemon {
+	my $port = shift;
+
+	my $server = IO::Socket::INET->new(
+	Proto => 'tcp',
+	LocalAddr => '127.0.0.1:' . $port,
+	Listen => 5,
+	Reuse => 1
+	) or die "Can't create listening socket: $!\n";
+
+	local $SIG{PIPE} = 'IGNORE';
+
+	while (my $client = $server->accept()) {
+		$client->autoflush(1);
+
+		my $headers = '';
+		my $uri = '';
+
+		while (<$client>) {
+			$headers .= $_;
+			last if (/^\x0d?\x0a?$/);
+		}
+
+		$uri = $1 if $headers =~ /^\S+\s+([^ ]+)\s+HTTP/i;
+
+		if ($uri eq '/status_line') {
+			print $client
+				"HTTP/1.1 2A";
+
+		} elsif ($uri eq '/content_length') {
+			print $client
+				"HTTP/1.1 200 OK" . CRLF .
+				"Content-Length: " . CRLF .
+				"Connection: close" . CRLF .
+				CRLF;
+
+		} elsif ($uri eq '/header') {
+			print $client
+				"HTTP/1.1 200 OK" . CRLF .
+				"@#" . CRLF .
+				"Connection: close" . CRLF .
+				CRLF;
+
+		} elsif ($uri eq '/duplicate_header') {
+			print $client
+				"HTTP/1.1 200 OK" . CRLF .
+				"Foo: a" . CRLF .
+				"bar: c" . CRLF .
+				"Foo: b" . CRLF .
+				"Connection: close" . CRLF .
+				CRLF;
+
+		} elsif ($uri eq '/headers') {
+			print $client
+				"HTTP/1.1 200 OK" . CRLF .
+				"Connection: close" . CRLF;
+
+		} elsif ($uri eq '/length') {
+			print $client
+				"HTTP/1.1 200 OK" . CRLF .
+				"Content-Length: 100" . CRLF .
+				"Connection: close" . CRLF .
+				CRLF .
+				"unfinished" . CRLF;
+
+		} elsif ($uri eq '/big') {
+			print $client
+				"HTTP/1.1 200 OK" . CRLF .
+				"Content-Length: 100100" . CRLF .
+				"Connection: close" . CRLF .
+				CRLF;
+			for (1 .. 1000) {
+				print $client ("X" x 98) . CRLF;
+			}
+			print $client "unfinished" . CRLF;
+
+		} elsif ($uri eq '/big/ok') {
+			print $client
+				"HTTP/1.1 200 OK" . CRLF .
+				"Content-Length: 100010" . CRLF .
+				"Connection: close" . CRLF .
+				CRLF;
+			for (1 .. 1000) {
+				print $client ("X" x 98) . CRLF;
+			}
+			print $client "finished" . CRLF;
+
+		} elsif ($uri eq '/chunked') {
+			print $client
+				"HTTP/1.1 200 OK" . CRLF .
+				"Transfer-Encoding: chunked" . CRLF .
+				"Connection: close" . CRLF .
+				CRLF .
+				"ff" . CRLF .
+				"unfinished" . CRLF;
+
+		} elsif ($uri eq '/chunked/ok') {
+			print $client
+				"HTTP/1.1 200 OK" . CRLF .
+				"Transfer-Encoding: chunked" . CRLF .
+				"Connection: close" . CRLF .
+				CRLF .
+				"a" . CRLF .
+				"finished" . CRLF .
+				CRLF . "0" . CRLF . CRLF;
+		} elsif ($uri eq '/chunked/big') {
+			print $client
+				"HTTP/1.1 200 OK" . CRLF .
+				"Transfer-Encoding: chunked" . CRLF .
+				"Connection: close" . CRLF .
+				CRLF;
+
+			for (1 .. 100) {
+				print $client "ff" . CRLF . ("X" x 255) . CRLF;
+			}
+
+		    print $client  "0" . CRLF . CRLF;
+		}
+	}
+}
diff --git a/stream_js_fetch.t b/stream_js_fetch.t
new file mode 100644
--- /dev/null
+++ b/stream_js_fetch.t
@@ -0,0 +1,171 @@
+#!/usr/bin/perl
+
+# (C) Dmitry Volyntsev
+# (C) Nginx, Inc.
+
+# Tests for http stream module, fetch method.
+
+###############################################################################
+
+use warnings;
+use strict;
+
+use Test::More;
+
+BEGIN { use FindBin; chdir($FindBin::Bin); }
+
+use lib 'lib';
+use Test::Nginx;
+use Test::Nginx::Stream qw/ dgram stream /;
+
+###############################################################################
+
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+my $t = Test::Nginx->new()->has(qw/http stream/)
+	->write_file_expand('nginx.conf', <<'EOF');
+
+%%TEST_GLOBALS%%
+
+daemon off;
+
+events {
+}
+
+http {
+    %%TEST_GLOBALS_HTTP%%
+
+    js_import test.js;
+
+    server {
+        listen       127.0.0.1:8080;
+        server_name  localhost;
+
+        location /njs {
+            js_content test.njs;
+        }
+    }
+
+    server {
+        listen       127.0.0.1:8080;
+        server_name  aaa;
+
+        location /validate {
+            js_content test.validate;
+        }
+    }
+}
+
+stream {
+    %%TEST_GLOBALS_STREAM%%
+
+    js_import test.js;
+
+    server {
+        listen      127.0.0.1:8081;
+        js_preread  test.preread_verify;
+        proxy_pass  127.0.0.1:8090;
+    }
+
+}
+
+EOF
+
+$t->write_file('test.js', <<EOF);
+    function test_njs(r) {
+        r.return(200, njs.version);
+    }
+
+    function validate(r) {
+        r.return((r.requestText == 'QZ') ? 200 : 403);
+    }
+
+    function preread_verify(s) {
+        var collect = Buffer.from([]);
+
+        s.on('upstream', function (data, flags) {
+            collect = Buffer.concat([collect, data]);
+
+            if (collect.length >= 4 && collect.readUInt16BE(0) == 0xabcd) {
+                ngx.fetch('http://127.0.0.1:8080/validate',
+                          {headers: {Host: 'aaa'}, body: collect.slice(2,4)})
+                .then(reply => (reply.status == 200) ? s.done(): s.deny())
+
+            } else if (collect.length) {
+                s.deny();
+            }
+        });
+    }
+
+    export default {njs: test_njs, validate, preread_verify}
+EOF
+
+$t->run_daemon(\&stream_daemon, port(8090));
+$t->try_run('no stream njs available')->plan(4);
+$t->waitforsocket('127.0.0.1:' . port(8090));
+
+###############################################################################
+
+TODO: {
+local $TODO = 'not yet'
+	unless http_get('/njs') =~ /^([.0-9]+)$/m && $1 ge '0.5.1';
+
+is(stream('127.0.0.1:' . port(8081))->io('###'), '', 'preread not enough');
+is(stream('127.0.0.1:' . port(8081))->io("\xAB\xCDQZ##"), "\xAB\xCDQZ##",
+	'preread validated');
+is(stream('127.0.0.1:' . port(8081))->io("\xAC\xCDQZ##"), '',
+	'preread invalid magic');
+is(stream('127.0.0.1:' . port(8081))->io("\xAB\xCDQQ##"), '',
+	'preread validation failed');
+
+}
+
+$t->stop();
+
+###############################################################################
+
+sub stream_daemon {
+	my $server = IO::Socket::INET->new(
+		Proto => 'tcp',
+		LocalAddr => '127.0.0.1:' . port(8090),
+		Listen => 5,
+		Reuse => 1
+	)
+		or die "Can't create listening socket: $!\n";
+
+	local $SIG{PIPE} = 'IGNORE';
+
+	while (my $client = $server->accept()) {
+		$client->autoflush(1);
+
+		log2c("(new connection $client)");
+
+		$client->sysread(my $buffer, 65536) or next;
+
+		log2i("$client $buffer");
+
+		log2o("$client $buffer");
+
+		$client->syswrite($buffer);
+
+		close $client;
+	}
+}
+
+sub log2i { Test::Nginx::log_core('|| <<', @_); }
+sub log2o { Test::Nginx::log_core('|| >>', @_); }
+sub log2c { Test::Nginx::log_core('||', @_); }
+
+sub get {
+	my ($url, %extra) = @_;
+
+	my $s = IO::Socket::INET->new(
+		Proto => 'tcp',
+		PeerAddr => '127.0.0.1:' . port(8079)
+	) or die "Can't connect to nginx: $!\n";
+
+	return http_get($url, socket => $s);
+}
+
+###############################################################################