Introduce test infrastructure for QuicTransport (#22844)

This change introduces test infrastructure for QuicTransport.
See also: https://github.com/web-platform-tests/rfcs/blob/master/rfcs/quic.md

tools/quic contains the test server and files needed by the server such as
certificate files (TODO: we will switch to the same certificate used by
wptserve once aioquic 0.8.8 is released).

tools/quic/quic_transport_server.py is based on
https://github.com/aiortc/aioquic/blob/master/examples/http3_server.py

webtransport/quic contains a test example and a sample custom handler.

This change doesn't contain a means to run the QuicTransport server
automatically.

Tracking issue: #19114

tools/quic/README.md

@@ -0,0 +1,30 @@
+This directory contains
+[QUIC](https://tools.ietf.org/html/draft-ietf-quic-transport) related tools.
+
+# QuicTransport
+[quic_transport_server.py](./quic_transport_server.py) implements a simple
+[QuicTransport](https://tools.ietf.org/html/draft-vvv-webtransport-quic) server
+for testing. It uses [aioquic](https://github.com/aiortc/aioquic/), and test
+authors can implement custom handlers by putting python scripts in
+[wpt/webtransport/quic/handlers/](../../webtransport/quic/handlers/).
+
+## Custom Handlers
+The QuicTransportServer calls functions defined in each handler script.
+
+ - handle_client_indication is called during the client indication process.
+   This function is called with three arguments:
+
+   - connection: aioquic.asyncio.QuicConnectionProtocol
+   - origin: str The origin of the initiator.
+   - query: Dict[str, str] The dictionary of query parameters of the URL of the
+            connection.
+
+   A handler can abort the client indication process either by raising an
+   exception or closing the connection.
+
+ - handle_event is called when a QuicEvent arrives.
+   - connection: aioquic.asyncio.QuicConnectionProtocol
+   - event: aioquic.quic.events.QuicEvent
+
+   This function is not called until the client indication process finishes
+   successfully.

tools/quic/certs/README.md

@@ -0,0 +1,12 @@
+To generate cert.key and cert.pem:
+
+ 1. Remove web-platform.test.key and web-platform.test.pem in ../../certs.
+ 1. From the root, run
+    `./wpt serve --config tools/quic/certs/config.json` and terminate it
+    after it has started up.
+ 1. Move tools/certs/web-platform.test.key to tools/quic/certs/cert.key.
+ 1. Move tools/certs/web-platform.test.pem to tools/quic/certs/cert.pem.
+ 1. Recover the original web-platform.test.key and web-platform.test.pem in
+    ../../certs.
+
+See also: ../../certs/README.md

tools/quic/certs/cert.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDbyPuiBGpxhavF
+3j7pI6g+A0gC4BTLTqMObKSTkQWsjq1GOd2LA1lwTPLObwrvhIUFzbwoIbOwoPMe
+MkjssFCHG3FMj56cKiAQ2DFI6dK5PjGUVNSRxk/F4Hh2Zx9DTENl/Eb/cRT2yuu+
+W9HCu/BWfbWwlwwN5vxyneCoh5cBB/jd1KTORguYpuatHb85AD5BRhYLXwHF7yVH
+NVxHeuGlK31yuYCHNKvBHDgZF5Tp8FqKXnVU+PlKXCSU5602c2U5xHIFvKTyiZlM
+cYXNSUp0lcNw/iTSVtsPE4k1stu6qkWdT7H/uU+GUB+aLqO+svA/s9GGX8HZgt+r
+vSFE8lIJAgMBAAECggEBAJA96D9djIoygxhaEompmCoStzkD3UHMuyClVqFuRP4J
+qVh0c5xfN1yHc7bdk5y8KR00966S574c81G3CLslv8Pb09C+VQcCcob7i+ThaCWg
+1qMVxWhicUpZVlXGufLN41HUbrgIfAy4Al2tHw4hj8sDt7FMgGHDXZzPVnjke8r1
+O9YiJl1Qx4L7vMWruGa9QWjFgHnG+uhaKjsL2v7JQOGy5t8aboVyb7h8rGg/mC+e
+HIYOucV1aEMgYVaAnhGsKMHkx5A1xWpXBSruG+GRBx/kXWZ+kCNckLXuVdrhq4HI
+AdbxIzqQTPMXpO3RAujyrxkHabENMPA/FGH4szmdLoECgYEA9z8pe7/vSlWgfhsF
+z5QnwWHyFjruhgD/2sa4LB/cmwTQdGw8E5TNHDbCgmS499DZUXIZuBOTekdEVDQa
+ng8VyL3o7Dms+5iPi5cqscp1KkjLEMyPpqs4JTuixRpjmMfycdxVTpXhcuqnJpTL
+QC9pR5N/zZcAMDlBv0Fzc8T78XkCgYEA45DueWGHVf2u4uMYyWxyZhaNDagl13yx
+/oSSUTzoLvSpGQxKkv+fxSNqL3nu5Ia6uD4Gu5NubP4Hr/VeSKRfmkT1luvFcVfC
+kn8r8bssZq855AVJxXa5K1auWjCuFHj0pYf56sfhkPxpY0RQEgkvuE3iosQ12gFX
+vw147FtQURECgYEA85RpVP45S31iOPp8Vg16wRyyeE4ksSYI6kr+JJJbLummSBxd
+b1kYXSRhqj56r8I0ZvXG+r9men/9hAs08eSgrHzUHO2RSuj4+ie6Kx/vH/JJBErT
+dvqVvLCs4gvmdRz+8EeGT35/dkxQ0kSinKBY0ugwb6XEzL2L1VUw3awCHdkCgYEA
+qtQIgOv6uU2ndEDAQax8MDCrkF3yklHUGFkSsZNERMN7EQeOD81+9XFBbARflgOh
+tV8ylKr3ETCdOrS6I1PpRJiRt8qjvBMCSBDZPyygBzFxBsAFggs+s87tMV0rwMiP
+9pcdv+ZuaPVic5c7eF6XCQbGpCMgvdeWNCB77woZP9ECgYEAlobkPGDYCy/RaViU
+Fbq5Go6w0pMVnLzYbn4Gh1AJPeQKISqXtJZ7tqpdW+i7qzkLw74ELaYCBR2ZElrj
+EVe5aROx6TFN9RnjkFnyv9LeyYL+YPc8AIwVUCeSPikSGLFpJfa/jwDmWh3vHmmA
+NRUP40wbtBi42C2udrTxUWsHxqc=
+-----END PRIVATE KEY-----

tools/quic/certs/config.json

@@ -0,0 +1,17 @@
+{
+    "ports": {
+        "http": [],
+        "https": ["auto"],
+        "ws": [],
+        "wss": []
+    },
+    "check_subdomains": false,
+    "ssl": {
+        "type": "openssl",
+        "openssl": {
+            "duration": 3650,
+            "force_regenerate": false,
+            "base_path": "tools/certs"
+        }
+    }
+}

tools/quic/quic_transport_server.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+import argparse
+import asyncio
+import io
+import logging
+import os
+import re
+import struct
+import urllib.parse
+from typing import Dict, Optional
+
+from aioquic.asyncio import QuicConnectionProtocol, serve
+from aioquic.quic.configuration import QuicConfiguration
+from aioquic.quic.connection import END_STATES
+from aioquic.quic.events import StreamDataReceived, QuicEvent
+from aioquic.tls import SessionTicket
+
+SERVER_NAME = 'aioquic-transport'
+
+handlers_path = None
+
+
+class EventHandler:
+    def __init__(self, connection: QuicConnectionProtocol, global_dict: Dict):
+        self.connection = connection
+        self.global_dict = global_dict
+
+    def handle_client_indication(
+            self,
+            origin: str,
+            query: Dict[str, str]) -> None:
+        name = 'handle_client_indication'
+        if name in self.global_dict:
+            self.global_dict[name](self.connection, origin, query)
+
+    def handle_event(self, event: QuicEvent) -> None:
+        name = 'handle_event'
+        if name in self.global_dict:
+            self.global_dict[name](self.connection, event)
+
+
+class QuicTransportProtocol(QuicConnectionProtocol):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.streams = dict()
+        self.pending_events = []
+        self.client_indication_finished = False
+        self.client_indication_data = b''
+        self.handler = None
+
+    def quic_event_received(self, event: QuicEvent) -> None:
+        prefix = '!!'
+        logging.log(logging.INFO, 'QUIC event: %s' % type(event))
+        try:
+            if (not self.client_indication_finished and
+                    isinstance(event, StreamDataReceived) and
+                    event.stream_id == 2):
+                # client indication process
+                self.client_indication_data += event.data
+                if event.end_stream:
+                    prefix = 'Client inditation error: '
+                    self.process_client_indication()
+                    if self.is_closing_or_closed():
+                        return
+                    prefix = 'Event handling Error: '
+                    for e in self.pending_events:
+                        self.handler.handle_event(e)
+                    self.pending_events.clear()
+            elif not self.client_indication_finished:
+                self.pending_events.append(event)
+            elif self.handler is not None:
+                prefix = 'Event handling Error: '
+                self.handler.handle_event(event)
+        except Exception as e:
+            self.handler = None
+            logging.log(logging.WARN, prefix + str(e))
+            self.close()
+
+    def parse_client_indication(self, bs):
+        while True:
+            key_b = bs.read(2)
+            if len(key_b) == 0:
+                return
+            length_b = bs.read(2)
+            if len(key_b) != 2:
+                raise Exception('failed to get "Key" field')
+            if len(length_b) != 2:
+                raise Exception('failed to get "Length" field')
+            key = struct.unpack('!H', key_b)[0]
+            length = struct.unpack('!H', length_b)[0]
+            value = bs.read(length)
+            if len(value) != length:
+                raise Exception('truncated "Value" field')
+            yield (key, value)
+
+    def process_client_indication(self) -> None:
+        origin = None
+        origin_string = None
+        path = None
+        path_string = None
+        KEY_ORIGIN = 0
+        KEY_PATH = 1
+        for (key, value) in self.parse_client_indication(
+                io.BytesIO(self.client_indication_data)):
+            if key == KEY_ORIGIN:
+                origin_string = value.decode()
+                origin = urllib.parse.urlparse(origin_string)
+            elif key == KEY_PATH:
+                path_string = value.decode()
+                path = urllib.parse.urlparse(path_string)
+            else:
+                # We must ignore unrecognized fields.
+                pass
+        logging.log(logging.INFO,
+                    'origin = %s, path = %s' % (origin_string, path_string))
+        if origin is None:
+            raise Exception('No origin is given')
+        if path is None:
+            raise Exception('No path is given')
+        if origin.scheme != 'https' and origin.scheme != 'http':
+            raise Exception('Invalid origin: %s' % origin_string)
+        if origin.netloc == '':
+            raise Exception('Invalid origin: %s' % origin_string)
+
+        # To make the situation simple we accept only simple path strings.
+        m = re.compile('^/([a-zA-Z0-9\._\-]+)$').match(path.path)
+        if m is None:
+            raise Exception('Invalid path: %s' % path_string)
+
+        handler_name = m.group(1)
+        query = dict(urllib.parse.parse_qsl(path.query))
+        self.handler = self.create_event_handler(handler_name)
+        self.handler.handle_client_indication(origin_string, query)
+        if self.is_closing_or_closed():
+            return
+        self.client_indication_finished = True
+        logging.log(logging.INFO, 'Client indication finished')
+
+    def create_event_handler(self, handler_name: str) -> None:
+        global_dict = {}
+        with open(handlers_path + '/' + handler_name) as f:
+            exec(f.read(), global_dict)
+        return EventHandler(self, global_dict)
+
+    def is_closing_or_closed(self) -> bool:
+        if self._quic._close_pending:
+            return True
+        if self._quic._state in END_STATES:
+            return True
+        return False
+
+
+class SessionTicketStore:
+    '''
+    Simple in-memory store for session tickets.
+    '''
+
+    def __init__(self) -> None:
+        self.tickets: Dict[bytes, SessionTicket] = {}
+
+    def add(self, ticket: SessionTicket) -> None:
+        self.tickets[ticket.ticket] = ticket
+
+    def pop(self, label: bytes) -> Optional[SessionTicket]:
+        return self.tickets.pop(label, None)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='QUIC server')
+    parser.add_argument(
+        '-c',
+        '--certificate',
+        type=str,
+        required=True,
+        help='load the TLS certificate from the specified file',
+    )
+    parser.add_argument(
+        '--host',
+        type=str,
+        default='::',
+        help='listen on the specified address (defaults to ::)',
+    )
+    parser.add_argument(
+        '--port',
+        type=int,
+        default=4433,
+        help='listen on the specified port (defaults to 4433)',
+    )
+    parser.add_argument(
+        '-k',
+        '--private-key',
+        type=str,
+        required=True,
+        help='load the TLS private key from the specified file',
+    )
+    parser.add_argument(
+        '--handlers-path',
+        type=str,
+        required=True,
+        help='the directory path of QuicTransport event handlers',
+    )
+    parser.add_argument(
+        '-v',
+        '--verbose',
+        action='store_true',
+        help='increase logging verbosity'
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        format='%(asctime)s %(levelname)s %(name)s %(message)s',
+        level=logging.DEBUG if args.verbose else logging.INFO,
+    )
+
+    configuration = QuicConfiguration(
+        alpn_protocols=['wq-vvv-01'] + ['siduck'],
+        is_client=False,
+        max_datagram_frame_size=65536,
+    )
+
+    handlers_path = os.path.abspath(os.path.expanduser(args.handlers_path))
+    logging.log(logging.INFO, 'port = %s' % args.port)
+    logging.log(logging.INFO, 'handlers path = %s' % handlers_path)
+
+    # load SSL certificate and key
+    configuration.load_cert_chain(args.certificate, args.private_key)
+
+    ticket_store = SessionTicketStore()
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(
+        serve(
+            args.host,
+            args.port,
+            configuration=configuration,
+            create_protocol=QuicTransportProtocol,
+            session_ticket_fetcher=ticket_store.pop,
+            session_ticket_handler=ticket_store.add,
+        )
+    )
+    try:
+        loop.run_forever()
+    except KeyboardInterrupt:
+        pass

tools/wptserve/wptserve/sslutils/openssl.py

@@ -402,6 +402,8 @@ def _load_host_cert(self, hosts):
 
     def _generate_host_cert(self, hosts):
         host = hosts[0]
+        if not self.force_regenerate:
+            self._load_ca_cert()
         if self._ca_key_path is None:
             self._generate_ca(hosts)
         ca_key_path = self._ca_key_path

webtransport/quic/client-indication.any.js

@@ -0,0 +1,32 @@
+// META: quic=true
+// META: script=/common/get-host-info.sub.js
+
+const PORT = 8983;
+const {ORIGINAL_HOST: HOST, ORIGIN} = get_host_info();
+const BASE = `quic-transport://${HOST}:${PORT}`;
+
+promise_test(async (test) => {
+  function onClosed() {
+    assert_unreached('The closed promise should be ' +
+                     'fulfilled or rejected after getting a PASS signal.');
+  }
+  const qt = new QuicTransport(
+    `${BASE}/client-indication.quic.py?origin=${ORIGIN}`);
+  qt.closed.then(test.step_func(onClosed), test.step_func(onClosed));
+
+  const streams = qt.receiveStreams();
+  const {done, value} = await streams.getReader().read();
+  assert_false(done, 'getting an incoming stream');
+
+  const readable = value.readable.pipeThrough(new TextDecoderStream());
+  const reader = readable.getReader();
+  let result = '';
+  while (true) {
+    const {done, value} = await reader.read();
+    if (done) {
+        break;
+    }
+    result += value;
+  }
+  assert_equals(result, 'PASS');
+}, 'Client indication');

webtransport/quic/handlers/README.md

@@ -0,0 +1,2 @@
+This directory contains custom handlers for testing QuicTransport. Please see
+https://github.com/web-platform-tests/wpt/tools/quic.