From d69e9a445d4bcb15d4afaa1ef635158fd62844c5 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 19 Jul 2018 13:55:01 -0400 Subject: [PATCH 01/15] Add I/O interfaces to Roadmap.md --- Roadmap.md | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/Roadmap.md b/Roadmap.md index 96bccfbd80333f..23170c05d0ddab 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -243,3 +243,101 @@ Fetch API: ```ts fetch(input?: Request | string, init?: RequestInit): Promise; ``` + +#### I/O + +There will be two API layers for I/O: +1. A low-level abstraction with (for sockets) non-blocking read/writes operating +on FDs. (Yet unspecified - but probably very similar to the unix syscalls.) + +2. A high-level API that will closely follow Go's I/O interfaces. The tentative +intrefaces are outlined below: +```ts +interface Reader { + // Reader is the interface that wraps the basic Read method. + // + // Read reads up to p.byteLength bytes into p. It returns the number of bytes + // read (0 <= n <= p.byteLength) and any error encountered. Even if Read + // returns n < p.byteLength, it may use all of p as scratch space during the + // call. If some data is available but not p.byteLength bytes, Read + // conventionally returns what is available instead of waiting for more. + // + // When Read encounters an error or end-of-file condition after successfully + // reading n > 0 bytes, it returns the number of bytes read. It may return the + // (non-nil) error from the same call or return the error (and n == 0) from a + // subsequent call. An instance of this general case is that a Reader + // returning a non-zero number of bytes at the end of the input stream may + // return either err == EOF or err == nil. The next Read should return 0, EOF. + // + // Callers should always process the n > 0 bytes returned before considering + // the error err. Doing so correctly handles I/O errors that happen after + // reading some bytes and also both of the allowed EOF behaviors. + // + // Implementations of Read are discouraged from returning a zero byte count + // with a nil error, except when p.byteLength == 0. Callers should treat a + // return of 0 and nil as indicating that nothing happened; in particular it + // does not indicate EOF. + // + // Implementations must not retain p. + // + // Copied from Go https://golang.org/pkg/io/#Reader + async read(p: Uint8Array): [number, IOError]; +} + +interface Writer { + // Writer is the interface that wraps the basic Write method. + // + // Write writes p.byteLength bytes from p to the underlying data stream. It + // returns the number of bytes written from p (0 <= n <= p.byteLength) and any + // error encountered that caused the write to stop early. Write must return a + // non-nil error if it returns n < p.byteLength. Write must not modify the + // slice data, even temporarily. + // + // Implementations must not retain p. + // + // Copied from Go https://golang.org/pkg/io/#Writer + async write(p: Uint8Array): [number, IOError]; +} + +interface Closer { + // The behavior of Close after the first call is undefined. Specific + // implementations may document their own behavior. + // Copied from Go https://golang.org/pkg/io/#Closer + close(): IOError; +} + +interface Seeker { + // Seek sets the offset for the next Read or Write to offset, interpreted + // according to whence: SeekStart means relative to the start of the file, + // SeekCurrent means relative to the current offset, and SeekEnd means + // relative to the end. Seek returns the new offset relative to the start of + // the file and an error, if any. + // + // Seeking to an offset before the start of the file is an error. Seeking to + // any positive offset is legal, but the behavior of subsequent I/O operations + // on the underlying object is implementation-dependent. + // + // Copied from Go https://golang.org/pkg/io/#Seeker + seek(offset: number, whence: number): [number, IOError]; +} + +// https://golang.org/pkg/io/#ReadCloser +interface ReaderCloser extends Reader, Closer { } + +// https://golang.org/pkg/io/#WriteCloser +interface WriteCloser extends Writer, Closer { } + +// https://golang.org/pkg/io/#ReadSeeker +interface ReadSeeker extends Reader, Seeker { } + +// https://golang.org/pkg/io/#WriteSeeker +interface WriteSeeker extends Writer, Seeker { } + +// https://golang.org/pkg/io/#ReadWriteCloser +interface ReadWriteCloser extends Reader, Writer, Closer { } + +// https://golang.org/pkg/io/#ReadWriteSeeker +interface ReadWriteSeeker extends Reader, Writer, Seeker { } +``` + + From eaa77fdcbe189cfbd6a370ce5c51264907686823 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 19 Jul 2018 14:20:53 -0400 Subject: [PATCH 02/15] Add I/O utils --- Roadmap.md | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/Roadmap.md b/Roadmap.md index 23170c05d0ddab..60782295b4f114 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -253,9 +253,9 @@ on FDs. (Yet unspecified - but probably very similar to the unix syscalls.) 2. A high-level API that will closely follow Go's I/O interfaces. The tentative intrefaces are outlined below: ```ts +// Reader is the interface that wraps the basic Read method. +// https://golang.org/pkg/io/#Reader interface Reader { - // Reader is the interface that wraps the basic Read method. - // // Read reads up to p.byteLength bytes into p. It returns the number of bytes // read (0 <= n <= p.byteLength) and any error encountered. Even if Read // returns n < p.byteLength, it may use all of p as scratch space during the @@ -279,14 +279,12 @@ interface Reader { // does not indicate EOF. // // Implementations must not retain p. - // - // Copied from Go https://golang.org/pkg/io/#Reader async read(p: Uint8Array): [number, IOError]; } +// Writer is the interface that wraps the basic Write method. +// https://golang.org/pkg/io/#Writer interface Writer { - // Writer is the interface that wraps the basic Write method. - // // Write writes p.byteLength bytes from p to the underlying data stream. It // returns the number of bytes written from p (0 <= n <= p.byteLength) and any // error encountered that caused the write to stop early. Write must return a @@ -294,18 +292,17 @@ interface Writer { // slice data, even temporarily. // // Implementations must not retain p. - // - // Copied from Go https://golang.org/pkg/io/#Writer async write(p: Uint8Array): [number, IOError]; } +// https://golang.org/pkg/io/#Closer interface Closer { // The behavior of Close after the first call is undefined. Specific // implementations may document their own behavior. - // Copied from Go https://golang.org/pkg/io/#Closer close(): IOError; } +// https://golang.org/pkg/io/#Seeker interface Seeker { // Seek sets the offset for the next Read or Write to offset, interpreted // according to whence: SeekStart means relative to the start of the file, @@ -316,9 +313,7 @@ interface Seeker { // Seeking to an offset before the start of the file is an error. Seeking to // any positive offset is legal, but the behavior of subsequent I/O operations // on the underlying object is implementation-dependent. - // - // Copied from Go https://golang.org/pkg/io/#Seeker - seek(offset: number, whence: number): [number, IOError]; + async seek(offset: number, whence: number): [number, IOError]; } // https://golang.org/pkg/io/#ReadCloser @@ -339,5 +334,27 @@ interface ReadWriteCloser extends Reader, Writer, Closer { } // https://golang.org/pkg/io/#ReadWriteSeeker interface ReadWriteSeeker extends Reader, Writer, Seeker { } ``` - - +These interfaces are well specified, simple, and have very nice utility +functions that will be easy to port. Some example utilites: +```ts +// Copy copies from src to dst until either EOF is reached on src or an error +// occurs. It returns the number of bytes copied and the first error encountered +// while copying, if any. +// +// A successful Copy returns err == nil, not err == EOF. Because Copy is defined +// to read from src until EOF, it does not treat an EOF from Read as an error to +// be reported. +// +// https://golang.org/pkg/io/#Copy +async copy(dst: Writer, src: Reader): [written: number, err: IOError]; + +// MultiWriter creates a writer that duplicates its writes to all the provided +// writers, similar to the Unix tee(1) command. +// +// Each write is written to each listed writer, one at a time. If a listed +// writer returns an error, that overall write operation stops and returns the +// error; it does not continue down the list. +// +// https://golang.org/pkg/io/#MultiWriter +function multiWriter(writers: ...Writer): Writer; +``` From 394a620d8c9c36aef67735ebd8e173baa96c4457 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 19 Jul 2018 14:28:12 -0400 Subject: [PATCH 03/15] Fix syntax --- Roadmap.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Roadmap.md b/Roadmap.md index 60782295b4f114..476d34c6b96568 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -279,7 +279,7 @@ interface Reader { // does not indicate EOF. // // Implementations must not retain p. - async read(p: Uint8Array): [number, IOError]; + async read(p: Uint8Array): Promise<[number, IOError]>; } // Writer is the interface that wraps the basic Write method. @@ -292,7 +292,7 @@ interface Writer { // slice data, even temporarily. // // Implementations must not retain p. - async write(p: Uint8Array): [number, IOError]; + async write(p: Uint8Array): Promise<[number, IOError]>; } // https://golang.org/pkg/io/#Closer @@ -313,7 +313,7 @@ interface Seeker { // Seeking to an offset before the start of the file is an error. Seeking to // any positive offset is legal, but the behavior of subsequent I/O operations // on the underlying object is implementation-dependent. - async seek(offset: number, whence: number): [number, IOError]; + async seek(offset: number, whence: number): Promise<[number, IOError]>; } // https://golang.org/pkg/io/#ReadCloser @@ -346,7 +346,7 @@ functions that will be easy to port. Some example utilites: // be reported. // // https://golang.org/pkg/io/#Copy -async copy(dst: Writer, src: Reader): [written: number, err: IOError]; +async function copy(dst: Writer, src: Reader): Promise<[written: number, err: IOError]>; // MultiWriter creates a writer that duplicates its writes to all the provided // writers, similar to the Unix tee(1) command. From a5ae1e32545d7c5a1ec3f76ac41b2bb90dd9f19c Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 19 Jul 2018 14:29:49 -0400 Subject: [PATCH 04/15] Fix syntax --- Roadmap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Roadmap.md b/Roadmap.md index 476d34c6b96568..bafaa55e2e938e 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -346,7 +346,7 @@ functions that will be easy to port. Some example utilites: // be reported. // // https://golang.org/pkg/io/#Copy -async function copy(dst: Writer, src: Reader): Promise<[written: number, err: IOError]>; +async function copy(dst: Writer, src: Reader): Promise<[number, IOError]>; // MultiWriter creates a writer that duplicates its writes to all the provided // writers, similar to the Unix tee(1) command. From b01c1be2ea3ffb85cc585cf8a5357ceda1978030 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 19 Jul 2018 15:21:05 -0400 Subject: [PATCH 05/15] Use IOResult --- Roadmap.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Roadmap.md b/Roadmap.md index bafaa55e2e938e..3a30e643117fff 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -253,6 +253,11 @@ on FDs. (Yet unspecified - but probably very similar to the unix syscalls.) 2. A high-level API that will closely follow Go's I/O interfaces. The tentative intrefaces are outlined below: ```ts +interface IOResult { + n: number; + eof: boolean; +} + // Reader is the interface that wraps the basic Read method. // https://golang.org/pkg/io/#Reader interface Reader { @@ -279,7 +284,7 @@ interface Reader { // does not indicate EOF. // // Implementations must not retain p. - async read(p: Uint8Array): Promise<[number, IOError]>; + async read(p: Uint8Array): Promise; } // Writer is the interface that wraps the basic Write method. @@ -292,14 +297,14 @@ interface Writer { // slice data, even temporarily. // // Implementations must not retain p. - async write(p: Uint8Array): Promise<[number, IOError]>; + async write(p: Uint8Array): Promise; } // https://golang.org/pkg/io/#Closer interface Closer { // The behavior of Close after the first call is undefined. Specific // implementations may document their own behavior. - close(): IOError; + close(): void; } // https://golang.org/pkg/io/#Seeker @@ -313,7 +318,7 @@ interface Seeker { // Seeking to an offset before the start of the file is an error. Seeking to // any positive offset is legal, but the behavior of subsequent I/O operations // on the underlying object is implementation-dependent. - async seek(offset: number, whence: number): Promise<[number, IOError]>; + async seek(offset: number, whence: number): Promise; } // https://golang.org/pkg/io/#ReadCloser @@ -346,7 +351,7 @@ functions that will be easy to port. Some example utilites: // be reported. // // https://golang.org/pkg/io/#Copy -async function copy(dst: Writer, src: Reader): Promise<[number, IOError]>; +async function copy(dst: Writer, src: Reader): Promise; // MultiWriter creates a writer that duplicates its writes to all the provided // writers, similar to the Unix tee(1) command. From cdf7261182d1461e242c64af558f322dfb578042 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 19 Jul 2018 15:28:30 -0400 Subject: [PATCH 06/15] IOResult cleanup --- Roadmap.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Roadmap.md b/Roadmap.md index 3a30e643117fff..a3a96f5e2e806a 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -253,6 +253,8 @@ on FDs. (Yet unspecified - but probably very similar to the unix syscalls.) 2. A high-level API that will closely follow Go's I/O interfaces. The tentative intrefaces are outlined below: ```ts +// The bytes written or read (n) during an I/O call and a boolean indicating +// EOF. interface IOResult { n: number; eof: boolean; @@ -346,12 +348,11 @@ functions that will be easy to port. Some example utilites: // occurs. It returns the number of bytes copied and the first error encountered // while copying, if any. // -// A successful Copy returns err == nil, not err == EOF. Because Copy is defined -// to read from src until EOF, it does not treat an EOF from Read as an error to -// be reported. +// Because Copy is defined to read from src until EOF, it does not treat an EOF +// from Read as an error to be reported. // // https://golang.org/pkg/io/#Copy -async function copy(dst: Writer, src: Reader): Promise; +async function copy(dst: Writer, src: Reader): Promise; // MultiWriter creates a writer that duplicates its writes to all the provided // writers, similar to the Unix tee(1) command. From c014e195d212be77f1a08dfc205f7b8422f85974 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 19 Jul 2018 15:32:20 -0400 Subject: [PATCH 07/15] ReadWriteResult --- Roadmap.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Roadmap.md b/Roadmap.md index a3a96f5e2e806a..3ebfe4cf8a10ac 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -253,9 +253,9 @@ on FDs. (Yet unspecified - but probably very similar to the unix syscalls.) 2. A high-level API that will closely follow Go's I/O interfaces. The tentative intrefaces are outlined below: ```ts -// The bytes written or read (n) during an I/O call and a boolean indicating +// The bytes read or written (n) during an I/O call and a boolean indicating // EOF. -interface IOResult { +interface ReadWriteResult { n: number; eof: boolean; } @@ -286,7 +286,7 @@ interface Reader { // does not indicate EOF. // // Implementations must not retain p. - async read(p: Uint8Array): Promise; + async read(p: Uint8Array): Promise; } // Writer is the interface that wraps the basic Write method. @@ -299,7 +299,7 @@ interface Writer { // slice data, even temporarily. // // Implementations must not retain p. - async write(p: Uint8Array): Promise; + async write(p: Uint8Array): Promise; } // https://golang.org/pkg/io/#Closer @@ -320,7 +320,7 @@ interface Seeker { // Seeking to an offset before the start of the file is an error. Seeking to // any positive offset is legal, but the behavior of subsequent I/O operations // on the underlying object is implementation-dependent. - async seek(offset: number, whence: number): Promise; + async seek(offset: number, whence: number): Promise; } // https://golang.org/pkg/io/#ReadCloser From 96ae2f9aa0df920ee0e7f058fe6d8cbdb05d22da Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 19 Jul 2018 15:37:04 -0400 Subject: [PATCH 08/15] Add low-level io functions --- Roadmap.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Roadmap.md b/Roadmap.md index 3ebfe4cf8a10ac..40ea8b01c4ad10 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -247,11 +247,8 @@ fetch(input?: Request | string, init?: RequestInit): Promise; #### I/O There will be two API layers for I/O: -1. A low-level abstraction with (for sockets) non-blocking read/writes operating -on FDs. (Yet unspecified - but probably very similar to the unix syscalls.) - -2. A high-level API that will closely follow Go's I/O interfaces. The tentative -intrefaces are outlined below: +1. A low-level abstraction. For sockets this will mean non-blocking read/writes + operating on FDs. ```ts // The bytes read or written (n) during an I/O call and a boolean indicating // EOF. @@ -260,6 +257,20 @@ interface ReadWriteResult { eof: boolean; } +// Low-level non-blocking, non-async read. +function read(fd: number, p: Uint8Array, nbytes: number): ReadWriteResult; + +// Low-level non-blocking, non-async write. +function write(fd: number, p: Uint8Array, nbytes: number): ReadWriteResult; + +// Low-level close. +function close(fd: number): void; +``` + +2. A high-level API that will closely follow Go's I/O interfaces. The tentative +intrefaces are outlined below: +```ts + // Reader is the interface that wraps the basic Read method. // https://golang.org/pkg/io/#Reader interface Reader { From e7f3572c732fd732504a2b646a0d46e2a52803db Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 19 Jul 2018 15:43:50 -0400 Subject: [PATCH 09/15] cleanup --- Roadmap.md | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/Roadmap.md b/Roadmap.md index 40ea8b01c4ad10..424b6453e08aaa 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -246,29 +246,37 @@ fetch(input?: Request | string, init?: RequestInit): Promise; #### I/O -There will be two API layers for I/O: -1. A low-level abstraction. For sockets this will mean non-blocking read/writes - operating on FDs. +Deno aims to provide fast, zero-copy I/O for sockets and files. Plus easy +interfaces for managing streams of data. + +##### Low-level Non-blocking I/O + +A low-level abstraction. For sockets this will mean non-blocking read/writes +operating on FDs. + ```ts // The bytes read or written (n) during an I/O call and a boolean indicating // EOF. -interface ReadWriteResult { +interface ReadResult { n: number; eof: boolean; } // Low-level non-blocking, non-async read. -function read(fd: number, p: Uint8Array, nbytes: number): ReadWriteResult; +function read(fd: number, p: Uint8Array, nbytes: number): ReadResult; // Low-level non-blocking, non-async write. -function write(fd: number, p: Uint8Array, nbytes: number): ReadWriteResult; +function write(fd: number, p: Uint8Array, nbytes: number): number; // Low-level close. function close(fd: number): void; ``` -2. A high-level API that will closely follow Go's I/O interfaces. The tentative -intrefaces are outlined below: +##### High-level Go-inspired I/O interfaces + +Deno will not use Node's stream API. Instead The high-level API that will +closely follow Go's I/O interfaces. + ```ts // Reader is the interface that wraps the basic Read method. @@ -297,7 +305,7 @@ interface Reader { // does not indicate EOF. // // Implementations must not retain p. - async read(p: Uint8Array): Promise; + async read(p: Uint8Array): Promise; } // Writer is the interface that wraps the basic Write method. @@ -310,7 +318,7 @@ interface Writer { // slice data, even temporarily. // // Implementations must not retain p. - async write(p: Uint8Array): Promise; + async write(p: Uint8Array): Promise; } // https://golang.org/pkg/io/#Closer From 0007905da016a871911d93b628969b38e67faa31 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Thu, 19 Jul 2018 15:56:19 -0400 Subject: [PATCH 10/15] cleanup --- Roadmap.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Roadmap.md b/Roadmap.md index 424b6453e08aaa..5ed9600f281391 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -246,19 +246,19 @@ fetch(input?: Request | string, init?: RequestInit): Promise; #### I/O -Deno aims to provide fast, zero-copy I/O for sockets and files. Plus easy -interfaces for managing streams of data. +I/O in Deno is split into two APIs. The low-level API is a syscall-like +interface operating on file descriptors. The high-level API operates on +Reader and Writer interfaces inspired by Golang. ##### Low-level Non-blocking I/O -A low-level abstraction. For sockets this will mean non-blocking read/writes -operating on FDs. +Deno aims to provide fast, memory efficient, zero-copy I/O for sockets. Towards +this the low-level I/O interface is inspired by non-blocking unix syscalls. ```ts -// The bytes read or written (n) during an I/O call and a boolean indicating -// EOF. +// The bytes read during an I/O call and a boolean indicating EOF. interface ReadResult { - n: number; + nread: number; eof: boolean; } From dc60a006da7e63920a0866412c37013fa75a4068 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Fri, 20 Jul 2018 02:42:46 -0400 Subject: [PATCH 11/15] improve examples --- Roadmap.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/Roadmap.md b/Roadmap.md index 5ed9600f281391..174d9f3b0a293e 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -371,7 +371,17 @@ functions that will be easy to port. Some example utilites: // from Read as an error to be reported. // // https://golang.org/pkg/io/#Copy -async function copy(dst: Writer, src: Reader): Promise; +async function copy(dst: Writer, src: Reader): Promise { + let n = 0; + const b = new Uint8Array(1024); + let got_eof = false; + while (got_eof === false) { + let result = await src.read(b); + if (result.eof) got_eof = true; + n += await dst.write(b.subarray(0, result.nread)); + } + return n; +} // MultiWriter creates a writer that duplicates its writes to all the provided // writers, similar to the Unix tee(1) command. @@ -381,5 +391,14 @@ async function copy(dst: Writer, src: Reader): Promise; // error; it does not continue down the list. // // https://golang.org/pkg/io/#MultiWriter -function multiWriter(writers: ...Writer): Writer; +function multiWriter(writers: ...Writer): Writer { + return { + write: async (p: Uint8Array) => Promise { + let n; + let nwritten = await Promise.all(writers.map((w) => w.write(p))); + return nwritten[0]; + // TODO unsure of proper semantics for return value.. + } + }; +} ``` From 2d37090a32f27654287cf2dd908bac85377b8cb8 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Mon, 23 Jul 2018 14:32:54 -0400 Subject: [PATCH 12/15] Remove low-level portion, provide better justification --- Roadmap.md | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/Roadmap.md b/Roadmap.md index 174d9f3b0a293e..99c5aa3c6aea1e 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -246,14 +246,20 @@ fetch(input?: Request | string, init?: RequestInit): Promise; #### I/O -I/O in Deno is split into two APIs. The low-level API is a syscall-like -interface operating on file descriptors. The high-level API operates on -Reader and Writer interfaces inspired by Golang. +There are many OS constructs that perform I/O: files, sockets, pipes. +Deno aims to provide a unified lowest common denominator interface to work with +these objects. Deno needs to operate on all of these asynchronously in order +to not block the event loop and it. -##### Low-level Non-blocking I/O +Sockets and pipes support non-blocking reads and write. Generally file I/O is +blocking but it can be done in a thread pool to avoid blocking the main thread. +Although file I/O can be made asynchronous, it does not support the same +non-blocking reads and writes that sockets and pipes do. -Deno aims to provide fast, memory efficient, zero-copy I/O for sockets. Towards -this the low-level I/O interface is inspired by non-blocking unix syscalls. +The following interfaces support files, socket, and pipes and are heavily +inspired by Go. The main difference in porting to JavaScript is that errors will +be handled by exceptions, modulo EOF, which is returned as part of +`ReadResult`. ```ts // The bytes read during an I/O call and a boolean indicating EOF. @@ -262,23 +268,6 @@ interface ReadResult { eof: boolean; } -// Low-level non-blocking, non-async read. -function read(fd: number, p: Uint8Array, nbytes: number): ReadResult; - -// Low-level non-blocking, non-async write. -function write(fd: number, p: Uint8Array, nbytes: number): number; - -// Low-level close. -function close(fd: number): void; -``` - -##### High-level Go-inspired I/O interfaces - -Deno will not use Node's stream API. Instead The high-level API that will -closely follow Go's I/O interfaces. - -```ts - // Reader is the interface that wraps the basic Read method. // https://golang.org/pkg/io/#Reader interface Reader { From fed220bf4019cad13d939766e057d7f2615f9188 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Tue, 24 Jul 2018 15:19:18 -0400 Subject: [PATCH 13/15] Use ArrayBufferView --- Roadmap.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Roadmap.md b/Roadmap.md index 99c5aa3c6aea1e..433a661da2649e 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -294,7 +294,7 @@ interface Reader { // does not indicate EOF. // // Implementations must not retain p. - async read(p: Uint8Array): Promise; + async read(p: ArrayBufferView): Promise; } // Writer is the interface that wraps the basic Write method. @@ -307,7 +307,7 @@ interface Writer { // slice data, even temporarily. // // Implementations must not retain p. - async write(p: Uint8Array): Promise; + async write(p: ArrayBufferView): Promise; } // https://golang.org/pkg/io/#Closer @@ -362,7 +362,7 @@ functions that will be easy to port. Some example utilites: // https://golang.org/pkg/io/#Copy async function copy(dst: Writer, src: Reader): Promise { let n = 0; - const b = new Uint8Array(1024); + const b = new ArrayBufferView(1024); let got_eof = false; while (got_eof === false) { let result = await src.read(b); @@ -382,7 +382,7 @@ async function copy(dst: Writer, src: Reader): Promise { // https://golang.org/pkg/io/#MultiWriter function multiWriter(writers: ...Writer): Writer { return { - write: async (p: Uint8Array) => Promise { + write: async (p: ArrayBufferView) => Promise { let n; let nwritten = await Promise.all(writers.map((w) => w.write(p))); return nwritten[0]; From 3b21dce80a3b93138d682bbe054844a034ddc304 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 25 Jul 2018 13:02:13 -0400 Subject: [PATCH 14/15] Add readerIterator --- Roadmap.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Roadmap.md b/Roadmap.md index 433a661da2649e..11cf1995b3b375 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -391,3 +391,14 @@ function multiWriter(writers: ...Writer): Writer { }; } ``` + +A utility function will be provided to make any `Reader` into an +`AsyncIterator`, which has very similar semanatics. + +```ts +function readerIterator(r: deno.Reader): AsyncIterator; +// Example +for await (let buf of readerIterator(socket)) { + console.log(`read ${buf.byteLength} from socket`); +} +``` From ba809720df4b9a2e0a50f3dbbf8540963d6e0043 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Sat, 11 Aug 2018 13:25:51 -0400 Subject: [PATCH 15/15] review comments --- Roadmap.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Roadmap.md b/Roadmap.md index 11cf1995b3b375..67a5a66f5c85dd 100644 --- a/Roadmap.md +++ b/Roadmap.md @@ -268,27 +268,27 @@ interface ReadResult { eof: boolean; } -// Reader is the interface that wraps the basic Read method. +// Reader is the interface that wraps the basic read() method. // https://golang.org/pkg/io/#Reader interface Reader { - // Read reads up to p.byteLength bytes into p. It returns the number of bytes - // read (0 <= n <= p.byteLength) and any error encountered. Even if Read + // read() reads up to p.byteLength bytes into p. It returns the number of bytes + // read (0 <= n <= p.byteLength) and any error encountered. Even if read() // returns n < p.byteLength, it may use all of p as scratch space during the - // call. If some data is available but not p.byteLength bytes, Read + // call. If some data is available but not p.byteLength bytes, read() // conventionally returns what is available instead of waiting for more. // - // When Read encounters an error or end-of-file condition after successfully + // When read() encounters an error or end-of-file condition after successfully // reading n > 0 bytes, it returns the number of bytes read. It may return the // (non-nil) error from the same call or return the error (and n == 0) from a // subsequent call. An instance of this general case is that a Reader // returning a non-zero number of bytes at the end of the input stream may - // return either err == EOF or err == nil. The next Read should return 0, EOF. + // return either err == EOF or err == nil. The next read() should return 0, EOF. // // Callers should always process the n > 0 bytes returned before considering // the error err. Doing so correctly handles I/O errors that happen after // reading some bytes and also both of the allowed EOF behaviors. // - // Implementations of Read are discouraged from returning a zero byte count + // Implementations of read() are discouraged from returning a zero byte count // with a nil error, except when p.byteLength == 0. Callers should treat a // return of 0 and nil as indicating that nothing happened; in particular it // does not indicate EOF. @@ -297,13 +297,13 @@ interface Reader { async read(p: ArrayBufferView): Promise; } -// Writer is the interface that wraps the basic Write method. +// Writer is the interface that wraps the basic write() method. // https://golang.org/pkg/io/#Writer interface Writer { - // Write writes p.byteLength bytes from p to the underlying data stream. It + // write() writes p.byteLength bytes from p to the underlying data stream. It // returns the number of bytes written from p (0 <= n <= p.byteLength) and any - // error encountered that caused the write to stop early. Write must return a - // non-nil error if it returns n < p.byteLength. Write must not modify the + // error encountered that caused the write to stop early. write() must return a + // non-nil error if it returns n < p.byteLength. write() must not modify the // slice data, even temporarily. // // Implementations must not retain p. @@ -319,7 +319,7 @@ interface Closer { // https://golang.org/pkg/io/#Seeker interface Seeker { - // Seek sets the offset for the next Read or Write to offset, interpreted + // Seek sets the offset for the next read() or write() to offset, interpreted // according to whence: SeekStart means relative to the start of the file, // SeekCurrent means relative to the current offset, and SeekEnd means // relative to the end. Seek returns the new offset relative to the start of @@ -352,12 +352,12 @@ interface ReadWriteSeeker extends Reader, Writer, Seeker { } These interfaces are well specified, simple, and have very nice utility functions that will be easy to port. Some example utilites: ```ts -// Copy copies from src to dst until either EOF is reached on src or an error +// copy() copies from src to dst until either EOF is reached on src or an error // occurs. It returns the number of bytes copied and the first error encountered // while copying, if any. // -// Because Copy is defined to read from src until EOF, it does not treat an EOF -// from Read as an error to be reported. +// Because copy() is defined to read from src until EOF, it does not treat an EOF +// from read() as an error to be reported. // // https://golang.org/pkg/io/#Copy async function copy(dst: Writer, src: Reader): Promise { @@ -387,7 +387,7 @@ function multiWriter(writers: ...Writer): Writer { let nwritten = await Promise.all(writers.map((w) => w.write(p))); return nwritten[0]; // TODO unsure of proper semantics for return value.. - } + } }; } ```