Attend requests, unfinished

o compile what we can do with Path.stat
o blow up if we can't represent it in a string instead of truncating it
o split the reads into no more than 2^32-1 to luv
o use the version where we return something with an early EOF (not read_exact)
o use O_PATH

Needs some more nitbits, pushing to show direction

lib_eio/fs.ml

@@ -8,9 +8,27 @@ end
 
 type path = string
 
+type stat = {
+  st_dev : Int64.t;
+  st_ino : Int64.t;
+  (* st_kind : file_kind; (\* mode@luv mode@stat *\) *)
+  (* st_perm : file_perm; (\* mode@luv mode@stat *\) *)
+  st_nlink : Int64.t;
+  st_uid : Int64.t;
+  st_gid : Int64.t;
+  st_rdev : Int64.t;
+  st_size : Optint.Int63.t;
+  st_blksize : Int64.t;
+  st_blocks : Int64.t;
+  st_atime : float;
+  st_mtime : float;
+  st_ctime : float;
+}
+
 exception Already_exists of path * exn
 exception Not_found of path * exn
 exception Permission_denied of path * exn
+exception File_too_large of path * exn
 
 (** When to create a new file. *)
 type create = [
@@ -36,7 +54,7 @@ class virtual dir = object (_ : #Generic.t)
   method virtual unlink : path -> unit
   method virtual rmdir : path -> unit
   method virtual rename : path -> dir -> path -> unit
-  method virtual size : path -> Optint.Int63.t
+  method virtual stat : path -> stat
   method virtual pp : Format.formatter -> unit
 end
 and virtual dir_with_close = object

lib_eio/path.ml

@@ -15,7 +15,7 @@ let open_out ~sw ?(append=false) ~create ((t:#Fs.dir), path) = t#open_out ~sw ~a
 let open_dir ~sw ((t:#Fs.dir), path) = (t#open_dir ~sw path, "")
 let mkdir ~perm ((t:#Fs.dir), path) = t#mkdir ~perm path
 let read_dir ((t:#Fs.dir), path) = List.sort String.compare (t#read_dir path)
-let size ((t:#Fs.dir), (path: string)) : Optint.Int63.t = t#size path
+let stat ((t:#Fs.dir), (path: string)) : Fs.stat = t#stat path
 
 let with_open_in path fn =
   Switch.run @@ fun sw -> fn (open_in ~sw path)
@@ -31,17 +31,24 @@ let with_lines path fn =
   let buf = Buf_read.of_flow flow ~max_size:max_int in
   fn (Buf_read.lines buf)
 
-let load path =
-  let len = size path in
+let load ((t:#Fs.dir), path) =
+  let st = t#stat path in
+  let len = st.st_size in
   let mymax = min Sys.max_string_length Sys.max_array_length in
-  let len = match Optint.Int63.(compare len (of_int mymax)) with
-    | 1 -> mymax
-    | _ -> Optint.Int63.to_int len
-  in
-  with_open_in path @@ fun flow ->
+  if Optint.Int63.(compare len (of_int mymax)) = 1 then
+    raise (Fs.File_too_large
+             (path, Invalid_argument "can't represent a string that long"));
+  with_open_in (t, path) @@ fun flow ->
+  let len = Optint.Int63.to_int len in
   let buf = Cstruct.create len in
-  Flow.read_exact flow buf;
-  Cstruct.to_string buf
+  let rec loop buf got =
+    try
+      let n = Flow.read flow buf in
+      loop (Cstruct.shift buf n) (n + got)
+    with End_of_file -> got
+  in
+  let got = loop buf 0 in
+  Cstruct.to_string ~len:got buf
 
 let save ?append ~create path data =
   with_open_out ?append ~create path @@ fun flow ->

lib_eio/path.mli

@@ -40,8 +40,8 @@ val ( / ) : 'a t -> string -> 'a t
 val pp : _ t Fmt.t
 (** [pp] formats a [_ t] as "<label:path>", suitable for logging. *)
 
-val size : _ t -> Optint.Int63.t
-(** [size]: is the size in bytes of [t]. *)
+val stat : _ t -> stat
+(** [stat]: is the stat(2) of [t]. *)
 
 (** {1 Reading files} *)
 

lib_eio_linux/eio_linux.ml

@@ -36,6 +36,7 @@ let system_thread = Ctf.mint_id ()
 let wrap_errors path fn =
   try fn () with
   | Unix.Unix_error(Unix.EEXIST, _, _) as ex -> raise @@ Eio.Fs.Already_exists (path, ex)
+  | Unix.Unix_error(Unix.E2BIG, _, _)  as ex -> raise @@ Eio.Fs.File_too_large (path, ex)
   | Unix.Unix_error(Unix.ENOENT, _, _) as ex -> raise @@ Eio.Fs.Not_found (path, ex)
   | Unix.Unix_error(Unix.EXDEV, _, _)  as ex -> raise @@ Eio.Fs.Permission_denied (path, ex)
   | Eio.Fs.Permission_denied _         as ex -> raise @@ Eio.Fs.Permission_denied (path, ex)
@@ -1207,16 +1208,31 @@ class dir ~label (fd : dir_fd) = object
     | Dir_fd -> Some fd
     | _ -> None
 
-  method size path =
+  method stat path =
     Switch.run @@ fun sw ->
-    let fd = Low_level.openat ~sw fd path
+    let fd = Low_level.openat ~sw ~seekable:false fd path
           ~access:`R
-          ~flags:Uring.Open_flags.cloexec
+          ~flags:Uring.Open_flags.(path + cloexec)
           ~perm:0
     in
-    let st = Low_level.fstat fd in
+    let ust = Low_level.fstat fd in
     FD.close fd;
-    st.st_size |> Optint.Int63.of_int64
+    Eio.Fs.{
+      st_dev     = ust.st_dev   |> Int64.of_int;
+      st_ino     = ust.st_ino   |> Int64.of_int;
+      (* kind *)
+      (* perm *)
+      st_nlink   = ust.st_nlink |> Int64.of_int;
+      st_uid     = ust.st_uid   |> Int64.of_int;
+      st_gid     = ust.st_gid   |> Int64.of_int;
+      st_rdev    = ust.st_rdev  |> Int64.of_int;
+      st_size    = ust.st_size  |> Optint.Int63.of_int64;
+      st_blksize = 512          |> Int64.of_int;         (* XXX *)
+      st_blocks  = Int64.(div ust.st_size (of_int 512)); (* XXX *)
+      st_atime   = ust.st_atime;
+      st_mtime   = ust.st_mtime;
+      st_ctime   = ust.st_ctime;
+    }
 
   method open_in ~sw path =
     let fd = Low_level.openat ~sw fd path

lib_eio_luv/eio_luv.ml

@@ -56,6 +56,27 @@ let or_raise_path path = function
   | Ok x -> x
   | Error e -> raise (wrap_error ~path e)
 
+(* Luv can't handle buffers with more than 2^32-1 bytes, limit it to
+   31bit so we can also make sure 32bit archs don't overflow.
+   See https://github.com/ocaml-multicore/eio/issues/335 *)
+let cstruct_to_luv buf =
+  Cstruct.to_bigarray @@
+  if (Cstruct.length buf) <= 0x7FFFFFFF then
+    buf
+  else
+    Cstruct.sub buf 0 0x7FFFFFFF
+
+let float_of_timespec (ts : Luv.File.Stat.timespec) =
+  let s : float = Signed.Long.to_int64  ts.sec |> Int64.to_float in
+  let nsec : float = Signed.Long.to_int64  ts.nsec |> Int64.to_float in
+
+  let n = nsec /. 1_000_000_000.0 in
+  let t = s +. n in
+  if t = (s +. 1.0) then
+    Float.next_after t s
+  else
+    t
+
 module Suspended = struct
   type 'a t = {
     fiber : Eio.Private.Fiber_context.t;
@@ -582,20 +603,20 @@ let flow fd = object (_ : <source; sink; ..>)
     | _ -> None
 
   method read_into buf =
-    let buf = Cstruct.to_bigarray buf in
+    let buf = cstruct_to_luv buf in
     match File.read fd [buf] |> or_raise |> Unsigned.Size_t.to_int with
     | 0 -> raise End_of_file
     | got -> got
 
   method pread ~file_offset bufs =
-    let bufs = List.map Cstruct.to_bigarray bufs in
+    let bufs = List.map cstruct_to_luv bufs in
     let file_offset = Optint.Int63.to_int64 file_offset in
     match File.read ~file_offset fd bufs |> or_raise |> Unsigned.Size_t.to_int with
     | 0 -> raise End_of_file
     | got -> got
 
   method pwrite ~file_offset bufs =
-    let bufs = List.map Cstruct.to_bigarray bufs in
+    let bufs = List.map cstruct_to_luv bufs in
     let file_offset = Optint.Int63.to_int64 file_offset in
     File.write_single ~file_offset fd bufs |> or_raise |> Unsigned.Size_t.to_int
 
@@ -629,7 +650,7 @@ let socket sock = object
   method unix_fd op = Stream.to_unix_opt op sock |> Option.get
 
   method read_into buf =
-    let buf = Cstruct.to_bigarray buf in
+    let buf = cstruct_to_luv buf in
     Stream.read_into sock buf
 
   method! write bufs =
@@ -738,7 +759,7 @@ module Udp = struct
 
   let send t buf = function
   | `Udp (host, port) ->
-    let bufs = [ Cstruct.to_bigarray buf ] in
+    let bufs = [ cstruct_to_luv buf ] in
     match await (fun _loop _fiber -> Luv.UDP.send (Handle.get "send" t) bufs (luv_addr_of_eio host port)) with
     | Ok () -> ()
     | Error e -> raise (wrap_flow_error e)
@@ -751,7 +772,7 @@ let udp_socket endp = object
 
   method send sockaddr bufs = Udp.send endp bufs sockaddr
   method recv buf =
-    let buf = Cstruct.to_bigarray buf in
+    let buf = cstruct_to_luv buf in
     Udp.recv endp buf
 end
 
@@ -932,12 +953,27 @@ class dir ~label (dir_path : string) = object (self)
       | exception Eio.Fs.Permission_denied (dir, ex) ->
         raise (Eio.Fs.Permission_denied (Filename.concat dir leaf, ex))
 
-  method size path : Optint.Int63.t =
+  method stat path =
     Switch.run @@ fun sw ->
     let fd = File.open_ ~sw (self#resolve path) [`NOFOLLOW; `RDONLY] |> or_raise_path path in
-    let st = File.fstat fd |> or_raise_path path in
+    let lst : Luv.File.Stat.t = File.fstat fd |> or_raise_path path in
     File.close fd;
-    st.size |> Unsigned.UInt64.to_int64 |> Optint.Int63.of_int64
+    Eio.Fs.{
+      st_dev     = lst.dev     |> Unsigned.UInt64.to_int64; (* lst.st_dev; *)
+      st_ino     = lst.ino     |> Unsigned.UInt64.to_int64;
+      (* kind *)
+      (* perm *)
+      st_nlink   = lst.nlink   |> Unsigned.UInt64.to_int64;
+      st_uid     = lst.uid     |> Unsigned.UInt64.to_int64;
+      st_gid     = lst.gid     |> Unsigned.UInt64.to_int64;
+      st_rdev    = lst.rdev    |> Unsigned.UInt64.to_int64;
+      st_size    = lst.size    |> Unsigned.UInt64.to_int64 |> Optint.Int63.of_int64;
+      st_blksize = lst.blksize |> Unsigned.UInt64.to_int64;
+      st_blocks  = lst.blocks  |> Unsigned.UInt64.to_int64;
+      st_atime   = lst.atim    |> float_of_timespec;
+      st_mtime   = lst.mtim    |> float_of_timespec;
+      st_ctime   = lst.ctim    |> float_of_timespec;
+    }
 
   method open_in ~sw path =
     let fd = File.open_ ~sw (self#resolve path) [`NOFOLLOW; `RDONLY] |> or_raise_path path in