diff --git a/tools/toitlsp/cmd/repro.go b/tools/toitlsp/cmd/repro.go index 856a73841..a89921e3a 100644 --- a/tools/toitlsp/cmd/repro.go +++ b/tools/toitlsp/cmd/repro.go @@ -139,8 +139,8 @@ func serveRepro(cmd *cobra.Command, args []string) error { documents := lsp.NewDocuments(logger) fs := lsp.MultiFileSystem{lsp.NewDocsCacheFileSystem(documents), reproFS} - fileServer := compiler.NewFileServer(fs, logger, reproFS.sdkPath) - go fileServer.ListenAndServe(fmt.Sprintf(":%d", port)) + fileServer := compiler.NewPortFileServer(fs, logger, reproFS.sdkPath, fmt.Sprintf(":%d", port)) + go fileServer.Run() defer fileServer.Stop() for !fileServer.IsReady() { diff --git a/tools/toitlsp/lsp/compiler/archive.go b/tools/toitlsp/lsp/compiler/archive.go index 2f1f2de6b..0afb3280a 100644 --- a/tools/toitlsp/lsp/compiler/archive.go +++ b/tools/toitlsp/lsp/compiler/archive.go @@ -49,13 +49,13 @@ type ArchiveFile struct { } type WriteArchiveOptions struct { - Writer io.Writer - CompilerFlags []string - CompilerInput string - Info string - FileServer *FileServer - IncludeSDK bool - CWDPath *string + Writer io.Writer + CompilerFlags []string + CompilerInput string + Info string + CompilerFSProtocol *CompilerFSProtocol + IncludeSDK bool + CWDPath *string } // WriteArchive creates a tar file with all files that have been served. @@ -65,13 +65,13 @@ func WriteArchive(ctx context.Context, options WriteArchiveOptions) error { Directories: map[string][]string{}, } - sdkPath, hasSdkPath := options.FileServer.ServedSdkPath() + sdkPath, hasSdkPath := options.CompilerFSProtocol.ServedSdkPath() if !strings.HasSuffix(sdkPath, string(filepath.Separator)) { sdkPath += string(filepath.Separator) } sdkPath = path.ToCompilerPath(sdkPath) - packagePaths := options.FileServer.ServedPackageCachePaths() + packagePaths := options.CompilerFSProtocol.ServedPackageCachePaths() packagePaths = path.ToCompilerPaths(packagePaths...) w := tar.NewWriter(options.Writer) @@ -90,7 +90,7 @@ func WriteArchive(ctx context.Context, options WriteArchiveOptions) error { return err } - for p, file := range options.FileServer.ServedFiles() { + for p, file := range options.CompilerFSProtocol.ServedFiles() { path := path.ToCompilerPath(p) meta.Files[path] = ArchiveFile{ Exists: file.Exists, @@ -109,7 +109,7 @@ func WriteArchive(ctx context.Context, options WriteArchiveOptions) error { } } - meta.Directories = options.FileServer.ServedDirectories() + meta.Directories = options.CompilerFSProtocol.ServedDirectories() metaContent, err := json.Marshal(meta) if err != nil { diff --git a/tools/toitlsp/lsp/compiler/compiler.go b/tools/toitlsp/lsp/compiler/compiler.go index 30df7a542..b2196e422 100644 --- a/tools/toitlsp/lsp/compiler/compiler.go +++ b/tools/toitlsp/lsp/compiler/compiler.go @@ -81,7 +81,7 @@ type Compiler struct { settings Settings logger *zap.Logger fs FileSystem - fileServer *FileServer + fileServer FileServer parser *parser lastCompilerFlags []string @@ -100,7 +100,7 @@ func New(fs FileSystem, logger *zap.Logger, settings Settings) *Compiler { settings: settings, logger: logger, fs: fs, - fileServer: NewFileServer(fs, logger, settings.SDKPath), + fileServer: NewPortFileServer(fs, logger, settings.SDKPath, ":0"), parser: newParser(logger), } } @@ -273,13 +273,13 @@ func (c *Compiler) Archive(ctx context.Context, options ArchiveOptions) error { } return WriteArchive(ctx, WriteArchiveOptions{ - Writer: options.Writer, - CompilerFlags: compilerFlags, - CompilerInput: compilerInput, - Info: options.Info, - FileServer: c.fileServer, - IncludeSDK: options.IncludeSDK, - CWDPath: cwdPath, + Writer: options.Writer, + CompilerFlags: compilerFlags, + CompilerInput: compilerInput, + Info: options.Info, + CompilerFSProtocol: c.fileServer.Protocol(), + IncludeSDK: options.IncludeSDK, + CWDPath: cwdPath, }) } @@ -303,7 +303,7 @@ func (w *logWriter) Write(b []byte) (n int, err error) { type parserFn func(context.Context, io.Reader) func (c *Compiler) run(ctx context.Context, input string, parserFunc parserFn) error { - go c.fileServer.ListenAndServe(":0") + go c.fileServer.Run() defer c.fileServer.Stop() cmd := c.cmd(ctx, input) @@ -367,7 +367,7 @@ func (c *Compiler) cmd(ctx context.Context, input string) *exec.Cmd { c.lastCompilerFlags = args c.lastCompilerInput = input - input = fmt.Sprintf("%d\n%s", c.fileServer.Port(), input) + input = fmt.Sprintf("%s\n%s", c.fileServer.ConfigLine(), input) c.logger.Debug("running compiler", zap.String("input", input), zap.Stringer("cmd", cmd)) cmd.Stdin = strings.NewReader(input) cmd.Stderr = newLogWriter(c.logger.Named("toitc"), zapcore.WarnLevel) diff --git a/tools/toitlsp/lsp/compiler/compiler_fs_protocol.go b/tools/toitlsp/lsp/compiler/compiler_fs_protocol.go new file mode 100644 index 000000000..1906306a7 --- /dev/null +++ b/tools/toitlsp/lsp/compiler/compiler_fs_protocol.go @@ -0,0 +1,244 @@ +// Copyright (C) 2022 Toitware ApS. +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; version +// 2.1 only. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// The license can be found in the file `LICENSE` in the top level +// directory of this repository. + +package compiler + +import ( + "bufio" + "fmt" + "io" + "sync" + + cpath "github.com/toitware/toit.git/toitlsp/lsp/compiler/path" + "go.uber.org/zap" +) + +type FileSystem interface { + Read(path string) (File, error) + ListDirectory(path string) ([]string, error) + PackageCachePaths() ([]string, error) +} + +type File struct { + Path string + Exists bool + IsRegular bool + IsDirectory bool + Content []byte +} + +type CompilerFSProtocol struct { + // l handles the fields listener and closeCh + l sync.Mutex + + fileCache *fileCache + directoryCache *directoryCache + logger *zap.Logger + sdkPath string + fs FileSystem + + servedSdkPath *string + servedPackageCachePaths []string +} + +func NewCompilerFSProtocol(fs FileSystem, logger *zap.Logger, SDKPath string) *CompilerFSProtocol { + return &CompilerFSProtocol{ + fileCache: newFileCache(fs), + directoryCache: newDirectoryCache(fs), + logger: logger.Named("compiler_protocol"), + sdkPath: SDKPath, + fs: fs, + } +} + +func (cp *CompilerFSProtocol) HandleConn(conn io.ReadWriter) { + scanner := bufio.NewScanner(conn) + w := bufio.NewWriter(conn) + for scanner.Scan() { + line := scanner.Text() + switch line { + case "SDK PATH": + cp.l.Lock() + if cp.servedSdkPath == nil { + cp.servedSdkPath = &cp.sdkPath + } + cp.l.Unlock() + //s.logger.Info("SDK Path requested") + w.WriteString(cpath.ToCompilerPath(cp.sdkPath) + "\n") + case "PACKAGE CACHE PATHS": + cp.l.Lock() + if cp.servedPackageCachePaths == nil { + var err error + if cp.servedPackageCachePaths, err = cp.fs.PackageCachePaths(); err != nil { + cp.logger.Error("failed to get package cache paths", zap.Error(err)) + } + } + cp.l.Unlock() + + //s.logger.Info("PACKAGE CACHE PATHS requested") + w.WriteString(fmt.Sprintf("%d\n", len(cp.servedPackageCachePaths))) + for _, path := range cp.servedPackageCachePaths { + w.WriteString(cpath.ToCompilerPath(path) + "\n") + } + case "LIST DIRECTORY": + if !scanner.Scan() { + break + } + path := cpath.FromCompilerPath(scanner.Text()) + //s.logger.Info("LIST DIRECTORY requested", zap.String("path", path)) + entries, err := cp.directoryCache.Get(path) + if err != nil { + cp.logger.Error("failed to directory entries", zap.Error(err)) + return + } + w.WriteString(fmt.Sprintf("%d\n", len(entries))) + for _, e := range entries { + w.WriteString(cpath.ToCompilerPath(e) + "\n") + } + case "INFO": + if !scanner.Scan() { + break + } + path := cpath.FromCompilerPath(scanner.Text()) + + f, err := cp.fileCache.Get(path) + if err != nil { + cp.logger.Error("failed to get file", zap.Error(err)) + return + } + contentLength := -1 + if f.Exists { + contentLength = len(f.Content) + } + //s.logger.Info(fmt.Sprintf("Fetching file: %s - exists: %t - regular: %t - size: %d", path, f.Exists, f.IsRegular, contentLength)) + w.WriteString(fmt.Sprintf("%t\n%t\n%t\n%d\n", f.Exists, f.IsRegular, f.IsDirectory, contentLength)) + w.Write(f.Content) + default: + cp.logger.Error("unhandled line", zap.String("line", line)) + return + } + w.Flush() + } + if err := scanner.Err(); err != nil { + cp.logger.Error("read failed", zap.Error(err)) + } + return +} + +func (cp *CompilerFSProtocol) ServedPackageCachePaths() []string { + cp.l.Lock() + defer cp.l.Unlock() + return cp.servedPackageCachePaths +} + +func (cp *CompilerFSProtocol) ServedSdkPath() (string, bool) { + cp.l.Lock() + defer cp.l.Unlock() + if cp.servedSdkPath == nil { + return "", false + } + return *cp.servedSdkPath, true +} + +func (cp *CompilerFSProtocol) ServedFiles() map[string]File { + return cp.fileCache.Snapshot() +} + +func (cp *CompilerFSProtocol) ServedDirectories() map[string][]string { + return cp.directoryCache.Snapshot() +} + +type directoryCache struct { + l sync.Mutex + + directories map[string][]string + fs FileSystem +} + +func newDirectoryCache(fs FileSystem) *directoryCache { + return &directoryCache{ + directories: map[string][]string{}, + fs: fs, + } +} + +func (d *directoryCache) Get(path string) ([]string, error) { + d.l.Lock() + p, ok := d.directories[path] + d.l.Unlock() + + if !ok { + var err error + if p, err = d.fs.ListDirectory(path); err != nil { + return nil, err + } + d.l.Lock() + d.directories[path] = p + d.l.Unlock() + } + + return p, nil +} + +func (d *directoryCache) Snapshot() map[string][]string { + d.l.Lock() + defer d.l.Unlock() + res := map[string][]string{} + for k, v := range d.directories { + res[k] = v + } + return res +} + +type fileCache struct { + l sync.Mutex + + files map[string]File + fs FileSystem +} + +func newFileCache(fs FileSystem) *fileCache { + return &fileCache{ + files: map[string]File{}, + fs: fs, + } +} + +func (f *fileCache) Get(path string) (File, error) { + f.l.Lock() + defer f.l.Unlock() + + file, ok := f.files[path] + if ok { + return file, nil + } + + var err error + if file, err = f.fs.Read(path); err != nil { + return File{}, err + } + f.files[path] = file + return file, nil +} + +func (f *fileCache) Snapshot() map[string]File { + f.l.Lock() + defer f.l.Unlock() + res := map[string]File{} + for k, v := range f.files { + res[k] = v + } + return res +} diff --git a/tools/toitlsp/lsp/compiler/file_server.go b/tools/toitlsp/lsp/compiler/file_server.go index 42d5bd3b0..8b3dd3fff 100644 --- a/tools/toitlsp/lsp/compiler/file_server.go +++ b/tools/toitlsp/lsp/compiler/file_server.go @@ -16,58 +16,41 @@ package compiler import ( - "bufio" "fmt" "net" + "strconv" "sync" - cpath "github.com/toitware/toit.git/toitlsp/lsp/compiler/path" "go.uber.org/zap" ) -type FileSystem interface { - Read(path string) (File, error) - ListDirectory(path string) ([]string, error) - PackageCachePaths() ([]string, error) +type FileServer interface { + IsReady() bool + /// ConfigLine is the line that is sent to the compiler to be able to + /// communicate with the file server. + ConfigLine() string + Run() error + Stop() error + Protocol() *CompilerFSProtocol } -type File struct { - Path string - Exists bool - IsRegular bool - IsDirectory bool - Content []byte -} - -type FileServer struct { - // l handles the fields listener and closeCh - l sync.Mutex - - fileCache *fileCache - directoryCache *directoryCache - logger *zap.Logger - sdkPath string - fs FileSystem - +type PortFileServer struct { + address string + l sync.Mutex listener net.Listener closeCh chan struct{} - servedSdkPath *string - servedPackageCachePaths []string + cp *CompilerFSProtocol } -func NewFileServer(fs FileSystem, logger *zap.Logger, SDKPath string) *FileServer { - return &FileServer{ - fileCache: newFileCache(fs), - directoryCache: newDirectoryCache(fs), - logger: logger.Named("fileserver"), - sdkPath: SDKPath, - fs: fs, +func NewPortFileServer(fs FileSystem, logger *zap.Logger, SDKPath string, address string) *PortFileServer { + return &PortFileServer{ + cp: NewCompilerFSProtocol(fs, logger, SDKPath), } } -func (s *FileServer) ListenAndServe(address string) error { - ocl, closeCh, err := s.setup(address) +func (s *PortFileServer) Run() error { + ocl, closeCh, err := s.setup(s.address) if err != nil { return err } @@ -76,7 +59,15 @@ func (s *FileServer) ListenAndServe(address string) error { return s.serve(ocl, closeCh) } -func (s *FileServer) setup(address string) (*onceCloseListener, chan struct{}, error) { +func (s *PortFileServer) ConfigLine() string { + return strconv.Itoa(s.Port()) +} + +func (s *PortFileServer) Protocol() *CompilerFSProtocol { + return s.cp +} + +func (s *PortFileServer) setup(address string) (*onceCloseListener, chan struct{}, error) { s.l.Lock() defer s.l.Unlock() if s.listener != nil { @@ -96,14 +87,14 @@ func (s *FileServer) setup(address string) (*onceCloseListener, chan struct{}, e return ocl, closeCh, nil } -func (s *FileServer) clear() { +func (s *PortFileServer) clear() { s.l.Lock() defer s.l.Unlock() s.closeCh = nil s.listener = nil } -func (s *FileServer) serve(l net.Listener, closeCh chan struct{}) error { +func (s *PortFileServer) serve(l net.Listener, closeCh chan struct{}) error { defer close(closeCh) for { conn, err := l.Accept() @@ -115,82 +106,12 @@ func (s *FileServer) serve(l net.Listener, closeCh chan struct{}) error { } } -func (s *FileServer) handleConn(conn net.Conn) { +func (s *PortFileServer) handleConn(conn net.Conn) { defer conn.Close() - scanner := bufio.NewScanner(conn) - w := bufio.NewWriter(conn) - for scanner.Scan() { - line := scanner.Text() - switch line { - case "SDK PATH": - s.l.Lock() - if s.servedSdkPath == nil { - s.servedSdkPath = &s.sdkPath - } - s.l.Unlock() - //s.logger.Info("SDK Path requested") - w.WriteString(cpath.ToCompilerPath(s.sdkPath) + "\n") - case "PACKAGE CACHE PATHS": - s.l.Lock() - if s.servedPackageCachePaths == nil { - var err error - if s.servedPackageCachePaths, err = s.fs.PackageCachePaths(); err != nil { - s.logger.Error("failed to get package cache paths", zap.Error(err)) - } - } - s.l.Unlock() - - //s.logger.Info("PACKAGE CACHE PATHS requested") - w.WriteString(fmt.Sprintf("%d\n", len(s.servedPackageCachePaths))) - for _, path := range s.servedPackageCachePaths { - w.WriteString(cpath.ToCompilerPath(path) + "\n") - } - case "LIST DIRECTORY": - if !scanner.Scan() { - break - } - path := cpath.FromCompilerPath(scanner.Text()) - //s.logger.Info("LIST DIRECTORY requested", zap.String("path", path)) - entries, err := s.directoryCache.Get(path) - if err != nil { - s.logger.Error("failed to directory entries", zap.Error(err)) - return - } - w.WriteString(fmt.Sprintf("%d\n", len(entries))) - for _, e := range entries { - w.WriteString(cpath.ToCompilerPath(e) + "\n") - } - case "INFO": - if !scanner.Scan() { - break - } - path := cpath.FromCompilerPath(scanner.Text()) - - f, err := s.fileCache.Get(path) - if err != nil { - s.logger.Error("failed to get file", zap.Error(err)) - return - } - contentLength := -1 - if f.Exists { - contentLength = len(f.Content) - } - //s.logger.Info(fmt.Sprintf("Fetching file: %s - exists: %t - regular: %t - size: %d", path, f.Exists, f.IsRegular, contentLength)) - w.WriteString(fmt.Sprintf("%t\n%t\n%t\n%d\n", f.Exists, f.IsRegular, f.IsDirectory, contentLength)) - w.Write(f.Content) - default: - s.logger.Error("unhandled line", zap.String("line", line)) - return - } - w.Flush() - } - if err := scanner.Err(); err != nil { - s.logger.Error("read failed", zap.Error(err)) - } - return + s.cp.HandleConn(conn) } -func (s *FileServer) Stop() error { +func (s *PortFileServer) Stop() error { s.l.Lock() listener := s.listener closeCh := s.closeCh @@ -207,131 +128,25 @@ func (s *FileServer) Stop() error { return err } -func (s *FileServer) StopWait() <-chan struct{} { +func (s *PortFileServer) StopWait() <-chan struct{} { s.l.Lock() closeCh := s.closeCh s.l.Unlock() return closeCh } -func (s *FileServer) IsReady() bool { +func (s *PortFileServer) IsReady() bool { s.l.Lock() defer s.l.Unlock() return s.listener != nil } -func (s *FileServer) Port() int { +func (s *PortFileServer) Port() int { s.l.Lock() defer s.l.Unlock() return s.listener.Addr().(*net.TCPAddr).Port } -func (s *FileServer) ServedPackageCachePaths() []string { - s.l.Lock() - defer s.l.Unlock() - return s.servedPackageCachePaths -} - -func (s *FileServer) ServedSdkPath() (string, bool) { - s.l.Lock() - defer s.l.Unlock() - if s.servedSdkPath == nil { - return "", false - } - return *s.servedSdkPath, true -} - -func (s *FileServer) ServedFiles() map[string]File { - return s.fileCache.Snapshot() -} - -func (s *FileServer) ServedDirectories() map[string][]string { - return s.directoryCache.Snapshot() -} - -type directoryCache struct { - l sync.Mutex - - directories map[string][]string - fs FileSystem -} - -func newDirectoryCache(fs FileSystem) *directoryCache { - return &directoryCache{ - directories: map[string][]string{}, - fs: fs, - } -} - -func (d *directoryCache) Get(path string) ([]string, error) { - d.l.Lock() - p, ok := d.directories[path] - d.l.Unlock() - - if !ok { - var err error - if p, err = d.fs.ListDirectory(path); err != nil { - return nil, err - } - d.l.Lock() - d.directories[path] = p - d.l.Unlock() - } - - return p, nil -} - -func (d *directoryCache) Snapshot() map[string][]string { - d.l.Lock() - defer d.l.Unlock() - res := map[string][]string{} - for k, v := range d.directories { - res[k] = v - } - return res -} - -type fileCache struct { - l sync.Mutex - - files map[string]File - fs FileSystem -} - -func newFileCache(fs FileSystem) *fileCache { - return &fileCache{ - files: map[string]File{}, - fs: fs, - } -} - -func (f *fileCache) Get(path string) (File, error) { - f.l.Lock() - defer f.l.Unlock() - - file, ok := f.files[path] - if ok { - return file, nil - } - - var err error - if file, err = f.fs.Read(path); err != nil { - return File{}, err - } - f.files[path] = file - return file, nil -} - -func (f *fileCache) Snapshot() map[string]File { - f.l.Lock() - defer f.l.Unlock() - res := map[string]File{} - for k, v := range f.files { - res[k] = v - } - return res -} - type onceCloseListener struct { net.Listener once sync.Once