From cbe2a4dcf1033d1f55276667b9b7680e052a3d86 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 12 Sep 2024 18:10:18 +0200 Subject: [PATCH] Cache library scan results --- internal/arduino/libraries/libraries.go | 231 ++++++++++++++++++ .../librariesmanager/librariesmanager.go | 86 +++++-- 2 files changed, 300 insertions(+), 17 deletions(-) diff --git a/internal/arduino/libraries/libraries.go b/internal/arduino/libraries/libraries.go index 9ffa1e27eea..a4ee7717d9f 100644 --- a/internal/arduino/libraries/libraries.go +++ b/internal/arduino/libraries/libraries.go @@ -16,8 +16,11 @@ package libraries import ( + "bytes" + "encoding/binary" "errors" "fmt" + "io" "github.com/arduino/arduino-cli/internal/arduino/cores" "github.com/arduino/arduino-cli/internal/arduino/globals" @@ -90,6 +93,234 @@ func (library *Library) String() string { return library.Name + "@" + library.Version.String() } +func (library *Library) MarshalBinary() []byte { + buffer := bytes.NewBuffer(make([]byte, 0, 4096)) + writeString := func(in string) { + inBytes := []byte(in) + binary.Write(buffer, binary.NativeEndian, uint16(len(inBytes))) + buffer.Write(inBytes) + } + writeStringArray := func(in []string) { + binary.Write(buffer, binary.NativeEndian, uint16(len(in))) + for _, i := range in { + writeString(i) + } + } + writeMap := func(in map[string]bool) { + binary.Write(buffer, binary.NativeEndian, uint16(len(in))) + for k, v := range in { + writeString(k) + binary.Write(buffer, binary.NativeEndian, v) + } + } + writePath := func(in *paths.Path) { + if in == nil { + writeString("") + } else { + writeString(in.String()) + } + } + writeString(library.Name) + writeString(library.Author) + writeString(library.Maintainer) + writeString(library.Sentence) + writeString(library.Paragraph) + writeString(library.Website) + writeString(library.Category) + writeStringArray(library.Architectures) + writeStringArray(library.Types) + writePath(library.InstallDir) + writeString(library.DirName) + writePath(library.SourceDir) + writePath(library.UtilityDir) + binary.Write(buffer, binary.NativeEndian, int32(library.Location)) + // library.ContainerPlatform *cores.PlatformRelease `json:""` + binary.Write(buffer, binary.NativeEndian, int32(library.Layout)) + binary.Write(buffer, binary.NativeEndian, library.DotALinkage) + binary.Write(buffer, binary.NativeEndian, library.Precompiled) + binary.Write(buffer, binary.NativeEndian, library.PrecompiledWithSources) + writeString(library.LDflags) + binary.Write(buffer, binary.NativeEndian, library.IsLegacy) + binary.Write(buffer, binary.NativeEndian, library.InDevelopment) + writeString(library.Version.String()) + writeString(library.License) + //writeStringArray(library.Properties.AsSlice()) + writeStringArray(library.Examples.AsStrings()) + writeStringArray(library.declaredHeaders) + writeStringArray(library.sourceHeaders) + writeMap(library.CompatibleWith) + return buffer.Bytes() +} + +func (library *Library) UnmarshalBinary(in io.Reader) error { + readString := func() (string, error) { + var len uint16 + if err := binary.Read(in, binary.NativeEndian, &len); err != nil { + return "", err + } + res := make([]byte, len) + if _, err := in.Read(res); err != nil { + return "", err + } + return string(res), nil + } + readStringArray := func() ([]string, error) { + var len uint16 + if err := binary.Read(in, binary.NativeEndian, &len); err != nil { + return nil, err + } + res := make([]string, len) + for i := range res { + var err error + res[i], err = readString() + if err != nil { + return nil, err + } + } + return res, nil + } + readMap := func() (map[string]bool, error) { + var len uint16 + if err := binary.Read(in, binary.NativeEndian, &len); err != nil { + return nil, err + } + res := map[string]bool{} + for i := uint16(0); i < len; i++ { + k, err := readString() + if err != nil { + return nil, err + } + var v bool + if err := binary.Read(in, binary.NativeEndian, &v); err != nil { + return nil, err + } + res[k] = v + } + return res, nil + } + var err error + library.Name, err = readString() + if err != nil { + return err + } + library.Author, err = readString() + if err != nil { + return err + } + library.Maintainer, err = readString() + if err != nil { + return err + } + library.Sentence, err = readString() + if err != nil { + return err + } + library.Paragraph, err = readString() + if err != nil { + return err + } + library.Website, err = readString() + if err != nil { + return err + } + library.Category, err = readString() + if err != nil { + return err + } + library.Architectures, err = readStringArray() + if err != nil { + return err + } + library.Types, err = readStringArray() + if err != nil { + return err + } + installDir, err := readString() + if err != nil { + return err + } + library.InstallDir = paths.New(installDir) + library.DirName, err = readString() + if err != nil { + return err + } + sourceDir, err := readString() + library.SourceDir = paths.New(sourceDir) + if err != nil { + return err + } + utilityDir, err := readString() + if err != nil { + return err + } + library.UtilityDir = paths.New(utilityDir) + var location int32 + if err := binary.Read(in, binary.NativeEndian, &location); err != nil { + return err + } + library.Location = LibraryLocation(location) + // library.ContainerPlatform *cores.PlatformRelease `json:""` + var layout int32 + if err := binary.Read(in, binary.NativeEndian, &layout); err != nil { + return err + } + library.Layout = LibraryLayout(layout) + if err := binary.Read(in, binary.NativeEndian, &library.DotALinkage); err != nil { + return err + } + if err := binary.Read(in, binary.NativeEndian, &library.Precompiled); err != nil { + return err + } + if err := binary.Read(in, binary.NativeEndian, &library.PrecompiledWithSources); err != nil { + return err + } + library.LDflags, err = readString() + if err != nil { + return err + } + if err := binary.Read(in, binary.NativeEndian, &library.IsLegacy); err != nil { + return err + } + if err := binary.Read(in, binary.NativeEndian, &library.InDevelopment); err != nil { + return err + } + version, err := readString() + if err != nil { + return err + } + library.Version = semver.MustParse(version) + library.License, err = readString() + if err != nil { + return err + } + // props, err := readStringArray() + // if err != nil { + // return err + // } + // library.Properties, err = properties.LoadFromSlice(props) + // if err != nil { + // return err + // } + examples, err := readStringArray() + if err != nil { + return err + } + library.Examples = paths.NewPathList(examples...) + library.declaredHeaders, err = readStringArray() + if err != nil { + return err + } + library.sourceHeaders, err = readStringArray() + if err != nil { + return err + } + library.CompatibleWith, err = readMap() + if err != nil { + return err + } + return nil +} + // ToRPCLibrary converts this library into an rpc.Library func (library *Library) ToRPCLibrary() (*rpc.Library, error) { pathOrEmpty := func(p *paths.Path) string { diff --git a/internal/arduino/libraries/librariesmanager/librariesmanager.go b/internal/arduino/libraries/librariesmanager/librariesmanager.go index f5883d6673b..46cc68660eb 100644 --- a/internal/arduino/libraries/librariesmanager/librariesmanager.go +++ b/internal/arduino/libraries/librariesmanager/librariesmanager.go @@ -16,6 +16,7 @@ package librariesmanager import ( + "encoding/binary" "errors" "fmt" "os" @@ -201,30 +202,81 @@ func (lm *LibrariesManager) loadLibrariesFromDir(librariesDir *LibrariesDir) []* librariesDir.scanned = true - var libDirs paths.PathList - if librariesDir.IsSingleLibrary { - libDirs.Add(librariesDir.Path) + var loadedLibs []*libraries.Library + if cacheFilePath := librariesDir.Path.Join("libraries-loader-cache"); cacheFilePath.Exist() { + logrus.WithField("file", cacheFilePath).Info("Using library cache") + + // Load lib cache + cache, err := cacheFilePath.Open() + if err != nil { + s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err) + return append(statuses, s) + } + defer cache.Close() + var n int32 + if err := binary.Read(cache, binary.NativeEndian, &n); err != nil { + s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err) + return append(statuses, s) + } + loadedLibs = make([]*libraries.Library, n) + for i := range loadedLibs { + var lib libraries.Library + if err := lib.UnmarshalBinary(cache); err != nil { + s := status.Newf(codes.FailedPrecondition, "reading lib cache %[1]s: %[2]s", cacheFilePath, err) + return append(statuses, s) + } + loadedLibs[i] = &lib + } } else { - d, err := librariesDir.Path.ReadDir() - if os.IsNotExist(err) { - return statuses + var libDirs paths.PathList + if librariesDir.IsSingleLibrary { + libDirs.Add(librariesDir.Path) + } else { + d, err := librariesDir.Path.ReadDir() + if os.IsNotExist(err) { + return statuses + } + if err != nil { + s := status.Newf(codes.FailedPrecondition, i18n.Tr("reading dir %[1]s: %[2]s", librariesDir.Path, err)) + return append(statuses, s) + } + d.FilterDirs() + d.FilterOutHiddenFiles() + libDirs = d } + + for _, libDir := range libDirs { + library, err := libraries.Load(libDir, librariesDir.Location) + if err != nil { + s := status.Newf(codes.Internal, i18n.Tr("loading library from %[1]s: %[2]s", libDir, err)) + statuses = append(statuses, s) + continue + } + loadedLibs = append(loadedLibs, library) + } + + // Write lib cache + cache, err := cacheFilePath.Create() if err != nil { - s := status.Newf(codes.FailedPrecondition, i18n.Tr("reading dir %[1]s: %[2]s", librariesDir.Path, err)) + s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err) + return append(statuses, s) + } + defer cache.Close() + if err := binary.Write(cache, binary.NativeEndian, int32(len(loadedLibs))); err != nil { + cacheFilePath.Remove() + s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err) return append(statuses, s) } - d.FilterDirs() - d.FilterOutHiddenFiles() - libDirs = d + for _, lib := range loadedLibs { + if _, err := cache.Write(lib.MarshalBinary()); err != nil { + cacheFilePath.Remove() + s := status.Newf(codes.FailedPrecondition, "writing lib cache %[1]s: %[2]s", cacheFilePath, err) + return append(statuses, s) + } + } } - for _, libDir := range libDirs { - library, err := libraries.Load(libDir, librariesDir.Location) - if err != nil { - s := status.Newf(codes.Internal, i18n.Tr("loading library from %[1]s: %[2]s", libDir, err)) - statuses = append(statuses, s) - continue - } + for _, library := range loadedLibs { library.ContainerPlatform = librariesDir.PlatformRelease alternatives := lm.libraries[library.Name] alternatives.Add(library)