diff --git a/commands/service_compile.go b/commands/service_compile.go index 69865f3522c..6a739c30a96 100644 --- a/commands/service_compile.go +++ b/commands/service_compile.go @@ -165,7 +165,7 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu if buildPathArg := req.GetBuildPath(); buildPathArg != "" { buildPath = paths.New(req.GetBuildPath()).Canonical() if in, _ := buildPath.IsInsideDir(sk.FullPath); in && buildPath.IsDir() { - if sk.AdditionalFiles, err = removeBuildFromSketchFiles(sk.AdditionalFiles, buildPath); err != nil { + if sk.AdditionalFiles, err = removeBuildPathFromSketchFiles(sk.AdditionalFiles, buildPath); err != nil { return err } } @@ -214,10 +214,6 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu return err } - actualPlatform := buildPlatform - otherLibrariesDirs := paths.NewPathList(req.GetLibraries()...) - otherLibrariesDirs.Add(s.settings.LibrariesDir()) - var libsManager *librariesmanager.LibrariesManager if pme.GetProfile() != nil { libsManager = lm @@ -240,6 +236,11 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu Message: &rpc.CompileResponse_Progress{Progress: p}, }) } + + librariesDirs := paths.NewPathList(req.GetLibraries()...) // Array of collection of libraries directories + librariesDirs.Add(s.settings.LibrariesDir()) + libraryDirs := paths.NewPathList(req.GetLibrary()...) // Array of single-library directories + sketchBuilder, err := builder.NewBuilder( ctx, sk, @@ -251,16 +252,16 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu int(req.GetJobs()), req.GetBuildProperties(), s.settings.HardwareDirectories(), - otherLibrariesDirs, + librariesDirs, s.settings.IDEBuiltinLibrariesDir(), fqbn, req.GetClean(), req.GetSourceOverride(), req.GetCreateCompilationDatabaseOnly(), - targetPlatform, actualPlatform, + targetPlatform, buildPlatform, req.GetSkipLibrariesDiscovery(), libsManager, - paths.NewPathList(req.GetLibrary()...), + libraryDirs, outStream, errStream, req.GetVerbose(), req.GetWarnings(), progressCB, pme.GetEnvVarsForSpawnedProcess(), @@ -434,15 +435,15 @@ func maybePurgeBuildCache(compilationsBeforePurge uint, cacheTTL time.Duration) buildcache.New(paths.TempDir().Join("arduino", "sketches")).Purge(cacheTTL) } -// removeBuildFromSketchFiles removes the files contained in the build directory from +// removeBuildPathFromSketchFiles removes the files contained in the build directory from // the list of the sketch files -func removeBuildFromSketchFiles(files paths.PathList, build *paths.Path) (paths.PathList, error) { +func removeBuildPathFromSketchFiles(files paths.PathList, build *paths.Path) (paths.PathList, error) { var res paths.PathList ignored := false for _, file := range files { if isInside, _ := file.IsInsideDir(build); !isInside { - res = append(res, file) - } else if !ignored { + res.Add(file) + } else { ignored = true } } diff --git a/commands/service_monitor.go b/commands/service_monitor.go index 7c47e201ada..d384a05b885 100644 --- a/commands/service_monitor.go +++ b/commands/service_monitor.go @@ -27,7 +27,6 @@ import ( "github.com/arduino/arduino-cli/internal/arduino/cores" "github.com/arduino/arduino-cli/internal/arduino/cores/packagemanager" pluggableMonitor "github.com/arduino/arduino-cli/internal/arduino/monitor" - "github.com/arduino/arduino-cli/internal/i18n" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-properties-orderedmap" "github.com/djherbis/buffer" @@ -264,10 +263,7 @@ func findMonitorAndSettingsForProtocolAndBoard(pme *packagemanager.Explorer, pro } else if recipe, ok := boardPlatform.MonitorsDevRecipes[protocol]; ok { // If we have a recipe we must resolve it cmdLine := boardProperties.ExpandPropsInString(recipe) - cmdArgs, err := properties.SplitQuotedString(cmdLine, `"'`, false) - if err != nil { - return nil, nil, &cmderrors.InvalidArgumentError{Message: i18n.Tr("Invalid recipe in platform.txt"), Cause: err} - } + cmdArgs, _ := properties.SplitQuotedString(cmdLine, `"'`, false) id := fmt.Sprintf("%s-%s", boardPlatform, protocol) return pluggableMonitor.New(id, cmdArgs...), boardSettings, nil } diff --git a/commands/service_upload.go b/commands/service_upload.go index 330decc3c5e..0365cdd794b 100644 --- a/commands/service_upload.go +++ b/commands/service_upload.go @@ -703,10 +703,7 @@ func runTool(recipeID string, props *properties.Map, outStream, errStream io.Wri return errors.New(i18n.Tr("no upload port provided")) } cmdLine := props.ExpandPropsInString(recipe) - cmdArgs, err := properties.SplitQuotedString(cmdLine, `"'`, false) - if err != nil { - return errors.New(i18n.Tr("invalid recipe '%[1]s': %[2]s", recipe, err)) - } + cmdArgs, _ := properties.SplitQuotedString(cmdLine, `"'`, false) // Run Tool logrus.WithField("phase", "upload").Tracef("Executing upload tool: %s", cmdLine) diff --git a/internal/arduino/builder/builder.go b/internal/arduino/builder/builder.go index 608baead6ee..b7e415b0865 100644 --- a/internal/arduino/builder/builder.go +++ b/internal/arduino/builder/builder.go @@ -58,9 +58,6 @@ type Builder struct { // Parallel processes jobs int - // Custom build properties defined by user (line by line as "key=value" pairs) - customBuildProperties []string - // core related coreBuildCachePath *paths.Path extraCoreBuildCachePaths paths.PathList @@ -88,7 +85,7 @@ type Builder struct { lineOffset int targetPlatform *cores.PlatformRelease - actualPlatform *cores.PlatformRelease + buildPlatform *cores.PlatformRelease buildArtifacts *buildArtifacts @@ -124,19 +121,20 @@ func NewBuilder( coreBuildCachePath *paths.Path, extraCoreBuildCachePaths paths.PathList, jobs int, - requestBuildProperties []string, - hardwareDirs, otherLibrariesDirs paths.PathList, + customBuildProperties []string, + hardwareDirs paths.PathList, + librariesDirs paths.PathList, builtInLibrariesDirs *paths.Path, fqbn *cores.FQBN, clean bool, sourceOverrides map[string]string, onlyUpdateCompilationDatabase bool, - targetPlatform, actualPlatform *cores.PlatformRelease, + targetPlatform, buildPlatform *cores.PlatformRelease, useCachedLibrariesResolution bool, librariesManager *librariesmanager.LibrariesManager, - libraryDirs paths.PathList, + customLibraryDirs paths.PathList, stdout, stderr io.Writer, verbose bool, warningsLevel string, - progresCB rpc.TaskProgressCB, + progressCB rpc.TaskProgressCB, toolEnv []string, ) (*Builder, error) { buildProperties := properties.NewMap() @@ -145,14 +143,12 @@ func NewBuilder( } if sk != nil { buildProperties.SetPath("sketch_path", sk.FullPath) + buildProperties.Set("build.project_name", sk.MainFile.Base()) + buildProperties.SetPath("build.source.path", sk.FullPath) } if buildPath != nil { buildProperties.SetPath("build.path", buildPath) } - if sk != nil { - buildProperties.Set("build.project_name", sk.MainFile.Base()) - buildProperties.SetPath("build.source.path", sk.FullPath) - } if optimizeForDebug { if debugFlags, ok := buildProperties.GetOk("compiler.optimization_flags.debug"); ok { buildProperties.Set("compiler.optimization_flags", debugFlags) @@ -164,12 +160,11 @@ func NewBuilder( } // Add user provided custom build properties - customBuildProperties, err := properties.LoadFromSlice(requestBuildProperties) - if err != nil { + if p, err := properties.LoadFromSlice(customBuildProperties); err == nil { + buildProperties.Merge(p) + } else { return nil, fmt.Errorf("invalid build properties: %w", err) } - buildProperties.Merge(customBuildProperties) - customBuildPropertiesArgs := append(requestBuildProperties, "build.warn_data_percentage=75") sketchBuildPath, err := buildPath.Join("sketch").Abs() if err != nil { @@ -189,16 +184,20 @@ func NewBuilder( } logger := logger.New(stdout, stderr, verbose, warningsLevel) - libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader( - useCachedLibrariesResolution, librariesManager, - builtInLibrariesDirs, libraryDirs, otherLibrariesDirs, - actualPlatform, targetPlatform, + libsResolver, libsLoadingWarnings, err := detector.LibrariesLoader( + useCachedLibrariesResolution, + librariesManager, + builtInLibrariesDirs, + customLibraryDirs, + librariesDirs, + buildPlatform, + targetPlatform, ) if err != nil { return nil, err } if logger.Verbose() { - logger.Warn(string(verboseOut)) + logger.Warn(string(libsLoadingWarnings)) } diagnosticStore := diagnostics.NewStore() @@ -211,7 +210,6 @@ func NewBuilder( coreBuildPath: coreBuildPath, librariesBuildPath: librariesBuildPath, jobs: jobs, - customBuildProperties: customBuildPropertiesArgs, coreBuildCachePath: coreBuildCachePath, extraCoreBuildCachePaths: extraCoreBuildCachePaths, logger: logger, @@ -219,17 +217,19 @@ func NewBuilder( sourceOverrides: sourceOverrides, onlyUpdateCompilationDatabase: onlyUpdateCompilationDatabase, compilationDatabase: compilation.NewDatabase(buildPath.Join("compile_commands.json")), - Progress: progress.New(progresCB), + Progress: progress.New(progressCB), executableSectionsSize: []ExecutableSectionSize{}, buildArtifacts: &buildArtifacts{}, targetPlatform: targetPlatform, - actualPlatform: actualPlatform, + buildPlatform: buildPlatform, toolEnv: toolEnv, buildOptions: newBuildOptions( - hardwareDirs, otherLibrariesDirs, - builtInLibrariesDirs, buildPath, + hardwareDirs, + librariesDirs, + builtInLibrariesDirs, + buildPath, sk, - customBuildPropertiesArgs, + customBuildProperties, fqbn, clean, buildProperties.Get("compiler.optimization_flags"), @@ -238,7 +238,7 @@ func NewBuilder( ), diagnosticStore: diagnosticStore, libsDetector: detector.NewSketchLibrariesDetector( - libsManager, libsResolver, + libsResolver, useCachedLibrariesResolution, onlyUpdateCompilationDatabase, logger, @@ -321,10 +321,19 @@ func (b *Builder) preprocess() error { b.librariesBuildPath, b.buildProperties, b.targetPlatform.Platform.Architecture, + b.jobs, ) if err != nil { return err } + if b.libsDetector.IncludeFoldersChanged() && b.librariesBuildPath.Exist() { + if b.logger.Verbose() { + b.logger.Info(i18n.Tr("The list of included libraries has been changed... rebuilding all libraries.")) + } + if err := b.librariesBuildPath.RemoveAll(); err != nil { + return err + } + } b.Progress.CompleteStep() b.warnAboutArchIncompatibleLibraries(b.libsDetector.ImportedLibraries()) @@ -493,10 +502,7 @@ func (b *Builder) prepareCommandForRecipe(buildProperties *properties.Map, recip commandLine = properties.DeleteUnexpandedPropsFromString(commandLine) } - parts, err := properties.SplitQuotedString(commandLine, `"'`, false) - if err != nil { - return nil, err - } + args, _ := properties.SplitQuotedString(commandLine, `"'`, false) // if the overall commandline is too long for the platform // try reducing the length by making the filenames relative @@ -504,18 +510,18 @@ func (b *Builder) prepareCommandForRecipe(buildProperties *properties.Map, recip var relativePath string if len(commandLine) > 30000 { relativePath = buildProperties.Get("build.path") - for i, arg := range parts { + for i, arg := range args { if _, err := os.Stat(arg); os.IsNotExist(err) { continue } rel, err := filepath.Rel(relativePath, arg) if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) { - parts[i] = rel + args[i] = rel } } } - command, err := paths.NewProcess(b.toolEnv, parts...) + command, err := paths.NewProcess(b.toolEnv, args...) if err != nil { return nil, err } diff --git a/internal/arduino/builder/compilation.go b/internal/arduino/builder/compilation.go index d3a1459da25..f3d85908ea0 100644 --- a/internal/arduino/builder/compilation.go +++ b/internal/arduino/builder/compilation.go @@ -110,7 +110,6 @@ func (b *Builder) compileFiles( return objectFiles, nil } -// CompileFilesRecursive fixdoc func (b *Builder) compileFileWithRecipe( sourcePath *paths.Path, source *paths.Path, @@ -118,28 +117,21 @@ func (b *Builder) compileFileWithRecipe( includes []string, recipe string, ) (*paths.Path, error) { - properties := b.buildProperties.Clone() - properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+b.logger.WarningsLevel())) - properties.Set("includes", strings.Join(includes, " ")) - properties.SetPath("source_file", source) relativeSource, err := sourcePath.RelTo(source) if err != nil { return nil, err } depsFile := buildPath.Join(relativeSource.String() + ".d") objectFile := buildPath.Join(relativeSource.String() + ".o") - - properties.SetPath("object_file", objectFile) - err = objectFile.Parent().MkdirAll() - if err != nil { - return nil, err - } - - objIsUpToDate, err := utils.ObjFileIsUpToDate(source, objectFile, depsFile) - if err != nil { + if err := objectFile.Parent().MkdirAll(); err != nil { return nil, err } + properties := b.buildProperties.Clone() + properties.Set("compiler.warning_flags", properties.Get("compiler.warning_flags."+b.logger.WarningsLevel())) + properties.Set("includes", strings.Join(includes, " ")) + properties.SetPath("source_file", source) + properties.SetPath("object_file", objectFile) command, err := b.prepareCommandForRecipe(properties, recipe, false) if err != nil { return nil, err @@ -147,41 +139,50 @@ func (b *Builder) compileFileWithRecipe( if b.compilationDatabase != nil { b.compilationDatabase.Add(source, command) } - if !objIsUpToDate && !b.onlyUpdateCompilationDatabase { - commandStdout, commandStderr := &bytes.Buffer{}, &bytes.Buffer{} - command.RedirectStdoutTo(commandStdout) - command.RedirectStderrTo(commandStderr) + objIsUpToDate, err := utils.ObjFileIsUpToDate(source, objectFile, depsFile) + if err != nil { + return nil, err + } + if objIsUpToDate { if b.logger.Verbose() { - b.logger.Info(utils.PrintableCommand(command.GetArgs())) - } - // Since this compile could be multithreaded, we first capture the command output - if err := command.Start(); err != nil { - return nil, err + b.logger.Info(i18n.Tr("Using previously compiled file: %[1]s", objectFile)) } - err := command.Wait() - // and transfer all at once at the end... + return objectFile, nil + } + if b.onlyUpdateCompilationDatabase { if b.logger.Verbose() { - b.logger.WriteStdout(commandStdout.Bytes()) + b.logger.Info(i18n.Tr("Skipping compile of: %[1]s", objectFile)) } - b.logger.WriteStderr(commandStderr.Bytes()) + return objectFile, nil + } - // Parse the output of the compiler to gather errors and warnings... - if b.diagnosticStore != nil { - b.diagnosticStore.Parse(command.GetArgs(), commandStdout.Bytes()) - b.diagnosticStore.Parse(command.GetArgs(), commandStderr.Bytes()) - } + commandStdout, commandStderr := &bytes.Buffer{}, &bytes.Buffer{} + command.RedirectStdoutTo(commandStdout) + command.RedirectStderrTo(commandStderr) + if b.logger.Verbose() { + b.logger.Info(utils.PrintableCommand(command.GetArgs())) + } + // Since this compile could be multithreaded, we first capture the command output + if err := command.Start(); err != nil { + return nil, err + } + err = command.Wait() + // and transfer all at once at the end... + if b.logger.Verbose() { + b.logger.WriteStdout(commandStdout.Bytes()) + } + b.logger.WriteStderr(commandStderr.Bytes()) - // ...and then return the error - if err != nil { - return nil, err - } - } else if b.logger.Verbose() { - if objIsUpToDate { - b.logger.Info(i18n.Tr("Using previously compiled file: %[1]s", objectFile)) - } else { - b.logger.Info(i18n.Tr("Skipping compile of: %[1]s", objectFile)) - } + // Parse the output of the compiler to gather errors and warnings... + if b.diagnosticStore != nil { + b.diagnosticStore.Parse(command.GetArgs(), commandStdout.Bytes()) + b.diagnosticStore.Parse(command.GetArgs(), commandStderr.Bytes()) + } + + // ...and then return the error + if err != nil { + return nil, err } return objectFile, nil diff --git a/internal/arduino/builder/core.go b/internal/arduino/builder/core.go index f407ed60c41..c3e7d4bb04b 100644 --- a/internal/arduino/builder/core.go +++ b/internal/arduino/builder/core.go @@ -163,7 +163,7 @@ func (b *Builder) compileCore() (*paths.Path, paths.PathList, error) { b.logger.Info(i18n.Tr("Archiving built core (caching) in: %[1]s", targetArchivedCore)) } else if os.IsNotExist(err) { b.logger.Info(i18n.Tr("Unable to cache built core, please tell %[1]s maintainers to follow %[2]s", - b.actualPlatform, + b.buildPlatform, "https://arduino.github.io/arduino-cli/latest/platform-specification/#recipes-to-build-the-corea-archive-file")) } else { b.logger.Info(i18n.Tr("Error archiving built core (caching) in %[1]s: %[2]s", targetArchivedCore, err)) diff --git a/internal/arduino/builder/internal/detector/cache.go b/internal/arduino/builder/internal/detector/cache.go new file mode 100644 index 00000000000..62b3d355678 --- /dev/null +++ b/internal/arduino/builder/internal/detector/cache.go @@ -0,0 +1,132 @@ +// This file is part of arduino-cli. +// +// Copyright 2024 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package detector + +import ( + "encoding/json" + "fmt" + + "github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner" + "github.com/arduino/go-paths-helper" +) + +type detectorCache struct { + curr int + entries []*detectorCacheEntry +} + +type detectorCacheEntry struct { + AddedIncludePath *paths.Path `json:"added_include_path,omitempty"` + Compile *sourceFile `json:"compile,omitempty"` + CompileTask *runner.Task `json:"compile_task,omitempty"` + MissingIncludeH *string `json:"missing_include_h,omitempty"` +} + +func (e *detectorCacheEntry) String() string { + if e.AddedIncludePath != nil { + return "Added include path: " + e.AddedIncludePath.String() + } + if e.Compile != nil && e.CompileTask != nil { + return "Compiling: " + e.Compile.String() + " / " + e.CompileTask.String() + } + if e.MissingIncludeH != nil { + if *e.MissingIncludeH == "" { + return "No missing include files detected" + } + return "Missing include file: " + *e.MissingIncludeH + } + return "No operation" +} + +func (e *detectorCacheEntry) Equals(entry *detectorCacheEntry) bool { + return e.String() == entry.String() +} + +func newDetectorCache() *detectorCache { + return &detectorCache{} +} + +func (c *detectorCache) String() string { + res := "" + for _, entry := range c.entries { + res += fmt.Sprintln(entry) + } + return res +} + +// Load reads a saved cache from the given file. +// If the file do not exists, it does nothing. +func (c *detectorCache) Load(cacheFile *paths.Path) error { + if exist, err := cacheFile.ExistCheck(); err != nil { + return err + } else if !exist { + return nil + } + data, err := cacheFile.ReadFile() + if err != nil { + return err + } + var entries []*detectorCacheEntry + if err := json.Unmarshal(data, &entries); err != nil { + return err + } + c.curr = 0 + c.entries = entries + return nil +} + +// Expect adds an entry to the cache and checks if it matches the next expected entry. +func (c *detectorCache) Expect(entry *detectorCacheEntry) { + if c.curr < len(c.entries) { + if c.entries[c.curr].Equals(entry) { + // Cache hit, move to the next entry + c.curr++ + return + } + // Cache mismatch, invalidate and cut the remainder of the cache + c.entries = c.entries[:c.curr] + } + c.curr++ + c.entries = append(c.entries, entry) +} + +// Peek returns the next cache entry to be expected or nil if the cache is fully consumed. +func (c *detectorCache) Peek() *detectorCacheEntry { + if c.curr < len(c.entries) { + return c.entries[c.curr] + } + return nil +} + +// EntriesAhead returns the entries that are ahead of the current cache position. +func (c *detectorCache) EntriesAhead() []*detectorCacheEntry { + if c.curr < len(c.entries) { + return c.entries[c.curr:] + } + return nil +} + +// Save writes the current cache to the given file. +func (c *detectorCache) Save(cacheFile *paths.Path) error { + // Cut off the cache if it is not fully consumed + c.entries = c.entries[:c.curr] + + data, err := json.MarshalIndent(c.entries, "", " ") + if err != nil { + return err + } + return cacheFile.WriteFile(data) +} diff --git a/internal/arduino/builder/internal/detector/detector.go b/internal/arduino/builder/internal/detector/detector.go index 17de9377833..847c67e9cd2 100644 --- a/internal/arduino/builder/internal/detector/detector.go +++ b/internal/arduino/builder/internal/detector/detector.go @@ -30,6 +30,7 @@ import ( "github.com/arduino/arduino-cli/internal/arduino/builder/internal/diagnostics" "github.com/arduino/arduino-cli/internal/arduino/builder/internal/logger" "github.com/arduino/arduino-cli/internal/arduino/builder/internal/preprocessor" + "github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner" "github.com/arduino/arduino-cli/internal/arduino/builder/internal/utils" "github.com/arduino/arduino-cli/internal/arduino/cores" "github.com/arduino/arduino-cli/internal/arduino/globals" @@ -49,20 +50,21 @@ type libraryResolutionResult struct { // SketchLibrariesDetector todo type SketchLibrariesDetector struct { - librariesManager *librariesmanager.LibrariesManager librariesResolver *librariesresolver.Cpp useCachedLibrariesResolution bool + cache *detectorCache onlyUpdateCompilationDatabase bool importedLibraries libraries.List librariesResolutionResults map[string]libraryResolutionResult includeFolders paths.PathList logger *logger.BuilderLogger diagnosticStore *diagnostics.Store + preRunner *runner.Runner + detectedChangeInLibraries bool } // NewSketchLibrariesDetector todo func NewSketchLibrariesDetector( - lm *librariesmanager.LibrariesManager, libsResolver *librariesresolver.Cpp, useCachedLibrariesResolution bool, onlyUpdateCompilationDatabase bool, @@ -70,9 +72,9 @@ func NewSketchLibrariesDetector( diagnosticStore *diagnostics.Store, ) *SketchLibrariesDetector { return &SketchLibrariesDetector{ - librariesManager: lm, librariesResolver: libsResolver, useCachedLibrariesResolution: useCachedLibrariesResolution, + cache: newDetectorCache(), librariesResolutionResults: map[string]libraryResolutionResult{}, importedLibraries: libraries.List{}, includeFolders: paths.PathList{}, @@ -169,28 +171,21 @@ func (l *SketchLibrariesDetector) PrintUsedAndNotUsedLibraries(sketchError bool) time.Sleep(100 * time.Millisecond) } -// IncludeFolders fixdoc +// IncludeFolders returns the list of include folders detected as needed. func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList { - // TODO should we do a deep copy? return l.includeFolders } -// appendIncludeFolder todo should rename this, probably after refactoring the -// container_find_includes command. -// Original comment: -// Append the given folder to the include path and match or append it to -// the cache. sourceFilePath and include indicate the source of this -// include (e.g. what #include line in what file it was resolved from) -// and should be the empty string for the default include folders, like -// the core or variant. -func (l *SketchLibrariesDetector) appendIncludeFolder( - cache *includeCache, - sourceFilePath *paths.Path, - include string, - folder *paths.Path, -) { +// IncludeFoldersChanged returns true if the include folders list changed +// from the previous compile. +func (l *SketchLibrariesDetector) IncludeFoldersChanged() bool { + return l.detectedChangeInLibraries +} + +// addIncludeFolder add the given folder to the include path. +func (l *SketchLibrariesDetector) addIncludeFolder(folder *paths.Path) { l.includeFolders = append(l.includeFolders, folder) - cache.ExpectEntry(sourceFilePath, include, folder) + l.cache.Expect(&detectorCacheEntry{AddedIncludePath: folder}) } // FindIncludes todo @@ -204,8 +199,9 @@ func (l *SketchLibrariesDetector) FindIncludes( librariesBuildPath *paths.Path, buildProperties *properties.Map, platformArch string, + jobs int, ) error { - err := l.findIncludes(ctx, buildPath, buildCorePath, buildVariantPath, sketchBuildPath, sketch, librariesBuildPath, buildProperties, platformArch) + err := l.findIncludes(ctx, buildPath, buildCorePath, buildVariantPath, sketchBuildPath, sketch, librariesBuildPath, buildProperties, platformArch, jobs) if err != nil && l.onlyUpdateCompilationDatabase { l.logger.Info( fmt.Sprintf( @@ -229,28 +225,47 @@ func (l *SketchLibrariesDetector) findIncludes( librariesBuildPath *paths.Path, buildProperties *properties.Map, platformArch string, + jobs int, ) error { - librariesResolutionCache := buildPath.Join("libraries.cache") - if l.useCachedLibrariesResolution && librariesResolutionCache.Exist() { - d, err := librariesResolutionCache.ReadFile() + librariesResolutionCachePath := buildPath.Join("libraries.cache") + var cachedIncludeFolders paths.PathList + if librariesResolutionCachePath.Exist() { + d, err := librariesResolutionCachePath.ReadFile() if err != nil { return err } - if err := json.Unmarshal(d, &l.includeFolders); err != nil { + if err := json.Unmarshal(d, &cachedIncludeFolders); err != nil { return err } + } + if l.useCachedLibrariesResolution && librariesResolutionCachePath.Exist() { + l.includeFolders = cachedIncludeFolders if l.logger.Verbose() { - l.logger.Info("Using cached library discovery: " + librariesResolutionCache.String()) + l.logger.Info("Using cached library discovery: " + librariesResolutionCachePath.String()) } return nil } cachePath := buildPath.Join("includes.cache") - cache := readCache(cachePath) + if err := l.cache.Load(cachePath); err != nil { + l.logger.Warn(i18n.Tr("Failed to load library discovery cache: %[1]s", err)) + } - l.appendIncludeFolder(cache, nil, "", buildCorePath) + // Pre-run cache entries + l.preRunner = runner.New(ctx, jobs) + for _, entry := range l.cache.EntriesAhead() { + if entry.Compile != nil && entry.CompileTask != nil { + upToDate, _ := entry.Compile.ObjFileIsUpToDate() + if !upToDate { + l.preRunner.Enqueue(entry.CompileTask) + } + } + } + defer l.preRunner.Cancel() + + l.addIncludeFolder(buildCorePath) if buildVariantPath != nil { - l.appendIncludeFolder(cache, nil, "", buildVariantPath) + l.addIncludeFolder(buildVariantPath) } sourceFileQueue := &uniqueSourceFileQueue{} @@ -261,7 +276,7 @@ func (l *SketchLibrariesDetector) findIncludes( if err != nil { return err } - sourceFileQueue.push(mergedfile) + sourceFileQueue.Push(mergedfile) l.queueSourceFilesFromFolder(sourceFileQueue, sketchBuildPath, false /* recurse */, sketchBuildPath, sketchBuildPath) srcSubfolderPath := sketchBuildPath.Join("src") @@ -269,17 +284,25 @@ func (l *SketchLibrariesDetector) findIncludes( l.queueSourceFilesFromFolder(sourceFileQueue, srcSubfolderPath, true /* recurse */, sketchBuildPath, sketchBuildPath) } - for !sourceFileQueue.empty() { - err := l.findIncludesUntilDone(ctx, cache, sourceFileQueue, buildProperties, librariesBuildPath, platformArch) + for !sourceFileQueue.Empty() { + err := l.findMissingIncludesInCompilationUnit(ctx, sourceFileQueue, buildProperties, librariesBuildPath, platformArch) if err != nil { cachePath.Remove() return err } + + // Create a new pre-runner if the previous one was cancelled + if l.preRunner == nil { + l.preRunner = runner.New(ctx, jobs) + // Push in the remainder of the queue + for _, sourceFile := range *sourceFileQueue { + l.preRunner.Enqueue(l.gccPreprocessTask(sourceFile, buildProperties)) + } + } } // Finalize the cache - cache.ExpectEnd() - if err := writeCache(cache, cachePath); err != nil { + if err := l.cache.Save(cachePath); err != nil { return err } } @@ -290,26 +313,37 @@ func (l *SketchLibrariesDetector) findIncludes( if d, err := json.Marshal(l.includeFolders); err != nil { return err - } else if err := librariesResolutionCache.WriteFile(d); err != nil { + } else if err := librariesResolutionCachePath.WriteFile(d); err != nil { return err } - + l.detectedChangeInLibraries = !slices.Equal( + cachedIncludeFolders.AsStrings(), + l.includeFolders.AsStrings()) return nil } -func (l *SketchLibrariesDetector) findIncludesUntilDone( +func (l *SketchLibrariesDetector) gccPreprocessTask(sourceFile *sourceFile, buildProperties *properties.Map) *runner.Task { + // Libraries may require the "utility" directory to be added to the include + // search path, but only for the source code of the library, so we temporary + // copy the current search path list and add the library' utility directory + // if needed. + includeFolders := l.includeFolders + if extraInclude := sourceFile.ExtraIncludePath; extraInclude != nil { + includeFolders = append(includeFolders, extraInclude) + } + + return preprocessor.GCC(sourceFile.SourcePath, paths.NullPath(), includeFolders, buildProperties) +} + +func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit( ctx context.Context, - cache *includeCache, sourceFileQueue *uniqueSourceFileQueue, buildProperties *properties.Map, librariesBuildPath *paths.Path, platformArch string, ) error { - sourceFile := sourceFileQueue.pop() - sourcePath := sourceFile.SourcePath() - targetFilePath := paths.NullPath() - depPath := sourceFile.DepfilePath() - objPath := sourceFile.ObjectPath() + sourceFile := sourceFileQueue.Pop() + sourcePath := sourceFile.SourcePath // TODO: This should perhaps also compare against the // include.cache file timestamp. Now, it only checks if the file @@ -323,82 +357,95 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone( // TODO: This reads the dependency file, but the actual building // does it again. Should the result be somehow cached? Perhaps // remove the object file if it is found to be stale? - unchanged, err := utils.ObjFileIsUpToDate(sourcePath, objPath, depPath) + unchanged, err := sourceFile.ObjFileIsUpToDate() if err != nil { return err } first := true for { - cache.ExpectFile(sourcePath) - - // Libraries may require the "utility" directory to be added to the include - // search path, but only for the source code of the library, so we temporary - // copy the current search path list and add the library' utility directory - // if needed. - includeFolders := l.includeFolders - if extraInclude := sourceFile.ExtraIncludePath(); extraInclude != nil { - includeFolders = append(includeFolders, extraInclude) - } + preprocTask := l.gccPreprocessTask(sourceFile, buildProperties) + l.cache.Expect(&detectorCacheEntry{Compile: sourceFile, CompileTask: preprocTask}) var preprocErr error - var preprocFirstResult preprocessor.Result + var preprocResult *runner.Result var missingIncludeH string - if unchanged && cache.valid { - missingIncludeH = cache.Next().Include + if entry := l.cache.Peek(); unchanged && entry != nil && entry.MissingIncludeH != nil { + missingIncludeH = *entry.MissingIncludeH if first && l.logger.Verbose() { l.logger.Info(i18n.Tr("Using cached library dependencies for file: %[1]s", sourcePath)) } + first = false } else { - preprocFirstResult, preprocErr = preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties) + if l.preRunner != nil { + if r := l.preRunner.Results(preprocTask); r != nil { + preprocResult = r + preprocErr = preprocResult.Error + } + } + if preprocResult == nil { + // The pre-runner missed this task, maybe the cache is outdated + // or maybe the source code changed. + + // Stop the pre-runner + if l.preRunner != nil { + preRunner := l.preRunner + l.preRunner = nil + go preRunner.Cancel() + } + + // Run the actual preprocessor + preprocResult = preprocTask.Run(ctx) + preprocErr = preprocResult.Error + } if l.logger.Verbose() { - l.logger.WriteStdout(preprocFirstResult.Stdout()) + l.logger.WriteStdout(preprocResult.Stdout) } // Unwrap error and see if it is an ExitError. var exitErr *exec.ExitError if preprocErr == nil { // Preprocessor successful, done missingIncludeH = "" - } else if isExitErr := errors.As(preprocErr, &exitErr); !isExitErr || preprocFirstResult.Stderr() == nil { + } else if isExitErr := errors.As(preprocErr, &exitErr); !isExitErr || len(preprocResult.Stderr) == 0 { // Ignore ExitErrors (e.g. gcc returning non-zero status), but bail out on other errors return preprocErr } else { - missingIncludeH = IncludesFinderWithRegExp(string(preprocFirstResult.Stderr())) + missingIncludeH = IncludesFinderWithRegExp(string(preprocResult.Stderr)) if missingIncludeH == "" && l.logger.Verbose() { l.logger.Info(i18n.Tr("Error while detecting libraries included by %[1]s", sourcePath)) } } } + l.cache.Expect(&detectorCacheEntry{MissingIncludeH: &missingIncludeH}) + if missingIncludeH == "" { // No missing includes found, we're done - cache.ExpectEntry(sourcePath, "", nil) return nil } library := l.resolveLibrary(missingIncludeH, platformArch) if library == nil { // Library could not be resolved, show error - if preprocErr == nil || preprocFirstResult.Stderr() == nil { - // Filename came from cache, so run preprocessor to obtain error to show - result, err := preprocessor.GCC(ctx, sourcePath, targetFilePath, includeFolders, buildProperties) + + // If preprocess result came from cache, run the preprocessor to obtain the actual error to show + if preprocErr == nil || len(preprocResult.Stderr) == 0 { + preprocResult = preprocTask.Run(ctx) + preprocErr = preprocResult.Error if l.logger.Verbose() { - l.logger.WriteStdout(result.Stdout()) + l.logger.WriteStdout(preprocResult.Stdout) } - if err == nil { + if preprocErr == nil { // If there is a missing #include in the cache, but running // gcc does not reproduce that, there is something wrong. // Returning an error here will cause the cache to be // deleted, so hopefully the next compilation will succeed. return errors.New(i18n.Tr("Internal error in cache")) } - l.diagnosticStore.Parse(result.Args(), result.Stderr()) - l.logger.WriteStderr(result.Stderr()) - return err } - l.diagnosticStore.Parse(preprocFirstResult.Args(), preprocFirstResult.Stderr()) - l.logger.WriteStderr(preprocFirstResult.Stderr()) + l.diagnosticStore.Parse(preprocResult.Args, preprocResult.Stderr) + l.logger.WriteStderr(preprocResult.Stderr) return preprocErr } @@ -406,7 +453,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone( // include path and queue its source files for further // include scanning l.AppendImportedLibraries(library) - l.appendIncludeFolder(cache, sourcePath, missingIncludeH, library.SourceDir) + l.addIncludeFolder(library.SourceDir) if library.Precompiled && library.PrecompiledWithSources { // Fully precompiled libraries should have no dependencies to avoid ABI breakage @@ -419,7 +466,6 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone( library.SourceDir, librariesBuildPath.Join(library.DirName), library.UtilityDir) } } - first = false } } @@ -445,7 +491,7 @@ func (l *SketchLibrariesDetector) queueSourceFilesFromFolder( if err != nil { return err } - sourceFileQueue.push(sourceFile) + sourceFileQueue.Push(sourceFile) } return nil @@ -502,104 +548,16 @@ func findIncludeForOldCompilers(source string) string { return "" } -type sourceFile struct { - // Path to the source file within the sketch/library root folder - relativePath *paths.Path - - // ExtraIncludePath contains an extra include path that must be - // used to compile this source file. - // This is mainly used for source files that comes from old-style libraries - // (Arduino IDE <1.5) requiring an extra include path to the "utility" folder. - extraIncludePath *paths.Path - - // The source root for the given origin, where its source files - // can be found. Prepending this to SourceFile.RelativePath will give - // the full path to that source file. - sourceRoot *paths.Path - - // The build root for the given origin, where build products will - // be placed. Any directories inside SourceFile.RelativePath will be - // appended here. - buildRoot *paths.Path -} - -// Equals fixdoc -func (f *sourceFile) Equals(g *sourceFile) bool { - return f.relativePath.EqualsTo(g.relativePath) && - f.buildRoot.EqualsTo(g.buildRoot) && - f.sourceRoot.EqualsTo(g.sourceRoot) -} - -// makeSourceFile containing the given source file path within the -// given origin. The given path can be absolute, or relative within the -// origin's root source folder -func makeSourceFile( - sourceDir *paths.Path, - buildDir *paths.Path, - sourceFilePath *paths.Path, - extraIncludePath ...*paths.Path, -) (*sourceFile, error) { - res := &sourceFile{ - buildRoot: buildDir, - sourceRoot: sourceDir, - } - - if len(extraIncludePath) > 1 { - panic("only one extra include path allowed") - } - if len(extraIncludePath) > 0 { - res.extraIncludePath = extraIncludePath[0] - } - // switch o := origin.(type) { - // case *sketch.Sketch: - // res.buildRoot = sketchBuildPath - // res.sourceRoot = sketchBuildPath - // case *libraries.Library: - // res.buildRoot = librariesBuildPath.Join(o.DirName) - // res.sourceRoot = o.SourceDir - // res.extraIncludePath = o.UtilityDir - // default: - // panic("Unexpected origin for SourceFile: " + fmt.Sprint(origin)) - // } - - if sourceFilePath.IsAbs() { - var err error - sourceFilePath, err = res.sourceRoot.RelTo(sourceFilePath) - if err != nil { - return nil, err - } - } - res.relativePath = sourceFilePath - return res, nil -} - -// ExtraIncludePath fixdoc -func (f *sourceFile) ExtraIncludePath() *paths.Path { - return f.extraIncludePath -} - -// SourcePath fixdoc -func (f *sourceFile) SourcePath() *paths.Path { - return f.sourceRoot.JoinPath(f.relativePath) -} - -// ObjectPath fixdoc -func (f *sourceFile) ObjectPath() *paths.Path { - return f.buildRoot.Join(f.relativePath.String() + ".o") -} - -// DepfilePath fixdoc -func (f *sourceFile) DepfilePath() *paths.Path { - return f.buildRoot.Join(f.relativePath.String() + ".d") -} - // LibrariesLoader todo func LibrariesLoader( useCachedLibrariesResolution bool, librariesManager *librariesmanager.LibrariesManager, - builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList, - actualPlatform, targetPlatform *cores.PlatformRelease, -) (*librariesmanager.LibrariesManager, *librariesresolver.Cpp, []byte, error) { + builtInLibrariesDir *paths.Path, + customLibraryDirs paths.PathList, + librariesDirs paths.PathList, + buildPlatform *cores.PlatformRelease, + targetPlatform *cores.PlatformRelease, +) (*librariesresolver.Cpp, []byte, error) { verboseOut := &bytes.Buffer{} lm := librariesManager if useCachedLibrariesResolution { @@ -610,21 +568,20 @@ func LibrariesLoader( if librariesManager == nil { lmb := librariesmanager.NewBuilder() - builtInLibrariesFolders := builtInLibrariesDirs - if builtInLibrariesFolders != nil { - if err := builtInLibrariesFolders.ToAbs(); err != nil { - return nil, nil, nil, err + if builtInLibrariesDir != nil { + if err := builtInLibrariesDir.ToAbs(); err != nil { + return nil, nil, err } lmb.AddLibrariesDir(librariesmanager.LibrariesDir{ - Path: builtInLibrariesFolders, + Path: builtInLibrariesDir, Location: libraries.IDEBuiltIn, }) } - if actualPlatform != targetPlatform { + if buildPlatform != targetPlatform { lmb.AddLibrariesDir(librariesmanager.LibrariesDir{ - PlatformRelease: actualPlatform, - Path: actualPlatform.GetLibrariesDir(), + PlatformRelease: buildPlatform, + Path: buildPlatform.GetLibrariesDir(), Location: libraries.ReferencedPlatformBuiltIn, }) } @@ -634,9 +591,9 @@ func LibrariesLoader( Location: libraries.PlatformBuiltIn, }) - librariesFolders := otherLibrariesDirs + librariesFolders := librariesDirs if err := librariesFolders.ToAbs(); err != nil { - return nil, nil, nil, err + return nil, nil, err } for _, folder := range librariesFolders { lmb.AddLibrariesDir(librariesmanager.LibrariesDir{ @@ -645,7 +602,7 @@ func LibrariesLoader( }) } - for _, dir := range libraryDirs { + for _, dir := range customLibraryDirs { lmb.AddLibrariesDir(librariesmanager.LibrariesDir{ Path: dir, Location: libraries.Unmanaged, @@ -655,152 +612,12 @@ func LibrariesLoader( newLm, libsLoadingWarnings := lmb.Build() for _, status := range libsLoadingWarnings { - // With the refactoring of the initialization step of the CLI we changed how - // errors are returned when loading platforms and libraries, that meant returning a list of - // errors instead of a single one to enhance the experience for the user. - // I have no intention right now to start a refactoring of the legacy package too, so - // here's this shitty solution for now. - // When we're gonna refactor the legacy package this will be gone. verboseOut.Write([]byte(status.Message())) } lm = newLm } allLibs := lm.FindAllInstalled() - resolver := librariesresolver.NewCppResolver(allLibs, targetPlatform, actualPlatform) - return lm, resolver, verboseOut.Bytes(), nil -} - -type includeCacheEntry struct { - Sourcefile *paths.Path - Include string - Includepath *paths.Path -} - -// String fixdoc -func (entry *includeCacheEntry) String() string { - return fmt.Sprintf("SourceFile: %s; Include: %s; IncludePath: %s", - entry.Sourcefile, entry.Include, entry.Includepath) -} - -// Equals fixdoc -func (entry *includeCacheEntry) Equals(other *includeCacheEntry) bool { - return entry.String() == other.String() -} - -type includeCache struct { - // Are the cache contents valid so far? - valid bool - // Index into entries of the next entry to be processed. Unused - // when the cache is invalid. - next int - entries []*includeCacheEntry -} - -// Next Return the next cache entry. Should only be called when the cache is -// valid and a next entry is available (the latter can be checked with -// ExpectFile). Does not advance the cache. -func (cache *includeCache) Next() *includeCacheEntry { - return cache.entries[cache.next] -} - -// ExpectFile check that the next cache entry is about the given file. If it is -// not, or no entry is available, the cache is invalidated. Does not -// advance the cache. -func (cache *includeCache) ExpectFile(sourcefile *paths.Path) { - if cache.valid && (cache.next >= len(cache.entries) || !cache.Next().Sourcefile.EqualsTo(sourcefile)) { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } -} - -// ExpectEntry check that the next entry matches the given values. If so, advance -// the cache. If not, the cache is invalidated. If the cache is -// invalidated, or was already invalid, an entry with the given values -// is appended. -func (cache *includeCache) ExpectEntry(sourcefile *paths.Path, include string, librarypath *paths.Path) { - entry := &includeCacheEntry{Sourcefile: sourcefile, Include: include, Includepath: librarypath} - if cache.valid { - if cache.next < len(cache.entries) && cache.Next().Equals(entry) { - cache.next++ - } else { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } - } - - if !cache.valid { - cache.entries = append(cache.entries, entry) - } -} - -// ExpectEnd check that the cache is completely consumed. If not, the cache is -// invalidated. -func (cache *includeCache) ExpectEnd() { - if cache.valid && cache.next < len(cache.entries) { - cache.valid = false - cache.entries = cache.entries[:cache.next] - } -} - -// Read the cache from the given file -func readCache(path *paths.Path) *includeCache { - bytes, err := path.ReadFile() - if err != nil { - // Return an empty, invalid cache - return &includeCache{} - } - result := &includeCache{} - err = json.Unmarshal(bytes, &result.entries) - if err != nil { - // Return an empty, invalid cache - return &includeCache{} - } - result.valid = true - return result -} - -// Write the given cache to the given file if it is invalidated. If the -// cache is still valid, just update the timestamps of the file. -func writeCache(cache *includeCache, path *paths.Path) error { - // If the cache was still valid all the way, just touch its file - // (in case any source file changed without influencing the - // includes). If it was invalidated, overwrite the cache with - // the new contents. - if cache.valid { - path.Chtimes(time.Now(), time.Now()) - } else { - bytes, err := json.MarshalIndent(cache.entries, "", " ") - if err != nil { - return err - } - err = path.WriteFile(bytes) - if err != nil { - return err - } - } - return nil -} - -type uniqueSourceFileQueue []*sourceFile - -func (queue *uniqueSourceFileQueue) push(value *sourceFile) { - if !queue.contains(value) { - *queue = append(*queue, value) - } -} - -func (queue uniqueSourceFileQueue) contains(target *sourceFile) bool { - return slices.ContainsFunc(queue, target.Equals) -} - -func (queue *uniqueSourceFileQueue) pop() *sourceFile { - old := *queue - x := old[0] - *queue = old[1:] - return x -} - -func (queue uniqueSourceFileQueue) empty() bool { - return len(queue) == 0 + resolver := librariesresolver.NewCppResolver(allLibs, targetPlatform, buildPlatform) + return resolver, verboseOut.Bytes(), nil } diff --git a/internal/arduino/builder/internal/detector/source_file.go b/internal/arduino/builder/internal/detector/source_file.go new file mode 100644 index 00000000000..d99cb1d862a --- /dev/null +++ b/internal/arduino/builder/internal/detector/source_file.go @@ -0,0 +1,115 @@ +// This file is part of arduino-cli. +// +// Copyright 2024 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package detector + +import ( + "fmt" + "slices" + + "github.com/arduino/arduino-cli/internal/arduino/builder/internal/utils" + "github.com/arduino/go-paths-helper" +) + +type sourceFile struct { + // SourcePath is the path to the source file + SourcePath *paths.Path `json:"source_path"` + + // ObjectPath is the path to the object file that will be generated + ObjectPath *paths.Path `json:"object_path"` + + // DepfilePath is the path to the dependency file that will be generated + DepfilePath *paths.Path `json:"depfile_path"` + + // ExtraIncludePath contains an extra include path that must be + // used to compile this source file. + // This is mainly used for source files that comes from old-style libraries + // (Arduino IDE <1.5) requiring an extra include path to the "utility" folder. + ExtraIncludePath *paths.Path `json:"extra_include_path,omitempty"` +} + +func (f *sourceFile) String() string { + return fmt.Sprintf("SourcePath:%s SourceRoot:%s BuildRoot:%s ExtraInclude:%s", + f.SourcePath, f.ObjectPath, f.DepfilePath, f.ExtraIncludePath) +} + +// Equals checks if a sourceFile is equal to another. +func (f *sourceFile) Equals(g *sourceFile) bool { + return f.SourcePath.EqualsTo(g.SourcePath) && + f.ObjectPath.EqualsTo(g.ObjectPath) && + f.DepfilePath.EqualsTo(g.DepfilePath) && + ((f.ExtraIncludePath == nil && g.ExtraIncludePath == nil) || + (f.ExtraIncludePath != nil && g.ExtraIncludePath != nil && f.ExtraIncludePath.EqualsTo(g.ExtraIncludePath))) +} + +// makeSourceFile create a sourceFile object for the given source file path. +// The given sourceFilePath can be absolute, or relative within the sourceRoot root folder. +func makeSourceFile(sourceRoot, buildRoot, sourceFilePath *paths.Path, extraIncludePaths ...*paths.Path) (*sourceFile, error) { + if len(extraIncludePaths) > 1 { + panic("only one extra include path allowed") + } + var extraIncludePath *paths.Path + if len(extraIncludePaths) > 0 { + extraIncludePath = extraIncludePaths[0] + } + + if sourceFilePath.IsAbs() { + var err error + sourceFilePath, err = sourceRoot.RelTo(sourceFilePath) + if err != nil { + return nil, err + } + } + res := &sourceFile{ + SourcePath: sourceRoot.JoinPath(sourceFilePath), + ObjectPath: buildRoot.Join(sourceFilePath.String() + ".o"), + DepfilePath: buildRoot.Join(sourceFilePath.String() + ".d"), + ExtraIncludePath: extraIncludePath, + } + return res, nil +} + +// ObjFileIsUpToDate checks if the compile object file is up to date. +func (f *sourceFile) ObjFileIsUpToDate() (unchanged bool, err error) { + return utils.ObjFileIsUpToDate(f.SourcePath, f.ObjectPath, f.DepfilePath) +} + +// uniqueSourceFileQueue is a queue of source files that does not allow duplicates. +type uniqueSourceFileQueue []*sourceFile + +// Push adds a source file to the queue if it is not already present. +func (queue *uniqueSourceFileQueue) Push(value *sourceFile) { + if !queue.Contains(value) { + *queue = append(*queue, value) + } +} + +// Contains checks if the queue Contains a source file. +func (queue uniqueSourceFileQueue) Contains(target *sourceFile) bool { + return slices.ContainsFunc(queue, target.Equals) +} + +// Pop removes and returns the first element of the queue. +func (queue *uniqueSourceFileQueue) Pop() *sourceFile { + old := *queue + x := old[0] + *queue = old[1:] + return x +} + +// Empty returns true if the queue is Empty. +func (queue uniqueSourceFileQueue) Empty() bool { + return len(queue) == 0 +} diff --git a/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go b/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go index deabc04c4f0..dd847c26eda 100644 --- a/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go +++ b/internal/arduino/builder/internal/preprocessor/arduino_preprocessor.go @@ -22,6 +22,7 @@ import ( "path/filepath" "runtime" + "github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner" "github.com/arduino/arduino-cli/internal/arduino/builder/internal/utils" "github.com/arduino/arduino-cli/internal/arduino/sketch" "github.com/arduino/arduino-cli/internal/i18n" @@ -35,7 +36,7 @@ func PreprocessSketchWithArduinoPreprocessor( ctx context.Context, sk *sketch.Sketch, buildPath *paths.Path, includeFolders paths.PathList, lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase bool, -) (*Result, error) { +) (*runner.Result, error) { verboseOut := &bytes.Buffer{} normalOut := &bytes.Buffer{} if err := buildPath.Join("preproc").MkdirAll(); err != nil { @@ -44,52 +45,48 @@ func PreprocessSketchWithArduinoPreprocessor( sourceFile := buildPath.Join("sketch", sk.MainFile.Base()+".cpp") targetFile := buildPath.Join("preproc", "sketch_merged.cpp") - gccResult, err := GCC(ctx, sourceFile, targetFile, includeFolders, buildProperties) - verboseOut.Write(gccResult.Stdout()) - verboseOut.Write(gccResult.Stderr()) - if err != nil { - return nil, err + gccResult := GCC(sourceFile, targetFile, includeFolders, buildProperties).Run(ctx) + verboseOut.Write(gccResult.Stdout) + verboseOut.Write(gccResult.Stderr) + if gccResult.Error != nil { + return nil, gccResult.Error } - arduiniPreprocessorProperties := properties.NewMap() - arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}") - arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor") - arduiniPreprocessorProperties.Set("tools.arduino-preprocessor.pattern", `"{cmd.path}" "{source_file}" -- -std=gnu++11`) - arduiniPreprocessorProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") - arduiniPreprocessorProperties.Merge(buildProperties) - arduiniPreprocessorProperties.Merge(arduiniPreprocessorProperties.SubTree("tools").SubTree("arduino-preprocessor")) - arduiniPreprocessorProperties.SetPath("source_file", targetFile) - pattern := arduiniPreprocessorProperties.Get("pattern") + arduinoPreprocessorProperties := properties.NewMap() + arduinoPreprocessorProperties.Set("tools.arduino-preprocessor.path", "{runtime.tools.arduino-preprocessor.path}") + arduinoPreprocessorProperties.Set("tools.arduino-preprocessor.cmd.path", "{path}/arduino-preprocessor") + arduinoPreprocessorProperties.Set("tools.arduino-preprocessor.pattern", `"{cmd.path}" "{source_file}" -- -std=gnu++11`) + arduinoPreprocessorProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") + arduinoPreprocessorProperties.Merge(buildProperties) + arduinoPreprocessorProperties.Merge(arduinoPreprocessorProperties.SubTree("tools").SubTree("arduino-preprocessor")) + arduinoPreprocessorProperties.SetPath("source_file", targetFile) + pattern := arduinoPreprocessorProperties.Get("pattern") if pattern == "" { return nil, errors.New(i18n.Tr("arduino-preprocessor pattern is missing")) } - commandLine := arduiniPreprocessorProperties.ExpandPropsInString(pattern) - parts, err := properties.SplitQuotedString(commandLine, `"'`, false) - if err != nil { - return nil, err - } - - command, err := paths.NewProcess(nil, parts...) + commandLine := arduinoPreprocessorProperties.ExpandPropsInString(pattern) + args, _ := properties.SplitQuotedString(commandLine, `"'`, false) + command, err := paths.NewProcess(nil, args...) if err != nil { return nil, err } if runtime.GOOS == "windows" { // chdir in the uppermost directory to avoid UTF-8 bug in clang (https://github.com/arduino/arduino-preprocessor/issues/2) - command.SetDir(filepath.VolumeName(parts[0]) + "/") + command.SetDir(filepath.VolumeName(args[0]) + "/") } verboseOut.WriteString(commandLine) commandStdOut, commandStdErr, err := command.RunAndCaptureOutput(ctx) verboseOut.Write(commandStdErr) if err != nil { - return &Result{args: gccResult.Args(), stdout: verboseOut.Bytes(), stderr: normalOut.Bytes()}, err + return &runner.Result{Args: gccResult.Args, Stdout: verboseOut.Bytes(), Stderr: normalOut.Bytes()}, err } result := utils.NormalizeUTF8(commandStdOut) destFile := buildPath.Join(sk.MainFile.Base() + ".cpp") if err := destFile.WriteFile(result); err != nil { - return &Result{args: gccResult.Args(), stdout: verboseOut.Bytes(), stderr: normalOut.Bytes()}, err + return &runner.Result{Args: gccResult.Args, Stdout: verboseOut.Bytes(), Stderr: normalOut.Bytes()}, err } - return &Result{args: gccResult.Args(), stdout: verboseOut.Bytes(), stderr: normalOut.Bytes()}, err + return &runner.Result{Args: gccResult.Args, Stdout: verboseOut.Bytes(), Stderr: normalOut.Bytes()}, err } diff --git a/internal/arduino/builder/internal/preprocessor/ctags.go b/internal/arduino/builder/internal/preprocessor/ctags.go index fe36cfc89e5..6393cd86e3d 100644 --- a/internal/arduino/builder/internal/preprocessor/ctags.go +++ b/internal/arduino/builder/internal/preprocessor/ctags.go @@ -27,6 +27,7 @@ import ( "github.com/arduino/arduino-cli/internal/arduino/builder/cpp" "github.com/arduino/arduino-cli/internal/arduino/builder/internal/preprocessor/internal/ctags" + "github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner" "github.com/arduino/arduino-cli/internal/arduino/sketch" "github.com/arduino/arduino-cli/internal/i18n" "github.com/arduino/go-paths-helper" @@ -43,7 +44,7 @@ func PreprocessSketchWithCtags( sketch *sketch.Sketch, buildPath *paths.Path, includes paths.PathList, lineOffset int, buildProperties *properties.Map, onlyUpdateCompilationDatabase, verbose bool, -) (*Result, error) { +) (*runner.Result, error) { // Create a temporary working directory tmpDir, err := paths.MkTempDir("", "") if err != nil { @@ -56,12 +57,12 @@ func PreprocessSketchWithCtags( // Run GCC preprocessor sourceFile := buildPath.Join("sketch", sketch.MainFile.Base()+".cpp") - result, err := GCC(ctx, sourceFile, ctagsTarget, includes, buildProperties) - stdout.Write(result.Stdout()) - stderr.Write(result.Stderr()) - if err != nil { + result := GCC(sourceFile, ctagsTarget, includes, buildProperties).Run(ctx) + stdout.Write(result.Stdout) + stderr.Write(result.Stderr) + if err := result.Error; err != nil { if !onlyUpdateCompilationDatabase { - return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err + return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err } // Do not bail out if we are generating the compile commands database @@ -69,17 +70,17 @@ func PreprocessSketchWithCtags( i18n.Tr("An error occurred adding prototypes"), i18n.Tr("the compilation database may be incomplete or inaccurate"))) if err := sourceFile.CopyTo(ctagsTarget); err != nil { - return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err + return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err } } if src, err := ctagsTarget.ReadFile(); err == nil { filteredSource := filterSketchSource(sketch, bytes.NewReader(src), false) if err := ctagsTarget.WriteFile([]byte(filteredSource)); err != nil { - return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err + return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err } } else { - return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err + return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err } // Run CTags on gcc-preprocessed source @@ -88,7 +89,7 @@ func PreprocessSketchWithCtags( stderr.Write(ctagsStdErr) } if err != nil { - return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err + return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err } // Parse CTags output @@ -103,13 +104,13 @@ func PreprocessSketchWithCtags( if sourceData, err := sourceFile.ReadFile(); err == nil { source = string(sourceData) } else { - return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err + return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err } source = strings.ReplaceAll(source, "\r\n", "\n") source = strings.ReplaceAll(source, "\r", "\n") sourceRows := strings.Split(source, "\n") if isFirstFunctionOutsideOfSource(firstFunctionLine, sourceRows) { - return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, nil + return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, nil } insertionLine := firstFunctionLine + lineOffset - 1 @@ -135,7 +136,7 @@ func PreprocessSketchWithCtags( // Write back arduino-preprocess output to the sourceFile err = sourceFile.WriteFile([]byte(preprocessedSource)) - return &Result{args: result.Args(), stdout: stdout.Bytes(), stderr: stderr.Bytes()}, err + return &runner.Result{Args: result.Args, Stdout: stdout.Bytes(), Stderr: stderr.Bytes()}, err } func composePrototypeSection(line int, prototypes []*ctags.Prototype) string { @@ -193,19 +194,15 @@ func RunCTags(ctx context.Context, sourceFile *paths.Path, buildProperties *prop } commandLine := ctagsBuildProperties.ExpandPropsInString(pattern) - parts, err := properties.SplitQuotedString(commandLine, `"'`, false) - if err != nil { - return nil, nil, err - } - proc, err := paths.NewProcess(nil, parts...) + args, _ := properties.SplitQuotedString(commandLine, `"'`, false) + proc, err := paths.NewProcess(nil, args...) if err != nil { return nil, nil, err } stdout, stderr, err := proc.RunAndCaptureOutput(ctx) // Append ctags arguments to stderr - args := fmt.Sprintln(strings.Join(parts, " ")) - stderr = append([]byte(args), stderr...) + stderr = append([]byte(fmt.Sprintln(strings.Join(args, " "))), stderr...) return stdout, stderr, err } diff --git a/internal/arduino/builder/internal/preprocessor/gcc.go b/internal/arduino/builder/internal/preprocessor/gcc.go index bfcc9512e0c..59500c3522d 100644 --- a/internal/arduino/builder/internal/preprocessor/gcc.go +++ b/internal/arduino/builder/internal/preprocessor/gcc.go @@ -16,14 +16,11 @@ package preprocessor import ( - "context" - "errors" - "fmt" "strings" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/internal/arduino/builder/cpp" - "github.com/arduino/arduino-cli/internal/i18n" + "github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" ) @@ -31,10 +28,9 @@ import ( // GCC performs a run of the gcc preprocess (macro/includes expansion). The function outputs the result // to targetFilePath. Returns the stdout/stderr of gcc if any. func GCC( - ctx context.Context, sourceFilePath, targetFilePath *paths.Path, includes paths.PathList, buildProperties *properties.Map, -) (Result, error) { +) *runner.Task { gccBuildProperties := properties.NewMap() gccBuildProperties.Set("preproc.macros.flags", "-w -x c++ -E -CC") gccBuildProperties.Merge(buildProperties) @@ -58,29 +54,12 @@ func GCC( } pattern := gccBuildProperties.Get(gccPreprocRecipeProperty) - if pattern == "" { - return Result{}, errors.New(i18n.Tr("%s pattern is missing", gccPreprocRecipeProperty)) - } - commandLine := gccBuildProperties.ExpandPropsInString(pattern) commandLine = properties.DeleteUnexpandedPropsFromString(commandLine) - args, err := properties.SplitQuotedString(commandLine, `"'`, false) - if err != nil { - return Result{}, err - } + args, _ := properties.SplitQuotedString(commandLine, `"'`, false) // Remove -MMD argument if present. Leaving it will make gcc try // to create a /dev/null.d dependency file, which won't work. args = f.Filter(args, f.NotEquals("-MMD")) - - proc, err := paths.NewProcess(nil, args...) - if err != nil { - return Result{}, err - } - stdout, stderr, err := proc.RunAndCaptureOutput(ctx) - - // Append gcc arguments to stdout - stdout = append([]byte(fmt.Sprintln(strings.Join(args, " "))), stdout...) - - return Result{args: proc.GetArgs(), stdout: stdout, stderr: stderr}, err + return runner.NewTask(args...) } diff --git a/internal/arduino/builder/internal/preprocessor/result.go b/internal/arduino/builder/internal/preprocessor/result.go deleted file mode 100644 index 3fa45e40974..00000000000 --- a/internal/arduino/builder/internal/preprocessor/result.go +++ /dev/null @@ -1,34 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2024 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package preprocessor - -type Result struct { - args []string - stdout []byte - stderr []byte -} - -func (r Result) Args() []string { - return r.args -} - -func (r Result) Stdout() []byte { - return r.stdout -} - -func (r Result) Stderr() []byte { - return r.stderr -} diff --git a/internal/arduino/builder/internal/runner/runner.go b/internal/arduino/builder/internal/runner/runner.go new file mode 100644 index 00000000000..89565bdbc4c --- /dev/null +++ b/internal/arduino/builder/internal/runner/runner.go @@ -0,0 +1,117 @@ +// This file is part of arduino-cli. +// +// Copyright 2024 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package runner + +import ( + "context" + "runtime" + "sync" +) + +// Runner is a helper to run commands in a queue, the commands are immediately exectuded +// in a goroutine as they are enqueued. The runner can be stopped by calling Cancel. +type Runner struct { + lock sync.Mutex + queue chan<- *enqueuedCommand + results map[string]<-chan *Result + ctx context.Context + ctxCancel func() + wg sync.WaitGroup +} + +type enqueuedCommand struct { + task *Task + accept func(*Result) +} + +func (cmd *enqueuedCommand) String() string { + return cmd.task.String() +} + +// New creates a new Runner with the given number of workers. +// If workers is 0, the number of workers will be the number of available CPUs. +func New(inCtx context.Context, workers int) *Runner { + ctx, cancel := context.WithCancel(inCtx) + queue := make(chan *enqueuedCommand, 1000) + r := &Runner{ + ctx: ctx, + ctxCancel: cancel, + queue: queue, + results: map[string]<-chan *Result{}, + } + + // Spawn workers + if workers == 0 { + workers = runtime.NumCPU() + } + for i := 0; i < workers; i++ { + r.wg.Add(1) + go func() { + worker(ctx, queue) + r.wg.Done() + }() + } + + return r +} + +func worker(ctx context.Context, queue <-chan *enqueuedCommand) { + done := ctx.Done() + for { + select { + case <-done: + return + default: + } + + select { + case <-done: + return + case cmd := <-queue: + result := cmd.task.Run(ctx) + cmd.accept(result) + } + } +} + +func (r *Runner) Enqueue(task *Task) { + r.lock.Lock() + defer r.lock.Unlock() + + result := make(chan *Result, 1) + r.results[task.String()] = result + r.queue <- &enqueuedCommand{ + task: task, + accept: func(res *Result) { + result <- res + }, + } +} + +func (r *Runner) Results(task *Task) *Result { + r.lock.Lock() + result, ok := r.results[task.String()] + r.lock.Unlock() + if !ok { + return nil + } + return <-result +} + +func (r *Runner) Cancel() { + r.ctxCancel() + r.wg.Wait() +} diff --git a/internal/arduino/builder/internal/runner/runner_test.go b/internal/arduino/builder/internal/runner/runner_test.go new file mode 100644 index 00000000000..ed499299912 --- /dev/null +++ b/internal/arduino/builder/internal/runner/runner_test.go @@ -0,0 +1,53 @@ +// This file is part of arduino-cli. +// +// Copyright 2024 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package runner_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner" + "github.com/stretchr/testify/require" +) + +func TestRunMultipleTask(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + r := runner.New(ctx, 0) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 1 ; echo -n 0")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 2 ; echo -n 1")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 3 ; echo -n 2")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 4 ; echo -n 3")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 5 ; echo -n 4")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 6 ; echo -n 5")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 7 ; echo -n 6")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 8 ; echo -n 7")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 9 ; echo -n 8")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 10 ; echo -n 9")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 11 ; echo -n 10")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 12 ; echo -n 11")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 13 ; echo -n 12")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 14 ; echo -n 13")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 15 ; echo -n 14")) + r.Enqueue(runner.NewTask("bash", "-c", "sleep 16 ; echo -n 15")) + require.Nil(t, r.Results(runner.NewTask("bash", "-c", "echo -n 5"))) + fmt.Println(string(r.Results(runner.NewTask("bash", "-c", "sleep 3 ; echo -n 2")).Stdout)) + fmt.Println("Cancelling") + r.Cancel() + fmt.Println("Runner completed") +} diff --git a/internal/arduino/builder/internal/runner/task.go b/internal/arduino/builder/internal/runner/task.go new file mode 100644 index 00000000000..dd44b2fa332 --- /dev/null +++ b/internal/arduino/builder/internal/runner/task.go @@ -0,0 +1,60 @@ +// This file is part of arduino-cli. +// +// Copyright 2024 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package runner + +import ( + "context" + "fmt" + "strings" + + "github.com/arduino/go-paths-helper" +) + +// Task is a command to be executed +type Task struct { + Args []string `json:"args"` +} + +// NewTask creates a new Task +func NewTask(args ...string) *Task { + return &Task{Args: args} +} + +func (t *Task) String() string { + return strings.Join(t.Args, " ") +} + +// Result contains the output of a command execution +type Result struct { + Args []string + Stdout []byte + Stderr []byte + Error error +} + +// Run executes the command and returns the result +func (t *Task) Run(ctx context.Context) *Result { + proc, err := paths.NewProcess(nil, t.Args...) + if err != nil { + return &Result{Args: t.Args, Error: err} + } + stdout, stderr, err := proc.RunAndCaptureOutput(ctx) + + // Append arguments to stdout + stdout = append([]byte(fmt.Sprintln(t)), stdout...) + + return &Result{Args: proc.GetArgs(), Stdout: stdout, Stderr: stderr, Error: err} +} diff --git a/internal/arduino/builder/preprocess_sketch.go b/internal/arduino/builder/preprocess_sketch.go index 86d7bd7e7e9..b4508304aa3 100644 --- a/internal/arduino/builder/preprocess_sketch.go +++ b/internal/arduino/builder/preprocess_sketch.go @@ -30,10 +30,10 @@ func (b *Builder) preprocessSketch(includes paths.PathList) error { ) if result != nil { if b.logger.Verbose() { - b.logger.WriteStdout(result.Stdout()) + b.logger.WriteStdout(result.Stdout) } - b.logger.WriteStdout(result.Stderr()) - b.diagnosticStore.Parse(result.Args(), result.Stderr()) + b.logger.WriteStdout(result.Stderr) + b.diagnosticStore.Parse(result.Args, result.Stderr) } return err diff --git a/internal/arduino/builder/sizer.go b/internal/arduino/builder/sizer.go index 84cf8012a32..87ec1f24870 100644 --- a/internal/arduino/builder/sizer.go +++ b/internal/arduino/builder/sizer.go @@ -196,15 +196,17 @@ func (b *Builder) checkSize() (ExecutablesFileSections, error) { return executableSectionsSize, errors.New(i18n.Tr("data section exceeds available space in board")) } + warnDataPercentage := 75 if w := properties.Get("build.warn_data_percentage"); w != "" { - warnDataPercentage, err := strconv.Atoi(w) - if err != nil { - return executableSectionsSize, err - } - if maxDataSize > 0 && dataSize > maxDataSize*warnDataPercentage/100 { - b.logger.Warn(i18n.Tr("Low memory available, stability problems may occur.")) + if p, err := strconv.Atoi(w); err == nil { + warnDataPercentage = p + } else { + b.logger.Warn(i18n.Tr("Invalid value for build.warn_data_percentage: %s", w)) } } + if maxDataSize > 0 && dataSize > maxDataSize*warnDataPercentage/100 { + b.logger.Warn(i18n.Tr("Low memory available, stability problems may occur.")) + } return executableSectionsSize, nil } diff --git a/internal/arduino/cores/packagemanager/loader.go b/internal/arduino/cores/packagemanager/loader.go index 14e1d6df912..e6067cfb125 100644 --- a/internal/arduino/cores/packagemanager/loader.go +++ b/internal/arduino/cores/packagemanager/loader.go @@ -721,11 +721,8 @@ func (pme *Explorer) loadDiscoveries(release *cores.PlatformRelease) []error { } cmd := configuration.ExpandPropsInString(pattern) - if cmdArgs, err := properties.SplitQuotedString(cmd, `"'`, true); err != nil { - merr = append(merr, err) - } else { - pme.discoveryManager.Add(discoveryID, cmdArgs...) - } + cmdArgs, _ := properties.SplitQuotedString(cmd, `"'`, true) + pme.discoveryManager.Add(discoveryID, cmdArgs...) } return merr diff --git a/internal/integrationtest/compile_4/compile_test.go b/internal/integrationtest/compile_4/compile_test.go index a909cab3ab8..5ae8b6b10a4 100644 --- a/internal/integrationtest/compile_4/compile_test.go +++ b/internal/integrationtest/compile_4/compile_test.go @@ -1038,15 +1038,14 @@ func TestBuildOptionsFile(t *testing.T) { "sketchLocation" ]`) requirejson.Query(t, buildOptionsBytes, ".fqbn", `"arduino:avr:uno"`) - requirejson.Query(t, buildOptionsBytes, ".customBuildProperties", `"build.warn_data_percentage=75"`) // Recompiling a second time should provide the same result _, _, err = cli.Run("compile", "-b", "arduino:avr:uno", "--build-path", buildPath.String(), sketchPath.String()) require.NoError(t, err) - buildOptionsBytes, err = buildPath.Join("build.options.json").ReadFile() + buildOptionsBytes2, err := buildPath.Join("build.options.json").ReadFile() require.NoError(t, err) - requirejson.Query(t, buildOptionsBytes, ".customBuildProperties", `"build.warn_data_percentage=75"`) + require.Equal(t, buildOptionsBytes, buildOptionsBytes2) // Recompiling with a new build option must produce a new `build.options.json` _, _, err = cli.Run("compile", "-b", "arduino:avr:uno", "--build-path", buildPath.String(), @@ -1055,7 +1054,7 @@ func TestBuildOptionsFile(t *testing.T) { ) require.NoError(t, err) - buildOptionsBytes, err = buildPath.Join("build.options.json").ReadFile() + buildOptionsBytes3, err := buildPath.Join("build.options.json").ReadFile() require.NoError(t, err) - requirejson.Query(t, buildOptionsBytes, ".customBuildProperties", `"custom=prop,build.warn_data_percentage=75"`) + require.NotEqual(t, buildOptionsBytes, buildOptionsBytes3) } diff --git a/internal/integrationtest/compile_4/lib_discovery_caching_test.go b/internal/integrationtest/compile_4/lib_discovery_caching_test.go new file mode 100644 index 00000000000..9d8bd4b5277 --- /dev/null +++ b/internal/integrationtest/compile_4/lib_discovery_caching_test.go @@ -0,0 +1,110 @@ +// This file is part of arduino-cli. +// +// Copyright 2023 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package compile_test + +import ( + "testing" + + "github.com/arduino/arduino-cli/internal/integrationtest" + "github.com/arduino/go-paths-helper" + "github.com/stretchr/testify/require" + "go.bug.st/testifyjson/requirejson" +) + +func TestLibDiscoveryCache(t *testing.T) { + env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t) + t.Cleanup(env.CleanUp) + + // Install Arduino AVR Boards + _, _, err := cli.Run("core", "install", "arduino:avr@1.8.6") + require.NoError(t, err) + + // Copy the testdata sketchbook + testdata, err := paths.New("testdata", "libraries_discovery_caching").Abs() + require.NoError(t, err) + sketchbook := cli.SketchbookDir() + require.NoError(t, sketchbook.RemoveAll()) + require.NoError(t, testdata.CopyDirTo(cli.SketchbookDir())) + + buildpath, err := paths.MkTempDir("", "tmpbuildpath") + require.NoError(t, err) + t.Cleanup(func() { buildpath.RemoveAll() }) + + { + sketchA := sketchbook.Join("SketchA") + { + outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String()) + require.NoError(t, err) + j := requirejson.Parse(t, outjson) + j.MustContain(`{"builder_result":{ + "used_libraries": [ + { "name": "LibA" }, + { "name": "LibB" } + ], + }}`) + } + + // Update SketchA + require.NoError(t, sketchA.Join("SketchA.ino").WriteFile([]byte(` +#include +#include +void setup() {} +void loop() {libAFunction();} +`))) + + { + // This compile should FAIL! + outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String()) + require.Error(t, err) + j := requirejson.Parse(t, outjson) + j.MustContain(`{ +"builder_result":{ + "used_libraries": [ + { "name": "LibC" }, + { "name": "LibA" } + ], + "diagnostics": [ + { + "severity": "ERROR", + "message": "'libAFunction' was not declared in this scope\n void loop() {libAFunction();}\n ^~~~~~~~~~~~" + } + ] +}}`) + j.Query(".compiler_out").MustContain(`"The list of included libraries has been changed... rebuilding all libraries."`) + } + + { + // This compile should FAIL! + outjson, _, err := cli.Run("compile", "-v", "-b", "arduino:avr:uno", "--build-path", buildpath.String(), "--json", sketchA.String()) + require.Error(t, err) + j := requirejson.Parse(t, outjson) + j.MustContain(`{ +"builder_result":{ + "used_libraries": [ + { "name": "LibC" }, + { "name": "LibA" } + ], + "diagnostics": [ + { + "severity": "ERROR", + "message": "'libAFunction' was not declared in this scope\n void loop() {libAFunction();}\n ^~~~~~~~~~~~" + } + ] +}}`) + j.Query(".compiler_out").MustNotContain(`"The list of included libraries has changed... rebuilding all libraries."`) + } + } +} diff --git a/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/SketchA/SketchA.ino b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/SketchA/SketchA.ino new file mode 100644 index 00000000000..b95755d1f82 --- /dev/null +++ b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/SketchA/SketchA.ino @@ -0,0 +1,3 @@ +#include +void setup() {} +void loop() {libAFunction();} diff --git a/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/LibA.h b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/LibA.h new file mode 100644 index 00000000000..5cc5459e363 --- /dev/null +++ b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/LibA.h @@ -0,0 +1,6 @@ + +#include + +#ifndef CHECK +void libAFunction(); +#endif diff --git a/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/file1.cpp b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/file1.cpp new file mode 100644 index 00000000000..08474b5de80 --- /dev/null +++ b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibA/file1.cpp @@ -0,0 +1,5 @@ +#include + +#ifndef CHECK +void libAFunction() {} +#endif diff --git a/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibB/LibB.h b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibB/LibB.h new file mode 100644 index 00000000000..e69de29bb2d diff --git a/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibB.h b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibB.h new file mode 100644 index 00000000000..4dbb6c43193 --- /dev/null +++ b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibB.h @@ -0,0 +1,2 @@ + +#define CHECK diff --git a/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibC.h b/internal/integrationtest/compile_4/testdata/libraries_discovery_caching/libraries/LibC/LibC.h new file mode 100644 index 00000000000..e69de29bb2d