Added depsusage plugin using channels for package & UsageEvidence acc…

…umulation

Signed-off-by: Omkar Phansopkar <omkarphansopkar@gmail.com>

go.mod

@@ -178,6 +178,7 @@ require (
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/safedep/code v0.0.0-20250121051057-a3616a726431 // indirect
 	github.com/sagikazarmark/locafero v0.7.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect

go.sum

@@ -866,6 +866,8 @@ github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/safedep/code v0.0.0-20250121051057-a3616a726431 h1:xGyszHYtLFWewwQYiEfKTXC5KrBAIn8EM6qUWnX6BW0=
+github.com/safedep/code v0.0.0-20250121051057-a3616a726431/go.mod h1:oZJ1skQ0nAnqneDMbSN08IM1tl8DKRGD57fEaSXpyuQ=
 github.com/safedep/dry v0.0.0-20241128083908-2f8ecd48dc2c h1:qUSfzPPlEcLKF6cKvkkXU6ddu4NGSz0UdS7Xjcrenvw=
 github.com/safedep/dry v0.0.0-20241128083908-2f8ecd48dc2c/go.mod h1:dtGFDAnRo+WqwEyqPc2hTwuVGwWLq2jHnP4Q8BO1u7g=
 github.com/safedep/dry v0.0.0-20250106055453-e0772cda4a25 h1:vkW9YyId5WHPnnGhnrmucKL53xTNUE8mBLBdmTBOGBc=

pkg/models/models.go

@@ -11,6 +11,7 @@ import (
 	malysisv1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/malysis/v1"
 	packagev1 "buf.build/gen/go/safedep/api/protocolbuffers/go/safedep/messages/package/v1"
 	"github.com/google/osv-scanner/pkg/lockfile"
+	"github.com/safedep/code/plugin/depsusage"
 	"github.com/safedep/vet/gen/insightapi"
 
 	modelspec "github.com/safedep/vet/gen/models"
@@ -361,6 +362,9 @@ type Package struct {
 
 	// Manifest from where this package was found directly or indirectly
 	Manifest *PackageManifest `json:"-"`
+
+	// Usage evidences for this package obtained by depsusage plugin in code analysis
+	UsageEvidences []*depsusage.UsageEvidence `json:"usage_evidences"`
 }
 
 // Id returns a unique identifier for this package within a manifest

pkg/scanner/callbacks.go

@@ -1,26 +1,37 @@
 package scanner
 
-import "github.com/safedep/vet/pkg/models"
+import (
+	"github.com/safedep/code/core"
+	"github.com/safedep/vet/pkg/models"
+)
 
 type ScannerCallbackOnManifestFn func(manifest *models.PackageManifest)
 
 type ScannerCallbackOnPackageFn func(pkg *models.Package)
 
+type CodeanalysisCallbackOnFileAnalysed func(file core.File)
+
+type CodeanalysisCallbackOnFileSkipped func(file core.File)
+
 type ScannerCallbackErrArgFn func(error)
 
 type ScannerCallbackNoArgFn func()
 
 type ScannerCallbacks struct {
-	OnStartEnumerateManifest ScannerCallbackNoArgFn      // Manifest enumeration is starting
-	OnEnumerateManifest      ScannerCallbackOnManifestFn // A manifest is read by reader
-	OnStart                  ScannerCallbackNoArgFn      // Manifest scan phase is starting
-	OnStartManifest          ScannerCallbackOnManifestFn // A manifest is starting to be scanned
-	OnStartPackage           ScannerCallbackOnPackageFn  // A package analysis is starting
-	OnAddTransitivePackage   ScannerCallbackOnPackageFn  // A transitive dependency is discovered
-	OnDonePackage            ScannerCallbackOnPackageFn  // A package analysis is finished
-	OnDoneManifest           ScannerCallbackOnManifestFn // A manifest analysis is finished
-	BeforeFinish             ScannerCallbackNoArgFn      // Scan is about to finish
-	OnStop                   ScannerCallbackErrArgFn     // Scan is finished
+	OnStartEnumerateManifest ScannerCallbackNoArgFn             // Manifest enumeration is starting
+	OnEnumerateManifest      ScannerCallbackOnManifestFn        // A manifest is read by reader
+	OnStart                  ScannerCallbackNoArgFn             // Manifest scan phase is starting
+	OnStartManifest          ScannerCallbackOnManifestFn        // A manifest is starting to be scanned
+	OnStartPackage           ScannerCallbackOnPackageFn         // A package analysis is starting
+	OnAddTransitivePackage   ScannerCallbackOnPackageFn         // A transitive dependency is discovered
+	OnDonePackage            ScannerCallbackOnPackageFn         // A package analysis is finished
+	OnDoneManifest           ScannerCallbackOnManifestFn        // A manifest analysis is finished
+	OnStartCodeAnalysis      ScannerCallbackNoArgFn             // Code analysis is starting
+	OnFileAnalysed           CodeanalysisCallbackOnFileAnalysed // A file is analysed in code analysis
+	OnFileSkipped            CodeanalysisCallbackOnFileSkipped  // A file is skipped in code analysis
+	OnDoneCodeAnalysis       ScannerCallbackNoArgFn             // Code analysis is finished
+	BeforeFinish             ScannerCallbackNoArgFn             // Scan is about to finish
+	OnStop                   ScannerCallbackErrArgFn            // Scan is finished
 }
 
 func (s *packageManifestScanner) WithCallbacks(callbacks ScannerCallbacks) {
@@ -75,6 +86,30 @@ func (s *packageManifestScanner) dispatchOnDoneManifest(manifest *models.Package
 	}
 }
 
+func (s *packageManifestScanner) dispatchOnStartCodeAnalysis() {
+	if s.callbacks.OnStartCodeAnalysis != nil {
+		s.callbacks.OnStartCodeAnalysis()
+	}
+}
+
+// func (s *packageManifestScanner) dispatchOnFileAnalysed(file core.File) {
+// 	if s.callbacks.OnFileAnalysed != nil {
+// 		s.callbacks.OnFileAnalysed(file)
+// 	}
+// }
+
+// func (s *packageManifestScanner) dispatchOnFileSkipped(file core.File) {
+// 	if s.callbacks.OnFileSkipped != nil {
+// 		s.callbacks.OnFileSkipped(file)
+// 	}
+// }
+
+func (s *packageManifestScanner) dispatchOnDoneCodeAnalysis() {
+	if s.callbacks.OnDoneCodeAnalysis != nil {
+		s.callbacks.OnDoneCodeAnalysis()
+	}
+}
+
 func (s *packageManifestScanner) dispatchBeforeFinish() {
 	if s.callbacks.BeforeFinish != nil {
 		s.callbacks.BeforeFinish()

pkg/scanner/scanner.go

@@ -4,6 +4,12 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/safedep/code/core"
+	"github.com/safedep/code/fs"
+	"github.com/safedep/code/lang"
+	"github.com/safedep/code/parser"
+	"github.com/safedep/code/plugin"
+	"github.com/safedep/code/plugin/depsusage"
 	dryutils "github.com/safedep/dry/utils"
 	"github.com/safedep/vet/pkg/analyzer"
 	"github.com/safedep/vet/pkg/common/logger"
@@ -14,11 +20,13 @@ import (
 )
 
 type Config struct {
-	ExcludePatterns    []string
-	ConcurrentAnalyzer int
-	TransitiveAnalysis bool
-	TransitiveDepth    int
-	Experimental       bool
+	ExcludePatterns               []string
+	ConcurrentAnalyzer            int
+	TransitiveAnalysis            bool
+	CodeDirectories               []string
+	EnableDependencyUsageAnalysis bool
+	TransitiveDepth               int
+	Experimental                  bool
 }
 
 type packageManifestScanner struct {
@@ -36,7 +44,8 @@ func NewPackageManifestScanner(config Config,
 	readers []readers.PackageManifestReader,
 	enrichers []PackageMetaEnricher,
 	analyzers []analyzer.Analyzer,
-	reporters []reporter.Reporter) *packageManifestScanner {
+	reporters []reporter.Reporter,
+) *packageManifestScanner {
 	return &packageManifestScanner{
 		config:    config,
 		readers:   readers,
@@ -49,29 +58,53 @@ func NewPackageManifestScanner(config Config,
 func (s *packageManifestScanner) Start() error {
 	s.dispatchOnStart()
 
-	// The manifest processing go routine will close the doneChannel
-	doneChannel := make(chan bool)
-
 	// We will close the scanner channel
 	scannerChannel := make(chan *models.PackageManifest, 100)
 
 	ctx := context.Background()
 	defer ctx.Done()
 
-	go s.startManifestScanner(ctx, scannerChannel, doneChannel)
+	// Accumulate packages from the scanner
+	packagesChannel := make(chan *models.Package)
+	packages := []*models.Package{}
+	go func() {
+		for pkg := range packagesChannel {
+			packages = append(packages, pkg)
+		}
+	}()
+
+	// Accumulate usage evidences from the code analysis
+	usageEvidencesChannel := make(chan *depsusage.UsageEvidence)
+	packageUsageEvidenceMapping := make(map[string]*depsusage.UsageEvidence)
+	go func() {
+		for evidence := range usageEvidencesChannel {
+			packageUsageEvidenceMapping[evidence.Module] = evidence
+		}
+	}()
+
+	// The manifest processing go routine will close the doneManifestProcessingChannel
+	doneManifestProcessingChannel := make(chan bool)
+	go s.startManifestScanner(ctx, scannerChannel, packagesChannel, doneManifestProcessingChannel)
+
+	// Code analysis go routine will close the doneCodeAnalysisChannel
+	doneCodeAnalysisChannel := make(chan bool)
+	if s.shouldPerformCodeAnalysis() {
+		go s.startCodeAnalysis(ctx, usageEvidencesChannel, doneCodeAnalysisChannel)
+	} else {
+		close(usageEvidencesChannel)
+		close(doneCodeAnalysisChannel)
+	}
 
 	s.dispatchStartManifestEnumeration()
 
 	for _, reader := range s.readers {
 		err := reader.EnumManifests(func(manifest *models.PackageManifest,
-			_ readers.PackageReader) error {
-
+			_ readers.PackageReader,
+		) error {
 			s.dispatchOnManifestEnumeration(manifest)
 			scannerChannel <- manifest
-
 			return nil
 		})
-
 		if err != nil {
 			return err
 		}
@@ -82,7 +115,18 @@ func (s *packageManifestScanner) Start() error {
 	close(scannerChannel)
 
 	// Wait for manifest scanner to finish
-	<-doneChannel
+	<-doneManifestProcessingChannel
+
+	// Wait for code analysis to finish
+	<-doneCodeAnalysisChannel
+
+	logger.Debugf("Mark evidences for %d packages", len(packages))
+	for _, pkg := range packages {
+		if evidence, ok := packageUsageEvidenceMapping[pkg.Name]; ok {
+			logger.Debugf("%s used with evidence %v", pkg.Name, evidence)
+			pkg.UsageEvidences = append(pkg.UsageEvidences, evidence)
+		}
+	}
 
 	s.dispatchBeforeFinish()
 
@@ -99,7 +143,9 @@ func (s *packageManifestScanner) Start() error {
 // mechanism where we can scan a manifest whenever it is available instead of waiting
 // for all manifests to be available
 func (s *packageManifestScanner) startManifestScanner(ctx context.Context,
-	incoming <-chan *models.PackageManifest, done chan bool) {
+	incoming <-chan *models.PackageManifest, packagesChannel chan *models.Package, done chan bool,
+) {
+	defer close(packagesChannel)
 	defer close(done)
 
 	// Start the scan phases per manifest
@@ -121,7 +167,7 @@ func (s *packageManifestScanner) startManifestScanner(ctx context.Context,
 		s.dispatchOnStartManifest(manifest)
 
 		// Enrich each packages in a manifest with metadata
-		err := s.enrichManifest(manifest)
+		err := s.enrichManifest(manifest, packagesChannel)
 		if err != nil {
 			logger.Errorf("Failed to enrich %s manifest %s : %v",
 				manifest.Ecosystem, manifest.GetPath(), err)
@@ -145,6 +191,66 @@ func (s *packageManifestScanner) startManifestScanner(ctx context.Context,
 	}
 }
 
+func (s *packageManifestScanner) startCodeAnalysis(ctx context.Context, packageUsageEvidencesChannel chan *depsusage.UsageEvidence, doneChannel chan bool) {
+	defer close(packageUsageEvidencesChannel)
+	defer close(doneChannel)
+
+	s.dispatchOnStartCodeAnalysis()
+
+	logger.Debugf("Performing code analysis on directories: %v\n", s.config.CodeDirectories)
+
+	fileSystem, err := fs.NewLocalFileSystem(fs.LocalFileSystemConfig{
+		AppDirectories: s.config.CodeDirectories,
+	})
+	if err != nil {
+		logger.Errorf("failed to create local filesystem: %v", err)
+		return
+	}
+
+	// @TODO - Enable walking for all available languages ?
+	language, err := lang.NewPythonLanguage()
+	if err != nil {
+		logger.Errorf("failed to get language: %v", err)
+		return
+	}
+
+	walker, err := fs.NewSourceWalker(fs.SourceWalkerConfig{}, language)
+	if err != nil {
+		logger.Errorf("failed to create source walker: %v", err)
+		return
+	}
+
+	treeWalker, err := parser.NewWalkingParser(walker, language)
+	if err != nil {
+		logger.Errorf("failed to create tree walker: %v", err)
+		return
+	}
+
+	plugins := []core.Plugin{}
+
+	if s.config.EnableDependencyUsageAnalysis {
+		// consume usage evidences
+		var usageCallback depsusage.DependencyUsageCallback = func(ctx context.Context, evidence *depsusage.UsageEvidence) error {
+			packageUsageEvidencesChannel <- evidence
+			return nil
+		}
+		plugins = append(plugins, depsusage.NewDependencyUsagePlugin(usageCallback))
+	}
+
+	pluginExecutor, err := plugin.NewTreeWalkPluginExecutor(treeWalker, plugins)
+	if err != nil {
+		logger.Errorf("failed to create plugin executor: %v", err)
+		return
+	}
+
+	err = pluginExecutor.Execute(ctx, fileSystem)
+	if err != nil {
+		logger.Errorf("failed to execute plugins: %v", err)
+	}
+
+	s.dispatchOnDoneCodeAnalysis()
+}
+
 func (s *packageManifestScanner) analyzeManifest(manifest *models.PackageManifest) error {
 	for _, task := range s.analyzers {
 		err := task.Analyze(manifest, func(event *analyzer.AnalyzerEvent) error {
@@ -154,7 +260,6 @@ func (s *packageManifestScanner) analyzeManifest(manifest *models.PackageManifes
 
 			return s.internalHandleAnalyzerEvent(event)
 		})
-
 		if err != nil {
 			logger.Errorf("Analyzer %s failed: %v", task.Name(), err)
 		}
@@ -210,7 +315,7 @@ func (s *packageManifestScanner) finishAnalyzers() {
 	}
 }
 
-func (s *packageManifestScanner) enrichManifest(manifest *models.PackageManifest) error {
+func (s *packageManifestScanner) enrichManifest(manifest *models.PackageManifest, packagesChannel chan *models.Package) error {
 	if len(s.enrichers) == 0 {
 		return nil
 	}
@@ -237,6 +342,7 @@ func (s *packageManifestScanner) enrichManifest(manifest *models.PackageManifest
 	q.Start()
 
 	readers.NewManifestModelReader(manifest).EnumPackages(func(pkg *models.Package) error {
+		packagesChannel <- pkg
 		q.Add(pkg)
 		return nil
 	})
@@ -281,7 +387,8 @@ func (s *packageManifestScanner) packageEnrichWorkQueueHandler(pm *models.Packag
 
 func (s *packageManifestScanner) packageDependencyHandler(pm *models.PackageManifest,
 	_ *models.Package,
-	q *utils.WorkQueue[*models.Package]) PackageDependencyCallbackFn {
+	q *utils.WorkQueue[*models.Package],
+) PackageDependencyCallbackFn {
 	return func(pkg *models.Package) error {
 		// Check and queue for further analysis
 		if !s.config.TransitiveAnalysis {
@@ -361,3 +468,8 @@ func (s *packageManifestScanner) finaliseDependencyGraph(manifest *models.Packag
 
 	manifest.DependencyGraph.SetPresent(true)
 }
+
+func (s *packageManifestScanner) shouldPerformCodeAnalysis() bool {
+	// Checks for flags which require code analysis
+	return s.config.EnableDependencyUsageAnalysis
+}