fix: Bug with backoff/retry loop

pkg/scanner/enrich_malware.go

@@ -29,7 +29,7 @@ func DefaultMalysisMalwareEnricherConfig() MalysisMalwareEnricherConfig {
 	return MalysisMalwareEnricherConfig{
 		Timeout:              5 * time.Minute,
 		QueryWorkerCount:     10,
-		GrpcOperationTimeout: 5 * time.Second,
+		GrpcOperationTimeout: 10 * time.Second,
 		MaxQueryRetries:      10,
 	}
 }
@@ -166,14 +166,27 @@ func (e *malysisMalwareEnricher) Wait() error {
 }
 
 func (e *malysisMalwareEnricher) enqueueAnalysisForQuery(req *analysisQueryRequest) {
-	// If we have already crossed next retry time, we will reset the retry count
-	if time.Now().After(req.nextRetryAt) {
-		// Always be time.Now() for the first retry
-		req.nextRetryAt = time.Now().Add(time.Duration(req.retryCount) * 5 * time.Second)
+	// If we are in the backoff period, we will schedule an enqueue
+	// operation in the future else we will submit the request immediately.
+	if time.Now().Before(req.nextRetryAt) {
+		go func(ctx context.Context) {
+			timer := time.NewTimer(time.Until(req.nextRetryAt.Add(100 * time.Millisecond)))
+			defer timer.Stop()
+
+			select {
+			case <-ctx.Done():
+				return
+			case <-timer.C:
+			}
+
+			e.queryChannel <- req
+		}(e.ctx)
+	} else {
 		req.retryCount++
-	}
+		req.nextRetryAt = time.Now().Add(5 * time.Second * time.Duration(req.retryCount))
 
-	e.queryChannel <- req
+		e.queryChannel <- req
+	}
 }
 
 func (e *malysisMalwareEnricher) submitPackageForAnalysis(pkg *models.Package) (*analysisSubmissionResult, error) {
@@ -308,23 +321,10 @@ func (e *malysisMalwareEnricher) startQueryWorker(ctx context.Context) error {
 						logger.Debugf("[Malware Analysis] Retrying query for analysis report: %s in %s",
 							req.analysisId, time.Until(req.nextRetryAt))
 
-						// Schedule the retry in future while handling
-						// global context cancellation
-						go func(ctx context.Context) {
-							timer := time.NewTimer(time.Until(req.nextRetryAt))
-							defer timer.Stop()
-
-							select {
-							case <-ctx.Done():
-								return
-							case <-timer.C:
-							}
-
-							e.resultsChannel <- &analysisQueryResult{
-								req: req,
-								err: errMalysisPollRetryInFuture,
-							}
-						}(ctx)
+						e.resultsChannel <- &analysisQueryResult{
+							req: req,
+							err: errMalysisPollRetryInFuture,
+						}
 
 						continue
 					}

pkg/scanner/scanner.go

@@ -259,7 +259,7 @@ func (s *packageManifestScanner) packageEnricherWait() error {
 
 		err := enricher.Wait()
 		if err != nil {
-			return err
+			logger.Errorf("Failed to wait for enricher %s: %v", enricher.Name(), err)
 		}
 	}