Add skip archive support (#2257)

hack/snifftest/main.go

@@ -203,6 +203,7 @@ func main() {
 						}
 					},
 					true,
+					false,
 				)
 
 				logger.Info("scanning repo", "repo", r)

pkg/handlers/archive.go

@@ -46,6 +46,7 @@ type Archive struct {
 	size         int
 	currentDepth int
 	skipBinaries bool
+	skipArchives bool
 }
 
 // New creates a new Archive handler with the provided options.
@@ -72,6 +73,10 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
 
 // FromFile extracts the files from an archive.
 func (a *Archive) FromFile(originalCtx logContext.Context, data io.Reader) chan []byte {
+	if a.skipArchives {
+		return nil
+	}
+
 	archiveChan := make(chan []byte, defaultBufferSize)
 	go func() {
 		ctx, cancel := logContext.WithTimeout(originalCtx, maxTimeout)

pkg/handlers/archive_test.go

@@ -311,6 +311,31 @@ func TestExtractDebContent(t *testing.T) {
 	assert.Equal(t, expectedLength, len(string(content)))
 }
 
+func TestSkipArchive(t *testing.T) {
+	file, err := os.Open("testdata/test.tgz")
+	assert.Nil(t, err)
+	defer file.Close()
+
+	reader, err := diskbufferreader.New(file)
+	assert.NoError(t, err)
+
+	ctx := logContext.Background()
+
+	chunkCh := make(chan *sources.Chunk)
+	go func() {
+		defer close(chunkCh)
+		ok := HandleFile(ctx, reader, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh}, WithSkipArchives(true))
+		assert.False(t, ok)
+	}()
+
+	wantCount := 0
+	count := 0
+	for range chunkCh {
+		count++
+	}
+	assert.Equal(t, wantCount, count)
+}
+
 func TestExtractTarContent(t *testing.T) {
 	file, err := os.Open("testdata/test.tgz")
 	assert.Nil(t, err)

pkg/handlers/handlers.go

@@ -36,6 +36,15 @@ func WithSkipBinaries(skip bool) Option {
 	}
 }
 
+// WithSkipArchives returns a Option that configures whether to skip archive files.
+func WithSkipArchives(skip bool) Option {
+	return func(h Handler) {
+		if a, ok := h.(*Archive); ok {
+			a.skipArchives = skip
+		}
+	}
+}
+
 type Handler interface {
 	FromFile(logContext.Context, io.Reader) chan []byte
 	IsFiletype(logContext.Context, io.Reader) (io.Reader, bool)
@@ -84,6 +93,10 @@ func processHandler(ctx logContext.Context, h Handler, reReader *diskbufferreade
 }
 
 func handleChunks(ctx logContext.Context, handlerChan chan []byte, chunkSkel *sources.Chunk, reporter sources.ChunkReporter) bool {
+	if handlerChan == nil {
+		return false
+	}
+
 	for {
 		select {
 		case data, open := <-handlerChan:

pkg/sources/git/git.go

@@ -58,6 +58,7 @@ type Git struct {
 	metrics            metrics
 	concurrency        *semaphore.Weighted
 	skipBinaries       bool
+	skipArchives       bool
 }
 
 type metrics struct {
@@ -66,6 +67,7 @@ type metrics struct {
 
 func NewGit(sourceType sourcespb.SourceType, jobID sources.JobID, sourceID sources.SourceID, sourceName string, verify bool, concurrency int,
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData, skipBinaries bool,
+	skipArchives bool,
 ) *Git {
 	return &Git{
 		sourceType:         sourceType,
@@ -76,6 +78,7 @@ func NewGit(sourceType sourcespb.SourceType, jobID sources.JobID, sourceID sourc
 		verify:             verify,
 		concurrency:        semaphore.NewWeighted(int64(concurrency)),
 		skipBinaries:       skipBinaries,
+		skipArchives:       skipArchives,
 	}
 }
 
@@ -178,6 +181,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, so
 			}
 		},
 		conn.GetSkipBinaries(),
+		conn.GetSkipArchives(),
 	)
 	return nil
 }
@@ -1014,6 +1018,10 @@ func (s *Git) handleBinary(ctx context.Context, gitDir string, reporter sources.
 		}
 	}
 
+	if s.skipArchives {
+		handlerOpts = append(handlerOpts, handlers.WithSkipArchives(true))
+	}
+
 	cmd := exec.Command("git", "-C", gitDir, "cat-file", "blob", commitHash.String()+":"+path)
 
 	var stderr bytes.Buffer

pkg/sources/github/github.go

@@ -277,6 +277,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
 			}
 		},
 		conn.GetSkipBinaries(),
+		conn.GetSkipArchives(),
 	)
 
 	return nil

pkg/sources/gitlab/gitlab.go

@@ -138,6 +138,7 @@ func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourc
 			}
 		},
 		conn.GetSkipBinaries(),
+		conn.GetSkipArchives(),
 	)
 
 	return nil

proto/sources.proto

@@ -96,6 +96,7 @@ message Bitbucket {
   repeated string repositories = 5;
   repeated string ignore_repos = 6;
   bool skip_binaries = 7;
+  bool skip_archives = 8;
 }
 
 message CircleCI {
@@ -196,6 +197,7 @@ message Git {
   // like head, base, bare, etc.
   string uri = 13; // repository URL. https://, file://, or ssh://
   bool skip_binaries = 14;
+  bool skip_archives = 15;
 }
 
 message GitLab {
@@ -208,6 +210,7 @@ message GitLab {
   repeated string repositories = 5;
   repeated string ignore_repos = 6;
   bool skip_binaries = 7;
+  bool skip_archives = 8;
 }
 
 message GitHub {
@@ -230,6 +233,7 @@ message GitHub {
   bool include_issue_comments = 15;
   bool include_gist_comments = 16;
   bool skip_binaries = 17;
+  bool skip_archives = 18;
 }
 
 message GoogleDrive {
@@ -301,6 +305,7 @@ message Gerrit {
   }
   repeated string projects = 4;
   bool skip_binaries = 5;
+  bool skip_archives = 6;
 }
 
 message Jenkins {
@@ -369,4 +374,5 @@ message AzureRepos {
   repeated string include_projects = 10;
   repeated string ignore_projects = 11;
   bool skip_binaries = 12;
+  bool skip_archives = 13;
 }