Removed stored logger from HuggingFace source (#4328)

Author: kashifkhan0771

pkg/sources/huggingface/client.go

@@ -150,11 +150,11 @@ func (c *HFClient) get(ctx context.Context, url string, target interface{}) erro
 	}
 
 	if resp.StatusCode == http.StatusUnauthorized {
-		return errors.New("invalid API key.")
+		return errors.New("invalid API key")
 	}
 
 	if resp.StatusCode == http.StatusForbidden {
-		return errors.New("access to this repo is restricted and you are not in the authorized list. Visit the repository to ask for access.")
+		return errors.New("access to this repo is restricted and you are not in the authorized list. Visit the repository to ask for access")
 	}
 
 	defer resp.Body.Close()

pkg/sources/huggingface/huggingface.go

@@ -68,7 +68,6 @@ type Source struct {
 	scanOptions *git.ScanOptions
 
 	apiClient       *HFClient
-	log             logr.Logger
 	conn            *sourcespb.Huggingface
 	jobPool         *errgroup.Group
 	resumeInfoMutex sync.Mutex
@@ -112,21 +111,21 @@ type filteredRepoCache struct {
 	include, exclude []glob.Glob
 }
 
-func (s *Source) newFilteredRepoCache(c cache.Cache[string], include, exclude []string) *filteredRepoCache {
+func (s *Source) newFilteredRepoCache(ctx context.Context, c cache.Cache[string], include, exclude []string) *filteredRepoCache {
 	includeGlobs := make([]glob.Glob, 0, len(include))
 	excludeGlobs := make([]glob.Glob, 0, len(exclude))
 	for _, ig := range include {
 		g, err := glob.Compile(ig)
 		if err != nil {
-			s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)
+			ctx.Logger().V(1).Info("invalid include glob", "include_value", ig, "err", err)
 			continue
 		}
 		includeGlobs = append(includeGlobs, g)
 	}
 	for _, eg := range exclude {
 		g, err := glob.Compile(eg)
 		if err != nil {
-			s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
+			ctx.Logger().V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
 			continue
 		}
 		excludeGlobs = append(excludeGlobs, g)
@@ -169,14 +168,12 @@ func (c *filteredRepoCache) includeRepo(s string) bool {
 }
 
 // Init returns an initialized HuggingFace source.
-func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(ctx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
 	err := git.CmdCheck()
 	if err != nil {
 		return err
 	}
 
-	s.log = aCtx.Logger()
-
 	s.name = name
 	s.sourceID = sourceID
 	s.jobID = jobID
@@ -208,17 +205,17 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
 		return err
 	}
 
-	s.filteredModelsCache = s.newFilteredRepoCache(simple.NewCache[string](),
+	s.filteredModelsCache = s.newFilteredRepoCache(ctx, simple.NewCache[string](),
 		append(s.conn.GetModels(), s.conn.GetIncludeModels()...),
 		s.conn.GetIgnoreModels(),
 	)
 
-	s.filteredSpacesCache = s.newFilteredRepoCache(simple.NewCache[string](),
+	s.filteredSpacesCache = s.newFilteredRepoCache(ctx, simple.NewCache[string](),
 		append(s.conn.GetSpaces(), s.conn.GetIncludeSpaces()...),
 		s.conn.GetIgnoreSpaces(),
 	)
 
-	s.filteredDatasetsCache = s.newFilteredRepoCache(simple.NewCache[string](),
+	s.filteredDatasetsCache = s.newFilteredRepoCache(ctx, simple.NewCache[string](),
 		append(s.conn.GetDatasets(), s.conn.GetIncludeDatasets()...),
 		s.conn.GetIgnoreDatasets(),
 	)
@@ -249,8 +246,8 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
 						Link:         giturl.GenerateLink(repository, commit, file, line),
 						Timestamp:    sanitizer.UTF8(timestamp),
 						Line:         line,
-						Visibility:   s.visibilityOf(aCtx, repository),
-						ResourceType: s.getResourceType(aCtx, repository),
+						Visibility:   s.visibilityOf(ctx, repository),
+						ResourceType: s.getResourceType(ctx, repository),
 					},
 				},
 			}
@@ -369,7 +366,7 @@ func (s *Source) enumerate(ctx context.Context) error {
 		}
 	}
 
-	s.log.Info("Completed enumeration", "num_models", len(s.models), "num_spaces", len(s.spaces), "num_datasets", len(s.datasets))
+	ctx.Logger().Info("Completed enumeration", "num_models", len(s.models), "num_spaces", len(s.spaces), "num_datasets", len(s.datasets))
 
 	// We must sort the repos so we can resume later if necessary.
 	sort.Strings(s.models)
@@ -507,7 +504,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
 
 	repos := s.getReposListByType(resourceType)
 
-	s.log.V(2).Info("Found "+resourceType+" to scan", "count", len(repos))
+	ctx.Logger().V(2).Info("Found "+resourceType+" to scan", "count", len(repos))
 
 	// If there is resume information available, limit this scan to only the repos that still need scanning.
 	reposToScan, progressIndexOffset := sources.FilterReposToResume(repos, s.GetProgress().EncodedResumeInfo)
@@ -539,7 +536,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
 			if !ok {
 				// This should never happen.
 				err := fmt.Errorf("no repoInfo for URL: %s", repoURL)
-				s.log.Error(err, "failed to scan "+resourceType)
+				ctx.Logger().Error(err, "failed to scan "+resourceType)
 				return nil
 			}
 			repoCtx := context.WithValues(ctx, resourceType, repoURL)
@@ -565,7 +562,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
 
 	_ = s.jobPool.Wait()
 	if scanErrs.Count() > 0 {
-		s.log.V(0).Info("failed to scan some repositories", "error_count", scanErrs.Count(), "errors", scanErrs.String())
+		ctx.Logger().V(0).Info("failed to scan some repositories", "error_count", scanErrs.Count(), "errors", scanErrs.String())
 	}
 	s.SetProgressComplete(len(repos), len(repos), "Completed HuggingFace "+resourceType+" scan", "")
 	return nil

pkg/sources/huggingface/repo.go

@@ -56,11 +56,13 @@ func (s *Source) cloneRepo(
 
 	switch s.conn.GetCredential().(type) {
 	case *sourcespb.Huggingface_Unauthenticated:
+		ctx.Logger().V(2).Info("cloning repo without authentication", "repo_url", repoURL)
 		path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL)
 		if err != nil {
 			return "", nil, err
 		}
 	case *sourcespb.Huggingface_Token:
+		ctx.Logger().V(2).Info("cloning repo with token authentication", "repo_url", repoURL)
 		path, repo, err = git.CloneRepoUsingToken(ctx, s.huggingfaceToken, repoURL, "", true)
 		if err != nil {
 			return "", nil, err