Refactor - 2: Application create - part 1 (#46)

- Remove unnecessary methods in helpers pkg
- Move some of the helper to respective dir
- Use the newer methods in `application create` and refactor few things.
- Will be continuing rest in subsequent PRs

Author: mayuka-c

ai-services/cmd/ai-services/cmd/application/create.go

@@ -20,10 +20,13 @@ import (
 	"github.com/project-ai-services/ai-services/internal/pkg/cli/templates"
 	"github.com/project-ai-services/ai-services/internal/pkg/constants"
 	"github.com/project-ai-services/ai-services/internal/pkg/logger"
+	"github.com/project-ai-services/ai-services/internal/pkg/models"
 	"github.com/project-ai-services/ai-services/internal/pkg/runtime"
 	"github.com/project-ai-services/ai-services/internal/pkg/runtime/podman"
+	"github.com/project-ai-services/ai-services/internal/pkg/specs"
 	"github.com/project-ai-services/ai-services/internal/pkg/spinner"
 	"github.com/project-ai-services/ai-services/internal/pkg/utils"
+	"github.com/project-ai-services/ai-services/internal/pkg/validators"
 	"github.com/project-ai-services/ai-services/internal/pkg/vars"
 )
 
@@ -105,31 +108,20 @@ var createCmd = &cobra.Command{
 		}
 		s.Stop("SMT level configured successfully")
 
-		// Fetch all the application Template names
-		appTemplateNames, err := helpers.FetchApplicationTemplatesNames()
-		if err != nil {
-			return fmt.Errorf("failed to list templates: %w", err)
-		}
-
-		var appTemplateName string
+		tp := templates.NewEmbedTemplateProvider(templates.EmbedOptions{})
 
-		if index := fetchAppTemplateIndex(appTemplateNames, templateName); index == -1 {
-			return errors.New("provided template name is wrong. Please provide a valid template name")
-		} else {
-			appTemplateName = appTemplateNames[index]
+		// validate whether the provided template name is correct
+		if err := validators.ValidateAppTemplateExist(tp, templateName); err != nil {
+			return err
 		}
 
-		applicationPodTemplatesPath := applicationPath + appTemplateName + "/templates"
-
-		tmpls, err := helpers.LoadAllTemplates(applicationPodTemplatesPath)
+		tmpls, err := tp.LoadAllTemplates(templateName + "/templates")
 		if err != nil {
 			return fmt.Errorf("failed to parse the templates: %w", err)
 		}
 
-		tp := templates.NewEmbedTemplateProvider(templates.EmbedOptions{})
-
 		// load metadata.yml to read the app metadata
-		appMetadata, err := tp.LoadMetadata(appTemplateName)
+		appMetadata, err := tp.LoadMetadata(templateName)
 		if err != nil {
 			return fmt.Errorf("failed to read the app metadata: %w", err)
 		}
@@ -141,7 +133,7 @@ var createCmd = &cobra.Command{
 		// ---- Validate Spyre card Requirements ----
 
 		// calculate the required spyre cards
-		reqSpyreCardsCount, err := calculateReqSpyreCards(utils.ExtractMapKeys(tmpls), applicationPodTemplatesPath)
+		reqSpyreCardsCount, err := calculateReqSpyreCards(tp, utils.ExtractMapKeys(tmpls), templateName)
 		if err != nil {
 			return err
 		}
@@ -165,7 +157,7 @@ var createCmd = &cobra.Command{
 		if !skipModelDownload {
 			s = spinner.New("Downloading models as part of application creation...")
 			s.Start(ctx)
-			models, err := helpers.ListModels(appTemplateName)
+			models, err := helpers.ListModels(templateName)
 			if err != nil {
 				s.Fail("failed to list models")
 				return err
@@ -196,15 +188,15 @@ var createCmd = &cobra.Command{
 		s = spinner.New("Deploying application '" + appName + "'...")
 		s.Start(ctx)
 		// execute the pod Templates
-		if err := executePodTemplates(runtime, appName, appMetadata, tmpls, applicationPodTemplatesPath, pciAddresses); err != nil {
+		if err := executePodTemplates(runtime, tp, appName, appMetadata, tmpls, pciAddresses); err != nil {
 			return err
 		}
 		s.Stop("Application '" + appName + "' deployed successfully")
 
 		logger.Infoln("-------")
 
 		// print the next steps to be performed at the end of create
-		if err := helpers.PrintNextSteps(runtime, appName, appTemplateName); err != nil {
+		if err := helpers.PrintNextSteps(runtime, appName, templateName); err != nil {
 			// do not want to fail the overall create if we cannot print next steps
 			logger.Infof("failed to display next steps: %v\n", err)
 			return nil
@@ -306,44 +298,22 @@ func setSMTLevel() error {
 }
 
 func getTargetSMTLevel() (*int, error) {
-	appTemplateNames, err := helpers.FetchApplicationTemplatesNames()
-	if err != nil {
-		return nil, fmt.Errorf("failed to list templates: %w", err)
-	}
-
-	var appTemplateName string
+	tp := templates.NewEmbedTemplateProvider(templates.EmbedOptions{})
 
-	if index := fetchAppTemplateIndex(appTemplateNames, templateName); index == -1 {
-		return nil, errors.New("provided template name is wrong. Please provide a valid template name")
-	} else {
-		appTemplateName = appTemplateNames[index]
+	// validate whether the provided template name is correct
+	if err := validators.ValidateAppTemplateExist(tp, templateName); err != nil {
+		return nil, err
 	}
 
-	tp := templates.NewEmbedTemplateProvider(templates.EmbedOptions{})
-
 	// load metadata.yml to read the app metadata
-	appMetadata, err := tp.LoadMetadata(appTemplateName)
+	appMetadata, err := tp.LoadMetadata(templateName)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read the app metadata: %w", err)
 	}
 
 	return appMetadata.SMTLevel, nil
 }
 
-// fetchAppTemplateIndex -> Returns the index of app template if exists, otherwise -1
-func fetchAppTemplateIndex(appTemplateNames []string, templateName string) int {
-	appTemplateIndex := -1
-
-	for index, appTemplateName := range appTemplateNames {
-		if strings.EqualFold(appTemplateName, templateName) {
-			appTemplateIndex = index
-			break
-		}
-	}
-
-	return appTemplateIndex
-}
-
 func verifyPodTemplateExists(tmpls map[string]*template.Template, appMetadata *templates.AppMetadata) error {
 	flattenPodTemplateExecutions := utils.FlattenArray(appMetadata.PodTemplateExecutions)
 
@@ -361,8 +331,8 @@ func verifyPodTemplateExists(tmpls map[string]*template.Template, appMetadata *t
 	return nil
 }
 
-func executePodTemplates(runtime runtime.Runtime, appName string, appMetadata *templates.AppMetadata,
-	tmpls map[string]*template.Template, podTemplatesPath string, pciAddresses []string) error {
+func executePodTemplates(runtime runtime.Runtime, tp templates.Template, appName string, appMetadata *templates.AppMetadata,
+	tmpls map[string]*template.Template, pciAddresses []string) error {
 
 	globalParams := map[string]any{
 		"AppName":         appName,
@@ -390,10 +360,8 @@ func executePodTemplates(runtime runtime.Runtime, appName string, appMetadata *t
 				// Shallow Copy globalParams Map
 				params := utils.CopyMap(globalParams)
 
-				podTemplateFilePath := podTemplatesPath + "/" + podTemplateName
-
 				// fetch pod Spec
-				podSpec, err := fetchPodSpec(podTemplateFilePath)
+				podSpec, err := fetchPodSpec(tp, templateName, podTemplateName)
 				if err != nil {
 					errCh <- err
 				}
@@ -496,18 +464,15 @@ func validateSpyreCardRequirements(req int, actual int) error {
 	return nil
 }
 
-func calculateReqSpyreCards(podTemplateFileNames []string, podTemplatesPath string) (int, error) {
+func calculateReqSpyreCards(tp templates.Template, podTemplateFileNames []string, appTemplateName string) (int, error) {
 	totalReqSpyreCounts := 0
 
 	// Calculate Req Spyre Counts
 	for _, podTemplateFileName := range podTemplateFileNames {
-
-		podTemplateFilePath := podTemplatesPath + "/" + podTemplateFileName
-
-		// load the pod Template
-		podSpec, err := helpers.LoadPodTemplate(podTemplateFilePath)
+		// fetch pod spec
+		podSpec, err := fetchPodSpec(tp, appTemplateName, podTemplateFileName)
 		if err != nil {
-			return totalReqSpyreCounts, fmt.Errorf("failed to load pod Template: %s with error: %w", podTemplateFilePath, err)
+			return totalReqSpyreCounts, fmt.Errorf("failed to load pod Template: '%s' for appTemplate: '%s' with error: %w", podTemplateFileName, appTemplateName, err)
 		}
 
 		// fetch the spyreCount for all containers from the annotations
@@ -550,23 +515,23 @@ func fetchSpyreCardsFromPodAnnotations(annotations map[string]string) (int, map[
 	return spyreCards, spyreCardContainerMap, nil
 }
 
-func fetchPodSpec(podTemplateFilePath string) (*helpers.PodSpec, error) {
-	podSpec, err := helpers.LoadPodTemplate(podTemplateFilePath)
+func fetchPodSpec(tp templates.Template, appTemplateName, podTemplateFileName string) (*models.PodSpec, error) {
+	podSpec, err := tp.LoadPodTemplateWithDummyParams(appTemplateName, podTemplateFileName)
 	if err != nil {
-		return nil, fmt.Errorf("failed to load pod Template: %s with error: %w", podTemplateFilePath, err)
+		return nil, fmt.Errorf("failed to load pod Template: '%s' for appTemplate: '%s' with error: %w", podTemplateFileName, appTemplateName, err)
 	}
 
 	return podSpec, nil
 }
 
-func fetchPodAnnotations(podSpec *helpers.PodSpec) map[string]string {
-	return helpers.FetchPodAnnotations(*podSpec)
+func fetchPodAnnotations(podSpec *models.PodSpec) map[string]string {
+	return specs.FetchPodAnnotations(*podSpec)
 }
 
-func returnEnvParamsForPod(podSpec *helpers.PodSpec, podAnnotations map[string]string, pciAddresses *[]string) (map[string]map[string]string, error) {
+func returnEnvParamsForPod(podSpec *models.PodSpec, podAnnotations map[string]string, pciAddresses *[]string) (map[string]map[string]string, error) {
 
 	env := map[string]map[string]string{}
-	podContainerNames := helpers.FetchContainerNames(*podSpec)
+	podContainerNames := specs.FetchContainerNames(*podSpec)
 
 	// populate env with empty map
 	for _, containerName := range podContainerNames {

ai-services/cmd/ai-services/cmd/application/templates.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/project-ai-services/ai-services/internal/pkg/cli/helpers"
+	"github.com/project-ai-services/ai-services/internal/pkg/cli/templates"
 	"github.com/project-ai-services/ai-services/internal/pkg/logger"
 )
 
@@ -15,7 +15,9 @@ var templatesCmd = &cobra.Command{
 	Short: "Lists the offered application templates",
 	Long:  `Retrieves information about the offered application templates`,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		appTemplateNames, err := helpers.FetchApplicationTemplatesNames()
+		tp := templates.NewEmbedTemplateProvider(templates.EmbedOptions{})
+
+		appTemplateNames, err := tp.ListApplications()
 		if err != nil {
 			return fmt.Errorf("failed to list application templates: %w", err)
 		}

ai-services/internal/pkg/cli/helpers/helper.go

@@ -2,79 +2,18 @@ package helpers
 
 import (
 	"fmt"
-	"io/fs"
 	"log"
 	"os"
 	"os/exec"
-	"regexp"
-	"slices"
 	"strings"
-	"text/template"
 	"time"
 
 	"github.com/containers/podman/v5/libpod/define"
-	v1 "github.com/containers/podman/v5/pkg/k8s.io/api/core/v1"
-	"sigs.k8s.io/yaml"
 
-	"github.com/project-ai-services/ai-services/assets"
 	"github.com/project-ai-services/ai-services/internal/pkg/logger"
 	"github.com/project-ai-services/ai-services/internal/pkg/runtime"
 )
 
-func FetchApplicationTemplatesNames() ([]string, error) {
-	apps := []string{}
-
-	err := fs.WalkDir(assets.ApplicationFS, "applications", func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-		if d.IsDir() {
-			return nil
-		}
-
-		// Templates Pattern :- "assets/applications/<AppName>/templates/*.yaml.tmpl"
-		parts := strings.Split(path, "/")
-
-		if len(parts) >= 4 {
-			appName := parts[1]
-			if slices.Contains(apps, appName) {
-				return nil
-			}
-			apps = append(apps, appName)
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return apps, nil
-}
-
-// LoadAllTemplates -> Loads all templates under a specified root path
-func LoadAllTemplates(rootPath string) (map[string]*template.Template, error) {
-	tmpls := make(map[string]*template.Template)
-
-	err := fs.WalkDir(assets.ApplicationFS, rootPath, func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-		if d.IsDir() || !strings.HasSuffix(d.Name(), ".tmpl") {
-			return nil
-		}
-
-		t, err := template.ParseFS(assets.ApplicationFS, path)
-		if err != nil {
-			return fmt.Errorf("parse %s: %w", path, err)
-		}
-
-		// key should be just the template file name (Eg:- pod1.yaml.tmpl)
-		tmpls[strings.TrimPrefix(path, fmt.Sprintf("%s/", rootPath))] = t
-		return nil
-	})
-	return tmpls, err
-}
-
 type HealthStatus string
 
 const (
@@ -186,45 +125,6 @@ func FindFreeSpyreCards() ([]string, error) {
 	return free_spyre_dev_id_list, nil
 }
 
-type PodSpec struct {
-	v1.Pod
-}
-
-func sanitizeTemplateForYaml(input []byte) []byte {
-	re := regexp.MustCompile(`{{.*?}}`)
-	return re.ReplaceAll(input, []byte("# template removed"))
-}
-
-func LoadPodTemplate(path string) (*PodSpec, error) {
-	data, err := assets.ApplicationFS.ReadFile(path)
-	if err != nil {
-		return nil, fmt.Errorf("read metadata: %w", err)
-	}
-
-	// comment out template literals
-	// Note: This will sanitize by removing the template literals
-	// If we want to use with templating values, then apply templating and then read the file
-	data = sanitizeTemplateForYaml(data)
-
-	var podSpec PodSpec
-	if err := yaml.Unmarshal(data, &podSpec); err != nil {
-		return nil, err
-	}
-	return &podSpec, nil
-}
-
-func FetchPodAnnotations(podspec PodSpec) map[string]string {
-	return podspec.Annotations
-}
-
-func FetchContainerNames(podspec PodSpec) []string {
-	var containerNames []string
-	for _, v1Container := range podspec.Spec.Containers {
-		containerNames = append(containerNames, v1Container.Name)
-	}
-	return containerNames
-}
-
 func ParseSkipChecks(skipChecks []string) map[string]bool {
 	skipMap := make(map[string]bool)
 	for _, check := range skipChecks {

ai-services/internal/pkg/cli/helpers/images.go

@@ -23,19 +23,13 @@ func ListImages(template string) ([]string, error) {
 		return nil, fmt.Errorf("error loading templates for %s: %w", template, err)
 	}
 
-	dummyParams := map[string]any{
-		"AppName":         "dummy-app",
-		"AppTemplateName": "",
-		"Version":         "",
-	}
-
 	images := []string{
 		// include tool image as well which is used for all the housekeeping tasks
 		vars.ToolImage,
 	}
 
 	for _, tmpl := range tmpls {
-		ps, err := tp.LoadPodTemplate(template, tmpl.Name(), dummyParams)
+		ps, err := tp.LoadPodTemplateWithDummyParams(template, tmpl.Name())
 		if err != nil {
 			return nil, fmt.Errorf("error loading pod template: %w", err)
 		}