Dev

src/api/v1/file_manager_impl.go

@@ -2,8 +2,8 @@ package v1
 
 import (
 	"encoding/json"
-	. "github.com/makeopensource/leviathan/common"
-	. "github.com/makeopensource/leviathan/service/file_manager"
+	com "github.com/makeopensource/leviathan/common"
+	fm "github.com/makeopensource/leviathan/service/file_manager"
 	"github.com/rs/zerolog/log"
 	"mime/multipart"
 	"net/http"
@@ -20,15 +20,15 @@ type FileManagerHandler struct {
 	BasePath             string
 	UploadLabPath        string
 	UploadSubmissionPath string
-	service              FileManagerService
+	service              fm.FileManagerService
 }
 
 func NewFileManagerHandler(basePath string) *FileManagerHandler {
 	return &FileManagerHandler{
 		BasePath:             basePath,
 		UploadLabPath:        basePath + "/upload/lab",
 		UploadSubmissionPath: basePath + "/upload/submission",
-		service:              FileManagerService{},
+		service:              fm.FileManagerService{},
 	}
 }
 
@@ -44,7 +44,6 @@ func (f *FileManagerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	w.WriteHeader(http.StatusMethodNotAllowed)
-	return
 }
 
 func (f *FileManagerHandler) UploadLabData(w http.ResponseWriter, r *http.Request) {
@@ -58,12 +57,12 @@ func (f *FileManagerHandler) UploadLabData(w http.ResponseWriter, r *http.Reques
 	if err != nil {
 		http.Error(
 			w,
-			ErrLog("Failed to get dockerfile in form", err, log.Error()).Error(),
+			com.ErrLog("Failed to get dockerfile in form", err, log.Error()).Error(),
 			http.StatusBadRequest,
 		)
 		return
 	}
-	defer CloseFile(dockerFile)
+	defer com.CloseFile(dockerFile)
 
 	jobFiles, ok := r.MultipartForm.File[LabFilesKey]
 	if !ok || len(jobFiles) == 0 {
@@ -76,9 +75,9 @@ func (f *FileManagerHandler) UploadLabData(w http.ResponseWriter, r *http.Reques
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
-	defer func(files []*FileInfo) {
+	defer func(files []*fm.FileInfo) {
 		for _, file := range files {
-			CloseFile(file.Reader)
+			com.CloseFile(file.Reader)
 		}
 	}(fileInfos)
 
@@ -88,7 +87,7 @@ func (f *FileManagerHandler) UploadLabData(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	sendResponse(w, err, folderID)
+	sendResponse(w, folderID)
 }
 
 func (f *FileManagerHandler) UploadSubmissionData(w http.ResponseWriter, r *http.Request) {
@@ -109,9 +108,9 @@ func (f *FileManagerHandler) UploadSubmissionData(w http.ResponseWriter, r *http
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
-	defer func(files []*FileInfo) {
+	defer func(files []*fm.FileInfo) {
 		for _, file := range files {
-			CloseFile(file.Reader)
+			com.CloseFile(file.Reader)
 		}
 	}(fileInfos)
 
@@ -121,11 +120,11 @@ func (f *FileManagerHandler) UploadSubmissionData(w http.ResponseWriter, r *http
 		return
 	}
 
-	sendResponse(w, err, folderID)
+	sendResponse(w, folderID)
 }
 
-func sendResponse(w http.ResponseWriter, err error, folderID string) {
-	jsonData, err := toJson(folderID, err)
+func sendResponse(w http.ResponseWriter, folderID string) {
+	jsonData, err := toJson(folderID)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
@@ -135,32 +134,32 @@ func sendResponse(w http.ResponseWriter, err error, folderID string) {
 	if err != nil {
 		http.Error(
 			w,
-			ErrLog("Failed to write response", err, log.Error()).Error(),
+			com.ErrLog("Failed to write response", err, log.Error()).Error(),
 			http.StatusInternalServerError,
 		)
 		return
 	}
 }
 
-func toJson(folderID string, err error) ([]byte, error) {
+func toJson(folderID string) ([]byte, error) {
 	resultMap := map[string]string{
 		"folderId": folderID,
 	}
 	jsonData, err := json.Marshal(resultMap)
 	if err != nil {
-		return nil, ErrLog("Failed to marshal json", err, log.Error())
+		return nil, com.ErrLog("Failed to marshal json", err, log.Error())
 	}
 	return jsonData, nil
 }
 
-func mapToFileInfo(jobFiles []*multipart.FileHeader) ([]*FileInfo, error) {
-	var fileInfos []*FileInfo
+func mapToFileInfo(jobFiles []*multipart.FileHeader) ([]*fm.FileInfo, error) {
+	var fileInfos []*fm.FileInfo
 	for _, jobFile := range jobFiles {
 		file, err := jobFile.Open()
 		if err != nil {
-			return fileInfos, ErrLog("unable to open file: "+err.Error(), err, log.Error())
+			return fileInfos, com.ErrLog("unable to open file: "+err.Error(), err, log.Error())
 		}
-		fileInfos = append(fileInfos, &FileInfo{
+		fileInfos = append(fileInfos, &fm.FileInfo{
 			Reader:   file,
 			Filename: jobFile.Filename,
 		})

src/common/logger.go

@@ -52,8 +52,8 @@ func FileConsoleLogger() zerolog.Logger {
 //
 // This hides implementation details from users while ensuring full error information is available for debugging.
 func ErrLog(message string, err error, eventLevel *zerolog.Event) error {
-	eventLevel.Err(err).Msgf(message)
-	return fmt.Errorf(message)
+	eventLevel.Err(err).Msg(message)
+	return fmt.Errorf("%s", message)
 }
 
 func ConsoleLogger() zerolog.Logger {

src/common/utils.go

@@ -11,10 +11,7 @@ import (
 
 func FileExists(filename string) bool {
 	_, err := os.Stat(filename)
-	if os.IsNotExist(err) {
-		return false
-	}
-	return true
+	return !os.IsNotExist(err)
 }
 
 func CloseFile(file io.ReadCloser) {

src/models/job_error.go

@@ -0,0 +1,35 @@
+package models
+
+type JobError interface {
+	// Reason will be displayed to the end user, providing a user-friendly message.
+	Reason() string
+	// Err err parameter holds the underlying error, used for debugging purposes.
+	Err() error
+	// ErrStr returns string from the error, if nil return empty string
+	ErrStr() string
+}
+
+// JErr implements JobError
+type JErr struct {
+	reason string
+	err    error
+}
+
+func JError(reason string, err error) JErr {
+	return JErr{reason: reason, err: err}
+}
+
+func (err JErr) Reason() string {
+	return err.reason
+}
+
+func (err JErr) Err() error {
+	return err.err
+}
+
+func (err JErr) ErrStr() string {
+	if err.err != nil {
+		return err.err.Error()
+	}
+	return ""
+}

src/models/worker_semaphore.go

@@ -0,0 +1,25 @@
+package models
+
+type WorkerSemaphore struct {
+	semaphore chan struct{}
+}
+
+func NewWorkerSemaphore(maxWorkers int) *WorkerSemaphore {
+	sem := &WorkerSemaphore{
+		semaphore: make(chan struct{}, maxWorkers),
+	}
+	for i := 0; i < maxWorkers; i++ {
+		sem.Release() // fill the semaphore with maxWorkers
+	}
+	return sem
+}
+
+// Acquire returns the semaphore channel for select statements
+// will block until resource is available
+func (s *WorkerSemaphore) Acquire() <-chan struct{} {
+	return s.semaphore
+}
+
+func (s *WorkerSemaphore) Release() {
+	s.semaphore <- struct{}{}
+}

src/service/docker/docker_utils_ssh.go

@@ -9,7 +9,7 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
-	. "github.com/makeopensource/leviathan/common"
+	com "github.com/makeopensource/leviathan/common"
 	"github.com/makeopensource/leviathan/models"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
@@ -70,7 +70,7 @@ func saveHostKey(machine models.MachineOptions) func(hostname string, remote net
 }
 
 func writeMachineToConfigFile(machine models.MachineOptions) {
-	machineKey := fmt.Sprintf("%s.%s", ClientsSSH.ConfigKey, machine.Name())
+	machineKey := fmt.Sprintf("%s.%s", com.ClientsSSH.ConfigKey, machine.Name())
 	viper.Set(machineKey, machine)
 	err := viper.WriteConfig()
 	if err != nil {
@@ -109,7 +109,7 @@ func GenerateKeyPair() (privateKey []byte, publicKey []byte, err error) {
 //
 // the generated keys can be found in common.SSHConfigFolder
 func initKeyPairFile() {
-	basePath := SSHConfigFolder.GetStr()
+	basePath := com.SSHConfigFolder.GetStr()
 	privateKeyPath := fmt.Sprintf("%s/%s", basePath, "id_rsa")
 	publicKeyPath := fmt.Sprintf("%s/%s", basePath, "id_rsa.pub")
 
@@ -120,7 +120,7 @@ func initKeyPairFile() {
 		Str("private_key_file", privateKeyPath).
 		Str("public_key_file", publicKeyPath)
 
-	if FileExists(privateKeyPath) && FileExists(publicKeyPath) {
+	if com.FileExists(privateKeyPath) && com.FileExists(publicKeyPath) {
 		logF.Msg("found existing keys... skipping generation")
 		return
 	}
@@ -143,7 +143,7 @@ func initKeyPairFile() {
 func LoadPrivateKey() ([]byte, error) {
 	return os.ReadFile(fmt.Sprintf(
 		"%s/%s",
-		SSHConfigFolder.GetStr(),
+		com.SSHConfigFolder.GetStr(),
 		"id_rsa",
 	))
 }

src/service/file_manager/file_manager_service.go

@@ -3,7 +3,7 @@ package file_manager
 import (
 	"fmt"
 	"github.com/google/uuid"
-	. "github.com/makeopensource/leviathan/common"
+	com "github.com/makeopensource/leviathan/common"
 	"github.com/rs/zerolog/log"
 	"io"
 	"os"
@@ -36,7 +36,7 @@ func (f *FileManagerService) CreateTmpLabFolder(dockerfile io.Reader, jobFiles .
 	jobDataDir := filepath.Join(basePath, JobDataFolderName)
 	err = os.MkdirAll(jobDataDir, os.ModePerm)
 	if err != nil {
-		return "", ErrLog("unable to create job data folder", err, log.Error())
+		return "", com.ErrLog("unable to create job data folder", err, log.Error())
 	}
 
 	if err = f.SaveFile(basePath, DockerfileName, dockerfile); err != nil {
@@ -71,14 +71,14 @@ func (f *FileManagerService) CreateSubmissionFolder(jobFiles ...*FileInfo) (stri
 func (f *FileManagerService) createBaseFolder() (string, string, error) {
 	folderUUID, err := uuid.NewUUID()
 	if err != nil {
-		return "", "", ErrLog("Unable to generate uuid", err, log.Error())
+		return "", "", com.ErrLog("Unable to generate uuid", err, log.Error())
 	}
 	stringUuid := folderUUID.String()
-	basePath := filepath.Join(TmpUploadFolder.GetStr(), stringUuid)
+	basePath := filepath.Join(com.TmpUploadFolder.GetStr(), stringUuid)
 
-	err = os.Mkdir(basePath, DefaultFilePerm)
+	err = os.Mkdir(basePath, com.DefaultFilePerm)
 	if err != nil {
-		return "", "", ErrLog("Unable to create tmp folder", err, log.Error())
+		return "", "", com.ErrLog("Unable to create tmp folder", err, log.Error())
 	}
 
 	return folderUUID.String(), basePath, nil
@@ -90,7 +90,7 @@ func (f *FileManagerService) SaveFile(basePath string, filename string, file io.
 	dst, err := os.Create(fPath)
 
 	if err != nil {
-		return ErrLog(
+		return com.ErrLog(
 			"Failed to create destination file",
 			err,
 			log.Error(),
@@ -106,7 +106,7 @@ func (f *FileManagerService) SaveFile(basePath string, filename string, file io.
 	// Copy the file contents
 	written, err := io.Copy(dst, file)
 	if err != nil {
-		return ErrLog(
+		return com.ErrLog(
 			"Failed to write file",
 			err,
 			log.Error(),
@@ -122,14 +122,14 @@ func (f *FileManagerService) SaveFile(basePath string, filename string, file io.
 }
 
 func (f *FileManagerService) DeleteFolder(folderUuid string) {
-	basePath := filepath.Join(TmpUploadFolder.GetStr(), folderUuid)
+	basePath := filepath.Join(com.TmpUploadFolder.GetStr(), folderUuid)
 	if err := os.RemoveAll(basePath); err != nil {
 		log.Warn().Err(err).Msgf("failed to delete tmp folder %s", folderUuid)
 	}
 }
 
 func (f *FileManagerService) GetLabFilePaths(folderUuid string) (basePath string, err error) {
-	basePath = filepath.Join(TmpUploadFolder.GetStr(), folderUuid)
+	basePath = filepath.Join(com.TmpUploadFolder.GetStr(), folderUuid)
 	jobData := filepath.Join(basePath, JobDataFolderName)
 	dockerFile := filepath.Join(basePath, DockerfileName)
 
@@ -144,12 +144,12 @@ func (f *FileManagerService) GetLabFilePaths(folderUuid string) (basePath string
 }
 
 func (f *FileManagerService) GetSubmissionPath(uuid string) (string, error) {
-	path := filepath.Join(TmpUploadFolder.GetStr(), uuid)
+	path := filepath.Join(com.TmpUploadFolder.GetStr(), uuid)
 	return f.checkFolder(path)
 }
 
 func (f *FileManagerService) checkFolder(path string) (jobData string, err error) {
-	if !FileExists(path) {
+	if !com.FileExists(path) {
 		return "", fmt.Errorf("could not find path")
 	}
 	return path, err