From 690336a53253f9d8cc5ad31040d70a753b70cb03 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 26 Jan 2024 16:40:43 -0500 Subject: [PATCH 01/27] add a new GetList with call back --- blobstore/list.go | 120 +++++++++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 54 deletions(-) diff --git a/blobstore/list.go b/blobstore/list.go index d9688b1..7295aba 100644 --- a/blobstore/list.go +++ b/blobstore/list.go @@ -88,16 +88,17 @@ func (bh *BlobHandler) HandleListByPrefix(c echo.Context) error { } } - listOutput, err := s3Ctrl.GetList(bucket, prefix, delimiter) - if err != nil { - log.Error("HandleListByPrefix: Error getting list:", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + var objectKeys []string + processPage := func(page *s3.ListObjectsV2Output) error { + for _, object := range page.Contents { + objectKeys = append(objectKeys, aws.StringValue(object.Key)) + } + return nil } - // Convert the list of object keys to strings - var objectKeys []string - for _, object := range listOutput.Contents { - objectKeys = append(objectKeys, aws.StringValue(object.Key)) + err = s3Ctrl.GetListWithCallBack(bucket, prefix, delimiter, processPage) + if err != nil { + return c.JSON(http.StatusInternalServerError, fmt.Sprintf("Error processing objects: %v", err)) } log.Info("HandleListByPrefix: Successfully retrieved list by prefix:", prefix) @@ -139,72 +140,83 @@ func (bh *BlobHandler) HandleListByPrefixWithDetail(c echo.Context) error { prefix = strings.Trim(prefix, "/") + "/" } - query := &s3.ListObjectsV2Input{ - Bucket: aws.String(bucket), - Prefix: aws.String(prefix), - Delimiter: aws.String("/"), - MaxKeys: aws.Int64(1000), - } - - result := []ListResult{} - truncatedListing := true + var results []ListResult var count int - for truncatedListing { - resp, err := s3Ctrl.S3Svc.ListObjectsV2(query) - if err != nil { - log.Error("HandleListByPrefixWithDetail: error retrieving list with the following query ", err) - errMsg := fmt.Errorf("HandleListByPrefixWithDetail: error retrieving list, %s", err.Error()) - return c.JSON(http.StatusInternalServerError, errMsg.Error()) - } - - for _, cp := range resp.CommonPrefixes { - w := ListResult{ + processPage := func(page *s3.ListObjectsV2Output) error { + for _, cp := range page.CommonPrefixes { + // Handle directories (common prefixes) + dir := ListResult{ ID: count, - Name: filepath.Base(*cp.Prefix), - Size: "", + Bucket: bucket, + Name: filepath.Base(strings.TrimSuffix(*cp.Prefix, "/")), Path: *cp.Prefix, - Type: "", IsDir: true, - ModifiedBy: "", + Modified: time.Time{}, // Directory might not have a modified time + ModifiedBy: "", // Information might not be available } + results = append(results, dir) count++ - result = append(result, w) } - for _, object := range resp.Contents { - parts := strings.Split(filepath.Dir(*object.Key), "/") - isSelf := filepath.Base(*object.Key) == parts[len(parts)-1] - - if !isSelf { - w := ListResult{ - ID: count, - Name: filepath.Base(*object.Key), - Size: strconv.FormatInt(*object.Size, 10), - Path: filepath.Dir(*object.Key), - Type: filepath.Ext(*object.Key), - IsDir: false, - Modified: *object.LastModified, - ModifiedBy: "", - } - - count++ - result = append(result, w) + for _, object := range page.Contents { + // Handle files + file := ListResult{ + ID: count, + Bucket: bucket, + Name: filepath.Base(*object.Key), + Size: strconv.FormatInt(*object.Size, 10), + Path: *object.Key, + Type: filepath.Ext(*object.Key), + IsDir: false, + Modified: *object.LastModified, + ModifiedBy: "", // Information might not be available } + results = append(results, file) + count++ } + return nil + } - query.ContinuationToken = resp.NextContinuationToken - truncatedListing = *resp.IsTruncated + err = s3Ctrl.GetListWithCallBack(bucket, prefix, true, processPage) + if err != nil { + log.Error("HandleListByPrefixWithDetail: Error processing objects:", err) + return c.JSON(http.StatusInternalServerError, fmt.Sprintf("Error processing objects: %v", err)) } - log.Info("HandleListByPrefix: Successfully retrieved list by prefix with detail:", prefix) - return c.JSON(http.StatusOK, result) + log.Info("HandleListByPrefixWithDetail: Successfully retrieved detailed list by prefix:", prefix) + return c.JSON(http.StatusOK, results) } // GetList retrieves a list of objects in the specified S3 bucket with the given prefix. // if delimiter is set to true then it is going to search for any objects within the prefix provided, if no object sare found it will // return null even if there was prefixes within the user provided prefix. If delimiter is set to false then it will look for all prefixes // that start with the user provided prefix. +func (s3Ctrl *S3Controller) GetListWithCallBack(bucket, prefix string, delimiter bool, processPage func(*s3.ListObjectsV2Output) error) error { + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(prefix), + MaxKeys: aws.Int64(1000), // Adjust the MaxKeys as needed + } + + if delimiter { + input.SetDelimiter("/") + } + + var lastError error // Variable to capture the last error + + // Iterate over the pages of results + err := s3Ctrl.S3Svc.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, _ bool) bool { + lastError = processPage(page) + return lastError == nil && *page.IsTruncated // Continue if no error and more pages are available + }) + + if lastError != nil { + return lastError // Return the last error encountered in the processPage function + } + return err // Return any errors encountered in the pagination process +} + func (s3Ctrl *S3Controller) GetList(bucket, prefix string, delimiter bool) (*s3.ListObjectsV2Output, error) { // Set up input parameters for the ListObjectsV2 API input := &s3.ListObjectsV2Input{ From 91445183f5fb4974119f17f2f4d405c080b33772 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 26 Jan 2024 16:40:55 -0500 Subject: [PATCH 02/27] fix get size to use new get list --- blobstore/metadata.go | 61 +++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/blobstore/metadata.go b/blobstore/metadata.go index 03c0f78..2e461b7 100644 --- a/blobstore/metadata.go +++ b/blobstore/metadata.go @@ -12,25 +12,22 @@ import ( log "github.com/sirupsen/logrus" ) -func (bh *BlobHandler) GetSize(list *s3.ListObjectsV2Output) (uint64, uint32, error) { - if list == nil { - return 0, 0, errors.New("getSize: input list is nil") +func (bh *BlobHandler) GetSize(page *s3.ListObjectsV2Output, totalSize *uint64, fileCount *uint64) error { + if page == nil { + return errors.New("input page is nil") } - var size uint64 = 0 - fileCount := uint32(len(list.Contents)) - - for _, file := range list.Contents { + for _, file := range page.Contents { if file.Size == nil { - return 0, 0, errors.New("getSize: file size is nil") + return errors.New("file size is nil") } - size += uint64(*file.Size) + *totalSize += uint64(*file.Size) + *fileCount++ } - return size, fileCount, nil + return nil } -// HandleGetSize retrieves the total size and the number of files in the specified S3 bucket with the given prefix. // HandleGetSize retrieves the total size and the number of files in the specified S3 bucket with the given prefix. func (bh *BlobHandler) HandleGetSize(c echo.Context) error { prefix := c.QueryParam("prefix") @@ -51,42 +48,44 @@ func (bh *BlobHandler) HandleGetSize(c echo.Context) error { // Check if the prefix points directly to an object isObject, err := s3Ctrl.KeyExists(bucket, prefix) if err != nil { - log.Error("HandleGetSize: Error checking if prefix is an object:", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error checking if prefix is an object: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } if isObject { - // Prefix points directly to an object instead of a collection of objects - return c.JSON(http.StatusTeapot, "The provided prefix points to a single object rather than a collection") - } - list, err := s3Ctrl.GetList(bucket, prefix, false) - if err != nil { - log.Error("HandleGetSize: Error getting list:", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("the provided prefix %s points to a single object rather than a collection", prefix) + log.Error(errMsg.Error()) + return c.JSON(http.StatusTeapot, errMsg.Error()) } - if len(list.Contents) == 0 { - // No objects found with the provided prefix - return c.JSON(http.StatusNotFound, "Prefix not found") - } + var totalSize uint64 + var fileCount uint64 + err = s3Ctrl.GetListWithCallBack(bucket, prefix, false, func(page *s3.ListObjectsV2Output) error { + return bh.GetSize(page, &totalSize, &fileCount) + }) - size, fileCount, err := bh.GetSize(list) if err != nil { - log.Error("HandleGetSize: Error getting size:", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error processing objects: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) + } + if totalSize == 0 { + errMsg := fmt.Errorf("prefix %s not found", prefix) + log.Error(errMsg.Error()) + return c.JSON(http.StatusNotFound, errMsg.Error()) } - response := struct { Size uint64 `json:"size"` - FileCount uint32 `json:"file_count"` + FileCount uint64 `json:"file_count"` Prefix string `json:"prefix"` }{ - Size: size, + Size: totalSize, FileCount: fileCount, Prefix: prefix, } - log.Info("HandleGetSize: Successfully retrieved size for prefix:", prefix) + log.Info("Successfully retrieved size for prefix:", prefix) return c.JSON(http.StatusOK, response) } From 3bd4cce77b7d3a7ed87d34f647156b4b24f1a258 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 26 Jan 2024 22:09:33 -0500 Subject: [PATCH 03/27] fix delete prefix to use new list functionality --- blobstore/delete.go | 74 ++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 45 deletions(-) diff --git a/blobstore/delete.go b/blobstore/delete.go index 97ab3e7..c4b4b86 100644 --- a/blobstore/delete.go +++ b/blobstore/delete.go @@ -13,39 +13,37 @@ import ( ) func (s3Ctrl *S3Controller) RecursivelyDeleteObjects(bucket, prefix string) error { - prefixPath := strings.Trim(prefix, "/") + "/" - query := &s3.ListObjectsV2Input{ - Bucket: aws.String(bucket), - Prefix: aws.String(prefixPath), - } - resp, err := s3Ctrl.S3Svc.ListObjectsV2(query) - if err != nil { - return fmt.Errorf("recursivelyDeleteObjects: error listing objects: %s", err) - } - if len(resp.Contents) > 0 { - var objectsToDelete []*s3.ObjectIdentifier + processPage := func(page *s3.ListObjectsV2Output) error { + if len(page.Contents) == 0 { + return nil // No objects to delete in this page + } - for _, obj := range resp.Contents { - objectsToDelete = append(objectsToDelete, &s3.ObjectIdentifier{ - Key: obj.Key, - }) + var objectsToDelete []*s3.ObjectIdentifier + for _, obj := range page.Contents { + objectsToDelete = append(objectsToDelete, &s3.ObjectIdentifier{Key: obj.Key}) } - if len(objectsToDelete) > 0 { - _, err = s3Ctrl.S3Svc.DeleteObjects(&s3.DeleteObjectsInput{ - Bucket: aws.String(bucket), - Delete: &s3.Delete{ - Objects: objectsToDelete, - }, - }) - - if err != nil { - return fmt.Errorf("recursivelyDeleteObjects: error Deleting objects %v: %s", objectsToDelete, err) - } + // Perform the delete operation for the current page + _, err := s3Ctrl.S3Svc.DeleteObjects(&s3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &s3.Delete{ + Objects: objectsToDelete, + Quiet: aws.Bool(true), + }, + }) + if err != nil { + return fmt.Errorf("error deleting objects: %v", err) } - } else { - return fmt.Errorf("recursivelyDeleteObjects: object %s not found and no objects were deleted", prefixPath) + + log.Infof("Successfully deleted %d objects", len(objectsToDelete)) + return nil + } + + err := s3Ctrl.GetListWithCallBack(bucket, prefix, false, processPage) + if err != nil { + return fmt.Errorf("error processing objects for deletion: %v", err) } + return nil } @@ -104,7 +102,6 @@ func (bh *BlobHandler) HandleDeletePrefix(c echo.Context) error { log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } - prefix := c.QueryParam("prefix") if prefix == "" { err = errors.New("parameter 'prefix' is required") @@ -114,27 +111,14 @@ func (bh *BlobHandler) HandleDeletePrefix(c echo.Context) error { if !strings.HasSuffix(prefix, "/") { prefix = prefix + "/" } - response, err := s3Ctrl.GetList(bucket, prefix, false) - if err != nil { - log.Errorf("HandleDeleteObjects: Error getting list: %s", err.Error()) - return c.JSON(http.StatusInternalServerError, err) - } - if *response.KeyCount == 0 { - err := fmt.Errorf("the specified prefix %s does not exist in S3", prefix) - log.Errorf("HandleDeleteObjects: %s", err.Error()) - return c.JSON(http.StatusNotFound, err.Error()) - } - // This will recursively delete all objects with the specified prefix err = s3Ctrl.RecursivelyDeleteObjects(bucket, prefix) if err != nil { - msg := fmt.Sprintf("error deleting objects. %s", err.Error()) - log.Errorf("HandleDeleteObjects: %s", msg) - return c.JSON(http.StatusInternalServerError, msg) + errMsg := fmt.Errorf("error deleting objects: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } - log.Info("HandleDeleteObjects: Successfully deleted prefix and its contents for prefix:", prefix) return c.JSON(http.StatusOK, "Successfully deleted prefix and its contents") - } func (s3Ctrl *S3Controller) DeleteKeys(bucket string, key []string) error { From 72f052a6d6900d5ee0f076d99b90c3105ed86df0 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Mon, 29 Jan 2024 10:46:03 -0500 Subject: [PATCH 04/27] refactor page processor in delete prefix --- blobstore/delete.go | 51 ++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/blobstore/delete.go b/blobstore/delete.go index c4b4b86..0115029 100644 --- a/blobstore/delete.go +++ b/blobstore/delete.go @@ -12,34 +12,37 @@ import ( log "github.com/sirupsen/logrus" ) -func (s3Ctrl *S3Controller) RecursivelyDeleteObjects(bucket, prefix string) error { - processPage := func(page *s3.ListObjectsV2Output) error { - if len(page.Contents) == 0 { - return nil // No objects to delete in this page - } - - var objectsToDelete []*s3.ObjectIdentifier - for _, obj := range page.Contents { - objectsToDelete = append(objectsToDelete, &s3.ObjectIdentifier{Key: obj.Key}) - } +func (s3Ctrl *S3Controller) DeleteList(page *s3.ListObjectsV2Output, bucket string) error { + if len(page.Contents) == 0 { + return nil // No objects to delete in this page + } - // Perform the delete operation for the current page - _, err := s3Ctrl.S3Svc.DeleteObjects(&s3.DeleteObjectsInput{ - Bucket: aws.String(bucket), - Delete: &s3.Delete{ - Objects: objectsToDelete, - Quiet: aws.Bool(true), - }, - }) - if err != nil { - return fmt.Errorf("error deleting objects: %v", err) - } + var objectsToDelete []*s3.ObjectIdentifier + for _, obj := range page.Contents { + objectsToDelete = append(objectsToDelete, &s3.ObjectIdentifier{Key: obj.Key}) + } - log.Infof("Successfully deleted %d objects", len(objectsToDelete)) - return nil + // Perform the delete operation for the current page + _, err := s3Ctrl.S3Svc.DeleteObjects(&s3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &s3.Delete{ + Objects: objectsToDelete, + Quiet: aws.Bool(true), + }, + }) + if err != nil { + return fmt.Errorf("error deleting objects: %v", err) } - err := s3Ctrl.GetListWithCallBack(bucket, prefix, false, processPage) + log.Infof("Successfully deleted %d objects", len(objectsToDelete)) + return nil +} + +func (s3Ctrl *S3Controller) RecursivelyDeleteObjects(bucket, prefix string) error { + + err := s3Ctrl.GetListWithCallBack(bucket, prefix, false, func(page *s3.ListObjectsV2Output) error { + return s3Ctrl.DeleteList(page, bucket) + }) if err != nil { return fmt.Errorf("error processing objects for deletion: %v", err) } From 1539cf5b6388f49e460bb53310274709a0dd12a8 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Mon, 29 Jan 2024 10:46:30 -0500 Subject: [PATCH 05/27] make move prefix use teh get list with call back --- blobstore/move.go | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/blobstore/move.go b/blobstore/move.go index b84e798..f93c821 100644 --- a/blobstore/move.go +++ b/blobstore/move.go @@ -47,27 +47,36 @@ func (bh *BlobHandler) HandleMovePrefix(c echo.Context) error { } func (s3Ctrl *S3Controller) CopyPrefix(bucket, srcPrefix, destPrefix string) error { - // List objects within the source prefix - listOutput, err := s3Ctrl.GetList(bucket, srcPrefix, true) - if err != nil { - return errors.New("error listing objects with prefix " + srcPrefix + " in bucket " + bucket + ", " + err.Error()) - } + processPage := func(page *s3.ListObjectsV2Output) error { + for _, object := range page.Contents { + srcObjectKey := aws.StringValue(object.Key) + destObjectKey := strings.Replace(srcObjectKey, srcPrefix, destPrefix, 1) + + // Copy the object to the new location + copyInput := &s3.CopyObjectInput{ + Bucket: aws.String(bucket), + CopySource: aws.String(bucket + "/" + srcObjectKey), + Key: aws.String(destObjectKey), + } + _, err := s3Ctrl.S3Svc.CopyObject(copyInput) + if err != nil { + return fmt.Errorf("error copying object %s to %s: %v", srcObjectKey, destObjectKey, err) + } - if len(listOutput.Contents) == 0 { - return errors.New("source prefix " + srcPrefix + " does not exist") + } + err := s3Ctrl.DeleteList(page, bucket) + if err != nil { + errMsg := fmt.Errorf("error deleting from source prefix %s: %v", srcPrefix, err) + return errMsg + } + return nil } - // Copy each object to the destination prefix - for _, object := range listOutput.Contents { - srcObjectKey := aws.StringValue(object.Key) - destObjectKey := strings.Replace(srcObjectKey, srcPrefix, destPrefix, 1) - - copyErr := s3Ctrl.CopyObject(bucket, srcObjectKey, destObjectKey) - if copyErr != nil { - // If an error occurs during copying, return immediately - return copyErr - } + err := s3Ctrl.GetListWithCallBack(bucket, srcPrefix, false, processPage) + if err != nil { + return fmt.Errorf("error processing objects for move: %v", err) } + return nil } From 4fafd403f21bdb30f80f46bdf8366a7895e9815b Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Mon, 29 Jan 2024 20:21:08 -0500 Subject: [PATCH 06/27] Change download script presigned url to use get list with call back --- blobstore/presigned_url.go | 121 ++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 70 deletions(-) diff --git a/blobstore/presigned_url.go b/blobstore/presigned_url.go index df1721d..af31590 100644 --- a/blobstore/presigned_url.go +++ b/blobstore/presigned_url.go @@ -183,7 +183,8 @@ func (bh *BlobHandler) HandleGetPresignedURLMultiObj(c echo.Context) error { return c.JSON(http.StatusNotFound, errMsg.Error()) } //check if size is below 5GB - size, _, err := bh.GetSize(response) + var size, fileCount uint64 + err = bh.GetSize(response, &size, &fileCount) if err != nil { errMsg := fmt.Errorf("error getting size: %s", err.Error()) log.Error(errMsg.Error()) @@ -248,53 +249,20 @@ func (bh *BlobHandler) HandleGetPresignedURLMultiObj(c echo.Context) error { func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { prefix := c.QueryParam("prefix") - bucket := c.QueryParam("bucket") - if prefix == "" || bucket == "" { - errMsg := fmt.Errorf("`prefix` and `bucket` query params are required") - log.Error(errMsg.Error()) - return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) - } - if !strings.HasSuffix(prefix, "/") { - prefix += "/" + if prefix == "" { + return c.JSON(http.StatusBadRequest, "Prefix parameter is required") } + bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("error getting controller for bucket %s: %s", bucket, err) - log.Error(errMsg.Error()) - return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) - } - - //list objects within the prefix and test if empty - response, err := s3Ctrl.GetList(bucket, prefix, false) - if err != nil { - errMsg := fmt.Errorf("error listing objects in bucket %s with prefix %s: %s", bucket, prefix, err) - log.Error(errMsg.Error()) - return c.JSON(http.StatusInternalServerError, errMsg.Error()) - } - if len(response.Contents) == 0 { - errMsg := fmt.Errorf("prefix %s is empty or does not exist", prefix) - log.Error(errMsg.Error()) - return c.JSON(http.StatusBadRequest, errMsg.Error()) - } - size, _, err := bh.GetSize(response) - if err != nil { - errMsg := fmt.Errorf("error retrieving size of prefix: %s", err.Error()) - log.Error(errMsg.Error()) - return c.JSON(http.StatusInternalServerError, errMsg.Error()) + return c.JSON(http.StatusInternalServerError, fmt.Sprintf("Error getting controller for bucket %s: %v", bucket, err)) } - limit := uint64(1024 * 1024 * 1024 * bh.Config.DefaultScriptDownloadSizeLimit) - if size > limit { - errMsg := fmt.Errorf("request entity is larger than %v GB, current prefix size is: %v GB", bh.Config.DefaultScriptDownloadSizeLimit, float64(size)/(1024*1024*1024)) - log.Error(errMsg.Error()) - return c.JSON(http.StatusRequestEntityTooLarge, errMsg.Error()) - } - - //expiration period from the env - + var totalSize uint64 var scriptBuilder strings.Builder createdDirs := make(map[string]bool) - // Add download instructions at the beginning of the script + basePrefix := filepath.Base(strings.TrimSuffix(prefix, "/")) + scriptBuilder.WriteString(fmt.Sprintf("mkdir \"%s\"\n", basePrefix)) scriptBuilder.WriteString("REM Download Instructions\n") scriptBuilder.WriteString("REM To download the selected directory or file, please follow these steps:\n\n") scriptBuilder.WriteString("REM 1. Locate the Downloaded File: Find the file you just downloaded. It should have a .txt file extension.\n") @@ -302,41 +270,54 @@ func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { scriptBuilder.WriteString("REM 3. Rename the File: Right-click on the file, select \"Rename,\" and change the file extension from \".txt\" to \".bat.\" For example, if the file is named \"script.txt,\" rename it to \"script.bat.\"\n") scriptBuilder.WriteString("REM 4. Initiate the Download: Double-click the renamed \".bat\" file to initiate the download process. Windows might display a warning message to protect your PC.\n") scriptBuilder.WriteString("REM 5. Windows Defender SmartScreen (Optional): If you see a message like \"Windows Defender SmartScreen prevented an unrecognized app from starting,\" click \"More info\" and then click \"Run anyway\" to proceed with the download.\n\n") - //iterate over every object and check if it has any sub-prefixes to maintain a directory structure - - basePrefix := filepath.Base(strings.TrimSuffix(prefix, "/")) scriptBuilder.WriteString(fmt.Sprintf("mkdir \"%s\"\n", basePrefix)) - for _, item := range response.Contents { - // Remove the prefix up to the base, keeping the structure under the base prefix - relativePath := strings.TrimPrefix(*item.Key, filepath.Dir(prefix)+"/") - - // Calculate the directory path for the relative path - dirPath := filepath.Join(basePrefix, filepath.Dir(relativePath)) + // Define the processPage function + processPage := func(page *s3.ListObjectsV2Output) error { + fmt.Println("getting here?") + for _, item := range page.Contents { + fmt.Println("getting here 2? ", item) + // Size checking + if item.Size != nil { + fmt.Println("getting here 3? ") + totalSize += uint64(*item.Size) + if totalSize > uint64(bh.Config.DefaultScriptDownloadSizeLimit*1024*1024*1024) { + return fmt.Errorf("size limit of %d GB exceeded", bh.Config.DefaultScriptDownloadSizeLimit) + } - // Create directory if it does not exist and is not the root - if _, exists := createdDirs[dirPath]; !exists && dirPath != basePrefix { - scriptBuilder.WriteString(fmt.Sprintf("mkdir \"%s\"\n", dirPath)) - createdDirs[dirPath] = true - } + } - // Create the full path for the object including the base prefix - fullPath := filepath.Join(basePrefix, relativePath) + fmt.Println(totalSize) + // Script generation logic (replicating your directory creation and URL logic) + relativePath := strings.TrimPrefix(*item.Key, filepath.Dir(prefix)+"/") + dirPath := filepath.Join(basePrefix, filepath.Dir(relativePath)) + if _, exists := createdDirs[dirPath]; !exists && dirPath != basePrefix { + scriptBuilder.WriteString(fmt.Sprintf("mkdir \"%s\"\n", dirPath)) + createdDirs[dirPath] = true + } - presignedURL, err := s3Ctrl.GetDownloadPresignedURL(bucket, *item.Key, bh.Config.DefaultDownloadPresignedUrlExpiration) - if err != nil { - errMsg := fmt.Errorf("error generating presigned URL for %s: %s", *item.Key, err) - log.Error(errMsg.Error()) - return c.JSON(http.StatusInternalServerError, errMsg.Error()) - } - url, err := url.QueryUnescape(presignedURL) //to remove url encoding which causes errors when executed in terminal - if err != nil { - errMsg := fmt.Errorf("error Unescaping url encoding: %s", err.Error()) - log.Error(errMsg.Error()) - return c.JSON(http.StatusInternalServerError, errMsg.Error()) + fullPath := filepath.Join(basePrefix, relativePath) + presignedURL, err := s3Ctrl.GetDownloadPresignedURL(bucket, *item.Key, bh.Config.DefaultDownloadPresignedUrlExpiration) + if err != nil { + return fmt.Errorf("error generating presigned URL for object %s: %v", *item.Key, err) + } + url, err := url.QueryUnescape(presignedURL) + if err != nil { + return fmt.Errorf("error unescaping URL encoding: %v", err) + } + encodedURL := strings.ReplaceAll(url, " ", "%20") + fmt.Println(presignedURL) + scriptBuilder.WriteString(fmt.Sprintf("if exist \"%s\" (echo skipping existing file) else (curl -v -o \"%s\" \"%s\")\n", fullPath, fullPath, encodedURL)) } - encodedURL := strings.ReplaceAll(url, " ", "%20") - scriptBuilder.WriteString(fmt.Sprintf("if exist \"%s\" (echo skipping existing file) else (curl -v -o \"%s\" \"%s\")\n", fullPath, fullPath, encodedURL)) + return nil + } + + // Call GetList with the processPage function + err = s3Ctrl.GetListWithCallBack(bucket, prefix, false, processPage) + fmt.Println(err) + if err != nil { + // Handle errors, including size limit exceeded + return c.JSON(http.StatusInternalServerError, fmt.Errorf("error processing objects: %v", err).Error()) } txtBatFileName := fmt.Sprintf("%s_download_script.txt", strings.TrimSuffix(prefix, "/")) From c987ad858b1c1d6dd5d8b5c8954369adeabe3080 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Mon, 29 Jan 2024 20:31:40 -0500 Subject: [PATCH 07/27] check if prefix is empty --- blobstore/delete.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/blobstore/delete.go b/blobstore/delete.go index 0115029..262c810 100644 --- a/blobstore/delete.go +++ b/blobstore/delete.go @@ -39,14 +39,20 @@ func (s3Ctrl *S3Controller) DeleteList(page *s3.ListObjectsV2Output, bucket stri } func (s3Ctrl *S3Controller) RecursivelyDeleteObjects(bucket, prefix string) error { - + var objectsFound bool err := s3Ctrl.GetListWithCallBack(bucket, prefix, false, func(page *s3.ListObjectsV2Output) error { + if len(page.Contents) > 0 { + objectsFound = true + } return s3Ctrl.DeleteList(page, bucket) }) if err != nil { return fmt.Errorf("error processing objects for deletion: %v", err) } + if !objectsFound { + return fmt.Errorf("prefix not found") + } return nil } @@ -116,6 +122,10 @@ func (bh *BlobHandler) HandleDeletePrefix(c echo.Context) error { } err = s3Ctrl.RecursivelyDeleteObjects(bucket, prefix) if err != nil { + if strings.Contains(err.Error(), "prefix not found") { + log.Infof("No objects found with prefix: %s", prefix) + return c.JSON(http.StatusNotFound, "Prefix not found") + } errMsg := fmt.Errorf("error deleting objects: %s", err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusInternalServerError, errMsg.Error()) From 36d1be29171ad20cdc95167c36d45239224d8043 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Mon, 29 Jan 2024 20:38:24 -0500 Subject: [PATCH 08/27] add prefix not found for prefix move --- blobstore/move.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/blobstore/move.go b/blobstore/move.go index f93c821..1bbf642 100644 --- a/blobstore/move.go +++ b/blobstore/move.go @@ -34,11 +34,11 @@ func (bh *BlobHandler) HandleMovePrefix(c echo.Context) error { log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } - err = s3Ctrl.CopyPrefix(bucket, srcPrefix, destPrefix) if err != nil { - if strings.Contains(err.Error(), "does not exist") { - return c.JSON(http.StatusNotFound, err.Error()) + if strings.Contains(err.Error(), "source prefix not found") { + log.Infof("No objects found with source prefix: %s", srcPrefix) + return c.JSON(http.StatusNotFound, "Source prefix not found") } return c.JSON(http.StatusInternalServerError, err.Error()) } @@ -47,7 +47,14 @@ func (bh *BlobHandler) HandleMovePrefix(c echo.Context) error { } func (s3Ctrl *S3Controller) CopyPrefix(bucket, srcPrefix, destPrefix string) error { + var objectsFound bool + processPage := func(page *s3.ListObjectsV2Output) error { + if len(page.Contents) == 0 { + return nil + } + objectsFound = true // Objects found, set the flag + for _, object := range page.Contents { srcObjectKey := aws.StringValue(object.Key) destObjectKey := strings.Replace(srcObjectKey, srcPrefix, destPrefix, 1) @@ -69,6 +76,10 @@ func (s3Ctrl *S3Controller) CopyPrefix(bucket, srcPrefix, destPrefix string) err errMsg := fmt.Errorf("error deleting from source prefix %s: %v", srcPrefix, err) return errMsg } + if !objectsFound { + return fmt.Errorf("source prefix not found") + } + return nil } From 37d5c8f73c0c719c417cc40e07e208cd242b1ecb Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Mon, 29 Jan 2024 21:02:05 -0500 Subject: [PATCH 09/27] fix prefix exists checker --- blobstore/move.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/blobstore/move.go b/blobstore/move.go index 1bbf642..21e877c 100644 --- a/blobstore/move.go +++ b/blobstore/move.go @@ -51,7 +51,7 @@ func (s3Ctrl *S3Controller) CopyPrefix(bucket, srcPrefix, destPrefix string) err processPage := func(page *s3.ListObjectsV2Output) error { if len(page.Contents) == 0 { - return nil + return nil // No objects to process in this page } objectsFound = true // Objects found, set the flag @@ -69,17 +69,15 @@ func (s3Ctrl *S3Controller) CopyPrefix(bucket, srcPrefix, destPrefix string) err if err != nil { return fmt.Errorf("error copying object %s to %s: %v", srcObjectKey, destObjectKey, err) } - } + + // Deleting the source objects should be handled carefully + // Ensure that your application logic requires this before proceeding err := s3Ctrl.DeleteList(page, bucket) if err != nil { errMsg := fmt.Errorf("error deleting from source prefix %s: %v", srcPrefix, err) return errMsg } - if !objectsFound { - return fmt.Errorf("source prefix not found") - } - return nil } @@ -88,6 +86,11 @@ func (s3Ctrl *S3Controller) CopyPrefix(bucket, srcPrefix, destPrefix string) err return fmt.Errorf("error processing objects for move: %v", err) } + // Check if objects were found after processing all pages + if !objectsFound { + return fmt.Errorf("source prefix not found") + } + return nil } From ae80715c8599312cfc85392f787aa23d9c10de14 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Tue, 30 Jan 2024 10:34:24 -0500 Subject: [PATCH 10/27] code clean up --- blobstore/delete.go | 9 +++---- blobstore/list.go | 52 ++++++++++++++++++++------------------ blobstore/presigned_url.go | 19 +++++++------- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/blobstore/delete.go b/blobstore/delete.go index 262c810..568f7a8 100644 --- a/blobstore/delete.go +++ b/blobstore/delete.go @@ -34,7 +34,6 @@ func (s3Ctrl *S3Controller) DeleteList(page *s3.ListObjectsV2Output, bucket stri return fmt.Errorf("error deleting objects: %v", err) } - log.Infof("Successfully deleted %d objects", len(objectsToDelete)) return nil } @@ -107,14 +106,14 @@ func (bh *BlobHandler) HandleDeletePrefix(c echo.Context) error { bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("parameter `bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } prefix := c.QueryParam("prefix") if prefix == "" { - err = errors.New("parameter 'prefix' is required") - log.Errorf("HandleDeleteObjects: %s", err.Error()) + err = errors.New("parameter `prefix` is required") + log.Error(err.Error()) return c.JSON(http.StatusUnprocessableEntity, err.Error()) } if !strings.HasSuffix(prefix, "/") { @@ -130,7 +129,7 @@ func (bh *BlobHandler) HandleDeletePrefix(c echo.Context) error { log.Error(errMsg.Error()) return c.JSON(http.StatusInternalServerError, errMsg.Error()) } - log.Info("HandleDeleteObjects: Successfully deleted prefix and its contents for prefix:", prefix) + log.Info("Successfully deleted prefix and its contents for prefix:", prefix) return c.JSON(http.StatusOK, "Successfully deleted prefix and its contents") } diff --git a/blobstore/list.go b/blobstore/list.go index 7295aba..3e7f8a4 100644 --- a/blobstore/list.go +++ b/blobstore/list.go @@ -192,31 +192,6 @@ func (bh *BlobHandler) HandleListByPrefixWithDetail(c echo.Context) error { // if delimiter is set to true then it is going to search for any objects within the prefix provided, if no object sare found it will // return null even if there was prefixes within the user provided prefix. If delimiter is set to false then it will look for all prefixes // that start with the user provided prefix. -func (s3Ctrl *S3Controller) GetListWithCallBack(bucket, prefix string, delimiter bool, processPage func(*s3.ListObjectsV2Output) error) error { - input := &s3.ListObjectsV2Input{ - Bucket: aws.String(bucket), - Prefix: aws.String(prefix), - MaxKeys: aws.Int64(1000), // Adjust the MaxKeys as needed - } - - if delimiter { - input.SetDelimiter("/") - } - - var lastError error // Variable to capture the last error - - // Iterate over the pages of results - err := s3Ctrl.S3Svc.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, _ bool) bool { - lastError = processPage(page) - return lastError == nil && *page.IsTruncated // Continue if no error and more pages are available - }) - - if lastError != nil { - return lastError // Return the last error encountered in the processPage function - } - return err // Return any errors encountered in the pagination process -} - func (s3Ctrl *S3Controller) GetList(bucket, prefix string, delimiter bool) (*s3.ListObjectsV2Output, error) { // Set up input parameters for the ListObjectsV2 API input := &s3.ListObjectsV2Input{ @@ -251,3 +226,30 @@ func (s3Ctrl *S3Controller) GetList(bucket, prefix string, delimiter bool) (*s3. return response, nil } + +// GetListWithCallBack is the same as GetList, except instead of returning the entire list at once, it gives you the option of processing page by page +// this method is safer than GetList as it avoid memory overload for large datasets since it does not store the entire list in memory but rather processes it on the go. +func (s3Ctrl *S3Controller) GetListWithCallBack(bucket, prefix string, delimiter bool, processPage func(*s3.ListObjectsV2Output) error) error { + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(prefix), + MaxKeys: aws.Int64(1000), // Adjust the MaxKeys as needed + } + + if delimiter { + input.SetDelimiter("/") + } + + var lastError error // Variable to capture the last error + + // Iterate over the pages of results + err := s3Ctrl.S3Svc.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, _ bool) bool { + lastError = processPage(page) + return lastError == nil && *page.IsTruncated // Continue if no error and more pages are available + }) + + if lastError != nil { + return lastError // Return the last error encountered in the processPage function + } + return err // Return any errors encountered in the pagination process +} diff --git a/blobstore/presigned_url.go b/blobstore/presigned_url.go index af31590..879f506 100644 --- a/blobstore/presigned_url.go +++ b/blobstore/presigned_url.go @@ -250,12 +250,16 @@ func (bh *BlobHandler) HandleGetPresignedURLMultiObj(c echo.Context) error { func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { prefix := c.QueryParam("prefix") if prefix == "" { - return c.JSON(http.StatusBadRequest, "Prefix parameter is required") + errMsg := fmt.Errorf("`prefix` and `bucket` query params are required") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - return c.JSON(http.StatusInternalServerError, fmt.Sprintf("Error getting controller for bucket %s: %v", bucket, err)) + errMsg := fmt.Errorf("error getting controller for bucket %s: %s", bucket, err) + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } var totalSize uint64 @@ -274,12 +278,9 @@ func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { // Define the processPage function processPage := func(page *s3.ListObjectsV2Output) error { - fmt.Println("getting here?") for _, item := range page.Contents { - fmt.Println("getting here 2? ", item) // Size checking if item.Size != nil { - fmt.Println("getting here 3? ") totalSize += uint64(*item.Size) if totalSize > uint64(bh.Config.DefaultScriptDownloadSizeLimit*1024*1024*1024) { return fmt.Errorf("size limit of %d GB exceeded", bh.Config.DefaultScriptDownloadSizeLimit) @@ -287,7 +288,6 @@ func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { } - fmt.Println(totalSize) // Script generation logic (replicating your directory creation and URL logic) relativePath := strings.TrimPrefix(*item.Key, filepath.Dir(prefix)+"/") dirPath := filepath.Join(basePrefix, filepath.Dir(relativePath)) @@ -306,7 +306,6 @@ func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { return fmt.Errorf("error unescaping URL encoding: %v", err) } encodedURL := strings.ReplaceAll(url, " ", "%20") - fmt.Println(presignedURL) scriptBuilder.WriteString(fmt.Sprintf("if exist \"%s\" (echo skipping existing file) else (curl -v -o \"%s\" \"%s\")\n", fullPath, fullPath, encodedURL)) } return nil @@ -314,10 +313,10 @@ func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { // Call GetList with the processPage function err = s3Ctrl.GetListWithCallBack(bucket, prefix, false, processPage) - fmt.Println(err) if err != nil { - // Handle errors, including size limit exceeded - return c.JSON(http.StatusInternalServerError, fmt.Errorf("error processing objects: %v", err).Error()) + errMsg := fmt.Errorf("error processing objects: %v", err) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } txtBatFileName := fmt.Sprintf("%s_download_script.txt", strings.TrimSuffix(prefix, "/")) From b1623f4556e4846bd95c044a85be9e53f4c54017 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Tue, 30 Jan 2024 13:54:21 -0500 Subject: [PATCH 11/27] make error handeling uniform --- blobstore/buckets.go | 4 +- blobstore/delete.go | 74 ++++++++++++++++----------------- blobstore/list.go | 67 ++++++++++++++++-------------- blobstore/metadata.go | 44 ++++++++++---------- blobstore/move.go | 32 ++++++++------- blobstore/object_content.go | 18 ++++----- blobstore/presigned_url.go | 81 ++++++++++++++++++++----------------- blobstore/upload.go | 67 +++++++++++++++--------------- 8 files changed, 204 insertions(+), 183 deletions(-) diff --git a/blobstore/buckets.go b/blobstore/buckets.go index b2d12c1..c4df444 100644 --- a/blobstore/buckets.go +++ b/blobstore/buckets.go @@ -112,9 +112,7 @@ func (bh *BlobHandler) HandleListBuckets(c echo.Context) error { } } bh.Mu.Unlock() - - log.Info("HandleListBuckets: Successfully retrieved list of buckets") - + log.Info("Successfully retrieved list of buckets") return c.JSON(http.StatusOK, allBuckets) } diff --git a/blobstore/delete.go b/blobstore/delete.go index 568f7a8..4a0e8a9 100644 --- a/blobstore/delete.go +++ b/blobstore/delete.go @@ -1,7 +1,6 @@ package blobstore import ( - "errors" "fmt" "net/http" "strings" @@ -62,28 +61,29 @@ func (bh *BlobHandler) HandleDeleteObject(c echo.Context) error { bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("parameter `bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } key := c.QueryParam("key") if key == "" { - err := errors.New("parameter 'key' is required") - log.Errorf("HandleDeleteObjects: %s", err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("parameter `key` is required") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } // If the key is not a folder, proceed with deleting a single object keyExist, err := s3Ctrl.KeyExists(bucket, key) if err != nil { - log.Errorf("HandleDeleteObjects: Error checking if key exists: %s", err.Error()) - return c.JSON(http.StatusInternalServerError, err) + errMsg := fmt.Errorf("error checking if key exists: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } if !keyExist { - err := fmt.Errorf("object %s not found", key) - log.Errorf("HandleDeleteObjects: %s", err.Error()) - return c.JSON(http.StatusNotFound, err.Error()) + errMsg := fmt.Errorf("object %s not found", key) + log.Error(errMsg.Error()) + return c.JSON(http.StatusNotFound, errMsg.Error()) } deleteInput := &s3.DeleteObjectInput{ @@ -93,12 +93,12 @@ func (bh *BlobHandler) HandleDeleteObject(c echo.Context) error { _, err = s3Ctrl.S3Svc.DeleteObject(deleteInput) if err != nil { - msg := fmt.Sprintf("error deleting object. %s", err.Error()) - log.Errorf("HandleDeleteObjects: %s", err.Error()) - return c.JSON(http.StatusInternalServerError, msg) + errMsg := fmt.Errorf("error deleting object. %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } - log.Info("HandleDeleteObjects: Successfully deleted file with key:", key) + log.Infof("successfully deleted file with key: %s", key) return c.JSON(http.StatusOK, fmt.Sprintf("Successfully deleted object: %s", key)) } @@ -112,9 +112,9 @@ func (bh *BlobHandler) HandleDeletePrefix(c echo.Context) error { } prefix := c.QueryParam("prefix") if prefix == "" { - err = errors.New("parameter `prefix` is required") - log.Error(err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("parameter `prefix` is required") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } if !strings.HasSuffix(prefix, "/") { prefix = prefix + "/" @@ -122,8 +122,9 @@ func (bh *BlobHandler) HandleDeletePrefix(c echo.Context) error { err = s3Ctrl.RecursivelyDeleteObjects(bucket, prefix) if err != nil { if strings.Contains(err.Error(), "prefix not found") { - log.Infof("No objects found with prefix: %s", prefix) - return c.JSON(http.StatusNotFound, "Prefix not found") + errMsg := fmt.Errorf("no objects found with prefix: %s", prefix) + log.Error(errMsg.Error()) + return c.JSON(http.StatusNotFound, errMsg.Error()) } errMsg := fmt.Errorf("error deleting objects: %s", err.Error()) log.Error(errMsg.Error()) @@ -153,7 +154,7 @@ func (s3Ctrl *S3Controller) DeleteKeys(bucket string, key []string) error { _, err := s3Ctrl.S3Svc.DeleteObjects(input) if err != nil { - return fmt.Errorf("deleteKeys: error Deleting objects: %s", err.Error()) + return fmt.Errorf("error Deleting objects: %s", err.Error()) } return nil } @@ -165,21 +166,22 @@ func (bh *BlobHandler) HandleDeleteObjectsByList(c echo.Context) error { } var deleteRequest DeleteRequest if err := c.Bind(&deleteRequest); err != nil { - log.Errorf("HandleDeleteObjectsByList: Error parsing request body: %s" + err.Error()) - return c.JSON(http.StatusBadRequest, "Invalid request body") + errMsg := fmt.Errorf("error parsing request body: %s" + err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusBadRequest, errMsg.Error()) } // Ensure there are keys to delete if len(deleteRequest.Keys) == 0 { - errMsg := "No keys to delete. Please provide 'keys' in the request body." - log.Errorf("HandleDeleteObjectsByList: %s", errMsg) - return c.JSON(http.StatusUnprocessableEntity, errMsg) + errMsg := fmt.Errorf("no keys to delete. Please provide 'keys' in the request body") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } @@ -193,14 +195,14 @@ func (bh *BlobHandler) HandleDeleteObjectsByList(c echo.Context) error { // Check if the key exists before appending it to the keys list keyExists, err := s3Ctrl.KeyExists(bucket, s3Path) if err != nil { - msg := fmt.Errorf("error checking if key exists. %s", err.Error()) - log.Errorf("HandleDeleteObjectsByList: %s", msg) - return c.JSON(http.StatusInternalServerError, msg) + errMsg := fmt.Errorf("error checking if key exists. %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg) } if !keyExists { - errMsg := fmt.Sprintf("object %s not found", s3Path) - log.Errorf("HandleDeleteObjectsByList: %s", errMsg) - return c.JSON(http.StatusNotFound, errMsg) + errMsg := fmt.Errorf("object %s not found", s3Path) + log.Error(errMsg.Error()) + return c.JSON(http.StatusNotFound, errMsg.Error()) } keys = append(keys, *key) @@ -209,11 +211,11 @@ func (bh *BlobHandler) HandleDeleteObjectsByList(c echo.Context) error { // Delete the objects using the deleteKeys function err = s3Ctrl.DeleteKeys(bucket, keys) if err != nil { - msg := fmt.Sprintf("error deleting objects. %s", err.Error()) - log.Errorf("HandleDeleteObjectsByList: %s", msg) - return c.JSON(http.StatusInternalServerError, msg) + errMsg := fmt.Errorf("error deleting objects. %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg) } - log.Info("HandleDeleteObjectsByList: Successfully deleted objects:", deleteRequest.Keys) + log.Info("Successfully deleted objects:", deleteRequest.Keys) return c.JSON(http.StatusOK, "Successfully deleted objects") } diff --git a/blobstore/list.go b/blobstore/list.go index 3e7f8a4..a150883 100644 --- a/blobstore/list.go +++ b/blobstore/list.go @@ -1,7 +1,6 @@ package blobstore import ( - "errors" "fmt" "net/http" "path/filepath" @@ -32,15 +31,15 @@ type ListResult struct { func (bh *BlobHandler) HandleListByPrefix(c echo.Context) error { prefix := c.QueryParam("prefix") if prefix == "" { - err := errors.New("request must include a `prefix` parameter") - log.Error("HandleListByPrefix: " + err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("request must include a `prefix` parameter") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } @@ -51,14 +50,15 @@ func (bh *BlobHandler) HandleListByPrefix(c echo.Context) error { var err error delimiter, err = strconv.ParseBool(delimiterParam) if err != nil { - log.Error("HandleListByPrefix: Error parsing `delimiter` param:", err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("error parsing `delimiter` param: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } } else { - err := errors.New("request must include a `delimiter`, options are `true` or `false`") - log.Error("HandleListByPrefix: " + err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("request must include a `delimiter`, options are `true` or `false`") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } if delimiter { @@ -69,22 +69,24 @@ func (bh *BlobHandler) HandleListByPrefix(c echo.Context) error { isObject, err := s3Ctrl.KeyExists(bucket, prefix) if err != nil { - log.Error("HandleListByPrefix: can't find bucket or object " + err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("can't find bucket or object %s" + err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } if isObject { objMeta, err := s3Ctrl.GetMetaData(bucket, prefix) if err != nil { - log.Error("HandleListByPrefix: " + err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error getting metadata: %s" + err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } if *objMeta.ContentLength == 0 { - log.Infof("HandleListByPrefix: Detected a zero byte directory marker within prefix: %s", prefix) + log.Infof("Detected a zero byte directory marker within prefix: %s", prefix) } else { - err = fmt.Errorf("`%s` is an object, not a prefix. please see options for keys or pass a prefix", prefix) - log.Error("HandleListByPrefix: " + err.Error()) - return c.JSON(http.StatusTeapot, err.Error()) + errMsg := fmt.Errorf("`%s` is an object, not a prefix. please see options for keys or pass a prefix", prefix) + log.Error(errMsg.Error()) + return c.JSON(http.StatusTeapot, errMsg.Error()) } } @@ -101,7 +103,7 @@ func (bh *BlobHandler) HandleListByPrefix(c echo.Context) error { return c.JSON(http.StatusInternalServerError, fmt.Sprintf("Error processing objects: %v", err)) } - log.Info("HandleListByPrefix: Successfully retrieved list by prefix:", prefix) + log.Info("Successfully retrieved list by prefix:", prefix) return c.JSON(http.StatusOK, objectKeys) } @@ -112,7 +114,7 @@ func (bh *BlobHandler) HandleListByPrefixWithDetail(c echo.Context) error { bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } @@ -120,21 +122,23 @@ func (bh *BlobHandler) HandleListByPrefixWithDetail(c echo.Context) error { if prefix != "" && prefix != "./" && prefix != "/" { isObject, err := s3Ctrl.KeyExists(bucket, prefix) if err != nil { - log.Error("HandleListByPrefixWithDetail: " + err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error checking if key exists: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } if isObject { objMeta, err := s3Ctrl.GetMetaData(bucket, prefix) if err != nil { - log.Error("HandleListByPrefixWithDetail: " + err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error checking for object's metadata: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } if *objMeta.ContentLength == 0 { - log.Infof("HandleListByPrefixWithDetail: Detected a zero byte directory marker within prefix: %s", prefix) + log.Infof("detected a zero byte directory marker within prefix: %s", prefix) } else { - err = fmt.Errorf("`%s` is an object, not a prefix. please see options for keys or pass a prefix", prefix) - log.Error("HandleListByPrefixWithDetail: " + err.Error()) - return c.JSON(http.StatusTeapot, err.Error()) + errMsg := fmt.Errorf("`%s` is an object, not a prefix. please see options for keys or pass a prefix", prefix) + log.Error(errMsg.Error()) + return c.JSON(http.StatusTeapot, errMsg.Error()) } } prefix = strings.Trim(prefix, "/") + "/" @@ -180,11 +184,12 @@ func (bh *BlobHandler) HandleListByPrefixWithDetail(c echo.Context) error { err = s3Ctrl.GetListWithCallBack(bucket, prefix, true, processPage) if err != nil { - log.Error("HandleListByPrefixWithDetail: Error processing objects:", err) - return c.JSON(http.StatusInternalServerError, fmt.Sprintf("Error processing objects: %v", err)) + errMsg := fmt.Errorf("error processing objects: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } - log.Info("HandleListByPrefixWithDetail: Successfully retrieved detailed list by prefix:", prefix) + log.Info("successfully retrieved detailed list by prefix:", prefix) return c.JSON(http.StatusOK, results) } diff --git a/blobstore/metadata.go b/blobstore/metadata.go index 2e461b7..9ecb2ad 100644 --- a/blobstore/metadata.go +++ b/blobstore/metadata.go @@ -32,15 +32,15 @@ func (bh *BlobHandler) GetSize(page *s3.ListObjectsV2Output, totalSize *uint64, func (bh *BlobHandler) HandleGetSize(c echo.Context) error { prefix := c.QueryParam("prefix") if prefix == "" { - err := errors.New("request must include a `prefix` parameter") - log.Error("HandleGetSize: " + err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := errors.New("request must include a `prefix` parameter") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } @@ -92,15 +92,15 @@ func (bh *BlobHandler) HandleGetSize(c echo.Context) error { func (bh *BlobHandler) HandleGetMetaData(c echo.Context) error { key := c.QueryParam("key") if key == "" { - err := errors.New("request must include a `key` parameter") - log.Error("HandleGetMetaData: " + err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("request must include a `key` parameter") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } @@ -108,40 +108,42 @@ func (bh *BlobHandler) HandleGetMetaData(c echo.Context) error { result, err := s3Ctrl.GetMetaData(bucket, key) if err != nil { if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" { - err := fmt.Errorf("object %s not found", key) - log.Error("HandleGetMetaData: " + err.Error()) - return c.JSON(http.StatusNotFound, err.Error()) + errMsg := fmt.Errorf("object %s not found", key) + log.Error(errMsg.Error()) + return c.JSON(http.StatusNotFound, errMsg.Error()) } - log.Error("HandleGetMetaData: Error getting metadata:", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error getting metadata: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } - log.Info("HandleGetMetaData: Successfully retrieved metadata for key:", key) + log.Info("successfully retrieved metadata for key:", key) return c.JSON(http.StatusOK, result) } func (bh *BlobHandler) HandleGetObjExist(c echo.Context) error { key := c.QueryParam("key") if key == "" { - err := errors.New("request must include a `key` parameter") - log.Error("HandleGetObjExist: " + err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("request must include a `key` parameter") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } result, err := s3Ctrl.KeyExists(bucket, key) if err != nil { - log.Error("HandleGetObjExist: " + err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error checking if key exists: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } - log.Info("HandleGetObjExist: Successfully retrieved metadata for key:", key) + log.Info("successfully retrieved metadata for key:", key) return c.JSON(http.StatusOK, result) } diff --git a/blobstore/move.go b/blobstore/move.go index 21e877c..26fd1ff 100644 --- a/blobstore/move.go +++ b/blobstore/move.go @@ -16,9 +16,9 @@ func (bh *BlobHandler) HandleMovePrefix(c echo.Context) error { srcPrefix := c.QueryParam("src_prefix") destPrefix := c.QueryParam("dest_prefix") if srcPrefix == "" || destPrefix == "" { - err := errors.New("parameters `src_key` and `dest_key` are required") - log.Error("HandleCopyPrefix", err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("parameters `src_key` and `dest_key` are required") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } if !strings.HasSuffix(srcPrefix, "/") { srcPrefix = srcPrefix + "/" @@ -30,15 +30,16 @@ func (bh *BlobHandler) HandleMovePrefix(c echo.Context) error { bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } err = s3Ctrl.CopyPrefix(bucket, srcPrefix, destPrefix) if err != nil { if strings.Contains(err.Error(), "source prefix not found") { - log.Infof("No objects found with source prefix: %s", srcPrefix) - return c.JSON(http.StatusNotFound, "Source prefix not found") + errMsg := fmt.Errorf("no objects found with source prefix: %s", srcPrefix) + log.Error(errMsg.Error()) + return c.JSON(http.StatusNotFound, errMsg.Error()) } return c.JSON(http.StatusInternalServerError, err.Error()) } @@ -75,8 +76,7 @@ func (s3Ctrl *S3Controller) CopyPrefix(bucket, srcPrefix, destPrefix string) err // Ensure that your application logic requires this before proceeding err := s3Ctrl.DeleteList(page, bucket) if err != nil { - errMsg := fmt.Errorf("error deleting from source prefix %s: %v", srcPrefix, err) - return errMsg + return fmt.Errorf("error deleting from source prefix %s: %v", srcPrefix, err) } return nil } @@ -98,15 +98,15 @@ func (bh *BlobHandler) HandleMoveObject(c echo.Context) error { srcObjectKey := c.QueryParam("src_key") destObjectKey := c.QueryParam("dest_key") if srcObjectKey == "" || destObjectKey == "" { - err := errors.New("paramters `src_key` and `dest_key` are required") - log.Error("HandleCopyObject", err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("paramters `src_key` and `dest_key` are required") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } @@ -114,14 +114,18 @@ func (bh *BlobHandler) HandleMoveObject(c echo.Context) error { err = s3Ctrl.CopyObject(bucket, srcObjectKey, destObjectKey) if err != nil { if strings.Contains(err.Error(), "keys are identical; no action taken") { + log.Error(err.Error()) return c.JSON(http.StatusBadRequest, err.Error()) // 400 Bad Request } else if strings.Contains(err.Error(), "already exists in the bucket; duplication will cause an overwrite") { + log.Error(err.Error()) return c.JSON(http.StatusConflict, err.Error()) // 409 Conflict } else if strings.Contains(err.Error(), "does not exist") { + log.Error(err.Error()) return c.JSON(http.StatusNotFound, err.Error()) } - log.Error("HandleCopyObject: Error when implementing copyObject", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error when copying object: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } return c.JSON(http.StatusOK, fmt.Sprintf("Succesfully moved object from %s to %s", srcObjectKey, destObjectKey)) diff --git a/blobstore/object_content.go b/blobstore/object_content.go index 06d69b1..c4e6304 100644 --- a/blobstore/object_content.go +++ b/blobstore/object_content.go @@ -1,7 +1,6 @@ package blobstore import ( - "errors" "fmt" "io" "net/http" @@ -41,30 +40,31 @@ func (s3Ctrl *S3Controller) FetchObjectContent(bucket string, key string) ([]byt func (bh *BlobHandler) HandleObjectContents(c echo.Context) error { key := c.QueryParam("key") if key == "" { - err := errors.New("parameter 'key' is required") - log.Error("HandleObjectContents: " + err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("parameter 'key' is required") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } body, err := s3Ctrl.FetchObjectContent(bucket, key) if err != nil { - log.Error("HandleObjectContents: " + err.Error()) + errMsg := fmt.Errorf("error fetching object's content: %s", err.Error()) + log.Error(errMsg.Error()) if strings.Contains(err.Error(), "object") { - return c.JSON(http.StatusNotFound, err.Error()) + return c.JSON(http.StatusNotFound, errMsg.Error()) } else { - return c.JSON(http.StatusInternalServerError, err.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } } - log.Info("HandleObjectContents: Successfully fetched object data for key:", key) + log.Info("successfully fetched object data for key:", key) //TODO: add contentType return c.Blob(http.StatusOK, "", body) } diff --git a/blobstore/presigned_url.go b/blobstore/presigned_url.go index 879f506..9c7bddd 100644 --- a/blobstore/presigned_url.go +++ b/blobstore/presigned_url.go @@ -4,7 +4,6 @@ import ( "archive/tar" "bytes" "compress/gzip" - "errors" "fmt" "io" "net/http" @@ -118,52 +117,54 @@ func (bh *BlobHandler) HandleGetPresignedDownloadURL(c echo.Context) error { bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } key := c.QueryParam("key") if key == "" { - err := errors.New("parameter `key` is required") - log.Error("HandleGetPresignedURL: " + err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("parameter `key` is required") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } keyExist, err := s3Ctrl.KeyExists(bucket, key) if err != nil { - log.Error("HandleGetPresignedURL: Error checking if key exists:", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("checking if key exists: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } if !keyExist { - err := fmt.Errorf("object %s not found", key) - log.Error("HandleGetPresignedURL: " + err.Error()) - return c.JSON(http.StatusNotFound, err.Error()) + errMsg := fmt.Errorf("object %s not found", key) + log.Error(errMsg.Error()) + return c.JSON(http.StatusNotFound, errMsg.Error()) } // Set the expiration time for the pre-signed URL url, err := s3Ctrl.GetDownloadPresignedURL(bucket, key, bh.Config.DefaultDownloadPresignedUrlExpiration) if err != nil { - log.Error("HandleGetPresignedURL: Error getting presigned URL:", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error getting presigned URL: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } - log.Info("HandleGetPresignedURL: Successfully generated presigned URL for key:", key) + log.Info("successfully generated presigned URL for key:", key) return c.JSON(http.StatusOK, url) } func (bh *BlobHandler) HandleGetPresignedURLMultiObj(c echo.Context) error { prefix := c.QueryParam("prefix") if prefix == "" { - err := errors.New("request must include a `prefix` parameter") - log.Error("HandleGetPresignedURLMultiObj: " + err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("request must include a `prefix` parameter") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } @@ -174,12 +175,13 @@ func (bh *BlobHandler) HandleGetPresignedURLMultiObj(c echo.Context) error { response, err := s3Ctrl.GetList(bucket, prefix, false) if err != nil { - log.Error("HandleGetPresignedURLMultiObj: Error getting list:", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error getting list: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } if *response.KeyCount == 0 { errMsg := fmt.Errorf("the specified prefix %s does not exist in S3", prefix) - log.Error("HandleGetPresignedURLMultiObj: " + errMsg.Error()) + log.Error(errMsg.Error()) return c.JSON(http.StatusNotFound, errMsg.Error()) } //check if size is below 5GB @@ -193,9 +195,9 @@ func (bh *BlobHandler) HandleGetPresignedURLMultiObj(c echo.Context) error { limit := uint64(1024 * 1024 * 1024 * bh.Config.DefaultZipDownloadSizeLimit) if size >= limit { - err := fmt.Errorf("request entity is larger than %v GB, current prefix size is: %v GB", bh.Config.DefaultZipDownloadSizeLimit, float64(size)/(1024*1024*1024)) - log.Error("HandleGetPresignedURLMultiObj: ", err.Error()) - return c.JSON(http.StatusRequestEntityTooLarge, err.Error()) + errMsg := fmt.Errorf("request entity is larger than %v GB, current prefix size is: %v GB", bh.Config.DefaultZipDownloadSizeLimit, float64(size)/(1024*1024*1024)) + log.Error(errMsg.Error()) + return c.JSON(http.StatusRequestEntityTooLarge, errMsg.Error()) } filename := fmt.Sprintf("%s.%s", strings.TrimSuffix(prefix, "/"), "tar.gz") @@ -204,8 +206,9 @@ func (bh *BlobHandler) HandleGetPresignedURLMultiObj(c echo.Context) error { // Check if the tar.gz file already exists in S3 tarFileResponse, err := s3Ctrl.GetList(bucket, outputFile, false) if err != nil { - log.Error("Error checking if tar.gz file exists in S3:", err) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error checking if tar.gz file exists in S3: %s", err) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } if len(tarFileResponse.Contents) > 0 { @@ -213,8 +216,9 @@ func (bh *BlobHandler) HandleGetPresignedURLMultiObj(c echo.Context) error { // Tar.gz file exists, now compare modification dates mostRecentModTime, err := s3Ctrl.getMostRecentModTime(bucket, prefix) if err != nil { - log.Error("Error getting most recent modification time:", err) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error getting most recent modification time: %s", err) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } if tarFileResponse.Contents[0].LastModified.After(mostRecentModTime) { @@ -223,8 +227,9 @@ func (bh *BlobHandler) HandleGetPresignedURLMultiObj(c echo.Context) error { // Existing tar.gz file is up-to-date, return pre-signed URL href, err := s3Ctrl.GetDownloadPresignedURL(bucket, outputFile, bh.Config.DefaultDownloadPresignedUrlExpiration) if err != nil { - log.Error("Error getting presigned:", err) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error getting presigned: %s", err) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } return c.JSON(http.StatusOK, string(href)) } @@ -233,17 +238,19 @@ func (bh *BlobHandler) HandleGetPresignedURLMultiObj(c echo.Context) error { err = s3Ctrl.tarS3Files(response, bucket, outputFile, prefix) if err != nil { - log.Error("HandleGetPresignedURLMultiObj: Error tarring S3 files:", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error tarring S3 files: %s", err) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } href, err := s3Ctrl.GetDownloadPresignedURL(bucket, outputFile, bh.Config.DefaultDownloadPresignedUrlExpiration) if err != nil { - log.Error("HandleGetPresignedURLMultiObj: Error getting presigned URL:", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error getting presigned URL: %s", err) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } - log.Info("HandleGetPresignedURLMultiObj: Successfully generated presigned URL for prefix:", prefix) + log.Info("successfully generated presigned URL for prefix:", prefix) return c.JSON(http.StatusOK, string(href)) } @@ -314,7 +321,7 @@ func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { // Call GetList with the processPage function err = s3Ctrl.GetListWithCallBack(bucket, prefix, false, processPage) if err != nil { - errMsg := fmt.Errorf("error processing objects: %v", err) + errMsg := fmt.Errorf("error processing objects: %s", err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusInternalServerError, errMsg.Error()) } @@ -331,7 +338,7 @@ func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { ContentType: aws.String("binary/octet-stream"), }) if err != nil { - errMsg := fmt.Errorf("error uploading %s to S3: %s", txtBatFileName, err) + errMsg := fmt.Errorf("error uploading %s to S3: %s", txtBatFileName, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusInternalServerError, errMsg.Error()) } @@ -343,6 +350,6 @@ func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { return c.JSON(http.StatusInternalServerError, errMsg.Error()) } - log.Infof("Successfully generated download script for prefix %s in bucket %s", prefix, bucket) + log.Infof("successfully generated download script for prefix %s in bucket %s", prefix, bucket) return c.JSON(http.StatusOK, href) } diff --git a/blobstore/upload.go b/blobstore/upload.go index ef83ecd..cfad93c 100644 --- a/blobstore/upload.go +++ b/blobstore/upload.go @@ -2,7 +2,6 @@ package blobstore import ( "bytes" - "errors" "fmt" "io" "net/http" @@ -24,7 +23,7 @@ func (s3Ctrl *S3Controller) UploadS3Obj(bucket string, key string, body io.ReadC resp, err := s3Ctrl.S3Svc.CreateMultipartUpload(params) if err != nil { - return fmt.Errorf("uploadS3Obj: error initializing multipart upload. %s", err.Error()) + return fmt.Errorf("error initializing multipart upload. %s", err.Error()) } // Create the variables that will track upload progress @@ -41,7 +40,7 @@ func (s3Ctrl *S3Controller) UploadS3Obj(bucket string, key string, body io.ReadC // This would be a true error while reading if err != nil && err != io.EOF { - return fmt.Errorf("uploadS3Obj: error copying POST body to S3. %s", err.Error()) + return fmt.Errorf("error copying POST body to S3. %s", err.Error()) } // Add the buffer data to the buffer @@ -59,7 +58,7 @@ func (s3Ctrl *S3Controller) UploadS3Obj(bucket string, key string, body io.ReadC result, err := s3Ctrl.S3Svc.UploadPart(params) if err != nil { - return fmt.Errorf("uploadS3Obj: error streaming POST body to S3. %s, %+v", err.Error(), result) + return fmt.Errorf("error streaming POST body to S3. %s, %+v", err.Error(), result) } totalBytes += int64(buffer.Len()) @@ -88,7 +87,7 @@ func (s3Ctrl *S3Controller) UploadS3Obj(bucket string, key string, body io.ReadC result, err := s3Ctrl.S3Svc.UploadPart(params2) if err != nil { - return fmt.Errorf("uploadS3Obj: error streaming POST body to S3. %s, %+v", err.Error(), result) + return fmt.Errorf("error streaming POST body to S3. %s, %+v", err.Error(), result) } totalBytes += int64(buffer.Len()) @@ -106,7 +105,7 @@ func (s3Ctrl *S3Controller) UploadS3Obj(bucket string, key string, body io.ReadC } _, err = s3Ctrl.S3Svc.CompleteMultipartUpload(completeParams) if err != nil { - return fmt.Errorf("uploadS3Obj: error completing multipart upload. %s", err.Error()) + return fmt.Errorf("error completing multipart upload. %s", err.Error()) } return nil @@ -116,15 +115,15 @@ func (bh *BlobHandler) HandleMultipartUpload(c echo.Context) error { // Add overwrite check and parameter key := c.QueryParam("key") if key == "" { - err := errors.New("parameter 'key' is required") - log.Error("HandleMultipartUpload: " + err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("parameter 'key' is required") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } @@ -144,23 +143,24 @@ func (bh *BlobHandler) HandleMultipartUpload(c echo.Context) error { var err error override, err = strconv.ParseBool(c.QueryParam("override")) if err != nil { - log.Errorf("HandleMultipartUpload: Error parsing 'override' parameter: %s", err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("error parsing 'override' parameter: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } } else { - err := errors.New("request must include a `override`, options are `true` or `false`") - log.Errorf("HandleMultipartUpload: %s", err.Error()) - return c.JSON(http.StatusUnprocessableEntity, err.Error()) + errMsg := fmt.Errorf("request must include a `override`, options are `true` or `false`") + log.Error(errMsg.Error()) + return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } // Check if the request body is empty buf := make([]byte, 1) _, err = c.Request().Body.Read(buf) if err == io.EOF { - err := errors.New("no file provided in the request body") - log.Error("HandleMultipartUpload: " + err.Error()) - return c.JSON(http.StatusBadRequest, err.Error()) // Return 400 Bad Request + errMsg := fmt.Errorf("no file provided in the request body") + log.Error(errMsg.Error()) + return c.JSON(http.StatusBadRequest, errMsg.Error()) // Return 400 Bad Request } // Reset the request body to its original state @@ -168,13 +168,14 @@ func (bh *BlobHandler) HandleMultipartUpload(c echo.Context) error { keyExist, err := s3Ctrl.KeyExists(bucket, key) if err != nil { - log.Errorf("HandleMultipartUpload: Error checking if key exists: %s", err.Error()) - return c.JSON(http.StatusInternalServerError, err) + errMsg := fmt.Errorf("error checking if key exists: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } if keyExist && !override { - err := fmt.Errorf("object %s already exists and override is set to %t", key, override) - log.Errorf("HandleMultipartUpload: %s" + err.Error()) - return c.JSON(http.StatusConflict, err.Error()) + errMsg := fmt.Errorf("object %s already exists and override is set to %t", key, override) + log.Errorf(errMsg.Error()) + return c.JSON(http.StatusConflict, errMsg.Error()) } body := c.Request().Body @@ -182,11 +183,12 @@ func (bh *BlobHandler) HandleMultipartUpload(c echo.Context) error { err = s3Ctrl.UploadS3Obj(bucket, key, body) if err != nil { - log.Errorf("HandleMultipartUpload: Error uploading S3 object: %s", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error uploading S3 object: %s", err.Error()) + log.Errorf(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } - log.Infof("HandleMultipartUpload: Successfully uploaded file with key: %s", key) + log.Infof("Successfully uploaded file with key: %s", key) return c.JSON(http.StatusOK, "Successfully uploaded file") } @@ -236,7 +238,7 @@ func (bh *BlobHandler) HandleGetPresignedUploadURL(c echo.Context) error { //get controller for bucket s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } @@ -259,8 +261,9 @@ func (bh *BlobHandler) HandleGetPresignedUploadURL(c echo.Context) error { } presignedURL, err := s3Ctrl.GetUploadPartPresignedURL(bucket, key, uploadID, int64(partNumber), bh.Config.DefaultUploadPresignedUrlExpiration) if err != nil { - log.Errorf("error generating presigned part URL: %s", err.Error()) - return c.JSON(http.StatusInternalServerError, err.Error()) + errMsg := fmt.Errorf("error generating presigned part URL: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } log.Infof("successfully generated presigned part URL for key: %s", key) return c.JSON(http.StatusOK, presignedURL) @@ -305,7 +308,7 @@ func (bh *BlobHandler) HandleGetMultipartUploadID(c echo.Context) error { //get controller for bucket s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } @@ -354,7 +357,7 @@ func (bh *BlobHandler) HandleCompleteMultipartUpload(c echo.Context) error { bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } @@ -422,7 +425,7 @@ func (bh *BlobHandler) HandleAbortMultipartUpload(c echo.Context) error { bucket := c.QueryParam("bucket") s3Ctrl, err := bh.GetController(bucket) if err != nil { - errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error()) + errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } From c8b59880072fc3fc3b88edef055bf95e0884d299 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Tue, 30 Jan 2024 16:16:37 -0500 Subject: [PATCH 12/27] add new endpoints to e2e test --- blobstore/blobhandler.go | 8 +- blobstore/upload.go | 47 +++- e2e-test/e2eCollection.json | 477 +++++++++++++++++++++++++++++++++++- e2e-test/e2eEnv.json | 20 +- e2e-test/missing_huc8s.xlsx | Bin 0 -> 13128 bytes 5 files changed, 527 insertions(+), 25 deletions(-) create mode 100644 e2e-test/missing_huc8s.xlsx diff --git a/blobstore/blobhandler.go b/blobstore/blobhandler.go index 6e16342..051a8d4 100644 --- a/blobstore/blobhandler.go +++ b/blobstore/blobhandler.go @@ -21,6 +21,7 @@ type S3Controller struct { Sess *session.Session S3Svc *s3.S3 Buckets []string + S3Mock bool } // Config holds the configuration settings for the REST API server. @@ -92,13 +93,12 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) { } // Configure the BlobHandler with MinIO session and bucket information - config.S3Controllers = []S3Controller{{Sess: sess, S3Svc: s3SVC, Buckets: []string{creds.Bucket}}} + config.S3Controllers = []S3Controller{{Sess: sess, S3Svc: s3SVC, Buckets: []string{creds.Bucket}, S3Mock: true}} // Return the configured BlobHandler return &config, nil } // Using AWS S3 - // Load AWS credentials from the provided .env.json file log.Debug("looking for .env.json") awsConfig, err := newAWSConfig(envJson) @@ -127,7 +127,7 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) { return nil, errMsg } - S3Ctrl := S3Controller{Sess: sess, S3Svc: s3SVC} + S3Ctrl := S3Controller{Sess: sess, S3Svc: s3SVC, S3Mock: false} // Retrieve the list of buckets for each account result, err := S3Ctrl.ListBuckets() if err != nil { @@ -153,7 +153,7 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) { } if len(bucketNames) > 0 { - config.S3Controllers = append(config.S3Controllers, S3Controller{Sess: sess, S3Svc: s3SVC, Buckets: bucketNames}) + config.S3Controllers = append(config.S3Controllers, S3Controller{Sess: sess, S3Svc: s3SVC, Buckets: bucketNames, S3Mock: false}) } } diff --git a/blobstore/upload.go b/blobstore/upload.go index cfad93c..60091b5 100644 --- a/blobstore/upload.go +++ b/blobstore/upload.go @@ -9,8 +9,10 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/labstack/echo/v4" + log "github.com/sirupsen/logrus" ) @@ -211,16 +213,43 @@ func (s3Ctrl *S3Controller) GetUploadPresignedURL(bucket string, key string, exp // function to retrieve presigned url for a multipart upload part. func (s3Ctrl *S3Controller) GetUploadPartPresignedURL(bucket string, key string, uploadID string, partNumber int64, expMin int) (string, error) { duration := time.Duration(expMin) * time.Minute - req, _ := s3Ctrl.S3Svc.UploadPartRequest(&s3.UploadPartInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - UploadId: aws.String(uploadID), - PartNumber: aws.Int64(partNumber), - }) + var urlStr string + var err error + if s3Ctrl.S3Mock { + // Create a temporary S3 client with the modified endpoint + tempS3Svc, err := session.NewSession(&aws.Config{ + Endpoint: aws.String("http://localhost:9000"), + Region: s3Ctrl.S3Svc.Config.Region, + Credentials: s3Ctrl.S3Svc.Config.Credentials, + S3ForcePathStyle: aws.Bool(true), + }) + if err != nil { + return "", fmt.Errorf("error creating temporary s3 session: %s", err.Error()) + } - urlStr, err := req.Presign(duration) - if err != nil { - return "", err + // Generate the request using the temporary client + req, _ := s3.New(tempS3Svc).UploadPartRequest(&s3.UploadPartInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: aws.String(uploadID), + PartNumber: aws.Int64(partNumber), + }) + urlStr, err = req.Presign(duration) + if err != nil { + return "", err + } + } else { + // Generate the request using the original client + req, _ := s3Ctrl.S3Svc.UploadPartRequest(&s3.UploadPartInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: aws.String(uploadID), + PartNumber: aws.Int64(partNumber), + }) + urlStr, err = req.Presign(duration) + if err != nil { + return "", err + } } return urlStr, nil diff --git a/e2e-test/e2eCollection.json b/e2e-test/e2eCollection.json index 00a59d3..10834d4 100644 --- a/e2e-test/e2eCollection.json +++ b/e2e-test/e2eCollection.json @@ -48,7 +48,7 @@ "script": { "exec": [ "pm.test('ping with auth did not work as expected', function () {\r", - " pm.response.to.have.status(401);\r", + " pm.response.to.have.status(500);\r", "});" ], "type": "text/javascript" @@ -296,7 +296,7 @@ { "key": "", "type": "file", - "src": "staff_table.csv" + "src": "FFRMS/missing_huc8s.xlsx" } ] }, @@ -776,6 +776,45 @@ }, "response": [] }, + { + "name": "10.2/prefix/download_script", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{bearer_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{s3_api_root_url}}/prefix/download/script?prefix={{e2ePathToObj}}&bucket={{bucket}}", + "host": [ + "{{s3_api_root_url}}" + ], + "path": [ + "prefix", + "download", + "script" + ], + "query": [ + { + "key": "prefix", + "value": "{{e2ePathToObj}}" + }, + { + "key": "bucket", + "value": "{{bucket}}" + } + ] + } + }, + "response": [] + }, { "name": "11/object/upload 2", "event": [ @@ -817,7 +856,7 @@ { "key": "", "type": "file", - "src": "staff_table.csv" + "src": "FFRMS/missing_huc8s.xlsx" } ] }, @@ -1040,7 +1079,7 @@ { "key": "", "type": "file", - "src": "staff_table.csv" + "src": "FFRMS/missing_huc8s.xlsx" } ] }, @@ -1112,7 +1151,7 @@ { "key": "", "type": "file", - "src": "staff_table.csv" + "src": "FFRMS/missing_huc8s.xlsx" } ] }, @@ -1195,6 +1234,362 @@ } }, "response": [] + }, + { + "name": "18/object/presigned_upload", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{bearer_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{s3_api_root_url}}/object/presigned_upload?key={{e2eObjName}}&bucket={{bucket}}", + "host": [ + "{{s3_api_root_url}}" + ], + "path": [ + "object", + "presigned_upload" + ], + "query": [ + { + "key": "key", + "value": "{{e2eObjName}}" + }, + { + "key": "bucket", + "value": "{{bucket}}" + } + ] + } + }, + "response": [] + }, + { + "name": "19/object/multipart_upload_id", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "var jsonData = pm.response.json();\r", + "pm.environment.set(\"uploadId\", jsonData);\r", + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{bearer_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{s3_api_root_url}}/object/multipart_upload_id?key={{e2eObjName}}&bucket={{bucket}}", + "host": [ + "{{s3_api_root_url}}" + ], + "path": [ + "object", + "multipart_upload_id" + ], + "query": [ + { + "key": "key", + "value": "{{e2eObjName}}" + }, + { + "key": "bucket", + "value": "{{bucket}}" + } + ] + } + }, + "response": [] + }, + { + "name": "19/object/presigned_upload_multipart", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "var jsonData = pm.response.json()//.replace('minio', 'localhost');\r", + "pm.environment.set(\"presignedUploadUrl\", jsonData);\r", + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{bearer_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{s3_api_root_url}}/object/presigned_upload?key={{e2eObjName}}&bucket={{bucket}}&upload_id={{uploadId}}&part_number=1", + "host": [ + "{{s3_api_root_url}}" + ], + "path": [ + "object", + "presigned_upload" + ], + "query": [ + { + "key": "key", + "value": "{{e2eObjName}}" + }, + { + "key": "bucket", + "value": "{{bucket}}" + }, + { + "key": "upload_id", + "value": "{{uploadId}}" + }, + { + "key": "part_number", + "value": "1" + } + ] + } + }, + "response": [] + }, + { + "name": "presigned_url_test", + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + }, + { + "listen": "test", + "script": { + "exec": [ + "// Extract ETag from the response header\r", + "var etag = pm.response.headers.get(\"ETag\");\r", + "\r", + "\r", + "// Remove extra quotation marks if present\r", + "etag = etag.replace(/\"/g, '');\r", + "\r", + "\r", + "// Set the ETag as an environment variable\r", + "pm.environment.set(\"etag\", etag);\r", + "" + ], + "type": "text/javascript" + } + } + ], + "protocolProfileBehavior": { + "disabledSystemHeaders": {} + }, + "request": { + "method": "PUT", + "header": [], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "", + "type": "file", + "src": "FFRMS/missing_huc8s.xlsx" + } + ] + }, + "url": { + "raw": "{{presignedUploadUrl}}", + "host": [ + "{{presignedUploadUrl}}" + ] + } + }, + "response": [] + }, + { + "name": "20/object/complete_multipart_upload", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{bearer_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\r\n \"uploadId\": \"{{uploadId}}\",\r\n \"parts\": [{\r\n \"partNumber\":1,\r\n \"eTag\":\"{{etag}}\"\r\n }]\r\n}\r\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{s3_api_root_url}}/object/complete_multipart_upload?key={{e2eObjName}}&bucket={{bucket}}", + "host": [ + "{{s3_api_root_url}}" + ], + "path": [ + "object", + "complete_multipart_upload" + ], + "query": [ + { + "key": "key", + "value": "{{e2eObjName}}" + }, + { + "key": "bucket", + "value": "{{bucket}}" + } + ] + } + }, + "response": [] + }, + { + "name": "20/object/multipart_upload_id", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "var jsonData = pm.response.json();\r", + "pm.environment.set(\"uploadId\", jsonData);\r", + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{bearer_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{s3_api_root_url}}/object/multipart_upload_id?key={{e2eObjName}}&bucket={{bucket}}", + "host": [ + "{{s3_api_root_url}}" + ], + "path": [ + "object", + "multipart_upload_id" + ], + "query": [ + { + "key": "key", + "value": "{{e2eObjName}}" + }, + { + "key": "bucket", + "value": "{{bucket}}" + } + ] + } + }, + "response": [] + }, + { + "name": "21/object/abort_multipart_upload", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{bearer_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "url": { + "raw": "{{s3_api_root_url}}/object/abort_multipart_upload?key={{e2eObjName}}&bucket={{bucket}}&upload_id={{uploadId}}", + "host": [ + "{{s3_api_root_url}}" + ], + "path": [ + "object", + "abort_multipart_upload" + ], + "query": [ + { + "key": "key", + "value": "{{e2eObjName}}" + }, + { + "key": "bucket", + "value": "{{bucket}}" + }, + { + "key": "upload_id", + "value": "{{uploadId}}" + } + ] + } + }, + "response": [] } ], "event": [ @@ -1267,7 +1662,7 @@ { "key": "", "type": "file", - "src": "staff_table.csv" + "src": "FFRMS/missing_huc8s.xlsx" } ] }, @@ -1339,7 +1734,7 @@ { "key": "", "type": "file", - "src": "staff_table.csv" + "src": "FFRMS/missing_huc8s.xlsx" } ] }, @@ -1438,7 +1833,7 @@ { "key": "", "type": "file", - "src": "staff_table.csv" + "src": "missing_huc8s.xlsx" } ] }, @@ -1514,7 +1909,7 @@ { "key": "", "type": "file", - "src": "staff_table.csv" + "src": "missing_huc8s.xlsx" } ] }, @@ -1587,7 +1982,7 @@ { "key": "", "type": "file", - "src": "staff_table.csv" + "src": "missing_huc8s.xlsx" } ] }, @@ -2183,6 +2578,57 @@ }, "response": [] }, + { + "name": "10.2/prefix/download/script", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{bearer_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{s3_api_root_url}}/prefix/download/script?bucket={{bucket}}", + "host": [ + "{{s3_api_root_url}}" + ], + "path": [ + "prefix", + "download", + "script" + ], + "query": [ + { + "key": "prefix", + "value": "{{e2ePathToObj}}", + "disabled": true + }, + { + "key": "bucket", + "value": "{{bucket}}" + } + ] + } + }, + "response": [] + }, { "name": "12/prefix/move", "event": [ @@ -3094,7 +3540,10 @@ { "key": "", "type": "file", - "src": "staff_table.csv" + "src": [ + "missing_huc8s.xlsx", + "FFRMS/missing_huc8s.xlsx" + ] } ] }, @@ -3403,5 +3852,11 @@ } ] } + ], + "variable": [ + { + "key": "presignedUploadUrl", + "value": "\"\"" + } ] } \ No newline at end of file diff --git a/e2e-test/e2eEnv.json b/e2e-test/e2eEnv.json index d1e1dc9..f5b8e48 100644 --- a/e2e-test/e2eEnv.json +++ b/e2e-test/e2eEnv.json @@ -52,7 +52,7 @@ }, { "key": "e2eObjName", - "value": "staff_table3.txt", + "value": "missing_huc8s.xlsx", "type": "default", "enabled": true }, @@ -73,6 +73,24 @@ "value": "test-bucket", "type": "default", "enabled": true + }, + { + "key": "uploadId", + "value": "", + "type": "any", + "enabled": true + }, + { + "key": "presignedUploadUrl", + "value": "", + "type": "any", + "enabled": true + }, + { + "key": "etag", + "value": "", + "type": "any", + "enabled": true } ], "_postman_variable_scope": "environment", diff --git a/e2e-test/missing_huc8s.xlsx b/e2e-test/missing_huc8s.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..d18cf6fa3dfbdaee03c5421655c33eef3b1e2d04 GIT binary patch literal 13128 zcmeHug03ZNhAnQoj z**cloIvHrVzcF*vd*WteO`VU5%$y5AM$rH7^Ixoiio`EUom{w*7cxI3zH!JcRKLLB z-S_XsXH^yX+?CK%W}=gAVR4roc7rRPM`ZczBmT&OC->11tlHM5-aoYGlNvC>ukW*_ zKG`$Q?%q9yb`ns$gZAr#Ttc!JPeBF-QP2!Ps(nMJ8V_VuOs?WFjEGpocRg#aR|{{s zrwLf5UGltd9=@WawS>n%P&$FTK1MKUX~E*Df3nUYiyA@exj^iq|4xPGl{$;DT9d$|ldvSRA!Kb!ol!`o$t4qUA%d~r9t=QVVbiF*omFOP7%JQfQY(Bh_EzpFtc{#d~$#NUpW33 z$KXFLy&_&wxswYRd?52Pr0-&CF&0-`-c?NI6OD$quj~R&T~q-**Y0`4;%r31l ztmTa*dGh_sbW#&XkdL?{PhLKLNbrt4m{>T?PrFC)^^)=Bd!#9G_5I4=s)lE|I|(Bh zKGVrX+gPHZ&lUE@(}?;Ujm>A@d-Yk4$UXDXuH`xUlI7m@>|tkl-(kOh1XalrqC)+9kcgst?E9bq07HmU zhJ(DAaor9IJot@Qv-Kx;Eu`A%m{pCG=i{9l*GN=RK)e{5gy@P7Ln>|+w*JxTH zEcbnGlCClJ>w+R7wmxHmWxAF53*r)08w&4VeL>VVj)B-T-q~4lXjfGY4MP57MM`S~-UluP3 zZ(J2&jMOxyT=@G!jju^YX+`FffxOq)G=#;5hiB;o6H-53pjL*WUhDPG*+R-s z)K@HNv#*IaT5OTl{4^EXfVs_Do2wfpooMbQ)O#E*b(Ov!&_%5~45OIfU&mB=zn~pl zYG)(Wex!R#D9VQwDOzagnHewAhxQf(4wOqv#qn*-Ju{6tdhSTy8jhTxGM<(=9eU zEzY);eww7~A1m#)1sr+A{?gr_D_>4+3|H2?B`&w~t!{f+6OyNvgGyt4uFr*n3XFY} zqzjX-U*dRFkKnasYe~^HyYAav>YqgJYn7(uAK98UlR3UWD|ba^=v_RdqL*_ZYC!Tk z=-dBwmx;Q<+@Tsgtp+>1<8Ja-TZlDh1}3)8Twzl#o z7d(lxyQWNk7j2iu@1PsmF2U9D$Wh7T^$)A}xy8*WXd-jz3Hyx!LsjP8WKl02@5C&B zS&&||mHSTZyL{Oy@>CDewwxFIO6UciirYXZ4m@+8Cj{8)<(Sy9Fxgiwb^i;oq z!AX}m6ji$Aj~qOX_-xKh7V$mr%VIqU0`Gd?@^1-~8$1m+lMU3xj2DGRScpwO-F~6= z$=CP5v-;z1ZxI$@7SOy8?iaQ;;~1M^S}zFiXLqi7R?YhJjLD8}nKZiyaz0vGCL7{h z(qnBWOgb&t*``F*%&AMydT>*GH+1wM|HD?;@a%qAbj^p8jh~R^awm76l&l@{p>=wJ zMNw_`{PyJSoOQe&QN~G9%=bNFMLo<35mbms{P$S(6G==tK|E%>XBr@4!R(O=6l&qxFIK3yhb zZPt7BRh|>YF4+BGsMqC!7kdqb;0(@I5{{4RL129#h-No*GK7L*ykAM_FO5ckzqhkn zzW)e2-I1V4B)*S5n~?weB{Kskq@aiva@sT=%wsvs<;-pAI7%^q$nRfjzxiGVs_#|$ z*ETdzr?Wse?wTa#&mZLYyHkywfF8;UVYvgwwkB!VF5!~r7{@I?p^EOqR`7yr;0AfQ zJY^ZLuvf4Xbx5Byl@Ck2<3@hkC(VOZC-IvBmiGB$_AjRg&~N>{Q_BcT|GUfhXdfYm z5dZ+JkpKWGg5vM#-4SMH=H$rv`}Xuc&u7N##AI;cN-eXUiBh)*_{C!7#!fM^?6hgN zwhBTqw7l}_9o47qmi(A!laBE#QO6AjpMx(mL~{~>*oIkL&9t(F2>_mFQKR$WKd)bh zJb##J-fT1uE9nD~yX$-X;t_rP$&tbrCpK44QsCewUaeP9Wl*vRJ<3kogy`fF)-ZWS z{0i({NUykl8OF-U(}XK`gV9-Axl+aF_2I?Mqb`3_${W|Lt;M9GOFI7UJ=^7wg%V0U zGMXUK7!`bG{WjaWkJfbdKh)!&Qf9L2vn(Z)Lv`E1EbvPEqBFes?RT3uM~}2sdh7~s zuBbZbL3M9%JCWM*#$)Od73?ZbvG}g2_qHF9d8cq(7e3jc9Z5L+QW4t8u9ULs zP?s5;Fy*AZx$VHGLvq(=yo4dNYE<@FTqiE2(5pW_bF|w_Vbl&u`r($q*#dtV}%*KwM`pgXX36$Ggjv^4!vM_B4Q1M4wtR{#3o z`z29V{Yl9Xt!Y#mbH>SI^4{2dfpqF~(ch;1eM!t3izaWvlJqfK#R-*HbAEZzVMXoh z)Aa_pH^KOdOzHT7MpA8x;F9=U2*_nZ;9piKuR6)H)7h}x?tS}f`{ZJTO7!Q|h|k^e zrrp8ZCxf=z-4m+iy#|B*i@Vy}lN5uyqh)1-_P1Bpy%)<%Ha=trBHov4MLrF84L(;; zpS#P(w{Caq>me5gd%fD)L(ZeRIaqZjpX}a#x@~DinmdX9bvbixvpiRS*V0N++aT>Z ze?C+H)q80PtJd|)d3s$#R8E5{`u^hXt@Cb<+Y(BxtA^*$lEIz(9c6B-bX~*zW%zk^ zZmW5%xrx<*vXI?;pVyMU$4s`U&ss3ycz*Zhd95pRPa|Gj>QUy@$bJ=ZH@h{25h|p! z@Li}SqL@+KqBL=n&)RxYouIUKa=`J^>x`Y!uvgk;-HFCEs&z8o#??48u0ALZ9Y~>? z^gRE1f#8X^C`~KAsv1iiVNBmH1$i2K?nqQ1L%_nkj;T<-I&7!fszRvkgnEex{?O@U zWchV{>4gl>#4=3kEDl!m;jH_%A$2ti))Tqul}R@dh<*S8Lo+$jF*6Pc@398jUZGbA z`V=TOnbj&4`dB2$oC-%h%56uC$E0V>*pZ?PAvkh!&^BBA0x!s8UX=&!oF`K)mY>18>w;s7|XeH=Jj0VF| zwYH+Ke+}XGlDS-UL;+(t^O}#6t;9rdUD*#Yd#1(EONb5HXaB8OWT-#|1MDxbZ?;%ksrz5;>&mMbz9RNVi%O;BZQcn zTfJKj3(KlK&OKk;kx01W!aapmg)>5_z1rqpB4x!=ufHepo$N;*68;ucTwi$D?Ma{h zoR!atgl#(D-XoxmZZW?_T*21!m)7rb>frD0w&K%&7fkw+{OEo>e(g*PU1u3Fo)b4c z^K=i({IuST0ML|lews9_Cs}LRF|6*;Zy2fu&MvBgb{pZ04VvtmKZ#z>zIVZzP1>Hf z_y{Fi_Nbk+-d@rpyoe-*4SJbyKL;KYQnpF6vHmSPMGt9GG!L!!HvGQWGkZo~qtaXn`V0uB{ zg;&u`gWXBu7Pt>dmZET-y?SctSlXMTM=4k^F|@Ck$@rE0(Ta-bvMFj;nF;6Px~5~8 zYea>q=on~|nAKyr=D9QWr#uOznryWT`eL9OnFPw90L7;qV9E5uRo^6~lZjVndq5m! z5o#snBpwj8O5kUO-ldHiIuH@vVHHQVh);u~k1rh&gW;f`ta_U=eZd-PONzrR%;jA9 zy2rZYhPyw(SH@{gZ$$Tpm2(UO|;NX~e=`!ax zH@HzU#8aIv--8tE{hlYo?r=)YnjT;QPA=f5Q5)wW^yO*Nq7!|l@&q*wWgmj>RMRf5B%vQ z$2d=x4_pu+U>oKy4@I(nQ28Q2MyKQ{w;P~k1cKEORANgznM-7a%U>B7I_98JE0(K1 z5JW2Wg7WR^1mB*62klBOk^Rb*UR2aFsaWgrTy{Z%Nua65B>XAZuv!seAndhMDJ5mX zDI#~=xO_~BNJi>)%8UYf-^rhlLovZ^6mv8LbScL6>vNq2p!V|(p6{pWqX&&Byt2`7%{L3W zAG$HLV3KTIy?}ucGCx31t6R`zg?O8CVL`&H#_PUOX4LVKq*E7frXG{HXe(g1F;0x?vn7!F0cpYy5BFj9S zK3gI%tHLCX)(U&qNS6$+bo3U$tZa*wWS8JfhPP)VuC^#*9PdH1yaLe`EQb*k@#9lP zyLzq4UEDL$JvBpp8UwcZoDGl6xYTYtrf3rLNCt2tv*SO(9~ycP58yuZVNeX@K4tZI zqx;ro3U6TLYZ}(kJBdI(iaelg>mh|_@mkFr1VPOye6X^VFD`O;4V{v!)IMYi=c|LM z@_QHEdqUx5b8=edlt`6eeFj|YiMVgr0>+>PkMonKIO~Zwrkc3nW}*Hp4o~8$-jjUc zY7dx{8F4LZL?O#(caDC4X%9M zv{g?+-@$;s*-bH1A5umIpAtY1DN#I&@Sqg4!(zqaN>gXeAYEb^SdK5UY z)iAh3nC2$TNrNk(laJeM>VDGOo5I+tBA_u~D=hfM_*5fV4!}!q45*&518JvAhgX8n z&l4VA?ZJF%VZm2th2xC*pyt=b(c}wiH~bpnc_c$Cyu$R2Q&Gy)Cbew>$E|tfuC3J_1f&lp%qIg525zra01!pAgT1uiJ+^3btG@;NfNiZZ_yYc`5X{y36 zC`h`eW23{NhqD%{0ikf*q0x~tOpaz~F!ESH!fzJ6^BS7!W%n^*N7 z^9ia%rD>pHuy+Sgze|iGMr53w}MNZG(rHuU^D*@@)41vw5Ntyi!x*Uy@Ia`=R?xI06c%jz6*PB={K0aXHXSaaU(R*qkI#jU zlD$}IE=u=~_0)C(Sl1-SqfMVLPBtG@!1q#>AjTGg0j-v_fi~M-yT9%x9s_D^R4)}5 z1I$GGJxrN&KFrh2GLDTgr_&6;2JD#xsvoOup34-ydVnzCwWM^%h>np5_m2-rAk(?| z0k&Memo|oEF76}`^FdN$Kvh+pC_DtD3B}M>?Kd46GS8zi`tpWJO2t3-m4;+5M*7jKWre^n24%R0>i+!?mLYwPP^N zL!H&@!@d^O@B|bCl5(U;1S2~@KW9{mH12slDCo8Xi)S{lgTP%`qL#xoyA+t_81b)$ zlmo#Cy8Im0B1!&E=1iJC@sHA^d+{M%AR(1Ru6@G>q0DKaLALNNUYt-UsVw-?E5+-_ z3$6g250LBbJxKYOKj`~vT&67j7p^jHiG_>8$!NpiI4%jsan0I5R&eA=PspQD7gQVcV_b&U}xPx4nP>fuSN+}HJ{p#&B_f1vIf>pUByoTv-!n6wfL2J3X z!-)Fi528IOZxJdR-U>L3>)7m4a>vZ4EfdYUmhzPH$X3X+%N z>SRu($>ptkoHq_!XCGQYzdoLweJA0pJ}tDMAp(K}*8@T1TVPJUSWRGh0*CY5HIj#pIGBG|qI*hb0HN|$OxIEc%jG)YSH8+v9lz^9s= zXBf=f$C*R1U1Ip)!y3Z}1Je7%>e#^I9B3OxipamcbpkhG_kugUavUxYDpGw1LLSsw zzlRVD{#5EkftbHPh`ngab?R@Drw&a;Rlj~R4CYT$`AFLgHqK0uy4+88h>VTKgocjG zS1Tr(l|<0d>M$)ch3|$#Ltm-h+dm&femm*9dcW>M&`)}`D**_?3dms*^5gf8EwCN? zi1{t`eXgu=K~lv*q?@zZqUL;~d(_0pNDUazhc?}aa0nN-E*ue|TnNE$L&h0de?#i1 zL~MV1nsQnCmK=fnnuI?wQqG4r#5FtzY)erUK^+4Yz=HXs)Nd3K10!a0oMHyMAK7F0 z>ecc_O+n)mi(ux>Cm|S!+7BjAemya-hzgEG82g-&fzAou#c%Y$)wx=+mORTY z%p^C>k!qy1g7+mZE5KJmqQTp~zgp2r)Cvwf+AqRL*)v5uI~n32FpjM77P54N1GDjQ zbtH5yR5O~$(F_@vna1MwF{~nmBsmsZ;*3jjcC(4^>R+`f<9`L+X&Hr5wS6&1#QxsP zcNUz<7xy)i;)mmsCsS<`9l~kpP&0Jv)8#RsGvz!Ih49H6gu{62gIAGSelA3;Msa4fy^=YXDYU=3e#%fe&$J(yjTgkB`*kI=lt+#`%P(siJPx%o)-_jfU* zOFlQZyVD0bB0hINk9__*jHzakQBXcpWS1KmWYg-CRb>09$vp~wsm45@Fyi}B{`k@7 z{31C_*((Nyt0pdeDNAEHOT19?io)3B90TO_t@+ulluZ&d67#~M`Sq&Lk< zB(JOAQ`G`<$G4CXATvBaQ)4cCmm+Q_oH+gh1*`Qm<{^FUHq{u@&JZ;unFqw&^M59!oL9r4KqZ7RpYnUOQ8^b^TGR{YF7Udl3wk$7egZLc1& z|CFE4@pizo>Bs1uLAioh=Ot6gKA&eL?#2j?Yy1r5_giL+?3L-D%OUP1rwCX|C1!i4 zR!K@1f2r?ptm65Wn}%)=D@o4Bv$REd^=X_qRwVQC%d(FQc^}t1tHncuw$y%5(99hh zWj__9T=N{iV8+MyhFvk-Zjv-`5|3#yLN*?}maxr5yxUMP^m;Ysc)Sg_9H+(z(Dr<1 zfKsB{d$xP4-CjRlj0EHHPb#XP5!g+#jWxnjxt_}V^zuOS3-QvZfnH4$MOjSHbgqU^ zRWKE>c}4Qk2Qd~A-Oi9KXtDY3bS|NRPUp1zzP?Jssg(*u)}b-@Nl=}oYttN&X~$MH z!4fb)tTmoxL^T$xd{0eX;T7$(4eiJu+# WHox(wf=6P@+d}iV+8Z_m9BY)2@HI@ zV0=cyMhzA$InB!tB7VP$=)&u{xC@i?48vZLshztScl9gO|76RZdPdEeE$sR0aQh^r zUD)Sz>#EbBx~Yy;-WHufUH!*YjKSUQEy(^Dqi5TbOh+AE?LV)+q_p2137l1} zEf8_Gy*)nOP9Rx+d$y?@Loqi`PLX;@6ZY}&{i?|UV}y|(_RSHJXjT`Ocvt)jye*fK zA7A>A^>wIsI-INaJLbt>^`Nx~wOh?LQyn$YN2h&|`iQn*66r$fDzH+P{^juSO}4^f z!bwxur?AmDs~=C~DgYOo;M|vMYO_&QX5X>G@xzMv;p6JaUea`srP9TT{YeM70&#TPH{ur=Nk zK?;r&cA1G%5ptYUi0|w2X2hnEjB^c;CGm=x6@&;&kgHTbU1)e%-*F42ZAANKnV!fL zdTlmL@v$W`t(A(z$-gSXY)+_trfw(BlLnO2KGOMpF4k;LHW*0|^HdW?!_Wo|f%wc%Tg;$Rui*kDW8<`p& zS<%>vE4;`XRNo>HZ%)y+xZ*#x%kD;Dyf#nqApYT{_lC{euBO+}U`cqt4He{N38Fr4 zI3L1QwelA-3dX<$k}Ud%68+MJWR+`M|47W3$>765A(YZ;B}n)d)jbkc&GgfCfYKGo z@c~+l&e2VxKnAnfCd9Ai;RO}Dl?N<);q#mL3dgVRnvVKYmv=X1&JdHpml!|cYTZ6G8T!>RWh&I%+zFwG)#M$&YQ}< zkVwa|ZqF#n#61@?3e=V`Tl}0Eu5hN|24>7PDGYIDBgz@Ayiq!Fi;0?0GHlt$Tw%Oj@(%GV{RkDzSF4Czciff$9pJbI?a47labx(6z$RxLNlW`N719&G22hU29 zmZ{N?kx_lLOVIYJ&Jv@6m5#^iieQVhbK31e6Wyc^Nxpic3rIHG_R`fT8AMVY0!b9m zO;8$m@H8&fGqH=qfWs#Vi)V~H--hj_FRqC8dio1CstkKvC&4~9ZaAGq@FJ;dwi1=! zs`~pkzskCoDJtk^$vax9qGDQ61U8@H6~21K$%Y;s<{J^z&2RuV z-4Z_jJTaP1AVS3@Kh-q5gE#E`;FvNS6{+%UIy`UpiBM`8%6Y5u;xnqL0mlax-_47< zS$8I#61GDGzhRXhOH5X?zO%hm;&6Cx^`r?Kf6861Wp(mx1PYvsBWI9X@pX$*XLwDG z>U=Gj9RsL5$#-evEX6N;^qxtyFKbZS`33u~T-N3Fz(UIu6y4QN$m5LnM!nmgh_ycit@4=!FzkQ z;;b0Psp?`+>|P`38jVW2SbXT6e)3~|gJUQNFCs$0V34G4^?I#Mch<}P0FOHBrfwkj zwT+8q5VTI18MRfZV07=RK#+c=&HqEN@NQ02h{@>hJzzbA)@IuXQ2fWvd#ZzX<{a|j8Cd;_2g3Ql^cHvE76anHy`f#b))2>PcBeD z<{hTAIpH|6=%_ZHp@dm$(&JFcO|BP2`c!i#yj(H3U4#e@q}weiD;W(pa~olA>tAk{ zubh8$-e@*R880tzP`fR+X8O?`)G$p8A#8 z!Pj1IACw*Dv^a(isNbOMmjILlWwVpDVFBba=vlkM;c|SC1vU6|@J&^^+>`SF!SiVS zeTmF(qn-!!*}X;|--JH0Hzx8#Ug=Yh9eG-SIg=Z3TWd$%>Mu)Xc-`m&u>&qj+j9%N zxDIr0{#xYUFLJTIHJ}c+M2BE4Ti@6h2nuPgm)KlOG00^->jJ&DaG?>b(ua38kAODOBE$L%gxixpgMu zdXA1!Yuofj_tuZgP;0RXE$`*$tMch*n~7)wsvpzqJA|`DpqJJ@#zb~zUg6gru5)L& zAdz_-{ZiPDW7V+mD^ku^s#b2nK1&>;gD+hr$p>Z*__A38FhtE{chrmT)ZbC*Y$M=2e1dM6G|@PowcDWu@0a^`Sr zk1SM4{O(7)$#&y6V6c7`F29FXt~&v9I;ST`_7JN}m+Uq(4Db z&b4~DJ@X)4c1QtYXQ$ifb$8+hb?cRE<#Lpn>}<0Tqq*XFknYi_$N34HFrr|LBdjU} zha2ef>FHp8nfyX0dv$g#RMf<2p^&IlBqp&SA>4hu1?ogM5{ggq7B*iH{REde2UB+6 zpOr8hZN#TOIDpr9bM2ESX%w;5soglQbvt6V|Gqmm{mwWKK?JS^B1G{J{Sc;hCMphg zZyY&I>>SMg-gW!GHVB03BIED7P6;8uyH0t-PGVWB;x&XOu{6Z1ARntr2F&8zdWMB$^AU}=im5MON}bpB>>CpzzKz4c>=-io_d>v`Yn^dpCE5N5ur0ple}iVr!SoQK}N(S)AeoBJrNOG+U}KdLU_##e3MsZ zGZ)%`_Sw7R>>a=PY0{O?1-Oww-sk~&d(Q^#Tmj`1+TroWKCM}X{tq(xEzTo%sR#bp zB*(aG%U!e#+t%Vj$UFEsUeVW&{*&t&$v(slL%1FuqIC}UpIy)B&71#u9>V4R{>h9N zvzzAv1|6VWQvnyro-e8Z{VI%QUY=GNA;s65D;AmLRnv!7#O+PT)Q?ThdhA4f^K`q0 z$S#r#7@WqAb*qGJUk5P#Xqb&disJP+ILOLz)LNIoWRt<7fl+q z0wSNNrIv4#2W~WcUNzpe09xZaFtdJazREY3f{ol?Ufq)Xh8lx+GG&{Pxu{E1nF_12 zfa3Dd-cI~+9-Qa-Hydnz`N8(PCjlnFY_;La35sKoiYj%gnoQ5KN(r4Ec05l`)4Ev zATQoJHQ40&#jw?dB&ReA$z~6YA+pgbx&3Az$}dYk#tFf$FVBzqxQ>+g=l$#i9EYFj zrzu}Ec~X2b?T8CfKkE!K_mcVKW`&_Y?JlA*|JAhQ3QcKO1Zz^*j{AnjbTPduR9pY*k=9FfBGl$zqY{r#R>qxFn=@ue;VTcl=Ej_)L)V^ zfd4m%f9sO^Q_7#mGk;0Zrv8tIHGc~DbA$PpfLW&B0{+@={>l1hG5?FTg!ON%f0p(? zCH!mo{EHvK%?1GcTQU8U{a;hyKeL~3{1f}%6QYVd8p65&04Cz=i>TMtT=)O|Ke79Q AAOHXW literal 0 HcmV?d00001 From a13d4deb244760aada3526d1495b5944354411ae Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Tue, 30 Jan 2024 16:18:50 -0500 Subject: [PATCH 13/27] change status code back to 401 for e2e test --- e2e-test/e2eCollection.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-test/e2eCollection.json b/e2e-test/e2eCollection.json index 10834d4..9c39e12 100644 --- a/e2e-test/e2eCollection.json +++ b/e2e-test/e2eCollection.json @@ -48,7 +48,7 @@ "script": { "exec": [ "pm.test('ping with auth did not work as expected', function () {\r", - " pm.response.to.have.status(500);\r", + " pm.response.to.have.status(401);\r", "});" ], "type": "text/javascript" From ae72cb60f4d3d4e0309b99fe2c288690cc8009fa Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Tue, 30 Jan 2024 16:22:24 -0500 Subject: [PATCH 14/27] correct file path for e2e --- e2e-test/e2eCollection.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/e2e-test/e2eCollection.json b/e2e-test/e2eCollection.json index 9c39e12..6291595 100644 --- a/e2e-test/e2eCollection.json +++ b/e2e-test/e2eCollection.json @@ -296,7 +296,7 @@ { "key": "", "type": "file", - "src": "FFRMS/missing_huc8s.xlsx" + "src": "missing_huc8s.xlsx" } ] }, @@ -856,7 +856,7 @@ { "key": "", "type": "file", - "src": "FFRMS/missing_huc8s.xlsx" + "src": "missing_huc8s.xlsx" } ] }, @@ -1079,7 +1079,7 @@ { "key": "", "type": "file", - "src": "FFRMS/missing_huc8s.xlsx" + "src": "missing_huc8s.xlsx" } ] }, @@ -1151,7 +1151,7 @@ { "key": "", "type": "file", - "src": "FFRMS/missing_huc8s.xlsx" + "src": "missing_huc8s.xlsx" } ] }, @@ -1438,7 +1438,7 @@ { "key": "", "type": "file", - "src": "FFRMS/missing_huc8s.xlsx" + "src": "missing_huc8s.xlsx" } ] }, @@ -1662,7 +1662,7 @@ { "key": "", "type": "file", - "src": "FFRMS/missing_huc8s.xlsx" + "src": "missing_huc8s.xlsx" } ] }, @@ -1734,7 +1734,7 @@ { "key": "", "type": "file", - "src": "FFRMS/missing_huc8s.xlsx" + "src": "missing_huc8s.xlsx" } ] }, @@ -3542,7 +3542,7 @@ "type": "file", "src": [ "missing_huc8s.xlsx", - "FFRMS/missing_huc8s.xlsx" + "missing_huc8s.xlsx" ] } ] From 18915412ba9755794753e15499742efb0c5f53e8 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Tue, 30 Jan 2024 16:27:49 -0500 Subject: [PATCH 15/27] remove unnecessary postgress env vars from e2e --- .github/workflows/e2e-test.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index d86ea3a..c928046 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -30,11 +30,6 @@ jobs: echo AWS_S3_BUCKET='test-bucket' >> .env echo S3API_SERVICE_PORT='5005' >> .env echo AUTH_LEVEL=0 >> .env - echo POSTGRES_CONN_STRING='postgres://user:password@postgres:5432/db?sslmode=disable' >> .env - echo POSTGRES_PASSWORD='password' >> .env - echo POSTGRES_USER='user' >> .env - echo POSTGRES_DB='db' >> .env - echo PG_LOG_CHECKPOINTS='off' >> .env - name: Substitute secret variables in JSON env: From 921f31f05f1c1d8068c8cc73455c4d9686dcec02 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Wed, 31 Jan 2024 10:04:32 -0500 Subject: [PATCH 16/27] delete unused test file --- e2e-test/staff_table.csv | 1 - 1 file changed, 1 deletion(-) delete mode 100644 e2e-test/staff_table.csv diff --git a/e2e-test/staff_table.csv b/e2e-test/staff_table.csv deleted file mode 100644 index 6f9539d..0000000 --- a/e2e-test/staff_table.csv +++ /dev/null @@ -1 +0,0 @@ -TEST,TEST,TEST,TEST,TEST,TEST \ No newline at end of file From be3b5478a0c747f7190f14d8d664d8b7cce07633 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Wed, 31 Jan 2024 10:36:38 -0500 Subject: [PATCH 17/27] fix list with details to return the same thing --- blobstore/list.go | 13 ++++++------- blobstore/object_content.go | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/blobstore/list.go b/blobstore/list.go index a150883..285178f 100644 --- a/blobstore/list.go +++ b/blobstore/list.go @@ -152,12 +152,12 @@ func (bh *BlobHandler) HandleListByPrefixWithDetail(c echo.Context) error { // Handle directories (common prefixes) dir := ListResult{ ID: count, - Bucket: bucket, - Name: filepath.Base(strings.TrimSuffix(*cp.Prefix, "/")), + Name: filepath.Base(*cp.Prefix), + Size: "", Path: *cp.Prefix, + Type: "", IsDir: true, - Modified: time.Time{}, // Directory might not have a modified time - ModifiedBy: "", // Information might not be available + ModifiedBy: "", } results = append(results, dir) count++ @@ -167,14 +167,13 @@ func (bh *BlobHandler) HandleListByPrefixWithDetail(c echo.Context) error { // Handle files file := ListResult{ ID: count, - Bucket: bucket, Name: filepath.Base(*object.Key), Size: strconv.FormatInt(*object.Size, 10), - Path: *object.Key, + Path: filepath.Dir(*object.Key), Type: filepath.Ext(*object.Key), IsDir: false, Modified: *object.LastModified, - ModifiedBy: "", // Information might not be available + ModifiedBy: "", } results = append(results, file) count++ diff --git a/blobstore/object_content.go b/blobstore/object_content.go index c4e6304..401e001 100644 --- a/blobstore/object_content.go +++ b/blobstore/object_content.go @@ -57,7 +57,7 @@ func (bh *BlobHandler) HandleObjectContents(c echo.Context) error { if err != nil { errMsg := fmt.Errorf("error fetching object's content: %s", err.Error()) log.Error(errMsg.Error()) - if strings.Contains(err.Error(), "object") { + if strings.Contains(err.Error(), "not found") { return c.JSON(http.StatusNotFound, errMsg.Error()) } else { return c.JSON(http.StatusInternalServerError, errMsg.Error()) From b6afe784554310763629dad190696257f673c2c9 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Wed, 12 Jun 2024 12:55:23 -0400 Subject: [PATCH 18/27] remove unnecessary s3Mock declaration --- blobstore/blobhandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blobstore/blobhandler.go b/blobstore/blobhandler.go index 051a8d4..bcaf571 100644 --- a/blobstore/blobhandler.go +++ b/blobstore/blobhandler.go @@ -127,7 +127,7 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) { return nil, errMsg } - S3Ctrl := S3Controller{Sess: sess, S3Svc: s3SVC, S3Mock: false} + S3Ctrl := S3Controller{Sess: sess, S3Svc: s3SVC} // Retrieve the list of buckets for each account result, err := S3Ctrl.ListBuckets() if err != nil { From ec737bc24935808524345e823b794bf92b9d1694 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Wed, 12 Jun 2024 12:56:26 -0400 Subject: [PATCH 19/27] rename copyPrefix to MovePrefix --- blobstore/move.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blobstore/move.go b/blobstore/move.go index 26fd1ff..8981cc0 100644 --- a/blobstore/move.go +++ b/blobstore/move.go @@ -34,7 +34,7 @@ func (bh *BlobHandler) HandleMovePrefix(c echo.Context) error { log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } - err = s3Ctrl.CopyPrefix(bucket, srcPrefix, destPrefix) + err = s3Ctrl.MovePrefix(bucket, srcPrefix, destPrefix) if err != nil { if strings.Contains(err.Error(), "source prefix not found") { errMsg := fmt.Errorf("no objects found with source prefix: %s", srcPrefix) @@ -47,7 +47,7 @@ func (bh *BlobHandler) HandleMovePrefix(c echo.Context) error { return c.JSON(http.StatusOK, fmt.Sprintf("Successfully moved prefix from %s to %s", srcPrefix, destPrefix)) } -func (s3Ctrl *S3Controller) CopyPrefix(bucket, srcPrefix, destPrefix string) error { +func (s3Ctrl *S3Controller) MovePrefix(bucket, srcPrefix, destPrefix string) error { var objectsFound bool processPage := func(page *s3.ListObjectsV2Output) error { From 78f11018e5d435a5ad57c822478714abed9f9b9d Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Wed, 12 Jun 2024 13:02:44 -0400 Subject: [PATCH 20/27] change all occurences of errors.New to fmt.errorf --- blobstore/buckets.go | 6 +++--- blobstore/metadata.go | 7 +++---- blobstore/move.go | 9 ++++----- env-checker/env_checker.go | 3 +-- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/blobstore/buckets.go b/blobstore/buckets.go index c4df444..45eac21 100644 --- a/blobstore/buckets.go +++ b/blobstore/buckets.go @@ -120,7 +120,7 @@ func (bh *BlobHandler) HandleListBuckets(c echo.Context) error { // bucketName := c.QueryParam("name") // if bucketName == "" { -// err := errors.New("request must include a `name` parameter") +// err := fmt.Errorf("request must include a `name` parameter") // log.Info("HandleCreateBucket: " + err.Error()) // return c.JSON(http.StatusBadRequest, err.Error()) // } @@ -155,7 +155,7 @@ func (bh *BlobHandler) HandleListBuckets(c echo.Context) error { // bucketName := c.QueryParam("name") // if bucketName == "" { -// err := errors.New("request must include a `name` parameter") +// err := fmt.Errorf("request must include a `name` parameter") // log.Info("HandleDeleteBucket: " + err.Error()) // return c.JSON(http.StatusBadRequest, err.Error()) // } @@ -175,7 +175,7 @@ func (bh *BlobHandler) HandleListBuckets(c echo.Context) error { // bucketName := c.QueryParam("name") // if bucketName == "" { -// err := errors.New("request must include a `name` parameter") +// err := fmt.Errorf("request must include a `name` parameter") // log.Info("HandleGetBucketACL: " + err.Error()) // return c.JSON(http.StatusBadRequest, err.Error()) // } diff --git a/blobstore/metadata.go b/blobstore/metadata.go index 9ecb2ad..95629a4 100644 --- a/blobstore/metadata.go +++ b/blobstore/metadata.go @@ -1,7 +1,6 @@ package blobstore import ( - "errors" "fmt" "net/http" @@ -14,12 +13,12 @@ import ( func (bh *BlobHandler) GetSize(page *s3.ListObjectsV2Output, totalSize *uint64, fileCount *uint64) error { if page == nil { - return errors.New("input page is nil") + return fmt.Errorf("input page is nil") } for _, file := range page.Contents { if file.Size == nil { - return errors.New("file size is nil") + return fmt.Errorf("file size is nil") } *totalSize += uint64(*file.Size) *fileCount++ @@ -32,7 +31,7 @@ func (bh *BlobHandler) GetSize(page *s3.ListObjectsV2Output, totalSize *uint64, func (bh *BlobHandler) HandleGetSize(c echo.Context) error { prefix := c.QueryParam("prefix") if prefix == "" { - errMsg := errors.New("request must include a `prefix` parameter") + errMsg := fmt.Errorf("request must include a `prefix` parameter") log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } diff --git a/blobstore/move.go b/blobstore/move.go index 8981cc0..07b0b49 100644 --- a/blobstore/move.go +++ b/blobstore/move.go @@ -1,7 +1,6 @@ package blobstore import ( - "errors" "fmt" "net/http" "strings" @@ -142,7 +141,7 @@ func (s3Ctrl *S3Controller) CopyObject(bucket, srcObjectKey, destObjectKey strin return fmt.Errorf("error checking if object %s exists: %s", destObjectKey, err.Error()) } if !oldKeyExists { - return errors.New("`srcObjectKey` " + srcObjectKey + " does not exist") + return fmt.Errorf("`srcObjectKey` " + srcObjectKey + " does not exist") } // Check if the new key already exists in the bucket newKeyExists, err := s3Ctrl.KeyExists(bucket, destObjectKey) @@ -150,7 +149,7 @@ func (s3Ctrl *S3Controller) CopyObject(bucket, srcObjectKey, destObjectKey strin return fmt.Errorf("error checking if object %s exists: %s", destObjectKey, err.Error()) } if newKeyExists { - return errors.New(destObjectKey + " already exists in the bucket; duplication will cause an overwrite. Please rename dest_key to a different name") + return fmt.Errorf(destObjectKey + " already exists in the bucket; duplication will cause an overwrite. Please rename dest_key to a different name") } // Set up input parameters for the CopyObject API to rename the object copyInput := &s3.CopyObjectInput{ @@ -162,7 +161,7 @@ func (s3Ctrl *S3Controller) CopyObject(bucket, srcObjectKey, destObjectKey strin // Copy the object to the new key (effectively renaming) _, err = s3Ctrl.S3Svc.CopyObject(copyInput) if err != nil { - return errors.New("error copying object" + srcObjectKey + "with the new key" + destObjectKey + ", " + err.Error()) + return fmt.Errorf("error copying object" + srcObjectKey + "with the new key" + destObjectKey + ", " + err.Error()) } // Delete the source object @@ -171,7 +170,7 @@ func (s3Ctrl *S3Controller) CopyObject(bucket, srcObjectKey, destObjectKey strin Key: aws.String(srcObjectKey), }) if err != nil { - return errors.New("error deleting old object " + srcObjectKey + " in bucket " + bucket + ", " + err.Error()) + return fmt.Errorf("error deleting old object " + srcObjectKey + " in bucket " + bucket + ", " + err.Error()) } return nil diff --git a/env-checker/env_checker.go b/env-checker/env_checker.go index d903e52..e76af80 100644 --- a/env-checker/env_checker.go +++ b/env-checker/env_checker.go @@ -1,7 +1,6 @@ package envcheck import ( - "errors" "fmt" "os" ) @@ -20,7 +19,7 @@ func CheckEnvVariablesExist(envVars []string) error { if len(missingVars) > 0 { errMsg := fmt.Sprintf("The following environment variables are missing: %v", missingVars) - return errors.New(errMsg) + return fmt.Errorf(errMsg) } return nil From a14f5e64a23ff0f7b12ab67a91cf216ec97f7a92 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Wed, 12 Jun 2024 13:30:58 -0400 Subject: [PATCH 21/27] refactor prefix checks --- blobstore/list.go | 91 +++++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 51 deletions(-) diff --git a/blobstore/list.go b/blobstore/list.go index 285178f..2a29f56 100644 --- a/blobstore/list.go +++ b/blobstore/list.go @@ -27,6 +27,30 @@ type ListResult struct { ModifiedBy string `json:"modified_by"` } +// CheckAndAdjustPrefix checks if the prefix is an object and adjusts the prefix accordingly. +// Returns the adjusted prefix, an error message (if any), and the HTTP status code. +func CheckAndAdjustPrefix(s3Ctrl *S3Controller, bucket, prefix string) (string, string, int) { + if prefix != "" && prefix != "./" && prefix != "/" { + isObject, err := s3Ctrl.KeyExists(bucket, prefix) + if err != nil { + return "", fmt.Sprintf("error checking if key exists: %s", err.Error()), http.StatusInternalServerError + } + if isObject { + objMeta, err := s3Ctrl.GetMetaData(bucket, prefix) + if err != nil { + return "", fmt.Sprintf("error checking for object's metadata: %s", err.Error()), http.StatusInternalServerError + } + if *objMeta.ContentLength == 0 { + log.Infof("detected a zero byte directory marker within prefix: %s", prefix) + } else { + return "", fmt.Sprintf("`%s` is an object, not a prefix. please see options for keys or pass a prefix", prefix), http.StatusTeapot + } + } + prefix = strings.Trim(prefix, "/") + "/" + } + return prefix, "", http.StatusOK +} + // HandleListByPrefix handles the API endpoint for listing objects by prefix in S3 bucket. func (bh *BlobHandler) HandleListByPrefix(c echo.Context) error { prefix := c.QueryParam("prefix") @@ -47,7 +71,6 @@ func (bh *BlobHandler) HandleListByPrefix(c echo.Context) error { delimiterParam := c.QueryParam("delimiter") var delimiter bool if delimiterParam == "true" || delimiterParam == "false" { - var err error delimiter, err = strconv.ParseBool(delimiterParam) if err != nil { errMsg := fmt.Errorf("error parsing `delimiter` param: %s", err.Error()) @@ -61,34 +84,16 @@ func (bh *BlobHandler) HandleListByPrefix(c echo.Context) error { return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } - if delimiter { - if !strings.HasSuffix(prefix, "/") { - prefix = prefix + "/" - } - } - - isObject, err := s3Ctrl.KeyExists(bucket, prefix) - if err != nil { - errMsg := fmt.Errorf("can't find bucket or object %s" + err.Error()) - log.Error(errMsg.Error()) - return c.JSON(http.StatusInternalServerError, errMsg.Error()) + if delimiter && !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" } - if isObject { - objMeta, err := s3Ctrl.GetMetaData(bucket, prefix) - if err != nil { - errMsg := fmt.Errorf("error getting metadata: %s" + err.Error()) - log.Error(errMsg.Error()) - return c.JSON(http.StatusInternalServerError, errMsg.Error()) - } - if *objMeta.ContentLength == 0 { - log.Infof("Detected a zero byte directory marker within prefix: %s", prefix) - } else { - errMsg := fmt.Errorf("`%s` is an object, not a prefix. please see options for keys or pass a prefix", prefix) - log.Error(errMsg.Error()) - return c.JSON(http.StatusTeapot, errMsg.Error()) - } + adjustedPrefix, errMsg, statusCode := CheckAndAdjustPrefix(s3Ctrl, bucket, prefix) + if errMsg != "" { + log.Error(errMsg) + return c.JSON(statusCode, errMsg) } + prefix = adjustedPrefix var objectKeys []string processPage := func(page *s3.ListObjectsV2Output) error { @@ -100,7 +105,9 @@ func (bh *BlobHandler) HandleListByPrefix(c echo.Context) error { err = s3Ctrl.GetListWithCallBack(bucket, prefix, delimiter, processPage) if err != nil { - return c.JSON(http.StatusInternalServerError, fmt.Sprintf("Error processing objects: %v", err)) + errMsg := fmt.Errorf("error processing objects: %s", err.Error()) + log.Error(errMsg.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) } log.Info("Successfully retrieved list by prefix:", prefix) @@ -119,30 +126,12 @@ func (bh *BlobHandler) HandleListByPrefixWithDetail(c echo.Context) error { return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } - if prefix != "" && prefix != "./" && prefix != "/" { - isObject, err := s3Ctrl.KeyExists(bucket, prefix) - if err != nil { - errMsg := fmt.Errorf("error checking if key exists: %s", err.Error()) - log.Error(errMsg.Error()) - return c.JSON(http.StatusInternalServerError, errMsg.Error()) - } - if isObject { - objMeta, err := s3Ctrl.GetMetaData(bucket, prefix) - if err != nil { - errMsg := fmt.Errorf("error checking for object's metadata: %s", err.Error()) - log.Error(errMsg.Error()) - return c.JSON(http.StatusInternalServerError, errMsg.Error()) - } - if *objMeta.ContentLength == 0 { - log.Infof("detected a zero byte directory marker within prefix: %s", prefix) - } else { - errMsg := fmt.Errorf("`%s` is an object, not a prefix. please see options for keys or pass a prefix", prefix) - log.Error(errMsg.Error()) - return c.JSON(http.StatusTeapot, errMsg.Error()) - } - } - prefix = strings.Trim(prefix, "/") + "/" + adjustedPrefix, errMsg, statusCode := CheckAndAdjustPrefix(s3Ctrl, bucket, prefix) + if errMsg != "" { + log.Error(errMsg) + return c.JSON(statusCode, errMsg) } + prefix = adjustedPrefix var results []ListResult var count int @@ -188,7 +177,7 @@ func (bh *BlobHandler) HandleListByPrefixWithDetail(c echo.Context) error { return c.JSON(http.StatusInternalServerError, errMsg.Error()) } - log.Info("successfully retrieved detailed list by prefix:", prefix) + log.Info("Successfully retrieved detailed list by prefix:", prefix) return c.JSON(http.StatusOK, results) } From 5111774d897e4d3abcd9fc160d26b4d47902c9f6 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Wed, 12 Jun 2024 13:36:43 -0400 Subject: [PATCH 22/27] consistent error language --- blobstore/delete.go | 4 ++-- blobstore/list.go | 2 +- blobstore/metadata.go | 2 +- blobstore/presigned_url.go | 3 +-- blobstore/upload.go | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/blobstore/delete.go b/blobstore/delete.go index 4a0e8a9..92dfa83 100644 --- a/blobstore/delete.go +++ b/blobstore/delete.go @@ -76,7 +76,7 @@ func (bh *BlobHandler) HandleDeleteObject(c echo.Context) error { // If the key is not a folder, proceed with deleting a single object keyExist, err := s3Ctrl.KeyExists(bucket, key) if err != nil { - errMsg := fmt.Errorf("error checking if key exists: %s", err.Error()) + errMsg := fmt.Errorf("error checking if object exists: %s", err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusInternalServerError, errMsg.Error()) } @@ -195,7 +195,7 @@ func (bh *BlobHandler) HandleDeleteObjectsByList(c echo.Context) error { // Check if the key exists before appending it to the keys list keyExists, err := s3Ctrl.KeyExists(bucket, s3Path) if err != nil { - errMsg := fmt.Errorf("error checking if key exists. %s", err.Error()) + errMsg := fmt.Errorf("error checking if object exists. %s", err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusInternalServerError, errMsg) } diff --git a/blobstore/list.go b/blobstore/list.go index 2a29f56..3b7a4fd 100644 --- a/blobstore/list.go +++ b/blobstore/list.go @@ -33,7 +33,7 @@ func CheckAndAdjustPrefix(s3Ctrl *S3Controller, bucket, prefix string) (string, if prefix != "" && prefix != "./" && prefix != "/" { isObject, err := s3Ctrl.KeyExists(bucket, prefix) if err != nil { - return "", fmt.Sprintf("error checking if key exists: %s", err.Error()), http.StatusInternalServerError + return "", fmt.Sprintf("error checking if object exists: %s", err.Error()), http.StatusInternalServerError } if isObject { objMeta, err := s3Ctrl.GetMetaData(bucket, prefix) diff --git a/blobstore/metadata.go b/blobstore/metadata.go index 95629a4..d6bdcc1 100644 --- a/blobstore/metadata.go +++ b/blobstore/metadata.go @@ -138,7 +138,7 @@ func (bh *BlobHandler) HandleGetObjExist(c echo.Context) error { result, err := s3Ctrl.KeyExists(bucket, key) if err != nil { - errMsg := fmt.Errorf("error checking if key exists: %s", err.Error()) + errMsg := fmt.Errorf("error checking if object exists: %s", err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusInternalServerError, errMsg.Error()) } diff --git a/blobstore/presigned_url.go b/blobstore/presigned_url.go index 9c7bddd..7fb431d 100644 --- a/blobstore/presigned_url.go +++ b/blobstore/presigned_url.go @@ -257,7 +257,7 @@ func (bh *BlobHandler) HandleGetPresignedURLMultiObj(c echo.Context) error { func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { prefix := c.QueryParam("prefix") if prefix == "" { - errMsg := fmt.Errorf("`prefix` and `bucket` query params are required") + errMsg := fmt.Errorf("`prefix` query params are required") log.Error(errMsg.Error()) return c.JSON(http.StatusUnprocessableEntity, errMsg.Error()) } @@ -281,7 +281,6 @@ func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { scriptBuilder.WriteString("REM 3. Rename the File: Right-click on the file, select \"Rename,\" and change the file extension from \".txt\" to \".bat.\" For example, if the file is named \"script.txt,\" rename it to \"script.bat.\"\n") scriptBuilder.WriteString("REM 4. Initiate the Download: Double-click the renamed \".bat\" file to initiate the download process. Windows might display a warning message to protect your PC.\n") scriptBuilder.WriteString("REM 5. Windows Defender SmartScreen (Optional): If you see a message like \"Windows Defender SmartScreen prevented an unrecognized app from starting,\" click \"More info\" and then click \"Run anyway\" to proceed with the download.\n\n") - scriptBuilder.WriteString(fmt.Sprintf("mkdir \"%s\"\n", basePrefix)) // Define the processPage function processPage := func(page *s3.ListObjectsV2Output) error { diff --git a/blobstore/upload.go b/blobstore/upload.go index 60091b5..ac3b974 100644 --- a/blobstore/upload.go +++ b/blobstore/upload.go @@ -170,7 +170,7 @@ func (bh *BlobHandler) HandleMultipartUpload(c echo.Context) error { keyExist, err := s3Ctrl.KeyExists(bucket, key) if err != nil { - errMsg := fmt.Errorf("error checking if key exists: %s", err.Error()) + errMsg := fmt.Errorf("error checking if object exists: %s", err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusInternalServerError, errMsg.Error()) } From 286bfe7d6b7b7544eaa9c724861167cf1a3c1773 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Wed, 12 Jun 2024 13:42:26 -0400 Subject: [PATCH 23/27] change no prefix found check to test file count and not file size --- blobstore/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blobstore/metadata.go b/blobstore/metadata.go index d6bdcc1..c0ae9a5 100644 --- a/blobstore/metadata.go +++ b/blobstore/metadata.go @@ -69,7 +69,7 @@ func (bh *BlobHandler) HandleGetSize(c echo.Context) error { log.Error(errMsg.Error()) return c.JSON(http.StatusInternalServerError, errMsg.Error()) } - if totalSize == 0 { + if fileCount == 0 { errMsg := fmt.Errorf("prefix %s not found", prefix) log.Error(errMsg.Error()) return c.JSON(http.StatusNotFound, errMsg.Error()) From 3efcb656e0ddedfd1975bf4aca2bd8106223c2bb Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Wed, 12 Jun 2024 13:54:11 -0400 Subject: [PATCH 24/27] test without temp minio client --- blobstore/upload.go | 46 +++++++++------------------------------------ 1 file changed, 9 insertions(+), 37 deletions(-) diff --git a/blobstore/upload.go b/blobstore/upload.go index ac3b974..10224c3 100644 --- a/blobstore/upload.go +++ b/blobstore/upload.go @@ -9,7 +9,6 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/labstack/echo/v4" @@ -213,43 +212,16 @@ func (s3Ctrl *S3Controller) GetUploadPresignedURL(bucket string, key string, exp // function to retrieve presigned url for a multipart upload part. func (s3Ctrl *S3Controller) GetUploadPartPresignedURL(bucket string, key string, uploadID string, partNumber int64, expMin int) (string, error) { duration := time.Duration(expMin) * time.Minute - var urlStr string - var err error - if s3Ctrl.S3Mock { - // Create a temporary S3 client with the modified endpoint - tempS3Svc, err := session.NewSession(&aws.Config{ - Endpoint: aws.String("http://localhost:9000"), - Region: s3Ctrl.S3Svc.Config.Region, - Credentials: s3Ctrl.S3Svc.Config.Credentials, - S3ForcePathStyle: aws.Bool(true), - }) - if err != nil { - return "", fmt.Errorf("error creating temporary s3 session: %s", err.Error()) - } - // Generate the request using the temporary client - req, _ := s3.New(tempS3Svc).UploadPartRequest(&s3.UploadPartInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - UploadId: aws.String(uploadID), - PartNumber: aws.Int64(partNumber), - }) - urlStr, err = req.Presign(duration) - if err != nil { - return "", err - } - } else { - // Generate the request using the original client - req, _ := s3Ctrl.S3Svc.UploadPartRequest(&s3.UploadPartInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - UploadId: aws.String(uploadID), - PartNumber: aws.Int64(partNumber), - }) - urlStr, err = req.Presign(duration) - if err != nil { - return "", err - } + req, _ := s3Ctrl.S3Svc.UploadPartRequest(&s3.UploadPartInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: aws.String(uploadID), + PartNumber: aws.Int64(partNumber), + }) + urlStr, err := req.Presign(duration) + if err != nil { + return "", err } return urlStr, nil From 92f47712c21bc6699c9bca9cf00d390900803c99 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Wed, 12 Jun 2024 13:57:14 -0400 Subject: [PATCH 25/27] revert temp minio client --- blobstore/upload.go | 46 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/blobstore/upload.go b/blobstore/upload.go index 10224c3..ac3b974 100644 --- a/blobstore/upload.go +++ b/blobstore/upload.go @@ -9,6 +9,7 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/labstack/echo/v4" @@ -212,16 +213,43 @@ func (s3Ctrl *S3Controller) GetUploadPresignedURL(bucket string, key string, exp // function to retrieve presigned url for a multipart upload part. func (s3Ctrl *S3Controller) GetUploadPartPresignedURL(bucket string, key string, uploadID string, partNumber int64, expMin int) (string, error) { duration := time.Duration(expMin) * time.Minute + var urlStr string + var err error + if s3Ctrl.S3Mock { + // Create a temporary S3 client with the modified endpoint + tempS3Svc, err := session.NewSession(&aws.Config{ + Endpoint: aws.String("http://localhost:9000"), + Region: s3Ctrl.S3Svc.Config.Region, + Credentials: s3Ctrl.S3Svc.Config.Credentials, + S3ForcePathStyle: aws.Bool(true), + }) + if err != nil { + return "", fmt.Errorf("error creating temporary s3 session: %s", err.Error()) + } - req, _ := s3Ctrl.S3Svc.UploadPartRequest(&s3.UploadPartInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - UploadId: aws.String(uploadID), - PartNumber: aws.Int64(partNumber), - }) - urlStr, err := req.Presign(duration) - if err != nil { - return "", err + // Generate the request using the temporary client + req, _ := s3.New(tempS3Svc).UploadPartRequest(&s3.UploadPartInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: aws.String(uploadID), + PartNumber: aws.Int64(partNumber), + }) + urlStr, err = req.Presign(duration) + if err != nil { + return "", err + } + } else { + // Generate the request using the original client + req, _ := s3Ctrl.S3Svc.UploadPartRequest(&s3.UploadPartInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + UploadId: aws.String(uploadID), + PartNumber: aws.Int64(partNumber), + }) + urlStr, err = req.Presign(duration) + if err != nil { + return "", err + } } return urlStr, nil From 36b1c3bc7a90192074479a11b15e53e2fa0efe16 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Wed, 12 Jun 2024 14:20:25 -0400 Subject: [PATCH 26/27] add explanatory comments --- blobstore/list.go | 1 + blobstore/upload.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/blobstore/list.go b/blobstore/list.go index 3b7a4fd..3efeb1e 100644 --- a/blobstore/list.go +++ b/blobstore/list.go @@ -40,6 +40,7 @@ func CheckAndAdjustPrefix(s3Ctrl *S3Controller, bucket, prefix string) (string, if err != nil { return "", fmt.Sprintf("error checking for object's metadata: %s", err.Error()), http.StatusInternalServerError } + //this is because AWS considers empty prefixes with a .keep as an object, so we ignnore and log if *objMeta.ContentLength == 0 { log.Infof("detected a zero byte directory marker within prefix: %s", prefix) } else { diff --git a/blobstore/upload.go b/blobstore/upload.go index ac3b974..e26ea70 100644 --- a/blobstore/upload.go +++ b/blobstore/upload.go @@ -217,6 +217,8 @@ func (s3Ctrl *S3Controller) GetUploadPartPresignedURL(bucket string, key string, var err error if s3Ctrl.S3Mock { // Create a temporary S3 client with the modified endpoint + //this is done so that the presigned url starts with localhost:9000 instead of + //minio:9000 which would cause an error due to cors origin policy tempS3Svc, err := session.NewSession(&aws.Config{ Endpoint: aws.String("http://localhost:9000"), Region: s3Ctrl.S3Svc.Config.Region, From 0c976c9527c50421a7185f6a49ccf9d97d1401ef Mon Sep 17 00:00:00 2001 From: ShaneMPutnam Date: Wed, 12 Jun 2024 19:51:29 +0000 Subject: [PATCH 27/27] Add comment, reorder download script --- blobstore/delete.go | 2 +- blobstore/list.go | 3 ++- blobstore/presigned_url.go | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/blobstore/delete.go b/blobstore/delete.go index 92dfa83..47fafe1 100644 --- a/blobstore/delete.go +++ b/blobstore/delete.go @@ -154,7 +154,7 @@ func (s3Ctrl *S3Controller) DeleteKeys(bucket string, key []string) error { _, err := s3Ctrl.S3Svc.DeleteObjects(input) if err != nil { - return fmt.Errorf("error Deleting objects: %s", err.Error()) + return fmt.Errorf("error deleting objects: %s", err.Error()) } return nil } diff --git a/blobstore/list.go b/blobstore/list.go index 3efeb1e..b8d96c1 100644 --- a/blobstore/list.go +++ b/blobstore/list.go @@ -30,6 +30,7 @@ type ListResult struct { // CheckAndAdjustPrefix checks if the prefix is an object and adjusts the prefix accordingly. // Returns the adjusted prefix, an error message (if any), and the HTTP status code. func CheckAndAdjustPrefix(s3Ctrl *S3Controller, bucket, prefix string) (string, string, int) { + //As of 6/12/24, unsure why ./ is included here, may be needed for an edge case, but could also cause problems if prefix != "" && prefix != "./" && prefix != "/" { isObject, err := s3Ctrl.KeyExists(bucket, prefix) if err != nil { @@ -40,7 +41,7 @@ func CheckAndAdjustPrefix(s3Ctrl *S3Controller, bucket, prefix string) (string, if err != nil { return "", fmt.Sprintf("error checking for object's metadata: %s", err.Error()), http.StatusInternalServerError } - //this is because AWS considers empty prefixes with a .keep as an object, so we ignnore and log + //this is because AWS considers empty prefixes with a .keep as an object, so we ignore and log if *objMeta.ContentLength == 0 { log.Infof("detected a zero byte directory marker within prefix: %s", prefix) } else { diff --git a/blobstore/presigned_url.go b/blobstore/presigned_url.go index 7fb431d..a4f7c55 100644 --- a/blobstore/presigned_url.go +++ b/blobstore/presigned_url.go @@ -131,7 +131,7 @@ func (bh *BlobHandler) HandleGetPresignedDownloadURL(c echo.Context) error { keyExist, err := s3Ctrl.KeyExists(bucket, key) if err != nil { - errMsg := fmt.Errorf("checking if key exists: %s", err.Error()) + errMsg := fmt.Errorf("checking if object exists: %s", err.Error()) log.Error(errMsg.Error()) return c.JSON(http.StatusInternalServerError, errMsg.Error()) } @@ -273,7 +273,6 @@ func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { var scriptBuilder strings.Builder createdDirs := make(map[string]bool) basePrefix := filepath.Base(strings.TrimSuffix(prefix, "/")) - scriptBuilder.WriteString(fmt.Sprintf("mkdir \"%s\"\n", basePrefix)) scriptBuilder.WriteString("REM Download Instructions\n") scriptBuilder.WriteString("REM To download the selected directory or file, please follow these steps:\n\n") scriptBuilder.WriteString("REM 1. Locate the Downloaded File: Find the file you just downloaded. It should have a .txt file extension.\n") @@ -281,6 +280,7 @@ func (bh *BlobHandler) HandleGenerateDownloadScript(c echo.Context) error { scriptBuilder.WriteString("REM 3. Rename the File: Right-click on the file, select \"Rename,\" and change the file extension from \".txt\" to \".bat.\" For example, if the file is named \"script.txt,\" rename it to \"script.bat.\"\n") scriptBuilder.WriteString("REM 4. Initiate the Download: Double-click the renamed \".bat\" file to initiate the download process. Windows might display a warning message to protect your PC.\n") scriptBuilder.WriteString("REM 5. Windows Defender SmartScreen (Optional): If you see a message like \"Windows Defender SmartScreen prevented an unrecognized app from starting,\" click \"More info\" and then click \"Run anyway\" to proceed with the download.\n\n") + scriptBuilder.WriteString(fmt.Sprintf("mkdir \"%s\"\n", basePrefix)) // Define the processPage function processPage := func(page *s3.ListObjectsV2Output) error {