Stage provider support amazon, minio and COS (#18839)

1. Remove the check that the provider only supports minio. 2. add 'COS' and 'AMAZON' provider support. Approved by: @badboynt1, @m-schen
matrixorigin · Sep 18, 2024 · 5456f61 · 5456f61
1 parent e2a5a4b
commit 5456f61
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 3 deletions.
diff --git a/pkg/sql/plan/function/stage_test.go b/pkg/sql/plan/function/stage_test.go
@@ -17,8 +17,26 @@ package function
 import (
 	"reflect"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+func TestS3ServiceProvider(t *testing.T) {
+	protocol, err := getS3ServiceFromProvider("cos")
+	require.Nil(t, err)
+	assert.Equal(t, protocol, "s3")
+
+	protocol, err = getS3ServiceFromProvider("amazon")
+	require.Nil(t, err)
+	assert.Equal(t, protocol, "s3")
+
+	protocol, err = getS3ServiceFromProvider("minio")
+	require.Nil(t, err)
+	assert.Equal(t, protocol, "minio")
+
+}
+
 func TestParseDatalink(t *testing.T) {
 
 	type testCase struct {

diff --git a/pkg/sql/plan/function/stage_util.go b/pkg/sql/plan/function/stage_util.go
@@ -50,6 +50,7 @@ const PARAMKEY_PROVIDER = "provider"
 
 const S3_PROVIDER_AMAZON = "amazon"
 const S3_PROVIDER_MINIO = "minio"
+const S3_PROVIDER_COS = "cos"
 
 const S3_SERVICE = "s3"
 const MINIO_SERVICE = "minio"
@@ -154,6 +155,8 @@ func (s *StageDef) ToPath() (mopath string, query string, err error) {
 func getS3ServiceFromProvider(provider string) (string, error) {
 	provider = strings.ToLower(provider)
 	switch provider {
+	case S3_PROVIDER_COS:
+		return S3_SERVICE, nil
 	case S3_PROVIDER_AMAZON:
 		return S3_SERVICE, nil
 	case S3_PROVIDER_MINIO:

diff --git a/pkg/sql/plan/utils.go b/pkg/sql/plan/utils.go
@@ -1598,9 +1598,6 @@ func GetForETLWithType(param *tree.ExternParam, prefix string) (res fileservice.
 		w := csv.NewWriter(buf)
 		opts := []string{"s3-opts", "endpoint=" + param.S3Param.Endpoint, "region=" + param.S3Param.Region, "key=" + param.S3Param.APIKey, "secret=" + param.S3Param.APISecret,
 			"bucket=" + param.S3Param.Bucket, "role-arn=" + param.S3Param.RoleArn, "external-id=" + param.S3Param.ExternalId}
-		if strings.ToLower(param.S3Param.Provider) != "" && strings.ToLower(param.S3Param.Provider) != "minio" {
-			return nil, "", moerr.NewBadConfig(param.Ctx, "the provider only support 'minio' now")
-		}
 		if strings.ToLower(param.S3Param.Provider) == "minio" {
 			opts = append(opts, "is-minio=true")
 		}