Skip to content

Commit d09c25c

Browse files
zhoufeng1989zhoufeng1989
authored
Add universal option for object_storage and queue. (#20)
* Add universal option for object_storage and queue. * Rename APIs. * Add path in ObjectStorageOption.
1 parent 5267762 commit d09c25c

File tree

12 files changed

+407
-153
lines changed

12 files changed

+407
-153
lines changed

‎cloud/common.go‎

Lines changed: 65 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package cloud
22

33
import (
44
"errors"
5-
"fmt"
65

76
"github.com/aws/aws-sdk-go/aws"
87
"github.com/aws/aws-sdk-go/aws/credentials"
@@ -13,14 +12,17 @@ import (
1312
type Provider string
1413

1514
const (
16-
AWSProvider Provider = "aws"
17-
TencentCloudProvider Provider = "tencentcloud"
15+
AWSProvider Provider = "aws"
16+
TencentCloudProvider Provider = "tencentcloud"
17+
AliCloudProvider Provider = "alicloud"
18+
1819
StandaloneRedisProvider Provider = "standalone_redis"
1920
ClusterRedisProvider Provider = "cluster_redis"
2021
StandaloneRedisProviderV7 Provider = "standalone_redis_v7"
2122
ClusterRedisProviderV7 Provider = "cluster_redis_v7"
22-
AliCloudStorageProvider Provider = "alicloud_storage"
23-
AliCloudMNSQueueProvider Provider = "alicloud_mns_queue"
23+
24+
// AliCloudStorageProvider is deprecated, use AliCloudProvider instead.
25+
AliCloudStorageProvider Provider = "alicloud_storage"
2426
)
2527

2628
type AliCloudCredentialType string
@@ -41,7 +43,7 @@ const (
4143
)
4244

4345
var (
44-
ErrUnsupportedCloudProvider = fmt.Errorf("unsupported provider, only support %s, %s and %s", AWSProvider, TencentCloudProvider, StandaloneRedisProvider)
46+
ErrUnsupportedCloudProvider = errors.New("unsupported provider")
4547
ErrProviderNotTencentCloud = errors.New("provider is not tencentcloud")
4648
ErrProviderNotAWS = errors.New("provider is not aws")
4749
ErrProviderNotStandaloneRedis = errors.New("provider is not standalone redis")
@@ -66,13 +68,35 @@ type Option interface {
6668
CheckAliCloudStorage() error
6769
}
6870

71+
type CommonCloudOption struct {
72+
SecretID string `json:"secret_id" yaml:"secret_id"`
73+
SecretKey string `json:"secret_key" yaml:"secret_key"`
74+
AssumeRoleArn string `json:"assume_role_arn" yaml:"assume_role_arn"`
75+
Region string `json:"region" yaml:"region"`
76+
AssumeRegion string `json:"assume_region" yaml:"assume_region"`
77+
}
78+
79+
type AWSOption CommonCloudOption
80+
81+
func (option AWSOption) Check() error {
82+
if option.Region == "" {
83+
return ErrEmptyRegion
84+
}
85+
validKeyConfig := option.SecretID != "" && option.SecretKey != ""
86+
validAssumeRoleConfig := option.AssumeRoleArn != "" && option.AssumeRegion != ""
87+
if !validKeyConfig && !validAssumeRoleConfig {
88+
return errors.New("must have valid key pairs config or assume role config")
89+
}
90+
return nil
91+
}
92+
6993
type CommonOption struct {
70-
Provider Provider
71-
SecretID string
72-
SecretKey string
73-
AssumeRoleArn string
74-
Region string
75-
AssumeRegion string
94+
Provider Provider `json:"provider" yaml:"provider"`
95+
SecretID string `json:"secret_id" yaml:"secret_id"`
96+
SecretKey string `json:"secret_key" yaml:"secret_key"`
97+
AssumeRoleArn string `json:"assume_role_arn" yaml:"assume_role_arn"`
98+
Region string `json:"region" yaml:"region"`
99+
AssumeRegion string `json:"assume_region" yaml:"assume_region"`
76100
}
77101

78102
func (option CommonOption) GetProvider() Provider {
@@ -134,7 +158,7 @@ func (option CommonOption) CheckTencentCloud() error {
134158
}
135159

136160
func (option CommonOption) CheckAliCloudStorage() error {
137-
if option.Provider != AliCloudStorageProvider {
161+
if option.Provider != AliCloudStorageProvider && option.Provider != AliCloudProvider {
138162
return ErrProviderNotAliCloudStorage
139163
}
140164
return option.check()
@@ -173,3 +197,31 @@ func AwsNewSession(option Option) (*session.Session, *aws.Config, error) {
173197
}
174198
return sess, nil, nil
175199
}
200+
201+
func AwsNewSessionWithOption(option AWSOption) (*session.Session, *aws.Config, error) {
202+
if err := option.Check(); err != nil {
203+
return nil, nil, err
204+
}
205+
var creds *credentials.Credentials
206+
if option.SecretID != "" && option.SecretKey != "" {
207+
creds = credentials.NewStaticCredentials(option.SecretID, option.SecretKey, "")
208+
}
209+
210+
sess, err := session.NewSessionWithOptions(session.Options{
211+
Config: aws.Config{
212+
CredentialsChainVerboseErrors: aws.Bool(true),
213+
Credentials: creds, // 可能是nil
214+
// LogLevel: aws.LogLevel(aws.LogDebug),
215+
Region: aws.String(option.Region),
216+
},
217+
SharedConfigFiles: []string{},
218+
})
219+
if err != nil {
220+
return nil, nil, err
221+
}
222+
if roleArn := option.AssumeRoleArn; roleArn != "" { // 切换 assumeRole
223+
assumeRoleCreds := stscreds.NewCredentials(sess, roleArn)
224+
return sess, aws.NewConfig().WithCredentials(assumeRoleCreds).WithRegion(option.AssumeRegion), nil
225+
}
226+
return sess, nil, nil
227+
}

‎cloud/examples/object_storage/object_storage.go‎

Lines changed: 35 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,49 +5,60 @@ import (
55
"context"
66
"errors"
77
"fmt"
8-
"github.com/aws/aws-sdk-go/aws"
98
"io"
109
"log"
1110
"net/http"
1211
"time"
1312

13+
"github.com/aws/aws-sdk-go/aws"
14+
1415
"github.com/byte-power/gorich/cloud"
1516
"github.com/byte-power/gorich/cloud/object_storage"
1617
)
1718

1819
// Configure secret_id, secret_key, region and bucket_name to run this example.
1920
func main() {
20-
optionForTencentCloud := cloud.CommonOption{
21-
Provider: cloud.TencentCloudProvider,
22-
SecretID: "tencentcloud_secret_id_xxx",
23-
SecretKey: "tencentcloud_secret_key_xxx",
24-
Region: "tencentcloud_region_xxx",
21+
option := object_storage.ObjectStorageOption{
22+
Provider: cloud.TencentCloudProvider,
23+
Bucket: "tencent_bucket_name_xxx",
24+
COS: object_storage.COSOption{
25+
SecretID: "tencentcloud_secret_id_xxx",
26+
SecretKey: "tencentcloud_secret_key_xxx",
27+
Region: "tencentcloud_region_xxx",
28+
},
2529
}
26-
object_storage_examples("tencentcloud_bucket_name_xxx", optionForTencentCloud)
30+
object_storage_examples(option)
2731

28-
optionForAWS := cloud.CommonOption{
29-
Provider: cloud.AWSProvider,
30-
SecretID: "aws_secret_id_xxx",
31-
SecretKey: "aws_secret_key_xxx",
32-
Region: "aws_region_xxx",
32+
option = object_storage.ObjectStorageOption{
33+
Provider: cloud.AWSProvider,
34+
Bucket: "aws_bucket_name_xxx",
35+
S3: cloud.AWSOption{
36+
SecretID: "aws_secret_id_xxx",
37+
SecretKey: "aws_secret_key_xxx",
38+
Region: "aws_region_xxx",
39+
},
3340
}
34-
object_storage_examples("aws_bucket_name_xxx", optionForAWS)
41+
object_storage_examples(option)
3542

36-
optionForAliOSS := object_storage.AliCloudStorageOption{
37-
//CredentialType: "oidc_role_arn",
38-
//EndPoint: "oss-cn-zhangjiakou.aliyuncs.com",
39-
//SessionName: "test-rrsa-oidc-token",
43+
option = object_storage.ObjectStorageOption{
44+
Provider: cloud.AliCloudProvider,
45+
Bucket: "alicloud_bucket_name_xxx",
46+
OSS: object_storage.AliCloudStorageOption{
47+
//CredentialType: "oidc_role_arn",
48+
//EndPoint: "oss-cn-zhangjiakou.aliyuncs.com",
49+
//SessionName: "test-rrsa-oidc-token",
4050

41-
CredentialType: "ak",
42-
EndPoint: "oss-cn-beijing.aliyuncs.com",
43-
AccessKeyID: "alicloud_access_key_id_xxx",
44-
AccessKeySecret: "alicloud_access_key_secret_xxx",
51+
CredentialType: "ak",
52+
EndPoint: "oss-cn-beijing.aliyuncs.com",
53+
AccessKeyID: "alicloud_access_key_id_xxx",
54+
AccessKeySecret: "alicloud_access_key_secret_xxx",
55+
},
4556
}
46-
object_storage_examples("alicloud_bucket_name_xxx", optionForAliOSS)
57+
object_storage_examples(option)
4758
}
4859

49-
func object_storage_examples(bucketName string, option cloud.Option) {
50-
service, err := object_storage.GetObjectStorageService(bucketName, option)
60+
func object_storage_examples(option object_storage.ObjectStorageOption) {
61+
service, err := object_storage.GetObjectStorageServiceWithOption(option)
5162
if err != nil {
5263
fmt.Printf("GetObjectStorageService error:%s\n", err)
5364
return

‎cloud/examples/queue/queue.go‎

Lines changed: 74 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -14,86 +14,99 @@ import (
1414
// Configure secret_id, secret_key, region, and queue_name to run this example.
1515
func main() {
1616

17-
// Redis 单节点
18-
//optionForBaseRedis := queue.StandaloneRedisQueueOption{
19-
// Addr: "localhost:6379",
20-
// Password: "",
21-
// ConsumerGroup: "save_task_consumer_group",
22-
// Idle: 10,
23-
//}
24-
25-
// Redis 集群
26-
optionForBaseRedis := queue.ClusterRedisQueueOption{
27-
Addrs: []string{
28-
"localhost:7000",
29-
"localhost:7001",
30-
"localhost:7002",
31-
"localhost:7003",
32-
"localhost:7004",
33-
"localhost:7005",
17+
option := queue.QueueOption{
18+
Provider: cloud.ClusterRedisProvider,
19+
QueueName: "test_queue_name",
20+
ClusterRedis: queue.ClusterRedisQueueOption{
21+
Addrs: []string{
22+
"localhost:7000",
23+
"localhost:7001",
24+
"localhost:7002",
25+
"localhost:7003",
26+
"localhost:7004",
27+
"localhost:7005",
28+
},
29+
Password: "",
30+
ConsumerGroup: "save_task_consumer_group",
31+
Idle: 10,
3432
},
35-
Password: "",
36-
ConsumerGroup: "save_task_consumer_group",
37-
Idle: 10,
3833
}
34+
queue_examples(option)
3935

40-
queue_examples("test_queue_name", optionForBaseRedis)
41-
42-
optionForTencentCloud := queue.TencentCloudQueueOption{
43-
Token: "access_jwt_token_xxx",
44-
URL: "http://pulsar-xxxxxxxxx.tdmq.ap-gz.public.tencenttdmq.com:8080",
45-
}
4636
topicName := "pulsar-xxxxxx/namespace_name/topic_name"
4737
subscriptionName := "subscription_name"
4838
topicSub := queue.GenerateTopicAndSubName(topicName, subscriptionName)
49-
queue_examples(topicSub, optionForTencentCloud)
39+
option = queue.QueueOption{
40+
Provider: cloud.TencentCloudProvider,
41+
QueueName: topicSub,
42+
Pulsar: queue.TencentCloudQueueOption{
43+
Token: "access_jwt_token_xxx",
44+
URL: "http://pulsar-xxxxxxxxx.tdmq.ap-gz.public.tencenttdmq.com:8080",
45+
},
46+
}
47+
queue_examples(option)
5048

51-
optionForAWS := cloud.CommonOption{
49+
option = queue.QueueOption{
5250
Provider: cloud.AWSProvider,
53-
SecretID: "aws_secret_id_xxxx",
54-
SecretKey: "aws_secret_key_xxxx",
55-
Region: "aws_region_xxx",
51+
QueueName: "aws_queue_name",
52+
SQS: cloud.AWSOption{
53+
SecretID: "aws_secret_id_xxxx",
54+
SecretKey: "aws_secret_key_xxxx",
55+
Region: "aws_region_xxx",
56+
},
5657
}
57-
queue_examples("aws_queue_name", optionForAWS)
58+
59+
queue_examples(option)
5860

5961
dialTimeout := 5 * time.Second
60-
clusterRedisQueueOptionV7 := queue.ClusterRedisQueueOptionV7{
61-
ClusterRedisQueueOption: queue.ClusterRedisQueueOption{
62-
Addrs: []string{
63-
"localhost:30001",
64-
"localhost:30002",
65-
"localhost:30003",
62+
option = queue.QueueOption{
63+
Provider: cloud.ClusterRedisProviderV7,
64+
QueueName: "redis_cluster_queue_v7",
65+
ClusterRedisV7: queue.ClusterRedisQueueOptionV7{
66+
ClusterRedisQueueOption: queue.ClusterRedisQueueOption{
67+
Addrs: []string{
68+
"localhost:30001",
69+
"localhost:30002",
70+
"localhost:30003",
71+
},
72+
ConsumerGroup: "save_task_consumer_group_2",
73+
DialTimeout: &dialTimeout,
74+
Idle: 10,
6675
},
67-
ConsumerGroup: "save_task_consumer_group_2",
68-
DialTimeout: &dialTimeout,
69-
Idle: 10,
7076
},
7177
}
72-
queue_examples("redis_cluster_queue_v7", clusterRedisQueueOptionV7)
78+
queue_examples(option)
7379

7480
// alicloud MNS queue: access_key CredentialType
75-
mnsQueueOption := queue.AliMNSClientOption{
76-
EndPoint: "http://account-id.mns.region.aliyuncs.com",
77-
CredentialType: cloud.AliCloudAccessKeyCredentialType,
78-
AccessKeyId: "alicloud_access_key_id",
79-
AccessKeySecret: "alicloud_access_key_secret",
80-
MessagePriority: 10,
81+
option = queue.QueueOption{
82+
Provider: cloud.AliCloudProvider,
83+
QueueName: "mns_queue_name",
84+
MNS: queue.AliMNSClientOption{
85+
EndPoint: "http://account-id.mns.region.aliyuncs.com",
86+
CredentialType: cloud.AliCloudAccessKeyCredentialType,
87+
AccessKeyId: "alicloud_access_key_id",
88+
AccessKeySecret: "alicloud_access_key_secret",
89+
MessagePriority: 10,
90+
},
8191
}
82-
queue_examples("mns_queue_name", mnsQueueOption)
92+
queue_examples(option)
8393

8494
// alicloud MNS queue: ecs_ram_role credentialType
85-
mnsQueueOption = queue.AliMNSClientOption{
86-
EndPoint: "http://account-id.mns.region.aliyuncs.com",
87-
CredentialType: cloud.AliCloudECSRamRoleCredentialType,
88-
}
89-
queue_examples("mns_queue_name", mnsQueueOption)
90-
alicloud_mns_queue_examples("mns_queue_name", mnsQueueOption)
95+
option = queue.QueueOption{
96+
Provider: cloud.AliCloudProvider,
97+
QueueName: "mns_queue_name",
98+
MNS: queue.AliMNSClientOption{
99+
EndPoint: "http://account-id.mns.region.aliyuncs.com",
100+
CredentialType: cloud.AliCloudECSRamRoleCredentialType,
101+
}}
102+
queue_examples(option)
103+
alicloud_mns_queue_examples(option)
91104
}
92105

93-
func queue_examples(queueOrTopicName string, option cloud.Option) {
94-
service, err := queue.GetQueueService(queueOrTopicName, option)
106+
func queue_examples(option queue.QueueOption) {
107+
service, err := queue.GetQueueServiceWithOption(option)
95108
if err != nil {
96-
fmt.Printf("get queue service error %s %+v %s\n", queueOrTopicName, option, err)
109+
fmt.Printf("get queue service error %s %+v %s\n", option.QueueName, option, err)
97110
return
98111
}
99112
defer service.Close()
@@ -145,10 +158,10 @@ func queue_examples(queueOrTopicName string, option cloud.Option) {
145158
}
146159

147160
// The following examples show mns queue specific examples: How to set message priority; how to set long polling period seconds.
148-
func alicloud_mns_queue_examples(queueOrTopicName string, option cloud.Option) {
149-
service, err := queue.GetQueueService(queueOrTopicName, option)
161+
func alicloud_mns_queue_examples(option queue.QueueOption) {
162+
service, err := queue.GetQueueServiceWithOption(option)
150163
if err != nil {
151-
fmt.Printf("get queue service error %s %+v %s\n", queueOrTopicName, option, err)
164+
fmt.Printf("get queue service error %s %+v %s\n", option.QueueName, option, err)
152165
return
153166
}
154167
defer service.Close()

‎cloud/object_storage/alicloud_object_storage.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,13 @@ var (
2525
)
2626

2727
type AliCloudStorageOption struct {
28-
CredentialType string // eg: "oidc_role_arn" or "ak"
29-
EndPoint string // eg: "oss-cn-zhangjiakou.aliyuncs.com"
30-
SessionName string // eg: "test-rrsa-oidc-token"
28+
CredentialType string `yaml:"credential_type" json:"credential_type"` // eg: "oidc_role_arn" or "ak"
29+
EndPoint string `yaml:"endpoint" json:"endpoint"` // eg: "oss-cn-zhangjiakou.aliyuncs.com"
30+
SessionName string `yaml:"session_name" json:"session_name"` // eg: "test-rrsa-oidc-token"
3131

3232
// "ak" required
33-
AccessKeyID string
34-
AccessKeySecret string
33+
AccessKeyID string `yaml:"access_key_id" json:"access_key_id"`
34+
AccessKeySecret string `yaml:"access_key_secret" json:"access_key_secret"`
3535
}
3636

3737
func (option AliCloudStorageOption) GetProvider() cloud.Provider {

0 commit comments

Comments
 (0)