管理云商凭证，将不同云商，不同地域的api的key，secret保存到表中，再遍历这些key和secret，将云商的资源进行同步
云商凭证模块
secret的proto文件
apps/secret/pb/secret.proto
syntax = "proto3";
package go8.devcloud.cmdb.secret;
option go_package = "gitee.com/go-course/go8/projects/devcloud/cmdb/apps/secret";
import "apps/resource/pb/resource.proto";
// 基于用户名密码的凭证
// 记忆API Key的凭证
enum TYPE {
API_KEY = 0;
PASSWORD = 1;
}
message CreateSecretRequest {
// 描述
// @gotags: json:"description" validate:"required,lte=100"
string description = 1;
// 厂商
// @gotags: json:"vendor"
resource.VENDOR vendor = 2;
// 允许同步的区域
// @gotags: json:"allow_regions"
string allow_regions = 3;
// 凭证类型
// @gotags: json:"crendential_type"
TYPE crendential_type = 4;
// 服务地址, 云商不用填写
// @gotags: json:"address"
string address = 5;
// key
// @gotags: json:"api_key" validate:"required,lte=100"
string api_key = 6;
// api_secret
// @gotags: json:"api_secret" validate:"required,lte=100"
string api_secret = 7;
// 请求速率限制, 默认1秒5个
// @gotags: json:"request_rate"
int32 request_rate = 8;
// 所属Domain
// @gotags: json:"domain" validate:"required"
string domain = 9;
// 所属Namespace
// @gotags: json:"namespace" validate:"required"
string namespace = 10;
}
message Secret {
// 全局唯一Id
// @gotags: json:"id"
string id = 1;
// 创建时间
// @gotags: json:"create_at"
int64 create_at = 2;
// 创建信息
// @gotags: json:"spec"
CreateSecretRequest spec = 3;
}
message SecretSet {
// @gotags: json:"total"
int64 total = 1;
// @gotags: json:"items"
repeated Secret items = 2;
}
apps/secret/pb/rpc.proto
syntax = "proto3";
package go8.devcloud.cmdb.secret;
option go_package = "gitee.com/go-course/go8/projects/devcloud/cmdb/apps/secret";
import "apps/secret/pb/secret.proto";
import "github.com/infraboard/mcube/pb/page/page.proto";
service RPC {
// 创建
rpc CreateSecret(CreateSecretRequest) returns(Secret);
// 凭证查询
rpc QuerySecret (QuerySecretRequest) returns (SecretSet);
// 基于凭证来进行任务同步
rpc SyncResource(SyncResourceRequest) returns (SyncResourceResponse);
}
message SyncResourceRequest {
// 凭证的id列表
// @gotags: json:"secrets"
repeated string secrets = 1;
}
message SyncResourceResponse {
}
message QuerySecretRequest {
// 分页参数
// @gotags: json:"page"
infraboard.mcube.page.PageRequest page = 1;
// 关键字参数
// @gotags: json:"keywords"
string keywords = 2;
// 所属Domain
// @gotags: json:"domain"
string domain = 14;
// 所属Namespace
// @gotags: json:"namespace"
string namespace = 15;
}
make gen生成protobuf文件
secret模块的全局app
apps/secret/app.go
设置secret模块对应的模块名，接口以及一些构造函数
const (
AppName = "secret"
)
type Service interface {
RPCServer
}
func NewSecretSet() *SecretSet {
return &SecretSet{
Items: []*Secret{},
}
}
func (s *SecretSet) Get(secretId string) *Secret {
for i := range s.Items {
if s.Items[i].Id == secretId {
return s.Items[i]
}
}
return nil
}
func NewQuerySecretRequest() *QuerySecretRequest {
return &QuerySecretRequest{
Page: request.NewDefaultPageRequest(),
}
}
secret的逻辑实现部分
apps/secret/impl/impl.go
将模块实现IOC
var (
// Service 服务实例
svr = &service{}
)
type service struct {
db *gorm.DB
log logger.Logger
secret.UnimplementedRPCServer
host host.Service
}
func (s *service) Config() error {
db, err := conf.C().MySQL.ORM()
if err != nil {
return err
}
if conf.C().Log.Level == "debug" {
db = db.Debug()
}
db = db.Clauses(clause.OnConflict{
UpdateAll: true,
})
if err != nil {
return err
}
s.db = db
s.log = zap.L().Named(s.Name())
s.host = app.GetInternalApp(host.AppName).(host.Service)
return nil
}
func (s *service) Name() string {
return secret.AppName
}
func (s *service) Registry(server *grpc.Server) {
secret.RegisterRPCServer(server, svr)
}
func init() {
app.RegistryGrpcApp(svr)
app.RegistryInternalApp(svr)
}
apps/internal.go
将模块导入到内部IOC
package apps
import (
// 注册所有GRPC服务模块, 暴露给框架GRPC服务器加载, 注意 导入有先后顺序
_ "gitee.com/wendao365/mycmdb/apps/host/impl"
_ "gitee.com/wendao365/mycmdb/apps/resource/impl"
_ "gitee.com/wendao365/mycmdb/apps/secret/impl"
)
apps/secret/impl/secret.go
创建和查询secret的逻辑
func (s *service) CreateSecret(ctx context.Context, req *secret.CreateSecretRequest) (*secret.Secret, error) {
ins := New(req)
err := s.db.WithContext(ctx).Create(ins).Error
if err != nil {
return nil, err
}
return ins.Secret(), nil
}
func (s *service) QuerySecret(ctx context.Context, req *secret.QuerySecretRequest) (*secret.SecretSet, error) {
set := NewSecretSet()
err := s.db.WithContext(ctx).Find(&set.Items).Error
if err != nil {
return nil, err
}
return set.SecretSet(), nil
}
apps/secret/impl/secret_dao.go
type Secret struct {
Id string `json:"id"`
CreateAt int64 `json:"create_at"`
*secret.CreateSecretRequest
}
type SecretSet struct {
Items []*Secret
}
func NewSecretSet() *SecretSet {
return &SecretSet{
Items: []*Secret{},
}
}
func New(req *secret.CreateSecretRequest) *Secret {
return &Secret{
Id: xid.New().String(),
CreateAt: time.Now().Unix(),
CreateSecretRequest: req,
}
}
func (s *Secret) Secret() *secret.Secret {
return &secret.Secret{
Id: s.Id,
CreateAt: s.CreateAt,
Spec: s.CreateSecretRequest,
}
}
func (s *SecretSet) SecretSet() *secret.SecretSet {
set := secret.NewSecretSet()
for i := range s.Items {
set.Items = append(set.Items, s.Items[i].Secret())
}
return set
}
func (s *Secret) TableName() string {
return "secret"
}
创建和查询secret的单元测试
apps/secret/impl/impl_test.go
func init() {
tools.DevelopmentSetup()
impl = app.GetInternalApp(secret.AppName).(secret.Service)
}
var (
impl secret.Service
ctx = context.Background()
)
func TestCreateSecret(t *testing.T) {
ins, err := impl.CreateSecret(ctx, &secret.CreateSecretRequest{
Description: "腾讯云凭证",
CrendentialType: secret.TYPE_API_KEY,
Vendor: resource.VENDOR_TENCENT,
AllowRegions: "ap-guangzhou",
ApiKey: os.Getenv("TX_CLOUD_SECRET_ID"),
ApiSecret: os.Getenv("TX_CLOUD_SECRET_KEY"),
RequestRate: 5,
Domain: "default",
Namespace: "default",
})
if err != nil {
t.Fatal(err)
}
t.Log(ins)
}
测试结果
=== RUN TestCreateSecret
2023/01/27 00:00:49 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/secret/impl/secret.go:17
[44.667ms] [rows:1] INSERT INTO `secret` (`create_at`,`description`,`vendor`,`allow_regions`,`crendential_type`,`address`,`api_key`,`api_secret`,`request_rate`,`domain`,`namespace`,`id`) VALUES (1674748849,'腾讯云凭证',1,'ap-shanghai',0,'','AKIDCG8nDRHxQglfMq9IE xTnwsb8VeaKq9Ea','CF5i8LM9lBONzOaZNxJF2d2GJBpERVLw',5,'default','default','cf9a7cbha7hl9a4o5670') ON DUPLICATE KEY UPDATE `create_at`=VALUES(`create_at`),`description`=VALUES(`description`),`vendor`=VALUES(`vendor`),`allow_regions`=VALUES(`allow_regions`),`crendential_type`=VALUES(`crendential_type`),`address`=VALUES(`address`),`api_key`=VALUES(`api_key`),`api_secret`=VALUES(`api_secret`),`request_rate`=VALUES(`request_rate`),`domain`=VALUES(`domain`),`namespace`=VALUES(`namespace`)
impl_test.go:58: id:"cf9a7cbha7hl9a4o5670" create_at:1674748849 spec:{description:"腾讯云凭证" vendor:TENCENT allow_regions:"ap-sh anghai" api_key:"xx" api_secret:"xx" request_rate:5 domain:"default" namespace:"default"}
--- PASS: TestCreateSecret (0.05s)
PASS
apps/secret/impl/impl_test.go
查询secret的单元测试
func TestQuerySecret(t *testing.T) {
set, err := impl.QuerySecret(ctx, &secret.QuerySecretRequest{})
if err != nil {
t.Fatal(err)
}
t.Log(set)
}
根据查询的secret同步云商资源
使用凭证来同步
apps/secret/pb/rpc.proto
增加rpc方法
service RPC {
// 创建
rpc CreateSecret(CreateSecretRequest) returns(Secret);
// 凭证查询
rpc QuerySecret (QuerySecretRequest) returns (SecretSet);
// 基于凭证来进行任务同步
rpc SyncResource(SyncResourceRequest) returns (SyncResourceResponse);
}
message SyncResourceRequest {
// 凭证的id列表
// @gotags: json:"secrets"
repeated string secrets = 1;
}
message SyncResourceResponse {
}
apps/secret/impl/secret.go
同步资源的具体实现，先查询secret，再根据secret对应的region去同步云商的主机信息
func (s *service) SyncResource(ctx context.Context, req *secret.SyncResourceRequest) (
*secret.SyncResourceResponse, error) {
// 1. 查询出凭证
set, err := s.QuerySecret(ctx, &secret.QuerySecretRequest{})
if err != nil {
return nil, err
}
for i := range req.Secrets {
sc := set.Get(req.Secrets[i])
fmt.Println(sc)
// 2. 基于凭证构造Operator
switch sc.Spec.Vendor {
case resource.VENDOR_TENCENT:
// 3. 执行Operator操作
for _, region := range strings.Split(sc.Spec.AllowRegions, ",") {
// 3.1创建客户端
client := &connectivity.TencentCloudClient{
Region: region,
SecretID: sc.Spec.ApiKey,
SecretKey: sc.Spec.ApiSecret,
}
// 3.2 创建cvm operator
ac, err := client.Account()
if err != nil {
panic(err)
}
// 3.3 执行查询
op := cvm.NewCVMOperator(client.CvmClient(), ac)
pager := op.PageCvmQuery()
for pager.Next() {
set := host.NewHostSet()
if err := pager.Scan(ctx, set); err != nil {
return nil, err
}
// 3.4 保持
for i := range set.Items {
item := set.Items[i]
item.Resource.Meta.Domain = sc.Spec.Domain
item.Resource.Meta.Namespace = sc.Spec.Namespace
item.Resource.Meta.SyncAt = time.Now().Unix()
ins, err := s.host.CreateHost(ctx, item)
if err != nil {
s.log.Warnf("save host error, %s", err)
} else {
s.log.Debugf("save host %s success", ins.Resource.Spec.Name)
}
}
}
}
default:
return nil, fmt.Errorf("unsport %s", sc.Spec.Vendor.String())
}
}
return &secret.SyncResourceResponse{}, nil
}
apps/secret/impl/impl_test.go
func TestSyncResource(t *testing.T) {
set, err := impl.SyncResource(ctx, &secret.SyncResourceRequest{
Secrets: []string{"cdj1mfp3n7pjed28hr10"},
})
if err != nil {
t.Fatal(err)
}
t.Log(set)
}
SyncResource测试结果
自动根据数据库中的secret同步云商资源，同步的写入操作是事务，一个失败就会回滚
=== RUN TestSyncResource
2023/01/09 22:30:00 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/secret/impl/secret.go:28
[19.136ms] [rows:1] SELECT * FROM `secret`
items:{id:"cejhq83ha7hlfb0kss0g" create_at:1671896352 spec:{description:"腾讯云凭证" vendor:TENCENT allow_regions:"ap-shanghai" api_key: "xx" api_secret:"" request_rate:5 domain:"default" namespace:"default"}}
id:"cejhq83ha7hlfb0kss0g" create_at:1671896352 spec:{description:"腾讯云凭证" vendor:TENCENT allow_regions:"ap-shanghai" api_key:"xx" api_secret:"xx" request_rate:5 domain:"default" namespace:"default"}
2023-01-09 22:30:01 DEBUG [pager.cvm] cvm/cvm_pagger.go:77 resp length: 1, page size: 20
resource:{meta:{resource_id:"ins-rbcz77vt" domain:"default" namespace:"default" sync_at:1673274601 create_at:1604505186 serial_number:"5332f8eb-65c4-4612-ae94-3e2b8b6591be"} spec:{vendor:TENCENT region:"ap-shanghai" zone:"ap-shanghai-2" owner:"100009615835" name:"zhiqi_cloud" type:"S4.SMALL2" expire_at:1699545186 cpu:1 memory:2 band_width:1} cost:{pay_mode:PRE_PAY} status:{phase:"RUNNING" public_ip:"182.254.217.98" private_ip:"172.17.0.6"}} spec:{os_name:"CentOS 7.8 64bit" image_id:"img-3la7wgnt"}
2023/01/09 22:30:01 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/resource/impl/resource_dao.go:23
[26.663ms] [rows:1] INSERT INTO `resource_meta` (`resource_id`,`domain`,`namespace`,`env`,`sync_at`,`credential_id`,`create_at`,`serial_number`) VALUES ('ins-rbcz77vt','default','default','',1673274601,'',1604505186,'5332f8eb-65c4-4612-ae94-3e2b8b6591be')
2023/01/09 22:30:01 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/resource/impl/resource_dao.go:27
[26.908ms] [rows:1] INSERT INTO `resource_spec` (`resource_id`,`vendor`,`resource_type`,`region`,`zone`,`owner`,`name`,`category`,`type`,`description`,`expire_at`,`update_at`,`release_protection`,`cpu`,`gpu`,`memory`,`storage`,`band_width`) VALUES ('ins-rbcz77vt',1,0,'ap-shanghai','ap-shanghai-2','100009615835','zhiqi_cloud','','S4.SMALL2','',1699545186,0,NULL,1,0,2,0,1)
2023/01/09 22:30:01 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/resource/impl/resource_dao.go:31
[26.473ms] [rows:1] INSERT INTO `resource_status` (`resource_id`,`phase`,`lock_mode`,`lock_reason`,`public_ip`,`private_ip`) VALUES ('ins-rbcz77vt','RUNNING','','','182.254.217.98','172.17.0.6')
2023/01/09 22:30:01 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/resource/impl/resource_dao.go:35
[26.926ms] [rows:1] INSERT INTO `resource_cost` (`resource_id`,`pay_mode`,`pay_mode_detail`,`sale_price`,`real_cost`,`policy`,`unit_price`) VALUES ('ins-rbcz77vt','PRE_PAY','',0.000000,0.000000,0.000000,0.000000)
2023/01/09 22:30:01 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/resource/impl/resource_dao.go:40
[28.785ms] [rows:0] DELETE FROM `resource_tag` WHERE resource_id = 'ins-rbcz77vt'
2023/01/09 22:30:01 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/host/impl/host.go:21
[25.784ms] [rows:1] INSERT INTO `resource_host` (`resource_id`,`gpu_spec`,`os_type`,`os_name`,`image_id`,`internet_max_bandwidth_out`,`internet_max_bandwidth_in`) VALUES ('ins-rbcz77vt','','','CentOS 7.8 64bit','img-3la7wgnt',0,0) ON DUPLICATE KEY UPDATE `resource_id`=VALUES(`resource_id`),`gpu_spec`=VALUES(`gpu_spec`),`os_type`=VALUES(`os_type`),`os_name`=VALUES(`os_name`),`image_id`=VALUES(`image_id`),`internet_max_bandwidth_out`=VALUES(`internet_max_bandwidth_out`),`internet_max_bandwidth_in`=VALUES(`internet_max_bandwidth_in`)
2023-01-09 22:30:01 DEBUG [secret] impl/secret.go:89 save host zhiqi_cloud success
impl_test.go:76:
--- PASS: TestSyncResource (0.94s)
PASS
评论区