侧边栏壁纸
博主头像
问道

问道的小花园,总能给你带来惊喜

  • 累计撰写 71 篇文章
  • 累计创建 37 个标签
  • 累计收到 6 条评论

混合云资产管理项目(六)

问道
2023-01-27 / 0 评论 / 0 点赞 / 658 阅读 / 12,580 字 / 正在检测是否收录...
温馨提示:
本文最后更新于 2023-01-27,若内容或图片失效,请留言反馈。部分素材来自网络,若不小心影响到您的利益,请联系我们删除。

管理云商凭证,将不同云商,不同地域的api的key,secret保存到表中,再遍历这些key和secret,将云商的资源进行同步

云商凭证模块

secret的proto文件

apps/secret/pb/secret.proto

syntax = "proto3";

package go8.devcloud.cmdb.secret;
option go_package = "gitee.com/go-course/go8/projects/devcloud/cmdb/apps/secret";

import "apps/resource/pb/resource.proto";

// 基于用户名密码的凭证
// 记忆API Key的凭证
enum TYPE {
    API_KEY = 0;
    PASSWORD = 1;
}

message CreateSecretRequest {
    // 描述
    // @gotags: json:"description" validate:"required,lte=100"
    string description  = 1;
    // 厂商
    // @gotags: json:"vendor"
    resource.VENDOR vendor = 2;
    // 允许同步的区域
    // @gotags: json:"allow_regions"
    string allow_regions = 3;
    // 凭证类型
    // @gotags: json:"crendential_type"
    TYPE crendential_type = 4;
    // 服务地址, 云商不用填写
    // @gotags: json:"address"
    string address = 5;
    // key
    // @gotags: json:"api_key" validate:"required,lte=100"
    string api_key = 6;
    // api_secret
    // @gotags: json:"api_secret" validate:"required,lte=100"
    string api_secret = 7;
    // 请求速率限制, 默认1秒5个
    // @gotags: json:"request_rate"
    int32 request_rate = 8;
    // 所属Domain
    // @gotags: json:"domain" validate:"required"
    string domain = 9;
    // 所属Namespace
    // @gotags: json:"namespace" validate:"required"
    string namespace = 10;
}

message Secret {
    // 全局唯一Id
    // @gotags: json:"id"
    string id = 1;
    // 创建时间
    // @gotags: json:"create_at"
    int64 create_at = 2;
    // 创建信息
    // @gotags: json:"spec"
    CreateSecretRequest spec = 3;
}

message SecretSet {
    // @gotags: json:"total"
    int64 total = 1;
    // @gotags: json:"items"
    repeated Secret items = 2;
}

apps/secret/pb/rpc.proto

syntax = "proto3";

package go8.devcloud.cmdb.secret;
option go_package = "gitee.com/go-course/go8/projects/devcloud/cmdb/apps/secret";

import "apps/secret/pb/secret.proto";
import "github.com/infraboard/mcube/pb/page/page.proto";

service RPC {
    // 创建
    rpc CreateSecret(CreateSecretRequest) returns(Secret);
    // 凭证查询
    rpc QuerySecret (QuerySecretRequest) returns (SecretSet);
    // 基于凭证来进行任务同步
    rpc SyncResource(SyncResourceRequest) returns (SyncResourceResponse);
}

message SyncResourceRequest {
    // 凭证的id列表
    // @gotags: json:"secrets"
    repeated string secrets = 1;
}

message SyncResourceResponse {

}

message QuerySecretRequest {
    // 分页参数
    // @gotags: json:"page"
    infraboard.mcube.page.PageRequest page = 1;
    // 关键字参数
    // @gotags: json:"keywords"
    string keywords = 2;
    // 所属Domain
    // @gotags: json:"domain"
    string domain = 14;
    // 所属Namespace
    // @gotags: json:"namespace"
    string namespace = 15;
}

make gen生成protobuf文件

secret模块的全局app

apps/secret/app.go

设置secret模块对应的模块名,接口以及一些构造函数

const (
  AppName = "secret"
)

type Service interface {
  RPCServer
}

func NewSecretSet() *SecretSet {
  return &SecretSet{
    Items: []*Secret{},
  }
}
func (s *SecretSet) Get(secretId string) *Secret {
  for i := range s.Items {
    if s.Items[i].Id == secretId {
      return s.Items[i]
    }
  }

  return nil
}

func NewQuerySecretRequest() *QuerySecretRequest {
  return &QuerySecretRequest{
    Page: request.NewDefaultPageRequest(),
  }
}

secret的逻辑实现部分

apps/secret/impl/impl.go

将模块实现IOC

var (
  // Service 服务实例
  svr = &service{}
)

type service struct {
  db *gorm.DB

  log logger.Logger
  secret.UnimplementedRPCServer
  host host.Service
}

func (s *service) Config() error {
  db, err := conf.C().MySQL.ORM()
  if err != nil {
    return err
  }

  if conf.C().Log.Level == "debug" {
    db = db.Debug()
  }

  db = db.Clauses(clause.OnConflict{
    UpdateAll: true,
  })
  if err != nil {
    return err
  }
  s.db = db

  s.log = zap.L().Named(s.Name())
  s.host = app.GetInternalApp(host.AppName).(host.Service)
  return nil
}

func (s *service) Name() string {
  return secret.AppName
}

func (s *service) Registry(server *grpc.Server) {
  secret.RegisterRPCServer(server, svr)
}

func init() {
  app.RegistryGrpcApp(svr)
  app.RegistryInternalApp(svr)
}

apps/internal.go

将模块导入到内部IOC

package apps

import (
  // 注册所有GRPC服务模块, 暴露给框架GRPC服务器加载, 注意 导入有先后顺序
  _ "gitee.com/wendao365/mycmdb/apps/host/impl"
  _ "gitee.com/wendao365/mycmdb/apps/resource/impl"
  _ "gitee.com/wendao365/mycmdb/apps/secret/impl"
)

apps/secret/impl/secret.go

创建和查询secret的逻辑
func (s *service) CreateSecret(ctx context.Context, req *secret.CreateSecretRequest) (*secret.Secret, error) {
  ins := New(req)
  err := s.db.WithContext(ctx).Create(ins).Error
  if err != nil {
    return nil, err
  }
  return ins.Secret(), nil
}

func (s *service) QuerySecret(ctx context.Context, req *secret.QuerySecretRequest) (*secret.SecretSet, error) {
  set := NewSecretSet()
  err := s.db.WithContext(ctx).Find(&set.Items).Error
  if err != nil {
    return nil, err
  }
  return set.SecretSet(), nil
}

apps/secret/impl/secret_dao.go

type Secret struct {
  Id       string `json:"id"`
  CreateAt int64  `json:"create_at"`
  *secret.CreateSecretRequest
}

type SecretSet struct {
  Items []*Secret
}

func NewSecretSet() *SecretSet {
  return &SecretSet{
    Items: []*Secret{},
  }
}

func New(req *secret.CreateSecretRequest) *Secret {
  return &Secret{
    Id:                  xid.New().String(),
    CreateAt:            time.Now().Unix(),
    CreateSecretRequest: req,
  }
}


func (s *Secret) Secret() *secret.Secret {
  return &secret.Secret{
    Id:       s.Id,
    CreateAt: s.CreateAt,
    Spec:     s.CreateSecretRequest,
  }
}

func (s *SecretSet) SecretSet() *secret.SecretSet {
  set := secret.NewSecretSet()
  for i := range s.Items {
    set.Items = append(set.Items, s.Items[i].Secret())
  }

  return set
}

func (s *Secret) TableName() string {
  return "secret"
}

创建和查询secret的单元测试

apps/secret/impl/impl_test.go

func init() {
  tools.DevelopmentSetup()
  impl = app.GetInternalApp(secret.AppName).(secret.Service)
}

var (
  impl secret.Service
  ctx  = context.Background()
)

func TestCreateSecret(t *testing.T) {
  ins, err := impl.CreateSecret(ctx, &secret.CreateSecretRequest{
    Description:     "腾讯云凭证",
    CrendentialType: secret.TYPE_API_KEY,
    Vendor:          resource.VENDOR_TENCENT,
    AllowRegions:    "ap-guangzhou",
    ApiKey:          os.Getenv("TX_CLOUD_SECRET_ID"),
    ApiSecret:       os.Getenv("TX_CLOUD_SECRET_KEY"),
    RequestRate:     5,
    Domain:          "default",
    Namespace:       "default",
  })
  if err != nil {
    t.Fatal(err)
  }
  t.Log(ins)
}

测试结果

=== RUN   TestCreateSecret

2023/01/27 00:00:49 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/secret/impl/secret.go:17
[44.667ms] [rows:1] INSERT INTO `secret` (`create_at`,`description`,`vendor`,`allow_regions`,`crendential_type`,`address`,`api_key`,`api_secret`,`request_rate`,`domain`,`namespace`,`id`) VALUES (1674748849,'腾讯云凭证',1,'ap-shanghai',0,'','AKIDCG8nDRHxQglfMq9IE     xTnwsb8VeaKq9Ea','CF5i8LM9lBONzOaZNxJF2d2GJBpERVLw',5,'default','default','cf9a7cbha7hl9a4o5670') ON DUPLICATE KEY UPDATE `create_at`=VALUES(`create_at`),`description`=VALUES(`description`),`vendor`=VALUES(`vendor`),`allow_regions`=VALUES(`allow_regions`),`crendential_type`=VALUES(`crendential_type`),`address`=VALUES(`address`),`api_key`=VALUES(`api_key`),`api_secret`=VALUES(`api_secret`),`request_rate`=VALUES(`request_rate`),`domain`=VALUES(`domain`),`namespace`=VALUES(`namespace`)
    impl_test.go:58: id:"cf9a7cbha7hl9a4o5670"  create_at:1674748849  spec:{description:"腾讯云凭证"  vendor:TENCENT  allow_regions:"ap-sh     anghai"  api_key:"xx"  api_secret:"xx"  request_rate:5  domain:"default"  namespace:"default"}
--- PASS: TestCreateSecret (0.05s)
PASS

apps/secret/impl/impl_test.go

查询secret的单元测试

func TestQuerySecret(t *testing.T) {
  set, err := impl.QuerySecret(ctx, &secret.QuerySecretRequest{})
  if err != nil {
    t.Fatal(err)
  }
  t.Log(set)
}

根据查询的secret同步云商资源

使用凭证来同步

apps/secret/pb/rpc.proto

增加rpc方法

service RPC {
    // 创建
    rpc CreateSecret(CreateSecretRequest) returns(Secret);
    // 凭证查询
    rpc QuerySecret (QuerySecretRequest) returns (SecretSet);
    // 基于凭证来进行任务同步
    rpc SyncResource(SyncResourceRequest) returns (SyncResourceResponse);
}

message SyncResourceRequest {
    // 凭证的id列表
    // @gotags: json:"secrets"
    repeated string secrets = 1;
}

message SyncResourceResponse {

}

apps/secret/impl/secret.go

同步资源的具体实现,先查询secret,再根据secret对应的region去同步云商的主机信息

func (s *service) SyncResource(ctx context.Context, req *secret.SyncResourceRequest) (
  *secret.SyncResourceResponse, error) {
  // 1. 查询出凭证
  set, err := s.QuerySecret(ctx, &secret.QuerySecretRequest{})
  if err != nil {
    return nil, err
  }

  for i := range req.Secrets {
    sc := set.Get(req.Secrets[i])
    fmt.Println(sc)
    // 2. 基于凭证构造Operator
    switch sc.Spec.Vendor {
    case resource.VENDOR_TENCENT:
      // 3. 执行Operator操作
      for _, region := range strings.Split(sc.Spec.AllowRegions, ",") {
        // 3.1创建客户端
        client := &connectivity.TencentCloudClient{
          Region:    region,
          SecretID:  sc.Spec.ApiKey,
          SecretKey: sc.Spec.ApiSecret,
        }

        // 3.2 创建cvm operator
        ac, err := client.Account()
        if err != nil {
          panic(err)
        }

        // 3.3 执行查询
        op := cvm.NewCVMOperator(client.CvmClient(), ac)
        pager := op.PageCvmQuery()
        for pager.Next() {
          set := host.NewHostSet()
          if err := pager.Scan(ctx, set); err != nil {
            return nil, err
          }
          // 3.4 保持
          for i := range set.Items {
            item := set.Items[i]
            item.Resource.Meta.Domain = sc.Spec.Domain
            item.Resource.Meta.Namespace = sc.Spec.Namespace
            item.Resource.Meta.SyncAt = time.Now().Unix()

            ins, err := s.host.CreateHost(ctx, item)
            if err != nil {
              s.log.Warnf("save host error, %s", err)
            } else {
              s.log.Debugf("save host %s success", ins.Resource.Spec.Name)
            }
          }
        }
      }

    default:
      return nil, fmt.Errorf("unsport %s", sc.Spec.Vendor.String())
    }
  }

  return &secret.SyncResourceResponse{}, nil
}

apps/secret/impl/impl_test.go

func TestSyncResource(t *testing.T) {
  set, err := impl.SyncResource(ctx, &secret.SyncResourceRequest{
    Secrets: []string{"cdj1mfp3n7pjed28hr10"},
  })
  if err != nil {
    t.Fatal(err)
  }
  t.Log(set)
}

SyncResource测试结果

自动根据数据库中的secret同步云商资源,同步的写入操作是事务,一个失败就会回滚

=== RUN   TestSyncResource

2023/01/09 22:30:00 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/secret/impl/secret.go:28
[19.136ms] [rows:1] SELECT * FROM `secret`
items:{id:"cejhq83ha7hlfb0kss0g" create_at:1671896352 spec:{description:"腾讯云凭证" vendor:TENCENT allow_regions:"ap-shanghai" api_key:     "xx" api_secret:"" request_rate:5 domain:"default" namespace:"default"}}
id:"cejhq83ha7hlfb0kss0g" create_at:1671896352 spec:{description:"腾讯云凭证" vendor:TENCENT allow_regions:"ap-shanghai" api_key:"xx" api_secret:"xx" request_rate:5 domain:"default" namespace:"default"}
2023-01-09 22:30:01  DEBUG  [pager.cvm]  cvm/cvm_pagger.go:77  resp length: 1, page size: 20
resource:{meta:{resource_id:"ins-rbcz77vt" domain:"default" namespace:"default" sync_at:1673274601 create_at:1604505186 serial_number:"5332f8eb-65c4-4612-ae94-3e2b8b6591be"} spec:{vendor:TENCENT region:"ap-shanghai" zone:"ap-shanghai-2" owner:"100009615835" name:"zhiqi_cloud" type:"S4.SMALL2" expire_at:1699545186 cpu:1 memory:2 band_width:1} cost:{pay_mode:PRE_PAY} status:{phase:"RUNNING" public_ip:"182.254.217.98" private_ip:"172.17.0.6"}} spec:{os_name:"CentOS 7.8 64bit" image_id:"img-3la7wgnt"}

2023/01/09 22:30:01 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/resource/impl/resource_dao.go:23
[26.663ms] [rows:1] INSERT INTO `resource_meta` (`resource_id`,`domain`,`namespace`,`env`,`sync_at`,`credential_id`,`create_at`,`serial_number`) VALUES ('ins-rbcz77vt','default','default','',1673274601,'',1604505186,'5332f8eb-65c4-4612-ae94-3e2b8b6591be')

2023/01/09 22:30:01 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/resource/impl/resource_dao.go:27
[26.908ms] [rows:1] INSERT INTO `resource_spec` (`resource_id`,`vendor`,`resource_type`,`region`,`zone`,`owner`,`name`,`category`,`type`,`description`,`expire_at`,`update_at`,`release_protection`,`cpu`,`gpu`,`memory`,`storage`,`band_width`) VALUES ('ins-rbcz77vt',1,0,'ap-shanghai','ap-shanghai-2','100009615835','zhiqi_cloud','','S4.SMALL2','',1699545186,0,NULL,1,0,2,0,1)

2023/01/09 22:30:01 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/resource/impl/resource_dao.go:31
[26.473ms] [rows:1] INSERT INTO `resource_status` (`resource_id`,`phase`,`lock_mode`,`lock_reason`,`public_ip`,`private_ip`) VALUES ('ins-rbcz77vt','RUNNING','','','182.254.217.98','172.17.0.6')

2023/01/09 22:30:01 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/resource/impl/resource_dao.go:35
[26.926ms] [rows:1] INSERT INTO `resource_cost` (`resource_id`,`pay_mode`,`pay_mode_detail`,`sale_price`,`real_cost`,`policy`,`unit_price`) VALUES ('ins-rbcz77vt','PRE_PAY','',0.000000,0.000000,0.000000,0.000000)

2023/01/09 22:30:01 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/resource/impl/resource_dao.go:40
[28.785ms] [rows:0] DELETE FROM `resource_tag` WHERE resource_id = 'ins-rbcz77vt'

2023/01/09 22:30:01 F:/go/project/0-shizhanxiangmu/myCMDB/mycmdb/apps/host/impl/host.go:21
[25.784ms] [rows:1] INSERT INTO `resource_host` (`resource_id`,`gpu_spec`,`os_type`,`os_name`,`image_id`,`internet_max_bandwidth_out`,`internet_max_bandwidth_in`) VALUES ('ins-rbcz77vt','','','CentOS 7.8 64bit','img-3la7wgnt',0,0) ON DUPLICATE KEY UPDATE `resource_id`=VALUES(`resource_id`),`gpu_spec`=VALUES(`gpu_spec`),`os_type`=VALUES(`os_type`),`os_name`=VALUES(`os_name`),`image_id`=VALUES(`image_id`),`internet_max_bandwidth_out`=VALUES(`internet_max_bandwidth_out`),`internet_max_bandwidth_in`=VALUES(`internet_max_bandwidth_in`)
2023-01-09 22:30:01  DEBUG  [secret]  impl/secret.go:89  save host zhiqi_cloud success
    impl_test.go:76: 
--- PASS: TestSyncResource (0.94s)
PASS

0

评论区