From 618018dd9e07a226cb59375a1e0b3da3de83fc6a Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Mon, 7 Oct 2024 22:53:51 +0500 Subject: [PATCH] feat(backup): support integrations --- db/Integration.go | 4 +- services/project/backup.go | 113 ++++++++++++++++++++++++++++++------ services/project/restore.go | 64 ++++++++++++++++++++ services/project/types.go | 31 +++++++--- 4 files changed, 184 insertions(+), 28 deletions(-) diff --git a/db/Integration.go b/db/Integration.go index 0aadf5ba..8a89e0ca 100644 --- a/db/Integration.go +++ b/db/Integration.go @@ -38,8 +38,8 @@ const ( type IntegrationMatcher struct { ID int `db:"id" json:"id" backup:"-"` + IntegrationID int `db:"integration_id" json:"integration_id" backup:"-"` Name string `db:"name" json:"name"` - IntegrationID int `db:"integration_id" json:"integration_id"` MatchType IntegrationMatchType `db:"match_type" json:"match_type"` Method IntegrationMatchMethodType `db:"method" json:"method"` BodyDataType IntegrationBodyDataType `db:"body_data_type" json:"body_data_type"` @@ -56,8 +56,8 @@ const ( type IntegrationExtractValue struct { ID int `db:"id" json:"id" backup:"-"` - Name string `db:"name" json:"name"` IntegrationID int `db:"integration_id" json:"integration_id" backup:"-"` + Name string `db:"name" json:"name"` ValueSource IntegrationExtractValueSource `db:"value_source" json:"value_source"` BodyDataType IntegrationBodyDataType `db:"body_data_type" json:"body_data_type"` Key string `db:"key" json:"key"` diff --git a/services/project/backup.go b/services/project/backup.go index 1864e162..4c5e0256 100644 --- a/services/project/backup.go +++ b/services/project/backup.go @@ -108,53 +108,89 @@ func (b *BackupDB) makeUniqueNames() { }, func(item *db.View, name string) { item.Title = name }) + + makeUniqueNames(b.integrations, func(item *db.Integration) string { + return item.Name + }, func(item *db.Integration, name string) { + item.Name = name + }) } -func (b *BackupDB) new(projectID int, store db.Store) (*BackupDB, error) { - var err error +func (b *BackupDB) load(projectID int, store db.Store) (err error) { b.templates, err = store.GetTemplates(projectID, db.TemplateFilter{}, db.RetrieveQueryParams{}) if err != nil { - return nil, err + return } b.repositories, err = store.GetRepositories(projectID, db.RetrieveQueryParams{}) if err != nil { - return nil, err + return } b.keys, err = store.GetAccessKeys(projectID, db.RetrieveQueryParams{}) if err != nil { - return nil, err + return } b.views, err = store.GetViews(projectID) if err != nil { - return nil, err + return } b.inventories, err = store.GetInventories(projectID, db.RetrieveQueryParams{}) if err != nil { - return nil, err + return } b.environments, err = store.GetEnvironments(projectID, db.RetrieveQueryParams{}) if err != nil { - return nil, err + return } + schedules, err := store.GetSchedules() if err != nil { - return nil, err + return } + b.schedules = getSchedulesByProject(projectID, schedules) + b.meta, err = store.GetProject(projectID) if err != nil { - return nil, err + return + } + + b.integrationProjAliases, err = store.GetIntegrationAliases(projectID, nil) + if err != nil { + return + } + + b.integrations, err = store.GetIntegrations(projectID, db.RetrieveQueryParams{}) + if err != nil { + return + } + + b.integrationAliases = make(map[int][]db.IntegrationAlias) + b.integrationMatchers = make(map[int][]db.IntegrationMatcher) + b.integrationExtractValues = make(map[int][]db.IntegrationExtractValue) + for _, o := range b.integrations { + b.integrationAliases[o.ID], err = store.GetIntegrationAliases(projectID, &o.ID) + if err != nil { + return + } + b.integrationMatchers[o.ID], err = store.GetIntegrationMatchers(projectID, db.RetrieveQueryParams{}, o.ID) + if err != nil { + return + } + b.integrationExtractValues[o.ID], err = store.GetIntegrationExtractValues(projectID, db.RetrieveQueryParams{}, o.ID) + if err != nil { + return + } } b.makeUniqueNames() - return b, nil + return } func (b *BackupDB) format() (*BackupFormat, error) { @@ -247,25 +283,64 @@ func (b *BackupDB) format() (*BackupFormat, error) { Vaults: vaults, } } + + integrations := make([]BackupIntegration, len(b.integrations)) + for i, o := range b.integrations { + + var aliases []string + + for _, a := range b.integrationAliases[o.ID] { + aliases = append(aliases, a.Alias) + } + + tplName, _ := findNameByID[db.Template](o.TemplateID, b.templates) + + if tplName == nil { + continue + } + + var keyName *string + + if o.AuthSecretID != nil { + keyName, _ = findNameByID[db.AccessKey](*o.AuthSecretID, b.keys) + } + + integrations[i] = BackupIntegration{ + Integration: o, + Aliases: aliases, + Matchers: b.integrationMatchers[o.ID], + ExtractValues: b.integrationExtractValues[o.ID], + Template: *tplName, + AuthSecret: keyName, + } + } + + var integrationAliases []string + + for _, alias := range b.integrationProjAliases { + integrationAliases = append(integrationAliases, alias.Alias) + } + return &BackupFormat{ Meta: BackupMeta{ b.meta, }, - Inventories: inventories, - Environments: environments, - Views: views, - Repositories: repositories, - Keys: keys, - Templates: templates, + Inventories: inventories, + Environments: environments, + Views: views, + Repositories: repositories, + Keys: keys, + Templates: templates, + Integration: integrations, + IntegrationAliases: integrationAliases, }, nil } func GetBackup(projectID int, store db.Store) (*BackupFormat, error) { backup := BackupDB{} - if _, err := backup.new(projectID, store); err != nil { + if err := backup.load(projectID, store); err != nil { return nil, err } - return backup.format() } diff --git a/services/project/restore.go b/services/project/restore.go index f451130c..7422dbb3 100644 --- a/services/project/restore.go +++ b/services/project/restore.go @@ -274,6 +274,7 @@ func (e BackupTemplate) Restore(store db.Store, b *BackupDB) error { return err } } + if e.Vaults != nil { for _, vault := range e.Vaults { var VaultKeyID int @@ -297,6 +298,55 @@ func (e BackupTemplate) Restore(store db.Store, b *BackupDB) error { return nil } +func (e BackupIntegration) Restore(store db.Store, b *BackupDB) error { + var authSecretID *int + + if e.AuthSecret == nil { + authSecretID = nil + } else if k := findEntityByName[db.AccessKey](e.AuthSecret, b.keys); k == nil { + authSecretID = nil + } else { + authSecretID = &((*k).ID) + } + + tpl := findEntityByName[db.Template](&e.Template, b.templates) + if tpl == nil { + return fmt.Errorf("template does not exist in templates[].name") + } + + integration := e.Integration + integration.ProjectID = b.meta.ID + integration.AuthSecretID = authSecretID + integration.TemplateID = tpl.ID + + newIntegration, err := store.CreateIntegration(integration) + if err != nil { + return err + } + b.integrations = append(b.integrations, newIntegration) + + for _, m := range e.Matchers { + m.IntegrationID = newIntegration.ID + _, _ = store.CreateIntegrationMatcher(b.meta.ID, m) + } + + for _, v := range e.ExtractValues { + v.IntegrationID = newIntegration.ID + _, _ = store.CreateIntegrationExtractValue(b.meta.ID, v) + } + + for _, a := range e.Aliases { + alias := db.IntegrationAlias{ + Alias: a, + ProjectID: b.meta.ID, + IntegrationID: &newIntegration.ID, + } + _, _ = store.CreateIntegrationAlias(alias) + } + + return nil +} + func (backup *BackupFormat) Verify() error { for i, o := range backup.Environments { if err := o.Verify(backup); err != nil { @@ -399,5 +449,19 @@ func (backup *BackupFormat) Restore(user db.User, store db.Store) (*db.Project, } } + for i, o := range backup.Integration { + if err := o.Restore(store, &b); err != nil { + return nil, fmt.Errorf("error at integrations[%d]: %s", i, err.Error()) + } + } + + for _, o := range backup.IntegrationAliases { + alias := db.IntegrationAlias{ + Alias: o, + ProjectID: b.meta.ID, + } + _, _ = store.CreateIntegrationAlias(alias) + } + return &newProject, nil } diff --git a/services/project/types.go b/services/project/types.go index cefcd883..33bd9b23 100644 --- a/services/project/types.go +++ b/services/project/types.go @@ -13,16 +13,24 @@ type BackupDB struct { inventories []db.Inventory environments []db.Environment schedules []db.Schedule + + integrationProjAliases []db.IntegrationAlias + integrations []db.Integration + integrationAliases map[int][]db.IntegrationAlias + integrationMatchers map[int][]db.IntegrationMatcher + integrationExtractValues map[int][]db.IntegrationExtractValue } type BackupFormat struct { - Meta BackupMeta `backup:"meta"` - Templates []BackupTemplate `backup:"templates"` - Repositories []BackupRepository `backup:"repositories"` - Keys []BackupAccessKey `backup:"keys"` - Views []BackupView `backup:"views"` - Inventories []BackupInventory `backup:"inventories"` - Environments []BackupEnvironment `backup:"environments"` + Meta BackupMeta `backup:"meta"` + Templates []BackupTemplate `backup:"templates"` + Repositories []BackupRepository `backup:"repositories"` + Keys []BackupAccessKey `backup:"keys"` + Views []BackupView `backup:"views"` + Inventories []BackupInventory `backup:"inventories"` + Environments []BackupEnvironment `backup:"environments"` + Integration []BackupIntegration `backup:"integrations"` + IntegrationAliases []string `backup:"integration_aliases"` } type BackupMeta struct { @@ -72,6 +80,15 @@ type BackupTemplateVault struct { VaultKey string `backup:"vault_key"` } +type BackupIntegration struct { + db.Integration + Aliases []string `backup:"aliases"` + Matchers []db.IntegrationMatcher `backup:"matchers"` + ExtractValues []db.IntegrationExtractValue `backup:"extract_values"` + Template string `backup:"template"` + AuthSecret *string `backup:"auth_secret"` +} + type BackupEntry interface { GetName() string Verify(backup *BackupFormat) error