From acd29797d4d6bc2b5d41985d5aca21efd10c5d8b Mon Sep 17 00:00:00 2001 From: Denis Gukov Date: Sun, 6 Oct 2024 01:16:25 +0500 Subject: [PATCH] refactor(backup): add tag backup and use it --- db/AccessKey.go | 13 +++--- db/Environment.go | 16 ++++---- db/Inventory.go | 18 ++++----- db/Project.go | 4 +- db/Repository.go | 8 ++-- db/Schedule.go | 10 ++--- db/Template.go | 22 +++++----- db/TemplateVault.go | 8 ++-- db/View.go | 6 +-- services/project/backup.go | 59 +++++++++------------------ services/project/restore.go | 63 ++++++++++++----------------- services/project/types.go | 80 ++++++++++++++----------------------- 12 files changed, 129 insertions(+), 178 deletions(-) diff --git a/db/AccessKey.go b/db/AccessKey.go index f776d8e6..cb2795e6 100644 --- a/db/AccessKey.go +++ b/db/AccessKey.go @@ -26,24 +26,27 @@ const ( // AccessKey represents a key used to access a machine with ansible from semaphore type AccessKey struct { - ID int `db:"id" json:"id"` + ID int `db:"id" json:"id" backup:"-"` Name string `db:"name" json:"name" binding:"required"` // 'ssh/login_password/none' Type AccessKeyType `db:"type" json:"type" binding:"required"` - ProjectID *int `db:"project_id" json:"project_id"` + ProjectID *int `db:"project_id" json:"project_id" backup:"-"` // Secret used internally, do not assign this field. // You should use methods SerializeSecret to fill this field. - Secret *string `db:"secret" json:"-"` + Secret *string `db:"secret" json:"-" backup:"-"` String string `db:"-" json:"string"` LoginPassword LoginPassword `db:"-" json:"login_password"` SshKey SshKey `db:"-" json:"ssh"` OverrideSecret bool `db:"-" json:"override_secret"` - EnvironmentID *int `db:"environment_id" json:"-"` - UserID *int `db:"user_id" json:"-"` + // EnvironmentID is an ID of environment which owns the access key. + EnvironmentID *int `db:"environment_id" json:"-" backup:"-"` + + // UserID is an ID of user which owns the access key. + UserID *int `db:"user_id" json:"-" backup:"-"` } type LoginPassword struct { diff --git a/db/Environment.go b/db/Environment.go index 1865b4b8..14126af6 100644 --- a/db/Environment.go +++ b/db/Environment.go @@ -31,13 +31,15 @@ type EnvironmentSecret struct { // Environment is used to pass additional arguments, in json form to ansible type Environment struct { - ID int `db:"id" json:"id"` - Name string `db:"name" json:"name" binding:"required"` - ProjectID int `db:"project_id" json:"project_id"` - Password *string `db:"password" json:"password"` - JSON string `db:"json" json:"json" binding:"required"` - ENV *string `db:"env" json:"env" binding:"required"` - Secrets []EnvironmentSecret `db:"-" json:"secrets"` + ID int `db:"id" json:"id" backup:"-"` + Name string `db:"name" json:"name" binding:"required"` + ProjectID int `db:"project_id" json:"project_id" backup:"-"` + Password *string `db:"password" json:"password"` + JSON string `db:"json" json:"json" binding:"required"` + ENV *string `db:"env" json:"env" binding:"required"` + + // Secrets is a field which used to update secrets associated with the environment. + Secrets []EnvironmentSecret `db:"-" json:"secrets" backup:"-"` } func (s *EnvironmentSecret) Validate() error { diff --git a/db/Inventory.go b/db/Inventory.go index 2b405c08..750d4173 100644 --- a/db/Inventory.go +++ b/db/Inventory.go @@ -14,17 +14,17 @@ const ( // Inventory is the model of an ansible inventory file type Inventory struct { - ID int `db:"id" json:"id"` + ID int `db:"id" json:"id" backup:"-"` Name string `db:"name" json:"name" binding:"required"` - ProjectID int `db:"project_id" json:"project_id"` + ProjectID int `db:"project_id" json:"project_id" backup:"-"` Inventory string `db:"inventory" json:"inventory"` // accesses hosts in inventory - SSHKeyID *int `db:"ssh_key_id" json:"ssh_key_id"` - SSHKey AccessKey `db:"-" json:"-"` + SSHKeyID *int `db:"ssh_key_id" json:"ssh_key_id" backup:"-"` + SSHKey AccessKey `db:"-" json:"-" backup:"-"` - BecomeKeyID *int `db:"become_key_id" json:"become_key_id"` - BecomeKey AccessKey `db:"-" json:"-"` + BecomeKeyID *int `db:"become_key_id" json:"become_key_id" backup:"-"` + BecomeKey AccessKey `db:"-" json:"-" backup:"-"` // static/file Type InventoryType `db:"type" json:"type"` @@ -33,12 +33,12 @@ type Inventory struct { // It is not used now but can be used in feature for // inventories which can not be used more than one template // at once. - HolderID *int `db:"holder_id" json:"holder_id"` + HolderID *int `db:"holder_id" json:"holder_id" backup:"-"` // RepositoryID is an ID of repo where inventory stored. // If null than inventory will be got from template repository. - RepositoryID *int `db:"repository_id" json:"repository_id"` - Repository *Repository `db:"-" json:"-"` + RepositoryID *int `db:"repository_id" json:"repository_id" backup:"-"` + Repository *Repository `db:"-" json:"-" backup:"-"` } func (e Inventory) GetFilename() string { diff --git a/db/Project.go b/db/Project.go index 725d26db..f933696c 100644 --- a/db/Project.go +++ b/db/Project.go @@ -6,9 +6,9 @@ import ( // Project is the top level structure in Semaphore type Project struct { - ID int `db:"id" json:"id"` + ID int `db:"id" json:"id" backup:"-"` Name string `db:"name" json:"name" binding:"required"` - Created time.Time `db:"created" json:"created"` + Created time.Time `db:"created" json:"created" backup:"-"` Alert bool `db:"alert" json:"alert"` AlertChat *string `db:"alert_chat" json:"alert_chat"` MaxParallelTasks int `db:"max_parallel_tasks" json:"max_parallel_tasks"` diff --git a/db/Repository.go b/db/Repository.go index 03d6a8e2..b74b5641 100644 --- a/db/Repository.go +++ b/db/Repository.go @@ -23,14 +23,14 @@ const ( // Repository is the model for code stored in a git repository type Repository struct { - ID int `db:"id" json:"id"` + ID int `db:"id" json:"id" backup:"-"` Name string `db:"name" json:"name" binding:"required"` - ProjectID int `db:"project_id" json:"project_id"` + ProjectID int `db:"project_id" json:"project_id" backup:"-"` GitURL string `db:"git_url" json:"git_url" binding:"required"` GitBranch string `db:"git_branch" json:"git_branch" binding:"required"` - SSHKeyID int `db:"ssh_key_id" json:"ssh_key_id" binding:"required"` + SSHKeyID int `db:"ssh_key_id" json:"ssh_key_id" binding:"required" backup:"-"` - SSHKey AccessKey `db:"-" json:"-"` + SSHKey AccessKey `db:"-" json:"-" backup:"-"` } func (r Repository) ClearCache() error { diff --git a/db/Schedule.go b/db/Schedule.go index f18f78dc..a9d75bfa 100644 --- a/db/Schedule.go +++ b/db/Schedule.go @@ -1,15 +1,15 @@ package db type Schedule struct { - ID int `db:"id" json:"id"` - ProjectID int `db:"project_id" json:"project_id"` - TemplateID int `db:"template_id" json:"template_id"` + ID int `db:"id" json:"id" backup:"-"` + ProjectID int `db:"project_id" json:"project_id" backup:"-"` + TemplateID int `db:"template_id" json:"template_id" backup:"-"` CronFormat string `db:"cron_format" json:"cron_format"` Name string `db:"name" json:"name"` Active bool `db:"active" json:"active"` - LastCommitHash *string `db:"last_commit_hash" json:"-"` - RepositoryID *int `db:"repository_id" json:"repository_id"` + LastCommitHash *string `db:"last_commit_hash" json:"-" backup:"-"` + RepositoryID *int `db:"repository_id" json:"repository_id" backup:"-"` } type ScheduleWithTpl struct { diff --git a/db/Template.go b/db/Template.go index 9c613747..e006cd7e 100644 --- a/db/Template.go +++ b/db/Template.go @@ -58,12 +58,12 @@ type TemplateFilter struct { // Template is a user defined model that is used to run a task type Template struct { - ID int `db:"id" json:"id"` + ID int `db:"id" json:"id" backup:"-"` - ProjectID int `db:"project_id" json:"project_id"` - InventoryID *int `db:"inventory_id" json:"inventory_id"` - RepositoryID int `db:"repository_id" json:"repository_id"` - EnvironmentID *int `db:"environment_id" json:"environment_id"` + ProjectID int `db:"project_id" json:"project_id" backup:"-"` + InventoryID *int `db:"inventory_id" json:"inventory_id" backup:"-"` + RepositoryID int `db:"repository_id" json:"repository_id" backup:"-"` + EnvironmentID *int `db:"environment_id" json:"environment_id" backup:"-"` // Name as described in https://github.com/ansible-semaphore/semaphore/issues/188 Name string `db:"name" json:"name"` @@ -76,15 +76,15 @@ type Template struct { Description *string `db:"description" json:"description"` - Vaults []TemplateVault `db:"-" json:"vaults"` + Vaults []TemplateVault `db:"-" json:"vaults" backup:"-"` Type TemplateType `db:"type" json:"type"` StartVersion *string `db:"start_version" json:"start_version"` - BuildTemplateID *int `db:"build_template_id" json:"build_template_id"` + BuildTemplateID *int `db:"build_template_id" json:"build_template_id" backup:"-"` - ViewID *int `db:"view_id" json:"view_id"` + ViewID *int `db:"view_id" json:"view_id" backup:"-"` - LastTask *TaskWithTpl `db:"-" json:"last_task"` + LastTask *TaskWithTpl `db:"-" json:"last_task" backup:"-"` Autorun bool `db:"autorun" json:"autorun"` @@ -92,13 +92,13 @@ type Template struct { // It is not used for store survey vars to database. // Do not use it in your code. Use SurveyVars instead. SurveyVarsJSON *string `db:"survey_vars" json:"-"` - SurveyVars []SurveyVar `db:"-" json:"survey_vars"` + SurveyVars []SurveyVar `db:"-" json:"survey_vars" backup:"-"` SuppressSuccessAlerts bool `db:"suppress_success_alerts" json:"suppress_success_alerts"` App TemplateApp `db:"app" json:"app"` - Tasks int `db:"tasks" json:"tasks"` + Tasks int `db:"tasks" json:"tasks" backup:"-"` } func (tpl *Template) Validate() error { diff --git a/db/TemplateVault.go b/db/TemplateVault.go index a3a964e9..98e36a0c 100644 --- a/db/TemplateVault.go +++ b/db/TemplateVault.go @@ -1,10 +1,10 @@ package db type TemplateVault struct { - ID int `db:"id" json:"id"` - ProjectID int `db:"project_id" json:"project_id"` - TemplateID int `db:"template_id" json:"template_id"` - VaultKeyID int `db:"vault_key_id" json:"vault_key_id"` + ID int `db:"id" json:"id" backup:"-"` + ProjectID int `db:"project_id" json:"project_id" backup:"-"` + TemplateID int `db:"template_id" json:"template_id" backup:"-"` + VaultKeyID int `db:"vault_key_id" json:"vault_key_id" backup:"-"` Name *string `db:"name" json:"name"` Vault *AccessKey `db:"-" json:"-"` diff --git a/db/View.go b/db/View.go index fb81a782..1a451eeb 100644 --- a/db/View.go +++ b/db/View.go @@ -1,8 +1,8 @@ package db type View struct { - ID int `db:"id" json:"id"` - ProjectID int `db:"project_id" json:"project_id"` + ID int `db:"id" json:"id" backup:"-"` + ProjectID int `db:"project_id" json:"project_id" backup:"-"` Title string `db:"title" json:"title"` Position int `db:"position" json:"position"` } @@ -12,4 +12,4 @@ func (view *View) Validate() error { return &ValidationError{"title can not be empty"} } return nil -} \ No newline at end of file +} diff --git a/services/project/backup.go b/services/project/backup.go index 837aa553..0404d520 100644 --- a/services/project/backup.go +++ b/services/project/backup.go @@ -156,21 +156,17 @@ func (b *BackupDB) new(projectID int, store db.Store) (*BackupDB, error) { } func (b *BackupDB) format() (*BackupFormat, error) { - keys := make([]BackupKey, len(b.keys)) + keys := make([]BackupAccessKey, len(b.keys)) for i, o := range b.keys { - keys[i] = BackupKey{ - Name: o.Name, - Type: o.Type, + keys[i] = BackupAccessKey{ + o, } } environments := make([]BackupEnvironment, len(b.environments)) for i, o := range b.environments { environments[i] = BackupEnvironment{ - Name: o.Name, - ENV: o.ENV, - JSON: o.JSON, - Password: o.Password, + o, } } @@ -185,9 +181,7 @@ func (b *BackupDB) format() (*BackupFormat, error) { BecomeKey, _ = findNameByID[db.AccessKey](*o.BecomeKeyID, b.keys) } inventories[i] = BackupInventory{ - Name: o.Name, - Inventory: o.Inventory, - Type: o.Type, + Inventory: o, SSHKey: SSHKey, BecomeKey: BecomeKey, } @@ -196,8 +190,7 @@ func (b *BackupDB) format() (*BackupFormat, error) { views := make([]BackupView, len(b.views)) for i, o := range b.views { views[i] = BackupView{ - Name: o.Title, - Position: o.Position, + o, } } @@ -205,10 +198,8 @@ func (b *BackupDB) format() (*BackupFormat, error) { for i, o := range b.repositories { SSHKey, _ := findNameByID[db.AccessKey](o.SSHKeyID, b.keys) repositories[i] = BackupRepository{ - Name: o.Name, - SSHKey: SSHKey, - GitURL: o.GitURL, - GitBranch: o.GitBranch, + Repository: o, + SSHKey: SSHKey, } } @@ -223,8 +214,8 @@ func (b *BackupDB) format() (*BackupFormat, error) { var vaultKey *string = nil vaultKey, _ = findNameByID[db.AccessKey](vault.VaultKeyID, b.keys) vaults = append(vaults, BackupTemplateVault{ - Name: vault.Name, - VaultKey: *vaultKey, + TemplateVault: vault, + VaultKey: *vaultKey, }) } @@ -244,31 +235,19 @@ func (b *BackupDB) format() (*BackupFormat, error) { } templates[i] = BackupTemplate{ - Name: o.Name, - AllowOverrideArgsInTask: o.AllowOverrideArgsInTask, - Arguments: o.Arguments, - Autorun: o.Autorun, - Description: o.Description, - Playbook: o.Playbook, - StartVersion: o.StartVersion, - SuppressSuccessAlerts: o.SuppressSuccessAlerts, - SurveyVars: o.SurveyVarsJSON, - Type: o.Type, - View: View, - Repository: *Repository, - Inventory: Inventory, - Environment: Environment, - BuildTemplate: BuildTemplate, - Cron: getScheduleByTemplate(o.ID, b.schedules), - Vaults: vaults, + Template: o, + View: View, + Repository: *Repository, + Inventory: Inventory, + Environment: Environment, + BuildTemplate: BuildTemplate, + Cron: getScheduleByTemplate(o.ID, b.schedules), + Vaults: vaults, } } return &BackupFormat{ Meta: BackupMeta{ - Name: b.meta.Name, - MaxParallelTasks: b.meta.MaxParallelTasks, - Alert: b.meta.Alert, - AlertChat: b.meta.AlertChat, + b.meta, }, Inventories: inventories, Environments: environments, diff --git a/services/project/restore.go b/services/project/restore.go index a8b66f67..6f53e7e8 100644 --- a/services/project/restore.go +++ b/services/project/restore.go @@ -37,46 +37,36 @@ func (e BackupEnvironment) Verify(backup *BackupFormat) error { } func (e BackupEnvironment) Restore(store db.Store, b *BackupDB) error { - environment, err := store.CreateEnvironment( - db.Environment{ - Name: e.Name, - Password: e.Password, - ProjectID: b.meta.ID, - JSON: e.JSON, - ENV: e.ENV, - }, - ) + env := e.Environment + env.ProjectID = b.meta.ID + newEnv, err := store.CreateEnvironment(env) if err != nil { return err } - b.environments = append(b.environments, environment) + b.environments = append(b.environments, newEnv) return nil } func (e BackupView) Verify(backup *BackupFormat) error { - return verifyDuplicate[BackupView](e.Name, backup.Views) + return verifyDuplicate[BackupView](e.Title, backup.Views) } func (e BackupView) Restore(store db.Store, b *BackupDB) error { - view, err := store.CreateView( - db.View{ - Title: e.Name, - ProjectID: b.meta.ID, - Position: e.Position, - }, - ) + v := e.View + v.ProjectID = b.meta.ID + newView, err := store.CreateView(v) if err != nil { return err } - b.views = append(b.views, view) + b.views = append(b.views, newView) return nil } -func (e BackupKey) Verify(backup *BackupFormat) error { - return verifyDuplicate[BackupKey](e.Name, backup.Keys) +func (e BackupAccessKey) Verify(backup *BackupFormat) error { + return verifyDuplicate[BackupAccessKey](e.Name, backup.Keys) } -func (e BackupKey) Restore(store db.Store, b *BackupDB) error { +func (e BackupAccessKey) Restore(store db.Store, b *BackupDB) error { key, err := store.CreateAccessKey( db.AccessKey{ Name: e.Name, @@ -95,10 +85,10 @@ func (e BackupInventory) Verify(backup *BackupFormat) error { if err := verifyDuplicate[BackupInventory](e.Name, backup.Inventories); err != nil { return err } - if e.SSHKey != nil && getEntryByName[BackupKey](e.SSHKey, backup.Keys) == nil { + if e.SSHKey != nil && getEntryByName[BackupAccessKey](e.SSHKey, backup.Keys) == nil { return fmt.Errorf("SSHKey does not exist in keys[].Name") } - if e.BecomeKey != nil && getEntryByName[BackupKey](e.BecomeKey, backup.Keys) == nil { + if e.BecomeKey != nil && getEntryByName[BackupAccessKey](e.BecomeKey, backup.Keys) == nil { return fmt.Errorf("BecomeKey does not exist in keys[].Name") } return nil @@ -121,20 +111,17 @@ func (e BackupInventory) Restore(store db.Store, b *BackupDB) error { } else { BecomeKeyID = &((*k).ID) } - inventory, err := store.CreateInventory( - db.Inventory{ - ProjectID: b.meta.ID, - Name: e.Name, - Type: e.Type, - SSHKeyID: SSHKeyID, - BecomeKeyID: BecomeKeyID, - Inventory: e.Inventory, - }, - ) + + inv := e.Inventory + inv.ProjectID = b.meta.ID + inv.SSHKeyID = SSHKeyID + inv.BecomeKeyID = BecomeKeyID + + newInventory, err := store.CreateInventory(inv) if err != nil { return err } - b.inventories = append(b.inventories, inventory) + b.inventories = append(b.inventories, newInventory) return nil } @@ -142,7 +129,7 @@ func (e BackupRepository) Verify(backup *BackupFormat) error { if err := verifyDuplicate[BackupRepository](e.Name, backup.Repositories); err != nil { return err } - if e.SSHKey != nil && getEntryByName[BackupKey](e.SSHKey, backup.Keys) == nil { + if e.SSHKey != nil && getEntryByName[BackupAccessKey](e.SSHKey, backup.Keys) == nil { return fmt.Errorf("SSHKey does not exist in keys[].Name") } return nil @@ -181,12 +168,12 @@ func (e BackupTemplate) Verify(backup *BackupFormat) error { if getEntryByName[BackupInventory](e.Inventory, backup.Inventories) == nil { return fmt.Errorf("inventory does not exist in inventories[].name") } - if e.VaultKey != nil && getEntryByName[BackupKey](e.VaultKey, backup.Keys) == nil { + if e.VaultKey != nil && getEntryByName[BackupAccessKey](e.VaultKey, backup.Keys) == nil { return fmt.Errorf("vault_key does not exist in keys[].name") } if e.Vaults != nil { for _, vault := range e.Vaults { - if getEntryByName[BackupKey](&vault.VaultKey, backup.Keys) == nil { + if getEntryByName[BackupAccessKey](&vault.VaultKey, backup.Keys) == nil { return fmt.Errorf("vaults[].vaultKey does not exist in keys[].name") } } diff --git a/services/project/types.go b/services/project/types.go index f5da67d7..cefcd883 100644 --- a/services/project/types.go +++ b/services/project/types.go @@ -16,80 +16,60 @@ type BackupDB struct { } type BackupFormat struct { - Meta BackupMeta `json:"meta"` - Templates []BackupTemplate `json:"templates"` - Repositories []BackupRepository `json:"repositories"` - Keys []BackupKey `json:"keys"` - Views []BackupView `json:"views"` - Inventories []BackupInventory `json:"inventories"` - Environments []BackupEnvironment `json:"environments"` + Meta BackupMeta `backup:"meta"` + Templates []BackupTemplate `backup:"templates"` + Repositories []BackupRepository `backup:"repositories"` + Keys []BackupAccessKey `backup:"keys"` + Views []BackupView `backup:"views"` + Inventories []BackupInventory `backup:"inventories"` + Environments []BackupEnvironment `backup:"environments"` } type BackupMeta struct { - Name string `json:"name"` - Alert bool `json:"alert"` - AlertChat *string `json:"alert_chat"` - MaxParallelTasks int `json:"max_parallel_tasks"` + db.Project } type BackupEnvironment struct { - Name string `json:"name"` - Password *string `json:"password"` - JSON string `json:"json"` - ENV *string `json:"env"` + db.Environment } -type BackupKey struct { - Name string `json:"name"` - Type db.AccessKeyType `json:"type"` +type BackupAccessKey struct { + db.AccessKey } type BackupView struct { - Name string `json:"name"` - Position int `json:"position"` + db.View } type BackupInventory struct { - Name string `json:"name"` - Inventory string `json:"inventory"` - SSHKey *string `json:"ssh_key"` - BecomeKey *string `json:"become_key"` - Type db.InventoryType `json:"type"` + db.Inventory + SSHKey *string `backup:"ssh_key"` + BecomeKey *string `backup:"become_key"` } type BackupRepository struct { - Name string `json:"name"` - GitURL string `json:"git_url"` - GitBranch string `json:"git_branch"` - SSHKey *string `json:"ssh_key"` + db.Repository + SSHKey *string `backup:"ssh_key"` } type BackupTemplate struct { - Inventory *string `json:"inventory"` - Repository string `json:"repository"` - Environment *string `json:"environment"` - Name string `json:"name"` - Playbook string `json:"playbook"` - Arguments *string `json:"arguments"` - AllowOverrideArgsInTask bool `json:"allow_override_args_in_task"` - Description *string `json:"description"` - Type db.TemplateType `json:"type"` - StartVersion *string `json:"start_version"` - BuildTemplate *string `json:"build_template"` - View *string `json:"view"` - Autorun bool `json:"autorun"` - SurveyVars *string `json:"survey_vars"` - SuppressSuccessAlerts bool `json:"suppress_success_alerts"` - Cron *string `json:"cron"` - Vaults []BackupTemplateVault `json:"vaults"` + db.Template + + Inventory *string `backup:"inventory"` + Repository string `backup:"repository"` + Environment *string `backup:"environment"` + BuildTemplate *string `backup:"build_template"` + View *string `backup:"view"` + Vaults []BackupTemplateVault `backup:"vaults"` + Cron *string `backup:"cron"` // Deprecated: Left here for compatibility with old backups VaultKey *string `json:"vault_key"` } type BackupTemplateVault struct { - Name *string `json:"name"` - VaultKey string `json:"vault_key"` + db.TemplateVault + VaultKey string `backup:"vault_key"` } type BackupEntry interface { @@ -106,7 +86,7 @@ func (e BackupInventory) GetName() string { return e.Name } -func (e BackupKey) GetName() string { +func (e BackupAccessKey) GetName() string { return e.Name } @@ -115,7 +95,7 @@ func (e BackupRepository) GetName() string { } func (e BackupView) GetName() string { - return e.Name + return e.Title } func (e BackupTemplate) GetName() string {