Kiu/Semaphore
Kiu/Semaphore
1
0
Fork 0
mirror of https://github.com/semaphoreui/semaphore.git synced 2025-01-20 15:29:28 +01:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:semaphoreui/semaphore into develop
Some checks are pending
Dev / build-local (push) Waiting to run
Details
Dev / migrate-boltdb (push) Blocked by required conditions
Details
Dev / migrate-mysql (push) Blocked by required conditions
Details
Dev / migrate-mariadb (push) Blocked by required conditions
Details
Dev / migrate-postgres (push) Blocked by required conditions
Details
Dev / integrate-boltdb (push) Blocked by required conditions
Details
Dev / integrate-mysql (push) Blocked by required conditions
Details
Dev / integrate-mariadb (push) Blocked by required conditions
Details
Dev / integrate-postgres (push) Blocked by required conditions
Details
Dev / deploy-server (push) Blocked by required conditions
Details
Dev / deploy-runner (push) Blocked by required conditions
Details

Browse Source
This commit is contained in:
Denis Gukov 2024-10-07 17:49:16 +05:00
commit ccc6fa2502
45 changed files with 37043 additions and 35776 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.dredd/hooks/helpers.go
View File

@ -56,6 +56,7 @@ func truncateAll() {
"project__inventory",
"project__repository",
"project__template",
"project__template_vault",
"project__schedule",
"project__user",
"user",

2
README.md
View File

@ -17,7 +17,7 @@ Try the latest version of Semaphore at [https://cloud.semaphoreui.com](https://c
## What is Semaphore UI?
Semaphore UI is a modern web interface for popular DevOps tools.
Semaphore UI is a modern web interface for managing popular DevOps tools.
Semaphore UI allows you to:
* Easily run Ansible playbooks, Terraform and OpenTofu code, as well as Bash and PowerShell scripts.

18
api-docs.yml
View File

@ -136,7 +136,7 @@ definitions:
ProjectBackup:
type: object
example: {"meta":{"name":"homelab","alert":true,"alert_chat":"Test","max_parallel_tasks":0},"templates":[{"inventory":"Build","repository":"Demo","environment":"Empty","name":"Build","playbook":"build.yml","arguments":"[]","allow_override_args_in_task":false,"description":"Build Job","vault_key":null,"type":"build","start_version":"1.0.0","build_template":null,"view":"Build","autorun":false,"survey_vars":"null","suppress_success_alerts":false,"cron":"* * * * *"}],"repositories":[{"name":"Demo","git_url":"https://github.com/semaphoreui/demo-project.git","git_branch":"main","ssh_key":"None"}],"keys":[{"name":"None","type":"none"},{"name":"Vault Password","type":"login_password"}],"views":[{"name":"Build","position":0}],"inventories":[{"name":"Build","inventory":"","ssh_key":"None","become_key":"None","type":"static"},{"name":"Dev","inventory":"","ssh_key":"None","become_key":"None","type":"file"},{"name":"Prod","inventory":"","ssh_key":"None","become_key":"None","type":"file"}],"environments":[{"name":"Empty","password":null,"json":"{}","env":null}]}
example: {"meta":{"name":"homelab","alert":true,"alert_chat":"Test","max_parallel_tasks":0},"templates":[{"inventory":"Build","repository":"Demo","environment":"Empty","name":"Build","playbook":"build.yml","arguments":"[]","allow_override_args_in_task":false,"description":"Build Job","vault_key":null,"type":"build","start_version":"1.0.0","build_template":null,"view":"Build","autorun":false,"survey_vars":"null","suppress_success_alerts":false,"cron":"* * * * *"}],"repositories":[{"name":"Demo","git_url":"https://github.com/semaphoreui/demo-project.git","git_branch":"main","ssh_key":"None"}],"keys":[{"name":"None","type":"none"},{"name":"Vault Password","type":"login_password"}],"views":[{"title":"Build","position":0}],"inventories":[{"name":"Build","inventory":"","ssh_key":"None","become_key":"None","type":"static"},{"name":"Dev","inventory":"","ssh_key":"None","become_key":"None","type":"file"},{"name":"Prod","inventory":"","ssh_key":"None","become_key":"None","type":"file"}],"environments":[{"name":"Empty","password":null,"json":"{}","env":null}]}
properties:
meta:
type: object
@ -684,9 +684,10 @@ definitions:
view_id:
type: integer
minimum: 1
vault_id:
type: integer
minimum: 1
vaults:
type: array
items:
$ref: '#/definitions/TemplateVault'
name:
type: string
example: Test
@ -767,6 +768,15 @@ definitions:
example: String => "", Integer => "int"
required:
type: boolean
TemplateVault:
type: object
properties:
vault_key_id:
type: integer
minimum: 1
name:
type: string
example: default
ScheduleRequest:
type: object

3
api/helpers/helpers.go
View File

@ -87,7 +87,8 @@ func WriteJSON(w http.ResponseWriter, code int, out interface{}) {
w.WriteHeader(code)
if err := json.NewEncoder(w).Encode(out); err != nil {
panic(err)
log.Error(err)
debug.PrintStack()
}
}

42
api/projects/backupRestore.go
View File

@ -1,7 +1,9 @@
package projects
import (
"io"
"net/http"
"strings"
"github.com/ansible-semaphore/semaphore/api/helpers"
"github.com/ansible-semaphore/semaphore/db"
@ -21,27 +23,47 @@ func GetBackup(w http.ResponseWriter, r *http.Request) {
helpers.WriteError(w, err)
return
}
helpers.WriteJSON(w, http.StatusOK, backup)
str, err := backup.Marshal()
if err != nil {
helpers.WriteError(w, err)
return
}
w.Header().Set("content-type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(str))
}
func Restore(w http.ResponseWriter, r *http.Request) {
user := context.Get(r, "user").(*db.User)
var backup projectService.BackupFormat
var p *db.Project
var err error
if !helpers.Bind(w, r, &backup) {
helpers.WriteJSON(w, http.StatusBadRequest, backup)
return
}
store := helpers.Store(r)
if err = backup.Verify(); err != nil {
buf := new(strings.Builder)
if _, err := io.Copy(buf, r.Body); err != nil {
log.Error(err)
helpers.WriteError(w, err)
return
}
if p, err = backup.Restore(*user, store); err != nil {
if err := backup.Unmarshal(buf.String()); err != nil {
log.Error(err)
helpers.WriteError(w, err)
return
}
store := helpers.Store(r)
if err := backup.Verify(); err != nil {
log.Error(err)
helpers.WriteError(w, err)
return
}
var p *db.Project
p, err := backup.Restore(*user, store)
if err != nil {
log.Error(err)
helpers.WriteError(w, err)
return

29
api/projects/projects.go
View File

@ -140,7 +140,8 @@ func createDemoProject(projectID int, noneKeyID int, emptyEnvID int, store db.St
return
}
_, err = store.CreateTemplate(db.Template{
var template db.Template
template, err = store.CreateTemplate(db.Template{
Name: "Deploy to Dev",
Type: db.TemplateDeploy,
Playbook: "deploy.yml",
@ -150,7 +151,6 @@ func createDemoProject(projectID int, noneKeyID int, emptyEnvID int, store db.St
RepositoryID: demoRepo.ID,
BuildTemplateID: &buildTpl.ID,
Autorun: true,
VaultKeyID: &vaultKey.ID,
App: db.AppAnsible,
})
@ -158,7 +158,18 @@ func createDemoProject(projectID int, noneKeyID int, emptyEnvID int, store db.St
return
}
_, err = store.CreateTemplate(db.Template{
_, err = store.CreateTemplateVault(db.TemplateVault{
ProjectID: projectID,
TemplateID: template.ID,
VaultKeyID: vaultKey.ID,
Name: nil,
})
if err != nil {
return
}
template, err = store.CreateTemplate(db.Template{
Name: "Deploy to Production",
Type: db.TemplateDeploy,
Playbook: "deploy.yml",
@ -167,10 +178,20 @@ func createDemoProject(projectID int, noneKeyID int, emptyEnvID int, store db.St
EnvironmentID: &emptyEnvID,
RepositoryID: demoRepo.ID,
BuildTemplateID: &buildTpl.ID,
VaultKeyID: &vaultKey.ID,
App: db.AppAnsible,
})
if err != nil {
return
}
_, err = store.CreateTemplateVault(db.TemplateVault{
ProjectID: projectID,
TemplateID: template.ID,
VaultKeyID: vaultKey.ID,
Name: nil,
})
return
}

12
api/runners/runners.go
View File

@ -89,12 +89,14 @@ func GetRunner(w http.ResponseWriter, r *http.Request) {
data.AccessKeys[*tsk.Inventory.BecomeKeyID] = tsk.Inventory.BecomeKey
}
if tsk.Template.VaultKeyID != nil {
err := tsk.Template.VaultKey.DeserializeSecret()
if err != nil {
// TODO: return error
if tsk.Template.Vaults != nil {
for _, vault := range tsk.Template.Vaults {
err := vault.Vault.DeserializeSecret()
if err != nil {
// TODO: return error
}
data.AccessKeys[vault.VaultKeyID] = *vault.Vault
}
data.AccessKeys[*tsk.Template.VaultKeyID] = tsk.Template.VaultKey
}
if tsk.Inventory.RepositoryID != nil {

15
db/AccessKey.go
View File

@ -26,24 +26,29 @@ const (
// AccessKey represents a key used to access a machine with ansible from semaphore
type AccessKey struct {
ID int `db:"id" json:"id"`
ID int `db:"id" json:"id" backup:"-"`
Name string `db:"name" json:"name" binding:"required"`
// 'ssh/login_password/none'
Type AccessKeyType `db:"type" json:"type" binding:"required"`
ProjectID *int `db:"project_id" json:"project_id"`
ProjectID *int `db:"project_id" json:"project_id" backup:"-"`
// Secret used internally, do not assign this field.
// You should use methods SerializeSecret to fill this field.
Secret *string `db:"secret" json:"-"`
Secret *string `db:"secret" json:"-" backup:"-"`
String string `db:"-" json:"string"`
LoginPassword LoginPassword `db:"-" json:"login_password"`
SshKey SshKey `db:"-" json:"ssh"`
OverrideSecret bool `db:"-" json:"override_secret"`
EnvironmentID *int `db:"environment_id" json:"-"`
UserID *int `db:"user_id" json:"-"`
// EnvironmentID is an ID of environment which owns the access key.
EnvironmentID *int `db:"environment_id" json:"-" backup:"-"`
// UserID is an ID of user which owns the access key.
UserID *int `db:"user_id" json:"-" backup:"-"`
Empty bool `db:"-" json:"empty"`
}
type LoginPassword struct {

16
db/Environment.go
View File

@ -31,13 +31,15 @@ type EnvironmentSecret struct {
// Environment is used to pass additional arguments, in json form to ansible
type Environment struct {
ID int `db:"id" json:"id"`
Name string `db:"name" json:"name" binding:"required"`
ProjectID int `db:"project_id" json:"project_id"`
Password *string `db:"password" json:"password"`
JSON string `db:"json" json:"json" binding:"required"`
ENV *string `db:"env" json:"env" binding:"required"`
Secrets []EnvironmentSecret `db:"-" json:"secrets"`
ID int `db:"id" json:"id" backup:"-"`
Name string `db:"name" json:"name" binding:"required"`
ProjectID int `db:"project_id" json:"project_id" backup:"-"`
Password *string `db:"password" json:"password"`
JSON string `db:"json" json:"json" binding:"required"`
ENV *string `db:"env" json:"env" binding:"required"`
// Secrets is a field which used to update secrets associated with the environment.
Secrets []EnvironmentSecret `db:"-" json:"secrets" backup:"-"`
}
func (s *EnvironmentSecret) Validate() error {

18
db/Inventory.go
View File

@ -14,17 +14,17 @@ const (
// Inventory is the model of an ansible inventory file
type Inventory struct {
ID int `db:"id" json:"id"`
ID int `db:"id" json:"id" backup:"-"`
Name string `db:"name" json:"name" binding:"required"`
ProjectID int `db:"project_id" json:"project_id"`
ProjectID int `db:"project_id" json:"project_id" backup:"-"`
Inventory string `db:"inventory" json:"inventory"`
// accesses hosts in inventory
SSHKeyID *int `db:"ssh_key_id" json:"ssh_key_id"`
SSHKey AccessKey `db:"-" json:"-"`
SSHKeyID *int `db:"ssh_key_id" json:"ssh_key_id" backup:"-"`
SSHKey AccessKey `db:"-" json:"-" backup:"-"`
BecomeKeyID *int `db:"become_key_id" json:"become_key_id"`
BecomeKey AccessKey `db:"-" json:"-"`
BecomeKeyID *int `db:"become_key_id" json:"become_key_id" backup:"-"`
BecomeKey AccessKey `db:"-" json:"-" backup:"-"`
// static/file
Type InventoryType `db:"type" json:"type"`
@ -33,12 +33,12 @@ type Inventory struct {
// It is not used now but can be used in feature for
// inventories which can not be used more than one template
// at once.
HolderID *int `db:"holder_id" json:"holder_id"`
HolderID *int `db:"holder_id" json:"holder_id" backup:"-"`
// RepositoryID is an ID of repo where inventory stored.
// If null than inventory will be got from template repository.
RepositoryID *int `db:"repository_id" json:"repository_id"`
Repository *Repository `db:"-" json:"-"`
RepositoryID *int `db:"repository_id" json:"repository_id" backup:"-"`
Repository *Repository `db:"-" json:"-" backup:"-"`
}
func (e Inventory) GetFilename() string {

1
db/Migration.go
View File

@ -71,6 +71,7 @@ func GetMigrations() []Migration {
{Version: "2.10.12"},
{Version: "2.10.15"},
{Version: "2.10.16"},
{Version: "2.10.24"},
{Version: "2.10.26"},
}
}

4
db/Project.go
View File

@ -6,9 +6,9 @@ import (
// Project is the top level structure in Semaphore
type Project struct {
ID int `db:"id" json:"id"`
ID int `db:"id" json:"id" backup:"-"`
Name string `db:"name" json:"name" binding:"required"`
Created time.Time `db:"created" json:"created"`
Created time.Time `db:"created" json:"created" backup:"-"`
Alert bool `db:"alert" json:"alert"`
AlertChat *string `db:"alert_chat" json:"alert_chat"`
MaxParallelTasks int `db:"max_parallel_tasks" json:"max_parallel_tasks"`

8
db/Repository.go
View File

@ -23,14 +23,14 @@ const (
// Repository is the model for code stored in a git repository
type Repository struct {
ID int `db:"id" json:"id"`
ID int `db:"id" json:"id" backup:"-"`
Name string `db:"name" json:"name" binding:"required"`
ProjectID int `db:"project_id" json:"project_id"`
ProjectID int `db:"project_id" json:"project_id" backup:"-"`
GitURL string `db:"git_url" json:"git_url" binding:"required"`
GitBranch string `db:"git_branch" json:"git_branch" binding:"required"`
SSHKeyID int `db:"ssh_key_id" json:"ssh_key_id" binding:"required"`
SSHKeyID int `db:"ssh_key_id" json:"ssh_key_id" binding:"required" backup:"-"`
SSHKey AccessKey `db:"-" json:"-"`
SSHKey AccessKey `db:"-" json:"-" backup:"-"`
}
func (r Repository) ClearCache() error {

10
db/Schedule.go
View File

@ -1,15 +1,15 @@
package db
type Schedule struct {
ID int `db:"id" json:"id"`
ProjectID int `db:"project_id" json:"project_id"`
TemplateID int `db:"template_id" json:"template_id"`
ID int `db:"id" json:"id" backup:"-"`
ProjectID int `db:"project_id" json:"project_id" backup:"-"`
TemplateID int `db:"template_id" json:"template_id" backup:"-"`
CronFormat string `db:"cron_format" json:"cron_format"`
Name string `db:"name" json:"name"`
Active bool `db:"active" json:"active"`
LastCommitHash *string `db:"last_commit_hash" json:"-"`
RepositoryID *int `db:"repository_id" json:"repository_id"`
LastCommitHash *string `db:"last_commit_hash" json:"-" backup:"-"`
RepositoryID *int `db:"repository_id" json:"repository_id" backup:"-"`
}
type ScheduleWithTpl struct {

15
db/Store.go
View File

@ -23,7 +23,9 @@ func GetParsedTime(t time.Time) time.Time {
}
func ObjectToJSON(obj interface{}) *string {
if obj == nil || (reflect.ValueOf(obj).Kind() == reflect.Ptr && reflect.ValueOf(obj).IsNil()) {
if obj == nil ||
(reflect.ValueOf(obj).Kind() == reflect.Ptr && reflect.ValueOf(obj).IsNil()) ||
(reflect.ValueOf(obj).Kind() == reflect.Slice && reflect.ValueOf(obj).IsZero()) {
return nil
}
bytes, err := json.Marshal(obj)
@ -262,6 +264,10 @@ type Store interface {
DeleteGlobalRunner(runnerID int) error
UpdateRunner(runner Runner) error
CreateRunner(runner Runner) (Runner, error)
GetTemplateVaults(projectID int, templateID int) ([]TemplateVault, error)
CreateTemplateVault(vault TemplateVault) (TemplateVault, error)
UpdateTemplateVaults(projectID int, templateID int, vaults []TemplateVault) error
}
var AccessKeyProps = ObjectProps{
@ -418,6 +424,13 @@ var OptionProps = ObjectProps{
IsGlobal: true,
}
var TemplateVaultProps = ObjectProps{
TableName: "project__template_vault",
Type: reflect.TypeOf(TemplateVault{}),
PrimaryColumnName: "id",
ReferringColumnSuffix: "template_id",
}
func (p ObjectProps) GetReferringFieldsFrom(t reflect.Type) (fields []string, err error) {
n := t.NumField()
for i := 0; i < n; i++ {

30
db/Template.go
View File

@ -58,12 +58,12 @@ type TemplateFilter struct {
// Template is a user defined model that is used to run a task
type Template struct {
ID int `db:"id" json:"id"`
ID int `db:"id" json:"id" backup:"-"`
ProjectID int `db:"project_id" json:"project_id"`
InventoryID *int `db:"inventory_id" json:"inventory_id"`
RepositoryID int `db:"repository_id" json:"repository_id"`
EnvironmentID *int `db:"environment_id" json:"environment_id"`
ProjectID int `db:"project_id" json:"project_id" backup:"-"`
InventoryID *int `db:"inventory_id" json:"inventory_id" backup:"-"`
RepositoryID int `db:"repository_id" json:"repository_id" backup:"-"`
EnvironmentID *int `db:"environment_id" json:"environment_id" backup:"-"`
// Name as described in https://github.com/ansible-semaphore/semaphore/issues/188
Name string `db:"name" json:"name"`
@ -76,16 +76,15 @@ type Template struct {
Description *string `db:"description" json:"description"`
VaultKeyID *int `db:"vault_key_id" json:"vault_key_id"`
VaultKey AccessKey `db:"-" json:"-"`
Vaults []TemplateVault `db:"-" json:"vaults" backup:"-"`
Type TemplateType `db:"type" json:"type"`
StartVersion *string `db:"start_version" json:"start_version"`
BuildTemplateID *int `db:"build_template_id" json:"build_template_id"`
BuildTemplateID *int `db:"build_template_id" json:"build_template_id" backup:"-"`
ViewID *int `db:"view_id" json:"view_id"`
ViewID *int `db:"view_id" json:"view_id" backup:"-"`
LastTask *TaskWithTpl `db:"-" json:"last_task"`
LastTask *TaskWithTpl `db:"-" json:"last_task" backup:"-"`
Autorun bool `db:"autorun" json:"autorun"`
@ -93,13 +92,13 @@ type Template struct {
// It is not used for store survey vars to database.
// Do not use it in your code. Use SurveyVars instead.
SurveyVarsJSON *string `db:"survey_vars" json:"-"`
SurveyVars []SurveyVar `db:"-" json:"survey_vars"`
SurveyVars []SurveyVar `db:"-" json:"survey_vars" backup:"-"`
SuppressSuccessAlerts bool `db:"suppress_success_alerts" json:"suppress_success_alerts"`
App TemplateApp `db:"app" json:"app"`
Tasks int `db:"tasks" json:"tasks"`
Tasks int `db:"tasks" json:"tasks" backup:"-"`
}
func (tpl *Template) Validate() error {
@ -128,13 +127,12 @@ func (tpl *Template) Validate() error {
}
func FillTemplate(d Store, template *Template) (err error) {
if template.VaultKeyID != nil {
template.VaultKey, err = d.GetAccessKey(template.ProjectID, *template.VaultKeyID)
}
var vaults []TemplateVault
vaults, err = d.GetTemplateVaults(template.ProjectID, template.ID)
if err != nil {
return
}
template.Vaults = vaults
var tasks []TaskWithTpl
tasks, err = d.GetTemplateTasks(template.ProjectID, template.ID, RetrieveQueryParams{Count: 1})

21
db/TemplateVault.go Normal file
View File

@ -0,0 +1,21 @@
package db
type TemplateVault struct {
ID int `db:"id" json:"id" backup:"-"`
ProjectID int `db:"project_id" json:"project_id" backup:"-"`
TemplateID int `db:"template_id" json:"template_id" backup:"-"`
VaultKeyID int `db:"vault_key_id" json:"vault_key_id" backup:"-"`
Name *string `db:"name" json:"name"`
Vault *AccessKey `db:"-" json:"-"`
}
func FillTemplateVault(d Store, projectID int, templateVault *TemplateVault) (err error) {
var vault AccessKey
vault, err = d.GetAccessKey(projectID, templateVault.VaultKeyID)
if err != nil {
return
}
templateVault.Vault = &vault
return
}

6
db/View.go
View File

@ -1,8 +1,8 @@
package db
type View struct {
ID int `db:"id" json:"id"`
ProjectID int `db:"project_id" json:"project_id"`
ID int `db:"id" json:"id" backup:"-"`
ProjectID int `db:"project_id" json:"project_id" backup:"-"`
Title string `db:"title" json:"title"`
Position int `db:"position" json:"position"`
}
@ -12,4 +12,4 @@ func (view *View) Validate() error {
return &ValidationError{"title can not be empty"}
}
return nil
}
}

2
db/bolt/migration.go
View File

@ -45,6 +45,8 @@ func (d *BoltDb) ApplyMigration(m db.Migration) (err error) {
err = migration_2_10_12{migration{d.db}}.Apply()
case "2.10.16":
err = migration_2_10_16{migration{d.db}}.Apply()
case "2.10.24":
err = migration_2_10_24{migration{d.db}}.Apply()
}
if err != nil {

46
db/bolt/migration_2_10_24.go Normal file
View File

@ -0,0 +1,46 @@
package bolt
import "fmt"
type migration_2_10_24 struct {
migration
}
func (d migration_2_10_24) Apply() (err error) {
projectIDs, err := d.getProjectIDs()
if err != nil {
return err
}
for _, projectID := range projectIDs {
templates, err := d.getObjects(projectID, "template")
if err != nil {
return err
}
var templateVaultID int = 1
for templateID, template := range templates {
if template["vault_key_id"] != nil {
templateVault := map[string]interface{}{
"id": templateVaultID,
"project_id": template["project_id"],
"template_id": template["id"],
"vault_key_id": template["vault_key_id"],
"name": nil,
}
err = d.setObject(projectID, "template_vault", fmt.Sprintf("%010d", templateVaultID), templateVault)
if err != nil {
return err
}
templateVaultID++
}
delete(template, "vault_key_id")
err = d.setObject(projectID, "template", templateID, template)
if err != nil {
return err
}
}
}
return
}

94
db/bolt/migration_2_10_24_test.go Normal file
View File

@ -0,0 +1,94 @@
package bolt
import (
"encoding/json"
"go.etcd.io/bbolt"
"testing"
)
func TestMigration_2_10_24_Apply(t *testing.T) {
store := CreateTestStore()
err := store.db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte("project"))
if err != nil {
return err
}
err = b.Put([]byte("0000000001"), []byte("{}"))
if err != nil {
return err
}
r, err := tx.CreateBucketIfNotExists([]byte("project__template_0000000001"))
if err != nil {
return err
}
err = r.Put([]byte("0000000001"),
[]byte("{\"id\":\"1\",\"project_id\":\"1\",\"vault_key_id\":\"1\"}"))
return err
})
if err != nil {
t.Fatal(err)
}
err = migration_2_10_24{migration{store.db}}.Apply()
if err != nil {
t.Fatal(err)
}
var template map[string]interface{}
err = store.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte("project__template_0000000001"))
str := string(b.Get([]byte("0000000001")))
return json.Unmarshal([]byte(str), &template)
})
if err != nil {
t.Fatal(err)
}
var templateVault map[string]interface{}
err = store.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte("project__template_vault_0000000001"))
str := string(b.Get([]byte("0000000001")))
return json.Unmarshal([]byte(str), &templateVault)
})
if err != nil {
t.Fatal(err)
}
if _, ok := template["vault_key_id"]; ok {
t.Fatal("vault_key_id must be deleted")
}
if templateVault["vault_key_id"].(string) != "1" {
t.Fatal("invalid vault_key_id: " + templateVault["vault_key_id"].(string))
}
}
func TestMigration_2_10_24_Apply2(t *testing.T) {
store := CreateTestStore()
err := store.db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte("project"))
if err != nil {
return err
}
err = b.Put([]byte("0000000001"), []byte("{}"))
return err
})
if err != nil {
t.Fatal(err)
}
err = migration_2_10_24{migration{store.db}}.Apply()
if err != nil {
t.Fatal(err)
}
}

15
db/bolt/template.go
View File

@ -19,6 +19,10 @@ func (d *BoltDb) CreateTemplate(template db.Template) (newTemplate db.Template,
return
}
newTemplate = newTpl.(db.Template)
err = d.UpdateTemplateVaults(template.ProjectID, newTemplate.ID, template.Vaults)
if err != nil {
return
}
err = db.FillTemplate(d, &newTemplate)
return
}
@ -31,7 +35,11 @@ func (d *BoltDb) UpdateTemplate(template db.Template) error {
}
template.SurveyVarsJSON = db.ObjectToJSON(template.SurveyVars)
return d.updateObject(template.ProjectID, db.TemplateProps, template)
err = d.updateObject(template.ProjectID, db.TemplateProps, template)
if err != nil {
return err
}
return d.UpdateTemplateVaults(template.ProjectID, template.ID, template.Vaults)
}
func (d *BoltDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams) (templates []db.Template, err error) {
@ -59,6 +67,7 @@ func (d *BoltDb) GetTemplates(projectID int, filter db.TemplateFilter, params db
templatesMap := make(map[int]*db.Template)
for i := 0; i < len(templates); i++ {
templates[i].Vaults, err = d.GetTemplateVaults(projectID, templates[i].ID)
templatesMap[templates[i].ID] = &templates[i]
}
@ -69,6 +78,10 @@ func (d *BoltDb) GetTemplates(projectID int, filter db.TemplateFilter, params db
err = d.apply(projectID, db.TaskProps, db.RetrieveQueryParams{}, func(i interface{}) error {
task := i.(db.Task)
if task.ProjectID != projectID {
return nil
}
tpl, ok := templatesMap[task.TemplateID]
if !ok {
return nil

63
db/bolt/template_vault.go Normal file
View File

@ -0,0 +1,63 @@
package bolt
import (
"github.com/ansible-semaphore/semaphore/db"
"go.etcd.io/bbolt"
)
func (d *BoltDb) GetTemplateVaults(projectID int, templateID int) (vaults []db.TemplateVault, err error) {
err = d.getObjects(projectID, db.TemplateVaultProps, db.RetrieveQueryParams{}, func(referringObj interface{}) bool {
return referringObj.(db.TemplateVault).TemplateID == templateID
}, &vaults)
if err != nil {
return
}
for i := range vaults {
err = db.FillTemplateVault(d, projectID, &vaults[i])
if err != nil {
return
}
}
return
}
func (d *BoltDb) CreateTemplateVault(vault db.TemplateVault) (newVault db.TemplateVault, err error) {
var newTpl interface{}
newTpl, err = d.createObject(vault.ProjectID, db.TemplateVaultProps, vault)
if err != nil {
return
}
newVault = newTpl.(db.TemplateVault)
return
}
func (d *BoltDb) UpdateTemplateVaults(projectID int, templateID int, vaults []db.TemplateVault) (err error) {
if vaults == nil {
vaults = []db.TemplateVault{}
}
var oldVaults []db.TemplateVault
oldVaults, err = d.GetTemplateVaults(projectID, templateID)
err = d.db.Update(func(tx *bbolt.Tx) error {
for _, vault := range oldVaults {
err = d.deleteObject(projectID, db.TemplateVaultProps, intObjectID(vault.ID), tx)
if err != nil {
return err
}
}
for _, vault := range vaults {
vault.ProjectID = projectID
vault.TemplateID = templateID
_, err = d.createObjectTx(tx, projectID, db.TemplateVaultProps, vault)
if err != nil {
return err
}
}
return nil
})
return
}

27
db/sql/migration.go
View File

@ -142,6 +142,16 @@ func (d *SqlDb) ApplyMigration(migration db.Migration) error {
return err
}
switch migration.Version {
case "2.10.24":
err = migration_2_10_24{db: d}.PreApply(tx)
}
if err != nil {
handleRollbackError(tx.Rollback())
return err
}
queries := getVersionSQL(getVersionPath(migration))
for i, query := range queries {
fmt.Printf("\r [%d/%d]", i+1, len(query))
@ -164,20 +174,21 @@ func (d *SqlDb) ApplyMigration(migration db.Migration) error {
}
}
_, err = tx.Exec(d.PrepareQuery("insert into migrations(version, upgraded_date) values (?, ?)"), migration.Version, time.Now())
switch migration.Version {
case "2.8.26":
err = migration_2_8_26{db: d}.PostApply(tx)
case "2.8.42":
err = migration_2_8_42{db: d}.PostApply(tx)
}
if err != nil {
handleRollbackError(tx.Rollback())
return err
}
switch migration.Version {
case "2.8.26":
err = migration_2_8_26{db: d}.Apply(tx)
case "2.8.42":
err = migration_2_8_42{db: d}.Apply(tx)
}
_, err = tx.Exec(d.PrepareQuery("insert into migrations(version, upgraded_date) values (?, ?)"), migration.Version, time.Now())
if err != nil {
handleRollbackError(tx.Rollback())
return err
}

19
db/sql/migration_2_10_24.go Normal file
View File

@ -0,0 +1,19 @@
package sql
import "github.com/go-gorp/gorp/v3"
type migration_2_10_24 struct {
db *SqlDb
}
func (m migration_2_10_24) PreApply(tx *gorp.Transaction) error {
switch m.db.sql.Dialect.(type) {
case gorp.MySQLDialect:
_, _ = tx.Exec(m.db.PrepareQuery("alter table `project__template` drop foreign key `project__template_ibfk_6`"))
case gorp.PostgresDialect:
_, err := tx.Exec(
m.db.PrepareQuery("alter table `project__template` drop constraint if exists `project__template_vault_key_id_fkey`"))
return err
}
return nil
}

2
db/sql/migration_2_8_28.go
View File

@ -9,7 +9,7 @@ type migration_2_8_26 struct {
db *SqlDb
}
func (m migration_2_8_26) Apply(tx *gorp.Transaction) error {
func (m migration_2_8_26) PostApply(tx *gorp.Transaction) error {
rows, err := tx.Query(m.db.PrepareQuery("SELECT id, git_url FROM project__repository"))
if err != nil {
return err

2
db/sql/migration_2_8_42.go
View File

@ -6,7 +6,7 @@ type migration_2_8_42 struct {
db *SqlDb
}
func (m migration_2_8_42) Apply(tx *gorp.Transaction) error {
func (m migration_2_8_42) PostApply(tx *gorp.Transaction) error {
switch m.db.sql.Dialect.(type) {
case gorp.MySQLDialect:
_, _ = tx.Exec(m.db.PrepareQuery("alter table `task` drop foreign key `task_ibfk_3`"))

18
db/sql/migrations/v2.10.24.sql Normal file
View File

@ -0,0 +1,18 @@
create table `project__template_vault` (
`id` integer primary key autoincrement,
`project_id` int not null,
`template_id` int not null,
`vault_key_id` int not null,
`name` varchar(255),
unique (`template_id`, `vault_key_id`, `name`),
foreign key (`project_id`) references project(`id`) on delete cascade,
foreign key (`template_id`) references project__template(`id`) on delete cascade,
foreign key (`vault_key_id`) references `access_key`(`id`) on delete cascade
);
insert into `project__template_vault` (template_id, project_id, vault_key_id)
select `id` as template_id, project_id, vault_key_id
from `project__template` where `vault_key_id` is not null;
alter table `project__template` drop column `vault_key_id`;

24
db/sql/template.go
View File

@ -17,9 +17,9 @@ func (d *SqlDb) CreateTemplate(template db.Template) (newTemplate db.Template, e
insertID, err := d.insert(
"id",
"insert into project__template (project_id, inventory_id, repository_id, environment_id, "+
"name, playbook, arguments, allow_override_args_in_task, description, vault_key_id, `type`, start_version,"+
"name, playbook, arguments, allow_override_args_in_task, description, `type`, start_version,"+
"build_template_id, view_id, autorun, survey_vars, suppress_success_alerts, app)"+
"values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
template.ProjectID,
template.InventoryID,
template.RepositoryID,
@ -29,7 +29,6 @@ func (d *SqlDb) CreateTemplate(template db.Template) (newTemplate db.Template, e
template.Arguments,
template.AllowOverrideArgsInTask,
template.Description,
template.VaultKeyID,
template.Type,
template.StartVersion,
template.BuildTemplateID,
@ -43,6 +42,11 @@ func (d *SqlDb) CreateTemplate(template db.Template) (newTemplate db.Template, e
return
}
err = d.UpdateTemplateVaults(template.ProjectID, insertID, template.Vaults)
if err != nil {
return
}
err = db.FillTemplate(d, &newTemplate)
if err != nil {
@ -71,7 +75,6 @@ func (d *SqlDb) UpdateTemplate(template db.Template) error {
"arguments=?, "+
"allow_override_args_in_task=?, "+
"description=?, "+
"vault_key_id=?, "+
"`type`=?, "+
"start_version=?,"+
"build_template_id=?, "+
@ -89,7 +92,6 @@ func (d *SqlDb) UpdateTemplate(template db.Template) error {
template.Arguments,
template.AllowOverrideArgsInTask,
template.Description,
template.VaultKeyID,
template.Type,
template.StartVersion,
template.BuildTemplateID,
@ -101,6 +103,12 @@ func (d *SqlDb) UpdateTemplate(template db.Template) error {
template.ID,
template.ProjectID,
)
if err != nil {
return err
}
err = d.UpdateTemplateVaults(template.ProjectID, template.ID, template.Vaults)
return err
}
@ -122,7 +130,6 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
"pt.playbook",
"pt.arguments",
"pt.allow_override_args_in_task",
"pt.vault_key_id",
"pt.build_template_id",
"pt.start_version",
"pt.view_id",
@ -216,6 +223,11 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.
}
}
template.Vaults, err = d.GetTemplateVaults(projectID, template.ID)
if err != nil {
return
}
templates = append(templates, template)
}

75
db/sql/template_vault.go Normal file
View File

@ -0,0 +1,75 @@
package sql
import (
"github.com/ansible-semaphore/semaphore/db"
"strconv"
"strings"
)
func (d *SqlDb) GetTemplateVaults(projectID int, templateID int) (vaults []db.TemplateVault, err error) {
vaults = []db.TemplateVault{}
_, err = d.selectAll(&vaults, "select * from project__template_vault where project_id=? and template_id=?", projectID, templateID)
if err != nil {
return
}
for i := range vaults {
err = db.FillTemplateVault(d, projectID, &vaults[i])
if err != nil {
return
}
}
return
}
func (d *SqlDb) CreateTemplateVault(vault db.TemplateVault) (newVault db.TemplateVault, err error) {
insertID, err := d.insert(
"id",
"insert into project__template_vault (project_id, template_id, vault_key_id, name) values (?, ?, ?, ?)",
vault.ProjectID,
vault.TemplateID,
vault.VaultKeyID,
vault.Name)
if err != nil {
return
}
newVault = vault
newVault.ID = insertID
return
}
func (d *SqlDb) UpdateTemplateVaults(projectID int, templateID int, vaults []db.TemplateVault) (err error) {
if vaults == nil {
vaults = []db.TemplateVault{}
}
var vaultIDs []string
for _, vault := range vaults {
if vault.ID == 0 {
// Insert new vaults
var vaultId int
vaultId, err = d.insert("id", "insert into project__template_vault (project_id, template_id, vault_key_id, name) values (?, ?, ?, ?)", projectID, templateID, vault.VaultKeyID, vault.Name)
if err != nil {
return
}
vaultIDs = append(vaultIDs, strconv.Itoa(vaultId))
} else {
// Update existing vaults
_, err = d.exec("update project__template_vault set project_id=?, template_id=?, vault_key_id=?, name=? where id=?", projectID, templateID, vault.VaultKeyID, vault.Name, vault.ID)
vaultIDs = append(vaultIDs, strconv.Itoa(vault.ID))
}
if err != nil {
return
}
}
// Delete removed vaults
if len(vaultIDs) == 0 {
_, err = d.exec("delete from project__template_vault where project_id=? and template_id=?", projectID, templateID)
} else {
_, err = d.exec("delete from project__template_vault where project_id=? and template_id=? and id not in ("+strings.Join(vaultIDs, ",")+")", projectID, templateID)
}
return
}

4
db_lib/AnsibleApp.go
View File

@ -95,7 +95,7 @@ func (t *AnsibleApp) installGalaxyRequirementsFile(requirementsType string, requ
requirementsHashFilePath := fmt.Sprintf("%s.md5", requirementsFilePath)
if _, err := os.Stat(requirementsFilePath); err != nil {
t.Log("No " + requirementsType + "/requirements.yml file found. Skip galaxy install process.\n")
t.Log("No " + requirementsFilePath + " file found. Skip galaxy install process.\n")
return nil
}
@ -113,7 +113,7 @@ func (t *AnsibleApp) installGalaxyRequirementsFile(requirementsType string, requ
return err
}
} else {
t.Log(requirementsType + "/requirements.yml has no changes. Skip galaxy install process.\n")
t.Log(requirementsFilePath + " has no changes. Skip galaxy install process.\n")
}
return nil

97
services/project/backup.go
View File

@ -1,7 +1,9 @@
package project
import (
"encoding/json"
"fmt"
"reflect"
"github.com/ansible-semaphore/semaphore/db"
"github.com/ansible-semaphore/semaphore/pkg/random"
@ -156,21 +158,17 @@ func (b *BackupDB) new(projectID int, store db.Store) (*BackupDB, error) {
}
func (b *BackupDB) format() (*BackupFormat, error) {
keys := make([]BackupKey, len(b.keys))
keys := make([]BackupAccessKey, len(b.keys))
for i, o := range b.keys {
keys[i] = BackupKey{
Name: o.Name,
Type: o.Type,
keys[i] = BackupAccessKey{
o,
}
}
environments := make([]BackupEnvironment, len(b.environments))
for i, o := range b.environments {
environments[i] = BackupEnvironment{
Name: o.Name,
ENV: o.ENV,
JSON: o.JSON,
Password: o.Password,
o,
}
}
@ -185,9 +183,7 @@ func (b *BackupDB) format() (*BackupFormat, error) {
BecomeKey, _ = findNameByID[db.AccessKey](*o.BecomeKeyID, b.keys)
}
inventories[i] = BackupInventory{
Name: o.Name,
Inventory: o.Inventory,
Type: o.Type,
Inventory: o,
SSHKey: SSHKey,
BecomeKey: BecomeKey,
}
@ -196,8 +192,7 @@ func (b *BackupDB) format() (*BackupFormat, error) {
views := make([]BackupView, len(b.views))
for i, o := range b.views {
views[i] = BackupView{
Name: o.Title,
Position: o.Position,
o,
}
}
@ -205,10 +200,8 @@ func (b *BackupDB) format() (*BackupFormat, error) {
for i, o := range b.repositories {
SSHKey, _ := findNameByID[db.AccessKey](o.SSHKeyID, b.keys)
repositories[i] = BackupRepository{
Name: o.Name,
SSHKey: SSHKey,
GitURL: o.GitURL,
GitBranch: o.GitBranch,
Repository: o,
SSHKey: SSHKey,
}
}
@ -218,9 +211,15 @@ func (b *BackupDB) format() (*BackupFormat, error) {
if o.ViewID != nil {
View, _ = findNameByID[db.View](*o.ViewID, b.views)
}
var VaultKey *string = nil
if o.VaultKeyID != nil {
VaultKey, _ = findNameByID[db.AccessKey](*o.VaultKeyID, b.keys)
var vaults []BackupTemplateVault = nil
for _, vault := range o.Vaults {
var vaultKey *string = nil
vaultKey, _ = findNameByID[db.AccessKey](vault.VaultKeyID, b.keys)
vaults = append(vaults, BackupTemplateVault{
TemplateVault: vault,
VaultKey: *vaultKey,
})
}
var Environment *string = nil
if o.EnvironmentID != nil {
@ -238,31 +237,19 @@ func (b *BackupDB) format() (*BackupFormat, error) {
}
templates[i] = BackupTemplate{
Name: o.Name,
AllowOverrideArgsInTask: o.AllowOverrideArgsInTask,
Arguments: o.Arguments,
Autorun: o.Autorun,
Description: o.Description,
Playbook: o.Playbook,
StartVersion: o.StartVersion,
SuppressSuccessAlerts: o.SuppressSuccessAlerts,
SurveyVars: o.SurveyVarsJSON,
Type: o.Type,
View: View,
VaultKey: VaultKey,
Repository: *Repository,
Inventory: Inventory,
Environment: Environment,
BuildTemplate: BuildTemplate,
Cron: getScheduleByTemplate(o.ID, b.schedules),
Template: o,
View: View,
Repository: *Repository,
Inventory: Inventory,
Environment: Environment,
BuildTemplate: BuildTemplate,
Cron: getScheduleByTemplate(o.ID, b.schedules),
Vaults: vaults,
}
}
return &BackupFormat{
Meta: BackupMeta{
Name: b.meta.Name,
MaxParallelTasks: b.meta.MaxParallelTasks,
Alert: b.meta.Alert,
AlertChat: b.meta.AlertChat,
b.meta,
},
Inventories: inventories,
Environments: environments,
@ -281,3 +268,31 @@ func GetBackup(projectID int, store db.Store) (*BackupFormat, error) {
return backup.format()
}
func (b *BackupFormat) Marshal() (res string, err error) {
data, err := marshalValue(reflect.ValueOf(b))
if err != nil {
return
}
bytes, err := json.Marshal(data)
if err != nil {
return
}
res = string(bytes)
return
}
func (b *BackupFormat) Unmarshal(res string) (err error) {
// Parse the JSON data into a map
var jsonData interface{}
if err = json.Unmarshal([]byte(res), &jsonData); err != nil {
return
}
// Start the recursive unmarshaling process
err = unmarshalValueWithBackupTags(jsonData, reflect.ValueOf(b))
return
}

293
services/project/backup_marshal.go Normal file
View File

@ -0,0 +1,293 @@
package project
import (
"fmt"
"reflect"
)
func marshalValue(v reflect.Value) (interface{}, error) {
// Handle pointers
if v.Kind() == reflect.Ptr {
if v.IsNil() {
return nil, nil
}
return marshalValue(v.Elem())
}
// Handle structs
if v.Kind() == reflect.Struct {
typeOfV := v.Type()
result := make(map[string]interface{})
for i := 0; i < v.NumField(); i++ {
fieldValue := v.Field(i)
fieldType := typeOfV.Field(i)
// Handle anonymous fields (embedded structs)
if fieldType.Anonymous {
embeddedValue, err := marshalValue(fieldValue)
if err != nil {
return nil, err
}
if embeddedMap, ok := embeddedValue.(map[string]interface{}); ok {
// Merge embedded struct fields into parent result map
for k, v := range embeddedMap {
result[k] = v
}
}
continue
}
tag := fieldType.Tag.Get("backup")
if tag == "-" {
continue // Skip fields with backup:"-"
}
// Check if the field should be backed up
if tag == "" {
// Get the field name from the "db" tag
tag = fieldType.Tag.Get("db")
if tag == "" || tag == "-" {
continue // Skip if "db" tag is empty or "-"
}
}
// Recursively process the field value
value, err := marshalValue(fieldValue)
if err != nil {
return nil, err
}
if value == nil {
continue
}
result[tag] = value
}
return result, nil
}
// Handle slices and arrays
if v.Kind() == reflect.Slice || v.Kind() == reflect.Array {
if v.IsNil() {
return make([]interface{}, 0), nil
}
var result []interface{} = make([]interface{}, 0)
for i := 0; i < v.Len(); i++ {
elemValue, err := marshalValue(v.Index(i))
if err != nil {
return nil, err
}
result = append(result, elemValue)
}
return result, nil
}
// Handle maps
if v.Kind() == reflect.Map {
if v.IsNil() {
return make(map[string]interface{}), nil
}
result := make(map[string]interface{})
for _, key := range v.MapKeys() {
// Assuming the key is a string
mapKey := fmt.Sprintf("%v", key.Interface())
mapValue, err := marshalValue(v.MapIndex(key))
if err != nil {
return nil, err
}
result[mapKey] = mapValue
}
return result, nil
}
// Handle other types (int, string, etc.)
return v.Interface(), nil
}
func setBasicType(data interface{}, v reflect.Value) error {
if !v.CanSet() {
return fmt.Errorf("cannot set value of type %v", v.Type())
}
switch v.Kind() {
case reflect.Bool:
b, ok := data.(bool)
if !ok {
return fmt.Errorf("expected bool for field, got %T", data)
}
v.SetBool(b)
case reflect.String:
s, ok := data.(string)
if !ok {
return fmt.Errorf("expected string for field, got %T", data)
}
v.SetString(s)
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
n, ok := toFloat64(data)
if !ok {
return fmt.Errorf("expected number for field, got %T", data)
}
v.SetInt(int64(n))
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
n, ok := toFloat64(data)
if !ok {
return fmt.Errorf("expected number for field, got %T", data)
}
v.SetUint(uint64(n))
case reflect.Float32, reflect.Float64:
n, ok := toFloat64(data)
if !ok {
return fmt.Errorf("expected number for field, got %T", data)
}
v.SetFloat(n)
default:
return fmt.Errorf("unsupported kind %v", v.Kind())
}
return nil
}
func toFloat64(data interface{}) (float64, bool) {
switch n := data.(type) {
case float64:
return n, true
case float32:
return float64(n), true
case int:
return float64(n), true
case int64:
return float64(n), true
case int32:
return float64(n), true
case int16:
return float64(n), true
case int8:
return float64(n), true
case uint:
return float64(n), true
case uint64:
return float64(n), true
case uint32:
return float64(n), true
case uint16:
return float64(n), true
case uint8:
return float64(n), true
default:
return 0, false
}
}
func unmarshalValueWithBackupTags(data interface{}, v reflect.Value) error {
// Handle pointers
if v.Kind() == reflect.Ptr {
// Initialize pointer if it's nil
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
return unmarshalValueWithBackupTags(data, v.Elem())
}
// Handle structs
if v.Kind() == reflect.Struct {
// Data should be a map
m, ok := data.(map[string]interface{})
if !ok {
return fmt.Errorf("expected object for struct, got %T", data)
}
return unmarshalStructWithBackupTags(m, v)
}
// Handle slices and arrays
if v.Kind() == reflect.Slice {
dataSlice, ok := data.([]interface{})
if !ok {
return fmt.Errorf("expected array for slice, got %T", data)
}
slice := reflect.MakeSlice(v.Type(), len(dataSlice), len(dataSlice))
for i := 0; i < len(dataSlice); i++ {
elem := slice.Index(i)
if err := unmarshalValueWithBackupTags(dataSlice[i], elem); err != nil {
return err
}
}
v.Set(slice)
return nil
}
// Handle maps
if v.Kind() == reflect.Map {
dataMap, ok := data.(map[string]interface{})
if !ok {
return fmt.Errorf("expected object for map, got %T", data)
}
mapType := v.Type()
mapValue := reflect.MakeMap(mapType)
for key, value := range dataMap {
keyVal := reflect.ValueOf(key).Convert(mapType.Key())
valVal := reflect.New(mapType.Elem()).Elem()
if err := unmarshalValueWithBackupTags(value, valVal); err != nil {
return err
}
mapValue.SetMapIndex(keyVal, valVal)
}
v.Set(mapValue)
return nil
}
// Handle basic types
return setBasicType(data, v)
}
func unmarshalStructWithBackupTags(data map[string]interface{}, v reflect.Value) error {
t := v.Type()
for i := 0; i < v.NumField(); i++ {
fieldType := t.Field(i)
fieldValue := v.Field(i)
// Handle anonymous fields (embedded structs)
if fieldType.Anonymous {
// Pass the entire data map to the embedded struct
if err := unmarshalStructWithBackupTags(data, fieldValue); err != nil {
return err
}
continue
}
// Skip fields with backup:"-"
backupTag := fieldType.Tag.Get("backup")
if backupTag == "-" {
continue
}
// Determine the JSON key to use
var jsonKey string
if backupTag != "" {
jsonKey = backupTag
} else {
dbTag := fieldType.Tag.Get("db")
if dbTag != "" && dbTag != "-" {
jsonKey = dbTag
} else {
continue // Skip if no backup or db tag
}
}
// Check if the key exists in the data
if value, ok := data[jsonKey]; ok {
if !fieldValue.CanSet() {
continue // Skip fields that cannot be set
}
if value == nil {
continue
}
if err := unmarshalValueWithBackupTags(value, fieldValue); err != nil {
return err
}
}
}
return nil
}

117
services/project/backup_test.go
View File

@ -1,6 +1,9 @@
package project
import (
"github.com/ansible-semaphore/semaphore/db"
"github.com/ansible-semaphore/semaphore/db/bolt"
"github.com/ansible-semaphore/semaphore/util"
"testing"
)
@ -8,6 +11,120 @@ type testItem struct {
Name string
}
func TestBackupProject(t *testing.T) {
util.Config = &util.ConfigType{
TmpPath: "/tmp",
}
store := bolt.CreateTestStore()
proj, err := store.CreateProject(db.Project{
Name: "Test 123",
})
if err != nil {
t.Fatal(err)
}
key, err := store.CreateAccessKey(db.AccessKey{
ProjectID: &proj.ID,
Type: db.AccessKeyNone,
})
if err != nil {
t.Fatal(err)
}
repo, err := store.CreateRepository(db.Repository{
ProjectID: proj.ID,
SSHKeyID: key.ID,
Name: "Test",
GitURL: "git@example.com:test/test",
GitBranch: "master",
})
if err != nil {
t.Fatal(err)
}
inv, err := store.CreateInventory(db.Inventory{
ProjectID: proj.ID,
ID: 1,
})
if err != nil {
t.Fatal(err)
}
env, err := store.CreateEnvironment(db.Environment{
ProjectID: proj.ID,
Name: "test",
JSON: `{"author": "Denis", "comment": "Hello, World!"}`,
})
if err != nil {
t.Fatal(err)
}
_, err = store.CreateTemplate(db.Template{
Name: "Test",
Playbook: "test.yml",
ProjectID: proj.ID,
RepositoryID: repo.ID,
InventoryID: &inv.ID,
EnvironmentID: &env.ID,
})
if err != nil {
t.Fatal(err)
}
backup, err := GetBackup(proj.ID, store)
if err != nil {
t.Fatal(err)
}
if backup.Meta.ID != proj.ID {
t.Fatal("backup meta ID wrong")
}
str, err := backup.Marshal()
if err != nil {
t.Fatal(err)
}
if str != `{"environments":[{"json":"{\"author\": \"Denis\", \"comment\": \"Hello, World!\"}","name":"test"}],"inventories":[{"inventory":"","name":"","type":""}],"keys":[{"name":"","type":"none"}],"meta":{"alert":false,"max_parallel_tasks":0,"name":"Test 123","type":""},"repositories":[{"git_branch":"master","git_url":"git@example.com:test/test","name":"Test","ssh_key":""}],"templates":[{"allow_override_args_in_task":false,"app":"","autorun":false,"environment":"test","inventory":"","name":"Test","playbook":"test.yml","repository":"Test","suppress_success_alerts":false,"type":"","vaults":[]}],"views":[]}` {
t.Fatal("Invalid backup content")
}
restoredBackup := &BackupFormat{}
err = restoredBackup.Unmarshal(str)
if err != nil {
t.Fatal(err)
}
if restoredBackup.Meta.Name != proj.Name {
t.Fatal("backup meta ID wrong")
}
user, err := store.CreateUser(db.UserWithPwd{
Pwd: "3412341234123",
User: db.User{
Username: "test",
Name: "Test",
Email: "test@example.com",
Admin: true,
},
})
if err != nil {
t.Fatal(err)
}
restoredProj, err := restoredBackup.Restore(user, store)
if err != nil {
t.Fatal(err)
}
if restoredProj.Name != proj.Name {
t.Fatal("backup meta ID wrong")
}
}
func isUnique(items []testItem) bool {
for i, item := range items {
for k, other := range items {

231
services/project/restore.go
View File

@ -37,57 +37,46 @@ func (e BackupEnvironment) Verify(backup *BackupFormat) error {
}
func (e BackupEnvironment) Restore(store db.Store, b *BackupDB) error {
environment, err := store.CreateEnvironment(
db.Environment{
Name: e.Name,
Password: e.Password,
ProjectID: b.meta.ID,
JSON: e.JSON,
ENV: e.ENV,
},
)
env := e.Environment
env.ProjectID = b.meta.ID
newEnv, err := store.CreateEnvironment(env)
if err != nil {
return err
}
b.environments = append(b.environments, environment)
b.environments = append(b.environments, newEnv)
return nil
}
func (e BackupView) Verify(backup *BackupFormat) error {
return verifyDuplicate[BackupView](e.Name, backup.Views)
return verifyDuplicate[BackupView](e.Title, backup.Views)
}
func (e BackupView) Restore(store db.Store, b *BackupDB) error {
view, err := store.CreateView(
db.View{
Title: e.Name,
ProjectID: b.meta.ID,
Position: e.Position,
},
)
v := e.View
v.ProjectID = b.meta.ID
newView, err := store.CreateView(v)
if err != nil {
return err
}
b.views = append(b.views, view)
b.views = append(b.views, newView)
return nil
}
func (e BackupKey) Verify(backup *BackupFormat) error {
return verifyDuplicate[BackupKey](e.Name, backup.Keys)
func (e BackupAccessKey) Verify(backup *BackupFormat) error {
return verifyDuplicate[BackupAccessKey](e.Name, backup.Keys)
}
func (e BackupKey) Restore(store db.Store, b *BackupDB) error {
key, err := store.CreateAccessKey(
db.AccessKey{
Name: e.Name,
ProjectID: &b.meta.ID,
Type: e.Type,
},
)
func (e BackupAccessKey) Restore(store db.Store, b *BackupDB) error {
key := e.AccessKey
key.ProjectID = &b.meta.ID
newKey, err := store.CreateAccessKey(key)
if err != nil {
return err
}
b.keys = append(b.keys, key)
b.keys = append(b.keys, newKey)
return nil
}
@ -95,10 +84,10 @@ func (e BackupInventory) Verify(backup *BackupFormat) error {
if err := verifyDuplicate[BackupInventory](e.Name, backup.Inventories); err != nil {
return err
}
if e.SSHKey != nil && getEntryByName[BackupKey](e.SSHKey, backup.Keys) == nil {
if e.SSHKey != nil && getEntryByName[BackupAccessKey](e.SSHKey, backup.Keys) == nil {
return fmt.Errorf("SSHKey does not exist in keys[].Name")
}
if e.BecomeKey != nil && getEntryByName[BackupKey](e.BecomeKey, backup.Keys) == nil {
if e.BecomeKey != nil && getEntryByName[BackupAccessKey](e.BecomeKey, backup.Keys) == nil {
return fmt.Errorf("BecomeKey does not exist in keys[].Name")
}
return nil
@ -121,20 +110,17 @@ func (e BackupInventory) Restore(store db.Store, b *BackupDB) error {
} else {
BecomeKeyID = &((*k).ID)
}
inventory, err := store.CreateInventory(
db.Inventory{
ProjectID: b.meta.ID,
Name: e.Name,
Type: e.Type,
SSHKeyID: SSHKeyID,
BecomeKeyID: BecomeKeyID,
Inventory: e.Inventory,
},
)
inv := e.Inventory
inv.ProjectID = b.meta.ID
inv.SSHKeyID = SSHKeyID
inv.BecomeKeyID = BecomeKeyID
newInventory, err := store.CreateInventory(inv)
if err != nil {
return err
}
b.inventories = append(b.inventories, inventory)
b.inventories = append(b.inventories, newInventory)
return nil
}
@ -142,7 +128,7 @@ func (e BackupRepository) Verify(backup *BackupFormat) error {
if err := verifyDuplicate[BackupRepository](e.Name, backup.Repositories); err != nil {
return err
}
if e.SSHKey != nil && getEntryByName[BackupKey](e.SSHKey, backup.Keys) == nil {
if e.SSHKey != nil && getEntryByName[BackupAccessKey](e.SSHKey, backup.Keys) == nil {
return fmt.Errorf("SSHKey does not exist in keys[].Name")
}
return nil
@ -155,19 +141,16 @@ func (e BackupRepository) Restore(store db.Store, b *BackupDB) error {
} else {
SSHKeyID = (*k).ID
}
repository, err := store.CreateRepository(
db.Repository{
ProjectID: b.meta.ID,
Name: e.Name,
GitBranch: e.GitBranch,
GitURL: e.GitURL,
SSHKeyID: SSHKeyID,
},
)
repo := e.Repository
repo.ProjectID = b.meta.ID
repo.SSHKeyID = SSHKeyID
newRepo, err := store.CreateRepository(repo)
if err != nil {
return err
}
b.repositories = append(b.repositories, repository)
b.repositories = append(b.repositories, newRepo)
return nil
}
@ -175,24 +158,39 @@ func (e BackupTemplate) Verify(backup *BackupFormat) error {
if err := verifyDuplicate[BackupTemplate](e.Name, backup.Templates); err != nil {
return err
}
if getEntryByName[BackupRepository](&e.Repository, backup.Repositories) == nil {
return fmt.Errorf("repository does not exist in repositories[].name")
}
if getEntryByName[BackupInventory](e.Inventory, backup.Inventories) == nil {
if e.Inventory != nil && getEntryByName[BackupInventory](e.Inventory, backup.Inventories) == nil {
return fmt.Errorf("inventory does not exist in inventories[].name")
}
if e.VaultKey != nil && getEntryByName[BackupKey](e.VaultKey, backup.Keys) == nil {
if e.VaultKey != nil && getEntryByName[BackupAccessKey](e.VaultKey, backup.Keys) == nil {
return fmt.Errorf("vault_key does not exist in keys[].name")
}
if e.Vaults != nil {
for _, vault := range e.Vaults {
if getEntryByName[BackupAccessKey](&vault.VaultKey, backup.Keys) == nil {
return fmt.Errorf("vaults[].vaultKey does not exist in keys[].name")
}
}
}
if e.View != nil && getEntryByName[BackupView](e.View, backup.Views) == nil {
return fmt.Errorf("view does not exist in views[].name")
}
if string(e.Type) == "deploy" && e.BuildTemplate == nil {
return fmt.Errorf("type is deploy but build_template is null")
}
if string(e.Type) != "deploy" && e.BuildTemplate != nil {
return fmt.Errorf("type is not deploy but build_template is not null")
}
if buildTemplate := getEntryByName[BackupTemplate](e.BuildTemplate, backup.Templates); string(e.Type) == "deploy" && buildTemplate == nil {
return fmt.Errorf("deploy is build but build_template does not exist in templates[].name")
}
@ -207,24 +205,33 @@ func (e BackupTemplate) Verify(backup *BackupFormat) error {
}
func (e BackupTemplate) Restore(store db.Store, b *BackupDB) error {
var InventoryID int
if k := findEntityByName[db.Inventory](e.Inventory, b.inventories); k == nil {
return fmt.Errorf("inventory does not exist in inventories[].name")
} else {
InventoryID = k.GetID()
var InventoryID *int
if e.Inventory != nil {
if k := findEntityByName[db.Inventory](e.Inventory, b.inventories); k == nil {
return fmt.Errorf("inventory does not exist in inventories[].name")
} else {
id := k.GetID()
InventoryID = &id
}
}
var EnvironmentID int
if k := findEntityByName[db.Environment](e.Environment, b.environments); k == nil {
return fmt.Errorf("environment does not exist in environments[].name")
} else {
EnvironmentID = k.GetID()
var EnvironmentID *int
if e.Environment != nil {
if k := findEntityByName[db.Environment](e.Environment, b.environments); k == nil {
return fmt.Errorf("environment does not exist in environments[].name")
} else {
id := k.GetID()
EnvironmentID = &id
}
}
var RepositoryID int
if k := findEntityByName[db.Repository](&e.Repository, b.repositories); k == nil {
return fmt.Errorf("repository does not exist in repositories[].name")
} else {
RepositoryID = k.GetID()
}
var BuildTemplateID *int
if string(e.Type) != "deploy" {
BuildTemplateID = nil
@ -233,38 +240,32 @@ func (e BackupTemplate) Restore(store db.Store, b *BackupDB) error {
} else {
BuildTemplateID = &(k.ID)
}
var ViewID *int
if k := findEntityByName[db.View](e.View, b.views); k == nil {
ViewID = nil
} else {
ViewID = &k.ID
}
template, err := store.CreateTemplate(
db.Template{
ProjectID: b.meta.ID,
InventoryID: &InventoryID,
EnvironmentID: &EnvironmentID,
RepositoryID: RepositoryID,
ViewID: ViewID,
Autorun: e.Autorun,
AllowOverrideArgsInTask: e.AllowOverrideArgsInTask,
SuppressSuccessAlerts: e.SuppressSuccessAlerts,
Name: e.Name,
Playbook: e.Playbook,
Arguments: e.Arguments,
Type: e.Type,
BuildTemplateID: BuildTemplateID,
},
)
template := e.Template
template.ProjectID = b.meta.ID
template.RepositoryID = RepositoryID
template.EnvironmentID = EnvironmentID
template.InventoryID = InventoryID
template.ViewID = ViewID
template.BuildTemplateID = BuildTemplateID
newTemplate, err := store.CreateTemplate(template)
if err != nil {
return err
}
b.templates = append(b.templates, template)
b.templates = append(b.templates, newTemplate)
if e.Cron != nil {
_, err := store.CreateSchedule(
db.Schedule{
ProjectID: b.meta.ID,
TemplateID: template.ID,
TemplateID: newTemplate.ID,
CronFormat: *e.Cron,
RepositoryID: &RepositoryID,
},
@ -273,6 +274,26 @@ func (e BackupTemplate) Restore(store db.Store, b *BackupDB) error {
return err
}
}
if e.Vaults != nil {
for _, vault := range e.Vaults {
var VaultKeyID int
if k := findEntityByName[db.AccessKey](&vault.VaultKey, b.keys); k == nil {
return fmt.Errorf("vaults[].vaultKey does not exist in keys[].name")
} else {
VaultKeyID = k.ID
}
tplVault := vault.TemplateVault
tplVault.ProjectID = b.meta.ID
tplVault.TemplateID = newTemplate.ID
tplVault.VaultKeyID = VaultKeyID
_, err := store.CreateTemplateVault(tplVault)
if err != nil {
return err
}
}
}
return nil
}
@ -312,43 +333,54 @@ func (backup *BackupFormat) Verify() error {
func (backup *BackupFormat) Restore(user db.User, store db.Store) (*db.Project, error) {
var b = BackupDB{}
project, err := store.CreateProject(
db.Project{
Name: backup.Meta.Name,
Alert: backup.Meta.Alert,
MaxParallelTasks: backup.Meta.MaxParallelTasks,
AlertChat: backup.Meta.AlertChat,
},
)
project := backup.Meta.Project
newProject, err := store.CreateProject(project)
if err != nil {
return nil, err
}
b.meta = project
if _, err = store.CreateProjectUser(db.ProjectUser{
ProjectID: newProject.ID,
UserID: user.ID,
Role: db.ProjectOwner,
}); err != nil {
return nil, err
}
b.meta = newProject
for i, o := range backup.Environments {
if err := o.Restore(store, &b); err != nil {
return nil, fmt.Errorf("error at environments[%d]: %s", i, err.Error())
}
}
for i, o := range backup.Views {
if err := o.Restore(store, &b); err != nil {
return nil, fmt.Errorf("error at views[%d]: %s", i, err.Error())
}
}
for i, o := range backup.Keys {
if err := o.Restore(store, &b); err != nil {
return nil, fmt.Errorf("error at keys[%d]: %s", i, err.Error())
}
}
for i, o := range backup.Repositories {
if err := o.Restore(store, &b); err != nil {
return nil, fmt.Errorf("error at repositories[%d]: %s", i, err.Error())
}
}
for i, o := range backup.Inventories {
if err := o.Restore(store, &b); err != nil {
return nil, fmt.Errorf("error at inventories[%d]: %s", i, err.Error())
}
}
deployTemplates := make([]int, 0)
for i, o := range backup.Templates {
if string(o.Type) == "deploy" {
@ -359,6 +391,7 @@ func (backup *BackupFormat) Restore(user db.User, store db.Store) (*db.Project,
return nil, fmt.Errorf("error at templates[%d]: %s", i, err.Error())
}
}
for _, i := range deployTemplates {
o := backup.Templates[i]
if err := o.Restore(store, &b); err != nil {
@ -366,13 +399,5 @@ func (backup *BackupFormat) Restore(user db.User, store db.Store) (*db.Project,
}
}
if _, err = store.CreateProjectUser(db.ProjectUser{
ProjectID: project.ID,
UserID: user.ID,
Role: db.ProjectOwner,
}); err != nil {
return nil, err
}
return &project, nil
return &newProject, nil
}

84
services/project/types.go
View File

@ -16,72 +16,60 @@ type BackupDB struct {
}
type BackupFormat struct {
Meta BackupMeta `json:"meta"`
Templates []BackupTemplate `json:"templates"`
Repositories []BackupRepository `json:"repositories"`
Keys []BackupKey `json:"keys"`
Views []BackupView `json:"views"`
Inventories []BackupInventory `json:"inventories"`
Environments []BackupEnvironment `json:"environments"`
Meta BackupMeta `backup:"meta"`
Templates []BackupTemplate `backup:"templates"`
Repositories []BackupRepository `backup:"repositories"`
Keys []BackupAccessKey `backup:"keys"`
Views []BackupView `backup:"views"`
Inventories []BackupInventory `backup:"inventories"`
Environments []BackupEnvironment `backup:"environments"`
}
type BackupMeta struct {
Name string `json:"name"`
Alert bool `json:"alert"`
AlertChat *string `json:"alert_chat"`
MaxParallelTasks int `json:"max_parallel_tasks"`
db.Project
}
type BackupEnvironment struct {
Name string `json:"name"`
Password *string `json:"password"`
JSON string `json:"json"`
ENV *string `json:"env"`
db.Environment
}
type BackupKey struct {
Name string `json:"name"`
Type db.AccessKeyType `json:"type"`
type BackupAccessKey struct {
db.AccessKey
}
type BackupView struct {
Name string `json:"name"`
Position int `json:"position"`
db.View
}
type BackupInventory struct {
Name string `json:"name"`
Inventory string `json:"inventory"`
SSHKey *string `json:"ssh_key"`
BecomeKey *string `json:"become_key"`
Type db.InventoryType `json:"type"`
db.Inventory
SSHKey *string `backup:"ssh_key"`
BecomeKey *string `backup:"become_key"`
}
type BackupRepository struct {
Name string `json:"name"`
GitURL string `json:"git_url"`
GitBranch string `json:"git_branch"`
SSHKey *string `json:"ssh_key"`
db.Repository
SSHKey *string `backup:"ssh_key"`
}
type BackupTemplate struct {
Inventory *string `json:"inventory"`
Repository string `json:"repository"`
Environment *string `json:"environment"`
Name string `json:"name"`
Playbook string `json:"playbook"`
Arguments *string `json:"arguments"`
AllowOverrideArgsInTask bool `json:"allow_override_args_in_task"`
Description *string `json:"description"`
VaultKey *string `json:"vault_key"`
Type db.TemplateType `json:"type"`
StartVersion *string `json:"start_version"`
BuildTemplate *string `json:"build_template"`
View *string `json:"view"`
Autorun bool `json:"autorun"`
SurveyVars *string `json:"survey_vars"`
SuppressSuccessAlerts bool `json:"suppress_success_alerts"`
Cron *string `json:"cron"`
db.Template
Inventory *string `backup:"inventory"`
Repository string `backup:"repository"`
Environment *string `backup:"environment"`
BuildTemplate *string `backup:"build_template"`
View *string `backup:"view"`
Vaults []BackupTemplateVault `backup:"vaults"`
Cron *string `backup:"cron"`
// Deprecated: Left here for compatibility with old backups
VaultKey *string `json:"vault_key"`
}
type BackupTemplateVault struct {
db.TemplateVault
VaultKey string `backup:"vault_key"`
}
type BackupEntry interface {
@ -98,7 +86,7 @@ func (e BackupInventory) GetName() string {
return e.Name
}
func (e BackupKey) GetName() string {
func (e BackupAccessKey) GetName() string {
return e.Name
}
@ -107,7 +95,7 @@ func (e BackupRepository) GetName() string {
}
func (e BackupView) GetName() string {
return e.Name
return e.Title
}
func (e BackupTemplate) GetName() string {

12
services/runners/job_pool.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/ansible-semaphore/semaphore/db"
"io"
"net/http"
"os"
@ -425,9 +426,16 @@ func (p *JobPool) checkNewJobs() {
taskRunner.job.Inventory.BecomeKey = response.AccessKeys[*taskRunner.job.Inventory.BecomeKeyID]
}
if taskRunner.job.Template.VaultKeyID != nil {
taskRunner.job.Template.VaultKey = response.AccessKeys[*taskRunner.job.Template.VaultKeyID]
var vaults []db.TemplateVault
if taskRunner.job.Template.Vaults != nil {
for _, vault := range taskRunner.job.Template.Vaults {
vault := vault
key := response.AccessKeys[vault.VaultKeyID]
vault.Vault = &key
vaults = append(vaults, vault)
}
}
taskRunner.job.Template.Vaults = vaults
if taskRunner.job.Inventory.RepositoryID != nil {
taskRunner.job.Inventory.Repository.SSHKey = response.AccessKeys[taskRunner.job.Inventory.Repository.SSHKeyID]

44
services/tasks/LocalJob.go
View File

@ -30,9 +30,9 @@ type LocalJob struct {
// Internal field
Process *os.Process
sshKeyInstallation db.AccessKeyInstallation
becomeKeyInstallation db.AccessKeyInstallation
vaultFileInstallation db.AccessKeyInstallation
sshKeyInstallation db.AccessKeyInstallation
becomeKeyInstallation db.AccessKeyInstallation
vaultFileInstallations map[string]db.AccessKeyInstallation
}
func (t *LocalJob) Kill() {
@ -335,9 +335,11 @@ func (t *LocalJob) getPlaybookArgs(username string, incomingVersion *string) (ar
args = append(args, "--check")
}
if t.Template.VaultKeyID != nil {
args = append(args, "--ask-vault-pass")
inputMap[db.AccessKeyRoleAnsiblePasswordVault] = t.vaultFileInstallation.Password
for name, install := range t.vaultFileInstallations {
if install.Password != "" {
args = append(args, fmt.Sprintf("--vault-id=%s@prompt", name))
inputs[fmt.Sprintf("Vault password (%s):", name)] = install.Password
}
}
extraVars, err := t.getEnvironmentExtraVarsJSON(username, incomingVersion)
@ -390,10 +392,6 @@ func (t *LocalJob) getPlaybookArgs(username string, incomingVersion *string) (ar
inputs["BECOME password"] = line
}
if line, ok := inputMap[db.AccessKeyRoleAnsiblePasswordVault]; ok {
inputs["Vault password:"] = line
}
return
}
@ -493,8 +491,8 @@ func (t *LocalJob) prepareRun() error {
return err
}
if err := t.installVaultKeyFile(); err != nil {
t.Log("Failed to install vault password file: " + err.Error())
if err := t.installVaultKeyFiles(); err != nil {
t.Log("Failed to install vault password files: " + err.Error())
return err
}
@ -573,12 +571,28 @@ func (t *LocalJob) checkoutRepository() error {
return nil
}
func (t *LocalJob) installVaultKeyFile() (err error) {
if t.Template.VaultKeyID == nil {
func (t *LocalJob) installVaultKeyFiles() (err error) {
t.vaultFileInstallations = make(map[string]db.AccessKeyInstallation)
if t.Template.Vaults == nil || len(t.Template.Vaults) == 0 {
return nil
}
t.vaultFileInstallation, err = t.Template.VaultKey.Install(db.AccessKeyRoleAnsiblePasswordVault, t.Logger)
for _, vault := range t.Template.Vaults {
var name string
if vault.Name != nil {
name = *vault.Name
} else {
name = "default"
}
var install db.AccessKeyInstallation
install, err = vault.Vault.Install(db.AccessKeyRoleAnsiblePasswordVault, t.Logger)
if err != nil {
return
}
t.vaultFileInstallations[name] = install
}
return
}

8
services/tasks/LocalJob_inventory.go
View File

@ -105,8 +105,10 @@ func (t *LocalJob) destroyKeys() {
t.Log("Can't destroy inventory become user key, error: " + err.Error())
}
err = t.vaultFileInstallation.Destroy()
if err != nil {
t.Log("Can't destroy inventory vault password file, error: " + err.Error())
for _, vault := range t.vaultFileInstallations {
err = vault.Destroy()
if err != nil {
t.Log("Can't destroy inventory vault password file, error: " + err.Error())
}
}
}

4
services/tasks/TaskRunner.go
View File

@ -167,7 +167,7 @@ func (t *TaskRunner) run() {
t.SetStatus(task_logger.TaskSuccessStatus)
}
templates, err := t.pool.store.GetTemplates(t.Task.ProjectID, db.TemplateFilter{
tpls, err := t.pool.store.GetTemplates(t.Task.ProjectID, db.TemplateFilter{
BuildTemplateID: &t.Task.TemplateID,
AutorunOnly: true,
}, db.RetrieveQueryParams{})
@ -176,7 +176,7 @@ func (t *TaskRunner) run() {
return
}
for _, tpl := range templates {
for _, tpl := range tpls {
_, err = t.pool.AddTask(db.Task{
TemplateID: tpl.ID,
ProjectID: tpl.ProjectID,

71032
web/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

1
web/package.json
View File

@ -14,6 +14,7 @@
"axios": "^0.28.0",
"core-js": "^3.23.2",
"cron-parser": "^4.9.0",
"dredd": "^13.1.2",
"moment": "^2.29.4",
"vue": "^2.6.14",
"vue-codemirror": "^4.0.6",

56
web/src/components/TemplateForm.vue
View File

@ -199,34 +199,23 @@
v-if="needField('environment')"
></v-select>
<v-select
<TemplateVaults
v-if="itemTypeIndex === 0 && needField('vault')"
v-model="item.vault_key_id"
:label="fieldLabel('vault')"
clearable
:items="loginPasswordKeys"
item-value="id"
item-text="name"
:disabled="formSaving"
outlined
dense
></v-select>
:project-id="this.projectId"
:vaults="item.vaults"
@change="setTemplateVaults"
></TemplateVaults>
</v-col>
<v-col cols="12" md="6" class="pb-0">
<v-select
<TemplateVaults
v-if="itemTypeIndex > 0 && needField('vault')"
v-model="item.vault_key_id"
:label="fieldLabel('vault')"
clearable
:items="loginPasswordKeys"
item-value="id"
item-text="name"
:disabled="formSaving"
outlined
dense
></v-select>
:project-id="this.projectId"
:vaults="item.vaults"
@change="setTemplateVaults"
></TemplateVaults>
<SurveyVars style="margin-top: -10px;" :vars="item.survey_vars" @change="setSurveyVars"/>
@ -313,6 +302,7 @@ import 'codemirror/mode/vue/vue.js';
import 'codemirror/addon/lint/json-lint.js';
import 'codemirror/addon/display/placeholder.js';
import ArgsPicker from '@/components/ArgsPicker.vue';
import TemplateVaults from '@/components/TemplateVaults.vue';
import { TEMPLATE_TYPE_ICONS, TEMPLATE_TYPE_TITLES } from '../lib/constants';
import SurveyVars from './SurveyVars';
@ -320,6 +310,7 @@ export default {
mixins: [ItemFormBase],
components: {
TemplateVaults,
ArgsPicker,
SurveyVars,
},
@ -360,7 +351,6 @@ export default {
indentWithTabs: false,
},
item: null,
keys: null,
inventory: null,
repositories: null,
environment: null,
@ -403,21 +393,13 @@ export default {
return true;
}
return this.keys != null
&& this.repositories != null
return this.repositories != null
&& this.inventory != null
&& this.environment != null
&& this.item != null
&& this.schedules != null
&& this.views != null;
},
loginPasswordKeys() {
if (this.keys == null) {
return null;
}
return this.keys.filter((key) => key.type === 'login_password');
},
},
methods: {
@ -441,6 +423,10 @@ export default {
this.item.survey_vars = v;
},
setTemplateVaults(v) {
this.item.vaults = v;
},
showHelpDialog(key) {
this.helpKey = key;
this.helpDialog = true;
@ -458,12 +444,6 @@ export default {
this.advancedOptions = this.item.arguments != null || this.item.allow_override_args_in_task;
this.keys = (await axios({
keys: 'get',
url: `/api/project/${this.projectId}/keys`,
responseType: 'json',
})).data;
this.repositories = (await axios({
keys: 'get',
url: `/api/project/${this.projectId}/repositories`,

190
web/src/components/TemplateVaults.vue Normal file
View File

@ -0,0 +1,190 @@
<template>
<div class="pb-6">
<v-dialog
v-model="editDialog"
hide-overlay
width="300"
>
<v-card :color="$vuetify.theme.dark ? '#212121' : 'white'">
<v-card-title></v-card-title>
<v-card-text class="pb-0">
<v-form
ref="form"
lazy-validation
v-if="editedVault != null"
>
<v-alert
:value="formError"
color="error"
>{{ formError }}
</v-alert>
<v-text-field
:label="$t('vaultName')"
placeholder="default"
v-model.trim="editedVault.name"
:rules="[v => this.vaultNameRules(v)]"
/>
<v-select
v-model="editedVault.vault_key_id"
:label="$t('vaultPassword2')"
:items="vaultKeys"
item-value="id"
item-text="name"
required
:rules="[(v) => !!v || $t('vaultRequired')]"
></v-select>
</v-form>
</v-card-text>
<v-card-actions>
<v-spacer></v-spacer>
<v-btn
color="blue darken-1"
text
@click="editDialog = false"
>
{{ $t('cancel') }}
</v-btn>
<v-btn
color="blue darken-1"
text
@click="saveVault()"
>
{{ editedVaultIndex == null ? $t('add') : $t('save') }}
</v-btn>
</v-card-actions>
</v-card>
</v-dialog>
<fieldset style="padding: 0 10px 2px 10px;
border: 1px solid rgba(0, 0, 0, 0.38);
border-radius: 4px;
font-size: 12px;"
:style="{
'border-color': $vuetify.theme.dark ?
'rgba(200, 200, 200, 0.38)' :
'rgba(0, 0, 0, 0.38)'
}">
<legend style="padding: 0 3px;">{{ $t('vaults') }}</legend>
<v-chip-group column style="margin-top: -4px;">
<v-chip
v-for="(v, i) in modifiedVaults"
close
@click:close="deleteVault(i)"
:key="v.name"
@click="editVault(i)"
color="gray"
>
{{ v.name || 'default' }}
</v-chip>
<v-chip @click="editVault(null)">
+ <span class="ml-1" v-if="modifiedVaults.length === 0">{{ $t('vaultAdd') }}</span>
</v-chip>
</v-chip-group>
</fieldset>
</div>
</template>
<style lang="scss">
</style>
<script>
import axios from 'axios';
export default {
props: {
projectId: Number,
vaults: Array,
},
watch: {
vaults(val) {
this.var = val || [];
},
},
async created() {
this.modifiedVaults = (this.vaults || []).map((v) => ({ ...v }));
this.keys = (await axios({
keys: 'get',
url: `/api/project/${this.projectId}/keys`,
responseType: 'json',
})).data;
},
data() {
return {
editDialog: null,
editedVault: null,
editedVaultIndex: null,
modifiedVaults: null,
formError: null,
keys: null,
};
},
computed: {
vaultKeys() {
if (this.keys == null) {
return null;
}
return this.keys.filter((key) => ['login_password'].includes(key.type));
},
},
methods: {
vaultNameRules(v) {
if (v == null || v === '') {
if (this.modifiedVaults.some((vault) => vault.name === null)) {
return this.$t('vaultNameDefault');
}
} else if (this.modifiedVaults.some((vault) => vault.name === v.toLowerCase().trim())) {
return this.$t('vaultNameUnique');
}
return true;
},
editVault(index) {
this.editedVault = index != null ? { ...this.modifiedVaults[index] } : {
name: null,
vault_key_id: null,
};
this.editedVaultIndex = index;
if (this.$refs.form) {
this.$refs.form.resetValidation();
}
this.editDialog = true;
},
saveVault() {
this.formError = null;
if (!this.$refs.form.validate()) {
return;
}
if (this.editedVault.name == null || this.editedVault.name === '') {
this.editedVault.name = null;
} else {
this.editedVault.name = this.editedVault.name.toLowerCase().trim();
}
if (this.editedVaultIndex != null) {
this.modifiedVaults[this.editedVaultIndex] = this.editedVault;
} else {
this.modifiedVaults.push(this.editedVault);
}
this.editDialog = false;
this.editedVault = null;
this.$emit('change', this.modifiedVaults);
},
deleteVault(index) {
this.modifiedVaults.splice(index, 1);
this.$emit('change', this.modifiedVaults);
},
},
};
</script>

8
web/src/lang/en.js
View File

@ -31,7 +31,7 @@ export default {
editUser: 'Edit User',
newProject: 'New Project',
close: 'Close',
newProject2: 'New project...',
newProject2: 'New Project...',
demoMode: 'DEMO MODE',
task: 'Task #{expr}',
youCanRunAnyTasks: 'You can run any tasks',
@ -123,6 +123,12 @@ export default {
key: '{expr}',
surveyVariables: 'Survey Variables',
addVariable: 'Add variable',
vaultName: 'Vault ID (name)',
vaultNameDefault: 'Only one `default` (empty) name may exist',
vaultNameUnique: 'Must be unique',
vaults: 'Vaults',
vaultAdd: 'Add Vault',
vaultRequired: 'Vault Password is required',
columns: 'Columns',
buildVersion: 'Build Version',
messageOptional: 'Message (Optional)',