diff --git a/.dredd/hooks/capabilities.go b/.dredd/hooks/capabilities.go index e41ee884..bb83cb71 100644 --- a/.dredd/hooks/capabilities.go +++ b/.dredd/hooks/capabilities.go @@ -100,7 +100,7 @@ func resolveCapability(caps []string, resolved []string, uid string) { case "template": res, err := store.Sql().Exec( "insert into project__template "+ - "(project_id, inventory_id, repository_id, environment_id, alias, playbook, arguments, override_args, description, view_id) "+ + "(project_id, inventory_id, repository_id, environment_id, alias, playbook, arguments, allow_override_args_in_task, description, view_id) "+ "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", userProject.ID, inventoryID, repoID, environmentID, "Test-"+uid, "test-playbook.yml", "", false, "Hello, World!", view.ID) printError(err) diff --git a/api-docs.yml b/api-docs.yml index 79526803..2bcd0991 100644 --- a/api-docs.yml +++ b/api-docs.yml @@ -338,8 +338,9 @@ definitions: description: type: string example: Hello, World! - override_args: + allow_override_args_in_task: type: boolean + example: false Template: type: object properties: @@ -372,8 +373,9 @@ definitions: description: type: string example: Hello, World! - override_args: + allow_override_args_in_task: type: boolean + example: false ScheduleRequest: type: object diff --git a/api/projects/keys.go b/api/projects/keys.go index e6f46760..d70082e1 100644 --- a/api/projects/keys.go +++ b/api/projects/keys.go @@ -111,7 +111,25 @@ func UpdateKey(w http.ResponseWriter, r *http.Request) { return } - if err := helpers.Store(r).UpdateAccessKey(key); err != nil { + repos, err := helpers.Store(r).GetRepositories(*key.ProjectID, db.RetrieveQueryParams{}) + if err != nil { + helpers.WriteError(w, err) + return + } + + for _, repo := range repos { + if repo.SSHKeyID != key.ID { + continue + } + err = repo.ClearCache() + if err != nil { + helpers.WriteError(w, err) + return + } + } + + err = helpers.Store(r).UpdateAccessKey(key) + if err != nil { helpers.WriteError(w, err) return } @@ -121,7 +139,7 @@ func UpdateKey(w http.ResponseWriter, r *http.Request) { desc := "Access Key " + key.Name + " updated" objType := db.EventKey - _, err := helpers.Store(r).CreateEvent(db.Event{ + _, err = helpers.Store(r).CreateEvent(db.Event{ UserID: &user.ID, ProjectID: oldKey.ProjectID, Description: &desc, diff --git a/api/projects/repository.go b/api/projects/repository.go index 2028a75a..f01800eb 100644 --- a/api/projects/repository.go +++ b/api/projects/repository.go @@ -4,40 +4,11 @@ import ( log "github.com/Sirupsen/logrus" "github.com/ansible-semaphore/semaphore/api/helpers" "github.com/ansible-semaphore/semaphore/db" - "net/http" - "os" - "path/filepath" - "strconv" - "github.com/ansible-semaphore/semaphore/util" "github.com/gorilla/context" + "net/http" ) -func removeAllByPattern(path string, filenamePattern string) error { - d, err := os.Open(path) - if err != nil { - return err - } - defer d.Close() - names, err := d.Readdirnames(-1) - if err != nil { - return err - } - for _, name := range names { - if matched, _ := filepath.Match(filenamePattern, name); !matched { - continue - } - if err := os.RemoveAll(filepath.Join(path, name)); err != nil { - return err - } - } - return nil -} - -func clearRepositoryCache(repository db.Repository) error { - return removeAllByPattern(util.Config.TmpPath, "repository_" + strconv.Itoa(repository.ID) + "_*") -} - // RepositoryMiddleware ensures a repository exists and loads it to the context func RepositoryMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -152,7 +123,7 @@ func UpdateRepository(w http.ResponseWriter, r *http.Request) { } if oldRepo.GitURL != repository.GitURL { - util.LogWarning(clearRepositoryCache(oldRepo)) + util.LogWarning(oldRepo.ClearCache()) } user := context.Get(r, "user").(*db.User) @@ -161,7 +132,7 @@ func UpdateRepository(w http.ResponseWriter, r *http.Request) { objType := db.EventRepository _, err = helpers.Store(r).CreateEvent(db.Event{ - UserID: &user.ID, + UserID: &user.ID, ProjectID: &repository.ProjectID, Description: &desc, ObjectID: &repository.ID, @@ -201,7 +172,7 @@ func RemoveRepository(w http.ResponseWriter, r *http.Request) { return } - util.LogWarning(clearRepositoryCache(repository)) + util.LogWarning(repository.ClearCache()) user := context.Get(r, "user").(*db.User) desc := "Repository (" + repository.GitURL + ") deleted" diff --git a/api/tasks/runner.go b/api/tasks/runner.go index 27998034..01ab1502 100644 --- a/api/tasks/runner.go +++ b/api/tasks/runner.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "os" "os/exec" - "path" "regexp" "strconv" "strings" @@ -47,11 +46,11 @@ type task struct { } func (t *task) getRepoName() string { - return "repository_" + strconv.Itoa(t.repository.ID) + "_" + strconv.Itoa(t.template.ID) + return t.repository.GetDirName(t.template.ID) } func (t *task) getRepoPath() string { - return path.Join(util.Config.TmpPath, t.getRepoName()) + return t.repository.GetPath(t.template.ID) } func (t *task) validateRepo() error { @@ -534,7 +533,7 @@ func (t *task) canRepositoryBePulled() bool { func (t *task) cloneRepository() error { cmd := t.makeGitCommand(util.Config.TmpPath) t.log("Cloning repository " + t.repository.GitURL) - cmd.Args = append(cmd.Args, "clone", "--recursive", "--branch", t.repository.GitBranch, t.repository.GitURL, t.getRepoName()) + cmd.Args = append(cmd.Args, "clone", "--recursive", "--branch", t.repository.GitBranch, t.repository.GetGitURL(), t.getRepoName()) t.logCmd(cmd) return cmd.Run() } @@ -782,18 +781,25 @@ func (t *task) getPlaybookArgs() (args []string, err error) { if t.template.Arguments != nil { err = json.Unmarshal([]byte(*t.template.Arguments), &templateExtraArgs) if err != nil { - t.log("Could not unmarshal arguments to []string") + t.log("Invalid format of the template extra arguments, must be valid JSON") return } } - if t.template.OverrideArguments { - args = templateExtraArgs - } else { - args = append(args, templateExtraArgs...) - args = append(args, playbookName) + var taskExtraArgs []string + + if t.template.AllowOverrideArgsInTask && t.task.Arguments != nil { + err = json.Unmarshal([]byte(*t.task.Arguments), &taskExtraArgs) + if err != nil { + t.log("Invalid format of the task extra arguments, must be valid JSON") + return + } } + args = append(args, templateExtraArgs...) + args = append(args, taskExtraArgs...) + args = append(args, playbookName) + return } @@ -802,6 +808,7 @@ func (t *task) setCmdEnvironment(cmd *exec.Cmd, gitSSHCommand string) { env = append(env, fmt.Sprintf("HOME=%s", util.Config.TmpPath)) env = append(env, fmt.Sprintf("PWD=%s", cmd.Dir)) env = append(env, fmt.Sprintln("PYTHONUNBUFFERED=1")) + env = append(env, fmt.Sprintln("GIT_TERMINAL_PROMPT=0")) env = append(env, extractCommandEnvironment(t.environment.JSON)...) if gitSSHCommand != "" { diff --git a/api/tasks/runner_test.go b/api/tasks/runner_test.go index e07e1999..dd934891 100644 --- a/api/tasks/runner_test.go +++ b/api/tasks/runner_test.go @@ -6,6 +6,7 @@ import ( "github.com/ansible-semaphore/semaphore/util" "math/rand" "os" + "path" "strconv" "strings" "testing" @@ -30,6 +31,7 @@ func TestPopulateDetails(t *testing.T) { key, err := store.CreateAccessKey(db.AccessKey{ ProjectID: &proj.ID, + Type: db.AccessKeyNone, }) if err != nil { t.Fatal(err) @@ -202,7 +204,7 @@ func TestTaskGetPlaybookArgs3(t *testing.T) { func TestCheckTmpDir(t *testing.T) { //It should be able to create a random dir in /tmp - dirName := os.TempDir() + "/" + randString(rand.Intn(10-4)+4) + dirName := path.Join(os.TempDir(), util.RandString(rand.Intn(10-4)+4)) err := checkTmpDir(dirName) if err != nil { t.Fatal(err) @@ -236,32 +238,3 @@ func TestCheckTmpDir(t *testing.T) { t.Log(err) } } - -//HELPERS - -//https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang -var src = rand.NewSource(time.Now().UnixNano()) - -const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -const ( - letterIdxBits = 6 // 6 bits to represent a letter index - letterIdxMask = 1<= 0; { - if remain == 0 { - cache, remain = src.Int63(), letterIdxMax - } - if idx := int(cache & letterIdxMask); idx < len(letterBytes) { - b[i] = letterBytes[idx] - i-- - } - cache >>= letterIdxBits - remain-- - } - return string(b) -} diff --git a/db/AccessKey.go b/db/AccessKey.go index 41416b45..1dff8d93 100644 --- a/db/AccessKey.go +++ b/db/AccessKey.go @@ -16,10 +16,13 @@ import ( "github.com/ansible-semaphore/semaphore/util" ) +type AccessKeyType string + const ( - AccessKeySSH = "ssh" - AccessKeyNone = "none" - AccessKeyLoginPassword = "login_password" + AccessKeySSH AccessKeyType = "ssh" + AccessKeyNone AccessKeyType = "none" + AccessKeyLoginPassword AccessKeyType = "login_password" + AccessKeyPAT AccessKeyType = "pat" ) // AccessKey represents a key used to access a machine with ansible from semaphore @@ -27,7 +30,7 @@ type AccessKey struct { ID int `db:"id" json:"id"` Name string `db:"name" json:"name" binding:"required"` // 'ssh/login_password/none' - Type string `db:"type" json:"type" binding:"required"` + Type AccessKeyType `db:"type" json:"type" binding:"required"` ProjectID *int `db:"project_id" json:"project_id"` @@ -39,6 +42,7 @@ type AccessKey struct { LoginPassword LoginPassword `db:"-" json:"login_password"` SshKey SshKey `db:"-" json:"ssh"` + PAT string `db:"-" json:"pat"` OverrideSecret bool `db:"-" json:"override_secret"` InstallationKey int64 `db:"-" json:"-"` @@ -199,9 +203,13 @@ func (key *AccessKey) SerializeSecret() error { if err != nil { return err } - default: + case AccessKeyPAT: + plaintext = []byte(key.PAT) + case AccessKeyNone: key.Secret = nil return nil + default: + return fmt.Errorf("invalid access token type") } encryptionString := util.Config.GetAccessKeyEncryption() @@ -253,6 +261,8 @@ func (key *AccessKey) unmarshalAppropriateField(secret []byte) (err error) { if err == nil { key.LoginPassword = loginPass } + case AccessKeyPAT: + key.PAT = string(secret) } return } @@ -261,6 +271,7 @@ func (key *AccessKey) ResetSecret() { //key.Secret = nil key.LoginPassword = LoginPassword{} key.SshKey = SshKey{} + key.PAT = "" } func (key *AccessKey) DeserializeSecret() error { diff --git a/db/Migration.go b/db/Migration.go index dae60421..203de261 100644 --- a/db/Migration.go +++ b/db/Migration.go @@ -51,6 +51,7 @@ func GetMigrations() []Migration { {Version: "2.8.20"}, {Version: "2.8.25"}, {Version: "2.8.26"}, + {Version: "2.8.36"}, } } diff --git a/db/Repository.go b/db/Repository.go index 805fa4f3..f54de198 100644 --- a/db/Repository.go +++ b/db/Repository.go @@ -1,5 +1,23 @@ package db +import ( + "github.com/ansible-semaphore/semaphore/util" + "os" + "path" + "regexp" + "strconv" + "strings" +) + +type RepositorySchema string + +const ( + RepositoryGit RepositorySchema = "git" + RepositorySSH RepositorySchema = "ssh" + RepositoryHTTPS RepositorySchema = "https" + RepositoryFile RepositorySchema = "file" +) + // Repository is the model for code stored in a git repository type Repository struct { ID int `db:"id" json:"id"` @@ -13,6 +31,73 @@ type Repository struct { SSHKey AccessKey `db:"-" json:"-"` } +func (r Repository) ClearCache() error { + dir, err := os.Open(util.Config.TmpPath) + if err != nil { + return err + } + + files, err := dir.ReadDir(0) + if err != nil { + return err + } + + for _, f := range files { + if !f.IsDir() { + continue + } + if strings.HasPrefix(f.Name(), r.getDirNamePrefix()) { + err = os.RemoveAll(path.Join(util.Config.TmpPath, f.Name())) + if err != nil { + return err + } + } + } + + return nil +} + +func (r Repository) getDirNamePrefix() string { + return "repository_" + strconv.Itoa(r.ID) + "_" +} + +func (r Repository) GetDirName(templateID int) string { + return r.getDirNamePrefix() + strconv.Itoa(templateID) +} + +func (r Repository) GetPath(templateID int) string { + return path.Join(util.Config.TmpPath, r.GetDirName(templateID)) +} + +func (r Repository) GetGitURL() string { + url := r.GitURL + + if r.getSchema() == RepositoryHTTPS { + auth := "" + switch r.SSHKey.Type { + case AccessKeyLoginPassword: + auth = r.SSHKey.LoginPassword.Login + ":" + r.SSHKey.LoginPassword.Password + case AccessKeyPAT: + auth = r.SSHKey.PAT + } + if auth != "" { + auth += "@" + } + url = "https://" + auth + r.GitURL[8:] + } + + return url +} + +func (r Repository) getSchema() RepositorySchema { + re := regexp.MustCompile(`^(\w+)://`) + m := re.FindStringSubmatch(r.GitURL) + if m == nil { + return RepositoryFile + } + return RepositorySchema(m[1]) +} + func (r Repository) Validate() error { if r.Name == "" { return &ValidationError{"repository name can't be empty"} diff --git a/db/Repository_test.go b/db/Repository_test.go new file mode 100644 index 00000000..ef70267e --- /dev/null +++ b/db/Repository_test.go @@ -0,0 +1,40 @@ +package db + +import ( + "github.com/ansible-semaphore/semaphore/util" + "math/rand" + "os" + "path" + "testing" +) + +func TestRepository_GetSchema(t *testing.T) { + repo := Repository{GitURL: "https://example.com/hello/world"} + schema := repo.getSchema() + if schema != "https" { + t.Fatal() + } +} + +func TestRepository_ClearCache(t *testing.T) { + util.Config = &util.ConfigType{ + TmpPath: path.Join(os.TempDir(), util.RandString(rand.Intn(10-4)+4)), + } + repoDir := path.Join(util.Config.TmpPath, "repository_123_55") + err := os.MkdirAll(repoDir, 0755) + if err != nil { + t.Fatal(err) + } + repo := Repository{ID: 123} + err = repo.ClearCache() + if err != nil { + t.Fatal(err) + } + _, err = os.Stat(repoDir) + if err == nil { + t.Fatal("repo directory not deleted") + } + if !os.IsNotExist(err) { + t.Fatal(err) + } +} diff --git a/db/Task.go b/db/Task.go index bf289b01..83e11f97 100644 --- a/db/Task.go +++ b/db/Task.go @@ -27,6 +27,8 @@ type Task struct { Message string `db:"message" json:"message"` + // CommitMessage is a git commit hash of playbook repository which + // was active when task was created. CommitHash *string `db:"commit_hash" json:"commit_hash"` // CommitMessage contains message retrieved from git repository after checkout to CommitHash. // It is readonly by API. @@ -37,6 +39,8 @@ type Task struct { // Version is a build version. // This field available only for Build tasks. Version *string `db:"version" json:"version"` + + Arguments *string `db:"arguments" json:"arguments"` } func (task *Task) GetIncomingVersion(d Store) *string { diff --git a/db/Template.go b/db/Template.go index 6ea1eeab..b4edcb41 100644 --- a/db/Template.go +++ b/db/Template.go @@ -49,7 +49,7 @@ type Template struct { // to fit into []string Arguments *string `db:"arguments" json:"arguments"` // if true, semaphore will not prepend any arguments to `arguments` like inventory, etc - OverrideArguments bool `db:"override_args" json:"override_args"` + AllowOverrideArgsInTask bool `db:"allow_override_args_in_task" json:"allow_override_args_in_task"` Removed bool `db:"removed" json:"-"` diff --git a/db/sql/migrations/v2.8.36.sql b/db/sql/migrations/v2.8.36.sql new file mode 100644 index 00000000..0ab74e3f --- /dev/null +++ b/db/sql/migrations/v2.8.36.sql @@ -0,0 +1,3 @@ +alter table `project__template` add allow_override_args_in_task bool not null default false; +alter table `task` add arguments text; +alter table `project__template` drop column `override_args`; diff --git a/db/sql/template.go b/db/sql/template.go index 59be5696..36510062 100644 --- a/db/sql/template.go +++ b/db/sql/template.go @@ -16,7 +16,7 @@ func (d *SqlDb) CreateTemplate(template db.Template) (newTemplate db.Template, e insertID, err := d.insert( "id", "insert into project__template (project_id, inventory_id, repository_id, environment_id, "+ - "alias, playbook, arguments, override_args, description, vault_key_id, `type`, start_version,"+ + "alias, playbook, arguments, allow_override_args_in_task, description, vault_key_id, `type`, start_version,"+ "build_template_id, view_id, autorun, survey_vars)"+ "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", template.ProjectID, @@ -26,7 +26,7 @@ func (d *SqlDb) CreateTemplate(template db.Template) (newTemplate db.Template, e template.Alias, template.Playbook, template.Arguments, - template.OverrideArguments, + template.AllowOverrideArgsInTask, template.Description, template.VaultKeyID, template.Type, @@ -66,7 +66,7 @@ func (d *SqlDb) UpdateTemplate(template db.Template) error { "alias=?, "+ "playbook=?, "+ "arguments=?, "+ - "override_args=?, "+ + "allow_override_args_in_task=?, "+ "description=?, "+ "vault_key_id=?, "+ "`type`=?, "+ @@ -82,7 +82,7 @@ func (d *SqlDb) UpdateTemplate(template db.Template) error { template.Alias, template.Playbook, template.Arguments, - template.OverrideArguments, + template.AllowOverrideArgsInTask, template.Description, template.VaultKeyID, template.Type, @@ -106,7 +106,7 @@ func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db. "pt.alias", "pt.playbook", "pt.arguments", - "pt.override_args", + "pt.allow_override_args_in_task", "pt.vault_key_id", "pt.view_id", "pt.`type`"). diff --git a/util/test_helpers.go b/util/test_helpers.go new file mode 100644 index 00000000..a0ae1350 --- /dev/null +++ b/util/test_helpers.go @@ -0,0 +1,35 @@ +package util + +import ( + "math/rand" + "time" +) + +//HELPERS + +//https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang +var src = rand.NewSource(time.Now().UnixNano()) + +const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + return string(b) +} diff --git a/web2/src/components/KeyForm.vue b/web2/src/components/KeyForm.vue index 8d186409..5a87d864 100644 --- a/web2/src/components/KeyForm.vue +++ b/web2/src/components/KeyForm.vue @@ -64,6 +64,13 @@ autocomplete="new-password" /> + +
{{ commitHash ? commitHash.substr(0, 10) : '' }} + >{{ (item.commit_hash || '').substr(0, 10) }}
-
{{ commitMessage }}
+
{{ sourceTask.commit_message }}
+
+ + Advanced + mdi-chevron-right + +
+ + +