From 67a29f763bfc0d3ef6989cb09a4dac0c0faf4f83 Mon Sep 17 00:00:00 2001 From: Matej Kramny Date: Fri, 8 Apr 2016 20:41:20 +0100 Subject: [PATCH] =?UTF-8?q?Running=20all=20the=20things=20=F0=9F=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Define extra (environment) vars to tasks - Able to create custom ansible jobs by defining parameters (infinite usabilities for user here) - Log output to db - Debug tasks - Ssh key for accessing inventory (separate from the key to access inventory api) - --- database/sql_migrations/v1.0.0.sql | 10 ++ migration/versionHistory.go | 1 + models/Environment.go | 8 +- models/Inventory.go | 8 +- models/Task.go | 10 +- models/Template.go | 21 ++-- models/accessKey.go | 11 ++- routes/sockets/handler.go | 2 +- routes/tasks/inventory.go | 32 +++++++ routes/tasks/logging.go | 35 +++++++ routes/tasks/runner.go | 148 ++++++++++++++++++----------- 11 files changed, 216 insertions(+), 70 deletions(-) create mode 100644 database/sql_migrations/v1.0.0.sql create mode 100644 routes/tasks/inventory.go create mode 100644 routes/tasks/logging.go diff --git a/database/sql_migrations/v1.0.0.sql b/database/sql_migrations/v1.0.0.sql new file mode 100644 index 00000000..acebcb01 --- /dev/null +++ b/database/sql_migrations/v1.0.0.sql @@ -0,0 +1,10 @@ +alter table task add `debug` tinyint(1) not null default 0; + +alter table project__template add `arguments` text null, + add `override_args` tinyint(1) not null default 0; + +alter table project__inventory add `ssh_key_id` int(11) not null, + add foreign key (`ssh_key_id`) references access_key(`id`); + +alter table task__output drop index `id`; +alter table task__output add key `task_id` (`task_id`); \ No newline at end of file diff --git a/migration/versionHistory.go b/migration/versionHistory.go index 54b5036c..1d8465d5 100644 --- a/migration/versionHistory.go +++ b/migration/versionHistory.go @@ -55,5 +55,6 @@ func (version *DBVersion) GetSQL(path string) []string { func init() { Versions = []*DBVersion{ {}, + {Major: 1}, } } diff --git a/models/Environment.go b/models/Environment.go index cdbe85b1..d7391ef6 100644 --- a/models/Environment.go +++ b/models/Environment.go @@ -3,10 +3,10 @@ package models import "github.com/ansible-semaphore/semaphore/database" type Environment struct { - ID int `db:"id" json:"id"` - ProjectID int `db:"project_id" json:"project_id"` - Password string `db:"password" json:"password"` - JSON string `db:"json" json:"json"` + ID int `db:"id" json:"id"` + ProjectID int `db:"project_id" json:"project_id"` + Password *string `db:"password" json:"password"` + JSON string `db:"json" json:"json"` } func init() { diff --git a/models/Inventory.go b/models/Inventory.go index 1c2cec9e..5251cab6 100644 --- a/models/Inventory.go +++ b/models/Inventory.go @@ -5,9 +5,15 @@ import "github.com/ansible-semaphore/semaphore/database" type Inventory struct { ID int `db:"id" json:"id"` ProjectID int `db:"project_id" json:"project_id"` - KeyID *int `db:"key_id" json:"key_id"` Inventory string `db:"inventory" json:"inventory"` + // accesses dynamic inventory + KeyID *int `db:"key_id" json:"key_id"` + Key AccessKey `db:"-" json:"-"` + // accesses hosts in inventory + SshKeyID *int `db:"ssh_key_id" json:"ssh_key_id"` + SshKey AccessKey `db:"-" json:"-"` + // static/aws/do/gcloud Type string `db:"type" json:"type"` } diff --git a/models/Task.go b/models/Task.go index 4427e553..e5faba38 100644 --- a/models/Task.go +++ b/models/Task.go @@ -4,9 +4,13 @@ import "time" import "github.com/ansible-semaphore/semaphore/database" type Task struct { - ID int `db:"id" json:"id"` - TemplateID int `db:"template_id" json:"template_id" binding:"required"` - Status string `db:"status" json:"status"` + ID int `db:"id" json:"id"` + TemplateID int `db:"template_id" json:"template_id" binding:"required"` + + Status string `db:"status" json:"status"` + Debug bool `db:"debug" json:"debug"` + + // override variables Playbook string `db:"playbook" json:"playbook"` Environment string `db:"environment" json:"environment"` } diff --git a/models/Template.go b/models/Template.go index c27be6cd..0893c09c 100644 --- a/models/Template.go +++ b/models/Template.go @@ -3,13 +3,20 @@ package models import "github.com/ansible-semaphore/semaphore/database" type Template struct { - ID int `db:"id" json:"id"` - SshKeyID int `db:"ssh_key_id" json:"ssh_key_id"` - ProjectID int `db:"project_id" json:"project_id"` - InventoryID int `db:"inventory_id" json:"inventory_id"` - RepositoryID int `db:"repository_id" json:"repository_id"` - EnvironmentID *int `db:"environment_id" json:"environment_id"` - Playbook string `db:"playbook" json:"playbook"` + ID int `db:"id" json:"id"` + + SshKeyID int `db:"ssh_key_id" json:"ssh_key_id"` + ProjectID int `db:"project_id" json:"project_id"` + InventoryID int `db:"inventory_id" json:"inventory_id"` + RepositoryID int `db:"repository_id" json:"repository_id"` + EnvironmentID *int `db:"environment_id" json:"environment_id"` + + // playbook name in the form of "some_play.yml" + Playbook string `db:"playbook" json:"playbook"` + // to fit into []string + Arguments *string `db:"arguments" json:"arguments"` + // if true, semaphore will not prepend any arguments to `arguments` like inventory, etc + OverrideArguments bool `db:"override_args" json:"override_args"` } type TemplateSchedule struct { diff --git a/models/accessKey.go b/models/accessKey.go index 310f8b1f..49b72292 100644 --- a/models/accessKey.go +++ b/models/accessKey.go @@ -1,6 +1,11 @@ package models -import "github.com/ansible-semaphore/semaphore/database" +import ( + "strconv" + + "github.com/ansible-semaphore/semaphore/database" + "github.com/ansible-semaphore/semaphore/util" +) type AccessKey struct { ID int `db:"id" json:"id"` @@ -16,3 +21,7 @@ type AccessKey struct { func init() { database.Mysql.AddTableWithName(AccessKey{}, "access_key").SetKeys(true, "id") } + +func (key AccessKey) GetPath() string { + return util.Config.TmpPath + "/access_key_" + strconv.Itoa(key.ID) +} diff --git a/routes/sockets/handler.go b/routes/sockets/handler.go index 080d7023..c5653701 100644 --- a/routes/sockets/handler.go +++ b/routes/sockets/handler.go @@ -115,6 +115,6 @@ func Handler(context *gin.Context) { } func Broadcast(msg []byte) { - fmt.Println("broadcasting", string(msg)) + fmt.Printf("-> %v\n", string(msg)) h.broadcast <- msg } diff --git a/routes/tasks/inventory.go b/routes/tasks/inventory.go new file mode 100644 index 00000000..8774565d --- /dev/null +++ b/routes/tasks/inventory.go @@ -0,0 +1,32 @@ +package tasks + +import ( + "io/ioutil" + "strconv" + + "github.com/ansible-semaphore/semaphore/util" +) + +func (t *task) installInventory() error { + if t.inventory.SshKeyID != nil { + // write inventory key + err := t.installKey(t.inventory.SshKey) + if err != nil { + return err + } + } + + switch t.inventory.Type { + case "static": + return t.installStaticInventory() + } + + return nil +} + +func (t *task) installStaticInventory() error { + t.log("installing static inventory") + + // create inventory file + return ioutil.WriteFile(util.Config.TmpPath+"/inventory_"+strconv.Itoa(t.task.ID), []byte(t.inventory.Inventory), 0664) +} diff --git a/routes/tasks/logging.go b/routes/tasks/logging.go new file mode 100644 index 00000000..7b22327a --- /dev/null +++ b/routes/tasks/logging.go @@ -0,0 +1,35 @@ +package tasks + +import ( + "bufio" + "os/exec" + + "github.com/ansible-semaphore/semaphore/database" + "github.com/ansible-semaphore/semaphore/routes/sockets" +) + +func (t *task) log(msg string) { + // TODO: broadcast to a set of users listening to this project + sockets.Broadcast([]byte(msg)) + + go func() { + _, err := database.Mysql.Exec("insert into task__output set task_id=?, output=?", t.task.ID, msg) + if err != nil { + sockets.Broadcast([]byte("Error logging task output" + err.Error())) + } + }() +} + +func (t *task) logPipe(scanner *bufio.Scanner) { + for scanner.Scan() { + t.log(scanner.Text()) + } +} + +func (t *task) logCmd(cmd *exec.Cmd) { + stderr, _ := cmd.StderrPipe() + stdout, _ := cmd.StdoutPipe() + + go t.logPipe(bufio.NewScanner(stderr)) + go t.logPipe(bufio.NewScanner(stdout)) +} diff --git a/routes/tasks/runner.go b/routes/tasks/runner.go index e665999d..c4514cf1 100644 --- a/routes/tasks/runner.go +++ b/routes/tasks/runner.go @@ -2,6 +2,7 @@ package tasks import ( "database/sql" + "encoding/json" "errors" "fmt" "io/ioutil" @@ -11,7 +12,6 @@ import ( "github.com/ansible-semaphore/semaphore/database" "github.com/ansible-semaphore/semaphore/models" - "github.com/ansible-semaphore/semaphore/routes/sockets" "github.com/ansible-semaphore/semaphore/util" ) @@ -24,10 +24,6 @@ type task struct { environment models.Environment } -func (t *task) log(msg string) { - sockets.Broadcast([]byte(msg)) -} - func (t *task) run() { pool.running = t @@ -53,6 +49,17 @@ func (t *task) run() { return } + if err := t.installInventory(); err != nil { + t.log("Failed to install inventory: " + err.Error()) + return + } + + // todo: write environment + + if err := t.runPlaybook(); err != nil { + t.log("Running playbook failed: " + err.Error()) + return + } } func (t *task) fetch(errMsg string, ptr interface{}, query string, args ...interface{}) error { @@ -90,6 +97,20 @@ func (t *task) populateDetails() error { return err } + // get inventory services key + if t.inventory.KeyID != nil { + if err := t.fetch("Inventory AccessKey not found!", &t.inventory.Key, "select * from access_key where id=?", *t.inventory.KeyID); err != nil { + return err + } + } + + // get inventory ssh key + if t.inventory.SshKeyID != nil { + if err := t.fetch("Inventory Ssh Key not found!", &t.inventory.SshKey, "select * from access_key where id=?", *t.inventory.SshKeyID); err != nil { + return err + } + } + // get repository if err := t.fetch("Repository not found!", &t.repository, "select * from project__repository where id=?", t.template.RepositoryID); err != nil { return err @@ -118,65 +139,86 @@ func (t *task) populateDetails() error { } func (t *task) installKey(key models.AccessKey) error { - t.log("installing Access key: " + key.Name) + t.log("access key " + key.Name + " installed") + err := ioutil.WriteFile(key.GetPath(), []byte(*key.Secret), 0600) - // create .ssh directory - err := os.MkdirAll(util.Config.TmpPath+"/.ssh", 448) - if err != nil { - return err - } - - err = ioutil.WriteFile(util.Config.TmpPath+"/.ssh/id_rsa", []byte(*key.Secret), 0600) - if err != nil { - return err - } - - err = ioutil.WriteFile(util.Config.TmpPath+"/.ssh/id_rsa.pub", []byte(*key.Key), 0644) - if err != nil { - return err - } - - t.log("key " + key.Name + " installed") - return nil + return err } func (t *task) updateRepository() error { repoName := "repository_" + strconv.Itoa(t.repository.ID) - _, err := os.Stat(util.Config.TmpPath + "/" + repoName) - if err != nil && os.IsNotExist(err) { - t.log("Cloning repository") - cmd := exec.Command("git", "clone", t.repository.GitUrl, repoName) - cmd.Env = []string{ - "HOME=" + util.Config.TmpPath, - "PWD=" + util.Config.TmpPath, - "GIT_SSH_COMMAND=ssh -o StrictHostKeyChecking=no -i " + util.Config.TmpPath + "/.ssh/id_rsa", - // "GIT_FLUSH=1", - } - cmd.Dir = util.Config.TmpPath - - out, err := cmd.CombinedOutput() - fmt.Println(string(out)) - - return err - } else if err != nil { - return err - } - - t.log("Updating repository") - - // update instead of cloning - cmd := exec.Command("git", "pull", "origin", "master") + cmd := exec.Command("git") + cmd.Dir = util.Config.TmpPath + "/" + repoName cmd.Env = []string{ "HOME=" + util.Config.TmpPath, "PWD=" + util.Config.TmpPath, - "GIT_SSH_COMMAND=ssh -o StrictHostKeyChecking=no -i " + util.Config.TmpPath + "/.ssh/id_rsa", + "GIT_SSH_COMMAND=ssh -o StrictHostKeyChecking=no -i " + t.repository.SshKey.GetPath(), + // "GIT_FLUSH=1", } - cmd.Dir = util.Config.TmpPath + "/" + repoName - out, err := cmd.CombinedOutput() - fmt.Println(string(out)) + if err != nil && os.IsNotExist(err) { + t.log("Cloning repository") + cmd.Args = append(cmd.Args, "clone", t.repository.GitUrl, repoName) + } else if err != nil { + return err + } else { + t.log("Updating repository") + cmd.Args = append(cmd.Args, "pull", "origin", "master") + } - return nil + t.logCmd(cmd) + return cmd.Run() +} + +func (t *task) runPlaybook() error { + playbookName := t.task.Playbook + if len(playbookName) == 0 { + playbookName = t.template.Playbook + } + + args := []string{ + "-i", util.Config.TmpPath + "/inventory_" + strconv.Itoa(t.task.ID), + } + + if t.inventory.SshKeyID != nil { + args = append(args, "--private-key="+t.inventory.SshKey.GetPath()) + } + + if t.task.Debug { + args = append(args, "-vvvv") + } + + if len(t.environment.JSON) > 0 { + args = append(args, "--extra-vars", t.environment.JSON) + } + + var extraArgs []string + if t.template.Arguments != nil { + err := json.Unmarshal([]byte(*t.template.Arguments), &extraArgs) + if err != nil { + t.log("Could not unmarshal arguments to []string") + return err + } + } + + if t.template.OverrideArguments { + args = extraArgs + } else { + args = append(args, extraArgs...) + args = append(args, playbookName) + } + + cmd := exec.Command("ansible-playbook", args...) + cmd.Dir = util.Config.TmpPath + "/repository_" + strconv.Itoa(t.repository.ID) + cmd.Env = []string{ + "HOME=" + util.Config.TmpPath, + "PWD=" + cmd.Dir, + "PYTHONUNBUFFERED=1", + // "GIT_FLUSH=1", + } + + t.logCmd(cmd) + return cmd.Run() }