Kiu/Semaphore
Kiu/Semaphore
1
0
Fork 0
mirror of https://github.com/semaphoreui/semaphore.git synced 2025-01-20 15:29:28 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Running all the things 🎉

Browse Source 
- Define extra (environment) vars to tasks
- Able to create custom ansible jobs by defining parameters (infinite usabilities for user here)
- Log output to db
- Debug tasks
- Ssh key for accessing inventory (separate from the key to access inventory api)
-
This commit is contained in:
Matej Kramny 2016-04-08 20:41:20 +01:00
parent 2e1ad91e7a
commit 67a29f763b
11 changed files with 216 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
database/sql_migrations/v1.0.0.sql Normal file
View File

@ -0,0 +1,10 @@
alter table task add `debug` tinyint(1) not null default 0;
alter table project__template add `arguments` text null,
add `override_args` tinyint(1) not null default 0;
alter table project__inventory add `ssh_key_id` int(11) not null,
add foreign key (`ssh_key_id`) references access_key(`id`);
alter table task__output drop index `id`;
alter table task__output add key `task_id` (`task_id`);

1
migration/versionHistory.go
View File

@ -55,5 +55,6 @@ func (version *DBVersion) GetSQL(path string) []string {
func init() {
Versions = []*DBVersion{
{},
{Major: 1},
}
}

8
models/Environment.go
View File

@ -3,10 +3,10 @@ package models
import "github.com/ansible-semaphore/semaphore/database"
type Environment struct {
ID int `db:"id" json:"id"`
ProjectID int `db:"project_id" json:"project_id"`
Password string `db:"password" json:"password"`
JSON string `db:"json" json:"json"`
ID int `db:"id" json:"id"`
ProjectID int `db:"project_id" json:"project_id"`
Password *string `db:"password" json:"password"`
JSON string `db:"json" json:"json"`
}
func init() {

8
models/Inventory.go
View File

@ -5,9 +5,15 @@ import "github.com/ansible-semaphore/semaphore/database"
type Inventory struct {
ID int `db:"id" json:"id"`
ProjectID int `db:"project_id" json:"project_id"`
KeyID *int `db:"key_id" json:"key_id"`
Inventory string `db:"inventory" json:"inventory"`
// accesses dynamic inventory
KeyID *int `db:"key_id" json:"key_id"`
Key AccessKey `db:"-" json:"-"`
// accesses hosts in inventory
SshKeyID *int `db:"ssh_key_id" json:"ssh_key_id"`
SshKey AccessKey `db:"-" json:"-"`
// static/aws/do/gcloud
Type string `db:"type" json:"type"`
}

10
models/Task.go
View File

@ -4,9 +4,13 @@ import "time"
import "github.com/ansible-semaphore/semaphore/database"
type Task struct {
ID int `db:"id" json:"id"`
TemplateID int `db:"template_id" json:"template_id" binding:"required"`
Status string `db:"status" json:"status"`
ID int `db:"id" json:"id"`
TemplateID int `db:"template_id" json:"template_id" binding:"required"`
Status string `db:"status" json:"status"`
Debug bool `db:"debug" json:"debug"`
// override variables
Playbook string `db:"playbook" json:"playbook"`
Environment string `db:"environment" json:"environment"`
}

21
models/Template.go
View File

@ -3,13 +3,20 @@ package models
import "github.com/ansible-semaphore/semaphore/database"
type Template struct {
ID int `db:"id" json:"id"`
SshKeyID int `db:"ssh_key_id" json:"ssh_key_id"`
ProjectID int `db:"project_id" json:"project_id"`
InventoryID int `db:"inventory_id" json:"inventory_id"`
RepositoryID int `db:"repository_id" json:"repository_id"`
EnvironmentID *int `db:"environment_id" json:"environment_id"`
Playbook string `db:"playbook" json:"playbook"`
ID int `db:"id" json:"id"`
SshKeyID int `db:"ssh_key_id" json:"ssh_key_id"`
ProjectID int `db:"project_id" json:"project_id"`
InventoryID int `db:"inventory_id" json:"inventory_id"`
RepositoryID int `db:"repository_id" json:"repository_id"`
EnvironmentID *int `db:"environment_id" json:"environment_id"`
// playbook name in the form of "some_play.yml"
Playbook string `db:"playbook" json:"playbook"`
// to fit into []string
Arguments *string `db:"arguments" json:"arguments"`
// if true, semaphore will not prepend any arguments to `arguments` like inventory, etc
OverrideArguments bool `db:"override_args" json:"override_args"`
}
type TemplateSchedule struct {

11
models/accessKey.go
View File

@ -1,6 +1,11 @@
package models
import "github.com/ansible-semaphore/semaphore/database"
import (
"strconv"
"github.com/ansible-semaphore/semaphore/database"
"github.com/ansible-semaphore/semaphore/util"
)
type AccessKey struct {
ID int `db:"id" json:"id"`
@ -16,3 +21,7 @@ type AccessKey struct {
func init() {
database.Mysql.AddTableWithName(AccessKey{}, "access_key").SetKeys(true, "id")
}
func (key AccessKey) GetPath() string {
return util.Config.TmpPath + "/access_key_" + strconv.Itoa(key.ID)
}

2
routes/sockets/handler.go
View File

@ -115,6 +115,6 @@ func Handler(context *gin.Context) {
}
func Broadcast(msg []byte) {
fmt.Println("broadcasting", string(msg))
fmt.Printf("-> %v\n", string(msg))
h.broadcast <- msg
}

32
routes/tasks/inventory.go Normal file
View File

@ -0,0 +1,32 @@
package tasks
import (
"io/ioutil"
"strconv"
"github.com/ansible-semaphore/semaphore/util"
)
func (t *task) installInventory() error {
if t.inventory.SshKeyID != nil {
// write inventory key
err := t.installKey(t.inventory.SshKey)
if err != nil {
return err
}
}
switch t.inventory.Type {
case "static":
return t.installStaticInventory()
}
return nil
}
func (t *task) installStaticInventory() error {
t.log("installing static inventory")
// create inventory file
return ioutil.WriteFile(util.Config.TmpPath+"/inventory_"+strconv.Itoa(t.task.ID), []byte(t.inventory.Inventory), 0664)
}

35
routes/tasks/logging.go Normal file
View File

@ -0,0 +1,35 @@
package tasks
import (
"bufio"
"os/exec"
"github.com/ansible-semaphore/semaphore/database"
"github.com/ansible-semaphore/semaphore/routes/sockets"
)
func (t *task) log(msg string) {
// TODO: broadcast to a set of users listening to this project
sockets.Broadcast([]byte(msg))
go func() {
_, err := database.Mysql.Exec("insert into task__output set task_id=?, output=?", t.task.ID, msg)
if err != nil {
sockets.Broadcast([]byte("Error logging task output" + err.Error()))
}
}()
}
func (t *task) logPipe(scanner *bufio.Scanner) {
for scanner.Scan() {
t.log(scanner.Text())
}
}
func (t *task) logCmd(cmd *exec.Cmd) {
stderr, _ := cmd.StderrPipe()
stdout, _ := cmd.StdoutPipe()
go t.logPipe(bufio.NewScanner(stderr))
go t.logPipe(bufio.NewScanner(stdout))
}

148
routes/tasks/runner.go
View File

@ -2,6 +2,7 @@ package tasks
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
@ -11,7 +12,6 @@ import (
"github.com/ansible-semaphore/semaphore/database"
"github.com/ansible-semaphore/semaphore/models"
"github.com/ansible-semaphore/semaphore/routes/sockets"
"github.com/ansible-semaphore/semaphore/util"
)
@ -24,10 +24,6 @@ type task struct {
environment models.Environment
}
func (t *task) log(msg string) {
sockets.Broadcast([]byte(msg))
}
func (t *task) run() {
pool.running = t
@ -53,6 +49,17 @@ func (t *task) run() {
return
}
if err := t.installInventory(); err != nil {
t.log("Failed to install inventory: " + err.Error())
return
}
// todo: write environment
if err := t.runPlaybook(); err != nil {
t.log("Running playbook failed: " + err.Error())
return
}
}
func (t *task) fetch(errMsg string, ptr interface{}, query string, args ...interface{}) error {
@ -90,6 +97,20 @@ func (t *task) populateDetails() error {
return err
}
// get inventory services key
if t.inventory.KeyID != nil {
if err := t.fetch("Inventory AccessKey not found!", &t.inventory.Key, "select * from access_key where id=?", *t.inventory.KeyID); err != nil {
return err
}
}
// get inventory ssh key
if t.inventory.SshKeyID != nil {
if err := t.fetch("Inventory Ssh Key not found!", &t.inventory.SshKey, "select * from access_key where id=?", *t.inventory.SshKeyID); err != nil {
return err
}
}
// get repository
if err := t.fetch("Repository not found!", &t.repository, "select * from project__repository where id=?", t.template.RepositoryID); err != nil {
return err
@ -118,65 +139,86 @@ func (t *task) populateDetails() error {
}
func (t *task) installKey(key models.AccessKey) error {
t.log("installing Access key: " + key.Name)
t.log("access key " + key.Name + " installed")
err := ioutil.WriteFile(key.GetPath(), []byte(*key.Secret), 0600)
// create .ssh directory
err := os.MkdirAll(util.Config.TmpPath+"/.ssh", 448)
if err != nil {
return err
}
err = ioutil.WriteFile(util.Config.TmpPath+"/.ssh/id_rsa", []byte(*key.Secret), 0600)
if err != nil {
return err
}
err = ioutil.WriteFile(util.Config.TmpPath+"/.ssh/id_rsa.pub", []byte(*key.Key), 0644)
if err != nil {
return err
}
t.log("key " + key.Name + " installed")
return nil
return err
}
func (t *task) updateRepository() error {
repoName := "repository_" + strconv.Itoa(t.repository.ID)
_, err := os.Stat(util.Config.TmpPath + "/" + repoName)
if err != nil && os.IsNotExist(err) {
t.log("Cloning repository")
cmd := exec.Command("git", "clone", t.repository.GitUrl, repoName)
cmd.Env = []string{
"HOME=" + util.Config.TmpPath,
"PWD=" + util.Config.TmpPath,
"GIT_SSH_COMMAND=ssh -o StrictHostKeyChecking=no -i " + util.Config.TmpPath + "/.ssh/id_rsa",
// "GIT_FLUSH=1",
}
cmd.Dir = util.Config.TmpPath
out, err := cmd.CombinedOutput()
fmt.Println(string(out))
return err
} else if err != nil {
return err
}
t.log("Updating repository")
// update instead of cloning
cmd := exec.Command("git", "pull", "origin", "master")
cmd := exec.Command("git")
cmd.Dir = util.Config.TmpPath + "/" + repoName
cmd.Env = []string{
"HOME=" + util.Config.TmpPath,
"PWD=" + util.Config.TmpPath,
"GIT_SSH_COMMAND=ssh -o StrictHostKeyChecking=no -i " + util.Config.TmpPath + "/.ssh/id_rsa",
"GIT_SSH_COMMAND=ssh -o StrictHostKeyChecking=no -i " + t.repository.SshKey.GetPath(),
// "GIT_FLUSH=1",
}
cmd.Dir = util.Config.TmpPath + "/" + repoName
out, err := cmd.CombinedOutput()
fmt.Println(string(out))
if err != nil && os.IsNotExist(err) {
t.log("Cloning repository")
cmd.Args = append(cmd.Args, "clone", t.repository.GitUrl, repoName)
} else if err != nil {
return err
} else {
t.log("Updating repository")
cmd.Args = append(cmd.Args, "pull", "origin", "master")
}
return nil
t.logCmd(cmd)
return cmd.Run()
}
func (t *task) runPlaybook() error {
playbookName := t.task.Playbook
if len(playbookName) == 0 {
playbookName = t.template.Playbook
}
args := []string{
"-i", util.Config.TmpPath + "/inventory_" + strconv.Itoa(t.task.ID),
}
if t.inventory.SshKeyID != nil {
args = append(args, "--private-key="+t.inventory.SshKey.GetPath())
}
if t.task.Debug {
args = append(args, "-vvvv")
}
if len(t.environment.JSON) > 0 {
args = append(args, "--extra-vars", t.environment.JSON)
}
var extraArgs []string
if t.template.Arguments != nil {
err := json.Unmarshal([]byte(*t.template.Arguments), &extraArgs)
if err != nil {
t.log("Could not unmarshal arguments to []string")
return err
}
}
if t.template.OverrideArguments {
args = extraArgs
} else {
args = append(args, extraArgs...)
args = append(args, playbookName)
}
cmd := exec.Command("ansible-playbook", args...)
cmd.Dir = util.Config.TmpPath + "/repository_" + strconv.Itoa(t.repository.ID)
cmd.Env = []string{
"HOME=" + util.Config.TmpPath,
"PWD=" + cmd.Dir,
"PYTHONUNBUFFERED=1",
// "GIT_FLUSH=1",
}
t.logCmd(cmd)
return cmd.Run()
}