Kiu/Semaphore
Kiu/Semaphore
1
0
Fork 0
mirror of https://github.com/semaphoreui/semaphore.git synced 2025-01-20 15:29:28 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Removed runner.go

Browse Source
This commit is contained in:
Jordan Hayes 2022-02-10 11:49:36 -08:00
commit d2b9b856c8
141 changed files with 5346 additions and 3334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.circleci/config.yml
View File

@ -39,7 +39,7 @@ aliases:
curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.5/install.sh | bash
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
nvm install 12.15.0 && nvm alias default 12.15.0
nvm install 12.15.0 && nvm name default 12.15.0
# Each step uses the same `$BASH_ENV`, so need to modify it
echo 'export NVM_DIR="$HOME/.nvm"' >> $BASH_ENV
echo "[ -s \"$NVM_DIR/nvm.sh\" ] && . \"$NVM_DIR/nvm.sh\"" >> $BASH_ENV

2
.dredd/hooks/capabilities.go
View File

@ -100,7 +100,7 @@ func resolveCapability(caps []string, resolved []string, uid string) {
case "template":
res, err := store.Sql().Exec(
"insert into project__template "+
"(project_id, inventory_id, repository_id, environment_id, alias, playbook, arguments, override_args, description, view_id) "+
"(project_id, inventory_id, repository_id, environment_id, name, playbook, arguments, allow_override_args_in_task, description, view_id) "+
"values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
userProject.ID, inventoryID, repoID, environmentID, "Test-"+uid, "test-playbook.yml", "", false, "Hello, World!", view.ID)
printError(err)

6
.gitignore vendored
View File

@ -8,12 +8,14 @@ web2/.nyc_output
web2/dist/**/*
/config.json
/.dredd/config.json
/database.bolt
/database.boltdb
/database.boltdb.lock
/database2.boltdb
.DS_Store
node_modules/
.idea/
/.idea/
/semaphore.iml
/bin/
*-packr.go

2
CONTRIBUTING.md
View File

@ -51,7 +51,7 @@ echo "create database semaphore;" | mysql -uroot -p
```
task compile
go run cli/main.go setup
go run cli/main.go --config ./config.json
go run cli/main.go service --config ./config.json
```
Open [localhost:3000](http://localhost:3000)

17
README.md
View File

@ -1,17 +1,22 @@
# Ansible Semaphore
[![Circle CI](https://circleci.com/gh/ansible-semaphore/semaphore.svg?style=svg&circle-token=3702872acf2bec629017fa7dd99fdbea56aef7df)](https://circleci.com/gh/ansible-semaphore/semaphore)
[![Codacy Badge](https://api.codacy.com/project/badge/Grade/89e0129c6ba64fe2b1ebe983f72a4eff)](https://www.codacy.com/app/ansible-semaphore/semaphore?utm_source=github.com&utm_medium=referral&utm_content=ansible-semaphore/semaphore&utm_campaign=Badge_Grade)
[![Codacy Badge](https://api.codacy.com/project/badge/Coverage/89e0129c6ba64fe2b1ebe983f72a4eff)](https://www.codacy.com/app/ansible-semaphore/semaphore?utm_source=github.com&utm_medium=referral&utm_content=ansible-semaphore/semaphore&utm_campaign=Badge_Coverage)
[![Join the chat at https://gitter.im/AnsibleSemaphore/semaphore](https://badges.gitter.im/AnsibleSemaphore/semaphore.svg)](https://gitter.im/AnsibleSemaphore/semaphore?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
Follow Semaphore on Twitter ([AnsibleSem](https://twitter.com/AnsibleSem)) and StackShare ([ansible-semaphore](https://stackshare.io/ansible-semaphore)).
[![Twitter](https://img.shields.io/twitter/follow/AnsibleSem?style=social&logo=twitter)](https://twitter.com/AnsibleSem)
[![Snap](https://img.shields.io/badge/snap-semaphore-005c63)](https://snapcraft.io/semaphore)
[![StackShare](https://img.shields.io/badge/tech-stack-008ff9)](https://stackshare.io/ansible-semaphore)
[![Join the chat at https://gitter.im/AnsibleSemaphore/semaphore](https://img.shields.io/gitter/room/AnsibleSemaphore/semaphore?logo=gitter)](https://gitter.im/AnsibleSemaphore/semaphore?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
<!-- [![Release](https://img.shields.io/github/v/release/ansible-semaphore/semaphore.svg)](https://stackshare.io/ansible-semaphore) -->
<!-- [![Godoc Reference](https://pkg.go.dev/badge/github.com/ansible-semaphore/semaphore?utm_source=godoc)](https://godoc.org/github.com/ansible-semaphore/semaphore) -->
<!-- [![Codacy Badge](https://api.codacy.com/project/badge/Grade/89e0129c6ba64fe2b1ebe983f72a4eff)](https://www.codacy.com/app/ansible-semaphore/semaphore?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=ansible-semaphore/semaphore&amp;utm_campaign=Badge_Grade)
[![Codacy Badge](https://api.codacy.com/project/badge/Coverage/89e0129c6ba64fe2b1ebe983f72a4eff)](https://www.codacy.com/app/ansible-semaphore/semaphore?utm_source=github.com&utm_medium=referral&utm_content=ansible-semaphore/semaphore&utm_campaign=Badge_Coverage) -->
Ansible Semaphore is a modern UI for Ansible. It lets you easily run Ansible playbooks, get notifications about fails, control access to deployment system.
If your project has grown and deploying from the terminal is no longer for you then Ansible Semaphore is what you need.
Follow Semaphore on Twitter ([AnsibleSem](https://twitter.com/AnsibleSem)) and StackShare ([ansible-semaphore](https://stackshare.io/ansible-semaphore)).
![responsive-ui-phone1](https://user-images.githubusercontent.com/914224/134777345-8789d9e4-ff0d-439c-b80e-ddc56b74fcee.png)
<!--
@ -52,7 +57,7 @@ Login / password: `demo / demo`.
Admin and user docs: https://docs.ansible-semaphore.com
API docs: https://ansible-semaphore.github.io/semaphore
API docs: https://ansible-semaphore.com/api/
## Contributing

20
api-docs.yml
View File

@ -253,6 +253,10 @@ definitions:
type: integer
git_url:
type: string
example: git@example.com
git_branch:
type: string
example: master
ssh_key_id:
type: integer
Repository:
@ -267,6 +271,10 @@ definitions:
type: integer
git_url:
type: string
example: git@example.com
git_branch:
type: string
example: master
ssh_key_id:
type: integer
@ -318,7 +326,7 @@ definitions:
view_id:
type: integer
minimum: 1
alias:
name:
type: string
example: Test
playbook:
@ -330,8 +338,9 @@ definitions:
description:
type: string
example: Hello, World!
override_args:
allow_override_args_in_task:
type: boolean
example: false
Template:
type: object
properties:
@ -352,7 +361,7 @@ definitions:
view_id:
type: integer
minimum: 1
alias:
name:
type: string
example: Test
playbook:
@ -364,8 +373,9 @@ definitions:
description:
type: string
example: Hello, World!
override_args:
allow_override_args_in_task:
type: boolean
example: false
ScheduleRequest:
type: object
@ -1211,7 +1221,7 @@ paths:
required: true
type: string
description: sorting name
enum: [alias, playbook, ssh_key, inventory, environment, repository]
enum: [name, playbook, ssh_key, inventory, environment, repository]
- name: order
in: query
required: true

9
api/auth.go
View File

@ -92,6 +92,15 @@ func authentication(next http.Handler) http.Handler {
return
}
if util.Config.DemoMode {
if !user.Admin && r.Method != "GET" &&
!strings.HasSuffix(r.URL.Path, "/tasks") &&
!strings.HasSuffix(r.URL.Path, "/stop") {
w.WriteHeader(http.StatusUnauthorized)
return
}
}
context.Set(r, "user", &user)
next.ServeHTTP(w, r)

26
api/helpers/helpers.go
View File

@ -1,13 +1,10 @@
package helpers
import (
"crypto/md5"
"encoding/json"
"fmt"
"io"
"github.com/ansible-semaphore/semaphore/services/tasks"
"net/http"
"net/url"
"os"
"runtime/debug"
"strconv"
"strings"
@ -24,6 +21,10 @@ func Store(r *http.Request) db.Store {
return context.Get(r, "store").(db.Store)
}
func TaskPool(r *http.Request) *tasks.TaskPool {
return context.Get(r, "task_pool").(*tasks.TaskPool)
}
func isXHR(w http.ResponseWriter, r *http.Request) bool {
accept := r.Header.Get("Accept")
return !strings.Contains(accept, "text/html")
@ -93,24 +94,9 @@ func WriteError(w http.ResponseWriter, err error) {
}
}
func GetMD5Hash(filepath string) (string, error) {
file, err := os.Open(filepath)
if err != nil {
return "", err
}
defer file.Close()
hash := md5.New()
if _, err := io.Copy(hash, file); err != nil {
return "", err
}
return fmt.Sprintf("%x", hash.Sum(nil)), nil
}
func QueryParams(url *url.URL) db.RetrieveQueryParams {
return db.RetrieveQueryParams{
SortBy: url.Query().Get("sort"),
SortBy: url.Query().Get("sort"),
SortInverted: url.Query().Get("order") == "desc",
}
}

89
api/login.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"github.com/ansible-semaphore/semaphore/api/helpers"
"github.com/ansible-semaphore/semaphore/db"
"github.com/go-ldap/ldap/v3"
"net/http"
"strings"
"time"
@ -13,10 +14,9 @@ import (
"github.com/ansible-semaphore/semaphore/util"
"golang.org/x/crypto/bcrypt"
"gopkg.in/ldap.v2"
)
func findLDAPUser(username, password string) (*db.User, error) {
func tryFindLDAPUser(username, password string) (*db.User, error) {
if !util.Config.LdapEnable {
return nil, fmt.Errorf("LDAP not configured")
}
@ -36,17 +36,6 @@ func findLDAPUser(username, password string) (*db.User, error) {
}
defer l.Close()
// Reconnect with TLS if needed
if util.Config.LdapNeedTLS {
// TODO: InsecureSkipVerify should be configurable
tlsConf := tls.Config{
InsecureSkipVerify: true, //nolint: gas
}
if err = l.StartTLS(&tlsConf); err != nil {
return nil, err
}
}
// First bind with a read only user
if err = l.Bind(util.Config.LdapBindDN, util.Config.LdapBindPassword); err != nil {
return nil, err
@ -66,8 +55,12 @@ func findLDAPUser(username, password string) (*db.User, error) {
return nil, err
}
if len(sr.Entries) != 1 {
return nil, fmt.Errorf("user does not exist or too many entries returned")
if len(sr.Entries) < 1 {
return nil, nil
}
if len(sr.Entries) > 1 {
return nil, fmt.Errorf("too many entries returned")
}
// Bind as the user to verify their password
@ -103,6 +96,8 @@ func findLDAPUser(username, password string) (*db.User, error) {
return &ldapUser, nil
}
// createSession creates session for passed user and stores session details
// in cookies.
func createSession(w http.ResponseWriter, r *http.Request, user db.User) {
newSession, err := helpers.Store(r).CreateSession(db.Session{
UserID: user.ID,
@ -132,59 +127,6 @@ func createSession(w http.ResponseWriter, r *http.Request, user db.User) {
})
}
func info(w http.ResponseWriter, r *http.Request) {
var info struct {
NewUserRequired bool `json:"newUserRequired"`
}
if util.Config.RegisterFirstUser {
hasPlaceholderUser, err := db.HasPlaceholderUser(helpers.Store(r))
if err != nil {
log.Error(err)
w.WriteHeader(http.StatusInternalServerError)
return
}
info.NewUserRequired = hasPlaceholderUser
}
helpers.WriteJSON(w, http.StatusOK, info)
}
func register(w http.ResponseWriter, r *http.Request) {
var user db.UserWithPwd
if !helpers.Bind(w, r, &user) {
return
}
if !util.Config.RegisterFirstUser {
w.WriteHeader(http.StatusBadRequest)
return
}
hasPlaceholderUser, err := db.HasPlaceholderUser(helpers.Store(r))
if err != nil {
log.Error(err)
w.WriteHeader(http.StatusInternalServerError)
return
}
if !hasPlaceholderUser {
w.WriteHeader(http.StatusBadRequest)
return
}
newUser, err := db.ReplacePlaceholderUser(helpers.Store(r), user)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
createSession(w, r, newUser)
w.WriteHeader(http.StatusNoContent)
}
//nolint: gocyclo
func login(w http.ResponseWriter, r *http.Request) {
var login struct {
@ -206,13 +148,16 @@ func login(w http.ResponseWriter, r *http.Request) {
login.Auth = strings.ToLower(login.Auth)
var err error
var ldapUser *db.User
if util.Config.LdapEnable {
// search LDAP for users
if lu, err := findLDAPUser(login.Auth, login.Password); err == nil {
ldapUser = lu
} else {
ldapUser, err = tryFindLDAPUser(login.Auth, login.Password)
if err != nil {
log.Info(err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
}

105
api/login_test.go
View File

@ -1,105 +0,0 @@
package api
import (
"bytes"
"encoding/base64"
"encoding/json"
"github.com/ansible-semaphore/semaphore/db"
"github.com/ansible-semaphore/semaphore/db/bolt"
"github.com/ansible-semaphore/semaphore/util"
"github.com/gorilla/context"
"github.com/gorilla/securecookie"
"math/rand"
"strconv"
"time"
//_ "github.com/snikch/goodman/hooks"
//_ "github.com/snikch/goodman/transaction"
"net/http"
"net/http/httptest"
"testing"
)
func TestAuthRegister(t *testing.T) {
util.Config = &util.ConfigType{}
body, err := json.Marshal(map[string]string{
"name": "Toby",
"email": "Toby@example.com",
})
if err != nil {
t.Fail()
}
req, _ := http.NewRequest("POST", "/api/auth/register", bytes.NewBuffer(body))
rr := httptest.NewRecorder()
r := Route()
r.ServeHTTP(rr, req)
if rr.Code != 400 {
t.Errorf("Response code should be 400 but got %d", rr.Code)
}
}
func createBoltDb() bolt.BoltDb {
r := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
fn := "/tmp/test_semaphore_db_" + strconv.Itoa(r.Int())
return bolt.BoltDb{
Filename: fn,
}
}
func createStore() db.Store {
store := createBoltDb()
err := store.Connect()
if err != nil {
panic(err)
}
return &store
}
func initCookies() {
var encryption []byte
hash, _ := base64.StdEncoding.DecodeString("ikt5uEXMZa7qinEqRa7tO1y9QpBAMG8goTxsJ57h6O8=")
encryption, _ = base64.StdEncoding.DecodeString("bEjxOq4fhKdiYO50mEF99aR1LJPnevvViVvXfhZV5QY=")
util.Cookie = securecookie.New(hash, encryption)
}
func TestAuthRegister2(t *testing.T) {
initCookies()
util.Config = &util.ConfigType{
RegisterFirstUser: true,
}
body, err := json.Marshal(map[string]string{
"name": "Toby",
"email": "Toby@example.com",
"username": "toby",
"password": "Test123",
})
if err != nil {
t.Fail()
}
req, _ := http.NewRequest("POST", "/api/auth/register", bytes.NewBuffer(body))
rr := httptest.NewRecorder()
r := Route()
r.Use(func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
context.Set(r, "store", createStore())
next.ServeHTTP(w, r)
})
})
r.ServeHTTP(rr, req)
if rr.Code != 204 {
t.Errorf("Response code should be 204 but got %d", rr.Code)
}
}

39
api/projects/environment.go
View File

@ -31,6 +31,17 @@ func EnvironmentMiddleware(next http.Handler) http.Handler {
})
}
func GetEnvironmentRefs(w http.ResponseWriter, r *http.Request) {
env := context.Get(r, "environment").(db.Environment)
refs, err := helpers.Store(r).GetEnvironmentRefs(env.ProjectID, env.ID)
if err != nil {
helpers.WriteError(w, err)
return
}
helpers.WriteJSON(w, http.StatusOK, refs)
}
// GetEnvironment retrieves sorted environments from the database
func GetEnvironment(w http.ResponseWriter, r *http.Request) {
@ -61,9 +72,9 @@ func UpdateEnvironment(w http.ResponseWriter, r *http.Request) {
}
if env.ID != oldEnv.ID {
helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
"error": "Environment ID in body and URL must be the same",
})
helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{
"error": "Environment ID in body and URL must be the same",
})
return
}
@ -109,7 +120,7 @@ func AddEnvironment(w http.ResponseWriter, r *http.Request) {
desc := "Environment " + newEnv.Name + " created"
_, err = helpers.Store(r).CreateEvent(db.Event{
UserID: &user.ID,
UserID: &user.ID,
ProjectID: &newEnv.ID,
ObjectType: &objType,
ObjectID: &newEnv.ID,
@ -129,19 +140,13 @@ func RemoveEnvironment(w http.ResponseWriter, r *http.Request) {
var err error
softDeletion := r.URL.Query().Get("setRemoved") == "1"
if softDeletion {
err = helpers.Store(r).DeleteEnvironmentSoft(env.ProjectID, env.ID)
} else {
err = helpers.Store(r).DeleteEnvironment(env.ProjectID, env.ID)
if err == db.ErrInvalidOperation {
helpers.WriteJSON(w, http.StatusBadRequest, map[string]interface{}{
"error": "Environment is in use by one or more templates",
"inUse": true,
})
return
}
err = helpers.Store(r).DeleteEnvironment(env.ProjectID, env.ID)
if err == db.ErrInvalidOperation {
helpers.WriteJSON(w, http.StatusBadRequest, map[string]interface{}{
"error": "Environment is in use by one or more templates",
"inUse": true,
})
return
}
if err != nil {

31
api/projects/inventory.go
View File

@ -34,6 +34,17 @@ func InventoryMiddleware(next http.Handler) http.Handler {
})
}
func GetInventoryRefs(w http.ResponseWriter, r *http.Request) {
inventory := context.Get(r, "inventory").(db.Inventory)
refs, err := helpers.Store(r).GetInventoryRefs(inventory.ProjectID, inventory.ID)
if err != nil {
helpers.WriteError(w, err)
return
}
helpers.WriteJSON(w, http.StatusOK, refs)
}
// GetInventory returns an inventory from the database
func GetInventory(w http.ResponseWriter, r *http.Request) {
if inventory := context.Get(r, "inventory"); inventory != nil {
@ -180,19 +191,13 @@ func RemoveInventory(w http.ResponseWriter, r *http.Request) {
inventory := context.Get(r, "inventory").(db.Inventory)
var err error
softDeletion := r.URL.Query().Get("setRemoved") == "1"
if softDeletion {
err = helpers.Store(r).DeleteInventorySoft(inventory.ProjectID, inventory.ID)
} else {
err = helpers.Store(r).DeleteInventory(inventory.ProjectID, inventory.ID)
if err == db.ErrInvalidOperation {
helpers.WriteJSON(w, http.StatusBadRequest, map[string]interface{}{
"error": "Inventory is in use by one or more templates",
"inUse": true,
})
return
}
err = helpers.Store(r).DeleteInventory(inventory.ProjectID, inventory.ID)
if err == db.ErrInvalidOperation {
helpers.WriteJSON(w, http.StatusBadRequest, map[string]interface{}{
"error": "Inventory is in use by one or more templates",
"inUse": true,
})
return
}
if err != nil {

53
api/projects/keys.go
View File

@ -30,6 +30,17 @@ func KeyMiddleware(next http.Handler) http.Handler {
})
}
func GetKeyRefs(w http.ResponseWriter, r *http.Request) {
key := context.Get(r, "accessKey").(db.AccessKey)
refs, err := helpers.Store(r).GetAccessKeyRefs(*key.ProjectID, key.ID)
if err != nil {
helpers.WriteError(w, err)
return
}
helpers.WriteJSON(w, http.StatusOK, refs)
}
// GetKeys retrieves sorted keys from the database
func GetKeys(w http.ResponseWriter, r *http.Request) {
if key := context.Get(r, "accessKey"); key != nil {
@ -111,7 +122,25 @@ func UpdateKey(w http.ResponseWriter, r *http.Request) {
return
}
if err := helpers.Store(r).UpdateAccessKey(key); err != nil {
repos, err := helpers.Store(r).GetRepositories(*key.ProjectID, db.RetrieveQueryParams{})
if err != nil {
helpers.WriteError(w, err)
return
}
for _, repo := range repos {
if repo.SSHKeyID != key.ID {
continue
}
err = repo.ClearCache()
if err != nil {
helpers.WriteError(w, err)
return
}
}
err = helpers.Store(r).UpdateAccessKey(key)
if err != nil {
helpers.WriteError(w, err)
return
}
@ -121,7 +150,7 @@ func UpdateKey(w http.ResponseWriter, r *http.Request) {
desc := "Access Key " + key.Name + " updated"
objType := db.EventKey
_, err := helpers.Store(r).CreateEvent(db.Event{
_, err = helpers.Store(r).CreateEvent(db.Event{
UserID: &user.ID,
ProjectID: oldKey.ProjectID,
Description: &desc,
@ -143,19 +172,13 @@ func RemoveKey(w http.ResponseWriter, r *http.Request) {
var err error
softDeletion := r.URL.Query().Get("setRemoved") == "1"
if softDeletion {
err = helpers.Store(r).DeleteAccessKeySoft(*key.ProjectID, key.ID)
} else {
err = helpers.Store(r).DeleteAccessKey(*key.ProjectID, key.ID)
if err == db.ErrInvalidOperation {
helpers.WriteJSON(w, http.StatusBadRequest, map[string]interface{}{
"error": "Access Key is in use by one or more templates",
"inUse": true,
})
return
}
err = helpers.Store(r).DeleteAccessKey(*key.ProjectID, key.ID)
if err == db.ErrInvalidOperation {
helpers.WriteJSON(w, http.StatusBadRequest, map[string]interface{}{
"error": "Access Key is in use by one or more templates",
"inUse": true,
})
return
}
if err != nil {

5
api/projects/projects.go
View File

@ -29,6 +29,11 @@ func AddProject(w http.ResponseWriter, r *http.Request) {
user := context.Get(r, "user").(*db.User)
if !user.Admin {
log.Warn(user.Username + " is not permitted to edit users")
w.WriteHeader(http.StatusUnauthorized)
return
}
if !helpers.Bind(w, r, &body) {
return

68
api/projects/repository.go
View File

@ -4,40 +4,11 @@ import (
log "github.com/Sirupsen/logrus"
"github.com/ansible-semaphore/semaphore/api/helpers"
"github.com/ansible-semaphore/semaphore/db"
"net/http"
"os"
"path/filepath"
"strconv"
"github.com/ansible-semaphore/semaphore/util"
"github.com/gorilla/context"
"net/http"
)
func removeAllByPattern(path string, filenamePattern string) error {
d, err := os.Open(path)
if err != nil {
return err
}
defer d.Close()
names, err := d.Readdirnames(-1)
if err != nil {
return err
}
for _, name := range names {
if matched, _ := filepath.Match(filenamePattern, name); !matched {
continue
}
if err := os.RemoveAll(filepath.Join(path, name)); err != nil {
return err
}
}
return nil
}
func clearRepositoryCache(repository db.Repository) error {
return removeAllByPattern(util.Config.TmpPath, "repository_" + strconv.Itoa(repository.ID) + "_*")
}
// RepositoryMiddleware ensures a repository exists and loads it to the context
func RepositoryMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -59,6 +30,17 @@ func RepositoryMiddleware(next http.Handler) http.Handler {
})
}
func GetRepositoryRefs(w http.ResponseWriter, r *http.Request) {
repo := context.Get(r, "repository").(db.Repository)
refs, err := helpers.Store(r).GetRepositoryRefs(repo.ProjectID, repo.ID)
if err != nil {
helpers.WriteError(w, err)
return
}
helpers.WriteJSON(w, http.StatusOK, refs)
}
// GetRepositories returns all repositories in a project sorted by type
func GetRepositories(w http.ResponseWriter, r *http.Request) {
if repo := context.Get(r, "repository"); repo != nil {
@ -152,7 +134,7 @@ func UpdateRepository(w http.ResponseWriter, r *http.Request) {
}
if oldRepo.GitURL != repository.GitURL {
util.LogWarning(clearRepositoryCache(oldRepo))
util.LogWarning(oldRepo.ClearCache())
}
user := context.Get(r, "user").(*db.User)
@ -161,7 +143,7 @@ func UpdateRepository(w http.ResponseWriter, r *http.Request) {
objType := db.EventRepository
_, err = helpers.Store(r).CreateEvent(db.Event{
UserID: &user.ID,
UserID: &user.ID,
ProjectID: &repository.ProjectID,
Description: &desc,
ObjectID: &repository.ID,
@ -181,19 +163,13 @@ func RemoveRepository(w http.ResponseWriter, r *http.Request) {
var err error
softDeletion := r.URL.Query().Get("setRemoved") == "1"
if softDeletion {
err = helpers.Store(r).DeleteRepositorySoft(repository.ProjectID, repository.ID)
} else {
err = helpers.Store(r).DeleteRepository(repository.ProjectID, repository.ID)
if err == db.ErrInvalidOperation {
helpers.WriteJSON(w, http.StatusBadRequest, map[string]interface{}{
"error": "Repository is in use by one or more templates",
"inUse": true,
})
return
}
err = helpers.Store(r).DeleteRepository(repository.ProjectID, repository.ID)
if err == db.ErrInvalidOperation {
helpers.WriteJSON(w, http.StatusBadRequest, map[string]interface{}{
"error": "Repository is in use by one or more templates",
"inUse": true,
})
return
}
if err != nil {
@ -201,7 +177,7 @@ func RemoveRepository(w http.ResponseWriter, r *http.Request) {
return
}
util.LogWarning(clearRepositoryCache(repository))
util.LogWarning(repository.ClearCache())
user := context.Get(r, "user").(*db.User)
desc := "Repository (" + repository.GitURL + ") deleted"

6
api/projects/schedules.go
View File

@ -3,8 +3,8 @@ package projects
import (
log "github.com/Sirupsen/logrus"
"github.com/ansible-semaphore/semaphore/api/helpers"
"github.com/ansible-semaphore/semaphore/api/schedules"
"github.com/ansible-semaphore/semaphore/db"
"github.com/ansible-semaphore/semaphore/services/schedules"
"github.com/gorilla/context"
"net/http"
"strconv"
@ -33,7 +33,7 @@ func SchedulesMiddleware(next http.Handler) http.Handler {
func refreshSchedulePool(r *http.Request) {
pool := context.Get(r, "schedule_pool").(schedules.SchedulePool)
pool.Refresh(helpers.Store(r))
pool.Refresh()
}
// GetSchedule returns single template by ID
@ -202,4 +202,4 @@ func RemoveSchedule(w http.ResponseWriter, r *http.Request) {
refreshSchedulePool(r)
w.WriteHeader(http.StatusNoContent)
}
}

163
api/tasks/http.go → api/projects/tasks.go
View File

@ -1,127 +1,15 @@
package tasks
package projects
import (
log "github.com/Sirupsen/logrus"
"github.com/ansible-semaphore/semaphore/api/helpers"
"github.com/ansible-semaphore/semaphore/db"
"net/http"
"regexp"
"strconv"
"strings"
"time"
log "github.com/Sirupsen/logrus"
"github.com/ansible-semaphore/semaphore/util"
"github.com/gorilla/context"
"net/http"
"strconv"
)
func getNextBuildVersion(startVersion string, currentVersion string) string {
re := regexp.MustCompile(`^(.*[^\d])?(\d+)([^\d].*)?$`)
m := re.FindStringSubmatch(startVersion)
if m == nil {
return startVersion
}
var prefix, suffix, body string
switch len(m) - 1 {
case 3:
prefix = m[1]
body = m[2]
suffix = m[3]
case 2:
if _, err := strconv.Atoi(m[1]); err == nil {
body = m[1]
suffix = m[2]
} else {
prefix = m[1]
body = m[2]
}
case 1:
body = m[1]
default:
return startVersion
}
if !strings.HasPrefix(currentVersion, prefix) ||
!strings.HasSuffix(currentVersion, suffix) {
return startVersion
}
curr, err := strconv.Atoi(currentVersion[len(prefix) : len(currentVersion)-len(suffix)])
if err != nil {
return startVersion
}
start, err := strconv.Atoi(body)
if err != nil {
panic(err)
}
var newVer int
if start > curr {
newVer = start
} else {
newVer = curr + 1
}
return prefix + strconv.Itoa(newVer) + suffix
}
func AddTaskToPool(d db.Store, taskObj db.Task, userID *int, projectID int) (newTask db.Task, err error) {
taskObj.Created = time.Now()
taskObj.Status = taskWaitingStatus
taskObj.UserID = userID
taskObj.ProjectID = projectID
tpl, err := d.GetTemplate(projectID, taskObj.TemplateID)
if err != nil {
return
}
err = taskObj.ValidateNewTask(tpl)
if err != nil {
return
}
if tpl.Type == db.TemplateBuild { // get next version for task if it is a Build
var builds []db.TaskWithTpl
builds, err = d.GetTemplateTasks(tpl, db.RetrieveQueryParams{Count: 1})
if err != nil {
return
}
if len(builds) == 0 {
taskObj.Version = tpl.StartVersion
} else {
v := getNextBuildVersion(*tpl.StartVersion, *builds[0].Version)
taskObj.Version = &v
}
}
newTask, err = d.CreateTask(taskObj)
if err != nil {
return
}
pool.register <- &task{
store: d,
task: newTask,
projectID: projectID,
}
objType := db.EventTask
desc := "Task ID " + strconv.Itoa(newTask.ID) + " queued for running"
_, err = d.CreateEvent(db.Event{
UserID: userID,
ProjectID: &projectID,
ObjectType: &objType,
ObjectID: &newTask.ID,
Description: &desc,
})
return
}
// AddTask inserts a task into the database and returns a header or returns error
func AddTask(w http.ResponseWriter, r *http.Request) {
project := context.Get(r, "project").(db.Project)
@ -133,7 +21,7 @@ func AddTask(w http.ResponseWriter, r *http.Request) {
return
}
newTask, err := AddTaskToPool(helpers.Store(r), taskObj, &user.ID, project.ID)
newTask, err := helpers.TaskPool(r).AddTask(taskObj, &user.ID, project.ID)
if err != nil {
util.LogErrorWithFields(err, log.Fields{"error": "Cannot write new event to database"})
@ -153,7 +41,7 @@ func GetTasksList(w http.ResponseWriter, r *http.Request, limit uint64) {
var tasks []db.TaskWithTpl
if tpl != nil {
tasks, err = helpers.Store(r).GetTemplateTasks(tpl.(db.Template), db.RetrieveQueryParams{
tasks, err = helpers.Store(r).GetTemplateTasks(tpl.(db.Template).ProjectID, tpl.(db.Template).ID, db.RetrieveQueryParams{
Count: int(limit),
})
} else {
@ -180,7 +68,7 @@ func GetAllTasks(w http.ResponseWriter, r *http.Request) {
func GetLastTasks(w http.ResponseWriter, r *http.Request) {
str := r.URL.Query().Get("limit")
limit, err := strconv.Atoi(str)
if err != nil || limit <= 0 || limit > 200 {
if err != nil || limit <= 0 || limit > 200 {
limit = 200
}
GetTasksList(w, r, uint64(limit))
@ -237,34 +125,15 @@ func StopTask(w http.ResponseWriter, r *http.Request) {
targetTask := context.Get(r, "task").(db.Task)
project := context.Get(r, "project").(db.Project)
activeTask := pool.getTask(targetTask.ID)
if targetTask.ProjectID != project.ID {
w.WriteHeader(http.StatusBadRequest)
return
}
if activeTask == nil { // task not active, but exists in database
activeTask = &task{
store: helpers.Store(r),
task: targetTask,
projectID: project.ID,
}
err := activeTask.populateDetails()
if err != nil {
helpers.WriteError(w, err)
return
}
activeTask.setStatus(taskStoppedStatus)
activeTask.createTaskEvent()
} else {
if activeTask.task.Status == taskRunningStatus {
if activeTask.process == nil {
panic("running process can not be nil")
}
if err := activeTask.process.Kill(); err != nil {
helpers.WriteError(w, err)
}
}
activeTask.setStatus(taskStoppingStatus)
err := helpers.TaskPool(r).StopTask(targetTask)
if err != nil {
helpers.WriteError(w, err)
return
}
w.WriteHeader(http.StatusNoContent)
@ -276,7 +145,7 @@ func RemoveTask(w http.ResponseWriter, r *http.Request) {
editor := context.Get(r, "user").(*db.User)
project := context.Get(r, "project").(db.Project)
activeTask := pool.getTask(targetTask.ID)
activeTask := helpers.TaskPool(r).GetTask(targetTask.ID)
if activeTask != nil {
// can't delete task in queue or running

15
api/projects/templates.go
View File

@ -36,11 +36,22 @@ func GetTemplate(w http.ResponseWriter, r *http.Request) {
helpers.WriteJSON(w, http.StatusOK, template)
}
func GetTemplateRefs(w http.ResponseWriter, r *http.Request) {
tpl := context.Get(r, "template").(db.Template)
refs, err := helpers.Store(r).GetTemplateRefs(tpl.ProjectID, tpl.ID)
if err != nil {
helpers.WriteError(w, err)
return
}
helpers.WriteJSON(w, http.StatusOK, refs)
}
// GetTemplates returns all templates for a project in a sort order
func GetTemplates(w http.ResponseWriter, r *http.Request) {
project := context.Get(r, "project").(db.Project)
templates, err := helpers.Store(r).GetTemplates(project.ID, helpers.QueryParams(r.URL))
templates, err := helpers.Store(r).GetTemplates(project.ID, db.TemplateFilter{}, helpers.QueryParams(r.URL))
if err != nil {
helpers.WriteError(w, err)
@ -164,4 +175,4 @@ func RemoveTemplate(w http.ResponseWriter, r *http.Request) {
}
w.WriteHeader(http.StatusNoContent)
}
}

3
api/projects/views.go
View File

@ -34,7 +34,7 @@ func GetViewTemplates(w http.ResponseWriter, r *http.Request) {
project := context.Get(r, "project").(db.Project)
view := context.Get(r, "view").(db.View)
templates, err := helpers.Store(r).GetViewTemplates(project.ID, view.ID, helpers.QueryParams(r.URL))
templates, err := helpers.Store(r).GetTemplates(project.ID, db.TemplateFilter{ViewID: &view.ID}, helpers.QueryParams(r.URL))
if err != nil {
helpers.WriteError(w, err)
@ -134,7 +134,6 @@ func SetViewPositions(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
// UpdateView updates key in database
// nolint: gocyclo
func UpdateView(w http.ResponseWriter, r *http.Request) {

45
api/router.go
View File

@ -9,8 +9,6 @@ import (
"github.com/ansible-semaphore/semaphore/api/projects"
"github.com/ansible-semaphore/semaphore/api/sockets"
"github.com/ansible-semaphore/semaphore/api/tasks"
"github.com/ansible-semaphore/semaphore/util"
"github.com/gobuffalo/packr"
"github.com/gorilla/mux"
@ -70,8 +68,6 @@ func Route() *mux.Router {
publicAPIRouter := r.PathPrefix(webPath + "api").Subrouter()
publicAPIRouter.Use(JSONMiddleware)
publicAPIRouter.HandleFunc("/auth/info", info).Methods("GET")
publicAPIRouter.HandleFunc("/auth/register", register).Methods("POST")
publicAPIRouter.HandleFunc("/auth/login", login).Methods("POST")
publicAPIRouter.HandleFunc("/auth/logout", logout).Methods("POST")
@ -88,7 +84,6 @@ func Route() *mux.Router {
authenticatedAPI.Path("/users").HandlerFunc(getUsers).Methods("GET", "HEAD")
authenticatedAPI.Path("/users").HandlerFunc(addUser).Methods("POST")
authenticatedAPI.Path("/user").HandlerFunc(getUser).Methods("GET", "HEAD")
tokenAPI := authenticatedAPI.PathPrefix("/user").Subrouter()
@ -131,9 +126,9 @@ func Route() *mux.Router {
projectUserAPI.Path("/environment").HandlerFunc(projects.GetEnvironment).Methods("GET", "HEAD")
projectUserAPI.Path("/environment").HandlerFunc(projects.AddEnvironment).Methods("POST")
projectUserAPI.Path("/tasks").HandlerFunc(tasks.GetAllTasks).Methods("GET", "HEAD")
projectUserAPI.HandleFunc("/tasks/last", tasks.GetLastTasks).Methods("GET", "HEAD")
projectUserAPI.Path("/tasks").HandlerFunc(tasks.AddTask).Methods("POST")
projectUserAPI.Path("/tasks").HandlerFunc(projects.GetAllTasks).Methods("GET", "HEAD")
projectUserAPI.HandleFunc("/tasks/last", projects.GetLastTasks).Methods("GET", "HEAD")
projectUserAPI.Path("/tasks").HandlerFunc(projects.AddTask).Methods("POST")
projectUserAPI.Path("/templates").HandlerFunc(projects.GetTemplates).Methods("GET", "HEAD")
projectUserAPI.Path("/templates").HandlerFunc(projects.AddTemplate).Methods("POST")
@ -162,10 +157,11 @@ func Route() *mux.Router {
projectUserManagement.HandleFunc("/{user_id}/admin", projects.MakeUserAdmin).Methods("DELETE")
projectUserManagement.HandleFunc("/{user_id}", projects.RemoveUser).Methods("DELETE")
projectKeyManagement := projectAdminUsersAPI.PathPrefix("/keys").Subrouter()
projectKeyManagement := projectUserAPI.PathPrefix("/keys").Subrouter()
projectKeyManagement.Use(projects.KeyMiddleware)
projectKeyManagement.HandleFunc("/{key_id}", projects.GetKeys).Methods("GET", "HEAD")
projectKeyManagement.HandleFunc("/{key_id}/refs", projects.GetKeyRefs).Methods("GET", "HEAD")
projectKeyManagement.HandleFunc("/{key_id}", projects.UpdateKey).Methods("PUT")
projectKeyManagement.HandleFunc("/{key_id}", projects.RemoveKey).Methods("DELETE")
@ -173,6 +169,7 @@ func Route() *mux.Router {
projectRepoManagement.Use(projects.RepositoryMiddleware)
projectRepoManagement.HandleFunc("/{repository_id}", projects.GetRepositories).Methods("GET", "HEAD")
projectRepoManagement.HandleFunc("/{repository_id}/refs", projects.GetRepositoryRefs).Methods("GET", "HEAD")
projectRepoManagement.HandleFunc("/{repository_id}", projects.UpdateRepository).Methods("PUT")
projectRepoManagement.HandleFunc("/{repository_id}", projects.RemoveRepository).Methods("DELETE")
@ -180,6 +177,7 @@ func Route() *mux.Router {
projectInventoryManagement.Use(projects.InventoryMiddleware)
projectInventoryManagement.HandleFunc("/{inventory_id}", projects.GetInventory).Methods("GET", "HEAD")
projectInventoryManagement.HandleFunc("/{inventory_id}/refs", projects.GetInventoryRefs).Methods("GET", "HEAD")
projectInventoryManagement.HandleFunc("/{inventory_id}", projects.UpdateInventory).Methods("PUT")
projectInventoryManagement.HandleFunc("/{inventory_id}", projects.RemoveInventory).Methods("DELETE")
@ -187,6 +185,7 @@ func Route() *mux.Router {
projectEnvManagement.Use(projects.EnvironmentMiddleware)
projectEnvManagement.HandleFunc("/{environment_id}", projects.GetEnvironment).Methods("GET", "HEAD")
projectEnvManagement.HandleFunc("/{environment_id}/refs", projects.GetEnvironmentRefs).Methods("GET", "HEAD")
projectEnvManagement.HandleFunc("/{environment_id}", projects.UpdateEnvironment).Methods("PUT")
projectEnvManagement.HandleFunc("/{environment_id}", projects.RemoveEnvironment).Methods("DELETE")
@ -196,17 +195,18 @@ func Route() *mux.Router {
projectTmplManagement.HandleFunc("/{template_id}", projects.UpdateTemplate).Methods("PUT")
projectTmplManagement.HandleFunc("/{template_id}", projects.RemoveTemplate).Methods("DELETE")
projectTmplManagement.HandleFunc("/{template_id}", projects.GetTemplate).Methods("GET")
projectTmplManagement.HandleFunc("/{template_id}/tasks", tasks.GetAllTasks).Methods("GET")
projectTmplManagement.HandleFunc("/{template_id}/tasks/last", tasks.GetLastTasks).Methods("GET")
projectTmplManagement.HandleFunc("/{template_id}/refs", projects.GetTemplateRefs).Methods("GET", "HEAD")
projectTmplManagement.HandleFunc("/{template_id}/tasks", projects.GetAllTasks).Methods("GET")
projectTmplManagement.HandleFunc("/{template_id}/tasks/last", projects.GetLastTasks).Methods("GET")
projectTmplManagement.HandleFunc("/{template_id}/schedules", projects.GetTemplateSchedules).Methods("GET")
projectTaskManagement := projectUserAPI.PathPrefix("/tasks").Subrouter()
projectTaskManagement.Use(tasks.GetTaskMiddleware)
projectTaskManagement.Use(projects.GetTaskMiddleware)
projectTaskManagement.HandleFunc("/{task_id}/output", tasks.GetTaskOutput).Methods("GET", "HEAD")
projectTaskManagement.HandleFunc("/{task_id}", tasks.GetTask).Methods("GET", "HEAD")
projectTaskManagement.HandleFunc("/{task_id}", tasks.RemoveTask).Methods("DELETE")
projectTaskManagement.HandleFunc("/{task_id}/stop", tasks.StopTask).Methods("POST")
projectTaskManagement.HandleFunc("/{task_id}/output", projects.GetTaskOutput).Methods("GET", "HEAD")
projectTaskManagement.HandleFunc("/{task_id}", projects.GetTask).Methods("GET", "HEAD")
projectTaskManagement.HandleFunc("/{task_id}", projects.RemoveTask).Methods("DELETE")
projectTaskManagement.HandleFunc("/{task_id}/stop", projects.StopTask).Methods("POST")
projectScheduleManagement := projectUserAPI.PathPrefix("/schedules").Subrouter()
projectScheduleManagement.Use(projects.SchedulesMiddleware)
@ -338,16 +338,16 @@ func getSystemInfo(w http.ResponseWriter, r *http.Request) {
return
}
updateAvailable, err := util.CheckUpdate()
//updateAvailable, err := util.CheckUpdate()
if err != nil {
helpers.WriteError(w, err)
return
}
//if err != nil {
// helpers.WriteError(w, err)
// return
//}
body := map[string]interface{}{
"version": util.Version,
"update": updateAvailable,
//"update": updateAvailable,
"config": map[string]string{
"dbHost": dbConfig.Hostname,
"dbName": dbConfig.DbName,
@ -356,6 +356,7 @@ func getSystemInfo(w http.ResponseWriter, r *http.Request) {
"cmdPath": util.FindSemaphore(),
},
"ansible": util.AnsibleVersion(),
"demo": util.Config.DemoMode,
}
helpers.WriteJSON(w, http.StatusOK, body)

97
api/schedules/pool.go
View File

@ -1,97 +0,0 @@
package schedules
import (
log "github.com/Sirupsen/logrus"
"github.com/ansible-semaphore/semaphore/api/tasks"
"github.com/ansible-semaphore/semaphore/db"
"github.com/robfig/cron/v3"
"sync"
)
type ScheduleRunner struct {
Store db.Store
Schedule db.Schedule
}
func (r ScheduleRunner) Run() {
_, err := tasks.AddTaskToPool(r.Store, db.Task{
TemplateID: r.Schedule.TemplateID,
ProjectID: r.Schedule.ProjectID,
}, nil, r.Schedule.ProjectID)
if err != nil {
log.Error(err)
}
}
type SchedulePool struct {
cron *cron.Cron
locker sync.Locker
}
func (p *SchedulePool) init() {
p.cron = cron.New()
p.locker = &sync.Mutex{}
}
func (p *SchedulePool) Refresh(d db.Store) {
defer p.locker.Unlock()
schedules, err := d.GetSchedules()
if err != nil {
log.Error(err)
return
}
p.locker.Lock()
p.clear()
for _, schedule := range schedules {
_, err := p.addRunner(ScheduleRunner{
Store: d,
Schedule: schedule,
})
if err != nil {
log.Error(err)
}
}
}
func (p *SchedulePool) addRunner(runner ScheduleRunner) (int, error) {
id, err := p.cron.AddJob(runner.Schedule.CronFormat, runner)
if err != nil {
return 0, err
}
return int(id), nil
}
func (p *SchedulePool) Run() {
p.cron.Run()
}
func (p *SchedulePool) clear() {
runners := p.cron.Entries()
for _, r := range runners {
p.cron.Remove(r.ID)
}
}
func (p *SchedulePool) Destroy() {
defer p.locker.Unlock()
p.locker.Lock()
p.cron.Stop()
p.clear()
p.cron = nil
}
func CreateSchedulePool(d db.Store) (pool SchedulePool) {
pool.init()
pool.Refresh(d)
return
}
func ValidateCronFormat(cronFormat string) error {
_, err := cron.ParseStandard(cronFormat)
return err
}

183
api/tasks/pool.go
View File

@ -1,183 +0,0 @@
package tasks
import (
"github.com/ansible-semaphore/semaphore/db"
"strconv"
"time"
log "github.com/Sirupsen/logrus"
"github.com/ansible-semaphore/semaphore/util"
)
type logRecord struct {
task *task
output string
time time.Time
}
type taskPool struct {
queue []*task
register chan *task
activeProj map[int]*task
activeNodes map[string]*task
running int
runningTasks map[int]*task
logger chan logRecord
}
type resourceLock struct {
lock bool
holder *task
}
var pool = taskPool{
queue: make([]*task, 0), // queue of waiting tasks
register: make(chan *task), // add task to queue
activeProj: make(map[int]*task),
activeNodes: make(map[string]*task),
running: 0, // number of running tasks
runningTasks: make(map[int]*task), // working tasks
logger: make(chan logRecord, 10000), // store log records to database
}
var resourceLocker = make(chan *resourceLock)
func (p *taskPool) getTask(id int) (task *task){
for _, t := range p.queue {
if t.task.ID == id {
task = t
break
}
}
if task == nil {
for _, t := range p.runningTasks {
if t.task.ID == id {
task = t
break
}
}
}
return
}
//nolint: gocyclo
func (p *taskPool) run() {
ticker := time.NewTicker(5 * time.Second)
defer func() {
close(resourceLocker)
ticker.Stop()
}()
// Lock or unlock resources when running a task
go func(locker <-chan *resourceLock) {
for l := range locker {
t := l.holder
if l.lock {
if p.blocks(t) {
panic("Trying to lock an already locked resource!")
}
p.activeProj[t.projectID] = t
for _, node := range t.hosts {
p.activeNodes[node] = t
}
p.running++
p.runningTasks[t.task.ID] = t
continue
}
if p.activeProj[t.projectID] == t {
delete(p.activeProj, t.projectID)
}
for _, node := range t.hosts {
delete(p.activeNodes, node)
}
p.running--
delete(p.runningTasks, t.task.ID)
}
}(resourceLocker)
for {
select {
case record := <-p.logger:
_, err := record.task.store.CreateTaskOutput(db.TaskOutput{
TaskID: record.task.task.ID,
Output: record.output,
Time: record.time,
})
if err != nil {
log.Error(err)
}
case task := <-p.register:
p.queue = append(p.queue, task)
log.Debug(task)
msg := "Task " + strconv.Itoa(task.task.ID) + " added to queue"
task.log(msg)
log.Info(msg)
case <-ticker.C:
if len(p.queue) == 0 {
continue
}
//get task from top of queue
t := p.queue[0]
if t.task.Status == taskFailStatus {
//delete failed task from queue
p.queue = p.queue[1:]
log.Info("Task " + strconv.Itoa(t.task.ID) + " removed from queue")
continue
}
if p.blocks(t) {
//move blocked task to end of queue
p.queue = append(p.queue[1:], t)
continue
}
log.Info("Set resource locker with task " + strconv.Itoa(t.task.ID))
resourceLocker <- &resourceLock{lock: true, holder: t}
if !t.prepared {
go t.prepareRun()
continue
}
go t.run()
p.queue = p.queue[1:]
log.Info("Task " + strconv.Itoa(t.task.ID) + " removed from queue")
}
}
}
func (p *taskPool) blocks(t *task) bool {
if p.running >= util.Config.MaxParallelTasks {
return true
}
switch util.Config.ConcurrencyMode {
case "project":
return p.activeProj[t.projectID] != nil
case "node":
for _, node := range t.hosts {
if p.activeNodes[node] != nil {
return true
}
}
return false
default:
return p.running > 0
}
}
// StartRunner begins the task pool, used as a goroutine
func StartRunner() {
pool.run()
}

868
api/tasks/runner.go
View File

@ -1,868 +0,0 @@
package tasks
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
log "github.com/Sirupsen/logrus"
"github.com/ansible-semaphore/semaphore/api/helpers"
"github.com/ansible-semaphore/semaphore/api/sockets"
"github.com/ansible-semaphore/semaphore/db"
"github.com/ansible-semaphore/semaphore/util"
)
const (
taskRunningStatus = "running"
taskWaitingStatus = "waiting"
taskStoppingStatus = "stopping"
taskStoppedStatus = "stopped"
taskSuccessStatus = "success"
taskFailStatus = "error"
gitURLFilePrefix = "file://"
)
type task struct {
store db.Store
task db.Task
template db.Template
inventory db.Inventory
repository db.Repository
environment db.Environment
users []int
projectID int
hosts []string
alertChat string
alert bool
prepared bool
process *os.Process
}
func (t *task) getRepoName() string {
return "repository_" + strconv.Itoa(t.repository.ID) + "_" + strconv.Itoa(t.template.ID)
}
func (t *task) getRepoPath() string {
return util.Config.TmpPath + "/" + t.getRepoName()
}
func (t *task) validateRepo() error {
path := t.getRepoPath()
_, err := os.Stat(path)
return err
}
func (t *task) setStatus(status string) {
if t.task.Status == taskStoppingStatus {
switch status {
case taskFailStatus:
status = taskStoppedStatus
case taskStoppedStatus:
default:
panic("stopping task cannot be " + status)
}
}
t.task.Status = status
t.updateStatus()
if status == taskFailStatus {
t.sendMailAlert()
}
if status == taskSuccessStatus || status == taskFailStatus {
t.sendTelegramAlert()
}
}
func (t *task) updateStatus() {
for _, user := range t.users {
b, err := json.Marshal(&map[string]interface{}{
"type": "update",
"start": t.task.Start,
"end": t.task.End,
"status": t.task.Status,
"task_id": t.task.ID,
"template_id": t.task.TemplateID,
"project_id": t.projectID,
"version": t.task.Version,
})
util.LogPanic(err)
sockets.Message(user, b)
}
if err := t.store.UpdateTask(t.task); err != nil {
t.panicOnError(err, "Failed to update task status")
}
}
func (t *task) fail() {
t.setStatus(taskFailStatus)
}
func (t *task) destroyKeys() {
err := t.repository.SSHKey.Destroy()
if err != nil {
t.log("Can't destroy repository key, error: " + err.Error())
}
err = t.inventory.SSHKey.Destroy()
if err != nil {
t.log("Can't destroy inventory user key, error: " + err.Error())
}
err = t.inventory.BecomeKey.Destroy()
if err != nil {
t.log("Can't destroy inventory become user key, error: " + err.Error())
}
err = t.template.VaultKey.Destroy()
if err != nil {
t.log("Can't destroy inventory vault password file, error: " + err.Error())
}
}
func (t *task) createTaskEvent() {
objType := db.EventTask
desc := "Task ID " + strconv.Itoa(t.task.ID) + " (" + t.template.Alias + ")" + " finished - " + strings.ToUpper(t.task.Status)
_, err := t.store.CreateEvent(db.Event{
UserID: t.task.UserID,
ProjectID: &t.projectID,
ObjectType: &objType,
ObjectID: &t.task.ID,
Description: &desc,
})
if err != nil {
t.panicOnError(err, "Fatal error inserting an event")
}
}
func (t *task) prepareRun() {
t.prepared = false
defer func() {
log.Info("Stopped preparing task " + strconv.Itoa(t.task.ID))
log.Info("Release resource locker with task " + strconv.Itoa(t.task.ID))
resourceLocker <- &resourceLock{lock: false, holder: t}
t.createTaskEvent()
}()
t.log("Preparing: " + strconv.Itoa(t.task.ID))
err := checkTmpDir(util.Config.TmpPath)
if err != nil {
t.log("Creating tmp dir failed: " + err.Error())
t.fail()
return
}
if err := t.populateDetails(); err != nil {
t.log("Error: " + err.Error())
t.fail()
return
}
objType := db.EventTask
desc := "Task ID " + strconv.Itoa(t.task.ID) + " (" + t.template.Alias + ")" + " is preparing"
_, err = t.store.CreateEvent(db.Event{
UserID: t.task.UserID,
ProjectID: &t.projectID,
ObjectType: &objType,
ObjectID: &t.task.ID,
Description: &desc,
})
if err != nil {
t.log("Fatal error inserting an event")
panic(err)
}
t.log("Prepare task with template: " + t.template.Alias + "\n")
t.updateStatus()
//if err := t.installKey(t.repository.SSHKey, db.AccessKeyUsagePrivateKey); err != nil {
if err := t.repository.SSHKey.Install(db.AccessKeyUsagePrivateKey); err != nil {
t.log("Failed installing ssh key for repository access: " + err.Error())
t.fail()
return
}
if strings.HasPrefix(t.repository.GitURL, gitURLFilePrefix) {
repositoryPath := strings.TrimPrefix(t.repository.GitURL, gitURLFilePrefix)
if _, err := os.Stat(repositoryPath); err != nil {
t.log("Failed in finding static repository at " + repositoryPath + ": " + err.Error())
t.fail()
return
}
} else {
if err := t.updateRepository(); err != nil {
t.log("Failed updating repository: " + err.Error())
t.fail()
return
}
}
if err := t.checkoutRepository(); err != nil {
t.log("Failed to checkout repository to required commit: " + err.Error())
t.fail()
return
}
if err := t.installInventory(); err != nil {
t.log("Failed to install inventory: " + err.Error())
t.fail()
return
}
if err := t.installRequirements(); err != nil {
t.log("Running galaxy failed: " + err.Error())
t.fail()
return
}
if err := t.installVaultKeyFile(); err != nil {
t.log("Failed to install vault password file: " + err.Error())
t.fail()
return
}
// todo: write environment
if stderr, err := t.listPlaybookHosts(); err != nil {
t.log("Listing playbook hosts failed: " + err.Error() + "\n" + stderr)
t.fail()
return
}
t.prepared = true
}
func (t *task) run() {
defer func() {
log.Info("Stopped running task " + strconv.Itoa(t.task.ID))
log.Info("Release resource locker with task " + strconv.Itoa(t.task.ID))
resourceLocker <- &resourceLock{lock: false, holder: t}
now := time.Now()
t.task.End = &now
t.updateStatus()
t.createTaskEvent()
t.destroyKeys()
}()
if t.task.Status == taskStoppingStatus {
t.setStatus(taskStoppedStatus)
return
}
now := time.Now()
t.task.Start = &now
t.setStatus(taskRunningStatus)
objType := db.EventTask
desc := "Task ID " + strconv.Itoa(t.task.ID) + " (" + t.template.Alias + ")" + " is running"
_, err := t.store.CreateEvent(db.Event{
UserID: t.task.UserID,
ProjectID: &t.projectID,
ObjectType: &objType,
ObjectID: &t.task.ID,
Description: &desc,
})
if err != nil {
t.log("Fatal error inserting an event")
panic(err)
}
t.log("Started: " + strconv.Itoa(t.task.ID))
t.log("Run task with template: " + t.template.Alias + "\n")
if t.task.Status == taskStoppingStatus {
t.setStatus(taskStoppedStatus)
return
}
if err := t.runPlaybook(); err != nil {
t.log("Running playbook failed: " + err.Error())
t.fail()
return
}
t.setStatus(taskSuccessStatus)
}
func (t *task) prepareError(err error, errMsg string) error {
if err == db.ErrNotFound {
t.log(errMsg)
return err
}
if err != nil {
t.fail()
panic(err)
}
return nil
}
//nolint: gocyclo
func (t *task) populateDetails() error {
// get template
var err error
t.template, err = t.store.GetTemplate(t.projectID, t.task.TemplateID)
if err != nil {
return t.prepareError(err, "Template not found!")
}
// get project alert setting
project, err := t.store.GetProject(t.template.ProjectID)
if err != nil {
return t.prepareError(err, "Project not found!")
}
t.alert = project.Alert
t.alertChat = project.AlertChat
// get project users
users, err := t.store.GetProjectUsers(t.template.ProjectID, db.RetrieveQueryParams{})
if err != nil {
return t.prepareError(err, "Users not found!")
}
t.users = []int{}
for _, user := range users {
t.users = append(t.users, user.ID)
}
// get inventory
t.inventory, err = t.store.GetInventory(t.template.ProjectID, t.template.InventoryID)
if err != nil {
return t.prepareError(err, "Template Inventory not found!")
}
// get repository
t.repository, err = t.store.GetRepository(t.template.ProjectID, t.template.RepositoryID)
if err != nil {
return err
}
// get environment
if t.template.EnvironmentID != nil {
t.environment, err = t.store.GetEnvironment(t.template.ProjectID, *t.template.EnvironmentID)
if err != nil {
return err
}
}
if t.task.Environment != "" {
environment := make(map[string]interface{})
if t.environment.JSON != "" {
err = json.Unmarshal([]byte(t.task.Environment), &environment)
if err != nil {
return err
}
}
taskEnvironment := make(map[string]interface{})
err = json.Unmarshal([]byte(t.environment.JSON), &taskEnvironment)
if err != nil {
return err
}
for k, v := range taskEnvironment {
environment[k] = v
}
var ev []byte
ev, err = json.Marshal(environment)
if err != nil {
return err
}
t.environment.JSON = string(ev)
}
return nil
}
func (t *task) installVaultKeyFile() error {
if t.template.VaultKeyID == nil {
return nil
}
return t.template.VaultKey.Install(db.AccessKeyUsageVault)
}
//func (t *task) installKey(key db.AccessKey, accessKeyUsage int) error {
// if key.Type != db.AccessKeySSH {
// return nil
// }
//
// t.log("access key " + key.Name + " installed")
//
// path := key.GetPath()
//
// err := key.DeserializeSecret()
//
// if err != nil {
// return err
// }
//
// if key.SshKey.Passphrase != "" {
// return fmt.Errorf("ssh key with passphrase not supported")
// }
//
// return ioutil.WriteFile(path, []byte(key.SshKey.PrivateKey+"\n"), 0600)
//}
func (t *task) checkoutRepository() error {
if t.task.CommitHash != nil { // checkout to commit if it is provided for task
err := t.validateRepo()
if err != nil {
return err
}
cmd := exec.Command("git")
cmd.Dir = t.getRepoPath()
t.log("Checkout repository to commit " + *t.task.CommitHash)
cmd.Args = append(cmd.Args, "checkout", *t.task.CommitHash)
t.logCmd(cmd)
return cmd.Run()
}
// store commit to task table
commitHash, err := t.getCommitHash()
if err != nil {
return err
}
commitMessage, _ := t.getCommitMessage()
t.task.CommitHash = &commitHash
t.task.CommitMessage = commitMessage
return t.store.UpdateTask(t.task)
}
// getCommitHash retrieves current commit hash from task repository
func (t *task) getCommitHash() (res string, err error) {
err = t.validateRepo()
if err != nil {
return
}
cmd := exec.Command("git")
cmd.Dir = t.getRepoPath()
t.log("Get current commit hash")
cmd.Args = append(cmd.Args, "rev-parse", "HEAD")
out, err := cmd.Output()
if err != nil {
return
}
res = strings.Trim(string(out), " \n")
return
}
// getCommitMessage retrieves current commit message from task repository
func (t *task) getCommitMessage() (res string, err error) {
err = t.validateRepo()
if err != nil {
return
}
cmd := exec.Command("git")
cmd.Dir = t.getRepoPath()
t.log("Get current commit message")
cmd.Args = append(cmd.Args, "show-branch", "--no-name", "HEAD")
out, err := cmd.Output()
if err != nil {
return
}
res = strings.Trim(string(out), " \n")
if len(res) > 100 {
res = res[0:100]
}
return
}
func (t *task) updateRepository() error {
var gitSSHCommand string
if t.repository.SSHKey.Type == db.AccessKeySSH {
gitSSHCommand = t.repository.SSHKey.GetSshCommand()
}
cmd := exec.Command("git") //nolint: gas
cmd.Dir = util.Config.TmpPath
t.setCmdEnvironment(cmd, gitSSHCommand)
repoURL, repoTag := t.repository.GitURL, "master"
if split := strings.Split(repoURL, "#"); len(split) > 1 {
repoURL, repoTag = split[0], split[1]
}
err := t.validateRepo()
if err != nil {
if !os.IsNotExist(err) {
return err
}
t.log("Cloning repository " + repoURL)
cmd.Args = append(cmd.Args, "clone", "--recursive", "--branch", repoTag, repoURL, t.getRepoName())
} else {
t.log("Updating repository " + repoURL)
cmd.Dir = t.getRepoPath()
cmd.Args = append(cmd.Args, "pull", "origin", repoTag)
}
t.logCmd(cmd)
return cmd.Run()
}
func (t *task) installRequirements() error {
requirementsFilePath := fmt.Sprintf("%s/roles/requirements.yml", t.getRepoPath())
requirementsHashFilePath := fmt.Sprintf("%s/requirements.md5", t.getRepoPath())
if _, err := os.Stat(requirementsFilePath); err != nil {
t.log("No roles/requirements.yml file found. Skip galaxy install process.\n")
return nil
}
if hasRequirementsChanges(requirementsFilePath, requirementsHashFilePath) {
if err := t.runGalaxy([]string{
"install",
"-r",
"roles/requirements.yml",
"--force",
}); err != nil {
return err
}
if err := writeMD5Hash(requirementsFilePath, requirementsHashFilePath); err != nil {
return err
}
} else {
t.log("roles/requirements.yml has no changes. Skip galaxy install process.\n")
}
return nil
}
func (t *task) runGalaxy(args []string) error {
cmd := exec.Command("ansible-galaxy", args...) //nolint: gas
cmd.Dir = t.getRepoPath()
t.setCmdEnvironment(cmd, t.repository.SSHKey.GetSshCommand())
t.logCmd(cmd)
return cmd.Run()
}
func (t *task) listPlaybookHosts() (string, error) {
if util.Config.ConcurrencyMode == "project" {
return "", nil
}
args, err := t.getPlaybookArgs()
if err != nil {
return "", err
}
args = append(args, "--list-hosts")
cmd := exec.Command("ansible-playbook", args...) //nolint: gas
cmd.Dir = t.getRepoPath()
t.setCmdEnvironment(cmd, "")
var errb bytes.Buffer
cmd.Stderr = &errb
out, err := cmd.Output()
re := regexp.MustCompile(`(?m)^\\s{6}(.*)$`)
matches := re.FindAllSubmatch(out, 20)
hosts := make([]string, len(matches))
for i := range matches {
hosts[i] = string(matches[i][1])
}
t.hosts = hosts
return errb.String(), err
}
func (t *task) runPlaybook() (err error) {
args, err := t.getPlaybookArgs()
if err != nil {
return
}
cmd := exec.Command("ansible-playbook", args...) //nolint: gas
cmd.Dir = t.getRepoPath()
t.setCmdEnvironment(cmd, "")
t.logCmd(cmd)
cmd.Stdin = strings.NewReader("")
err = cmd.Start()
if err != nil {
return
}
t.process = cmd.Process
err = cmd.Wait()
return
}
func (t *task) getExtraVars() (str string, err error) {
extraVars := make(map[string]interface{})
if t.environment.JSON != "" {
err = json.Unmarshal([]byte(t.environment.JSON), &extraVars)
if err != nil {
return
}
}
delete(extraVars, "ENV")
if util.Config.VariablesPassingMethod == util.VariablesPassingBoth ||
util.Config.VariablesPassingMethod == util.VariablesPassingExtra {
if t.task.Message != "" {
extraVars["semaphore_task_message"] = t.task.Message
}
if t.task.UserID != nil {
var user db.User
user, err = t.store.GetUser(*t.task.UserID)
if err != nil {
return
}
extraVars["semaphore_task_username"] = user.Username
}
if t.template.Type != db.TemplateTask {
extraVars["semaphore_task_type"] = t.template.Type
var version string
switch t.template.Type {
case db.TemplateBuild:
version = *t.task.Version
case db.TemplateDeploy:
buildTask, err := t.store.GetTask(t.task.ProjectID, *t.task.BuildTaskID)
if err != nil {
panic("Deploy task has no build task")
}
version = *buildTask.Version
}
extraVars["semaphore_task_version"] = version
}
}
ev, err := json.Marshal(extraVars)
if err != nil {
return
}
str = string(ev)
return
}
//nolint: gocyclo
func (t *task) getPlaybookArgs() (args []string, err error) {
playbookName := t.task.Playbook
if playbookName == "" {
playbookName = t.template.Playbook
}
var inventory string
switch t.inventory.Type {
case db.InventoryFile:
inventory = t.inventory.Inventory
default:
inventory = util.Config.TmpPath + "/inventory_" + strconv.Itoa(t.task.ID)
}
args = []string{
"-i", inventory,
}
if t.inventory.SSHKeyID != nil {
switch t.inventory.SSHKey.Type {
case db.AccessKeySSH:
args = append(args, "--private-key="+t.inventory.SSHKey.GetPath())
case db.AccessKeyLoginPassword:
args = append(args, "--extra-vars=@"+t.inventory.SSHKey.GetPath())
case db.AccessKeyNone:
default:
err = fmt.Errorf("access key does not suite for inventory's User Access Key")
return
}
}
if t.inventory.BecomeKeyID != nil {
switch t.inventory.BecomeKey.Type {
case db.AccessKeyLoginPassword:
args = append(args, "--extra-vars=@"+t.inventory.BecomeKey.GetPath())
case db.AccessKeyNone:
default:
err = fmt.Errorf("access key does not suite for inventory's Become User Access Key")
return
}
}
if t.task.Debug {
args = append(args, "-vvvv")
}
if t.task.DryRun {
args = append(args, "--check")
}
if t.task.Limit != "" {
t.log("--limit="+t.task.Limit)
args = append(args, "--limit="+t.task.Limit)
}
if t.template.VaultKeyID != nil {
args = append(args, "--vault-password-file", t.template.VaultKey.GetPath())
}
extraVars, err := t.getExtraVars()
if err != nil {
t.log(err.Error())
t.log("Could not remove command environment, if existant it will be passed to --extra-vars. This is not fatal but be aware of side effects")
} else if extraVars != "" {
args = append(args, "--extra-vars", extraVars)
}
var templateExtraArgs []string
if t.template.Arguments != nil {
err = json.Unmarshal([]byte(*t.template.Arguments), &templateExtraArgs)
if err != nil {
t.log("Could not unmarshal arguments to []string")
return
}
}
if t.template.OverrideArguments {
args = templateExtraArgs
} else {
args = append(args, templateExtraArgs...)
args = append(args, playbookName)
}
return
}
func (t *task) setCmdEnvironment(cmd *exec.Cmd, gitSSHCommand string) {
env := os.Environ()
env = append(env, fmt.Sprintf("HOME=%s", util.Config.TmpPath))
env = append(env, fmt.Sprintf("PWD=%s", cmd.Dir))
env = append(env, fmt.Sprintln("PYTHONUNBUFFERED=1"))
env = append(env, extractCommandEnvironment(t.environment.JSON)...)
if util.Config.VariablesPassingMethod == util.VariablesPassingBoth ||
util.Config.VariablesPassingMethod == util.VariablesPassingEnv {
if t.task.Message != "" {
env = append(env, "SEMAPHORE_TASK_MESSAGE="+t.task.Message)
}
if t.task.UserID != nil {
user, err := t.store.GetUser(*t.task.UserID)
if err != nil {
panic("Deploy task can't find user")
}
env = append(env, "SEMAPHORE_TASK_USERNAME="+user.Username)
}
if t.template.Type != db.TemplateTask {
env = append(env, "SEMAPHORE_TASK_TYPE="+string(t.template.Type))
var version string
switch t.template.Type {
case db.TemplateBuild:
version = *t.task.Version
case db.TemplateDeploy:
buildTask, err := t.store.GetTask(t.task.ProjectID, *t.task.BuildTaskID)
if err != nil {
panic("Deploy task has no build task")
}
version = *buildTask.Version
}
env = append(env, "SEMAPHORE_TASK_VERSION="+version)
}
}
if gitSSHCommand != "" {
env = append(env, fmt.Sprintf("GIT_SSH_COMMAND=%s", gitSSHCommand))
}
cmd.Env = env
}
func hasRequirementsChanges(requirementsFilePath string, requirementsHashFilePath string) bool {
oldFileMD5HashBytes, err := ioutil.ReadFile(requirementsHashFilePath)
if err != nil {
return true
}
newFileMD5Hash, err := helpers.GetMD5Hash(requirementsFilePath)
if err != nil {
return true
}
return string(oldFileMD5HashBytes) != newFileMD5Hash
}
func writeMD5Hash(requirementsFile string, requirementsHashFile string) error {
newFileMD5Hash, err := helpers.GetMD5Hash(requirementsFile)
if err != nil {
return err
}
return ioutil.WriteFile(requirementsHashFile, []byte(newFileMD5Hash), 0644)
}
// extractCommandEnvironment unmarshalls a json string, extracts the ENV key from it and returns it as
// []string where strings are in key=value format
func extractCommandEnvironment(envJSON string) []string {
env := make([]string, 0)
var js map[string]interface{}
err := json.Unmarshal([]byte(envJSON), &js)
if err == nil {
if cfg, ok := js["ENV"]; ok {
switch v := cfg.(type) {
case map[string]interface{}:
for key, val := range v {
env = append(env, fmt.Sprintf("%s=%s", key, val))
}
}
}
}
return env
}
// checkTmpDir checks to see if the temporary directory exists
// and if it does not attempts to create it
func checkTmpDir(path string) error {
var err error
if _, err = os.Stat(path); err != nil {
if os.IsNotExist(err) {
return os.MkdirAll(path, 0700)
}
}
return err
}

4
cli/cmd/migrate.go
View File

@ -15,6 +15,6 @@ var migrateCmd = &cobra.Command{
Run: func(cmd *cobra.Command, args []string) {
store := createStore()
defer store.Close()
fmt.Println("\n DB migrations run on startup automatically")
fmt.Println("\n db migrations run on startup automatically")
},
}
}

45
cli/cmd/root.go
View File

@ -4,11 +4,11 @@ import (
"fmt"
log "github.com/Sirupsen/logrus"
"github.com/ansible-semaphore/semaphore/api"
"github.com/ansible-semaphore/semaphore/api/schedules"
"github.com/ansible-semaphore/semaphore/api/sockets"
"github.com/ansible-semaphore/semaphore/api/tasks"
"github.com/ansible-semaphore/semaphore/db"
"github.com/ansible-semaphore/semaphore/db/factory"
"github.com/ansible-semaphore/semaphore/services/schedules"
"github.com/ansible-semaphore/semaphore/services/tasks"
"github.com/ansible-semaphore/semaphore/util"
"github.com/gorilla/context"
"github.com/gorilla/handlers"
@ -27,23 +27,12 @@ var rootCmd = &cobra.Command{
Source code is available at https://github.com/ansible-semaphore/semaphore.
Complete documentation is available at https://ansible-semaphore.com.`,
Run: func(cmd *cobra.Command, args []string) {
if len(args) == 0 && configPath == "" {
_ = cmd.Help()
os.Exit(0)
} else {
serviceCmd.Run(cmd, args)
}
_ = cmd.Help()
os.Exit(0)
},
}
func Execute() {
args := os.Args[1:]
if len(args) == 2 && args[0] == "-config" {
configPath = args[1]
runService()
return
}
rootCmd.PersistentFlags().StringVar(&configPath, "config", "", "Configuration file path")
if err := rootCmd.Execute(); err != nil {
fmt.Fprintln(os.Stderr, err)
@ -53,7 +42,8 @@ func Execute() {
func runService() {
store := createStore()
schedulePool := schedules.CreateSchedulePool(store)
taskPool := tasks.CreateTaskPool(store)
schedulePool := schedules.CreateSchedulePool(store, &taskPool)
defer store.Close()
defer schedulePool.Destroy()
@ -78,8 +68,8 @@ func runService() {
fmt.Printf("Port %v\n", util.Config.Port)
go sockets.StartWS()
go tasks.StartRunner()
go schedulePool.Run()
go taskPool.Run()
route := api.Route()
@ -87,6 +77,7 @@ func runService() {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
context.Set(r, "store", store)
context.Set(r, "schedule_pool", schedulePool)
context.Set(r, "task_pool", &taskPool)
next.ServeHTTP(w, r)
})
})
@ -113,32 +104,18 @@ func createStore() db.Store {
if err := store.Connect(); err != nil {
switch err {
case bbolt.ErrTimeout:
fmt.Println("\n BoltDB supports only one connection at a time. You should stop service when using CLI.")
fmt.Println("\n BoltDB supports only one connection at a time. You should stop Semaphore to use CLI.")
default:
fmt.Println("\n Have you run `semaphore setup`?")
}
panic(err)
os.Exit(1)
}
initialized, err := store.IsInitialized()
err := db.Migrate(store)
if err != nil {
panic(err)
}
err = store.Migrate()
if err != nil {
panic(err)
}
if !initialized && util.Config.RegisterFirstUser {
err = store.CreatePlaceholderUser()
if err != nil {
panic(err)
}
}
return store
}

10
cli/cmd/service.go → cli/cmd/server.go
View File

@ -7,12 +7,13 @@ import (
)
func init() {
rootCmd.AddCommand(serviceCmd)
rootCmd.AddCommand(serverCmd)
}
var serviceCmd = &cobra.Command{
Use: "service",
Short: "Run Semaphore service",
var serverCmd = &cobra.Command{
Use: "server",
Short: "Run in server mode",
Aliases: []string{"service"},
Run: func(cmd *cobra.Command, args []string) {
runService()
},
@ -26,4 +27,3 @@ func cropTrailingSlashMiddleware(next http.Handler) http.Handler {
next.ServeHTTP(w, r)
})
}

8
cli/cmd/setup.go
View File

@ -50,8 +50,8 @@ func doSetup() int {
os.Exit(1)
}
fmt.Println("Running DB Migrations..")
if err := store.Migrate(); err != nil {
fmt.Println("Running db Migrations..")
if err := db.Migrate(store); err != nil {
fmt.Printf("Database migrations failed!\n %v\n", err.Error())
os.Exit(1)
}
@ -83,8 +83,8 @@ func doSetup() int {
fmt.Printf("\n You are all setup %v!\n", user.Name)
}
fmt.Printf(" Re-launch this program pointing to the configuration file\n\n./semaphore --config %v\n\n", configPath)
fmt.Printf(" To run as daemon:\n\nnohup ./semaphore --config %v &\n\n", configPath)
fmt.Printf(" Re-launch this program pointing to the configuration file\n\n./semaphore server --config %v\n\n", configPath)
fmt.Printf(" To run as daemon:\n\nnohup ./semaphore server --config %v &\n\n", configPath)
fmt.Printf(" You can login with %v or %v.\n", user.Email, user.Username)
return 0

5
cli/cmd/upgrade.go
View File

@ -11,6 +11,7 @@ import (
"os"
"path/filepath"
"runtime"
"strings"
"time"
)
@ -75,7 +76,9 @@ func doUpgrade() error {
// findAsset returns the binary for this platform.
func findAsset(release *github.RepositoryRelease) *github.ReleaseAsset {
for _, asset := range release.Assets {
if *asset.Name == fmt.Sprintf("semaphore_%s_%s", runtime.GOOS, runtime.GOARCH) {
suffix := fmt.Sprintf("_%s_%s.tar.gz", runtime.GOOS, runtime.GOARCH)
if strings.HasPrefix(*asset.Name, "semaphore_") &&
strings.HasSuffix(*asset.Name, suffix) {
return &asset
}
}

18
cli/setup/setup.go
View File

@ -86,21 +86,21 @@ func scanBoltDb(conf *util.ConfigType) {
workingDirectory = os.TempDir()
}
defaultBoltDBPath := filepath.Join(workingDirectory, "database.boltdb")
askValue("DB filename", defaultBoltDBPath, &conf.BoltDb.Hostname)
askValue("db filename", defaultBoltDBPath, &conf.BoltDb.Hostname)
}
func scanMySQL(conf *util.ConfigType) {
askValue("DB Hostname", "127.0.0.1:3306", &conf.MySQL.Hostname)
askValue("DB User", "root", &conf.MySQL.Username)
askValue("DB Password", "", &conf.MySQL.Password)
askValue("DB Name", "semaphore", &conf.MySQL.DbName)
askValue("db Hostname", "127.0.0.1:3306", &conf.MySQL.Hostname)
askValue("db User", "root", &conf.MySQL.Username)
askValue("db Password", "", &conf.MySQL.Password)
askValue("db Name", "semaphore", &conf.MySQL.DbName)
}
func scanPostgres(conf *util.ConfigType) {
askValue("DB Hostname", "127.0.0.1:5432", &conf.Postgres.Hostname)
askValue("DB User", "root", &conf.Postgres.Username)
askValue("DB Password", "", &conf.Postgres.Password)
askValue("DB Name", "semaphore", &conf.Postgres.DbName)
askValue("db Hostname", "127.0.0.1:5432", &conf.Postgres.Hostname)
askValue("db User", "root", &conf.Postgres.Username)
askValue("db Password", "", &conf.Postgres.Password)
askValue("db Name", "semaphore", &conf.Postgres.DbName)
}
func scanErrorChecker(n int, err error) {

92
db/AccessKey.go
View File

@ -16,10 +16,13 @@ import (
"github.com/ansible-semaphore/semaphore/util"
)
type AccessKeyType string
const (
AccessKeySSH = "ssh"
AccessKeyNone = "none"
AccessKeyLoginPassword = "login_password"
AccessKeySSH AccessKeyType = "ssh"
AccessKeyNone AccessKeyType = "none"
AccessKeyLoginPassword AccessKeyType = "login_password"
AccessKeyPAT AccessKeyType = "pat"
)
// AccessKey represents a key used to access a machine with ansible from semaphore
@ -27,7 +30,7 @@ type AccessKey struct {
ID int `db:"id" json:"id"`
Name string `db:"name" json:"name" binding:"required"`
// 'ssh/login_password/none'
Type string `db:"type" json:"type" binding:"required"`
Type AccessKeyType `db:"type" json:"type" binding:"required"`
ProjectID *int `db:"project_id" json:"project_id"`
@ -35,10 +38,9 @@ type AccessKey struct {
// You should use methods SerializeSecret to fill this field.
Secret *string `db:"secret" json:"-"`
Removed bool `db:"removed" json:"removed"`
LoginPassword LoginPassword `db:"-" json:"login_password"`
SshKey SshKey `db:"-" json:"ssh"`
PAT string `db:"-" json:"pat"`
OverrideSecret bool `db:"-" json:"override_secret"`
InstallationKey int64 `db:"-" json:"-"`
@ -55,16 +57,16 @@ type SshKey struct {
PrivateKey string `json:"private_key"`
}
type AccessKeyUsage int
type AccessKeyRole int
const (
AccessKeyUsageAnsibleUser = iota
AccessKeyUsageAnsibleBecomeUser
AccessKeyUsagePrivateKey
AccessKeyUsageVault
AccessKeyRoleAnsibleUser = iota
AccessKeyRoleAnsibleBecomeUser
AccessKeyRoleAnsiblePasswordVault
AccessKeyRoleGit
)
func (key *AccessKey) Install(usage AccessKeyUsage) error {
func (key *AccessKey) Install(usage AccessKeyRole) error {
rnd, err := rand.Int(rand.Reader, big.NewInt(1000000000))
if err != nil {
return err
@ -85,17 +87,20 @@ func (key *AccessKey) Install(usage AccessKeyUsage) error {
}
switch usage {
case AccessKeyUsagePrivateKey:
if key.SshKey.Passphrase != "" {
return fmt.Errorf("ssh key with passphrase not supported")
case AccessKeyRoleGit:
switch key.Type {
case AccessKeySSH:
if key.SshKey.Passphrase != "" {
return fmt.Errorf("ssh key with passphrase not supported")
}
return ioutil.WriteFile(path, []byte(key.SshKey.PrivateKey+"\n"), 0600)
}
return ioutil.WriteFile(path, []byte(key.SshKey.PrivateKey + "\n"), 0600)
case AccessKeyUsageVault:
case AccessKeyRoleAnsiblePasswordVault:
switch key.Type {
case AccessKeyLoginPassword:
return ioutil.WriteFile(path, []byte(key.LoginPassword.Password), 0600)
}
case AccessKeyUsageAnsibleBecomeUser:
case AccessKeyRoleAnsibleBecomeUser:
switch key.Type {
case AccessKeyLoginPassword:
content := make(map[string]string)
@ -110,13 +115,13 @@ func (key *AccessKey) Install(usage AccessKeyUsage) error {
default:
return fmt.Errorf("access key type not supported for ansible user")
}
case AccessKeyUsageAnsibleUser:
case AccessKeyRoleAnsibleUser:
switch key.Type {
case AccessKeySSH:
if key.SshKey.Passphrase != "" {
return fmt.Errorf("ssh key with passphrase not supported")
}
return ioutil.WriteFile(path, []byte(key.SshKey.PrivateKey + "\n"), 0600)
return ioutil.WriteFile(path, []byte(key.SshKey.PrivateKey+"\n"), 0600)
case AccessKeyLoginPassword:
content := make(map[string]string)
content["ansible_user"] = key.LoginPassword.Login
@ -136,9 +141,10 @@ func (key *AccessKey) Install(usage AccessKeyUsage) error {
return nil
}
func (key *AccessKey) Destroy() error {
func (key AccessKey) Destroy() error {
path := key.GetPath()
if _, err := os.Stat(path); os.IsNotExist(err) {
_, err := os.Stat(path)
if os.IsNotExist(err) {
return nil
}
return os.Remove(path)
@ -149,18 +155,6 @@ func (key AccessKey) GetPath() string {
return util.Config.TmpPath + "/access_key_" + strconv.FormatInt(key.InstallationKey, 10)
}
func (key AccessKey) GetSshCommand() string {
if key.Type != AccessKeySSH {
panic("type must be ssh")
}
args := "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i " + key.GetPath()
if util.Config.SshConfigPath != "" {
args += " -F " + util.Config.SshConfigPath
}
return args
}
func (key AccessKey) Validate(validateSecretFields bool) error {
if key.Name == "" {
return fmt.Errorf("name can not be empty")
@ -199,18 +193,24 @@ func (key *AccessKey) SerializeSecret() error {
if err != nil {
return err
}
default:
case AccessKeyPAT:
plaintext = []byte(key.PAT)
case AccessKeyNone:
key.Secret = nil
return nil
default:
return fmt.Errorf("invalid access token type")
}
if util.Config.AccessKeyEncryption == "" {
encryptionString := util.Config.GetAccessKeyEncryption()
if encryptionString == "" {
secret := base64.StdEncoding.EncodeToString(plaintext)
key.Secret = &secret
return nil
}
encryption, err := base64.StdEncoding.DecodeString(util.Config.AccessKeyEncryption)
encryption, err := base64.StdEncoding.DecodeString(encryptionString)
if err != nil {
return err
@ -251,15 +251,17 @@ func (key *AccessKey) unmarshalAppropriateField(secret []byte) (err error) {
if err == nil {
key.LoginPassword = loginPass
}
case AccessKeyPAT:
key.PAT = string(secret)
}
return
}
func (key *AccessKey) ResetSecret() {
//key.Secret = nil
key.LoginPassword = LoginPassword{}
key.SshKey = SshKey{}
}
//func (key *AccessKey) ClearSecret() {
// key.LoginPassword = LoginPassword{}
// key.SshKey = SshKey{}
// key.PAT = ""
//}
func (key *AccessKey) DeserializeSecret() error {
if key.Secret == nil || *key.Secret == "" {
@ -283,7 +285,9 @@ func (key *AccessKey) DeserializeSecret() error {
return err
}
if util.Config.AccessKeyEncryption == "" {
encryptionString := util.Config.GetAccessKeyEncryption()
if encryptionString == "" {
err = key.unmarshalAppropriateField(ciphertext)
if _, ok := err.(*json.SyntaxError); ok {
err = fmt.Errorf("cannot decrypt access key, perhaps encryption key was changed")
@ -291,7 +295,7 @@ func (key *AccessKey) DeserializeSecret() error {
return err
}
encryption, err := base64.StdEncoding.DecodeString(util.Config.AccessKeyEncryption)
encryption, err := base64.StdEncoding.DecodeString(encryptionString)
if err != nil {
return err
}

4
db/AccessKey_test.go
View File

@ -77,7 +77,7 @@ func TestSetGetSecretWithEncryption(t *testing.T) {
t.Error(err)
}
accessKey.ResetSecret()
//accessKey.ClearSecret()
err = accessKey.DeserializeSecret()
@ -88,4 +88,4 @@ func TestSetGetSecretWithEncryption(t *testing.T) {
if accessKey.SshKey.PrivateKey != "qerphqeruqoweurqwerqqeuiqwpavqr" {
t.Error("invalid secret")
}
}
}

3
db/Environment.go
View File

@ -11,7 +11,6 @@ type Environment struct {
ProjectID int `db:"project_id" json:"project_id"`
Password *string `db:"password" json:"password"`
JSON string `db:"json" json:"json" binding:"required"`
Removed bool `db:"removed" json:"removed"`
}
func (env *Environment) Validate() error {
@ -24,4 +23,4 @@ func (env *Environment) Validate() error {
}
return nil
}
}

5
db/Inventory.go
View File

@ -2,8 +2,9 @@ package db
const (
InventoryStatic = "static"
InventoryFile = "file"
InventoryFile = "file"
)
// Inventory is the model of an ansible inventory file
type Inventory struct {
ID int `db:"id" json:"id"`
@ -20,8 +21,6 @@ type Inventory struct {
// static/file
Type string `db:"type" json:"type"`
Removed bool `db:"removed" json:"removed"`
}
func FillInventory(d Store, inventory *Inventory) (err error) {

88
db/Migration.go Normal file
View File

@ -0,0 +1,88 @@
package db
import (
"fmt"
"time"
)
// Migration represents sql schema version
type Migration struct {
Version string `db:"version" json:"version"`
UpgradedDate *time.Time `db:"upgraded_date" json:"upgraded_date"`
Notes *string `db:"notes" json:"notes"`
}
// HumanoidVersion adds a v to the VersionString
func (m Migration) HumanoidVersion() string {
return "v" + m.Version
}
func GetMigrations() []Migration {
return []Migration{
{Version: "0.0.0"},
{Version: "1.0.0"},
{Version: "1.2.0"},
{Version: "1.3.0"},
{Version: "1.4.0"},
{Version: "1.5.0"},
{Version: "1.6.0"},
{Version: "1.7.0"},
{Version: "1.8.0"},
{Version: "1.9.0"},
{Version: "2.2.1"},
{Version: "2.3.0"},
{Version: "2.3.1"},
{Version: "2.3.2"},
{Version: "2.4.0"},
{Version: "2.5.0"},
{Version: "2.5.2"},
{Version: "2.7.1"},
{Version: "2.7.4"},
{Version: "2.7.6"},
{Version: "2.7.8"},
{Version: "2.7.9"},
{Version: "2.7.10"},
{Version: "2.7.12"},
{Version: "2.7.13"},
{Version: "2.8.0"},
{Version: "2.8.1"},
{Version: "2.8.7"},
{Version: "2.8.8"},
{Version: "2.8.20"},
{Version: "2.8.25"},
{Version: "2.8.26"},
{Version: "2.8.36"},
{Version: "2.8.38"},
{Version: "2.8.39"},
{Version: "2.8.40"},
{Version: "2.8.42"},
}
}
func Migrate(d Store) error {
didRun := false
for _, version := range GetMigrations() {
if exists, err := d.IsMigrationApplied(version); err != nil || exists {
if exists {
continue
}
return err
}
didRun = true
fmt.Printf("Executing migration %s (at %v)...\n", version.HumanoidVersion(), time.Now())
if err := d.ApplyMigration(version); err != nil {
fmt.Printf("Rolling back %s (time: %v)...\n", version.HumanoidVersion(), time.Now())
d.TryRollbackMigration(version)
return err
}
}
if didRun {
fmt.Println("Migrations Finished")
}
return nil
}

2
db/Project.go
View File

@ -10,5 +10,5 @@ type Project struct {
Name string `db:"name" json:"name" binding:"required"`
Created time.Time `db:"created" json:"created"`
Alert bool `db:"alert" json:"alert"`
AlertChat string `db:"alert_chat" json:"alert_chat"`
AlertChat *string `db:"alert_chat" json:"alert_chat"`
}

103
db/Repository.go
View File

@ -1,13 +1,114 @@
package db
import (
"github.com/ansible-semaphore/semaphore/util"
"os"
"path"
"regexp"
"strconv"
"strings"
)
type RepositorySchema string
const (
RepositoryGit RepositorySchema = "git"
RepositorySSH RepositorySchema = "ssh"
RepositoryHTTPS RepositorySchema = "https"
RepositoryFile RepositorySchema = "file"
)
// Repository is the model for code stored in a git repository
type Repository struct {
ID int `db:"id" json:"id"`
Name string `db:"name" json:"name" binding:"required"`
ProjectID int `db:"project_id" json:"project_id"`
GitURL string `db:"git_url" json:"git_url" binding:"required"`
GitBranch string `db:"git_branch" json:"git_branch" binding:"required"`
SSHKeyID int `db:"ssh_key_id" json:"ssh_key_id" binding:"required"`
Removed bool `db:"removed" json:"removed"`
SSHKey AccessKey `db:"-" json:"-"`
}
func (r Repository) ClearCache() error {
dir, err := os.Open(util.Config.TmpPath)
if err != nil {
return err
}
files, err := dir.ReadDir(0)
if err != nil {
return err
}
for _, f := range files {
if !f.IsDir() {
continue
}
if strings.HasPrefix(f.Name(), r.getDirNamePrefix()) {
err = os.RemoveAll(path.Join(util.Config.TmpPath, f.Name()))
if err != nil {
return err
}
}
}
return nil
}
func (r Repository) getDirNamePrefix() string {
return "repository_" + strconv.Itoa(r.ID) + "_"
}
func (r Repository) GetDirName(templateID int) string {
return r.getDirNamePrefix() + strconv.Itoa(templateID)
}
func (r Repository) GetFullPath(templateID int) string {
return path.Join(util.Config.TmpPath, r.GetDirName(templateID))
}
func (r Repository) GetGitURL() string {
url := r.GitURL
if r.getSchema() == RepositoryHTTPS {
auth := ""
switch r.SSHKey.Type {
case AccessKeyLoginPassword:
auth = r.SSHKey.LoginPassword.Login + ":" + r.SSHKey.LoginPassword.Password
case AccessKeyPAT:
auth = r.SSHKey.PAT
}
if auth != "" {
auth += "@"
}
url = "https://" + auth + r.GitURL[8:]
}
return url
}
func (r Repository) getSchema() RepositorySchema {
re := regexp.MustCompile(`^(\w+)://`)
m := re.FindStringSubmatch(r.GitURL)
if m == nil {
return RepositoryFile
}
return RepositorySchema(m[1])
}
func (r Repository) Validate() error {
if r.Name == "" {
return &ValidationError{"repository name can't be empty"}
}
if r.GitURL == "" {
return &ValidationError{"repository url can't be empty"}
}
if r.GitBranch == "" {
return &ValidationError{"repository branch can't be empty"}
}
return nil
}

40
db/Repository_test.go Normal file
View File

@ -0,0 +1,40 @@
package db
import (
"github.com/ansible-semaphore/semaphore/util"
"math/rand"
"os"
"path"
"testing"
)
func TestRepository_GetSchema(t *testing.T) {
repo := Repository{GitURL: "https://example.com/hello/world"}
schema := repo.getSchema()
if schema != "https" {
t.Fatal()
}
}
func TestRepository_ClearCache(t *testing.T) {
util.Config = &util.ConfigType{
TmpPath: path.Join(os.TempDir(), util.RandString(rand.Intn(10-4)+4)),
}
repoDir := path.Join(util.Config.TmpPath, "repository_123_55")
err := os.MkdirAll(repoDir, 0755)
if err != nil {
t.Fatal(err)
}
repo := Repository{ID: 123}
err = repo.ClearCache()
if err != nil {
t.Fatal(err)
}
_, err = os.Stat(repoDir)
if err == nil {
t.Fatal("repo directory not deleted")
}
if !os.IsNotExist(err) {
t.Fatal(err)
}
}

10
db/Schedule.go
View File

@ -1,8 +1,10 @@
package db
type Schedule struct {
ID int `db:"id" json:"id"`
ProjectID int `db:"project_id" json:"project_id"`
TemplateID int `db:"template_id" json:"template_id"`
CronFormat string `db:"cron_format" json:"cron_format"`
ID int `db:"id" json:"id"`
ProjectID int `db:"project_id" json:"project_id"`
TemplateID int `db:"template_id" json:"template_id"`
CronFormat string `db:"cron_format" json:"cron_format"`
RepositoryID *int `db:"repository_id" json:"repository_id"`
LastCommitHash *string `db:"last_commit_hash" json:"-"`
}

253
db/Store.go
View File

@ -1,9 +1,11 @@
package db
import (
"encoding/json"
"errors"
log "github.com/Sirupsen/logrus"
"reflect"
"strings"
"time"
)
@ -19,6 +21,18 @@ func GetParsedTime(t time.Time) time.Time {
return parsedTime
}
func ObjectToJSON(obj interface{}) *string {
if obj == nil || (reflect.ValueOf(obj).Kind() == reflect.Ptr && reflect.ValueOf(obj).IsNil()) {
return nil
}
bytes, err := json.Marshal(obj)
if err != nil {
return nil
}
str := string(bytes)
return &str
}
type RetrieveQueryParams struct {
Offset int
Count int
@ -26,18 +40,29 @@ type RetrieveQueryParams struct {
SortInverted bool
}
// ObjectProperties describe database entities.
type ObjectReferrer struct {
ID int `json:"id"`
Name string `json:"name"`
}
type ObjectReferrers struct {
Templates []ObjectReferrer `json:"templates"`
Inventories []ObjectReferrer `json:"inventories"`
Repositories []ObjectReferrer `json:"repositories"`
}
// ObjectProps describe database entities.
// It mainly used for NoSQL implementations (currently BoltDB) to preserve same
// data structure of different implementations and easy change it if required.
type ObjectProperties struct {
TableName string
IsGlobal bool // doesn't belong to other table, for example to project or user.
ForeignColumnSuffix string
PrimaryColumnName string
SortableColumns []string
DefaultSortingColumn string
SortInverted bool // sort from high to low object ID by default. It is useful for some NoSQL implementations.
Type reflect.Type // to which type the table bust be mapped.
type ObjectProps struct {
TableName string
Type reflect.Type // to which type the table bust be mapped.
IsGlobal bool // doesn't belong to other table, for example to project or user.
ReferringColumnSuffix string
PrimaryColumnName string
SortableColumns []string
DefaultSortingColumn string
SortInverted bool // sort from high to low object ID by default. It is useful for some NoSQL implementations.
}
var ErrNotFound = errors.New("no rows in result set")
@ -54,39 +79,47 @@ func (e *ValidationError) Error() string {
type Store interface {
Connect() error
Close() error
// IsInitialized indicates is database already initialized, or it is empty.
// The method is useful for creating required entities in database during first run.
IsInitialized() (bool, error)
Migrate() error
// IsMigrationApplied queries the database to see if a migration table with
// this version id exists already
IsMigrationApplied(version Migration) (bool, error)
// ApplyMigration runs executes a database migration
ApplyMigration(version Migration) error
// TryRollbackMigration attempts to roll back the database to an earlier version
// if a rollback exists
TryRollbackMigration(version Migration)
GetEnvironment(projectID int, environmentID int) (Environment, error)
GetEnvironmentRefs(projectID int, environmentID int) (ObjectReferrers, error)
GetEnvironments(projectID int, params RetrieveQueryParams) ([]Environment, error)
UpdateEnvironment(env Environment) error
CreateEnvironment(env Environment) (Environment, error)
DeleteEnvironment(projectID int, templateID int) error
DeleteEnvironmentSoft(projectID int, templateID int) error
GetInventory(projectID int, inventoryID int) (Inventory, error)
GetInventoryRefs(projectID int, inventoryID int) (ObjectReferrers, error)
GetInventories(projectID int, params RetrieveQueryParams) ([]Inventory, error)
UpdateInventory(inventory Inventory) error
CreateInventory(inventory Inventory) (Inventory, error)
DeleteInventory(projectID int, inventoryID int) error
DeleteInventorySoft(projectID int, inventoryID int) error
GetRepository(projectID int, repositoryID int) (Repository, error)
GetRepositoryRefs(projectID int, repositoryID int) (ObjectReferrers, error)
GetRepositories(projectID int, params RetrieveQueryParams) ([]Repository, error)
UpdateRepository(repository Repository) error
CreateRepository(repository Repository) (Repository, error)
DeleteRepository(projectID int, repositoryID int) error
DeleteRepositorySoft(projectID int, repositoryID int) error
GetAccessKey(projectID int, accessKeyID int) (AccessKey, error)
GetAccessKeyRefs(projectID int, accessKeyID int) (ObjectReferrers, error)
GetAccessKeys(projectID int, params RetrieveQueryParams) ([]AccessKey, error)
UpdateAccessKey(accessKey AccessKey) error
CreateAccessKey(accessKey AccessKey) (AccessKey, error)
DeleteAccessKey(projectID int, accessKeyID int) error
DeleteAccessKeySoft(projectID int, accessKeyID int) error
GetUsers(params RetrieveQueryParams) ([]User, error)
CreateUserWithoutPassword(user User) (User, error)
@ -100,16 +133,14 @@ type Store interface {
GetUser(userID int) (User, error)
GetUserByLoginOrEmail(login string, email string) (User, error)
CreatePlaceholderUser() error
GetPlaceholderUser() (User, error)
GetProject(projectID int) (Project, error)
GetProjects(userID int) ([]Project, error)
CreateProject(project Project) (Project, error)
DeleteProject(projectID int) error
UpdateProject(project Project) error
GetTemplates(projectID int, params RetrieveQueryParams) ([]Template, error)
GetTemplates(projectID int, filter TemplateFilter, params RetrieveQueryParams) ([]Template, error)
GetTemplateRefs(projectID int, templateID int) (ObjectReferrers, error)
CreateTemplate(template Template) (Template, error)
UpdateTemplate(template Template) error
GetTemplate(projectID int, templateID int) (Template, error)
@ -119,6 +150,7 @@ type Store interface {
GetTemplateSchedules(projectID int, templateID int) ([]Schedule, error)
CreateSchedule(schedule Schedule) (Schedule, error)
UpdateSchedule(schedule Schedule) error
SetScheduleCommitHash(projectID int, scheduleID int, hash string) error
GetSchedule(projectID int, scheduleID int) (Schedule, error)
DeleteSchedule(projectID int, scheduleID int) error
@ -145,7 +177,7 @@ type Store interface {
CreateTask(task Task) (Task, error)
UpdateTask(task Task) error
GetTemplateTasks(template Template, params RetrieveQueryParams) ([]TaskWithTpl, error)
GetTemplateTasks(projectID int, templateID int, params RetrieveQueryParams) ([]TaskWithTpl, error)
GetProjectTasks(projectID int, params RetrieveQueryParams) ([]TaskWithTpl, error)
GetTask(projectID int, taskID int) (Task, error)
DeleteTaskWithOutputs(projectID int, taskID int) error
@ -154,138 +186,135 @@ type Store interface {
GetView(projectID int, viewID int) (View, error)
GetViews(projectID int) ([]View, error)
GetViewTemplates(projectID int, viewID int, params RetrieveQueryParams) ([]Template, error)
UpdateView(view View) error
CreateView(view View) (View, error)
DeleteView(projectID int, viewID int) error
SetViewPositions(projectID int, viewPositions map[int]int) error
}
func HasPlaceholderUser(d Store) (bool, error) {
_, err := d.GetPlaceholderUser()
if err == nil {
return true, nil
}
if err == ErrNotFound {
return false, nil
}
return false, err
var AccessKeyProps = ObjectProps{
TableName: "access_key",
Type: reflect.TypeOf(AccessKey{}),
PrimaryColumnName: "id",
ReferringColumnSuffix: "key_id",
SortableColumns: []string{"name", "type"},
DefaultSortingColumn: "name",
}
func ReplacePlaceholderUser(d Store, user UserWithPwd) (newUser User, err error) {
placeholder, err := d.GetPlaceholderUser()
if err != nil {
return
}
user.ID = placeholder.ID
err = d.UpdateUser(user)
if err != nil {
return
}
newUser = user.User
return
var EnvironmentProps = ObjectProps{
TableName: "project__environment",
Type: reflect.TypeOf(Environment{}),
PrimaryColumnName: "id",
ReferringColumnSuffix: "environment_id",
SortableColumns: []string{"name"},
DefaultSortingColumn: "name",
}
var AccessKeyProps = ObjectProperties{
TableName: "access_key",
SortableColumns: []string{"name", "type"},
ForeignColumnSuffix: "key_id",
PrimaryColumnName: "id",
Type: reflect.TypeOf(AccessKey{}),
DefaultSortingColumn: "name",
var InventoryProps = ObjectProps{
TableName: "project__inventory",
Type: reflect.TypeOf(Inventory{}),
PrimaryColumnName: "id",
ReferringColumnSuffix: "inventory_id",
SortableColumns: []string{"name"},
DefaultSortingColumn: "name",
}
var EnvironmentProps = ObjectProperties{
TableName: "project__environment",
SortableColumns: []string{"name"},
ForeignColumnSuffix: "environment_id",
PrimaryColumnName: "id",
Type: reflect.TypeOf(Environment{}),
DefaultSortingColumn: "name",
var RepositoryProps = ObjectProps{
TableName: "project__repository",
Type: reflect.TypeOf(Repository{}),
PrimaryColumnName: "id",
ReferringColumnSuffix: "repository_id",
DefaultSortingColumn: "name",
}
var InventoryProps = ObjectProperties{
TableName: "project__inventory",
SortableColumns: []string{"name"},
ForeignColumnSuffix: "inventory_id",
PrimaryColumnName: "id",
Type: reflect.TypeOf(Inventory{}),
DefaultSortingColumn: "name",
var TemplateProps = ObjectProps{
TableName: "project__template",
Type: reflect.TypeOf(Template{}),
PrimaryColumnName: "id",
ReferringColumnSuffix: "template_id",
SortableColumns: []string{"name"},
DefaultSortingColumn: "name",
}
var RepositoryProps = ObjectProperties{
TableName: "project__repository",
ForeignColumnSuffix: "repository_id",
PrimaryColumnName: "id",
Type: reflect.TypeOf(Repository{}),
DefaultSortingColumn: "name",
}
var TemplateProps = ObjectProperties{
TableName: "project__template",
SortableColumns: []string{"name"},
PrimaryColumnName: "id",
Type: reflect.TypeOf(Template{}),
DefaultSortingColumn: "alias",
}
var ScheduleProps = ObjectProperties{
var ScheduleProps = ObjectProps{
TableName: "project__schedule",
PrimaryColumnName: "id",
Type: reflect.TypeOf(Schedule{}),
PrimaryColumnName: "id",
}
var ProjectUserProps = ObjectProperties{
var ProjectUserProps = ObjectProps{
TableName: "project__user",
PrimaryColumnName: "user_id",
Type: reflect.TypeOf(ProjectUser{}),
PrimaryColumnName: "user_id",
}
var ProjectProps = ObjectProperties{
var ProjectProps = ObjectProps{
TableName: "project",
IsGlobal: true,
PrimaryColumnName: "id",
Type: reflect.TypeOf(Project{}),
PrimaryColumnName: "id",
DefaultSortingColumn: "name",
IsGlobal: true,
}
var UserProps = ObjectProperties{
var UserProps = ObjectProps{
TableName: "user",
IsGlobal: true,
PrimaryColumnName: "id",
Type: reflect.TypeOf(User{}),
}
var SessionProps = ObjectProperties{
TableName: "session",
PrimaryColumnName: "id",
Type: reflect.TypeOf(Session{}),
}
var TokenProps = ObjectProperties{
TableName: "user__token",
PrimaryColumnName: "id",
}
var TaskProps = ObjectProperties{
TableName: "task",
IsGlobal: true,
PrimaryColumnName: "id",
SortInverted: true,
Type: reflect.TypeOf(Task{}),
}
var TaskOutputProps = ObjectProperties{
var SessionProps = ObjectProps{
TableName: "session",
Type: reflect.TypeOf(Session{}),
PrimaryColumnName: "id",
}
var TokenProps = ObjectProps{
TableName: "user__token",
Type: reflect.TypeOf(APIToken{}),
PrimaryColumnName: "id",
}
var TaskProps = ObjectProps{
TableName: "task",
Type: reflect.TypeOf(Task{}),
PrimaryColumnName: "id",
IsGlobal: true,
SortInverted: true,
}
var TaskOutputProps = ObjectProps{
TableName: "task__output",
Type: reflect.TypeOf(TaskOutput{}),
}
var ViewProps = ObjectProperties{
var ViewProps = ObjectProps{
TableName: "project__view",
PrimaryColumnName: "id",
Type: reflect.TypeOf(View{}),
PrimaryColumnName: "id",
DefaultSortingColumn: "position",
}
func (p ObjectProps) GetReferringFieldsFrom(t reflect.Type) (fields []string, err error) {
n := t.NumField()
for i := 0; i < n; i++ {
if !strings.HasSuffix(t.Field(i).Tag.Get("db"), p.ReferringColumnSuffix) {
continue
}
fields = append(fields, t.Field(i).Tag.Get("db"))
}
for i := 0; i < n; i++ {
if t.Field(i).Tag != "" || t.Field(i).Type.Kind() != reflect.Struct {
continue
}
var nested []string
nested, err = p.GetReferringFieldsFrom(t.Field(i).Type)
if err != nil {
return
}
fields = append(fields, nested...)
}
return
}

33
db/Store_test.go Normal file
View File

@ -0,0 +1,33 @@
package db
import "testing"
func TestObjectToJSON(t *testing.T) {
v := &SurveyVar{
Name: "test",
Title: "Test",
}
s := ObjectToJSON(v)
if s == nil || *s != "{\"name\":\"test\",\"title\":\"Test\",\"required\":false,\"type\":\"\",\"description\":\"\"}" {
t.Fail()
}
}
func TestObjectToJSON2(t *testing.T) {
var v *SurveyVar = nil
s := ObjectToJSON(v)
if s != nil {
t.Fail()
}
}
func TestObjectToJSON3(t *testing.T) {
v := SurveyVar{
Name: "test",
Title: "Test",
}
s := ObjectToJSON(v)
if s == nil || *s != "{\"name\":\"test\",\"title\":\"Test\",\"required\":false,\"type\":\"\",\"description\":\"\"}" {
t.Fail()
}
}

46
db/Task.go
View File

@ -4,6 +4,15 @@ import (
"time"
)
const (
TaskRunningStatus = "running"
TaskWaitingStatus = "waiting"
TaskStoppingStatus = "stopping"
TaskStoppedStatus = "stopped"
TaskSuccessStatus = "success"
TaskFailStatus = "error"
)
//Task is a model of a task which will be executed by the runner
type Task struct {
ID int `db:"id" json:"id"`
@ -28,13 +37,43 @@ type Task struct {
Message string `db:"message" json:"message"`
// CommitMessage is a git commit hash of playbook repository which
// was active when task was created.
CommitHash *string `db:"commit_hash" json:"commit_hash"`
// CommitMessage contains message retrieved from git repository after checkout to CommitHash.
// It is readonly by API.
CommitMessage string `db:"commit_message" json:"commit_message"`
BuildTaskID *int `db:"build_task_id" json:"build_task_id"`
Version *string `db:"version" json:"version"`
BuildTaskID *int `db:"build_task_id" json:"build_task_id"`
// Version is a build version.
// This field available only for Build tasks.
Version *string `db:"version" json:"version"`
Arguments *string `db:"arguments" json:"arguments"`
}
func (task *Task) GetIncomingVersion(d Store) *string {
if task.BuildTaskID == nil {
return nil
}
buildTask, err := d.GetTask(task.ProjectID, *task.BuildTaskID)
if err != nil {
return nil
}
tpl, err := d.GetTemplate(task.ProjectID, buildTask.TemplateID)
if err != nil {
return nil
}
if tpl.Type == TemplateBuild {
return buildTask.Version
}
return buildTask.GetIncomingVersion(d)
}
func (task *Task) ValidateNewTask(template Template) error {
@ -49,6 +88,9 @@ func (task *Task) ValidateNewTask(template Template) error {
func (task *TaskWithTpl) Fill(d Store) error {
if task.BuildTaskID != nil {
build, err := d.GetTask(task.ProjectID, *task.BuildTaskID)
if err == ErrNotFound {
return nil
}
if err != nil {
return err
}

51
db/Template.go
View File

@ -12,6 +12,27 @@ const (
TemplateDeploy TemplateType = "deploy"
)
type SurveyVarType string
const (
SurveyVarStr TemplateType = ""
SurveyVarInt TemplateType = "int"
)
type SurveyVar struct {
Name string `json:"name"`
Title string `json:"title"`
Required bool `json:"required"`
Type SurveyVarType `json:"type"`
Description string `json:"description"`
}
type TemplateFilter struct {
ViewID *int
BuildTemplateID *int
AutorunOnly bool
}
// Template is a user defined model that is used to run a task
type Template struct {
ID int `db:"id" json:"id"`
@ -21,16 +42,14 @@ type Template struct {
RepositoryID int `db:"repository_id" json:"repository_id"`
EnvironmentID *int `db:"environment_id" json:"environment_id"`
// Alias as described in https://github.com/ansible-semaphore/semaphore/issues/188
Alias string `db:"alias" json:"alias"`
// Name as described in https://github.com/ansible-semaphore/semaphore/issues/188
Name string `db:"name" json:"name"`
// playbook name in the form of "some_play.yml"
Playbook string `db:"playbook" json:"playbook"`
// to fit into []string
Arguments *string `db:"arguments" json:"arguments"`
// if true, semaphore will not prepend any arguments to `arguments` like inventory, etc
OverrideArguments bool `db:"override_args" json:"override_args"`
Removed bool `db:"removed" json:"-"`
AllowOverrideArgsInTask bool `db:"allow_override_args_in_task" json:"allow_override_args_in_task"`
Description *string `db:"description" json:"description"`
@ -44,11 +63,19 @@ type Template struct {
ViewID *int `db:"view_id" json:"view_id"`
LastTask *TaskWithTpl `db:"-" json:"last_task"`
Autorun bool `db:"autorun" json:"autorun"`
// SurveyVarsJSON used internally for read from database.
// It is not used for store survey vars to database.
// Do not use it in your code. Use SurveyVars instead.
SurveyVarsJSON *string `db:"survey_vars" json:"-"`
SurveyVars []SurveyVar `db:"-" json:"survey_vars"`
}
func (tpl *Template) Validate() error {
if tpl.Alias == "" {
return &ValidationError{"template alias can not be empty"}
if tpl.Name == "" {
return &ValidationError{"template name can not be empty"}
}
if tpl.Playbook == "" {
@ -68,7 +95,7 @@ func FillTemplates(d Store, templates []Template) (err error) {
for i := range templates {
tpl := &templates[i]
var tasks []TaskWithTpl
tasks, err = d.GetTemplateTasks(*tpl, RetrieveQueryParams{Count: 1})
tasks, err = d.GetTemplateTasks(tpl.ProjectID, tpl.ID, RetrieveQueryParams{Count: 1})
if err != nil {
return
}
@ -91,5 +118,13 @@ func FillTemplate(d Store, template *Template) (err error) {
err = FillTemplates(d, []Template{*template})
if err != nil {
return
}
if template.SurveyVarsJSON != nil {
err = json.Unmarshal([]byte(*template.SurveyVarsJSON), &template.SurveyVars)
}
return
}

490
db/bolt/BoltDb.go
View File

@ -7,8 +7,10 @@ import (
"github.com/ansible-semaphore/semaphore/db"
"github.com/ansible-semaphore/semaphore/util"
"go.etcd.io/bbolt"
"math/rand"
"reflect"
"sort"
"strconv"
"strings"
"time"
)
@ -50,7 +52,7 @@ func (d strObjectID) ToBytes() []byte {
return []byte(d)
}
func makeBucketId(props db.ObjectProperties, ids ...int) []byte {
func makeBucketId(props db.ObjectProps, ids ...int) []byte {
n := len(ids)
id := props.TableName
@ -105,7 +107,7 @@ func (d *BoltDb) IsInitialized() (initialized bool, err error) {
return
}
func (d *BoltDb) getObject(bucketID int, props db.ObjectProperties, objectID objectID, object interface{}) (err error) {
func (d *BoltDb) getObject(bucketID int, props db.ObjectProps, objectID objectID, object interface{}) (err error) {
err = d.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(makeBucketId(props, bucketID))
if b == nil {
@ -257,7 +259,7 @@ func marshalObject(obj interface{}) ([]byte, error) {
return json.Marshal(copyObject(obj, newType))
}
func unmarshalObjects(rawData enumerable, props db.ObjectProperties, params db.RetrieveQueryParams, filter func(interface{}) bool, objects interface{}) (err error) {
func unmarshalObjects(rawData enumerable, props db.ObjectProps, params db.RetrieveQueryParams, filter func(interface{}) bool, objects interface{}) (err error) {
objectsValue := reflect.ValueOf(objects).Elem()
objType := objectsValue.Type().Elem()
@ -315,7 +317,7 @@ func unmarshalObjects(rawData enumerable, props db.ObjectProperties, params db.R
return
}
func (d *BoltDb) getObjects(bucketID int, props db.ObjectProperties, params db.RetrieveQueryParams, filter func(interface{}) bool, objects interface{}) error {
func (d *BoltDb) getObjects(bucketID int, props db.ObjectProps, params db.RetrieveQueryParams, filter func(interface{}) bool, objects interface{}) error {
return d.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(makeBucketId(props, bucketID))
var c enumerable
@ -328,18 +330,269 @@ func (d *BoltDb) getObjects(bucketID int, props db.ObjectProperties, params db.R
})
}
func isObjectBelongTo(props db.ObjectProperties, objID objectID, tpl interface{}) bool {
if props.ForeignColumnSuffix == "" {
func (d *BoltDb) deleteObject(bucketID int, props db.ObjectProps, objectID objectID, tx *bbolt.Tx) error {
for _, u := range []db.ObjectProps{db.TemplateProps, db.EnvironmentProps, db.InventoryProps, db.RepositoryProps} {
inUse, err := d.isObjectInUse(bucketID, props, objectID, u)
if err != nil {
return err
}
if inUse {
return db.ErrInvalidOperation
}
}
fn := func(tx *bbolt.Tx) error {
b := tx.Bucket(makeBucketId(props, bucketID))
if b == nil {
return db.ErrNotFound
}
return b.Delete(objectID.ToBytes())
}
if tx != nil {
return fn(tx)
}
return d.db.Update(fn)
}
// updateObject updates data for object in database.
func (d *BoltDb) updateObject(bucketID int, props db.ObjectProps, object interface{}) error {
return d.db.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(makeBucketId(props, bucketID))
if b == nil {
return db.ErrNotFound
}
idFieldName, err := getFieldNameByTagSuffix(reflect.TypeOf(object), "db", props.PrimaryColumnName)
if err != nil {
return err
}
idValue := reflect.ValueOf(object).FieldByName(idFieldName)
var objID objectID
switch idValue.Kind() {
case reflect.Int,
reflect.Int8,
reflect.Int16,
reflect.Int32,
reflect.Int64,
reflect.Uint,
reflect.Uint8,
reflect.Uint16,
reflect.Uint32,
reflect.Uint64:
objID = intObjectID(idValue.Int())
case reflect.String:
objID = strObjectID(idValue.String())
}
if objID == nil {
return fmt.Errorf("unsupported ID type")
}
if b.Get(objID.ToBytes()) == nil {
return db.ErrNotFound
}
str, err := marshalObject(object)
if err != nil {
return err
}
return b.Put(objID.ToBytes(), str)
})
}
func (d *BoltDb) createObject(bucketID int, props db.ObjectProps, object interface{}) (interface{}, error) {
err := d.db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists(makeBucketId(props, bucketID))
if err != nil {
return err
}
objPtr := reflect.ValueOf(&object).Elem()
tmpObj := reflect.New(objPtr.Elem().Type()).Elem()
tmpObj.Set(objPtr.Elem())
var objID objectID
if props.PrimaryColumnName != "" {
idFieldName, err2 := getFieldNameByTagSuffix(reflect.TypeOf(object), "db", props.PrimaryColumnName)
if err2 != nil {
return err2
}
idValue := tmpObj.FieldByName(idFieldName)
switch idValue.Kind() {
case reflect.Int,
reflect.Int8,
reflect.Int16,
reflect.Int32,
reflect.Int64,
reflect.Uint,
reflect.Uint8,
reflect.Uint16,
reflect.Uint32,
reflect.Uint64:
//if idValue.Int() == 0 {
id, err3 := b.NextSequence()
if err3 != nil {
return err3
}
if props.SortInverted {
id = MaxID - id
}
idValue.SetInt(int64(id))
//}
objID = intObjectID(idValue.Int())
case reflect.String:
if idValue.String() == "" {
return fmt.Errorf("object ID can not be empty string")
}
objID = strObjectID(idValue.String())
case reflect.Invalid:
id, err3 := b.NextSequence()
if err3 != nil {
return err3
}
objID = intObjectID(id)
default:
return fmt.Errorf("unsupported ID type")
}
} else {
id, err2 := b.NextSequence()
if err2 != nil {
return err2
}
if props.SortInverted {
id = MaxID - id
}
objID = intObjectID(id)
}
if objID == nil {
return fmt.Errorf("object ID can not be nil")
}
objPtr.Set(tmpObj)
str, err := marshalObject(object)
if err != nil {
return err
}
return b.Put(objID.ToBytes(), str)
})
return object, err
}
func (d *BoltDb) getObjectRefs(projectID int, objectProps db.ObjectProps, objectID int) (refs db.ObjectReferrers, err error) {
refs.Templates, err = d.getObjectRefsFrom(projectID, objectProps, intObjectID(objectID), db.TemplateProps)
if err != nil {
return
}
refs.Repositories, err = d.getObjectRefsFrom(projectID, objectProps, intObjectID(objectID), db.RepositoryProps)
if err != nil {
return
}
refs.Inventories, err = d.getObjectRefsFrom(projectID, objectProps, intObjectID(objectID), db.InventoryProps)
if err != nil {
return
}
templates, err := d.getObjectRefsFrom(projectID, objectProps, intObjectID(objectID), db.ScheduleProps)
for _, st := range templates {
exists := false
for _, tpl := range refs.Templates {
if tpl.ID == st.ID {
exists = true
break
}
}
if exists {
continue
}
refs.Templates = append(refs.Templates, st)
}
return
}
func (d *BoltDb) getObjectRefsFrom(projectID int, objProps db.ObjectProps, objID objectID, referringObjectProps db.ObjectProps) (referringObjs []db.ObjectReferrer, err error) {
referringObjs = make([]db.ObjectReferrer, 0)
_, err = objProps.GetReferringFieldsFrom(referringObjectProps.Type)
if err != nil {
return
}
var referringObjects reflect.Value
if referringObjectProps.Type == db.ScheduleProps.Type {
schedules := make([]db.Schedule, 0)
err = d.getObjects(projectID, db.ScheduleProps, db.RetrieveQueryParams{}, func(referringObj interface{}) bool {
return isObjectReferredBy(objProps, objID, referringObj)
}, &schedules)
if err != nil {
return
}
for _, schedule := range schedules {
var template db.Template
template, err = d.GetTemplate(projectID, schedule.TemplateID)
if err != nil {
return
}
referringObjs = append(referringObjs, db.ObjectReferrer{
ID: template.ID,
Name: template.Name,
})
}
} else {
referringObjects = reflect.New(reflect.SliceOf(referringObjectProps.Type))
err = d.getObjects(projectID, referringObjectProps, db.RetrieveQueryParams{}, func(referringObj interface{}) bool {
return isObjectReferredBy(objProps, objID, referringObj)
}, referringObjects.Interface())
if err != nil {
return
}
for i := 0; i < referringObjects.Elem().Len(); i++ {
referringObjs = append(referringObjs, db.ObjectReferrer{
ID: int(referringObjects.Elem().Index(i).FieldByName("ID").Int()),
Name: referringObjects.Elem().Index(i).FieldByName("Name").String(),
})
}
}
return
}
func isObjectReferredBy(props db.ObjectProps, objID objectID, referringObj interface{}) bool {
if props.ReferringColumnSuffix == "" {
return false
}
fieldName, err := getFieldNameByTagSuffix(reflect.TypeOf(tpl), "db", props.ForeignColumnSuffix)
fieldName, err := getFieldNameByTagSuffix(reflect.TypeOf(referringObj), "db", props.ReferringColumnSuffix)
if err != nil {
return false
}
f := reflect.ValueOf(tpl).FieldByName(fieldName)
f := reflect.ValueOf(referringObj).FieldByName(fieldName)
if f.IsZero() {
return false
@ -378,224 +631,31 @@ func isObjectBelongTo(props db.ObjectProperties, objID objectID, tpl interface{}
}
// isObjectInUse checks if objID associated with any object in foreignTableProps.
func (d *BoltDb) isObjectInUse(bucketID int, objProps db.ObjectProperties, objID objectID, foreignTableProps db.ObjectProperties) (inUse bool, err error) {
templates := reflect.New(reflect.SliceOf(foreignTableProps.Type))
func (d *BoltDb) isObjectInUse(bucketID int, objProps db.ObjectProps, objID objectID, referringObjectProps db.ObjectProps) (inUse bool, err error) {
referringObjects := reflect.New(reflect.SliceOf(referringObjectProps.Type))
err = d.getObjects(bucketID, foreignTableProps, db.RetrieveQueryParams{}, func(foreignObj interface{}) bool {
return isObjectBelongTo(objProps, objID, foreignObj)
}, templates.Interface())
err = d.getObjects(bucketID, referringObjectProps, db.RetrieveQueryParams{}, func(referringObj interface{}) bool {
return isObjectReferredBy(objProps, objID, referringObj)
}, referringObjects.Interface())
if err != nil {
return
}
inUse = templates.Elem().Len() > 0
inUse = referringObjects.Elem().Len() > 0
return
}
func (d *BoltDb) deleteObject(bucketID int, props db.ObjectProperties, objectID objectID) error {
for _, u := range []db.ObjectProperties{db.TemplateProps, db.EnvironmentProps, db.InventoryProps, db.RepositoryProps} {
inUse, err := d.isObjectInUse(bucketID, props, objectID, u)
if err != nil {
return err
}
if inUse {
return db.ErrInvalidOperation
}
func CreateTestStore() BoltDb {
r := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
fn := "/tmp/test_semaphore_db_" + strconv.Itoa(r.Int())
store := BoltDb{
Filename: fn,
}
return d.db.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(makeBucketId(props, bucketID))
if b == nil {
return db.ErrNotFound
}
return b.Delete(objectID.ToBytes())
})
}
func (d *BoltDb) deleteObjectSoft(bucketID int, props db.ObjectProperties, objectID objectID) error {
var data map[string]interface{}
// load data
err := d.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket(makeBucketId(props, bucketID))
if b == nil {
return db.ErrNotFound
}
d := b.Get(objectID.ToBytes())
if d == nil {
return db.ErrNotFound
}
return json.Unmarshal(d, &data)
})
err := store.Connect()
if err != nil {
return err
panic(err)
}
// mark as removed if "removed" exists
if _, ok := data["removed"]; !ok {
return fmt.Errorf("removed field not exists")
}
data["removed"] = true
// store data back
res, err := json.Marshal(data)
if err != nil {
return err
}
return d.db.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(makeBucketId(props, bucketID))
if b == nil {
return db.ErrNotFound
}
return b.Put(objectID.ToBytes(), res)
})
}
// updateObject updates data for object in database.
func (d *BoltDb) updateObject(bucketID int, props db.ObjectProperties, object interface{}) error {
return d.db.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket(makeBucketId(props, bucketID))
if b == nil {
return db.ErrNotFound
}
idFieldName, err := getFieldNameByTagSuffix(reflect.TypeOf(object), "db", props.PrimaryColumnName)
if err != nil {
return err
}
idValue := reflect.ValueOf(object).FieldByName(idFieldName)
var objectID objectID
switch idValue.Kind() {
case reflect.Int,
reflect.Int8,
reflect.Int16,
reflect.Int32,
reflect.Int64,
reflect.Uint,
reflect.Uint8,
reflect.Uint16,
reflect.Uint32,
reflect.Uint64:
objectID = intObjectID(idValue.Int())
case reflect.String:
objectID = strObjectID(idValue.String())
}
if objectID == nil {
return fmt.Errorf("unsupported ID type")
}
if b.Get(objectID.ToBytes()) == nil {
return db.ErrNotFound
}
str, err := marshalObject(object)
if err != nil {
return err
}
return b.Put(objectID.ToBytes(), str)
})
}
func (d *BoltDb) createObject(bucketID int, props db.ObjectProperties, object interface{}) (interface{}, error) {
err := d.db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists(makeBucketId(props, bucketID))
if err != nil {
return err
}
objPtr := reflect.ValueOf(&object).Elem()
tmpObj := reflect.New(objPtr.Elem().Type()).Elem()
tmpObj.Set(objPtr.Elem())
var objectID objectID
if props.PrimaryColumnName != "" {
idFieldName, err := getFieldNameByTagSuffix(reflect.TypeOf(object), "db", props.PrimaryColumnName)
if err != nil {
return err
}
idValue := tmpObj.FieldByName(idFieldName)
switch idValue.Kind() {
case reflect.Int,
reflect.Int8,
reflect.Int16,
reflect.Int32,
reflect.Int64,
reflect.Uint,
reflect.Uint8,
reflect.Uint16,
reflect.Uint32,
reflect.Uint64:
if idValue.Int() == 0 {
id, err2 := b.NextSequence()
if err2 != nil {
return err2
}
if props.SortInverted {
id = MaxID - id
}
idValue.SetInt(int64(id))
}
objectID = intObjectID(idValue.Int())
case reflect.String:
if idValue.String() == "" {
return fmt.Errorf("object ID can not be empty string")
}
objectID = strObjectID(idValue.String())
case reflect.Invalid:
id, err2 := b.NextSequence()
if err2 != nil {
return err2
}
objectID = intObjectID(id)
default:
return fmt.Errorf("unsupported ID type")
}
} else {
id, err2 := b.NextSequence()
if err2 != nil {
return err2
}
if props.SortInverted {
id = MaxID - id
}
objectID = intObjectID(id)
}
if objectID == nil {
return fmt.Errorf("object ID can not be nil")
}
objPtr.Set(tmpObj)
str, err := marshalObject(object)
if err != nil {
return err
}
return b.Put(objectID.ToBytes(), str)
})
return object, err
return store
}

296
db/bolt/BoltDb_test.go
View File

@ -3,83 +3,62 @@ package bolt
import (
"fmt"
"github.com/ansible-semaphore/semaphore/db"
"math/rand"
"reflect"
"strconv"
"testing"
"time"
)
type test1 struct {
ID int `db:"ID"`
FirstName string `db:"first_name" json:"firstName"`
LastName string `db:"last_name" json:"lastName"`
Password string `db:"-" json:"password"`
ID int `db:"ID"`
FirstName string `db:"first_name" json:"firstName"`
LastName string `db:"last_name" json:"lastName"`
Password string `db:"-" json:"password"`
PasswordRepeat string `db:"-" json:"passwordRepeat"`
PasswordHash string `db:"password" json:"-"`
Removed bool `db:"removed"`
PasswordHash string `db:"password" json:"-"`
Removed bool `db:"removed"`
}
var test1props = db.ObjectProperties{
IsGlobal: true,
TableName: "test1",
PrimaryColumnName: "ID",
}
//var test1props = db.ObjectProps{
// IsGlobal: true,
// TableName: "test1",
// PrimaryColumnName: "ID",
//}
func createBoltDb() BoltDb {
r := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
fn := "/tmp/test_semaphore_db_" + strconv.Itoa(r.Int())
return BoltDb{
Filename: fn,
}
}
func createStore() db.Store {
store := createBoltDb()
return &store
}
func TestDeleteObjectSoft(t *testing.T) {
store := createBoltDb()
err := store.Connect()
if err != nil {
t.Fatal(err.Error())
}
obj := test1{
FirstName: "Denis",
LastName: "Gukov",
}
newObj, err := store.createObject(0, test1props, obj)
if err != nil {
t.Fatal(err.Error())
}
objID := intObjectID(newObj.(test1).ID)
err = store.deleteObjectSoft(0, test1props, objID)
if err != nil {
t.Fatal(err.Error())
}
var found test1
err = store.getObject(0, test1props, objID, &found)
if err != nil {
t.Fatal(err.Error())
}
if found.ID != int(objID) ||
found.Removed != true ||
found.Password != obj.Password ||
found.LastName != obj.LastName {
t.Fatal()
}
}
//func TestDeleteObjectSoft(t *testing.T) {
// store := CreateTestStore()
//
// obj := test1{
// FirstName: "Denis",
// LastName: "Gukov",
// }
// newObj, err := store.createObject(0, test1props, obj)
//
// if err != nil {
// t.Fatal(err.Error())
// }
//
// objID := intObjectID(newObj.(test1).ID)
//
// err = store.deleteObjectSoft(0, test1props, objID)
//
// if err != nil {
// t.Fatal(err.Error())
// }
//
// var found test1
// err = store.getObject(0, test1props, objID, &found)
//
// if err != nil {
// t.Fatal(err.Error())
// }
//
// if found.ID != int(objID) ||
// found.Removed != true ||
// found.Password != obj.Password ||
// found.LastName != obj.LastName {
//
// t.Fatal()
// }
//}
func TestMarshalObject_UserWithPwd(t *testing.T) {
user := db.UserWithPwd{
@ -107,11 +86,11 @@ func TestMarshalObject_UserWithPwd(t *testing.T) {
func TestMarshalObject(t *testing.T) {
test1 := test1{
FirstName: "Denis",
LastName: "Gukov",
Password: "1234556",
FirstName: "Denis",
LastName: "Gukov",
Password: "1234556",
PasswordRepeat: "123456",
PasswordHash: "9347502348723",
PasswordHash: "9347502348723",
}
bytes, err := marshalObject(test1)
@ -151,23 +130,23 @@ func TestUnmarshalObject(t *testing.T) {
func TestSortObjects(t *testing.T) {
objects := []db.Inventory{
{
ID: 1,
ID: 1,
Name: "x",
},
{
ID: 2,
ID: 2,
Name: "a",
},
{
ID: 3,
ID: 3,
Name: "d",
},
{
ID: 4,
ID: 4,
Name: "b",
},
{
ID: 5,
ID: 5,
Name: "r",
},
}
@ -183,7 +162,6 @@ func TestSortObjects(t *testing.T) {
objects[3].Name == "r" &&
objects[4].Name == "x"
if !expected {
t.Fatal(fmt.Errorf("objects not sorted"))
}
@ -211,12 +189,7 @@ func TestGetFieldNameByTag2(t *testing.T) {
}
func TestIsObjectInUse(t *testing.T) {
store := createBoltDb()
err := store.Connect()
if err != nil {
t.Fatal(err.Error())
}
store := CreateTestStore()
proj, err := store.CreateProject(db.Project{
Name: "test",
@ -227,9 +200,9 @@ func TestIsObjectInUse(t *testing.T) {
}
_, err = store.CreateTemplate(db.Template{
Alias: "Test",
Playbook: "test.yml",
ProjectID: proj.ID,
Name: "Test",
Playbook: "test.yml",
ProjectID: proj.ID,
InventoryID: 10,
})
@ -250,12 +223,7 @@ func TestIsObjectInUse(t *testing.T) {
}
func TestIsObjectInUse_Environment(t *testing.T) {
store := createBoltDb()
err := store.Connect()
if err != nil {
t.Fatal(err.Error())
}
store := CreateTestStore()
proj, err := store.CreateProject(db.Project{
Name: "test",
@ -268,9 +236,9 @@ func TestIsObjectInUse_Environment(t *testing.T) {
envID := 10
_, err = store.CreateTemplate(db.Template{
Alias: "Test",
Playbook: "test.yml",
ProjectID: proj.ID,
Name: "Test",
Playbook: "test.yml",
ProjectID: proj.ID,
EnvironmentID: &envID,
})
@ -291,40 +259,150 @@ func TestIsObjectInUse_Environment(t *testing.T) {
}
func TestIsObjectInUse_EnvironmentNil(t *testing.T) {
store := createBoltDb()
err := store.Connect()
if err != nil {
t.Fatal(err.Error())
}
store := CreateTestStore()
proj, err := store.CreateProject(db.Project{
Name: "test",
})
if err != nil {
t.Fatal(err.Error())
t.Fatal(err)
}
_, err = store.CreateTemplate(db.Template{
Alias: "Test",
Playbook: "test.yml",
ProjectID: proj.ID,
Name: "Test",
Playbook: "test.yml",
ProjectID: proj.ID,
EnvironmentID: nil,
})
if err != nil {
t.Fatal(err.Error())
t.Fatal(err)
}
isUse, err := store.isObjectInUse(proj.ID, db.EnvironmentProps, intObjectID(10), db.TemplateProps)
if err != nil {
t.Fatal(err.Error())
t.Fatal(err)
}
if isUse {
t.Fatal()
}
}
func TestBoltDb_CreateAPIToken(t *testing.T) {
store := CreateTestStore()
user, err := store.CreateUser(db.UserWithPwd{
Pwd: "3412341234123",
User: db.User{
Username: "test",
Name: "Test",
Email: "test@example.com",
Admin: true,
},
})
if err != nil {
t.Fatal(err)
}
token, err := store.CreateAPIToken(db.APIToken{
ID: "f349gyhgqirgysfgsfg34973dsfad",
UserID: user.ID,
})
if err != nil {
t.Fatal(err)
}
token2, err := store.GetAPIToken(token.ID)
if err != nil {
t.Fatal(err)
}
if token2.ID != token.ID {
t.Fatal()
}
tokens, err := store.GetAPITokens(user.ID)
if err != nil {
t.Fatal(err)
}
if len(tokens) != 1 {
t.Fatal()
}
if tokens[0].ID != token.ID {
t.Fatal()
}
err = store.ExpireAPIToken(user.ID, token.ID)
if err != nil {
t.Fatal(err)
}
token2, err = store.GetAPIToken(token.ID)
if err != nil {
t.Fatal(err)
}
if !token2.Expired {
t.Fatal()
}
}
func TestBoltDb_GetRepositoryRefs(t *testing.T) {
store := CreateTestStore()
repo1, err := store.CreateRepository(db.Repository{
Name: "repo1",
GitURL: "git@example.com/repo1",
GitBranch: "master",
ProjectID: 1,
})
if err != nil {
t.Fatal(err)
}
_, err = store.CreateTemplate(db.Template{
Type: db.TemplateBuild,
Name: "tpl1",
Playbook: "build.yml",
RepositoryID: repo1.ID,
ProjectID: 1,
})
if err != nil {
t.Fatal(err)
}
tpl2, err := store.CreateTemplate(db.Template{
Type: db.TemplateBuild,
Name: "tpl12",
Playbook: "build.yml",
ProjectID: 1,
})
if err != nil {
t.Fatal(err)
}
_, err = store.CreateSchedule(db.Schedule{
CronFormat: "* * * * *",
TemplateID: tpl2.ID,
ProjectID: 1,
RepositoryID: &repo1.ID,
})
if err != nil {
t.Fatal(err)
}
refs, err := store.GetRepositoryRefs(1, repo1.ID)
if err != nil {
t.Fatal(err)
}
if len(refs.Templates) != 2 {
t.Fatal()
}
}

91
db/bolt/Task_test.go Normal file
View File

@ -0,0 +1,91 @@
package bolt
import (
"github.com/ansible-semaphore/semaphore/db"
"testing"
)
func TestTask_GetVersion(t *testing.T) {
VERSION := "1.54.48"
store := CreateTestStore()
build, err := store.CreateTemplate(db.Template{
ProjectID: 0,
Type: db.TemplateBuild,
Name: "Build",
Playbook: "build.yml",
})
if err != nil {
t.Fatal(err)
}
deploy, err := store.CreateTemplate(db.Template{
ProjectID: 0,
Type: db.TemplateDeploy,
BuildTemplateID: &build.ID,
Name: "Deploy",
Playbook: "deploy.yml",
})
if err != nil {
t.Fatal(err)
}
deploy2, err := store.CreateTemplate(db.Template{
ProjectID: 0,
Type: db.TemplateDeploy,
BuildTemplateID: &deploy.ID,
Name: "Deploy2",
Playbook: "deploy2.yml",
})
if err != nil {
t.Fatal(err)
}
buildTask, err := store.CreateTask(db.Task{
ProjectID: 0,
TemplateID: build.ID,
Version: &VERSION,
})
if err != nil {
t.Fatal(err)
}
deployTask, err := store.CreateTask(db.Task{
ProjectID: 0,
TemplateID: deploy.ID,
BuildTaskID: &buildTask.ID,
})
if err != nil {
t.Fatal(err)
}
deploy2Task, err := store.CreateTask(db.Task{
ProjectID: 0,
TemplateID: deploy2.ID,
BuildTaskID: &deployTask.ID,
})
if err != nil {
t.Fatal(err)
}
version := deployTask.GetIncomingVersion(&store)
if version == nil {
t.Fatal()
return
}
if *version != VERSION {
t.Fatal()
return
}
version = deploy2Task.GetIncomingVersion(&store)
if version == nil {
t.Fatal()
return
}
if *version != VERSION {
t.Fatal()
return
}
}

12
db/bolt/access_key.go
View File

@ -13,6 +13,10 @@ func (d *BoltDb) GetAccessKey(projectID int, accessKeyID int) (key db.AccessKey,
return
}
func (d *BoltDb) GetAccessKeyRefs(projectID int, accessKeyID int) (db.ObjectReferrers, error) {
return d.getObjectRefs(projectID, db.AccessKeyProps, accessKeyID)
}
func (d *BoltDb) GetAccessKeys(projectID int, params db.RetrieveQueryParams) ([]db.AccessKey, error) {
var keys []db.AccessKey
err := d.getObjects(projectID, db.AccessKeyProps, params, nil, &keys)
@ -43,7 +47,7 @@ func (d *BoltDb) UpdateAccessKey(key db.AccessKey) error {
return d.updateObject(*key.ProjectID, db.AccessKeyProps, key)
}
func (d *BoltDb) CreateAccessKey(key db.AccessKey) (db.AccessKey, error) {
func (d *BoltDb) CreateAccessKey(key db.AccessKey) (db.AccessKey, error) {
err := key.SerializeSecret()
if err != nil {
return db.AccessKey{}, err
@ -53,9 +57,5 @@ func (d *BoltDb) CreateAccessKey(key db.AccessKey) (db.AccessKey, error) {
}
func (d *BoltDb) DeleteAccessKey(projectID int, accessKeyID int) error {
return d.deleteObject(projectID, db.AccessKeyProps, intObjectID(accessKeyID))
return d.deleteObject(projectID, db.AccessKeyProps, intObjectID(accessKeyID), nil)
}
func (d *BoltDb) DeleteAccessKeySoft(projectID int, accessKeyID int) error {
return d.deleteObjectSoft(projectID, db.AccessKeyProps, intObjectID(accessKeyID))
}

10
db/bolt/environment.go
View File

@ -7,6 +7,10 @@ func (d *BoltDb) GetEnvironment(projectID int, environmentID int) (environment d
return
}
func (d *BoltDb) GetEnvironmentRefs(projectID int, environmentID int) (db.ObjectReferrers, error) {
return d.getObjectRefs(projectID, db.EnvironmentProps, environmentID)
}
func (d *BoltDb) GetEnvironments(projectID int, params db.RetrieveQueryParams) (environment []db.Environment, err error) {
err = d.getObjects(projectID, db.EnvironmentProps, params, nil, &environment)
return
@ -34,9 +38,5 @@ func (d *BoltDb) CreateEnvironment(env db.Environment) (db.Environment, error) {
}
func (d *BoltDb) DeleteEnvironment(projectID int, environmentID int) error {
return d.deleteObject(projectID, db.EnvironmentProps, intObjectID(environmentID))
}
func (d *BoltDb) DeleteEnvironmentSoft(projectID int, environmentID int) error {
return d.deleteObjectSoft(projectID, db.EnvironmentProps, intObjectID(environmentID))
return d.deleteObject(projectID, db.EnvironmentProps, intObjectID(environmentID), nil)
}

6
db/bolt/event.go
View File

@ -24,7 +24,7 @@ import (
//}
// getEvents filter and sort enumerable object passed via parameter.
func (d *BoltDb) getEvents(c enumerable, params db.RetrieveQueryParams, filter func (db.Event) bool) (events []db.Event, err error) {
func (d *BoltDb) getEvents(c enumerable, params db.RetrieveQueryParams, filter func(db.Event) bool) (events []db.Event, err error) {
i := 0 // offset counter
n := 0 // number of added items
@ -105,7 +105,7 @@ func (d *BoltDb) GetUserEvents(userID int, params db.RetrieveQueryParams) (event
}
c := b.Cursor()
events, err = d.getEvents(c, params, func (evt db.Event) bool {
events, err = d.getEvents(c, params, func(evt db.Event) bool {
if evt.ProjectID == nil {
return false
}
@ -127,7 +127,7 @@ func (d *BoltDb) GetEvents(projectID int, params db.RetrieveQueryParams) (events
}
c := b.Cursor()
events, err = d.getEvents(c, params, func (evt db.Event) bool {
events, err = d.getEvents(c, params, func(evt db.Event) bool {
if evt.ProjectID == nil {
return false
}

12
db/bolt/inventory.go
View File

@ -4,7 +4,6 @@ import (
"github.com/ansible-semaphore/semaphore/db"
)
func (d *BoltDb) GetInventory(projectID int, inventoryID int) (inventory db.Inventory, err error) {
err = d.getObject(projectID, db.InventoryProps, intObjectID(inventoryID), &inventory)
@ -21,12 +20,12 @@ func (d *BoltDb) GetInventories(projectID int, params db.RetrieveQueryParams) (i
return
}
func (d *BoltDb) DeleteInventory(projectID int, inventoryID int) error {
return d.deleteObject(projectID, db.InventoryProps, intObjectID(inventoryID))
func (d *BoltDb) GetInventoryRefs(projectID int, inventoryID int) (db.ObjectReferrers, error) {
return d.getObjectRefs(projectID, db.InventoryProps, inventoryID)
}
func (d *BoltDb) DeleteInventorySoft(projectID int, inventoryID int) error {
return d.deleteObjectSoft(projectID, db.InventoryProps, intObjectID(inventoryID))
func (d *BoltDb) DeleteInventory(projectID int, inventoryID int) error {
return d.deleteObject(projectID, db.InventoryProps, intObjectID(inventoryID), nil)
}
func (d *BoltDb) UpdateInventory(inventory db.Inventory) error {
@ -37,6 +36,3 @@ func (d *BoltDb) CreateInventory(inventory db.Inventory) (db.Inventory, error) {
newInventory, err := d.createObject(inventory.ProjectID, db.InventoryProps, inventory)
return newInventory.(db.Inventory), err
}

117
db/bolt/migration.go Normal file
View File

@ -0,0 +1,117 @@
package bolt
import (
"encoding/json"
"github.com/ansible-semaphore/semaphore/db"
"go.etcd.io/bbolt"
)
func (d *BoltDb) IsMigrationApplied(migration db.Migration) (bool, error) {
err := d.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte("migrations"))
if b == nil {
return db.ErrNotFound
}
d := b.Get([]byte(migration.Version))
if d == nil {
return db.ErrNotFound
}
return nil
})
if err == nil {
return true, nil
}
if err == db.ErrNotFound {
return false, nil
}
return false, err
}
func (d *BoltDb) ApplyMigration(m db.Migration) (err error) {
switch m.Version {
case "2.8.26":
err = migration_2_8_28{migration{d.db}}.Apply()
case "2.8.40":
err = migration_2_8_40{migration{d.db}}.Apply()
}
if err != nil {
return
}
return d.db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte("migrations"))
if err != nil {
return err
}
j, err := json.Marshal(m)
if err != nil {
return err
}
return b.Put([]byte(m.Version), j)
})
}
func (d *BoltDb) TryRollbackMigration(m db.Migration) {
switch m.Version {
case "2.8.26":
}
}
type migration struct {
db *bbolt.DB
}
func (d migration) getProjectIDs() (projectIDs []string, err error) {
err = d.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte("project"))
if b == nil {
return nil
}
return b.ForEach(func(id, _ []byte) error {
projectIDs = append(projectIDs, string(id))
return nil
})
})
return
}
func (d migration) getObjects(projectID string, objectPrefix string) (map[string]map[string]interface{}, error) {
repos := make(map[string]map[string]interface{})
err := d.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte("project__" + objectPrefix + "_" + projectID))
if b == nil {
return nil
}
return b.ForEach(func(id, body []byte) error {
r := make(map[string]interface{})
repos[string(id)] = r
return json.Unmarshal(body, &r)
})
})
return repos, err
}
func (d migration) setObject(projectID string, objectPrefix string, objectID string, object map[string]interface{}) error {
return d.db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte("project__" + objectPrefix + "_" + projectID))
if err != nil {
return err
}
j, err := json.Marshal(object)
if err != nil {
return err
}
return b.Put([]byte(objectID), j)
})
}

46
db/bolt/migration_2_8_28.go Normal file
View File

@ -0,0 +1,46 @@
package bolt
import (
"strings"
)
type migration_2_8_28 struct {
migration
}
func (d migration_2_8_28) Apply() (err error) {
projectIDs, err := d.getProjectIDs()
if err != nil {
return
}
repos := make(map[string]map[string]map[string]interface{})
for _, projectID := range projectIDs {
var err2 error
repos[projectID], err2 = d.getObjects(projectID, "repository")
if err2 != nil {
return err2
}
}
for projectID, projectRepos := range repos {
for repoID, repo := range projectRepos {
branch := "master"
url := repo["git_url"].(string)
parts := strings.Split(url, "#")
if len(parts) > 1 {
url, branch = parts[0], parts[1]
}
repo["git_url"] = url
repo["git_branch"] = branch
err = d.setObject(projectID, "repository", repoID, repo)
if err != nil {
return err
}
}
}
return nil
}

84
db/bolt/migration_2_8_28_test.go Normal file
View File

@ -0,0 +1,84 @@
package bolt
import (
"encoding/json"
"go.etcd.io/bbolt"
"testing"
)
func TestMigration_2_8_28_Apply(t *testing.T) {
store := CreateTestStore()
err := store.db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte("project"))
if err != nil {
return err
}
err = b.Put([]byte("0000000001"), []byte("{}"))
if err != nil {
return err
}
r, err := tx.CreateBucketIfNotExists([]byte("project__repository_0000000001"))
if err != nil {
return err
}
err = r.Put([]byte("0000000001"),
[]byte("{\"id\":\"1\",\"project_id\":\"1\",\"git_url\": \"git@github.com/test/test#main\"}"))
return err
})
if err != nil {
t.Fatal(err)
}
err = migration_2_8_28{migration{store.db}}.Apply()
if err != nil {
t.Fatal(err)
}
var repo map[string]interface{}
err = store.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte("project__repository_0000000001"))
str := string(b.Get([]byte("0000000001")))
return json.Unmarshal([]byte(str), &repo)
})
if err != nil {
t.Fatal(err)
}
if repo["git_url"].(string) != "git@github.com/test/test" {
t.Fatal("invalid url")
}
if repo["git_branch"].(string) != "main" {
t.Fatal("invalid branch")
}
}
func TestMigration_2_8_28_Apply2(t *testing.T) {
store := CreateTestStore()
err := store.db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte("project"))
if err != nil {
return err
}
err = b.Put([]byte("0000000001"), []byte("{}"))
return err
})
if err != nil {
t.Fatal(err)
}
err = migration_2_8_28{migration{store.db}}.Apply()
if err != nil {
t.Fatal(err)
}
}

36
db/bolt/migration_2_8_40.go Normal file
View File

@ -0,0 +1,36 @@
package bolt
type migration_2_8_40 struct {
migration
}
func (d migration_2_8_40) Apply() (err error) {
projectIDs, err := d.getProjectIDs()
if err != nil {
return
}
templates := make(map[string]map[string]map[string]interface{})
for _, projectID := range projectIDs {
var err2 error
templates[projectID], err2 = d.getObjects(projectID, "template")
if err2 != nil {
return err2
}
}
for projectID, projectTemplates := range templates {
for repoID, tpl := range projectTemplates {
tpl["name"] = tpl["alias"]
delete(tpl, "alias")
err = d.setObject(projectID, "template", repoID, tpl)
if err != nil {
return err
}
}
}
return
}

84
db/bolt/migration_2_8_40_test.go Normal file
View File

@ -0,0 +1,84 @@
package bolt
import (
"encoding/json"
"go.etcd.io/bbolt"
"testing"
)
func TestMigration_2_8_40_Apply(t *testing.T) {
store := CreateTestStore()
err := store.db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte("project"))
if err != nil {
return err
}
err = b.Put([]byte("0000000001"), []byte("{}"))
if err != nil {
return err
}
r, err := tx.CreateBucketIfNotExists([]byte("project__template_0000000001"))
if err != nil {
return err
}
err = r.Put([]byte("0000000001"),
[]byte("{\"id\":\"1\",\"project_id\":\"1\",\"alias\": \"test123\"}"))
return err
})
if err != nil {
t.Fatal(err)
}
err = migration_2_8_40{migration{store.db}}.Apply()
if err != nil {
t.Fatal(err)
}
var repo map[string]interface{}
err = store.db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte("project__template_0000000001"))
str := string(b.Get([]byte("0000000001")))
return json.Unmarshal([]byte(str), &repo)
})
if err != nil {
t.Fatal(err)
}
if repo["name"].(string) != "test123" {
t.Fatal("invalid name")
}
if repo["alias"] != nil {
t.Fatal("alias must be deleted")
}
}
func TestMigration_2_8_40_Apply2(t *testing.T) {
store := CreateTestStore()
err := store.db.Update(func(tx *bbolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte("project"))
if err != nil {
return err
}
err = b.Put([]byte("0000000001"), []byte("{}"))
return err
})
if err != nil {
t.Fatal(err)
}
err = migration_2_8_28{migration{store.db}}.Apply()
if err != nil {
t.Fatal(err)
}
}

2
db/bolt/project.go
View File

@ -47,7 +47,7 @@ func (d *BoltDb) GetProject(projectID int) (project db.Project, err error) {
}
func (d *BoltDb) DeleteProject(projectID int) error {
return d.deleteObject(0, db.ProjectProps, intObjectID(projectID))
return d.deleteObject(0, db.ProjectProps, intObjectID(projectID), nil)
}
func (d *BoltDb) UpdateProject(project db.Project) error {

27
db/bolt/project_test.go
View File

@ -7,18 +7,13 @@ import (
)
func TestGetProjects(t *testing.T) {
store := createStore()
err := store.Connect()
if err != nil {
t.Fatal(err.Error())
}
store := CreateTestStore()
usr, err := store.CreateUser(db.UserWithPwd{
Pwd: "123456",
User: db.User{
Email: "denguk@example.com",
Name: "Denis Gukov",
Email: "denguk@example.com",
Name: "Denis Gukov",
Username: "fiftin",
},
})
@ -29,7 +24,7 @@ func TestGetProjects(t *testing.T) {
proj1, err := store.CreateProject(db.Project{
Created: time.Now(),
Name: "Test1",
Name: "Test1",
})
if err != nil {
@ -38,8 +33,8 @@ func TestGetProjects(t *testing.T) {
_, err = store.CreateProjectUser(db.ProjectUser{
ProjectID: proj1.ID,
UserID: usr.ID,
Admin: true,
UserID: usr.ID,
Admin: true,
})
if err != nil {
@ -59,19 +54,13 @@ func TestGetProjects(t *testing.T) {
}
func TestGetProject(t *testing.T) {
store := createStore()
err := store.Connect()
if err != nil {
t.Fatal(err.Error())
}
store := CreateTestStore()
proj, err := store.CreateProject(db.Project{
Created: time.Now(),
Name: "Test1",
Name: "Test1",
})
if err != nil {
t.Fatal(err.Error())
}

19
db/bolt/repository.go
View File

@ -13,25 +13,32 @@ func (d *BoltDb) GetRepository(projectID int, repositoryID int) (repository db.R
return
}
func (d *BoltDb) GetRepositoryRefs(projectID int, repositoryID int) (db.ObjectReferrers, error) {
return d.getObjectRefs(projectID, db.RepositoryProps, repositoryID)
}
func (d *BoltDb) GetRepositories(projectID int, params db.RetrieveQueryParams) (repositories []db.Repository, err error) {
err = d.getObjects(projectID, db.RepositoryProps, params, nil, &repositories)
return
}
func (d *BoltDb) UpdateRepository(repository db.Repository) error {
err := repository.Validate()
if err != nil {
return err
}
return d.updateObject(repository.ProjectID, db.RepositoryProps, repository)
}
func (d *BoltDb) CreateRepository(repository db.Repository) (db.Repository, error) {
err := repository.Validate()
if err != nil {
return db.Repository{}, err
}
newRepo, err := d.createObject(repository.ProjectID, db.RepositoryProps, repository)
return newRepo.(db.Repository), err
}
func (d *BoltDb) DeleteRepository(projectID int, repositoryId int) error {
return d.deleteObject(projectID, db.RepositoryProps, intObjectID(repositoryId))
return d.deleteObject(projectID, db.RepositoryProps, intObjectID(repositoryId), nil)
}
func (d *BoltDb) DeleteRepositorySoft(projectID int, repositoryId int) error {
return d.deleteObjectSoft(projectID, db.RepositoryProps, intObjectID(repositoryId))
}

25
db/bolt/schedule.go
View File

@ -1,6 +1,9 @@
package bolt
import "github.com/ansible-semaphore/semaphore/db"
import (
"github.com/ansible-semaphore/semaphore/db"
"go.etcd.io/bbolt"
)
func (d *BoltDb) GetSchedules() (schedules []db.Schedule, err error) {
var allProjects []db.Project
@ -28,7 +31,6 @@ func (d *BoltDb) GetProjectSchedules(projectID int) (schedules []db.Schedule, er
return
}
func (d *BoltDb) GetTemplateSchedules(projectID int, templateID int) (schedules []db.Schedule, err error) {
schedules = make([]db.Schedule, 0)
@ -64,6 +66,21 @@ func (d *BoltDb) GetSchedule(projectID int, scheduleID int) (schedule db.Schedul
return
}
func (d *BoltDb) DeleteSchedule(projectID int, scheduleID int) error {
return d.deleteObject(projectID, db.ScheduleProps, intObjectID(scheduleID))
func (d *BoltDb) deleteSchedule(projectID int, scheduleID int, tx *bbolt.Tx) error {
return d.deleteObject(projectID, db.ScheduleProps, intObjectID(scheduleID), tx)
}
func (d *BoltDb) DeleteSchedule(projectID int, scheduleID int) error {
return d.db.Update(func(tx *bbolt.Tx) error {
return d.deleteSchedule(projectID, scheduleID, tx)
})
}
func (d *BoltDb) SetScheduleCommitHash(projectID int, scheduleID int, hash string) error {
schedule, err := d.GetSchedule(projectID, scheduleID)
if err != nil {
return err
}
schedule.LastCommitHash = &hash
return d.updateObject(projectID, db.ScheduleProps, schedule)
}

19
db/bolt/session.go
View File

@ -2,19 +2,21 @@ package bolt
import (
"github.com/ansible-semaphore/semaphore/db"
"reflect"
"time"
)
var globalTokenObject = db.ObjectProperties{
TableName: "token",
}
type globalToken struct {
ID string `db:"id" json:"id"`
UserID int `db:"user_id" json:"user_id"`
ID string `db:"id" json:"id"`
UserID int `db:"user_id" json:"user_id"`
}
var globalTokenObject = db.ObjectProps{
TableName: "token",
PrimaryColumnName: "id",
Type: reflect.TypeOf(globalToken{}),
IsGlobal: true,
}
func (d *BoltDb) CreateSession(session db.Session) (db.Session, error) {
newSession, err := d.createObject(session.UserID, db.SessionProps, session)
@ -89,7 +91,6 @@ func (d *BoltDb) TouchSession(userID int, sessionID int) (err error) {
}
func (d *BoltDb) GetAPITokens(userID int) (tokens []db.APIToken, err error) {
err = d.getObjects(userID, db.SessionProps, db.RetrieveQueryParams{}, nil, &tokens)
err = d.getObjects(userID, db.TokenProps, db.RetrieveQueryParams{}, nil, &tokens)
return
}

31
db/bolt/task.go
View File

@ -28,7 +28,7 @@ func (d *BoltDb) CreateTaskOutput(output db.TaskOutput) (db.TaskOutput, error) {
return newOutput.(db.TaskOutput), nil
}
func (d *BoltDb) getTasks(projectID int, template *db.Template, params db.RetrieveQueryParams) (tasksWithTpl []db.TaskWithTpl, err error) {
func (d *BoltDb) getTasks(projectID int, templateID *int, params db.RetrieveQueryParams) (tasksWithTpl []db.TaskWithTpl, err error) {
var tasks []db.Task
err = d.getObjects(0, db.TaskProps, params, func(tsk interface{}) bool {
@ -38,7 +38,7 @@ func (d *BoltDb) getTasks(projectID int, template *db.Template, params db.Retrie
return false
}
if template != nil && task.TemplateID != template.ID {
if templateID != nil && task.TemplateID != *templateID {
return false
}
@ -56,16 +56,16 @@ func (d *BoltDb) getTasks(projectID int, template *db.Template, params db.Retrie
for i, task := range tasks {
tpl, ok := templates[task.TemplateID]
if !ok {
if template == nil {
tpl, _ = d.GetTemplate(task.ProjectID, task.TemplateID)
if templateID == nil {
tpl, _ = d.getRawTemplate(task.ProjectID, task.TemplateID)
} else {
tpl = *template
tpl, _ = d.getRawTemplate(task.ProjectID, *templateID)
}
templates[task.TemplateID] = tpl
}
tasksWithTpl[i] = db.TaskWithTpl{Task: task}
tasksWithTpl[i].TemplatePlaybook = tpl.Playbook
tasksWithTpl[i].TemplateAlias = tpl.Alias
tasksWithTpl[i].TemplateAlias = tpl.Name
tasksWithTpl[i].TemplateType = tpl.Type
if task.UserID != nil {
usr, ok := users[*task.UserID]
@ -103,32 +103,33 @@ func (d *BoltDb) GetTask(projectID int, taskID int) (task db.Task, err error) {
return
}
func (d *BoltDb) GetTemplateTasks(template db.Template, params db.RetrieveQueryParams) ([]db.TaskWithTpl, error) {
return d.getTasks(template.ProjectID, &template, params)
func (d *BoltDb) GetTemplateTasks(projectID int, templateID int, params db.RetrieveQueryParams) ([]db.TaskWithTpl, error) {
return d.getTasks(projectID, &templateID, params)
}
func (d *BoltDb) GetProjectTasks(projectID int, params db.RetrieveQueryParams) ([]db.TaskWithTpl, error) {
return d.getTasks(projectID, nil, params)
}
func (d *BoltDb) DeleteTaskWithOutputs(projectID int, taskID int) (err error) {
func (d *BoltDb) deleteTaskWithOutputs(projectID int, taskID int, tx *bbolt.Tx) (err error) {
// check if task exists in the project
_, err = d.GetTask(projectID, taskID)
if err != nil {
return
}
err = d.deleteObject(0, db.TaskProps, intObjectID(taskID))
err = d.deleteObject(0, db.TaskProps, intObjectID(taskID), tx)
if err != nil {
return
}
_ = d.db.Update(func(tx *bbolt.Tx) error {
return tx.DeleteBucket(makeBucketId(db.TaskOutputProps, taskID))
return tx.DeleteBucket(makeBucketId(db.TaskOutputProps, taskID))
}
func (d *BoltDb) DeleteTaskWithOutputs(projectID int, taskID int) error {
return d.db.Update(func(tx *bbolt.Tx) error {
return d.deleteTaskWithOutputs(projectID, taskID, tx)
})
return
}
func (d *BoltDb) GetTaskOutputs(projectID int, taskID int) (outputs []db.TaskOutput, err error) {

77
db/bolt/template.go
View File

@ -2,6 +2,7 @@ package bolt
import (
"github.com/ansible-semaphore/semaphore/db"
"go.etcd.io/bbolt"
)
func (d *BoltDb) CreateTemplate(template db.Template) (newTemplate db.Template, err error) {
@ -11,6 +12,7 @@ func (d *BoltDb) CreateTemplate(template db.Template) (newTemplate db.Template,
return
}
template.SurveyVarsJSON = db.ObjectToJSON(template.SurveyVars)
newTpl, err := d.createObject(template.ProjectID, db.TemplateProps, template)
if err != nil {
return
@ -27,19 +29,27 @@ func (d *BoltDb) UpdateTemplate(template db.Template) error {
return err
}
template.SurveyVarsJSON = db.ObjectToJSON(template.SurveyVars)
return d.updateObject(template.ProjectID, db.TemplateProps, template)
}
func (d *BoltDb) getTemplates(projectID int, viewID *int, params db.RetrieveQueryParams) (templates []db.Template, err error) {
var filter func(interface{}) bool
if viewID != nil {
filter = func (tpl interface{}) bool {
template := tpl.(db.Template)
return template.ViewID != nil && *template.ViewID == *viewID
func (d *BoltDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams) (templates []db.Template, err error) {
var ftr = func(tpl interface{}) bool {
template := tpl.(db.Template)
var res = true
if filter.ViewID != nil {
res = res && template.ViewID != nil && *template.ViewID == *filter.ViewID
}
if filter.BuildTemplateID != nil {
res = res && template.BuildTemplateID != nil && *template.BuildTemplateID == *filter.BuildTemplateID
if filter.AutorunOnly {
res = res && template.Autorun
}
}
return res
}
err = d.getObjects(projectID, db.TemplateProps, params, filter, &templates)
err = d.getObjects(projectID, db.TemplateProps, params, ftr, &templates)
if err != nil {
return
@ -50,12 +60,13 @@ func (d *BoltDb) getTemplates(projectID int, viewID *int, params db.RetrieveQuer
return
}
func (d *BoltDb) GetTemplates(projectID int, params db.RetrieveQueryParams) ( []db.Template, error) {
return d.getTemplates(projectID, nil, params)
func (d *BoltDb) getRawTemplate(projectID int, templateID int) (template db.Template, err error) {
err = d.getObject(projectID, db.TemplateProps, intObjectID(templateID), &template)
return
}
func (d *BoltDb) GetTemplate(projectID int, templateID int) (template db.Template, err error) {
err = d.getObject(projectID, db.TemplateProps, intObjectID(templateID), &template)
template, err = d.getRawTemplate(projectID, templateID)
if err != nil {
return
}
@ -63,6 +74,48 @@ func (d *BoltDb) GetTemplate(projectID int, templateID int) (template db.Templat
return
}
func (d *BoltDb) DeleteTemplate(projectID int, templateID int) error {
return d.deleteObject(projectID, db.TemplateProps, intObjectID(templateID))
func (d *BoltDb) deleteTemplate(projectID int, templateID int, tx *bbolt.Tx) (err error) {
inUse, err := d.isObjectInUse(projectID, db.TemplateProps, intObjectID(templateID), db.TemplateProps)
if err != nil {
return err
}
if inUse {
return db.ErrInvalidOperation
}
tasks, err := d.GetTemplateTasks(projectID, templateID, db.RetrieveQueryParams{})
if err != nil {
return
}
for _, task := range tasks {
err = d.deleteTaskWithOutputs(projectID, task.ID, tx)
if err != nil {
return
}
}
schedules, err := d.GetTemplateSchedules(projectID, templateID)
if err != nil {
return
}
for _, sch := range schedules {
err = d.deleteSchedule(projectID, sch.ID, tx)
if err != nil {
return
}
}
return d.deleteObject(projectID, db.TemplateProps, intObjectID(templateID), tx)
}
func (d *BoltDb) DeleteTemplate(projectID int, templateID int) error {
return d.db.Update(func(tx *bbolt.Tx) error {
return d.deleteTemplate(projectID, templateID, tx)
})
}
func (d *BoltDb) GetTemplateRefs(projectID int, templateID int) (db.ObjectReferrers, error) {
return d.getObjectRefs(projectID, db.TemplateProps, templateID)
}

60
db/bolt/user.go
View File

@ -87,7 +87,7 @@ func (d *BoltDb) DeleteUser(userID int) error {
_ = d.DeleteProjectUser(p.ID, userID)
}
return d.deleteObject(0, db.UserProps, intObjectID(userID))
return d.deleteObject(0, db.UserProps, intObjectID(userID), nil)
}
func (d *BoltDb) UpdateUser(user db.UserWithPwd) error {
@ -164,7 +164,7 @@ func (d *BoltDb) UpdateProjectUser(projectUser db.ProjectUser) error {
}
func (d *BoltDb) DeleteProjectUser(projectID, userID int) error {
return d.deleteObject(projectID, db.ProjectUserProps, intObjectID(userID))
return d.deleteObject(projectID, db.ProjectUserProps, intObjectID(userID), nil)
}
//GetUser retrieves a user from the database by ID
@ -195,59 +195,3 @@ func (d *BoltDb) GetUserByLoginOrEmail(login string, email string) (existingUser
err = db.ErrNotFound
return
}
func (d *BoltDb) CreatePlaceholderUser() (err error) {
user := db.User{
Created: db.GetParsedTime(time.Now()),
}
_, err = d.createObject(0, db.UserProps, user)
return
}
func (d *BoltDb) GetPlaceholderUser() (placeholderUser db.User, err error) {
var users []db.User
err = d.getObjects(0, db.UserProps, db.RetrieveQueryParams{}, nil, &users)
if err != nil {
return
}
for _, usr := range users {
if usr.Username == "" {
placeholderUser = usr
return
}
}
err = db.ErrNotFound
return
}
//func (d *BoltDb) HasPlaceholderUser() (bool, error) {
// _, err := d.getPlaceholderUser()
//
// if err == nil {
// return true, nil
// }
//
// if err == db.ErrNotFound {
// return false, nil
// }
//
// return false, err
//}
//
//func (d *BoltDb) ReplacePlaceholderUser(user db.UserWithPwd) (newUser db.User, err error) {
// placeholder, err := d.getPlaceholderUser()
// if err != nil {
// return
// }
// user.ID = placeholder.ID
// err = d.UpdateUser(user)
// if err != nil {
// return
// }
// newUser = user.User
// return
//}

42
db/bolt/user_test.go
View File

@ -7,19 +7,13 @@ import (
)
func TestBoltDb_UpdateProjectUser(t *testing.T) {
store := createStore()
err := store.Connect()
if err != nil {
t.Fatal(err.Error())
}
store := CreateTestStore()
usr, err := store.CreateUser(db.UserWithPwd{
Pwd: "123456",
User: db.User{
Email: "denguk@example.com",
Name: "Denis Gukov",
Email: "denguk@example.com",
Name: "Denis Gukov",
Username: "fiftin",
},
})
@ -30,7 +24,7 @@ func TestBoltDb_UpdateProjectUser(t *testing.T) {
proj1, err := store.CreateProject(db.Project{
Created: time.Now(),
Name: "Test1",
Name: "Test1",
})
if err != nil {
@ -39,8 +33,8 @@ func TestBoltDb_UpdateProjectUser(t *testing.T) {
projUser, err := store.CreateProjectUser(db.ProjectUser{
ProjectID: proj1.ID,
UserID: usr.ID,
Admin: true,
UserID: usr.ID,
Admin: true,
})
if err != nil {
@ -56,18 +50,13 @@ func TestBoltDb_UpdateProjectUser(t *testing.T) {
}
func TestGetUsers(t *testing.T) {
store := createStore()
err := store.Connect()
store := CreateTestStore()
if err != nil {
t.Fatal(err.Error())
}
_, err = store.CreateUser(db.UserWithPwd{
_, err := store.CreateUser(db.UserWithPwd{
Pwd: "123456",
User: db.User{
Email: "denguk@example.com",
Name: "Denis Gukov",
Email: "denguk@example.com",
Name: "Denis Gukov",
Username: "fiftin",
},
})
@ -89,18 +78,13 @@ func TestGetUsers(t *testing.T) {
}
func TestGetUser(t *testing.T) {
store := createStore()
err := store.Connect()
if err != nil {
t.Fatal(err.Error())
}
store := CreateTestStore()
usr, err := store.CreateUser(db.UserWithPwd{
Pwd: "123456",
User: db.User{
Email: "denguk@example.com",
Name: "Denis Gukov",
Email: "denguk@example.com",
Name: "Denis Gukov",
Username: "fiftin",
},
})

7
db/bolt/view.go
View File

@ -22,7 +22,7 @@ func (d *BoltDb) CreateView(view db.View) (db.View, error) {
}
func (d *BoltDb) DeleteView(projectID int, viewID int) error {
return d.deleteObject(projectID, db.ViewProps, intObjectID(viewID))
return d.deleteObject(projectID, db.ViewProps, intObjectID(viewID), nil)
}
func (d *BoltDb) SetViewPositions(projectID int, positions map[int]int) error {
@ -39,8 +39,3 @@ func (d *BoltDb) SetViewPositions(projectID int, positions map[int]int) error {
}
return nil
}
func (d *BoltDb) GetViewTemplates(projectID int, viewID int, params db.RetrieveQueryParams) ( []db.Template, error) {
return d.getTemplates(projectID, &viewID, params)
}

14
db/bolt/view_test.go
View File

@ -8,12 +8,7 @@ import (
)
func TestGetViews(t *testing.T) {
store := createStore()
err := store.Connect()
if err != nil {
t.Fatal(err.Error())
}
store := CreateTestStore()
proj1, err := store.CreateProject(db.Project{
Created: time.Now(),
@ -56,12 +51,7 @@ func TestGetViews(t *testing.T) {
}
func TestSetViewPositions(t *testing.T) {
store := createStore()
err := store.Connect()
if err != nil {
t.Fatal(err.Error())
}
store := CreateTestStore()
proj1, err := store.CreateProject(db.Project{
Created: time.Now(),

140
db/sql/SqlDb.go
View File

@ -11,6 +11,7 @@ import (
"github.com/gobuffalo/packr"
_ "github.com/lib/pq"
"github.com/masterminds/squirrel"
"reflect"
"regexp"
"strconv"
"strings"
@ -91,7 +92,7 @@ func (d *SqlDb) prepareQueryWithDialect(query string, dialect gorp.Dialect) stri
return query
}
func (d *SqlDb) prepareQuery(query string) string {
func (d *SqlDb) PrepareQuery(query string) string {
return d.prepareQueryWithDialect(query, d.sql.Dialect)
}
@ -102,7 +103,7 @@ func (d *SqlDb) insert(primaryKeyColumnName string, query string, args ...interf
case gorp.PostgresDialect:
query += " returning " + primaryKeyColumnName
err := d.sql.QueryRow(d.prepareQuery(query), args...).Scan(&insertId)
err := d.sql.QueryRow(d.PrepareQuery(query), args...).Scan(&insertId)
if err != nil {
return 0, err
@ -125,16 +126,16 @@ func (d *SqlDb) insert(primaryKeyColumnName string, query string, args ...interf
}
func (d *SqlDb) exec(query string, args ...interface{}) (sql.Result, error) {
q := d.prepareQuery(query)
q := d.PrepareQuery(query)
return d.sql.Exec(q, args...)
}
func (d *SqlDb) selectOne(holder interface{}, query string, args ...interface{}) error {
return d.sql.SelectOne(holder, d.prepareQuery(query), args...)
return d.sql.SelectOne(holder, d.PrepareQuery(query), args...)
}
func (d *SqlDb) selectAll(i interface{}, query string, args ...interface{}) ([]interface{}, error) {
q := d.prepareQuery(query)
q := d.PrepareQuery(query)
return d.sql.Select(i, q, args...)
}
@ -182,7 +183,7 @@ func createDb() error {
return nil
}
func (d *SqlDb) getObject(projectID int, props db.ObjectProperties, objectID int, object interface{}) (err error) {
func (d *SqlDb) getObject(projectID int, props db.ObjectProps, objectID int, object interface{}) (err error) {
q := squirrel.Select("*").
From(props.TableName).
Where("id=?", objectID)
@ -208,7 +209,7 @@ func (d *SqlDb) getObject(projectID int, props db.ObjectProperties, objectID int
return
}
func (d *SqlDb) getObjects(projectID int, props db.ObjectProperties, params db.RetrieveQueryParams, objects interface{}) (err error) {
func (d *SqlDb) getObjects(projectID int, props db.ObjectProps, params db.RetrieveQueryParams, objects interface{}) (err error) {
q := squirrel.Select("*").
From(props.TableName+" pe").
Where("pe.project_id=?", projectID)
@ -238,7 +239,7 @@ func (d *SqlDb) getObjects(projectID int, props db.ObjectProperties, params db.R
return
}
func (d *SqlDb) deleteObject(projectID int, props db.ObjectProperties, objectID int) error {
func (d *SqlDb) deleteObject(projectID int, props db.ObjectProps, objectID int) error {
return validateMutationResult(
d.exec(
"delete from "+props.TableName+" where project_id=? and id=?",
@ -246,14 +247,6 @@ func (d *SqlDb) deleteObject(projectID int, props db.ObjectProperties, objectID
objectID))
}
func (d *SqlDb) deleteObjectSoft(projectID int, props db.ObjectProperties, objectID int) error {
return validateMutationResult(
d.exec(
"update "+props.TableName+" set removed=1 where project_id=? and id=?",
projectID,
objectID))
}
func (d *SqlDb) Close() error {
return d.sql.Db.Close()
}
@ -338,14 +331,119 @@ func getSqlForTable(tableName string, p db.RetrieveQueryParams) (string, []inter
return q.ToSql()
}
func (d *SqlDb) getObjectRefs(projectID int, objectProps db.ObjectProps, objectID int) (refs db.ObjectReferrers, err error) {
refs.Templates, err = d.getObjectRefsFrom(projectID, objectProps, objectID, db.TemplateProps)
if err != nil {
return
}
refs.Repositories, err = d.getObjectRefsFrom(projectID, objectProps, objectID, db.RepositoryProps)
if err != nil {
return
}
refs.Inventories, err = d.getObjectRefsFrom(projectID, objectProps, objectID, db.InventoryProps)
if err != nil {
return
}
templates, err := d.getObjectRefsFrom(projectID, objectProps, objectID, db.ScheduleProps)
if err != nil {
return
}
for _, st := range templates {
exists := false
for _, tpl := range refs.Templates {
if tpl.ID == st.ID {
exists = true
break
}
}
if exists {
continue
}
refs.Templates = append(refs.Templates, st)
}
return
}
func (d *SqlDb) getObjectRefsFrom(
projectID int,
objectProps db.ObjectProps,
objectID int,
referringObjectProps db.ObjectProps,
) (referringObjs []db.ObjectReferrer, err error) {
referringObjs = make([]db.ObjectReferrer, 0)
fields, err := objectProps.GetReferringFieldsFrom(referringObjectProps.Type)
cond := ""
vals := []interface{}{projectID}
for _, f := range fields {
if cond != "" {
cond += " or "
}
cond += f + " = ?"
vals = append(vals, objectID)
}
if cond == "" {
return
}
var referringObjects reflect.Value
if referringObjectProps.Type == db.ScheduleProps.Type {
var referringSchedules []db.Schedule
_, err = d.selectAll(&referringSchedules, "select template_id id from project__schedule where project_id = ? and ("+cond+")", vals...)
if err != nil {
return
}
if len(referringSchedules) == 0 {
return
}
var ids []string
for _, schedule := range referringSchedules {
ids = append(ids, strconv.Itoa(schedule.ID))
}
referringObjects = reflect.New(reflect.SliceOf(db.TemplateProps.Type))
_, err = d.selectAll(referringObjects.Interface(),
"select id, name from project__template where id in ("+strings.Join(ids, ",")+")")
} else {
referringObjects = reflect.New(reflect.SliceOf(referringObjectProps.Type))
_, err = d.selectAll(
referringObjects.Interface(),
"select id, name from "+referringObjectProps.TableName+" where project_id = ? and "+cond,
vals...)
}
if err != nil {
return
}
for i := 0; i < referringObjects.Elem().Len(); i++ {
id := int(referringObjects.Elem().Index(i).FieldByName("ID").Int())
name := referringObjects.Elem().Index(i).FieldByName("Name").String()
referringObjs = append(referringObjs, db.ObjectReferrer{ID: id, Name: name})
}
return
}
func (d *SqlDb) Sql() *gorp.DbMap {
return d.sql
}
func (d *SqlDb) IsInitialized() (bool, error) {
exists, err := d.sql.SelectInt(d.prepareQuery("select count(1) from migrations"))
if err != nil {
return false, nil
}
return exists > 0, nil
_, err := d.sql.SelectInt(d.PrepareQuery("select count(1) from migrations"))
return err == nil, nil
}

92
db/sql/Version.go
View File

@ -1,92 +0,0 @@
package sql
import (
"fmt"
"strings"
"time"
)
// Version represents an sql schema version
type Version struct {
Major int
Minor int
Patch int
Build string
UpgradedDate *time.Time
Notes *string
}
// Versions holds all sql schema version references
var Versions []*Version
// VersionString returns a well formatted string of the current Version
func (version *Version) VersionString() string {
s := fmt.Sprintf("%d.%d.%d", version.Major, version.Minor, version.Patch)
if len(version.Build) == 0 {
return s
}
return fmt.Sprintf("%s-%s", s, version.Build)
}
// HumanoidVersion adds a v to the VersionString
func (version *Version) HumanoidVersion() string {
return "v" + version.VersionString()
}
// GetPath is the humanoid version with the file format appended
func (version *Version) GetPath() string {
return version.HumanoidVersion() + ".sql"
}
//GetErrPath is the humanoid version with '.err' and file format appended
func (version *Version) GetErrPath() string {
return version.HumanoidVersion() + ".err.sql"
}
// GetSQL takes a path to an SQL file and returns it from packr as a slice of strings separated by newlines
func (version *Version) GetSQL(path string) (queries []string) {
sql, err := dbAssets.MustString(path)
if err != nil {
panic(err)
}
queries = strings.Split(strings.ReplaceAll(sql, ";\r\n", ";\n"), ";\n")
return
}
func init() {
Versions = []*Version{
{},
{Major: 1},
{Major: 1, Minor: 2},
{Major: 1, Minor: 3},
{Major: 1, Minor: 4},
{Major: 1, Minor: 5},
{Major: 1, Minor: 6},
{Major: 1, Minor: 7},
{Major: 1, Minor: 8},
{Major: 1, Minor: 9},
{Major: 2, Minor: 2, Patch: 1},
{Major: 2, Minor: 3},
{Major: 2, Minor: 3, Patch: 1},
{Major: 2, Minor: 3, Patch: 2},
{Major: 2, Minor: 4},
{Major: 2, Minor: 5},
{Major: 2, Minor: 5, Patch: 2},
{Major: 2, Minor: 7, Patch: 1},
{Major: 2, Minor: 7, Patch: 4},
{Major: 2, Minor: 7, Patch: 6},
{Major: 2, Minor: 7, Patch: 8},
{Major: 2, Minor: 7, Patch: 9},
{Major: 2, Minor: 7, Patch: 10},
{Major: 2, Minor: 7, Patch: 12},
{Major: 2, Minor: 7, Patch: 13},
{Major: 2, Minor: 8, Patch: 0},
{Major: 2, Minor: 8, Patch: 1},
{Major: 2, Minor: 8, Patch: 7},
{Major: 2, Minor: 8, Patch: 8},
{Major: 2, Minor: 8, Patch: 20},
}
}

8
db/sql/access_key.go
View File

@ -15,6 +15,10 @@ func (d *SqlDb) GetAccessKey(projectID int, accessKeyID int) (key db.AccessKey,
return
}
func (d *SqlDb) GetAccessKeyRefs(projectID int, keyID int) (db.ObjectReferrers, error) {
return d.getObjectRefs(projectID, db.AccessKeyProps, keyID)
}
func (d *SqlDb) GetAccessKeys(projectID int, params db.RetrieveQueryParams) ([]db.AccessKey, error) {
var keys []db.AccessKey
err := d.getObjects(projectID, db.AccessKeyProps, params, &keys)
@ -83,7 +87,3 @@ func (d *SqlDb) CreateAccessKey(key db.AccessKey) (newKey db.AccessKey, err erro
func (d *SqlDb) DeleteAccessKey(projectID int, accessKeyID int) error {
return d.deleteObject(projectID, db.AccessKeyProps, accessKeyID)
}
func (d *SqlDb) DeleteAccessKeySoft(projectID int, accessKeyID int) error {
return d.deleteObjectSoft(projectID, db.AccessKeyProps, accessKeyID)
}

12
db/sql/environment.go
View File

@ -1,6 +1,8 @@
package sql
import "github.com/ansible-semaphore/semaphore/db"
import (
"github.com/ansible-semaphore/semaphore/db"
)
func (d *SqlDb) GetEnvironment(projectID int, environmentID int) (db.Environment, error) {
var environment db.Environment
@ -8,6 +10,10 @@ func (d *SqlDb) GetEnvironment(projectID int, environmentID int) (db.Environment
return environment, err
}
func (d *SqlDb) GetEnvironmentRefs(projectID int, environmentID int) (db.ObjectReferrers, error) {
return d.getObjectRefs(projectID, db.EnvironmentProps, environmentID)
}
func (d *SqlDb) GetEnvironments(projectID int, params db.RetrieveQueryParams) ([]db.Environment, error) {
var environment []db.Environment
err := d.getObjects(projectID, db.EnvironmentProps, params, &environment)
@ -56,7 +62,3 @@ func (d *SqlDb) CreateEnvironment(env db.Environment) (newEnv db.Environment, er
func (d *SqlDb) DeleteEnvironment(projectID int, environmentID int) error {
return d.deleteObject(projectID, db.EnvironmentProps, environmentID)
}
func (d *SqlDb) DeleteEnvironmentSoft(projectID int, environmentID int) error {
return d.deleteObjectSoft(projectID, db.EnvironmentProps, environmentID)
}

11
db/sql/inventory.go
View File

@ -18,12 +18,12 @@ func (d *SqlDb) GetInventories(projectID int, params db.RetrieveQueryParams) ([]
return inventories, err
}
func (d *SqlDb) DeleteInventory(projectID int, inventoryID int) error {
return d.deleteObject(projectID, db.InventoryProps, inventoryID)
func (d *SqlDb) GetInventoryRefs(projectID int, inventoryID int) (db.ObjectReferrers, error) {
return d.getObjectRefs(projectID, db.InventoryProps, inventoryID)
}
func (d *SqlDb) DeleteInventorySoft(projectID int, inventoryID int) error {
return d.deleteObjectSoft(projectID, db.InventoryProps, inventoryID)
func (d *SqlDb) DeleteInventory(projectID int, inventoryID int) error {
return d.deleteObject(projectID, db.InventoryProps, inventoryID)
}
func (d *SqlDb) UpdateInventory(inventory db.Inventory) error {
@ -58,6 +58,3 @@ func (d *SqlDb) CreateInventory(inventory db.Inventory) (newInventory db.Invento
newInventory.ID = insertID
return
}

190
db/sql/migration.go
View File

@ -3,21 +3,49 @@ package sql
import (
"fmt"
log "github.com/Sirupsen/logrus"
"github.com/ansible-semaphore/semaphore/db"
"github.com/go-gorp/gorp/v3"
"regexp"
"strings"
"time"
)
var (
autoIncrementRE = regexp.MustCompile(`(?i)\bautoincrement\b`)
serialRE = regexp.MustCompile(`(?i)\binteger primary key autoincrement\b`)
dateTimeTypeRE = regexp.MustCompile(`(?i)\bdatetime\b`)
tinyintRE = regexp.MustCompile(`(?i)\btinyint\b`)
longtextRE = regexp.MustCompile(`(?i)\blongtext\b`)
ifExistsRE = regexp.MustCompile(`(?i)\bif exists\b`)
dropForeignKey = regexp.MustCompile(`(?i)\bdrop foreign key\b`)
serialRE = regexp.MustCompile(`(?i)\binteger primary key autoincrement\b`)
dateTimeTypeRE = regexp.MustCompile(`(?i)\bdatetime\b`)
tinyintRE = regexp.MustCompile(`(?i)\btinyint\b`)
longtextRE = regexp.MustCompile(`(?i)\blongtext\b`)
ifExistsRE = regexp.MustCompile(`(?i)\bif exists\b`)
changeRE = regexp.MustCompile(`^alter table \x60(\w+)\x60 change \x60(\w+)\x60 \x60(\w+)\x60 ([\w\(\)]+)( not null)?$`)
//dropForeignKeyRE = regexp.MustCompile(`^alter table \x60(\w+)\x60 drop foreign key \x60(\w+)\x60 /\* postgres:\x60(\w*)\x60 mysql:\x60(\w*)\x60 \*/$`)
dropForeignKey2RE = regexp.MustCompile(`(?i)\bdrop foreign key\b`)
)
// getVersionPath is the humanoid version with the file format appended
func getVersionPath(version db.Migration) string {
return version.HumanoidVersion() + ".sql"
}
// getVersionErrPath is the humanoid version with '.err' and file format appended
func getVersionErrPath(version db.Migration) string {
return version.HumanoidVersion() + ".err.sql"
}
// getVersionSQL takes a path to an SQL file and returns it from packr as
// a slice of strings separated by newlines
func getVersionSQL(path string) (queries []string) {
sql, err := dbAssets.MustString(path)
if err != nil {
panic(err)
}
queries = strings.Split(strings.ReplaceAll(sql, ";\r\n", ";\n"), ";\n")
for i := range queries {
queries[i] = strings.Trim(queries[i], "\r\n\t ")
}
return
}
// prepareMigration converts migration SQLite-query to current dialect.
// Supported MySQL and Postgres dialects.
func (d *SqlDb) prepareMigration(query string) string {
@ -26,44 +54,94 @@ func (d *SqlDb) prepareMigration(query string) string {
query = autoIncrementRE.ReplaceAllString(query, "auto_increment")
query = ifExistsRE.ReplaceAllString(query, "")
case gorp.PostgresDialect:
query = serialRE.ReplaceAllString(query, "serial primary key")
query = identifierQuoteRE.ReplaceAllString(query, "\"")
m := changeRE.FindStringSubmatch(query)
if m != nil {
tableName := m[1]
oldColumnName := m[2]
newColumnName := m[3]
columnType := m[4]
columnNotNull := m[5] != ""
var queries []string
queries = append(queries,
"alter table `"+tableName+"` alter column `"+oldColumnName+"` type "+columnType)
if columnNotNull {
queries = append(queries,
"alter table `"+tableName+"` alter column `"+oldColumnName+"` set not null")
} else {
queries = append(queries,
"alter table `"+tableName+"` alter column `"+oldColumnName+"` drop not null")
}
if oldColumnName != newColumnName {
queries = append(queries,
"alter table `"+tableName+"` rename column `"+oldColumnName+"` to `"+newColumnName+"`")
}
query = strings.Join(queries, "; ")
}
query = dateTimeTypeRE.ReplaceAllString(query, "timestamp")
query = tinyintRE.ReplaceAllString(query, "smallint")
query = longtextRE.ReplaceAllString(query, "text")
query = dropForeignKey.ReplaceAllString(query, "drop constraint")
query = serialRE.ReplaceAllString(query, "serial primary key")
query = dropForeignKey2RE.ReplaceAllString(query, "drop constraint")
query = identifierQuoteRE.ReplaceAllString(query, "\"")
}
return query
}
// isMigrationApplied queries the database to see if a migration table with this version id exists already
func (d *SqlDb) isMigrationApplied(version *Version) (bool, error) {
exists, err := d.sql.SelectInt(d.prepareQuery("select count(1) as ex from migrations where version=?"), version.VersionString())
// IsMigrationApplied queries the database to see if a migration table with this version id exists already
func (d *SqlDb) IsMigrationApplied(migration db.Migration) (bool, error) {
initialized, err := d.IsInitialized()
if err != nil {
fmt.Println("Creating migrations table")
query := d.prepareMigration(initialSQL)
if _, err = d.exec(query); err != nil {
panic(err)
}
return false, err
}
return d.isMigrationApplied(version)
if !initialized {
return false, nil
}
exists, err := d.sql.SelectInt(
d.PrepareQuery("select count(1) as ex from migrations where version = ?"),
migration.Version)
if err != nil {
return false, err
}
return exists > 0, nil
}
// Run executes a database migration
func (d *SqlDb) applyMigration(version *Version) error {
fmt.Printf("Executing migration %s (at %v)...\n", version.HumanoidVersion(), time.Now())
// ApplyMigration runs executes a database migration
func (d *SqlDb) ApplyMigration(migration db.Migration) error {
initialized, err := d.IsInitialized()
if err != nil {
return err
}
if !initialized {
fmt.Println("Creating migrations table")
query := d.prepareMigration(initialSQL)
if query == "" {
return nil
}
_, err = d.exec(query)
if err != nil {
return err
}
}
tx, err := d.sql.Begin()
if err != nil {
return err
}
query := version.GetSQL(version.GetPath())
for i, query := range query {
queries := getVersionSQL(getVersionPath(migration))
for i, query := range queries {
fmt.Printf("\r [%d/%d]", i+1, len(query))
if len(query) == 0 {
@ -71,6 +149,10 @@ func (d *SqlDb) applyMigration(version *Version) error {
}
q := d.prepareMigration(query)
if q == "" {
continue
}
_, err = tx.Exec(q)
if err != nil {
handleRollbackError(tx.Rollback())
@ -80,63 +162,47 @@ func (d *SqlDb) applyMigration(version *Version) error {
}
}
if _, err := tx.Exec(d.prepareQuery("insert into migrations(version, upgraded_date) values (?, ?)"), version.VersionString(), time.Now()); err != nil {
_, err = tx.Exec(d.PrepareQuery("insert into migrations(version, upgraded_date) values (?, ?)"), migration.Version, time.Now())
if err != nil {
handleRollbackError(tx.Rollback())
return err
}
switch migration.Version {
case "2.8.26":
err = migration_2_8_26{db: d}.Apply(tx)
case "2.8.42":
err = migration_2_8_42{db: d}.Apply(tx)
}
if err != nil {
return err
}
fmt.Println()
return tx.Commit()
}
// TryRollback attempts to rollback the database to an earlier version if a rollback exists
func (d *SqlDb) tryRollbackMigration(version *Version) {
fmt.Printf("Rolling back %s (time: %v)...\n", version.HumanoidVersion(), time.Now())
data := dbAssets.Bytes(version.GetErrPath())
// TryRollbackMigration attempts to rollback the database to an earlier version if a rollback exists
func (d *SqlDb) TryRollbackMigration(version db.Migration) {
data := dbAssets.Bytes(getVersionErrPath(version))
if len(data) == 0 {
fmt.Println("Rollback SQL does not exist.")
fmt.Println()
return
}
query := version.GetSQL(version.GetErrPath())
for _, query := range query {
queries := getVersionSQL(getVersionErrPath(version))
for _, query := range queries {
fmt.Printf(" [ROLLBACK] > %v\n", query)
if _, err := d.exec(d.prepareMigration(query)); err != nil {
q := d.prepareMigration(query)
if q == "" {
continue
}
if _, err := d.exec(q); err != nil {
fmt.Println(" [ROLLBACK] - Stopping")
return
}
}
}
func (d *SqlDb) Migrate() error {
fmt.Println("Checking DB migrations")
didRun := false
// go from beginning to the end
for _, version := range Versions {
if exists, err := d.isMigrationApplied(version); err != nil || exists {
if exists {
continue
}
return err
}
didRun = true
if err := d.applyMigration(version); err != nil {
d.tryRollbackMigration(version)
return err
}
}
if didRun {
fmt.Println("Migrations Finished")
}
return nil
}

53
db/sql/migration_2_8_28.go Normal file
View File

@ -0,0 +1,53 @@
package sql
import (
"github.com/go-gorp/gorp/v3"
"strings"
)
type migration_2_8_26 struct {
db *SqlDb
}
func (m migration_2_8_26) Apply(tx *gorp.Transaction) error {
rows, err := tx.Query(m.db.PrepareQuery("SELECT id, git_url FROM project__repository"))
if err != nil {
return err
}
repoUrls := make(map[string]string)
for rows.Next() {
var id, url string
err3 := rows.Scan(&id, &url)
if err3 != nil {
continue
}
repoUrls[id] = url
}
err = rows.Close()
if err != nil {
return err
}
for id, url := range repoUrls {
branch := "master"
parts := strings.Split(url, "#")
if len(parts) > 1 {
url, branch = parts[0], parts[1]
}
q := m.db.PrepareQuery("UPDATE project__repository " +
"SET git_url = ?, git_branch = ? " +
"WHERE id = ?")
_, err = tx.Exec(q, url, branch, id)
if err != nil {
return err
}
}
return nil
}

19
db/sql/migration_2_8_42.go Normal file
View File

@ -0,0 +1,19 @@
package sql
import "github.com/go-gorp/gorp/v3"
type migration_2_8_42 struct {
db *SqlDb
}
func (m migration_2_8_42) Apply(tx *gorp.Transaction) error {
switch m.db.sql.Dialect.(type) {
case gorp.MySQLDialect:
_, _ = tx.Exec(m.db.PrepareQuery("alter table `task` drop foreign key `task_ibfk_3`"))
case gorp.PostgresDialect:
_, err := tx.Exec(
m.db.PrepareQuery("alter table `task` drop constraint if exists `task_build_task_id_fkey`"))
return err
}
return nil
}

3
db/sql/migrations/v2.3.1.sql
View File

@ -1,6 +1,3 @@
-- alter table session change ip ip varchar(39) not null default '';
alter table session rename to session_backup;
create table session

11
db/sql/migrations/v2.8.20.sql
View File

@ -1,7 +1,4 @@
alter table `event`
add id integer primary key autoincrement;
alter table `event` rename to `event_backup_7534878`;
alter table `event` rename to `event_backup_5784568`;
create table `event`
(
@ -15,9 +12,3 @@ create table `event`
foreign key (`project_id`) references `project` (`id`) on delete cascade,
foreign key (`user_id`) references `user` (`id`) on delete set null
);
insert into `event`(id, project_id, object_id, object_type, description, created, user_id)
select id, project_id, object_id, object_type, description, created, user_id
from `event_backup_7534878`;
drop table `event_backup_7534878`;

4
db/sql/migrations/v2.8.25.sql Normal file
View File

@ -0,0 +1,4 @@
alter table `project__template` add survey_vars longtext;
alter table `project__template` add autorun boolean default false;
alter table `project__schedule` add repository_id int null references project__repository(`id`) on delete set null;
alter table `project__schedule` add last_commit_hash varchar(40);

1
db/sql/migrations/v2.8.26.sql Normal file
View File

@ -0,0 +1 @@
alter table `project__repository` add git_branch varchar(255) not null default '';

3
db/sql/migrations/v2.8.36.sql Normal file
View File

@ -0,0 +1,3 @@
alter table `project__template` add allow_override_args_in_task bool not null default false;
alter table `task` add arguments text;
alter table `project__template` drop column `override_args`;

39
db/sql/migrations/v2.8.38.sql Normal file
View File

@ -0,0 +1,39 @@
delete
from project__schedule
where template_id is null;
delete
from project__schedule
where (select count(*) from project__template where project__template.id = project__schedule.template_id) = 0;
delete
from project__schedule
where (select count(*) from project where project.id = project__schedule.project_id) = 0;
update project__schedule
set repository_id = null
where repository_id is not null
and (select count(*) from project__repository where project__repository.id = project__schedule.repository_id) = 0;
alter table `project__schedule`
rename to `project__schedule_backup_8436583`;
create table project__schedule
(
id integer primary key autoincrement,
template_id int not null,
project_id int not null,
cron_format varchar(255) not null,
repository_id int null,
last_commit_hash varchar(40) null,
foreign key (`template_id`) references project__template(`id`) on delete cascade,
foreign key (`project_id`) references project(`id`) on delete cascade,
foreign key (`repository_id`) references project__repository(`id`)
);
insert into project__schedule
select *
from project__schedule_backup_8436583;
drop table project__schedule_backup_8436583;

22
db/sql/migrations/v2.8.39.sql Normal file
View File

@ -0,0 +1,22 @@
alter table `task`
add constraint `task_build_task_id_fk_y38rt`
foreign key (`build_task_id`) references `task` (`id`)
on delete set null;
create table `project__template_backup_385025846` (
id int,
removed boolean default false
);
insert into `project__template_backup_385025846` select `id`, `removed` from `project__template`;
update `project__template`
set build_template_id = null
where (select t.`removed` from `project__template_backup_385025846` t where t.`id` = `build_template_id`) = true;
drop table `project__template_backup_385025846`;
delete from `project__template` where `removed` = true;
alter table `project__template` drop column `removed`;

11
db/sql/migrations/v2.8.40.sql Normal file
View File

@ -0,0 +1,11 @@
alter table `project` change `alert_chat` `alert_chat` varchar(30);
alter table `project__template` change `alias` `name` varchar(100) not null;
alter table `project__inventory` drop column `removed`;
alter table `project__environment` drop column `removed`;
alter table `access_key` drop column `removed`;
alter table `project__repository` drop column `removed`;

1
db/sql/migrations/v2.8.42.sql Normal file
View File

@ -0,0 +1 @@
-- see migration_2_8_42.go

2
db/sql/project.go
View File

@ -72,7 +72,7 @@ func (d *SqlDb) DeleteProject(projectID int) error {
}
for _, statement := range statements {
_, err = tx.Exec(d.prepareQuery(statement), projectID)
_, err = tx.Exec(d.PrepareQuery(statement), projectID)
if err != nil {
err = tx.Rollback()

29
db/sql/repository.go
View File

@ -18,6 +18,10 @@ func (d *SqlDb) GetRepository(projectID int, repositoryID int) (db.Repository, e
return repository, err
}
func (d *SqlDb) GetRepositoryRefs(projectID int, repositoryID int) (db.ObjectReferrers, error) {
return d.getObjectRefs(projectID, db.RepositoryProps, repositoryID)
}
func (d *SqlDb) GetRepositories(projectID int, params db.RetrieveQueryParams) (repositories []db.Repository, err error) {
q := squirrel.Select("*").
From("project__repository pr")
@ -52,10 +56,17 @@ func (d *SqlDb) GetRepositories(projectID int, params db.RetrieveQueryParams) (r
}
func (d *SqlDb) UpdateRepository(repository db.Repository) error {
_, err := d.exec(
"update project__repository set name=?, git_url=?, ssh_key_id=? where id=?",
err := repository.Validate()
if err != nil {
return err
}
_, err = d.exec(
"update project__repository set name=?, git_url=?, git_branch=?, ssh_key_id=? where id=?",
repository.Name,
repository.GitURL,
repository.GitBranch,
repository.SSHKeyID,
repository.ID)
@ -63,11 +74,18 @@ func (d *SqlDb) UpdateRepository(repository db.Repository) error {
}
func (d *SqlDb) CreateRepository(repository db.Repository) (newRepo db.Repository, err error) {
err = repository.Validate()
if err != nil {
return
}
insertID, err := d.insert(
"id",
"insert into project__repository(project_id, git_url, ssh_key_id, name) values (?, ?, ?, ?)",
"insert into project__repository(project_id, git_url, git_branch, ssh_key_id, name) values (?, ?, ?, ?, ?)",
repository.ProjectID,
repository.GitURL,
repository.GitBranch,
repository.SSHKeyID,
repository.Name)
@ -83,8 +101,3 @@ func (d *SqlDb) CreateRepository(repository db.Repository) (newRepo db.Repositor
func (d *SqlDb) DeleteRepository(projectID int, repositoryId int) error {
return d.deleteObject(projectID, db.RepositoryProps, repositoryId)
}
func (d *SqlDb) DeleteRepositorySoft(projectID int, repositoryId int) error {
return d.deleteObjectSoft(projectID, db.RepositoryProps, repositoryId)
}

33
db/sql/schedule.go
View File

@ -8,11 +8,12 @@ import (
func (d *SqlDb) CreateSchedule(schedule db.Schedule) (newSchedule db.Schedule, err error) {
insertID, err := d.insert(
"id",
"insert into project__schedule (project_id, template_id, cron_format)" +
"values (?, ?, ?)",
"insert into project__schedule (project_id, template_id, cron_format, repository_id)"+
"values (?, ?, ?, ?)",
schedule.ProjectID,
schedule.TemplateID,
schedule.CronFormat)
schedule.CronFormat,
schedule.RepositoryID)
if err != nil {
return
@ -24,12 +25,26 @@ func (d *SqlDb) CreateSchedule(schedule db.Schedule) (newSchedule db.Schedule, e
return
}
func (d *SqlDb) SetScheduleLastCommitHash(projectID int, scheduleID int, lastCommentHash string) error {
_, err := d.exec("update project__schedule set "+
"last_commit_hash=? "+
"where project_id=? and id=?",
lastCommentHash,
projectID,
scheduleID)
return err
}
func (d *SqlDb) UpdateSchedule(schedule db.Schedule) error {
_, err := d.exec("update project__schedule set cron_format=? where project_id=? and id=?",
_, err := d.exec("update project__schedule set "+
"cron_format=?, "+
"repository_id=?, "+
"last_commit_hash = NULL "+
"where project_id=? and id=?",
schedule.CronFormat,
schedule.RepositoryID,
schedule.ProjectID,
schedule.ID)
return err
}
@ -64,3 +79,11 @@ func (d *SqlDb) GetTemplateSchedules(projectID int, templateID int) (schedules [
templateID)
return
}
func (d *SqlDb) SetScheduleCommitHash(projectID int, scheduleID int, hash string) error {
_, err := d.exec("update project__schedule set last_commit_hash=? where project_id=? and id=?",
hash,
projectID,
scheduleID)
return err
}

5
db/sql/session.go
View File

@ -18,7 +18,7 @@ func (d *SqlDb) CreateAPIToken(token db.APIToken) (db.APIToken, error) {
}
func (d *SqlDb) GetAPIToken(tokenID string) (token db.APIToken, err error) {
err = d.selectOne(&token, d.prepareQuery("select * from user__token where id=? and expired=false"), tokenID)
err = d.selectOne(&token, d.PrepareQuery("select * from user__token where id=? and expired=false"), tokenID)
if err == sql.ErrNoRows {
err = db.ErrNotFound
@ -56,7 +56,7 @@ func (d *SqlDb) TouchSession(userID int, sessionID int) error {
}
func (d *SqlDb) GetAPITokens(userID int) (tokens []db.APIToken, err error) {
_, err = d.selectAll(&tokens, d.prepareQuery("select * from user__token where user_id=?"), userID)
_, err = d.selectAll(&tokens, d.PrepareQuery("select * from user__token where user_id=?"), userID)
if err == sql.ErrNoRows {
err = db.ErrNotFound
@ -64,4 +64,3 @@ func (d *SqlDb) GetAPITokens(userID int) (tokens []db.APIToken, err error) {
return
}

10
db/sql/task.go
View File

@ -31,12 +31,11 @@ func (d *SqlDb) CreateTaskOutput(output db.TaskOutput) (db.TaskOutput, error) {
return output, err
}
func (d *SqlDb) getTasks(projectID int, templateID* int, params db.RetrieveQueryParams, tasks *[]db.TaskWithTpl) (err error) {
func (d *SqlDb) getTasks(projectID int, templateID *int, params db.RetrieveQueryParams, tasks *[]db.TaskWithTpl) (err error) {
fields := "task.*"
fields += ", tpl.playbook as tpl_playbook" +
", `user`.name as user_name" +
", tpl.alias as tpl_alias" +
", tpl.name as tpl_alias" +
", tpl.type as tpl_type"
q := squirrel.Select(fields).
@ -69,7 +68,6 @@ func (d *SqlDb) getTasks(projectID int, templateID* int, params db.RetrieveQuery
return
}
func (d *SqlDb) GetTask(projectID int, taskID int) (task db.Task, err error) {
q := squirrel.Select("task.*").
From("task").
@ -96,8 +94,8 @@ func (d *SqlDb) GetTask(projectID int, taskID int) (task db.Task, err error) {
return
}
func (d *SqlDb) GetTemplateTasks(template db.Template, params db.RetrieveQueryParams) (tasks []db.TaskWithTpl, err error) {
err = d.getTasks(template.ProjectID, &template.ID, params, &tasks)
func (d *SqlDb) GetTemplateTasks(projectID int, templateID int, params db.RetrieveQueryParams) (tasks []db.TaskWithTpl, err error) {
err = d.getTasks(projectID, &templateID, params, &tasks)
return
}

92
db/sql/template.go
View File

@ -15,24 +15,26 @@ func (d *SqlDb) CreateTemplate(template db.Template) (newTemplate db.Template, e
insertID, err := d.insert(
"id",
"insert into project__template (project_id, inventory_id, repository_id, environment_id, " +
"alias, playbook, arguments, override_args, description, vault_key_id, `type`, start_version," +
"build_template_id, view_id)" +
"values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"insert into project__template (project_id, inventory_id, repository_id, environment_id, "+
"name, playbook, arguments, allow_override_args_in_task, description, vault_key_id, `type`, start_version,"+
"build_template_id, view_id, autorun, survey_vars)"+
"values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
template.ProjectID,
template.InventoryID,
template.RepositoryID,
template.EnvironmentID,
template.Alias,
template.Name,
template.Playbook,
template.Arguments,
template.OverrideArguments,
template.AllowOverrideArgsInTask,
template.Description,
template.VaultKeyID,
template.Type,
template.StartVersion,
template.BuildTemplateID,
template.ViewID)
template.ViewID,
template.Autorun,
db.ObjectToJSON(template.SurveyVars))
if err != nil {
return
@ -57,57 +59,68 @@ func (d *SqlDb) UpdateTemplate(template db.Template) error {
return err
}
_, err = d.exec("update project__template set " +
"inventory_id=?, " +
"repository_id=?, " +
"environment_id=?, " +
"alias=?, " +
"playbook=?, " +
"arguments=?, " +
"override_args=?, " +
"description=?, " +
"vault_key_id=?, " +
"`type`=?, " +
"start_version=?," +
"build_template_id=?, " +
"view_id=? " +
"where removed = false and id=? and project_id=?",
_, err = d.exec("update project__template set "+
"inventory_id=?, "+
"repository_id=?, "+
"environment_id=?, "+
"name=?, "+
"playbook=?, "+
"arguments=?, "+
"allow_override_args_in_task=?, "+
"description=?, "+
"vault_key_id=?, "+
"`type`=?, "+
"start_version=?,"+
"build_template_id=?, "+
"view_id=?, "+
"autorun=?, "+
"survey_vars=? "+
"where id=? and project_id=?",
template.InventoryID,
template.RepositoryID,
template.EnvironmentID,
template.Alias,
template.Name,
template.Playbook,
template.Arguments,
template.OverrideArguments,
template.AllowOverrideArgsInTask,
template.Description,
template.VaultKeyID,
template.Type,
template.StartVersion,
template.BuildTemplateID,
template.ViewID,
template.Autorun,
db.ObjectToJSON(template.SurveyVars),
template.ID,
template.ProjectID,
)
return err
}
func (d *SqlDb) getTemplates(projectID int, viewID *int, params db.RetrieveQueryParams) (templates []db.Template, err error) {
func (d *SqlDb) GetTemplates(projectID int, filter db.TemplateFilter, params db.RetrieveQueryParams) (templates []db.Template, err error) {
q := squirrel.Select("pt.id",
"pt.project_id",
"pt.inventory_id",
"pt.repository_id",
"pt.environment_id",
"pt.alias",
"pt.name",
"pt.playbook",
"pt.arguments",
"pt.override_args",
"pt.allow_override_args_in_task",
"pt.vault_key_id",
"pt.view_id",
"pt.`type`").
From("project__template pt").
Where("pt.removed = false")
From("project__template pt")
if viewID != nil {
q = q.Where("pt.view_id=?", *viewID)
if filter.ViewID != nil {
q = q.Where("pt.view_id=?", *filter.ViewID)
}
if filter.BuildTemplateID != nil {
q = q.Where("pt.build_template_id=?", *filter.BuildTemplateID)
if filter.AutorunOnly {
q = q.Where("pt.autorun=true")
}
}
order := "ASC"
@ -116,7 +129,7 @@ func (d *SqlDb) getTemplates(projectID int, viewID *int, params db.RetrieveQuery
}
switch params.SortBy {
case "alias", "playbook":
case "name", "playbook":
q = q.Where("pt.project_id=?", projectID).
OrderBy("pt." + params.SortBy + " " + order)
case "inventory":
@ -133,7 +146,7 @@ func (d *SqlDb) getTemplates(projectID int, viewID *int, params db.RetrieveQuery
OrderBy("pr.name " + order)
default:
q = q.Where("pt.project_id=?", projectID).
OrderBy("pt.alias " + order)
OrderBy("pt.name " + order)
}
query, args, err := q.ToSql()
@ -153,14 +166,10 @@ func (d *SqlDb) getTemplates(projectID int, viewID *int, params db.RetrieveQuery
return
}
func (d *SqlDb) GetTemplates(projectID int, params db.RetrieveQueryParams) ( []db.Template, error) {
return d.getTemplates(projectID, nil, params)
}
func (d *SqlDb) GetTemplate(projectID int, templateID int) (template db.Template, err error) {
err = d.selectOne(
&template,
"select * from project__template where project_id=? and id=? and removed = false",
"select * from project__template where project_id=? and id=?",
projectID,
templateID)
@ -177,7 +186,10 @@ func (d *SqlDb) GetTemplate(projectID int, templateID int) (template db.Template
}
func (d *SqlDb) DeleteTemplate(projectID int, templateID int) error {
_, err := d.exec("update project__template set removed=true where project_id=? and id=?", projectID, templateID)
_, err := d.exec("delete from project__template where project_id=? and id=?", projectID, templateID)
return err
}
func (d *SqlDb) GetTemplateRefs(projectID int, templateID int) (db.ObjectReferrers, error) {
return d.getObjectRefs(projectID, db.TemplateProps, templateID)
}

48
db/sql/user.go
View File

@ -206,7 +206,7 @@ func (d *SqlDb) GetUsers(params db.RetrieveQueryParams) (users []db.User, err er
func (d *SqlDb) GetUserByLoginOrEmail(login string, email string) (existingUser db.User, err error) {
err = d.selectOne(
&existingUser,
d.prepareQuery("select * from `user` where email=? or username=?"),
d.PrepareQuery("select * from `user` where email=? or username=?"),
email, login)
if err == sql.ErrNoRows {
@ -215,49 +215,3 @@ func (d *SqlDb) GetUserByLoginOrEmail(login string, email string) (existingUser
return
}
func (d *SqlDb) CreatePlaceholderUser() error {
user := db.User{
Created: db.GetParsedTime(time.Now()),
}
return d.sql.Insert(&user)
}
func (d *SqlDb) GetPlaceholderUser() (user db.User, err error) {
err = d.selectOne(&user, d.prepareQuery("select id from `user` where username=''"))
if err == sql.ErrNoRows {
err = db.ErrNotFound
}
return
}
//
//func (d *SqlDb) HasPlaceholderUser() (bool, error) {
// _, err := d.getPlaceholderUser()
//
// if err == nil {
// return true, nil
// }
//
// if err == db.ErrNotFound {
// return false, nil
// }
//
// return false, err
//}
//
//func (d *SqlDb) ReplacePlaceholderUser(user db.UserWithPwd) (newUser db.User, err error) {
// placeholder, err := d.getPlaceholderUser()
// if err != nil {
// return
// }
// user.ID = placeholder.ID
// err = d.UpdateUser(user)
// if err != nil {
// return
// }
// newUser = user.User
// return
//}

4
db/sql/view.go
View File

@ -56,7 +56,3 @@ func (d *SqlDb) SetViewPositions(projectID int, positions map[int]int) error {
}
return nil
}
func (d *SqlDb) GetViewTemplates(projectID int, viewID int, params db.RetrieveQueryParams) ( []db.Template, error) {
return d.getTemplates(projectID, &viewID, params)
}

2
deployment/docker/ci/Dockerfile
View File

@ -33,4 +33,4 @@ RUN deployment/docker/ci/bin/install
USER semaphore
EXPOSE 3000
ENTRYPOINT ["/usr/local/bin/semaphore-wrapper"]
CMD ["./bin/semaphore", "--config", "/etc/semaphore/config.json"]
CMD ["./bin/semaphore", "server", "--config", "/etc/semaphore/config.json"]

4
deployment/docker/common/semaphore-wrapper
View File

@ -36,6 +36,8 @@ SEMAPHORE_LDAP_MAPPING_USERNAME="${SEMAPHORE_LDAP_MAPPING_USERNAME:-uid}"
SEMAPHORE_LDAP_MAPPING_FULLNAME="${SEMAPHORE_LDAP_MAPPING_FULLNAME:-cn}"
SEMAPHORE_LDAP_MAPPING_EMAIL="${SEMAPHORE_LDAP_MAPPING_EMAIL:-mail}"
export SEMAPHORE_ACCESS_KEY_ENCRYPTION="${SEMAPHORE_ACCESS_KEY_ENCRYPTION:-cFcXI5qHzCDqtS4xCnblOACuNu5AmKHkvxK7abwR8Eg=}"
# create semaphore temporary directory if non existent
[ -d "${SEMAPHORE_TMP_PATH}" ] || mkdir -p "${SEMAPHORE_TMP_PATH}" || {
echo "Can't create Semaphore tmp path ${SEMAPHORE_TMP_PATH}."
@ -108,7 +110,7 @@ EOF
cat "${SEMAPHORE_TMP_PATH}/config.stdin"
$1 setup - < "${SEMAPHORE_TMP_PATH}/config.stdin"
echoerr "Run Semaphore with semaphore --config ${SEMAPHORE_CONFIG_PATH}/config.json"
echoerr "Run Semaphore with semaphore server --config ${SEMAPHORE_CONFIG_PATH}/config.json"
fi
# run our command

2
deployment/docker/dev/Dockerfile
View File

@ -33,4 +33,4 @@ RUN deployment/docker/dev/bin/install
USER semaphore
EXPOSE 3000
ENTRYPOINT ["/usr/local/bin/semaphore-wrapper"]
CMD ["./bin/semaphore", "--config", "/etc/semaphore/config.json"]
CMD ["./bin/semaphore", "server", "--config", "/etc/semaphore/config.json"]

2
deployment/docker/prod/Dockerfile
View File

@ -30,4 +30,4 @@ WORKDIR /home/semaphore
USER 1001
ENTRYPOINT ["/sbin/tini", "--"]
CMD ["/usr/local/bin/semaphore-wrapper", "/usr/local/bin/semaphore", "--config", "/etc/semaphore/config.json"]
CMD ["/usr/local/bin/semaphore-wrapper", "/usr/local/bin/semaphore", "server", "--config", "/etc/semaphore/config.json"]

4
deployment/systemd/semaphore.service
View File

@ -4,11 +4,11 @@ Requires=network.target
[Service]
EnvironmentFile=/etc/semaphore/env
ExecStart=/usr/bin/semaphore --config ${SEMAPHORE_CONFIG}
ExecStart=/usr/bin/semaphore server --config ${SEMAPHORE_CONFIG}
User=semaphore
Group=semaphore
Restart=always
RestartSec=3s
[Install]
WantedBy=multi-user.target
WantedBy=multi-user.target

3
go.mod
View File

@ -5,6 +5,7 @@ go 1.16
require (
github.com/Sirupsen/logrus v1.0.4
github.com/go-gorp/gorp/v3 v3.0.2
github.com/go-ldap/ldap/v3 v3.4.1
github.com/go-sql-driver/mysql v1.4.1
github.com/gobuffalo/packr v1.10.4
github.com/google/go-github v17.0.0+incompatible
@ -26,7 +27,5 @@ require (
go.etcd.io/bbolt v1.3.2
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225 // indirect
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
gopkg.in/ldap.v2 v2.5.1
)

11
go.sum
View File

@ -37,6 +37,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28=
github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Sirupsen/logrus v1.0.4 h1:yilvuj073Hm7wwwz12E96GjrdivMNuTMJk9ddjde+D8=
@ -71,11 +73,15 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-asn1-ber/asn1-ber v1.5.1 h1:pDbRAunXzIUXfx4CB2QJFv5IuPiuoW+sWvr/Us009o8=
github.com/go-asn1-ber/asn1-ber v1.5.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gorp/gorp/v3 v3.0.2 h1:ULqJXIekoqMx29FI5ekXXFoH1dT2Vc8UhnRzBg+Emz4=
github.com/go-gorp/gorp/v3 v3.0.2/go.mod h1:BJ3q1ejpV8cVALtcXvXaXyTOlMmJhWDxTmncaR6rwBY=
github.com/go-ldap/ldap/v3 v3.4.1 h1:fU/0xli6HY02ocbMuozHAYsaHLcnkLjvho2r5a34BUU=
github.com/go-ldap/ldap/v3 v3.4.1/go.mod h1:iYS1MdmrmceOJ1QOTnRXrIs7i3kloqtmGQjRvjKpyMg=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/gobuffalo/packr v1.10.4 h1:k/aQq1+9UlSxfr9+QXz1sQ8vJVFTXqrEDdKgeonlt4Y=
@ -293,6 +299,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -603,16 +610,12 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225 h1:JBwmEvLfCqgPcIq8MjVMQxsF3LVL4XG/HH0qiG0+IFY=
gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ldap.v2 v2.5.1 h1:wiu0okdNfjlBzg6UWvd1Hn8Y+Ux17/u/4nlk4CQr6tU=
gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

79
lib/AnsiblePlaybook.go Normal file
View File

@ -0,0 +1,79 @@
package lib
import (
"bytes"
"fmt"
"github.com/ansible-semaphore/semaphore/db"
"github.com/ansible-semaphore/semaphore/util"
"os"
"os/exec"
"regexp"
"strings"
)
type AnsiblePlaybook struct {
TemplateID int
Repository db.Repository
Logger Logger
}
func (p AnsiblePlaybook) makeCmd(command string, args []string) *exec.Cmd {
cmd := exec.Command(command, args...) //nolint: gas
cmd.Dir = p.GetFullPath()
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", util.Config.TmpPath))
cmd.Env = append(cmd.Env, fmt.Sprintf("PWD=%s", cmd.Dir))
cmd.Env = append(cmd.Env, fmt.Sprintln("PYTHONUNBUFFERED=1"))
return cmd
}
func (p AnsiblePlaybook) runCmd(command string, args []string) error {
cmd := p.makeCmd(command, args)
p.Logger.LogCmd(cmd)
return cmd.Run()
}
func (p AnsiblePlaybook) GetHosts(args []string) (hosts []string, err error) {
args = append(args, "--list-hosts")
cmd := p.makeCmd("ansible-playbook", args)
var errb bytes.Buffer
cmd.Stderr = &errb
out, err := cmd.Output()
if err != nil {
return
}
re := regexp.MustCompile(`(?m)^\\s{6}(.*)$`)
matches := re.FindAllSubmatch(out, 20)
hosts = make([]string, len(matches))
for i := range matches {
hosts[i] = string(matches[i][1])
}
return
}
func (p AnsiblePlaybook) RunPlaybook(args []string, cb func(*os.Process)) error {
cmd := p.makeCmd("ansible-playbook", args)
p.Logger.LogCmd(cmd)
cmd.Stdin = strings.NewReader("")
err := cmd.Start()
if err != nil {
return err
}
cb(cmd.Process)
return cmd.Wait()
}
func (p AnsiblePlaybook) RunGalaxy(args []string) error {
return p.runCmd("ansible-galaxy", args)
}
func (p AnsiblePlaybook) GetFullPath() (path string) {
path = p.Repository.GetFullPath(p.TemplateID)
return
}

166
lib/GitRepository.go Normal file
View File

@ -0,0 +1,166 @@
package lib
import (
"fmt"
"github.com/ansible-semaphore/semaphore/db"
"github.com/ansible-semaphore/semaphore/util"
"os"
"os/exec"
"strings"
)
type GitRepositoryDirType int
const (
GitRepositoryTmpDir GitRepositoryDirType = iota
GitRepositoryRepoDir
)
type GitRepository struct {
TemplateID int
Repository db.Repository
Logger Logger
}
func (r GitRepository) makeCmd(targetDir GitRepositoryDirType, args ...string) *exec.Cmd {
cmd := exec.Command("git") //nolint: gas
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, fmt.Sprintln("GIT_TERMINAL_PROMPT=0"))
if r.Repository.SSHKey.Type == db.AccessKeySSH {
sshCmd := "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i " + r.Repository.SSHKey.GetPath()
if util.Config.SshConfigPath != "" {
sshCmd += " -F " + util.Config.SshConfigPath
}
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_SSH_COMMAND=%s", sshCmd))
}
switch targetDir {
case GitRepositoryTmpDir:
cmd.Dir = util.Config.TmpPath
case GitRepositoryRepoDir:
cmd.Dir = r.GetFullPath()
default:
panic("unknown Repository directory type")
}
cmd.Args = append(cmd.Args, args...)
return cmd
}
func (r GitRepository) run(targetDir GitRepositoryDirType, args ...string) error {
err := r.Repository.SSHKey.Install(db.AccessKeyRoleGit)
if err != nil {
return err
}
defer r.Repository.SSHKey.Destroy() //nolint: errcheck
cmd := r.makeCmd(targetDir, args...)
r.Logger.LogCmd(cmd)
return cmd.Run()
}
func (r GitRepository) output(targetDir GitRepositoryDirType, args ...string) (out string, err error) {
err = r.Repository.SSHKey.Install(db.AccessKeyRoleGit)
if err != nil {
return
}
defer r.Repository.SSHKey.Destroy() //nolint: errcheck
bytes, err := r.makeCmd(targetDir, args...).Output()
if err != nil {
return
}
out = strings.Trim(string(bytes), " \n")
return
}
func (r GitRepository) Clone() error {
r.Logger.Log("Cloning Repository " + r.Repository.GitURL)
return r.run(GitRepositoryTmpDir,
"clone",
"--recursive",
"--branch",
r.Repository.GitBranch,
r.Repository.GetGitURL(),
r.Repository.GetDirName(r.TemplateID))
}
func (r GitRepository) Pull() error {
r.Logger.Log("Updating Repository " + r.Repository.GitURL)
return r.run(GitRepositoryRepoDir, "pull", "origin", r.Repository.GitBranch)
}
func (r GitRepository) Checkout(target string) error {
r.Logger.Log("Checkout repository to " + target)
return r.run(GitRepositoryRepoDir, "checkout", target)
}
func (r GitRepository) CanBePulled() bool {
err := r.run(GitRepositoryRepoDir, "fetch")
if err != nil {
return false
}
err = r.run(GitRepositoryRepoDir,
"merge-base", "--is-ancestor", "HEAD", "origin/"+r.Repository.GitBranch)
return err == nil
}
func (r GitRepository) GetLastCommitMessage() (msg string, err error) {
r.Logger.Log("Get current commit message")
msg, err = r.output(GitRepositoryRepoDir, "show-branch", "--no-name", "HEAD")
if err != nil {
return
}
if len(msg) > 100 {
msg = msg[0:100]
}
return
}
func (r GitRepository) GetLastCommitHash() (hash string, err error) {
r.Logger.Log("Get current commit hash")
hash, err = r.output(GitRepositoryRepoDir, "rev-parse", "HEAD")
return
}
func (r GitRepository) ValidateRepo() error {
_, err := os.Stat(r.GetFullPath())
return err
}
func (r GitRepository) GetFullPath() (path string) {
path = r.Repository.GetFullPath(r.TemplateID)
return
}
func (r GitRepository) GetLastRemoteCommitHash() (hash string, err error) {
out, err := r.output(GitRepositoryTmpDir, "ls-remote", r.Repository.GetGitURL(), r.Repository.GitBranch)
if err != nil {
return
}
firstSpaceIndex := strings.IndexAny(out, "\t ")
if firstSpaceIndex == -1 {
err = fmt.Errorf("can't retreave remote commit hash")
}
if err != nil {
return
}
hash = out[0:firstSpaceIndex]
return
}

8
lib/Logger.go Normal file
View File

@ -0,0 +1,8 @@
package lib
import "os/exec"
type Logger interface {
Log(msg string)
LogCmd(cmd *exec.Cmd)
}

Some files were not shown because too many files have changed in this diff Show More